openwrt-rebuild: update to use schroot

b5ffcbf5 · Paul Spooren · aff8132e · b5ffcbf5 · b5ffcbf5 · b5ffcbf5
Commit b5ffcbf5 authored Oct 04, 2019 by Paul Spooren
--- a/bin/reproducible_common.py
+++ b/bin/reproducible_common.py
+import os
+import subprocess
+import resource
+def e(var, default):
+    """Return env variable or default"""
+    return os.environ.get(var, default)
+# DBSUITE which version of diffoscope to use
+timeout = e("TIMEOUT", 30 * 60)  # 30m
+# DIFFOSCOPE_VIRT_LIMIT max RAM usage
+ds_virt_limit = int(e("DIFFOSCOPE_VIRT_LIMIT", 10 * 1024 ** 3))  # 10GB
+# TIMEOUT timeout for diffoscope in seconds
+dbsuite = e("DBDSUITE", "unstable")
+# SCHROOT to use
+schroot = e("SCHROOT", f"source:jenkins-reproducible-{dbsuite}-diffoscope")
+def limit_resources():
+    resource.setrlimit(resource.RLIMIT_CPU, (1, 1))
+    resource.setrlimit(resource.RLIMIT_AS, (ds_virt_limit, resource.RLIM_INFINITY))
+def diffoscope_version():
+    cmd = []
+    if schroot:
+        cmd.extend(["schroot", "-c", schroot])
+    cmd.extend(["diffoscope", "--", "--version"])
+    print(cmd)
+    return (
+        subprocess.run(cmd, capture_output=True, text=True, preexec_fn=limit_resources)
+        .stdout.strip()
+        .split()[1]
+    )
+def diffoscope_compare(path_a, path_b, path_output_html):
+    """
+    Run diffoscope in a schroot environment
+    Args:
+    - path_a path to first file to compare
+    - path_b path to second file a to compare
+    - path_output_html path where to store result html
+    """
+    cmd = []
+    if schroot:
+        cmd.extend(["schroot", "-c", schroot])
+    try:
+        cmd.extend(["diffoscope", "--", "--html", path_output_html, path_a, path_b])
+        result = subprocess.run(
+            cmd,
+            timeout=timeout,
+            capture_output=True,
+            text=True,
+            preexec_fn=limit_resources,
+        )
+        msg = f"diffoscope {diffoscope_version()} "
+        if result.returncode == 0:
+            print(msg + f"{path_a} reproducible, yay!")
+        else:
+            if result.returncode == 1:
+                print(msg + f"found issues, please investigate {path_a}")
+            elif result.returncode == 2:
+                with open(path_output_html, "w") as output_html_file:
+                    output_html_file.write(
+                        msg
+                        + f"""had trouble comparing the two builds. Please
+                                investigate {path_a}"""
+                    )
+    except subprocess.TimeoutExpired:
+        if os.path.exists(path_output_html):
+            print(
+                msg
+                + f"""produced no output comparing {path_a} with {path_b} and
+                    was killed after running into timeout after {timeout}..."""
+            )
+        else:
+            print(
+                msg
+                + """was killed after running into timeout after $TIMEOUT, but
+                    there is still {path_output_html}"""
+            )
--- a/bin/reproducible_openwrt_rebuild.py
+++ b/bin/reproducible_openwrt_rebuild.py
--- a/mustache-templates/openwrt/header.mustache
+++ b/mustache-templates/openwrt/header.mustache
@@ -4,8 +4,7 @@
 <head>
 	<meta charset="UTF-8">
 	<meta name="viewport" content="width=device-width">
-	<title>Reproducible OpenWrt ?</title>
+	<title>Reproducible OpenWrt?</title>
-	<link rel='stylesheet' id='kamikaze-style-css' href='cascade.css?ver=4.0' type='text/css' media='all'>
 </head>
 <style type="text/css">
 	html,
@@ -115,7 +114,8 @@
 <body>
 	<div id="content">
-		<pre>
+    <a href="{{ root }}" style="text-decoration: none;">
+		  <pre>
  _______                     ________        __
 |       |.-----.-----.-----.|  |  |  |.----.|  |_
 |   -   ||  _  |  -__|     ||  |  |  ||   _||   _|
@@ -124,7 +124,8 @@
 -----------------------------------------------------
 OpenWrt {{ version }}, {{ commit_string }}
 -----------------------------------------------------
-    </pre>
+      </pre>
+    </a>
 	</div>
 	<div id="main-content">
 		<h1>OpenWrt - <em>reproducible</em> wireless freedom!</h1>
--- a/mustache-templates/openwrt/index.mustache
+++ b/mustache-templates/openwrt/index.mustache
@@ -26,7 +26,7 @@
    <p>
      <ul>
        {{ #targets }}
-        <li><a href="{{ version }}/{{ name}}">{{ version }} - {{ name }}</a></li>
+        <li><a href="{{ root }}/{{ version }}/{{ name}}">{{ version }} - {{ name }}</a></li>
        {{ /targets }}
      </ul>
    </p>

--- a/mustache-templates/openwrt/target.mustache
+++ b/mustache-templates/openwrt/target.mustache
@@ -16,10 +16,10 @@
      {{ #images }}
      <tr>
 	  	{{ #repro }}
-        <td><img src="/userContent/reproducible/static/weather-clear.png" alt="reproducible icon" /> {{ name }} ({{ sha256sum }}, {{ size }}K) is reproducible.</td>
+        <td><img src="{{ root }}/static/weather-clear.png" alt="reproducible icon" /> {{ name }} ({{ sha256sum }}, {{ size }}K) is reproducible.</td>
 	  	{{ /repro }}
 	  	{{ ^repro }}
-		<td><a href="/{{ version }}/{{ target }}/{{ name }}.html"><img src="/userContent/reproducible/static/weather-showers-scattered.png" alt="unreproducible icon"> {{ name }}</a> ({{ size }}K) is unreproducible.</td>
+		<td><a href="{{ root }}/{{ version }}/{{ target }}/{{ name }}.html"><img src="{{ root }}/static/weather-showers-scattered.png" alt="unreproducible icon"> {{ name }}</a> ({{ size }}K) is unreproducible.</td>
 	  	{{ /repro }}
      </tr>
      {{ /images }}
@@ -31,7 +31,7 @@
      {{ #packages }}
 	  	{{ ^repro }}
      <tr>
-		<td><a href="/{{ version }}/{{ target }}/{{ name }}.html"><img src="/userContent/reproducible/static/weather-showers-scattered.png" alt="unreproducible icon"> {{ name }}</a> ({{ size }}K) is unreproducible.</td>
+		<td><a href="{{ root }}/{{ version }}/{{ target }}/{{ name }}.html"><img src="{{ root }}/static/weather-showers-scattered.png" alt="unreproducible icon"> {{ name }}</a> ({{ size }}K) is unreproducible.</td>
      </tr>
 	  	{{ /repro }}
      {{ /packages }}
@@ -43,7 +43,7 @@
      {{ #packages }}
 	    {{ #repro }}
 	  <tr>
-        <td><img src="/userContent/reproducible/static/weather-clear.png" alt="reproducible icon" /> {{ name }} ({{ sha256sum }}, {{ size }}K) is reproducible.</td>
+        <td><img src="{{ root }}/static/weather-clear.png" alt="reproducible icon" /> {{ name }} ({{ sha256sum }}, {{ size }}K) is reproducible.</td>
 	  </tr>
 	  	{{ /repro }}
 	  {{ /packages }}