reproducible: move diffoscope_compare() into its own class

Signed-off-by: Mattia Rizzolo <mattia@debian.org>

bin/rblib/commands.py

+# -*- coding: utf-8 -*-
+#
+# Copyright © 2015-2019 Mattia Rizzolo <mattia@debian.org>
+# Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>
+# Copyright © 2019      Paul Spooren <mail@aparcar.org>
+#
+# Licensed under GPL-2
+
+import os
+import subprocess
+import resource
+
+
+class Schroot():
+    def __init__(self, chroot, directory=None):
+        self.chroot = chroot
+        self._cmd = ['schroot', '-c', chroot]
+        self.set_directory(directory)
+        self._limits = []
+
+    def set_directory(self, directory):
+        if not directory:
+            self.directory = '/tmp'
+        self._cmd.extend(('--directory', self.directory))
+
+    def add_limit(self, what, how):
+        self._limits.append((what, how))
+
+    def set_default_limits(self):
+        # 10 GB of actually used memory
+        self._limits.append(
+            resource.RLIMIT_AS, (10 * 1024 ** 3, resource.RLIM_INFINITY)
+        )
+
+    def _preexec_limiter(self):
+        for limit in self._limits:
+            resource.setrlimit(*limit)
+
+    def run(self, command, *, check=False, timeout=False):
+        # separate the command name frome the optiosn with --
+        self._cmd.append(command[0])
+        self._cmd.append('--')
+        self._cmd.extend(command[1:])
+        return subprocess.run(
+            self._cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            check=check,
+            timeout=timeout,
+            text=check,
+            preexec_fn=self._preexec_limiter,
+        )
+
+
+class Diffoscope(Schroot):
+    def __init__(self, chroot=None, suite=None):
+        if suite is None:
+            suite = 'unstable'
+        if chroot is None:
+            chroot = f'jenkins-reproducible-{suite}-diffoscope'
+        super.__init__(chroot=chroot)
+        self.set_default_limits()
+
+    def version(self):
+        try:
+            p = self.run(('diffoscope', '--version'), check=True)
+        except subprocess.CalledProcessError:
+            return 'cannot get version'
+        return p.stdout.strip()
+
+    def compare_files(self, a, b, html):
+        self.set_directory(os.path.dirname(html))
+        diff_cmd = ['diffoscope', '--html', html]
+        msg = f'diffoscope {self.version()}'
+        timeout = 30*60  # 30 minutes
+        try:
+            p = self.run(diff_cmd + (a, b), timeout=timeout)
+            if p.returncode == 0:
+                print(f'{msg}: {a} is reproducible, yay!')
+            elif p.returncode == 1:
+                print(f'{msg}: {a} has issue, please investigate')
+            elif p.returncode == 2:
+                with open(html, 'w') as f:
+                    f.write(f'{msg} had errors comparing the two builds.\n')
+                    f.write(f'{diff_cmd}\n')
+                    f.write(p.stdout)
+        except subprocess.TimeoutExpired:
+            if os.path.exits(html):
+                text = (f'{msg} produced not output and was killed after '
+                        f'running into timeout after {timeout}.')
+                with open(html, 'w') as f:
+                    f.write(f'text\n')
+                    f.write(f'{diff_cmd}\n')
+                    f.write(p.stdout)
+                print(text)
+            else:
+                print(f'{msg} was killed after running into timeout after '
+                      f'{timeout}, but there still {html}')

bin/reproducible_common.py

-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# Copyright © 2019 Paul Spooren <mail@aparcar.org>
-#
-# Inspired by reproducible_common.sh
-#   © Holger Levsen <holger@layer-acht.org>
-#
-# Released under the GPLv2
-
-import os
-import subprocess
-import resource
-
-
-def e(var, default):
-    """Return env variable or default"""
-    return os.environ.get(var, default)
-
-
-# DBSUITE which version of diffoscope to use
-timeout = e("TIMEOUT", 30 * 60)  # 30m
-# DIFFOSCOPE_VIRT_LIMIT max RAM usage
-ds_virt_limit = int(e("DIFFOSCOPE_VIRT_LIMIT", 10 * 1024 ** 3))  # 10GB
-# TIMEOUT timeout for diffoscope in seconds
-dbsuite = e("DBDSUITE", "unstable")
-# SCHROOT to use
-schroot = e("SCHROOT", f"source:jenkins-reproducible-{dbsuite}-diffoscope")
-
-
-def limit_resources():
-    resource.setrlimit(resource.RLIMIT_CPU, (1, 1))
-    resource.setrlimit(resource.RLIMIT_AS, (ds_virt_limit, resource.RLIM_INFINITY))
-
-
-def diffoscope_version():
-    cmd = []
-    if schroot:
-        cmd.extend(["schroot", "--directory", "/tmp", "-c", schroot])
-
-    cmd.extend(["diffoscope", "--", "--version"])
-    print(cmd)
-    return (
-        subprocess.run(cmd, capture_output=True, text=True, preexec_fn=limit_resources)
-        .stdout.strip()
-        .split()[1]
-    )
-
-
-def diffoscope_compare(path_a, path_b, path_output_html):
-    """
-    Run diffoscope in a schroot environment
-
-    Args:
-    - path_a path to first file to compare
-    - path_b path to second file a to compare
-    - path_output_html path where to store result html
-    """
-    cmd = []
-    if schroot:
-        cmd.extend(
-            ["schroot", "--directory", os.path.dirname(path_output_html), "-c", schroot]
-        )
-
-    try:
-        cmd.extend(["diffoscope", "--", "--html", path_output_html, path_a, path_b])
-        result = subprocess.run(
-            cmd,
-            timeout=timeout,
-            capture_output=True,
-            text=True,
-            preexec_fn=limit_resources,
-        )
-        msg = f"diffoscope {diffoscope_version()} "
-        if result.returncode == 0:
-            print(msg + f"{path_a} reproducible, yay!")
-        else:
-            if result.returncode == 1:
-                print(msg + f"found issues, please investigate {path_a}")
-            elif result.returncode == 2:
-                with open(path_output_html, "w") as output_html_file:
-                    output_html_file.write(
-                        msg
-                        + f"""had trouble comparing the two builds. Please
-                                investigate {path_a}"""
-                    )
-
-    except subprocess.TimeoutExpired:
-        if os.path.exists(path_output_html):
-            print(
-                msg
-                + f"""produced no output comparing {path_a} with {path_b} and
-                    was killed after running into timeout after {timeout}..."""
-            )
-        else:
-            print(
-                msg
-                + """was killed after running into timeout after $TIMEOUT, but
-                    there is still {path_output_html}"""
-            )

bin/reproducible_openwrt_rebuild.py

@@ -23,7 +23,7 @@ from time import strftime, gmtime
 import shutil
 import json
 
-from reproducible_common import diffoscope_compare, diffoscope_version
+from .rblib.commands import Diffoscope
 from reproducible_openwrt_package_parser import insert_into_db, show_list_difference
 
 # target to be build
@@ -108,7 +108,7 @@ context = {
     "packages_repro_percent": 0,
     "packages_total": 0,
     "today": strftime("%Y-%m-%d", gmtime()),
-    "diffoscope_version": diffoscope_version(),
+    "diffoscope_version": Diffoscope().version(),
     "target": target,
     "images": [],
     "packages": [],
@@ -125,10 +125,10 @@ def diffoscope(origin_name):
         print("Error downloading {}".format(origin_name))
         return
 
-    diffoscope_compare(
+    Diffoscope().compare(
         file_origin.name,
         target_dir + "/" + origin_name,
-        results_target_dir + "/" + origin_name + ".html",
+        html=results_target_dir + "/" + origin_name + ".html",
     )
 
     file_origin.close()
@@ -145,7 +145,7 @@ def get_file(url, path=None):
     print("downloading {}".format(url))
     try:
         content = urlopen(url).read()
-    except:
+    except Exception:
         return 1
 
     if path: