reproducible: move diffoscope_compare() into its own class

# -*- coding: utf-8 -*-

#

# Copyright © 2015-2019 Mattia Rizzolo <mattia@debian.org>

# Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>

# Copyright © 2019      Paul Spooren <mail@aparcar.org>

#

# Licensed under GPL-2

import os

import subprocess

import resource

class Schroot():

    def __init__(self, chroot, directory=None):

        self.chroot = chroot

        self._cmd = ['schroot', '-c', chroot]

        self.set_directory(directory)

        self._limits = []

    def set_directory(self, directory):

        if not directory:

            self.directory = '/tmp'

        self._cmd.extend(('--directory', self.directory))

    def add_limit(self, what, how):

        self._limits.append((what, how))

    def set_default_limits(self):

        # 10 GB of actually used memory

        self._limits.append(

            resource.RLIMIT_AS, (10 * 1024 ** 3, resource.RLIM_INFINITY)

        )

    def _preexec_limiter(self):

        for limit in self._limits:

            resource.setrlimit(*limit)

    def run(self, command, *, check=False, timeout=False):

        # separate the command name frome the optiosn with --

        self._cmd.append(command[0])

        self._cmd.append('--')

        self._cmd.extend(command[1:])

        return subprocess.run(

            self._cmd,

            stdout=subprocess.PIPE,

            stderr=subprocess.STDOUT,

            check=check,

            timeout=timeout,

            text=check,

            preexec_fn=self._preexec_limiter,

        )

class Diffoscope(Schroot):

    def __init__(self, chroot=None, suite=None):

        if suite is None:

            suite = 'unstable'

        if chroot is None:

            chroot = f'jenkins-reproducible-{suite}-diffoscope'

        super.__init__(chroot=chroot)

        self.set_default_limits()

    def version(self):

        try:

            p = self.run(('diffoscope', '--version'), check=True)

        except subprocess.CalledProcessError:

            return 'cannot get version'

        return p.stdout.strip()

    def compare_files(self, a, b, html):

        self.set_directory(os.path.dirname(html))

        diff_cmd = ['diffoscope', '--html', html]

        msg = f'diffoscope {self.version()}'

        timeout = 30*60  # 30 minutes

        try:

            p = self.run(diff_cmd + (a, b), timeout=timeout)

            if p.returncode == 0:

                print(f'{msg}: {a} is reproducible, yay!')

            elif p.returncode == 1:

                print(f'{msg}: {a} has issue, please investigate')

            elif p.returncode == 2:

                with open(html, 'w') as f:

                    f.write(f'{msg} had errors comparing the two builds.\n')

                    f.write(f'{diff_cmd}\n')

                    f.write(p.stdout)

        except subprocess.TimeoutExpired:

            if os.path.exits(html):

                text = (f'{msg} produced not output and was killed after '

                        f'running into timeout after {timeout}.')

                with open(html, 'w') as f:

                    f.write(f'text\n')

                    f.write(f'{diff_cmd}\n')

                    f.write(p.stdout)

                print(text)

            else:

                print(f'{msg} was killed after running into timeout after '

                      f'{timeout}, but there still {html}')

#!/usr/bin/env python3

# -*- coding: utf-8 -*-

#

# Copyright © 2019 Paul Spooren <mail@aparcar.org>

#

# Inspired by reproducible_common.sh

#   © Holger Levsen <holger@layer-acht.org>

#

# Released under the GPLv2

import os

import subprocess

import resource

def e(var, default):

    """Return env variable or default"""

    return os.environ.get(var, default)

# DBSUITE which version of diffoscope to use

timeout = e("TIMEOUT", 30 * 60)  # 30m

# DIFFOSCOPE_VIRT_LIMIT max RAM usage

ds_virt_limit = int(e("DIFFOSCOPE_VIRT_LIMIT", 10 * 1024 ** 3))  # 10GB

# TIMEOUT timeout for diffoscope in seconds

dbsuite = e("DBDSUITE", "unstable")

# SCHROOT to use

schroot = e("SCHROOT", f"source:jenkins-reproducible-{dbsuite}-diffoscope")

def limit_resources():

    resource.setrlimit(resource.RLIMIT_CPU, (1, 1))

    resource.setrlimit(resource.RLIMIT_AS, (ds_virt_limit, resource.RLIM_INFINITY))

def diffoscope_version():

    cmd = []

    if schroot:

        cmd.extend(["schroot", "--directory", "/tmp", "-c", schroot])

    cmd.extend(["diffoscope", "--", "--version"])

    print(cmd)

    return (

        subprocess.run(cmd, capture_output=True, text=True, preexec_fn=limit_resources)

        .stdout.strip()

        .split()[1]

    )

def diffoscope_compare(path_a, path_b, path_output_html):

    """

    Run diffoscope in a schroot environment

    Args:

    - path_a path to first file to compare

    - path_b path to second file a to compare

    - path_output_html path where to store result html

    """

    cmd = []

    if schroot:

        cmd.extend(

            ["schroot", "--directory", os.path.dirname(path_output_html), "-c", schroot]

        )

    try:

        cmd.extend(["diffoscope", "--", "--html", path_output_html, path_a, path_b])

        result = subprocess.run(

            cmd,

            timeout=timeout,

            capture_output=True,

            text=True,

            preexec_fn=limit_resources,

        )

        msg = f"diffoscope {diffoscope_version()} "

        if result.returncode == 0:

            print(msg + f"{path_a} reproducible, yay!")

        else:

            if result.returncode == 1:

                print(msg + f"found issues, please investigate {path_a}")

            elif result.returncode == 2:

                with open(path_output_html, "w") as output_html_file:

                    output_html_file.write(

                        msg

                        + f"""had trouble comparing the two builds. Please

                                investigate {path_a}"""

                    )

    except subprocess.TimeoutExpired:

        if os.path.exists(path_output_html):

            print(

                msg

                + f"""produced no output comparing {path_a} with {path_b} and

                    was killed after running into timeout after {timeout}..."""

            )

        else:

            print(

                msg

                + """was killed after running into timeout after $TIMEOUT, but

                    there is still {path_output_html}"""

            )

import shutil

import json

from reproducible_common import diffoscope_compare, diffoscope_version

from .rblib.commands import Diffoscope

from reproducible_openwrt_package_parser import insert_into_db, show_list_difference

# target to be build

    "packages_repro_percent": 0,

    "packages_total": 0,

    "today": strftime("%Y-%m-%d", gmtime()),

    "diffoscope_version": diffoscope_version(),

    "diffoscope_version": Diffoscope().version(),

    "target": target,

    "images": [],

    "packages": [],

        print("Error downloading {}".format(origin_name))

        return

    diffoscope_compare(

    Diffoscope().compare(

        file_origin.name,

        target_dir + "/" + origin_name,

        results_target_dir + "/" + origin_name + ".html",

        html=results_target_dir + "/" + origin_name + ".html",

    )

    file_origin.close()

    print("downloading {}".format(url))

    try:

        content = urlopen(url).read()

    except:

    except Exception:

        return 1

    if path: