bin/rblib/__init__.py

+# -*- coding: utf-8 -*-
+#
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@debian.org>
+# Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>
+# Licensed under GPL-2
+
+from sqlalchemy import Table
+from sqlalchemy.exc import NoSuchTableError, OperationalError
+
+from .confparse import log
+from .const import PGDATABASE, DB_METADATA, conn_db
+from .utils import print_critical_message
+
+
+def db_table(table_name):
+    """Returns a SQLAlchemy Table objects to be used in queries
+    using SQLAlchemy's Expressive Language.
+
+    Arguments:
+        table_name: a string corrosponding to an existing table name
+    """
+    try:
+        return Table(table_name, DB_METADATA, autoload=True)
+    except NoSuchTableError:
+        log.error(
+            "Table %s does not exist or schema for %s could not be loaded",
+            table_name, PGDATABASE)
+        raise
+
+
+def query_db(query, *args, **kwargs):
+    """Excutes a raw SQL query. Return depends on query type.
+
+    Returns:
+        select:
+            list of tuples
+        update or delete:
+            the number of rows affected
+        insert:
+            None
+    """
+    try:
+        result = conn_db.execute(query, *args, **kwargs)
+    except OperationalError as ex:
+        print_critical_message('Error executing this query:\n' + query)
+        raise
+
+    if result.returns_rows:
+        return result.fetchall()
+    elif result.supports_sane_rowcount() and result.rowcount > -1:
+        return result.rowcount
+    else:
+        return None
+
+
+def get_status_icon(status):
+    table = {'reproducible': 'weather-clear.png',
+             'FTBFS': 'weather-storm.png',
+             'FTBR': 'weather-showers-scattered.png',
+             '404': 'weather-severe-alert.png',
+             'depwait': 'weather-snow.png',
+             'not for us': 'weather-few-clouds-night.png',
+             'not_for_us': 'weather-few-clouds-night.png',
+             'untested': 'weather-clear-night.png',
+             'blacklisted': 'error.png'}
+    spokenstatus = status
+    if status == 'unreproducible':
+            status = 'FTBR'
+    elif status == 'not for us':
+            status = 'not_for_us'
+    try:
+        return (status, table[status], spokenstatus)
+    except KeyError:
+        log.error('Status ' + status + ' not recognized')
+        return (status, '', spokenstatus)
+
+
+def get_trailing_bug_icon(bug, bugs, package=None):
+    html = ''
+    if not package:
+        for pkg in bugs.keys():
+            if get_trailing_bug_icon(bug, bugs, pkg):
+                return get_trailing_bug_icon(bug, bugs, pkg)
+    else:
+        try:
+            if bug in bugs[package].keys():
+                html += '<span class="'
+                if bugs[package][bug]['done']:
+                    html += 'bug-done" title="#' + str(bug) + ', done">#'
+                elif bugs[package][bug]['pending']:
+                    html += 'bug-pending" title="#' + str(bug) + ', pending">P'
+                elif bugs[package][bug]['patch']:
+                    html += 'bug-patch" title="#' + str(bug) + ', with patch">+'
+                else:
+                    html += 'bug">'
+                html += '</span>'
+        except KeyError:
+            pass
+    return html

bin/rblib/bugs.py

+# -*- coding: utf-8 -*-
+#
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@debian.org>
+# Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>
+# Licensed under GPL-2
+
+import psycopg2
+
+from .confparse import log
+
+
+class Udd:
+    __singleton = {}
+
+    def __init__(self):
+        self.__dict__ = self.__singleton
+        if not self.__singleton:
+            self._conn_udd = None
+
+    @property
+    def _conn(self):
+        if self._conn_udd is not None:
+            return self._conn_udd
+        username = "public-udd-mirror"
+        password = "public-udd-mirror"
+        host = "public-udd-mirror.xvm.mit.edu"
+        port = 5432
+        db = "udd"
+        try:
+            try:
+                log.debug("Starting connection to the UDD database")
+                conn = psycopg2.connect(
+                    dbname=db,
+                    user=username,
+                    password=password,
+                    host=host,
+                    port=port,
+                    connect_timeout=5,
+                )
+                conn.set_client_encoding('utf8')
+            except psycopg2.OperationalError as err:
+                if str(err) == 'timeout expired\n':
+                    log.error('Connection to the UDD database timed out.')
+                    log.error('Maybe the machine is offline or unavailable.')
+                    log.error('Failing nicely, all queries will return an '
+                              'empty response.')
+                    conn = False
+                else:
+                    raise
+        except Exception:
+            log.exception('Erorr connecting to the UDD database replica. '
+                          'The full error is:')
+            log.error('Failing nicely , all queries will return an empty '
+                      'response.')
+            conn = False
+        self._conn_udd = conn
+        return conn
+
+    def query(self, query):
+        if not self._conn:
+            log.error('There has been an error connecting to UDD. '
+                      'Look for a previous error for more information.')
+            log.error('Failing nicely, returning an empty response.')
+            return []
+        try:
+            cursor = self._conn.cursor()
+            cursor.execute(query)
+        except Exception:
+            log.exception('The UDD server encountered a issue while '
+                          'executing the query.  The full error is:')
+            log.error('Failing nicely, returning an empty response.')
+            return []
+        return cursor.fetchall()
+
+
+class Bugs:
+    __singleton = {}
+    _query = """
+        SELECT bugs.id, bugs.source, bugs.done, ARRAY_AGG(tags.tag), bugs.title
+        FROM bugs JOIN bugs_usertags ON bugs.id = bugs_usertags.id
+                  LEFT JOIN (
+                    SELECT id, tag FROM bugs_tags
+                    WHERE tag='patch' OR tag='pending'
+                  ) AS tags ON bugs.id = tags.id
+        WHERE
+            bugs_usertags.email = 'reproducible-builds@lists.alioth.debian.org'
+        AND bugs.id NOT IN (
+            SELECT id
+            FROM bugs_usertags
+            WHERE email = 'reproducible-builds@lists.alioth.debian.org'
+            AND (
+                bugs_usertags.tag = 'toolchain'
+                OR bugs_usertags.tag = 'infrastructure')
+            )
+        GROUP BY bugs.id, bugs.source, bugs.done
+    """
+
+    def __init__(self):
+        self.__dict__ = self.__singleton
+        if not self.__singleton:
+            self._bugs = {}
+
+    @property
+    def bugs(self):
+        """
+        This function returns a dict:
+        { "package_name": {
+            bug1: {patch: True, done: False, title: "string"},
+            bug2: {patch: False, done: False, title: "string"},
+           }
+        }
+        """
+        if self._bugs:
+            return self._bugs
+
+        log.info("Finding out which usertagged bugs have been closed or at "
+                 "least have patches")
+        # returns a list of tuples [(id, source, done)]
+        rows = Udd().query(self._query)
+        packages = {}
+        for bug in rows:
+            # bug[0] = bug_id
+            # bug[1] = source_name
+            # bug[2] = who_when_done
+            # bug[3] = tag (patch or pending)
+            # bug[4] = title
+            if bug[1] not in packages:
+                packages[bug[1]] = {}
+            packages[bug[1]][bug[0]] = {
+                'done': False,
+                'patch': False,
+                'pending': False,
+                'title': bug[4],
+            }
+            if bug[2]:  # if the bug is done
+                packages[bug[1]][bug[0]]['done'] = True
+            if 'patch' in bug[3]:  # the bug is patched
+                packages[bug[1]][bug[0]]['patch'] = True
+            if 'pending' in bug[3]:  # the bug is pending
+                packages[bug[1]][bug[0]]['pending'] = True
+        self._bugs = packages
+        return packages
+
+    def get_trailing_icon(self, package):
+        """
+        determine the HTML representation of the bug status
+        """
+        html = ''
+
+        if package in self.bugs:
+            bb = self.bugs[package]
+            for bug in bb:
+                html += '<a href="https://bugs.debian.org/{bug}">'
+                html += '<span class="'
+                if bb[bug]['done']:
+                    html += 'bug-done" title="#{bug}, done">#</span>'
+                elif bb[bug]['pending']:
+                    html += 'bug-pending" title"#{bug}, pending">P</span>'
+                elif bb[bug]['patch']:
+                    html += 'bug-patch" title="#{bug}, with patch">+</span>'
+                else:
+                    html += 'bug" title="#{bug}">#</span>'
+                html = html.format(bug=bug) + '</a>'
+        return html

bin/rblib/confparse.py

+# -*- coding: utf-8 -*-
+#
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@debian.org>
+# Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>
+# Licensed under GPL-2
+
+
+import os
+import sys
+import atexit
+import logging
+import argparse
+import configparser
+from datetime import datetime
+
+DEBUG = False
+QUIET = False
+
+__location__ = os.path.realpath(
+    os.path.join(os.getcwd(), os.path.dirname(__file__), '..'))
+
+CONFIG = os.path.join(__location__, 'reproducible.ini')
+
+# command line option parsing
+parser = argparse.ArgumentParser()
+group = parser.add_mutually_exclusive_group()
+parser.add_argument('--distro', help='name of the distribution to work on',
+                    default='debian', nargs='?')
+group.add_argument("-d", "--debug", action="store_true")
+group.add_argument("-q", "--quiet", action="store_true")
+parser.add_argument("--skip-database-connection", action="store_true",
+                    help="skip connecting to database")
+parser.add_argument("--ignore-missing-files", action="store_true",
+                    help="useful for local testing, where you don't have all "
+                    "the build logs, etc..")
+args, unknown_args = parser.parse_known_args()
+DISTRO = args.distro
+log_level = logging.INFO
+if args.debug or DEBUG:
+    DEBUG = True
+    log_level = logging.DEBUG
+if args.quiet or QUIET:
+    log_level = logging.ERROR
+log = logging.getLogger(__name__)
+log.setLevel(log_level)
+sh = logging.StreamHandler()
+sh.setFormatter(logging.Formatter(
+    '[%(asctime)s] %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S'))
+log.addHandler(sh)
+
+started_at = datetime.now()
+log.info('Starting at %s', started_at)
+
+
+# load configuration
+config = configparser.ConfigParser()
+config.read(CONFIG)
+try:
+    conf_distro = config[DISTRO]
+except KeyError:
+    log.critical('Distribution %s is not known.', DISTRO)
+    sys.exit(1)
+
+
+@atexit.register
+def print_time():
+    log.info('Finished at %s, took: %s', datetime.now(),
+             datetime.now()-started_at)

bin/rblib/const.py

+# -*- coding: utf-8 -*-
+#
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@debian.org>
+# Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>
+# Licensed under GPL-2
+
+import os
+import csv
+from urllib.parse import urljoin
+from sqlalchemy import MetaData, create_engine
+
+from .confparse import (
+    __location__,
+    args,
+    conf_distro,
+    log,
+    DISTRO,
+)
+
+# tested suites
+SUITES = conf_distro['suites'].split()
+# tested architectures
+ARCHS = conf_distro['archs'].split()
+# defaults
+defaultsuite = conf_distro['defaultsuite']
+defaultarch = conf_distro['defaultarch']
+
+BIN_PATH = __location__
+BASE = conf_distro['basedir']
+TEMPLATE_PATH = conf_distro['templates']
+PKGSET_DEF_PATH = '/srv/reproducible-results'
+TEMP_PATH = conf_distro['tempdir']
+
+REPRODUCIBLE_STYLES = os.path.join(BASE, conf_distro['css'])
+
+DISTRO_URI = '/' + conf_distro['distro_root']
+DISTRO_BASE = os.path.join(BASE, conf_distro['distro_root'])
+
+DBD_URI = os.path.join(DISTRO_URI, conf_distro['diffoscope_html'])
+DBDTXT_URI = os.path.join(DISTRO_URI, conf_distro['diffoscope_txt'])
+LOGS_URI = os.path.join(DISTRO_URI, conf_distro['buildlogs'])
+DIFFS_URI = os.path.join(DISTRO_URI, conf_distro['logdiffs'])
+NOTES_URI = os.path.join(DISTRO_URI, conf_distro['notes'])
+ISSUES_URI = os.path.join(DISTRO_URI, conf_distro['issues'])
+RB_PKG_URI = os.path.join(DISTRO_URI, conf_distro['packages'])
+RBUILD_URI = os.path.join(DISTRO_URI, conf_distro['rbuild'])
+HISTORY_URI = os.path.join(DISTRO_URI, conf_distro['pkghistory'])
+BUILDINFO_URI = os.path.join(DISTRO_URI, conf_distro['buildinfo'])
+DBD_PATH = BASE + DBD_URI
+DBDTXT_PATH = BASE + DBDTXT_URI
+LOGS_PATH = BASE + LOGS_URI
+DIFFS_PATH = BASE + DIFFS_URI
+NOTES_PATH = BASE + NOTES_URI
+ISSUES_PATH = BASE + ISSUES_URI
+RB_PKG_PATH = BASE + RB_PKG_URI
+RBUILD_PATH = BASE + RBUILD_URI
+HISTORY_PATH = BASE + HISTORY_URI
+BUILDINFO_PATH = BASE + BUILDINFO_URI
+
+REPRODUCIBLE_JSON = os.path.join(DISTRO_BASE, conf_distro['json_out'])
+REPRODUCIBLE_TRACKER_JSON = os.path.join(DISTRO_BASE, conf_distro['tracker.json_out'])
+
+REPRODUCIBLE_URL = conf_distro['base_url']
+DISTRO_URL = urljoin(REPRODUCIBLE_URL, conf_distro['distro_root'])
+DISTRO_DASHBOARD_URI = os.path.join(DISTRO_URI, conf_distro['landing_page'])
+JENKINS_URL = conf_distro['jenkins_url']
+
+# global package set definitions
+# META_PKGSET[pkgset_id] = (pkgset_name, pkgset_group)
+# csv file columns: (pkgset_group, pkgset_name)
+META_PKGSET = []
+with open(os.path.join(BIN_PATH, 'reproducible_pkgsets.csv'), newline='') as f:
+    for line in csv.reader(f):
+        META_PKGSET.append((line[1], line[0]))
+
+# DATABSE CONSTANT
+PGDATABASE = 'reproducibledb'
+
+
+# init the database data and connection
+if not args.skip_database_connection:
+    DB_ENGINE = create_engine("postgresql:///%s" % PGDATABASE)
+    DB_METADATA = MetaData(DB_ENGINE)  # Get all table definitions
+    conn_db = DB_ENGINE.connect()      # the local postgres reproducible db
+
+for key, value in conf_distro.items():
+    log.debug('%-16s: %s', key, value)
+log.debug("BIN_PATH:\t" + BIN_PATH)
+log.debug("BASE:\t\t" + BASE)
+log.debug("DISTRO:\t\t" + DISTRO)
+log.debug("DBD_URI:\t\t" + DBD_URI)
+log.debug("DBD_PATH:\t" + DBD_PATH)
+log.debug("DBDTXT_URI:\t" + DBDTXT_URI)
+log.debug("DBDTXT_PATH:\t" + DBDTXT_PATH)
+log.debug("LOGS_URI:\t" + LOGS_URI)
+log.debug("LOGS_PATH:\t" + LOGS_PATH)
+log.debug("DIFFS_URI:\t" + DIFFS_URI)
+log.debug("DIFFS_PATH:\t" + DIFFS_PATH)
+log.debug("NOTES_URI:\t" + NOTES_URI)
+log.debug("ISSUES_URI:\t" + ISSUES_URI)
+log.debug("NOTES_PATH:\t" + NOTES_PATH)
+log.debug("ISSUES_PATH:\t" + ISSUES_PATH)
+log.debug("RB_PKG_URI:\t" + RB_PKG_URI)
+log.debug("RB_PKG_PATH:\t" + RB_PKG_PATH)
+log.debug("RBUILD_URI:\t" + RBUILD_URI)
+log.debug("RBUILD_PATH:\t" + RBUILD_PATH)
+log.debug("HISTORY_URI:\t" + HISTORY_URI)
+log.debug("HISTORY_PATH:\t" + HISTORY_PATH)
+log.debug("BUILDINFO_URI:\t" + BUILDINFO_URI)
+log.debug("BUILDINFO_PATH:\t" + BUILDINFO_PATH)
+log.debug("REPRODUCIBLE_JSON:\t" + REPRODUCIBLE_JSON)
+log.debug("JENKINS_URL:\t\t" + JENKINS_URL)
+log.debug("REPRODUCIBLE_URL:\t" + REPRODUCIBLE_URL)
+log.debug("DISTRO_URL:\t" + DISTRO_URL)
+
+if args.ignore_missing_files:
+    log.warning("Missing files will be ignored!")
+
+try:
+    JOB_URL = os.environ['JOB_URL']
+except KeyError:
+    JOB_URL = ''
+    JOB_NAME = ''
+else:
+    JOB_NAME = os.path.basename(JOB_URL[:-1])
+
+
+# filter used on the index_FTBFS pages and for the reproducible.json
+filtered_issues = (
+    'ftbfs_in_jenkins_setup',
+    'ftbfs_build_depends_not_available_on_amd64',
+    'ftbfs_build-indep_not_build_on_some_archs'
+)
+filter_query = ''
+for issue in filtered_issues:
+    if filter_query == '':
+        filter_query = "n.issues LIKE '%%{}%%'".format(issue)
+        filter_html = '<a href="{}{}/$suite/{}_issue.html">{}</a>'.format(
+            REPRODUCIBLE_URL, ISSUES_URI, issue, issue)
+    else:
+        filter_query += " OR n.issues LIKE '%%{}%%'".format(issue)
+        filter_html = 'or <a href="{}{}/$suite/{}_issue.html">{}</a>'.format(
+            REPRODUCIBLE_URL, ISSUES_URI, issue, issue)

bin/rblib/html.py

+# -*- coding: utf-8 -*-
+#
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@debian.org>
+# Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>
+# Licensed under GPL-2
+
+import os
+import errno
+import hashlib
+import pystache
+from datetime import datetime
+
+from .confparse import log, conf_distro
+from .const import (
+    defaultsuite, defaultarch,
+    SUITES, ARCHS,
+    DISTRO_DASHBOARD_URI,
+    JENKINS_URL, JOB_URL, JOB_NAME,
+    TEMPLATE_PATH, REPRODUCIBLE_STYLES,
+)
+
+
+tab = '  '
+
+# take a SHA1 of the css page for style version
+_hasher = hashlib.sha1()
+with open(REPRODUCIBLE_STYLES, 'rb') as f:
+        _hasher.update(f.read())
+REPRODUCIBLE_STYLE_SHA1 = _hasher.hexdigest()
+
+# Templates used for creating package pages
+_renderer = pystache.Renderer()
+status_icon_link_template = _renderer.load_template(
+    TEMPLATE_PATH + '/status_icon_link')
+default_page_footer_template = _renderer.load_template(
+    TEMPLATE_PATH + '/default_page_footer')
+pkg_legend_template = _renderer.load_template(
+    TEMPLATE_PATH + '/pkg_symbol_legend')
+project_links_template = _renderer.load_template(
+    os.path.join(TEMPLATE_PATH, 'project_links'))
+main_navigation_template = _renderer.load_template(
+    os.path.join(TEMPLATE_PATH, 'main_navigation'))
+basic_page_template = _renderer.load_template(
+    os.path.join(TEMPLATE_PATH, 'basic_page'))
+
+
+def _create_default_page_footer(date):
+    return _renderer.render(default_page_footer_template, {
+            'date': date,
+            'job_url': JOB_URL,
+            'job_name': JOB_NAME,
+            'jenkins_url': JENKINS_URL,
+        })
+
+
+def _gen_suite_arch_nav_context(suite, arch, suite_arch_nav_template=None,
+                                ignore_experimental=False, no_suite=None,
+                                no_arch=None):
+    # if a template is not passed in to navigate between suite and archs the
+    # current page, we use the "default" suite/arch summary view.
+    default_nav_template = \
+            '/{{distro}}/{{suite}}/index_suite_{{arch}}_stats.html'
+    if not suite_arch_nav_template:
+        suite_arch_nav_template = default_nav_template
+
+    suite_list = []
+    if not no_suite:
+        for s in SUITES:
+            include_suite = True
+            if s == 'experimental' and ignore_experimental:
+                include_suite = False
+            suite_list.append({
+                's': s,
+                'class': 'current' if s == suite else '',
+                'uri': _renderer.render(suite_arch_nav_template,
+                                       {'distro': conf_distro['distro_root'],
+                                        'suite': s, 'arch': arch})
+                if include_suite else '',
+            })
+
+    arch_list = []
+    if not no_arch:
+        for a in ARCHS:
+            arch_list.append({
+                'a': a,
+                'class': 'current' if a == arch else '',
+                'uri': _renderer.render(suite_arch_nav_template,
+                                       {'distro': conf_distro['distro_root'],
+                                        'suite': suite, 'arch': a}),
+            })
+    return (suite_list, arch_list)
+
+
+# See bash equivelent: reproducible_common.sh's "write_page_header()"
+def create_main_navigation(suite=defaultsuite, arch=defaultarch,
+                           displayed_page=None, suite_arch_nav_template=None,
+                           ignore_experimental=False, no_suite=None,
+                           no_arch=None):
+    suite_list, arch_list = _gen_suite_arch_nav_context(suite, arch,
+            suite_arch_nav_template, ignore_experimental, no_suite, no_arch)
+    context = {
+        'suite': suite,
+        'arch': arch,
+        'project_links_html': _renderer.render(project_links_template),
+        'suite_nav': {
+            'suite_list': suite_list
+        } if len(suite_list) else '',
+        'arch_nav': {
+            'arch_list': arch_list
+        } if len(arch_list) else '',
+        'debian_uri': DISTRO_DASHBOARD_URI,
+        'cross_suite_arch_nav': True if suite_arch_nav_template else False,
+    }
+    if suite != 'experimental':
+        # there are not package sets in experimental
+        context['include_pkgset_link'] = True
+    # the "display_page" argument controls which of the main page navigation
+    # items will be highlighted.
+    if displayed_page:
+        context[displayed_page] = True
+    return _renderer.render(main_navigation_template, context)
+
+
+def write_html_page(title, body, destfile, no_header=False, style_note=False,
+                    noendpage=False, refresh_every=None, displayed_page=None,
+                    left_nav_html=None):
+    meta_refresh_html = '<meta http-equiv="refresh" content="%d"></meta>' % \
+        refresh_every if refresh_every is not None else ''
+    if style_note:
+        body += _renderer.render(pkg_legend_template, {})
+    if not noendpage:
+        now = datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC')
+        body += _create_default_page_footer(now)
+    context = {
+        'page_title': title,
+        'meta_refresh_html': meta_refresh_html,
+        'navigation_html': left_nav_html,
+        'main_header': title if not no_header else "",
+        'main_html': body,
+        'style_dot_css_sha1sum': REPRODUCIBLE_STYLE_SHA1,
+    }
+    html = _renderer.render(basic_page_template, context)
+
+    try:
+        os.makedirs(destfile.rsplit('/', 1)[0], exist_ok=True)
+    except OSError as e:
+        if e.errno != errno.EEXIST:  # that's 'File exists' error (errno 17)
+            raise
+    log.debug("Writing " + destfile)
+    with open(destfile, 'w', encoding='UTF-8') as fd:
+        fd.write(html)
+
+
+def gen_status_link_icon(status, spokenstatus, icon, suite, arch):
+    """
+    Returns the html for "<icon> <spokenstatus>" with both icon and status
+    linked to the appropriate index page for the status, arch and suite.
+
+    If icon is set to None, the icon will be ommited.
+    If spokenstatus is set to None, the spokenstatus link be ommited.
+    """
+    context = {
+        'status': status,
+        'spokenstatus': spokenstatus,
+        'icon': icon,
+        'suite': suite,
+        'arch': arch,
+        'untested': True if status == 'untested' else False,
+    }
+    return _renderer.render(status_icon_link_template, context)

bin/rblib/models.py

+# -*- coding: utf-8 -*-
+#
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@debian.org>
+# Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>
+# Licensed under GPL-2
+
+import os
+import json
+import os.path
+import functools
+import html as HTML
+
+from .const import (
+    ARCHS,
+    SUITES,
+    defaultarch,
+    defaultsuite,
+    RB_PKG_URI,
+    BUILDINFO_PATH, BUILDINFO_URI,
+    RBUILD_PATH, RBUILD_URI,
+    LOGS_PATH, LOGS_URI,
+    DIFFS_PATH, DIFFS_URI,
+)
+from .bugs import Bugs
+from .utils import strip_epoch
+from . import query_db
+
+
+def lazyproperty(fn):
+    attr_name = '_l_' + fn.__name__
+
+    @property
+    @functools.wraps(fn)
+    def _lazy(self):
+        if not hasattr(self, attr_name):
+            fn(self)
+        return getattr(self, attr_name)
+
+    return _lazy
+
+
+class Bug:
+    def __init__(self, bug):
+        self.bug = bug
+
+    def __str__(self):
+        return str(self.bug)
+
+
+class Issue:
+    def __init__(self, name):
+        self.name = name
+
+    @lazyproperty
+    def url(self):
+        self._set()
+
+    @lazyproperty
+    def desc(self):
+        self._set()
+
+    def _set(self):
+        query = "SELECT url, description  FROM issues WHERE name='{}'"
+        result = query_db(query.format(self.name))
+        try:
+            self._l_url = result[0][0]
+        except IndexError:
+            self._l_url = ''
+        try:
+            self._l_desc = result[0][1]
+        except IndexError:
+            self._l_desc = ''
+
+
+class Note:
+    def __init__(self, pkg, results):
+        self.issues = [Issue(x) for x in json.loads(results[0])]
+        self.bugs = [Bug(x) for x in json.loads(results[1])]
+        self.comment = results[2]
+
+
+class Build:
+    class __file:
+        def __init__(self, pkg, filename, base_path=None, base_url=None,
+                     path_templ=None, url_templ=None, formatter=None):
+            fmt = {
+                'pkg': pkg.package,
+                'eversion': strip_epoch(pkg.version),
+                'arch': pkg.arch,
+                'suite': pkg.suite,
+            }
+            if path_templ is None:
+                path_templ = os.path.join(base_path, pkg.suite, pkg.arch, filename)
+            if url_templ is None:
+                url_templ = base_url + '/{suite}/{arch}/{file}'
+            if formatter is not None:
+                fmt = {**fmt, **formatter}
+            if 'file' not in fmt:
+                fmt['file'] = filename.format_map(fmt)
+            self.path = path_templ.format_map(fmt)
+            self.url = url_templ.format_map(fmt)
+
+        def __bool__(self):
+            return os.access(self.path, os.R_OK)
+
+        @property
+        def size(self):
+            return os.stat(self.path).st_size
+
+    def __init__(self, package, suite, arch):
+        self.package = package
+        self.suite = suite
+        self.arch = arch
+
+    @lazyproperty
+    def status(self):
+        self._get_package_status()
+
+    @lazyproperty
+    def version(self):
+        self._get_package_status()
+
+    @lazyproperty
+    def build_date(self):
+        self._get_package_status()
+
+    def _get_package_status(self):
+        try:
+            query = """SELECT r.status, r.version, r.build_date
+                       FROM results AS r JOIN sources AS s
+                       ON r.package_id=s.id WHERE s.name='{}'
+                       AND s.architecture='{}' AND s.suite='{}'"""
+            query = query.format(self.package, self.arch, self.suite)
+            result = query_db(query)[0]
+        except IndexError:  # not tested, look whether it actually exists
+            query = """SELECT version FROM sources WHERE name='{}'
+                       AND suite='{}' AND architecture='{}'"""
+            query = query.format(self.package, self.suite, self.arch)
+            try:
+                result = query_db(query)[0][0]
+                if result:
+                    result = ('untested', str(result), None)
+            except IndexError:  # there is no package with this name in this
+                result = (None, None, None)  # suite/arch, or none at all
+        self._l_status = result[0]
+        self._l_version = result[1]
+        self._l_build_date = str(result[2]) + ' UTC' if result[2] else None
+
+    @lazyproperty
+    def note(self):
+        query = """
+            SELECT n.issues, n.bugs, n.comments
+            FROM sources AS s JOIN notes AS n ON s.id=n.package_id
+            WHERE s.name='{}' AND s.suite='{}' AND s.architecture='{}'
+        """
+        result = query_db(query.format(self.package, self.suite, self.arch))
+        try:
+            result = result[0]
+        except IndexError:
+            self._l_note = None
+        else:
+            self._l_note = Note(self, result)
+
+    @lazyproperty
+    def buildinfo(self):
+        filename = '{pkg}_{eversion}_{arch}.buildinfo'
+        self._l_buildinfo = self.__file(self, filename, BUILDINFO_PATH, BUILDINFO_URI)
+
+    @lazyproperty
+    def rbuild(self):
+        filename = '{pkg}_{eversion}.rbuild.log.gz'
+        self._l_rbuild = self.__file(self, filename, RBUILD_PATH, RBUILD_URI)
+
+    @lazyproperty
+    def build2(self):
+        filename = '{pkg}_{eversion}.build2.log.gz'
+        self._l_build2 = self.__file(self, filename, LOGS_PATH, LOGS_URI)
+
+    @lazyproperty
+    def logdiff(self):
+        filename = '{pkg}_{eversion}.diff.gz'
+        self._l_logdiff = self.__file(self, filename, DIFFS_PATH, DIFFS_URI)
+
+
+class _Package_cache:
+    __singleton = {}
+
+    def __init__(self):
+        self.__dict__ = self.__singleton
+        if not self.__singleton:
+            self._cache = {}
+
+    def get(self, pkgname):
+        try:
+            return self._cache[pkgname]
+        except KeyError:
+            self._cache[pkgname] = {}
+            return self._cache[pkgname]
+
+
+class Package:
+    def __init__(self, name, no_notes=False):
+        self.__dict__ = _Package_cache().get(name)
+        self.name = name
+
+    @lazyproperty
+    def builds(self):
+        self._l_builds = {}
+        for suite in SUITES:
+            self._l_builds[suite] = {}
+            for arch in ARCHS:
+                self._l_builds[suite][arch] = Build(self.name, suite, arch)
+
+    @lazyproperty
+    def status(self):
+        try:
+            self._l_status = self.builds[defaultsuite][defaultarch].status
+        except KeyError:
+            self._l_status = False
+
+    @lazyproperty
+    def note(self):
+        try:
+            self._l_note = self.builds[defaultsuite][defaultarch].note
+        except KeyError:
+            self._l_note = False
+
+    @lazyproperty
+    def notify_maint(self):
+        query = "SELECT notify_maintainer FROM sources WHERE name='{}'"
+        try:
+            result = int(query_db(query.format(self.name))[0][0])
+        except IndexError:
+            result = 0
+        self._l_notify_maint = '⚑' if result == 1 else ''
+
+    @lazyproperty
+    def history(self):
+        self._l_history = []
+        keys = [
+            'build ID', 'version', 'suite', 'architecture', 'result',
+            'build date', 'build duration', 'node1', 'node2', 'job',
+            'schedule message'
+        ]
+        query = """
+                SELECT id, version, suite, architecture, status, build_date,
+                    build_duration, node1, node2, job
+                FROM stats_build WHERE name='{}' ORDER BY build_date DESC
+            """.format(self.name)
+        results = query_db(query)
+        for record in results:
+            self._l_history.append(dict(zip(keys, record)))
+
+    def html_link(self, suite, arch, bugs=True, popcon=None, is_popular=None):
+        url = '/'.join((RB_PKG_URI, suite, arch, self.name+'.html'))
+        css_classes = []
+        title = ''
+        if is_popular:
+            css_classes.append('package-popular')
+        if popcon is not None:
+            title += 'popcon score: {}\n'.format(popcon)
+        notes = self.builds[suite][arch].note
+        if notes is None:
+            css_classes.append('package')
+        else:
+            css_classes.append('noted')
+            title += '\n'.join([x.name for x in notes.issues]) + '\n'
+            title += '\n'.join([str(x.bug) for x in notes.bugs]) + '\n'
+            if notes.comment:
+                title += HTML.escape(notes.comment)
+        html = '<a href="{url}" class="{cls}" title="{title}">{pkg}</a>{ico}\n'
+        bug_icon = Bugs().get_trailing_icon(self.name) if bugs else ''
+        return html.format(url=url, cls=' '.join(css_classes),
+                           title=title, pkg=self.name, ico=bug_icon)

bin/rblib/utils.py

+# -*- coding: utf-8 -*-
+#
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@debian.org>
+# Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>
+# Licensed under GPL-2
+
+import os
+import re
+import sys
+import subprocess
+from tempfile import NamedTemporaryFile
+
+from rblib.const import log, TEMP_PATH, JOB_NAME
+
+
+url2html = re.compile(r'((mailto\:|((ht|f)tps?)\://|file\:///){1}\S+)')
+
+
+class bcolors:
+    BOLD = '\033[1m' if sys.stdout.isatty() else ''
+    UNDERLINE = '\033[4m' if sys.stdout.isatty() else ''
+    RED = '\033[91m' if sys.stdout.isatty() else ''
+    GOOD = '\033[92m' if sys.stdout.isatty() else ''
+    WARN = '\033[93m' + UNDERLINE if sys.stdout.isatty() else ''
+    FAIL = RED + BOLD + UNDERLINE
+    ENDC = '\033[0m' if sys.stdout.isatty() else ''
+
+
+def print_critical_message(msg):
+    print('\n\n\n')
+    try:
+        for line in msg.splitlines():
+            log.critical(line)
+    except AttributeError:
+        log.critical(msg)
+    print('\n\n\n')
+
+
+def create_temp_file(mode='w+b'):
+    os.makedirs(TEMP_PATH, exist_ok=True)
+    return NamedTemporaryFile(suffix=JOB_NAME, dir=TEMP_PATH, mode=mode)
+
+
+def convert_into_hms_string(duration):
+    if not duration:
+        duration = ''
+    else:
+        duration = int(duration)
+        hours = int(duration/3600)
+        minutes = int((duration-(hours*3600))/60)
+        seconds = int(duration-(hours*3600)-(minutes*60))
+        duration = ''
+        if hours > 0:
+            duration = str(hours)+'h ' + str(minutes)+'m ' + str(seconds) + 's'
+        elif minutes > 0:
+            duration = str(minutes)+'m ' + str(seconds) + 's'
+        else:
+            duration = str(seconds)+'s'
+    return duration
+
+
+def strip_epoch(version):
+    """
+    Stip the epoch out of the version string. Some file (e.g. buildlogs, debs)
+    do not have epoch in their filenames.
+    """
+    try:
+        return version.split(':', 1)[1]
+    except IndexError:
+        return version
+
+
+def irc_msg(msg, channel='debian-reproducible'):
+    kgb = ['kgb-client', '--conf', '/srv/jenkins/kgb/%s.conf' % channel,
+           '--relay-msg']
+    kgb.extend(str(msg).strip().split())
+    subprocess.run(kgb)

bin/reproducible_common.sh

 #!/bin/bash
+# vim: set noexpandtab:
 
 # Copyright 2014-2018 Holger Levsen <holger@layer-acht.org>
-#              © 2015 Mattia Rizzolo <mattia@mapreri.org>
+#         © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
 # released under the GPLv=2
 #
 # included by all reproducible_*.sh scripts, so be quiet
@@ -539,34 +540,11 @@ publish_page() {
 	echo "Enjoy $REPRODUCIBLE_URL/$TARGET"
 }
 
-link_packages() {
-	set +x
-        local i
-	for (( i=1; i<$#+1; i=i+400 )) ; do
-		local string='['
-		local delimiter=''
-		local j
-		for (( j=0; j<400; j++)) ; do
-			local item=$(( $j+$i ))
-			if (( $item < $#+1 )) ; then
-				string+="${delimiter}\"${!item}\""
-				delimiter=','
-			fi
-		done
-		string+=']'
-		cd /srv/jenkins/bin
-		DATA=" $(python3 -c "from reproducible_common import link_packages; \
-				print(link_packages(${string}, '$SUITE', '$ARCH'))" 2> /dev/null)"
-		cd - > /dev/null
-		write_page "$DATA"
-	done
-	if "$DEBUG" ; then set -x ; fi
-}
-
 gen_package_html() {
 	cd /srv/jenkins/bin
 	python3 -c "import reproducible_html_packages as rep
-pkg = rep.Package('$1', no_notes=True)
+from rblib.models import Package
+pkg = Package('$1', no_notes=True)
 rep.gen_packages_html([pkg], no_clean=True)" || echo "Warning: cannot update HTML pages for $1"
 	cd - > /dev/null
 }

bin/reproducible_db_maintenance.py

 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
 # Copyright © 2015 Holger Levsen <holger@layer-acht.org>
 # Based on various reproducible_* files © 2014-2015 Holger Levsen <holger@layer-acht.org>
 # Licensed under GPL-2
@@ -11,7 +11,14 @@
 # Track the database schema and changes to it. Also allow simple creation
 # and migration of it.
 
-from reproducible_common import *
+import re
+import sys
+from datetime import datetime
+
+from rblib import query_db
+from rblib.confparse import log
+from rblib.const import DB_METADATA
+from rblib.utils import print_critiacal_message
 
 now = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
 

bin/reproducible_html_breakages.py

 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2015-2016 Mattia Rizzolo <mattia@mapreri.org>
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
 # Copyright © 2016-2017 Holger Levsen <holger@layer-acht.org>
 #
 # Licensed under GPL-2
@@ -10,9 +10,26 @@
 #
 # Build a page full of CI issues to investigate
 
-from reproducible_common import *
+import os
+import re
+import csv
 import time
 import os.path
+import datetime
+from subprocess import check_call
+from timedate import timedelta
+
+from rblib import query_db
+from rblib.confparse import log
+from rblib.models import Package
+from rblib.html import tab, create_main_navigation, write_html_page
+from rblib.utils import bcolors, create_temp_file, strip_epoch
+from rblib.const import (
+    BIN_PATH,
+    DISTRO_BASE, DISTRO_URL,
+    HISTORY_PATH, RB_PKG_PATH, DBD_PATH, DBDTXT_PATH,
+    BUILDINFO_PATH, LOGS_PATH, DIFFS_PATH, RBUILD_PATH,
+)
 
 def unrep_with_dbd_issues():
     log.info('running unrep_with_dbd_issues check...')
@@ -79,7 +96,9 @@ def lack_rbuild():
                ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
     results = query_db(query)
     for pkg, version, suite, arch in results:
-        if not pkg_has_rbuild(pkg, version, suite, arch):
+        rbuild = os.path.join(RBUILD_PATH, suite, arch) + \
+                '/{}_{}.rbuild.log.gz'.format(pkg, strip_epoch(version))
+        if not os.access(rbuild, os.R_OK):
             bad_pkgs.append((pkg, version, suite, arch))
             log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') has been '
                         'built, but a buildlog is missing.')
@@ -277,7 +296,7 @@ def _gen_packages_html(header, pkgs):
         html += header
         html += '<br/><pre>\n'
         for pkg in pkgs:
-            html += tab + link_package(pkg[0], pkg[2], pkg[3]).strip()
+            html += tab + Package(pkg[0]).html_link(pkg[2], pkg[3], bugs=False)
             html += ' (' + pkg[1] + ' in ' + pkg[2] + '/' + pkg[3] + ')\n'
         html += '</pre></p>\n'
     return html
@@ -394,7 +413,6 @@ def gen_html():
 
 
 if __name__ == '__main__':
-    bugs = get_bugs()
     html = '<p>This page lists unexpected things a human should look at and '
     html += 'fix, like packages with an incoherent status or files that '
     html += 'should not be there. Some of these breakages are caused by '

bin/reproducible_html_dd_list.py

@@ -2,26 +2,31 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Holger Levsen <holger@layer-acht.org>
-#           © 2015 Mattia Rizzolo <mattia@mapreri.org>
+#           © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
 # Licensed under GPL-2
 #
 # Depends: python3
 #
 # Get the output of dd-list(1) and turn it into some nice html
 
+import os
+import re
 import lzma
+import html as HTML
 from urllib.request import urlopen
 from subprocess import Popen, PIPE
 from tempfile import NamedTemporaryFile
 
-from reproducible_common import *
+from rblib import query_db
+from rblib.confparse import log
+from rblib.const import DISTRO_BASE, DISTRO_URI, DISTRO_URL, SUITES
+from rblib.models import Package
+from rblib.html import create_main_navigation, write_html_page
 
 
 arch = 'amd64' # the arch is only relevant for link targets here
 mirror = 'http://deb.debian.org/debian'
 
-bugs = get_bugs()
-
 for suite in SUITES:
     remotefile = mirror + '/dists/' + suite + '/main/source/Sources.xz'
     os.makedirs('/tmp/reproducible', exist_ok=True)
@@ -59,7 +64,7 @@ for suite in SUITES:
                 line = line.strip().split(None, 1)
                 html += '    '
                 # the final strip() is to avoid a newline
-                html += link_package(line[0], suite, arch, bugs).strip()
+                html += Package(line[0]).html_link(suite, arch).strip()
                 try:
                     html += ' ' + line[1]  # eventual uploaders sign
                 except IndexError:

bin/reproducible_html_indexes.py

 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@maprerii.org>
 # Copyright © 2015-2016 Holger Levsen <holger@layer-acht.org>
 # Based on reproducible_html_indexes.sh © 2014 Holger Levsen <holger@layer-acht.org>
 # Licensed under GPL-2
@@ -10,9 +10,23 @@
 #
 # Build quite all index_* pages
 
-from reproducible_common import *
+import sys
+from string import Template
+from datetime import datetime, timedelta
 from sqlalchemy import select, and_, or_, func, bindparam, desc
 
+from rblib import query_db, db_table, get_status_icon
+from rblib.confparse import log
+from rblib.models import Package
+from rblib.utils import print_critical_message
+from rblib.html import tab, create_main_navigation, write_html_page
+from rblib.const import (
+    DISTRO_BASE, DISTRO_URI, DISTRO_URL,
+    SUITES, ARCHS,
+    defaultsuite, defaultarch,
+    filtered_issues, filter_html,
+)
+
 """
 Reference doc for the folowing lists:
 
@@ -715,7 +729,7 @@ def build_page_section(page, section, suite, arch):
     html += '<p>\n' + tab + '<code>\n'
     for row in rows:
         pkg = row[0]
-        html += tab*2 + link_package(pkg, suite, arch, bugs)
+        html += tab*2 + Package(pkg).html_link(suite, arch)
     else:
         html += tab + '</code>\n'
         html += '</p>'
@@ -793,8 +807,6 @@ def build_page(page, suite=None, arch=None):
     log.info('"' + title + '" now available at ' + desturl)
 
 
-bugs = get_bugs() # this variable should not be global, else merely importing _html_indexes always queries UDD
-
 if __name__ == '__main__':
     for arch in ARCHS:
         for suite in SUITES:

bin/reproducible_html_live_status.py

@@ -2,18 +2,26 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>
+#           © 2018      Mattia Rizzolo <mattia@mapreri.org>
 # based on ~jenkins.d.n:~mattia/status.sh by Mattia Rizzolo <mattia@mapreri.org>
 # Licensed under GPL-2
 #
 # Depends: python3
-#
 
-from reproducible_common import *
-from reproducible_html_indexes import build_leading_text_section
+from string import Template
 from sqlalchemy import select, func, cast, Integer, and_, bindparam
-import glob
 
-bugs = get_bugs()
+from rblib import query_db, db_table, get_status_icon
+from rblib.confparse import log
+from rblib.models import Package
+from rblib.utils import convert_into_hms_string
+from rblib.html import tab, create_main_navigation, write_html_page
+from reproducible_html_indexes import build_leading_text_section
+from rblib.const import (
+    DISTRO_BASE, DISTRO_URL, DISTRO_URI,
+    ARCHS, SUITES,
+    defaultsuite,
+)
 
 # sqlalchemy table definitions needed for queries
 results = db_table('results')
@@ -82,7 +90,7 @@ def generate_schedule(arch):
         avg_duration = convert_into_hms_string(row[6])
         html += tab + '<tr><td>&nbsp;</td><td>' + row[0] + '</td>'
         html += '<td>' + row[1] + '</td><td>' + row[2] + '</td><td><code>'
-        html += link_package(pkg, row[1], row[2], bugs)
+        html += Package(pkg).html_link(row[1], row[2])
         html += '</code></td><td>'+convert_into_status_html(str(row[4]))+'</td><td>'+duration+'</td><td>' + avg_duration + '</td></tr>\n'
     html += '</table></p>\n'
     destfile = DISTRO_BASE + '/index_' + arch + '_scheduled.html'
@@ -146,7 +154,7 @@ def generate_live_status_table(arch):
         avg_duration = convert_into_hms_string(row[8])
         html += tab + '<tr><td>&nbsp;</td><td>' + str(row[0]) + '</td>'
         html += '<td>' + suite + '</td><td>' + arch + '</td>'
-        html += '<td><code>' + link_package(pkg, suite, arch) + '</code></td>'
+        html += '<td><code>' + Package(pkg).html_link(suite, arch, bugs=False) + '</code></td>'
         html += '<td>' + str(row[4]) + '</td><td>' + str(row[5]) + '</td>'
         html += '<td>' + convert_into_status_html(str(row[6])) + '</td><td>' + duration + '</td><td>' + avg_duration + '</td>'
         html += '<td><a href="https://tests.reproducible-builds.org/cgi-bin/nph-logwatch?' + str(row[9]) + '">' + str(row[9]) + '</a></td>'
@@ -187,7 +195,7 @@ def generate_oldies(arch):
             pkg = row[2]
             html += tab + '<tr><td>&nbsp;</td><td>' + row[0] + '</td>'
             html += '<td>' + row[1] + '</td><td><code>'
-            html += link_package(pkg, row[0], row[1], bugs)
+            html += Package(pkg).html_link(row[0], row[1])
             html += '</code></td><td>'+convert_into_status_html(str(row[3]))+'</td><td>' + row[4] + '</td></tr>\n'
         html += '</table></p>\n'
     destfile = DISTRO_BASE + '/index_' + arch + '_oldies.html'

bin/reproducible_html_notes.py

 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
 # Copyright © 2015 Holger Levsen <holger@layer-acht.org>
 # Based on reproducible_html_notes.sh © 2014 Holger Levsen <holger@layer-acht.org>
 # Licensed under GPL-2
@@ -10,17 +10,36 @@
 #
 # Build HTML pages based on the content of the notes.git repository
 
+import os
+import re
+import sys
 import copy
 import yaml
 import popcon
 import pystache
+from string import Template
 from collections import OrderedDict
 from math import sqrt
-from reproducible_common import *
+from rblib.models import Package
+from rblib.bugs import Bugs
 from reproducible_html_packages import gen_packages_html
 from reproducible_html_indexes import build_page
 from sqlalchemy import select, and_, bindparam
 
+from rblib import query_db, get_status_icon, db_table, get_trailing_bug_icon
+from rblib.confparse import log
+from rblib.html import tab, create_main_navigation, write_html_page
+from rblib.const import (
+    REPRODUCIBLE_URL,
+    TEMPLATE_PATH,
+    DISTRO_BASE, DISTRO_URL,
+    SUITES, ARCHS,
+    defaultsuite,
+    ISSUES_PATH, ISSUES_URI,
+    NOTES_PATH, NOTES_URI,
+)
+
+
 renderer = pystache.Renderer()
 notes_body_template = renderer.load_template(
     os.path.join(TEMPLATE_PATH, 'notes_body'))
@@ -30,6 +49,8 @@ ISSUES = 'issues.yml'
 
 NOTESGIT_DESCRIPTION = 'Our notes about issues affecting packages are stored in <a href="https://salsa.debian.org/reproducible-builds/reproducible-notes" target="_parent">notes.git</a> and are targeted at packages in Debian in \'unstable/amd64\' (unless they say otherwise).'
 
+url2html = re.compile(r'((mailto\:|((ht|f)tps?)\://|file\:///){1}\S+)')
+
 note_issues_html = Template((tab*3).join("""
 <tr>
   <td>
@@ -307,7 +328,7 @@ def gen_html_issue(issue, suite):
             pkgs_popcon = issues_popcon_annotate(pkgs)
             try:
                 for pkg, popcon, is_popular in sorted(pkgs_popcon, key=lambda x: x[0] in bugs):
-                    affected += tab*6 + link_package(pkg, suite, arch, bugs, popcon, is_popular)
+                    affected += tab*6 + Package(pkg).html_link(suite, arch, bugs, popcon, is_popular)
             except ValueError:
                 pass
             affected += tab*5 + '</code>\n'
@@ -477,7 +498,7 @@ def index_issues(issues, scorefuncs):
 
 if __name__ == '__main__':
     issues_count = {}
-    bugs = get_bugs()
+    bugs = Bugs().bugs
     notes = load_notes()
     issues = load_issues()
     iterate_over_notes(notes)

bin/reproducible_html_packages.py

 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
 # Copyright © 2016-2017 Valerie R Young <spectranaut@riseup.net>
 # Based on reproducible_html_packages.sh © 2014 Holger Levsen <holger@layer-acht.org>
 # Licensed under GPL-2
@@ -10,11 +10,32 @@
 #
 # Build rb-pkg pages (the pages that describe the package status)
 
-from reproducible_common import *
+import os
+import errno
 import pystache
 import apt_pkg
 apt_pkg.init_system()
 
+from rblib import query_db, get_status_icon
+from rblib.confparse import log, args
+from rblib.models import Package
+from rblib.utils import strip_epoch, convert_into_hms_string
+from rblib.html import gen_status_link_icon, write_html_page
+from rblib.const import (
+    TEMPLATE_PATH,
+    REPRODUCIBLE_URL,
+    DISTRO_URL,
+    SUITES, ARCHS,
+    RB_PKG_PATH, RB_PKG_URI,
+    HISTORY_PATH, HISTORY_URI,
+    NOTES_PATH, NOTES_URI,
+    DBDTXT_PATH, DBDTXT_URI,
+    DBD_PATH, DBD_URI,
+    DIFFS_PATH, DIFFS_URI,
+    LOGS_PATH, LOGS_URI,
+)
+
+
 # Templates used for creating package pages
 renderer = pystache.Renderer();
 package_page_template = renderer.load_template(
@@ -42,21 +63,6 @@ def sizeof_fmt(num):
     return str(int(round(float("%f" % num), 0))) + "%s" % ('Yi')
 
 
-def get_buildlog_links_context(package, eversion, suite, arch):
-    log = suite + '/' + arch + '/' + package + '_' + eversion + '.build2.log.gz'
-    diff = suite + '/' + arch + '/' + package + '_' + eversion + '.diff.gz'
-
-    context = {}
-    if os.access(LOGS_PATH+'/'+log, os.R_OK):
-        context['build2_uri'] = LOGS_URI + '/' + log
-        context['build2_size'] = sizeof_fmt(os.stat(LOGS_PATH+'/'+log).st_size)
-
-    if os.access(DIFFS_PATH+'/'+diff, os.R_OK):
-        context['diff_uri'] = DIFFS_URI + '/' + diff
-
-    return context
-
-
 def get_dbd_links(package, eversion, suite, arch):
     """Returns dictionary of links to diffoscope pages.
 
@@ -120,8 +126,8 @@ def get_and_clean_dbd_links(package, eversion, suite, arch, status):
 def gen_suitearch_details(package, version, suite, arch, status, spokenstatus,
                           build_date):
     eversion = strip_epoch(version) # epoch_free_version is too long
-    buildinfo_file = BUILDINFO_PATH + '/' + suite + '/' + arch + '/' + package + \
-                '_' + eversion + '_' + arch + '.buildinfo'
+    pkg = Package(package)
+    build = pkg.builds[suite][arch]
 
     context = {}
     default_view = ''
@@ -148,25 +154,26 @@ def gen_suitearch_details(package, version, suite, arch, status, spokenstatus,
         default_view = default_view if default_view else dbd_uri
 
     # Get buildinfo context
-    if pkg_has_buildinfo(package, version, suite, arch):
-        url = BUILDINFO_URI + '/' + suite + '/' + arch + '/' + package + \
-              '_' + eversion + '_' + arch + '.buildinfo'
-        context['buildinfo_uri'] = url
-        default_view = default_view if default_view else url
+    if build.buildinfo:
+        context['buildinfo_uri'] = build.buildinfo.url
+        default_view = default_view if default_view else build.buildinfo.url
     elif not args.ignore_missing_files and status not in \
         ('untested', 'blacklisted', 'FTBFS', 'not_for_us', 'depwait', '404'):
-            log.critical('buildinfo not detected at ' + buildinfo_file)
+            log.critical('buildinfo not detected at ' + build.buildinfo.path)
 
     # Get rbuild, build2 and build diffs context
-    rbuild = pkg_has_rbuild(package, version, suite, arch)
-    if rbuild:  # being a tuple (rbuild path, size), empty if non existant
-        url = RBUILD_URI + '/' + suite + '/' + arch + '/' + package + '_' + \
-              eversion + '.rbuild.log.gz'
-        context['rbuild_uri'] = url
-        context['rbuild_size'] = sizeof_fmt(rbuild[1])
-        default_view = default_view if default_view else url
-        context['buildlogs'] = get_buildlog_links_context(package, eversion,
-                                                          suite, arch)
+    if build.rbuild:
+        context['rbuild_uri'] = build.rbuild.url
+        context['rbuild_size'] = sizeof_fmt(build.rbuild.size)
+        default_view = default_view if default_view else build.rbuild.url
+        context['buildlogs'] = {}
+        if build.build2 and build.logdiff:
+            context['buildlogs']['build2_uri'] = build.build2.url
+            context['buildlogs']['build2_size'] = build.build2.size
+            context['buildlogs']['diff_uri'] = build.logdiff.url
+        else:
+            log.error('Either {} or {} is missing'.format(
+                build.build2.path, build.logdiff.path))
     elif status not in ('untested', 'blacklisted') and \
          not args.ignore_missing_files:
         log.critical(DISTRO_URL  + '/' + suite + '/' + arch + '/' + package +
@@ -182,7 +189,6 @@ def gen_suitearch_details(package, version, suite, arch, status, spokenstatus,
 
 
 def determine_reproducibility(status1, version1, status2, version2):
-    newstatus = ''
     versionscompared = apt_pkg.version_compare(version1, version2);
 
     # if version1 > version2,
@@ -217,10 +223,10 @@ def gen_suitearch_section(package, current_suite, current_arch):
         suites = []
         for s in SUITES:
 
-            status = package.get_status(s, a)
+            status = package.builds[s][a].status
             if not status:  # The package is not available in that suite/arch
                 continue
-            version = package.get_tested_version(s, a)
+            version = package.builds[s][a].version
 
             if not final_version or not final_status:
                 final_version = version
@@ -229,7 +235,7 @@ def gen_suitearch_section(package, current_suite, current_arch):
                 final_status, final_version = determine_reproducibility(
                     final_status, final_version, status, version)
 
-            build_date = package.get_build_date(s, a)
+            build_date = package.builds[s][a].build_date
             status, icon, spokenstatus = get_status_icon(status)
 
             if not (build_date and status != 'blacklisted'):
@@ -328,7 +334,7 @@ def gen_packages_html(packages, no_clean=False):
     packages should be a list of Package objects.
     """
     total = len(packages)
-    log.debug('Generating the pages of ' + str(total) + ' package(s)')
+    log.info('Generating the pages of ' + str(total) + ' package(s)')
     for package in sorted(packages, key=lambda x: x.name):
         assert isinstance(package, Package)
         gen_history_page(package)
@@ -345,10 +351,10 @@ def gen_packages_html(packages, no_clean=False):
         for suite in SUITES:
             for arch in ARCHS:
 
-                status = package.get_status(suite, arch)
-                version = package.get_tested_version(suite, arch)
-                build_date = package.get_build_date(suite, arch)
-                if status == False:  # the package is not in the checked suite
+                status = package.builds[suite][arch].status
+                version = package.builds[suite][arch].version
+                build_date = package.builds[suite][arch].build_date
+                if status is None:  # the package is not in the checked suite
                     continue
                 log.debug('Generating the page of %s/%s/%s @ %s built at %s',
                           pkg, suite, arch, version, build_date)

bin/reproducible_html_pkg_sets.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2016 Valerie Young <spectranaut@riseup.net>
+#           © 2018 Mattia Rizzolo <mattia@mapreri.org>
 # Based on reproducible_html_pkg_sets.sh:
 #           © 2014-2016 Holger Levsen <holger@layer-acht.org>
 #           © 2015 Mattia Rizzolo <mattia@debian.org>
@@ -11,13 +12,27 @@
 #
 # Build rb-pkg pages (the pages that describe the package status)
 
-from reproducible_common import *
-
+import os
 import csv
-import time
 import pystache
+from datetime import datetime, timedelta
+from subprocess import check_call
 from collections import OrderedDict
 
+from rblib import query_db, get_status_icon
+from rblib.bugs import Bugs
+from rblib.confpase import log
+from rblib.models import Package
+from rblib.utils import create_temp_file
+from rblib.html import create_main_navigation, write_html_page, gen_status_link_icon
+from rblib.const import (
+    BIN_PATH,
+    SUITES, ARCHS,
+    DISTRO_BASE, DISTRO_URI,
+    META_PKGSET, PKGSET_DEF_PATH,
+    TEMPLATE_PATH,
+)
+
 # Templates used for creating package pages
 renderer = pystache.Renderer()
 pkgset_navigation_template = renderer.load_template(
@@ -30,6 +45,11 @@ pkg_legend_template = renderer.load_template(
 # we only do stats up until yesterday
 YESTERDAY = (datetime.now()-timedelta(days=1)).strftime('%Y-%m-%d')
 
+
+def percent(part, whole):
+    return round(100 * float(part)/float(whole), 1)
+
+
 def gather_meta_stats(suite, arch, pkgset_name):
     pkgset_file = os.path.join(PKGSET_DEF_PATH, 'meta_pkgsets-' + suite,
                                pkgset_name + '.pkgset')
@@ -234,7 +254,7 @@ def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name):
         details_context = {
             'icon_html': icon_html,
             'description': description,
-            'package_list_html': link_packages(stats[cutename], suite, arch, bugs),
+            'package_list_html': ''.join([Package(x).html_link(suite, arch) for x in stats[cutename]]),
             'status_count': stats["count_" + cutename],
             'status_percent': stats["percent_" + cutename],
         }
@@ -293,7 +313,7 @@ def create_pkgset_graph(png_file, suite, arch, pkgset_name):
                     y_label, '1920', '960'])
 
 
-bugs = get_bugs()
+bugs = Bugs().bugs
 for arch in ARCHS:
     for suite in SUITES:
         if suite == 'experimental':

bin/reproducible_json.py

 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
 # Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>
 # Based on reproducible_json.sh © 2014 Holger Levsen <holger@layer-acht.org>
 # Licensed under GPL-2
@@ -10,15 +10,21 @@
 #
 # Build the reproducible.json and reproducibe-tracker.json files, to provide nice datasources
 
-from reproducible_common import *
 
-from apt_pkg import version_compare
-import aptsources.sourceslist
-import json
 import os
-import subprocess
+import json
+import apt_pkg
+apt_pkg.init_system()
 import tempfile
+import subprocess
 
+from rblib import query_db
+from rblib.confparse import log
+from rblib.const import (
+    DISTRO_URL,
+    REPRODUCIBLE_JSON, REPRODUCIBLE_TRACKER_JSON,
+    filter_query,
+)
 
 output = []
 output4tracker = []
@@ -61,7 +67,7 @@ for row in result:
             # compare the versions (only keep most up to date!)
             version1 = crossarch[package]['version']
             version2 = pkg['version']
-            versionscompared = version_compare(version1, version2);
+            versionscompared = apt_pkg.version_compare(version1, version2);
 
             # if version1 > version2,
             # skip the package results we are currently inspecting

bin/reproducible_notes.py

 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
 # Licensed under GPL-2
 #
 # Depends: python3 python-apt python3-yaml
 #
 # Import the content of the notes.git repository into the reproducible database
 
-from reproducible_common import *
 
 import os
-import apt
 import yaml
 import json
+import apt_pkg
+apt_pkg.init_system()
 from sqlalchemy import sql
-from apt_pkg import version_compare
+
+from rblib import db_table, query_db
+from rblib.confparse import log
+from rblib.const import conn_db
+from rblib.utils import print_critical_message, irc_msg
+
 
 NOTES = 'packages.yml'
 ISSUES = 'issues.yml'
@@ -67,7 +72,7 @@ def load_notes():
                 pkg_details = {}
 # https://image-store.slidesharecdn.com/c2c44a06-5e28-4296-8d87-419529750f6b-original.jpeg
                 try:
-                    if version_compare(str(original[pkg]['version']),
+                    if apt_pkg.version_compare(str(original[pkg]['version']),
                                        str(suite[1])) > 0:
                         continue
                 except KeyError:

bin/reproducible_remote_scheduler.py

 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Copyright © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
 # Licensed under GPL-2
 #
 # Depends: python3
@@ -13,18 +13,15 @@ import os
 import re
 import sys
 import time
+import subprocess
 from sqlalchemy import sql
-from reproducible_common import (
-    # Use an explicit list rather than a star import, because the previous code had
-    # a mysterious comment about not being able to do a star import prior to
-    # parsing the command line, & debugging the mystery via edit-compile-h01ger-run
-    # detours is not practical.
-    SUITES, ARCHS,
-    bcolors, log,
-    query_db, db_table, sql, conn_db,
-    datetime, timedelta,
-    irc_msg, unknown_args
-)
+from datetime import datetime, timedelta
+
+from rblib import query_db, db_table
+from rblib.const import SUITES, ARCHS, conn_db
+from rblib.confparse import unknown_args, log
+from rblib.utils import bcolors, irc_msg
+
 
 def packages_matching_criteria(arch, suite, criteria):
     "Return a list of packages in (SUITE, ARCH) matching the given CRITERIA."
@@ -180,7 +177,7 @@ def parse_args():
 
     if len(packages) > 50 and notify:
         log.critical(bcolors.RED + bcolors.BOLD)
-        call(['figlet', 'No.'])
+        subprocess.run(('figlet', 'No.'))
         log.critical(bcolors.FAIL + 'Do not reschedule more than 50 packages ',
                      'with notification.\nIf you think you need to do this, ',
                      'please discuss this with the IRC channel first.',