reproducible: first PoC of a buildinfo sqlite importer.

Current implementation takes ~1 hour to create a ~2GB DB with ~28M records out
of all buildinfos (for 3 suites testing/unstable/experimental and 3 archs amd64/i386/armhf)

bin/buildinfo_to_udd.pl

+#!/usr/bin/perl
+
+=comment
+CREATE TABLE buildinfo (
+    package text NOT NULL,
+    version debversion NOT NULL,
+    architecture text NOT NULL,
+    distribution text NOT NULL,
+    built_with_package text NOT NULL,
+    built_with_package_version debversion NOT NULL,
+    built_with_package_architecture text NOT NULL
+);
+=cut
+
+use strict;
+use warnings;
+
+use Dpkg ();
+use Dpkg::Control;
+use Dpkg::Deps;
+
+use Data::Dumper;
+
+sub is_arch_all {
+    my ($package, $version, $distribution) = @_;
+
+    #print "SELECT architecture FROM packages WHERE package = ? AND version = ? AND distribution = ?\n";
+    return 0;
+}
+
+my $distribution = 'debian';
+
+my $buildinfo = Dpkg::Control->new(type => 'CTRL_FILE_BUILDINFO');
+$buildinfo->load($ARGV[0]);
+#print Dumper($buildinfo);
+foreach my $package (split ' ', $buildinfo->{'Binary'}) {
+    $buildinfo->{'Checksums-Sha256'} =~ /\Q$package\E_\S+_(\S+)\./;
+    my $package_arch = $1;
+
+    my $deps = deps_parse($buildinfo->{'Build-Environment'}, host_arch => $buildinfo->{'Build-Architecture'});
+    foreach my $dep ($deps->get_deps()) {
+        my $dep_arch;
+        if (defined $dep->{'archqual'}) {
+            $dep_arch = $dep->{'archqual'};
+        } else {
+            $dep_arch = is_arch_all($dep->{'package'}, $dep->{'version'}, $distribution) ? 'all' : $buildinfo->{'Build-Architecture'};
+        }
+        printf "INSERT INTO buildinfo VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s');\n",
+            $package,
+            $buildinfo->{'Version'},
+            $package_arch,
+            $distribution,
+            $dep->{'package'},
+            $dep->{'version'},
+            $dep_arch;
+    }
+}

bin/reproducible_buildinfo.py

+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Licensed under GPL-2
+#
+# Depends: python3
+#
+# Push .buildinfo files into a sqlite DB
+
+import shutil
+from reproducible_common import *
+from deb822 import Changes, PkgRelation
+
+BUILDINFO_DB = '/home/mattia/rb-buildinfo.db'  # XXX
+
+create_q = """
+CREATE TABLE buildinfo (
+    source TEXT NOT NULL,
+    version TEXT NOT NULL,
+    architecture TEXT NOT NULL,
+    suite TEXT NOT NULL,
+    built_with_package TEXT NOT NULL,
+    built_with_package_version TEXT NOT NULL
+)"""
+
+
+class bcolors:
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+    RED = '\033[91m'
+    GOOD = '\033[92m'
+    WARN = '\033[93m' + UNDERLINE
+    FAIL = RED + BOLD + UNDERLINE
+    ENDC = '\033[0m'
+
+
+def find():
+    buildinfos = []
+    for root, dirs, files in os.walk(BUILDINFO_PATH):
+        if not files:
+            continue
+        for file in sorted(files):
+            buildinfos.append('/'.join((root, file)))
+    return buildinfos
+
+
+def parse(buildinfo, suite):
+    """ where `buildinfo` is a file object """
+    details = []
+    bi = Changes(buildinfo)
+    try:
+        installed_deps = bi['Build-Environment']
+    except KeyError:
+        installed_deps = bi['Installed-Build-Depends']
+    for deps in PkgRelation().parse_relations(installed_deps):
+        dep = deps[0]  # buildinfo does not have alternatives here...
+        details.append((
+            bi['Source'],
+            bi['Version'],
+            bi['Build-Architecture'],
+            suite,
+            dep['name'],
+            dep['version'][1],
+        ))
+    return details
+
+
+def push(stuff):
+    log.info('pushing to the DB now... This might take some time...')
+    now = datetime.now()
+    query = '''INSERT INTO buildinfo VALUES (?, ?, ?, ?, ?, ?)'''
+    conn_bi_db = sqlite3.connect(BUILDINFO_DB, timeout=60)
+    cursor = conn_bi_db.cursor()
+    try:
+        cursor.executemany(query, stuff)
+    except:
+        conn_bi_db.rollback()
+        raise
+    else:
+        conn_bi_db.commit()
+    conn_bi_db.close()
+    log.info('the DB push of %s records took %s', len(stuff), datetime.now()-now)
+
+
+def create():
+    log.info('discarding old db (if available) and creating a new one...')
+    try:
+        os.remove(BUILDINFO_DB)
+    except FileNotFoundError:
+        pass
+    conn_bi_db = sqlite3.connect(BUILDINFO_DB, timeout=60)
+    cursor = conn_bi_db.cursor()
+    try:
+        cursor.execute(create_q)
+    except:
+        conn_bi_db.rollback()
+        raise
+    else:
+        conn_bi_db.commit()
+    conn_bi_db.close()
+
+
+def main():
+    create()
+    # nothing will delete the files while they are queued for reading here,
+    # right ?!?!
+    buildinfos = find()
+    content = []
+    i = 0
+    total = len(buildinfos)
+    for buildinfo in buildinfos:
+        i += 1
+        log.info('%s%s/%s%s - %s', bcolors.GOOD, i, total, bcolors.ENDC,
+                 buildinfo)  # XXX maybe remove the colors once in jenkins ?
+        try:
+            with open(buildinfo) as fd:
+                suite = buildinfo.split('/')[8]
+                content.extend(parse(fd, suite))
+        except FileNotFoundError:
+            log.warning('%sThe file %s got removed while reading it%s',
+                        bcolors.WARN, buildinfo, bcolors.ENDC)
+            pass
+    push(content)
+    # XXX commenting out while testing
+    #log.info('copying the new DB on a public http location')
+    #shutil.copy(BUILDINFO_DB, BASE)
+
+
+if __name__ == '__main__':
+    main()

bin/reproducible_common.py

@@ -49,6 +49,7 @@ TEMPLATE_PATH = BIN_PATH + '/templates'
 REPRODUCIBLE_JSON = BASE + '/reproducible.json'
 REPRODUCIBLE_TRACKER_JSON = BASE + '/reproducible-tracker.json'
 REPRODUCIBLE_DB = '/var/lib/jenkins/reproducible.db'
+BUILDINFO_DB = '/var/lib/jenkins/rb-buildinfo.db'
 
 DBD_URI = '/debian/dbd'
 DBDTXT_URI = '/debian/dbdtxt'