diff --git a/bin/buildinfo_to_udd.pl b/bin/buildinfo_to_udd.pl new file mode 100755 index 0000000000000000000000000000000000000000..682e755a5222a5786104bfd012eedb7ed645a81b --- /dev/null +++ b/bin/buildinfo_to_udd.pl @@ -0,0 +1,57 @@ +#!/usr/bin/perl + +=comment +CREATE TABLE buildinfo ( + package text NOT NULL, + version debversion NOT NULL, + architecture text NOT NULL, + distribution text NOT NULL, + built_with_package text NOT NULL, + built_with_package_version debversion NOT NULL, + built_with_package_architecture text NOT NULL +); +=cut + +use strict; +use warnings; + +use Dpkg (); +use Dpkg::Control; +use Dpkg::Deps; + +use Data::Dumper; + +sub is_arch_all { + my ($package, $version, $distribution) = @_; + + #print "SELECT architecture FROM packages WHERE package = ? AND version = ? AND distribution = ?\n"; + return 0; +} + +my $distribution = 'debian'; + +my $buildinfo = Dpkg::Control->new(type => 'CTRL_FILE_BUILDINFO'); +$buildinfo->load($ARGV[0]); +#print Dumper($buildinfo); +foreach my $package (split ' ', $buildinfo->{'Binary'}) { + $buildinfo->{'Checksums-Sha256'} =~ /\Q$package\E_\S+_(\S+)\./; + my $package_arch = $1; + + my $deps = deps_parse($buildinfo->{'Build-Environment'}, host_arch => $buildinfo->{'Build-Architecture'}); + foreach my $dep ($deps->get_deps()) { + my $dep_arch; + if (defined $dep->{'archqual'}) { + $dep_arch = $dep->{'archqual'}; + } else { + $dep_arch = is_arch_all($dep->{'package'}, $dep->{'version'}, $distribution) ? 'all' : $buildinfo->{'Build-Architecture'}; + } + printf "INSERT INTO buildinfo VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s');\n", + $package, + $buildinfo->{'Version'}, + $package_arch, + $distribution, + $dep->{'package'}, + $dep->{'version'}, + $dep_arch; + } +} diff --git a/bin/reproducible_buildinfo.py b/bin/reproducible_buildinfo.py new file mode 100755 index 0000000000000000000000000000000000000000..fdb81cf03441edede91175589eadc58dfb97fb46 --- /dev/null +++ b/bin/reproducible_buildinfo.py @@ -0,0 +1,131 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# +# Copyright © 2015 Mattia Rizzolo +# Licensed under GPL-2 +# +# Depends: python3 +# +# Push .buildinfo files into a sqlite DB + +import shutil +from reproducible_common import * +from deb822 import Changes, PkgRelation + +BUILDINFO_DB = '/home/mattia/rb-buildinfo.db' # XXX + +create_q = """ +CREATE TABLE buildinfo ( + source TEXT NOT NULL, + version TEXT NOT NULL, + architecture TEXT NOT NULL, + suite TEXT NOT NULL, + built_with_package TEXT NOT NULL, + built_with_package_version TEXT NOT NULL +)""" + + +class bcolors: + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + RED = '\033[91m' + GOOD = '\033[92m' + WARN = '\033[93m' + UNDERLINE + FAIL = RED + BOLD + UNDERLINE + ENDC = '\033[0m' + + +def find(): + buildinfos = [] + for root, dirs, files in os.walk(BUILDINFO_PATH): + if not files: + continue + for file in sorted(files): + buildinfos.append('/'.join((root, file))) + return buildinfos + + +def parse(buildinfo, suite): + """ where `buildinfo` is a file object """ + details = [] + bi = Changes(buildinfo) + try: + installed_deps = bi['Build-Environment'] + except KeyError: + installed_deps = bi['Installed-Build-Depends'] + for deps in PkgRelation().parse_relations(installed_deps): + dep = deps[0] # buildinfo does not have alternatives here... + details.append(( + bi['Source'], + bi['Version'], + bi['Build-Architecture'], + suite, + dep['name'], + dep['version'][1], + )) + return details + + +def push(stuff): + log.info('pushing to the DB now... This might take some time...') + now = datetime.now() + query = '''INSERT INTO buildinfo VALUES (?, ?, ?, ?, ?, ?)''' + conn_bi_db = sqlite3.connect(BUILDINFO_DB, timeout=60) + cursor = conn_bi_db.cursor() + try: + cursor.executemany(query, stuff) + except: + conn_bi_db.rollback() + raise + else: + conn_bi_db.commit() + conn_bi_db.close() + log.info('the DB push of %s records took %s', len(stuff), datetime.now()-now) + + +def create(): + log.info('discarding old db (if available) and creating a new one...') + try: + os.remove(BUILDINFO_DB) + except FileNotFoundError: + pass + conn_bi_db = sqlite3.connect(BUILDINFO_DB, timeout=60) + cursor = conn_bi_db.cursor() + try: + cursor.execute(create_q) + except: + conn_bi_db.rollback() + raise + else: + conn_bi_db.commit() + conn_bi_db.close() + + +def main(): + create() + # nothing will delete the files while they are queued for reading here, + # right ?!?! + buildinfos = find() + content = [] + i = 0 + total = len(buildinfos) + for buildinfo in buildinfos: + i += 1 + log.info('%s%s/%s%s - %s', bcolors.GOOD, i, total, bcolors.ENDC, + buildinfo) # XXX maybe remove the colors once in jenkins ? + try: + with open(buildinfo) as fd: + suite = buildinfo.split('/')[8] + content.extend(parse(fd, suite)) + except FileNotFoundError: + log.warning('%sThe file %s got removed while reading it%s', + bcolors.WARN, buildinfo, bcolors.ENDC) + pass + push(content) + # XXX commenting out while testing + #log.info('copying the new DB on a public http location') + #shutil.copy(BUILDINFO_DB, BASE) + + +if __name__ == '__main__': + main() diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py index d9704052d6081ac1c01e59a680924f82c32d1e5a..19486810fb9fd93939fbe05f0f19bad6d593f3b6 100755 --- a/bin/reproducible_common.py +++ b/bin/reproducible_common.py @@ -49,6 +49,7 @@ TEMPLATE_PATH = BIN_PATH + '/templates' REPRODUCIBLE_JSON = BASE + '/reproducible.json' REPRODUCIBLE_TRACKER_JSON = BASE + '/reproducible-tracker.json' REPRODUCIBLE_DB = '/var/lib/jenkins/reproducible.db' +BUILDINFO_DB = '/var/lib/jenkins/rb-buildinfo.db' DBD_URI = '/debian/dbd' DBDTXT_URI = '/debian/dbdtxt'