reproducible_opensuse_import_json.py 2.76 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
#!/usr/bin/python3
# -*- coding: utf-8 -*-
#
# Copyright © 2018 Mattia Rizzolo <mattia@mapreri.org>
#
# Licensed under GPL-2
#
# Depends: python3

import json
from datetime import datetime
from urllib.request import urlopen
from sqlalchemy import select, and_, bindparam

from rblib import conn_db, query_db, db_table
from rblib.confparse import log

json_url = 'http://rb.zq1.de/compare.factory/reproducible.json'

log.info('Downloading status file…')
21
ofile = urlopen(json_url).read().decode('utf-8')
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
ostatus = json.loads(ofile)


distributions = db_table('distributions')
sources = db_table('sources')
results = db_table('results')

distro_id = query_db(
    select([distributions.c.id]).where(distributions.c.name == 'opensuse')
)[0][0]

pkgs = []
pkgs_b = {}

for pkg in ostatus:
    p = {
        'name': pkg['package'],
        'version': pkg['version'],
        'suite': 'factory',
        'architecture': pkg['architecture'],
        'distribution': distro_id
    }
    pkgs.append(p)
    data = {
        'status': pkg['status'],
        'build_date': datetime.fromtimestamp(pkg['build_date']),
        'build_duration': pkg['build_duration'],
    }
    pkgs_b[(pkg['package'], pkg['version'])] = data


# just insert everything for now

log.info('Dropping old data…')
transaction = conn_db.begin()
d = results.delete(results.c.package_id.in_(
    select([sources.c.id]).select_from(sources).where(sources.c.distribution == distro_id)
))
query_db(d)
d = sources.delete(sources.c.distribution == distro_id)
query_db(d)
transaction.commit()


log.info('Injecting new source packages…')
transaction = conn_db.begin()
conn_db.execute(sources.insert(), pkgs)
transaction.commit()


log.info('Injecting build results…')
cur_pkgs = select(
    [sources.c.id, sources.c.name, sources.c.version,
     sources.c.suite, sources.c.architecture]
).select_from(
    sources.join(distributions)
).where(
    and_(
        distributions.c.name == 'opensuse',
        sources.c.suite == bindparam('suite'),
        sources.c.architecture == bindparam('arch')
    )
)
cur_pkgs = query_db(cur_pkgs.params({'suite': 'factory', 'arch': 'x86_64'}))

builds = []
for pkg in cur_pkgs:
    # (id, name, version, suite, architecture)
    data = pkgs_b[(pkg[1], pkg[2])]
    if data['status'] == 'nobinaries':
        continue
    pkg_status = data['status'].\
        replace('unreproducible', 'FTBR').\
        replace('notforus', 'NFU').\
        replace('waitdep', 'depwait')
    p = {
        'package_id': pkg[0],
        'version': pkg[2],
        'status': pkg_status,
        'build_date': data['build_date'],
        'build_duration': data['build_duration'],
        'job': 'external',
    }
    builds.append(p)
if builds:
    transaction = conn_db.begin()
    conn_db.execute(results.insert(), builds)
transaction.commit()