Commit 380d632b authored by lynxis lazus's avatar lynxis lazus Committed by Holger Levsen

reproducible OpenWrt: rewrite db interaction

* Do not remove old OpenWrt source packages (which was completely wrong)
* Remove old results if same package/version present
* Be much slower when running against the database because
  there are a lot more queries.
* Actual fill the source and result tables with OpenWrt.
Signed-off-by: Holger Levsen's avatarHolger Levsen <holger@layer-acht.org>
parent 1371f7c1
#!/usr/bin/env python
# Copyright Alexander Couzens <lynxis@fe80.eu> 2018
# -*- coding: utf-8 -*-
# Copyright Alexander Couzens <lynxis@fe80.eu> 2018, 2019
#
# under the GPL2
#
......@@ -78,87 +79,108 @@ def insert_into_db(result, suite='trunk'):
same, alone_a, alone_b, differ = result
distributions = db_table('distributions')
results = db_table('results')
sources = db_table('sources')
results_tbl = db_table('results')
sources_tbl = db_table('sources')
distro_id = query_db(
select([distributions.c.id]).where(distributions.c.name == 'openwrt')
)[0][0]
# Delete all old data
transaction = conn_db.begin()
d = results.delete(results.c.package_id.in_(
select([sources.c.id]).select_from(sources).where(sources.c.distribution == distro_id)
))
query_db(d)
d = sources.delete(sources.c.distribution == distro_id)
query_db(d)
transaction.commit()
# create new data
pkgs = []
pkgs_b = {}
src_pkgs = []
build_pkgs = {}
now = datetime.now()
# query for a source package with name, version
query_src_pkg = select(
[sources_tbl.c.id, sources_tbl.c.name, sources_tbl.c.version,
sources_tbl.c.suite, sources_tbl.c.architecture]
).select_from(
sources_tbl.join(distributions)
).where(
and_(
distributions.c.name == 'openwrt',
sources_tbl.c.name == bindparam('name'),
sources_tbl.c.version == bindparam('version')
)
)
query_results_pkg = select(
[results_tbl.c.id]
).where(results_tbl.c.package_id == bindparam('package_id'))
def insert_pkg_list(pkg_list, state, timestamp):
# Add new data
for pkg in same:
p = {
'name': pkg['Package'],
'version': pkg['Version'],
for pkg in pkg_list:
entry = pkg_list[pkg][0]
package = {
'name': pkg,
'version': entry['Version'],
'suite': suite,
'architecture': pkg['Architecture'],
'architecture': entry['Architecture'],
'distribution': distro_id
}
pkgs.append(p)
src_pkgs.append(package)
data = {
'status': pkg['status'],
'status': state,
'build_date': timestamp,
'build_duration': 2342,
}
pkgs_b[(pkg['package'], pkg['version'])] = data
build_pkgs[(entry['package'], entry['version'])] = data
# parse the pkg lists into our structure
insert_pkg_list(same, "reproducible", now)
insert_pkg_list(alone_a, "FTBFS on B", now)
insert_pkg_list(alone_b, "FTBFS on A", now)
insert_pkg_list(differ, "unreproducible", now)
log.info('Injecting new source packages…')
# import new source packages if they are not yet present
new_src_pkgs = []
for pkg in src_pkgs:
db_pkg = query_db(query_src_pkg.params({'name': pkg['name'], 'version': pkg['version']}))
if db_pkg:
continue
new_src_pkgs.append(pkg)
if new_src_pkgs:
log.info('Found new source packages. Adding to the database')
transaction = conn_db.begin()
conn_db.execute(sources.insert(), pkgs)
conn_db.execute(sources_tbl.insert(), new_src_pkgs)
transaction.commit()
log.info('Injecting build results…')
cur_pkgs = select(
[sources.c.id, sources.c.name, sources.c.version,
sources.c.suite, sources.c.architecture]
).select_from(
sources.join(distributions)
).where(
and_(
distributions.c.name == 'openwrt',
sources.c.suite == bindparam('suite'),
sources.c.architecture == bindparam('arch')
)
)
cur_pkgs = query_db(cur_pkgs.params({'suite': 'factory', 'arch': 'x86_64'}))
builds = []
for pkg in cur_pkgs:
# (id, name, version, suite, architecture)
data = pkgs_b[(pkg[1], pkg[2])]
p = {
'package_id': pkg[0],
'version': pkg[2],
'status': pkg['status'],
results = []
for pkg in build_pkgs:
# pkg = ("<package_name>", "<package_version>"
data = build_pkgs[pkg]
# search for the source package
db_pkg = query_db(query_src_pkg.params({'name': pkg[0], 'version': pkg[1]}))
if not db_pkg:
log.warning("Could not find the source package for %s version %s", pkg[0], pkg[1])
continue
db_pkg = db_pkg[0]
# search results and remove it
result_pkg = query_db(query_results_pkg.params({'package_id': db_pkg[0]}))
if result_pkg:
transaction = conn_db.begin()
query_db(results_tbl.delete(results_tbl.c.package_id == db_pkg[0]))
transaction.commit()
bin_pkg = {
'package_id': db_pkg[0],
'version': pkg[1],
'status': data['status'],
'build_date': data['build_date'],
'build_duration': data['build_duration'],
'job': 'external',
}
builds.append(p)
if builds:
results.append(bin_pkg)
# add new results
if results:
transaction = conn_db.begin()
conn_db.execute(results.insert(), builds)
conn_db.execute(results_tbl.insert(), results)
transaction.commit()
def example():
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment