Commit 380d632b authored by lynxis lazus's avatar lynxis lazus Committed by Holger Levsen

reproducible OpenWrt: rewrite db interaction

* Do not remove old OpenWrt source packages (which was completely wrong)
* Remove old results if same package/version present
* Be much slower when running against the database because
  there are a lot more queries.
* Actual fill the source and result tables with OpenWrt.
Signed-off-by: Holger Levsen's avatarHolger Levsen <holger@layer-acht.org>
parent 1371f7c1
#!/usr/bin/env python #!/usr/bin/env python
# Copyright Alexander Couzens <lynxis@fe80.eu> 2018 # -*- coding: utf-8 -*-
# Copyright Alexander Couzens <lynxis@fe80.eu> 2018, 2019
# #
# under the GPL2 # under the GPL2
# #
...@@ -78,87 +79,108 @@ def insert_into_db(result, suite='trunk'): ...@@ -78,87 +79,108 @@ def insert_into_db(result, suite='trunk'):
same, alone_a, alone_b, differ = result same, alone_a, alone_b, differ = result
distributions = db_table('distributions') distributions = db_table('distributions')
results = db_table('results') results_tbl = db_table('results')
sources = db_table('sources') sources_tbl = db_table('sources')
distro_id = query_db( distro_id = query_db(
select([distributions.c.id]).where(distributions.c.name == 'openwrt') select([distributions.c.id]).where(distributions.c.name == 'openwrt')
)[0][0] )[0][0]
# Delete all old data src_pkgs = []
transaction = conn_db.begin() build_pkgs = {}
d = results.delete(results.c.package_id.in_(
select([sources.c.id]).select_from(sources).where(sources.c.distribution == distro_id)
))
query_db(d)
d = sources.delete(sources.c.distribution == distro_id)
query_db(d)
transaction.commit()
# create new data
pkgs = []
pkgs_b = {}
now = datetime.now() now = datetime.now()
# query for a source package with name, version
query_src_pkg = select(
[sources_tbl.c.id, sources_tbl.c.name, sources_tbl.c.version,
sources_tbl.c.suite, sources_tbl.c.architecture]
).select_from(
sources_tbl.join(distributions)
).where(
and_(
distributions.c.name == 'openwrt',
sources_tbl.c.name == bindparam('name'),
sources_tbl.c.version == bindparam('version')
)
)
query_results_pkg = select(
[results_tbl.c.id]
).where(results_tbl.c.package_id == bindparam('package_id'))
def insert_pkg_list(pkg_list, state, timestamp): def insert_pkg_list(pkg_list, state, timestamp):
# Add new data # Add new data
for pkg in same: for pkg in pkg_list:
p = { entry = pkg_list[pkg][0]
'name': pkg['Package'], package = {
'version': pkg['Version'], 'name': pkg,
'version': entry['Version'],
'suite': suite, 'suite': suite,
'architecture': pkg['Architecture'], 'architecture': entry['Architecture'],
'distribution': distro_id 'distribution': distro_id
} }
pkgs.append(p) src_pkgs.append(package)
data = { data = {
'status': pkg['status'], 'status': state,
'build_date': timestamp, 'build_date': timestamp,
'build_duration': 2342, 'build_duration': 2342,
} }
pkgs_b[(pkg['package'], pkg['version'])] = data build_pkgs[(entry['package'], entry['version'])] = data
# parse the pkg lists into our structure
insert_pkg_list(same, "reproducible", now) insert_pkg_list(same, "reproducible", now)
insert_pkg_list(alone_a, "FTBFS on B", now) insert_pkg_list(alone_a, "FTBFS on B", now)
insert_pkg_list(alone_b, "FTBFS on A", now) insert_pkg_list(alone_b, "FTBFS on A", now)
insert_pkg_list(differ, "unreproducible", now) insert_pkg_list(differ, "unreproducible", now)
log.info('Injecting new source packages…') # import new source packages if they are not yet present
new_src_pkgs = []
for pkg in src_pkgs:
db_pkg = query_db(query_src_pkg.params({'name': pkg['name'], 'version': pkg['version']}))
if db_pkg:
continue
new_src_pkgs.append(pkg)
if new_src_pkgs:
log.info('Found new source packages. Adding to the database')
transaction = conn_db.begin() transaction = conn_db.begin()
conn_db.execute(sources.insert(), pkgs) conn_db.execute(sources_tbl.insert(), new_src_pkgs)
transaction.commit() transaction.commit()
log.info('Injecting build results…') log.info('Injecting build results…')
cur_pkgs = select( results = []
[sources.c.id, sources.c.name, sources.c.version, for pkg in build_pkgs:
sources.c.suite, sources.c.architecture] # pkg = ("<package_name>", "<package_version>"
).select_from( data = build_pkgs[pkg]
sources.join(distributions)
).where( # search for the source package
and_( db_pkg = query_db(query_src_pkg.params({'name': pkg[0], 'version': pkg[1]}))
distributions.c.name == 'openwrt', if not db_pkg:
sources.c.suite == bindparam('suite'), log.warning("Could not find the source package for %s version %s", pkg[0], pkg[1])
sources.c.architecture == bindparam('arch') continue
) db_pkg = db_pkg[0]
)
cur_pkgs = query_db(cur_pkgs.params({'suite': 'factory', 'arch': 'x86_64'})) # search results and remove it
result_pkg = query_db(query_results_pkg.params({'package_id': db_pkg[0]}))
builds = [] if result_pkg:
for pkg in cur_pkgs: transaction = conn_db.begin()
# (id, name, version, suite, architecture) query_db(results_tbl.delete(results_tbl.c.package_id == db_pkg[0]))
data = pkgs_b[(pkg[1], pkg[2])] transaction.commit()
p = {
'package_id': pkg[0], bin_pkg = {
'version': pkg[2], 'package_id': db_pkg[0],
'status': pkg['status'], 'version': pkg[1],
'status': data['status'],
'build_date': data['build_date'], 'build_date': data['build_date'],
'build_duration': data['build_duration'], 'build_duration': data['build_duration'],
'job': 'external', 'job': 'external',
} }
builds.append(p) results.append(bin_pkg)
if builds:
# add new results
if results:
transaction = conn_db.begin() transaction = conn_db.begin()
conn_db.execute(results.insert(), builds) conn_db.execute(results_tbl.insert(), results)
transaction.commit() transaction.commit()
def example(): def example():
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment