Skip to content
Commits on Source (3)
Show whitespace changes
Inline Side-by-side
bin/reproducible_html_breakages.py
View file @ a7483b2c
#!/usr/bin/python3
# -*- coding: utf-8 -*-
#
# Copyright © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
# Copyright © 2016-2019 Holger Levsen <holger@layer-acht.org>
# Copyright © 2015-2019 Mattia Rizzolo <mattia@mapreri.org>
# Copyright © 2016-2018 Holger Levsen <holger@layer-acht.org>
#
# Licensed under GPL-2
#
......@@ -29,6 +29,7 @@ from rblib.const import (
BUILDINFO_PATH, LOGS_PATH, DIFFS_PATH, RBUILD_PATH,
)
def unrep_with_dbd_issues():
log.info('running unrep_with_dbd_issues check...')
without_dbd = []
......@@ -36,7 +37,8 @@ def unrep_with_dbd_issues():
sources_without_dbd = set()
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status='FTBR' AND s.suite IN ('stretch', 'buster', 'bullseye', 'unstable', 'experimental')
WHERE r.status='FTBR'
AND s.distribution = (SELECT id FROM distributions WHERE name = 'debian')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
......@@ -59,6 +61,7 @@ def unrep_with_dbd_issues():
sources_without_dbd.add(pkg)
return without_dbd, bad_dbd, sources_without_dbd
def count_pkgs(pkgs_to_count=[]):
counted_pkgs = []
for pkg, version, suite, arch in pkgs_to_count:
......@@ -72,7 +75,8 @@ def not_unrep_with_dbd_file():
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != 'FTBR' and s.suite IN ('stretch', 'buster', 'bullseye', 'unstable', 'experimental')
WHERE r.status != 'FTBR'
AND s.distribution = (SELECT id FROM distributions WHERE name = 'debian')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
......@@ -98,7 +102,7 @@ def lack_rbuild():
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status NOT IN ('blacklisted', '')
AND s.suite IN ('stretch', 'buster', 'bullseye', 'unstable', 'experimental')
AND s.distribution = (SELECT id FROM distributions WHERE name = 'debian')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
......@@ -118,7 +122,7 @@ def lack_buildinfo():
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status NOT IN
('blacklisted', 'NFU', 'FTBFS', 'timeout', 'depwait', 'E404')
AND s.suite IN ('stretch', 'buster', 'bullseye', 'unstable', 'experimental')
AND s.distribution = (SELECT id FROM distributions WHERE name = 'debian')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
......@@ -172,10 +176,16 @@ def alien_log(directory=None):
try:
if os.path.getmtime('/'.join([root, file])) < time.time()-86400:
os.remove('/'.join([root, file]))
log.warning('/'.join([root, file]) + ' should not be there and and was older than a day so it was removed.')
log.warning('%s should not be there and was older than a day so it was removed',
'/'.join([root, file]))
else:
bad_files.append('/'.join([root, file]))
log.info('/'.join([root, file]) + ' should not be there, but is also less than 24h old and will probably soon be gone. Probably diffoscope is running on that package right now.')
log.info(
'%s should not be there, but is also less than 24h '
'old and will probably soon be gone. Probably '
'diffoscope is running on that package right now.',
'/'.join([root, file])
)
except FileNotFoundError:
pass # that bad file is already gone.
return bad_files
......@@ -210,10 +220,13 @@ def alien_buildinfo():
try:
if os.path.getmtime('/'.join([root, file])) < time.time()-86400:
os.remove('/'.join([root, file]))
log.warning('/'.join([root, file]) + ' should not be there and and was older than a day so it was removed.')
log.warning('%s should not be there and and was older than a day so it was removed.',
'/'.join([root, file]))
else:
bad_files.append('/'.join([root, file]))
log.info('/'.join([root, file]) + ' should not be there, but is also less than 24h old and will probably soon be gone.')
log.info('%s should not be there, but is also less than'
' 24h old and will probably soon be gone.',
'/'.join([root, file]))
except FileNotFoundError:
pass # that bad file is already gone.
return bad_files
......@@ -282,6 +295,7 @@ def _gen_packages_html(header, pkgs):
html += '</pre></p>\n'
return html
def _gen_files_html(header, entries):
html = ''
if entries:
......@@ -293,6 +307,7 @@ def _gen_files_html(header, entries):
html += '</pre></p>\n'
return html
def create_breakages_graph(png_file, main_label):
png_fullpath = os.path.join(DISTRO_BASE, png_file)
table = "stats_breakages"
......@@ -330,7 +345,11 @@ def update_stats_breakages(diffoscope_timeouts, diffoscope_crashes):
if not result:
insert = "INSERT INTO stats_breakages VALUES ('{date}', " + \
"'{diffoscope_timeouts}', '{diffoscope_crashes}')"
query_db(insert.format(date=YESTERDAY, diffoscope_timeouts=diffoscope_timeouts, diffoscope_crashes=diffoscope_crashes))
query_db(insert.format(
date=YESTERDAY,
diffoscope_timeouts=diffoscope_timeouts,
diffoscope_crashes=diffoscope_crashes
))
log.info("Updating db table stats_breakages on %s with %s timeouts and %s crashes.",
YESTERDAY, diffoscope_timeouts, diffoscope_crashes)
else:
......@@ -355,7 +374,8 @@ def gen_html():
html += '<br> <a href="/debian/' + png_file + '"><img src="/debian/'
html += png_file + '" alt="' + main_label + '"></a>'
# link artifacts
html += '<br/> <a href="https://tests.reproducible-builds.org/debian/artifacts/">Artifacts diffoscope crashed</a> on are available for 48h for download.'
html += '<br/> <a href="https://tests.reproducible-builds.org/debian/artifacts/">'
html += 'Artifacts diffoscope crashed</a> on are available for 48h for download.'
html += _gen_packages_html('are marked as FTBR, but there is no ' +
'diffoscope output - so probably diffoscope ' +
......