Skip to content

Commits on Source 7

Show whitespace changes
Inline Side-by-side
bin/rblib/__init__.py
View file @ ca3416d7
......@@ -57,22 +57,21 @@ def get_status_icon(status):
table = {'reproducible': 'weather-clear.png',
'FTBFS': 'weather-storm.png',
'FTBR': 'weather-showers-scattered.png',
'404': 'weather-severe-alert.png',
'E404': 'weather-severe-alert.png',
'depwait': 'weather-snow.png',
'not for us': 'weather-few-clouds-night.png',
'not_for_us': 'weather-few-clouds-night.png',
'NFU': 'weather-few-clouds-night.png',
'untested': 'weather-clear-night.png',
'blacklisted': 'error.png'}
spokenstatus = status
if status == 'unreproducible':
status = 'FTBR'
elif status == 'not for us':
status = 'not_for_us'
elif status in ('not for us', 'not_for_us'):
status = 'NFU'
try:
return (status, table[status], spokenstatus)
except KeyError:
log.error('Status ' + status + ' not recognized')
return (status, '', spokenstatus)
log.exception('Status %s not recognized', status)
raise
def get_trailing_bug_icon(bug, bugs, package=None):
......
bin/reproducible_build.sh
View file @ ca3416d7
......@@ -133,15 +133,15 @@ update_db_and_html() {
query_db "SELECT status FROM results WHERE package_id='${SRCPKGID}'")
# irc+mail notifications for changing status in unstable and experimental
if [ "$SUITE" = "unstable" ] || [ "$SUITE" = "experimental" ] ; then
if ([ "$OLD_STATUS" = "reproducible" ] && ( [ "$STATUS" = "unreproducible" ] || [ "$STATUS" = "FTBFS" ] )) || \
([ "$OLD_STATUS" = "unreproducible" ] && [ "$STATUS" = "FTBFS" ] ); then
if ([ "$OLD_STATUS" = "reproducible" ] && ( [ "$STATUS" = "FTBR" ] || [ "$STATUS" = "FTBFS" ] )) || \
([ "$OLD_STATUS" = "FTBR" ] && [ "$STATUS" = "FTBFS" ] ); then
MESSAGE="${DEBIAN_URL}/${SUITE}/${ARCH}/${SRCPACKAGE} : ${OLD_STATUS} ➤ ${STATUS}"
log_info "$MESSAGE"
irc_message debian-reproducible-changes "$MESSAGE"
fi
if [ "$OLD_STATUS" != "$STATUS" ] && [ "$NOTIFY_MAINTAINER" -eq 1 ] && \
[ "$OLD_STATUS" != "depwait" ] && [ "$STATUS" != "depwait" ] && \
[ "$OLD_STATUS" != "404" ] && [ "$STATUS" != "404" ]; then
[ "$OLD_STATUS" != "E404" ] && [ "$STATUS" != "E404" ]; then
# spool notifications and mail them once a day
mkdir -p /srv/reproducible-results/notification-emails
echo "$(date -u +'%Y-%m-%d %H:%M') $DEBIAN_URL/$SUITE/$ARCH/$SRCPACKAGE changed from $OLD_STATUS -> $STATUS" >> /srv/reproducible-results/notification-emails/$SRCPACKAGE
......@@ -156,7 +156,7 @@ update_db_and_html() {
query_db "INSERT INTO results (package_id, version, status, build_date, build_duration, node1, node2, job) VALUES ('$SRCPKGID', '$VERSION', '$STATUS', '$DATE', '$DURATION', '$NODE1', '$NODE2', '$JOB')" || \
query_db "INSERT INTO results (package_id, version, status, build_date, build_duration, node1, node2, job) VALUES ('$SRCPKGID', '$VERSION', '$STATUS', '$DATE', '$DURATION', '$NODE1', '$NODE2', '$JOB')"
fi
if [ ! -z "$DURATION" ] ; then # this happens when not 404 and not_for_us
if [ ! -z "$DURATION" ] ; then # this happens when not E404 and NFU
query_db "INSERT INTO stats_build (name, version, suite, architecture, status, build_date, build_duration, node1, node2, job) VALUES ('$SRCPACKAGE', '$VERSION', '$SUITE', '$ARCH', '$STATUS', '$DATE', '$DURATION', '$NODE1', '$NODE2', '$JOB')" || \
query_db "INSERT INTO stats_build (name, version, suite, architecture, status, build_date, build_duration, node1, node2, job) VALUES ('$SRCPACKAGE', '$VERSION', '$SUITE', '$ARCH', '$STATUS', '$DATE', '$DURATION', '$NODE1', '$NODE2', '$JOB')"
fi
......@@ -203,13 +203,13 @@ diff_copy_buildlogs() {
fi
}
handle_404() {
handle_E404() {
log_warning "Download of ${SRCPACKAGE} sources from ${SUITE} failed."
ls -l ${SRCPACKAGE}* | log_file -
log_warning "Maybe there was a network problem, or ${SRCPACKAGE} is not a source package in ${SUITE}, or it was removed or renamed. Please investigate. Sleeping 30m as this should not happen."
DURATION=0
update_rbuildlog
update_db_and_html "404"
update_db_and_html "E404"
if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=0 ; fi
if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
sleep 30m
......@@ -226,12 +226,12 @@ handle_depwait() {
if [ -n "$NOTIFY" ] ; then NOTIFY="depwait" ; fi
}
handle_not_for_us() {
handle_NFU() {
# a list of valid architecture for this package should be passed to this function
log_info "Package ${SRCPACKAGE} (${VERSION}) shall only be build on \"$(echo "$@" | xargs echo )\" and thus was skipped."
DURATION=0
update_rbuildlog
update_db_and_html "not for us"
update_db_and_html "NFU"
if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=0 ; fi
if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
exit 0 # RBUILDLOG and SAVE_ARTIFACTS and NOTIFY are used in cleanup_all called at exit
......@@ -302,7 +302,7 @@ handle_ftbr() {
gzip -9n $DEBIAN_BASE/dbdtxt/$SUITE/$ARCH/$DBDTXT
fi
calculate_build_duration
update_db_and_html "unreproducible"
update_db_and_html "FTBR"
}
handle_reproducible() {
......@@ -550,9 +550,9 @@ get_source_package() {
download_again_if_needed # yes, this is called three times. this should really not happen
if [ "$(ls ${SRCPACKAGE}_${EVERSION}.dsc 2> /dev/null)" = "" ] || [ ! -z "$PARSED_RESULT" ] ; then
if [ "$MODE" = "master" ] ; then
handle_404
handle_E404
else
exit 404
exit E404
fi
fi
}
......@@ -580,7 +580,7 @@ check_suitability() {
fi
done
if ! $SUITABLE ; then handle_not_for_us $ARCHITECTURES ; fi
if ! $SUITABLE ; then handle_NFU $ARCHITECTURES ; fi
}
first_build() {
......@@ -739,7 +739,7 @@ remote_build() {
RESULT=$?
# 404-256=148... (ssh 'really' only 'supports' exit codes below 255...)
if [ $RESULT -eq 148 ] ; then
handle_404
handle_E404
elif [ $RESULT -eq 100 ] ; then
log_error "Version mismatch between main node and build $BUILDNR, aborting. Please upgrade the schroots..."
# reschedule the package for later and quit the build without saving anything
......
bin/reproducible_common.sh
View file @ ca3416d7
......@@ -140,17 +140,17 @@ set_icon() {
case "$1" in
reproducible) ICON=weather-clear.png
;;
unreproducible|FTBR) ICON=weather-showers-scattered.png
FTBR) ICON=weather-showers-scattered.png
STATE_TARGET_NAME="FTBR"
;;
FTBFS) ICON=weather-storm.png
;;
depwait) ICON=weather-snow.png
;;
404) ICON=weather-severe-alert.png
E404) ICON=weather-severe-alert.png
;;
not_for_us|"not for us") ICON=weather-few-clouds-night.png
STATE_TARGET_NAME="not_for_us"
NFU|not_for_us|"not for us") ICON=weather-few-clouds-night.png
STATE_TARGET_NAME="NFU"
;;
blacklisted) ICON=error.png
;;
......@@ -278,7 +278,7 @@ write_page_header() {
write_page " <a href=\"$JENKINS_URL/userContent/about.html#_reproducible_builds_jobs\">jenkins.debian.net</a>."
write_page " Thanks to <a href=\"https://www.profitbricks.co.uk\">Profitbricks</a> for donating the virtual machines this is running on!"
write_page "</ul>"
LATEST=$(query_db "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id = s.id WHERE r.status IN ('unreproducible') AND s.suite = 'unstable' AND s.architecture = 'amd64' AND s.id NOT IN (SELECT package_id FROM notes) ORDER BY build_date DESC LIMIT 23"|sort -R|head -1)
LATEST=$(query_db "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id = s.id WHERE r.status = 'FTBR' AND s.suite = 'unstable' AND s.architecture = 'amd64' AND s.id NOT IN (SELECT package_id FROM notes) ORDER BY build_date DESC LIMIT 23"|sort -R|head -1)
write_page "<form action=\"$REPRODUCIBLE_URL/redirect\" method=\"GET\">$REPRODUCIBLE_URL/"
write_page "<input type=\"text\" name=\"SrcPkg\" placeholder=\"Type my friend..\" value=\"$LATEST\" />"
write_page "<input type=\"submit\" value=\"submit source package name\" />"
......@@ -723,10 +723,10 @@ create_png_from_table() {
COALESCE(reproducible_buster,0) AS reproducible_buster,
COALESCE(reproducible_unstable,0) AS reproducible_unstable,
COALESCE(reproducible_experimental,0) AS reproducible_experimental,
COALESCE(unreproducible_stretch,0) AS unreproducible_stretch,
COALESCE(unreproducible_buster,0) AS unreproducible_buster,
COALESCE(unreproducible_unstable,0) AS unreproducible_unstable,
COALESCE(unreproducible_experimental,0) AS unreproducible_experimental,
COALESCE(FTBR_stretch,0) AS FTBR_stretch,
COALESCE(FTBR_buster,0) AS FTBR_buster,
COALESCE(FTBR_unstable,0) AS FTBR_unstable,
COALESCE(FTBR_experimental,0) AS FTBR_experimental,
COALESCE(FTBFS_stretch,0) AS FTBFS_stretch,
COALESCE(FTBFS_buster,0) AS FTBFS_buster,
COALESCE(FTBFS_unstable,0) AS FTBFS_unstable,
......@@ -740,10 +740,10 @@ create_png_from_table() {
COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA),0) AS reproducible_buster,
COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS reproducible_unstable,
COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS reproducible_experimental,
(SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS unreproducible_stretch,
(SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS unreproducible_buster,
(SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS unreproducible_unstable,
(SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS unreproducible_experimental,
(SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBR_stretch,
(SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBR_buster,
(SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBR_unstable,
(SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBR_experimental,
(SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBFS_stretch,
(SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBFS_buster,
(SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBFS_unstable,
......@@ -755,7 +755,7 @@ create_png_from_table() {
FROM stats_builds_per_day AS s $WHERE2_EXTRA GROUP BY s.datum) as stats
ORDER BY datum" >> ${TABLE[$1]}.csv
elif [ $1 -eq 2 ] ; then
# just make a graph of the oldest reproducible build (ignore FTBFS and unreproducible)
# just make a graph of the oldest reproducible build (ignore FTBFS and FTBR)
query_to_csv "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
elif [ $1 -eq 7 ] ; then
query_to_csv "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv
......
bin/reproducible_db_maintenance.py
View file @ ca3416d7
......@@ -14,11 +14,12 @@
import re
import sys
from datetime import datetime
from sqlalchemy.orm import sessionmaker
from rblib import query_db
from rblib.confparse import log
from rblib.const import DB_METADATA
from rblib.utils import print_critiacal_message
from rblib.const import DB_ENGINE, DB_METADATA
from rblib.utils import print_critical_message
now = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
......@@ -157,7 +158,7 @@ db_schema = [
# and here are some queries, split by update, that can be used to
# update the live schema
schema_updates = {
1: ["INSERT INTO rb_schema (version, date) VALUES (1, '" + now + "')"],
1: [],
2: [ # do a funny dance to add an id, suite and architecture values to
# the `suites` table
'''CREATE TABLE sources_new_tmp
......@@ -220,11 +221,11 @@ schema_updates = {
build_date TEXT NOT NULL,
build_duration TEXT NOT NULL,
UNIQUE (name, version, suite, architecture, build_date))''',
"INSERT INTO rb_schema (version, date) VALUES (2, '" + now + "')"],
],
3: [ # add columns to stats_bugs for new usertag umask
'''ALTER TABLE stats_bugs ADD COLUMN open_umask INTEGER''',
'''ALTER TABLE stats_bugs ADD COLUMN done_umask INTEGER''',
"INSERT INTO rb_schema (version, date) VALUES (3, '" + now + "')"],
],
4: [ # stats_pkg_state needs (datum, suite) as primary key
'''CREATE TABLE stats_pkg_state_tmp
(datum TEXT NOT NULL,
......@@ -241,7 +242,7 @@ schema_updates = {
FTBFS, other FROM stats_pkg_state;''',
'''DROP TABLE stats_pkg_state;''',
'''ALTER TABLE stats_pkg_state_tmp RENAME TO stats_pkg_state;''',
"INSERT INTO rb_schema (version, date) VALUES (4, '" + now + "')"],
],
5: [ # stats_builds_per_day needs (datum, suite) as primary key
'''CREATE TABLE stats_builds_per_day_tmp
(datum TEXT NOT NULL,
......@@ -257,7 +258,7 @@ schema_updates = {
FTBFS, other FROM stats_builds_per_day;''',
'''DROP TABLE stats_builds_per_day;''',
'''ALTER TABLE stats_builds_per_day_tmp RENAME TO stats_builds_per_day;''',
"INSERT INTO rb_schema (version, date) VALUES (5, '" + now + "')"],
],
6: [ # stats_builds_age needs (datum, suite) as primary key
'''CREATE TABLE stats_builds_age_tmp
(datum TEXT NOT NULL,
......@@ -272,7 +273,7 @@ schema_updates = {
oldest_FTBFS FROM stats_builds_age;''',
'''DROP TABLE stats_builds_age;''',
'''ALTER TABLE stats_builds_age_tmp RENAME TO stats_builds_age;''',
"INSERT INTO rb_schema (version, date) VALUES (6, '" + now + "')"],
],
7: [ # change build_duration field in results and stats_build from str to int
'''CREATE TABLE stats_build_tmp
(id INTEGER PRIMARY KEY,
......@@ -303,7 +304,7 @@ schema_updates = {
build_date, CAST (build_duration AS INTEGER) FROM results''',
'DROP TABLE results',
'ALTER TABLE results_tmp RENAME TO results',
"INSERT INTO rb_schema (version, date) VALUES (7, '" + now + "')"],
],
8: [ # add default value to stats_bugs to get a full 'done vs open bugs' graph
'''CREATE TABLE stats_bugs_tmp
(datum TEXT NOT NULL,
......@@ -339,7 +340,7 @@ schema_updates = {
'INSERT INTO stats_bugs_tmp SELECT * FROM stats_bugs',
'DROP TABLE stats_bugs',
'ALTER TABLE stats_bugs_tmp RENAME TO stats_bugs',
"INSERT INTO rb_schema (version, date) VALUES (8, '" + now + "')"],
],
9: [ # rename 'sid' to 'unstable'
"UPDATE sources SET suite = 'unstable' WHERE suite = 'sid'",
"UPDATE stats_build SET suite = 'unstable' WHERE suite = 'sid'",
......@@ -347,7 +348,7 @@ schema_updates = {
"UPDATE stats_builds_per_day SET suite = 'unstable' WHERE suite = 'sid'",
"UPDATE stats_builds_age SET suite = 'unstable' WHERE suite = 'sid'",
"UPDATE stats_meta_pkg_state SET suite = 'unstable' WHERE suite = 'sid'",
"INSERT INTO rb_schema (version, date) VALUES (9, '" + now + "')"],
],
10: [ # add the notes and issues tables
'''CREATE TABLE notes (
package_id INTEGER,
......@@ -362,18 +363,18 @@ schema_updates = {
description TEXT NOT NULL,
url TEXT,
PRIMARY KEY (name))''',
"INSERT INTO rb_schema (version, date) VALUES (10, '" + now + "')"],
],
11: [ # table with removed packages, to enable the maintenance job to do clean up
'''CREATE TABLE removed_packages (
name TEXT NOT NULL,
suite TEXT NOT NULL,
architecture TEXT NOT NULL,
PRIMARY KEY (name, suite, architecture))''',
"INSERT INTO rb_schema (version, date) VALUES (11, '" + now + "')"],
],
12: [ # refactor the artifacts handling, splitting artifacts saving from
# IRC notification
'ALTER TABLE schedule ADD COLUMN notify TEXT',
"INSERT INTO rb_schema (version, date) VALUES (12, '" + now + "')"],
],
13: [ # record manual scheduling done, to be able to limit people
'''CREATE TABLE manual_scheduler (
id INTEGER PRIMARY KEY,
......@@ -381,14 +382,14 @@ schema_updates = {
requester TEXT NOT NULL,
date_request INTEGER NOT NULL)''',
'ALTER TABLE schedule ADD COLUMN scheduler TEXT',
"INSERT INTO rb_schema (version, date) VALUES (13, '" + now + "')"],
],
14: [ # column to enable mail notification to maintainers
'ALTER TABLE sources ADD COLUMN notify_maintainer INTEGER NOT NULL DEFAULT 0',
"INSERT INTO rb_schema (version, date) VALUES (14, '" + now + "')"],
],
15: [ # add columns to stats_bugs for new usertag ftbfs
'''ALTER TABLE stats_bugs ADD COLUMN open_ftbfs INTEGER''',
'''ALTER TABLE stats_bugs ADD COLUMN done_ftbfs INTEGER''',
"INSERT INTO rb_schema (version, date) VALUES (15, '" + now + "')"],
],
16: [ # add default value to stats_bugs.(open|done)_ftbfs to get a full 'done vs open bugs' graph
'''CREATE TABLE stats_bugs_tmp
(datum TEXT NOT NULL,
......@@ -426,21 +427,21 @@ schema_updates = {
'INSERT INTO stats_bugs_tmp SELECT * FROM stats_bugs',
'DROP TABLE stats_bugs',
'ALTER TABLE stats_bugs_tmp RENAME TO stats_bugs',
"INSERT INTO rb_schema (version, date) VALUES (16, '" + now + "')"],
],
17: [ # add column to save which builders builds what
"ALTER TABLE schedule ADD COLUMN builder TEXT",
"ALTER TABLE results ADD COLUMN builder TEXT NOT NULL DEFAULT ''",
"ALTER TABLE stats_build ADD COLUMN builder TEXT NOT NULL DEFAULT ''",
"INSERT INTO rb_schema (version, date) VALUES (17, '" + now + "')"],
],
18: [ # add columns to stats_bugs for new usertag locale
'''ALTER TABLE stats_bugs ADD COLUMN open_locale INTEGER DEFAULT 0''',
'''ALTER TABLE stats_bugs ADD COLUMN done_locale INTEGER DEFAULT 0''',
"INSERT INTO rb_schema (version, date) VALUES (18, '" + now + "')"],
],
19: [ # add column architecture to stats_pkg_state, stats_builds_per_day and stats_builds_age tables and set previous values to amd64
"ALTER TABLE stats_pkg_state ADD COLUMN architecture TEXT NOT NULL DEFAULT 'amd64'",
"ALTER TABLE stats_builds_per_day ADD COLUMN architecture TEXT NOT NULL DEFAULT 'amd64'",
"ALTER TABLE stats_builds_age ADD COLUMN architecture TEXT NOT NULL DEFAULT 'amd64'",
"INSERT INTO rb_schema (version, date) VALUES (19, '" + now + "')"],
],
20: [ # use (datum, suite, architecture) as primary key for stats_pkg_state
'''CREATE TABLE stats_pkg_state_tmp
(datum TEXT NOT NULL,
......@@ -458,7 +459,7 @@ schema_updates = {
FTBFS, other FROM stats_pkg_state;''',
'''DROP TABLE stats_pkg_state;''',
'''ALTER TABLE stats_pkg_state_tmp RENAME TO stats_pkg_state;''',
"INSERT INTO rb_schema (version, date) VALUES (20, '" + now + "')"],
],
21: [ # use (datum, suite, architecture) as primary key for stats_builds_per_day
'''CREATE TABLE stats_builds_per_day_tmp
(datum TEXT NOT NULL,
......@@ -475,7 +476,7 @@ schema_updates = {
FTBFS, other FROM stats_builds_per_day;''',
'''DROP TABLE stats_builds_per_day;''',
'''ALTER TABLE stats_builds_per_day_tmp RENAME TO stats_builds_per_day;''',
"INSERT INTO rb_schema (version, date) VALUES (21, '" + now + "')"],
],
22: [ # use (datum, suite, architecture) as primary key for stats_builds_age
'''CREATE TABLE stats_builds_age_tmp
(datum TEXT NOT NULL,
......@@ -491,7 +492,7 @@ schema_updates = {
oldest_FTBFS FROM stats_builds_age;''',
'''DROP TABLE stats_builds_age;''',
'''ALTER TABLE stats_builds_age_tmp RENAME TO stats_builds_age;''',
"INSERT INTO rb_schema (version, date) VALUES (22, '" + now + "')"],
],
23: [ # save which builders built a package and change the name of the
# field keeping the job name
'''CREATE TABLE stats_build_tmp
......@@ -513,7 +514,7 @@ schema_updates = {
build_duration, builder FROM stats_build''',
'DROP TABLE stats_build',
'ALTER TABLE stats_build_tmp RENAME TO stats_build',
"INSERT INTO rb_schema (version, date) VALUES (23, '" + now + "')"],
],
24: [ # the same as #23 but for the results table
'''CREATE TABLE results_tmp
(id INTEGER PRIMARY KEY,
......@@ -533,7 +534,7 @@ schema_updates = {
builder FROM results''',
'DROP TABLE results',
'ALTER TABLE results_tmp RENAME TO results',
"INSERT INTO rb_schema (version, date) VALUES (24, '" + now + "')"],
],
25: [ # rename the builder column also in the schedule table.
'''CREATE TABLE schedule_tmp
(id INTEGER PRIMARY KEY,
......@@ -554,14 +555,14 @@ schema_updates = {
FROM schedule''',
'DROP TABLE schedule',
'ALTER TABLE schedule_tmp RENAME TO schedule',
"INSERT INTO rb_schema (version, date) VALUES (25, '" + now + "')"],
],
26: [ # add a column to the schedule table to save the schedule message
"ALTER TABLE schedule ADD COLUMN message TEXT",
"ALTER TABLE stats_build ADD COLUMN schedule_message TEXT NOT NULL DEFAULT ''",
"INSERT INTO rb_schema (version, date) VALUES (26, '" + now + "')"],
],
27: [ # add column architecture to stats_meta_pkg_state and set previous values to amd64
"ALTER TABLE stats_meta_pkg_state ADD COLUMN architecture TEXT NOT NULL DEFAULT 'amd64'",
"INSERT INTO rb_schema (version, date) VALUES (27, '" + now + "')"],
],
28: [ # use (datum, suite, architecture, meta_pkg) as primary key for stats_meta_pkg_state
'''CREATE TABLE stats_meta_pkg_state_tmp
(datum TEXT NOT NULL,
......@@ -579,7 +580,7 @@ schema_updates = {
FTBFS, other FROM stats_meta_pkg_state;''',
'''DROP TABLE stats_meta_pkg_state;''',
'''ALTER TABLE stats_meta_pkg_state_tmp RENAME TO stats_meta_pkg_state;''',
"INSERT INTO rb_schema (version, date) VALUES (28, '" + now + "')"],
],
# THE FOLLOWING UPDATES CAN ONLY BE PREFORMED ON POSTGRES DATABASE
......@@ -596,22 +597,18 @@ schema_updates = {
NEXTVAL('stats_build_id_seq')""",
"CREATE SEQUENCE results_id_seq",
"ALTER TABLE results ALTER id SET DEFAULT NEXTVAL('results_id_seq')",
"INSERT INTO rb_schema (version, date) VALUES (29, '" + now + "')"
],
30: [ # Add new table to track diffoscope breake
'''CREATE TABLE stats_breakages
(datum TEXT,
diffoscope_timeouts INTEGER,
diffoscope_crashes INTEGER,
PRIMARY KEY (datum))''',
"INSERT INTO rb_schema (version, date) VALUES (30, '" + now + "')"
],
31: # rename the 'testing' suite into 'stretch'
[ "UPDATE {} SET suite='stretch' WHERE suite='testing'".format(t)
31: [ # rename the 'testing' suite into 'stretch'
"UPDATE {} SET suite='stretch' WHERE suite='testing'".format(t)
for t in ("sources", "stats_pkg_state", "stats_builds_per_day",
"stats_builds_age", "stats_meta_pkg_state", "stats_build")] + [
"INSERT INTO rb_schema (version, date) VALUES (31, '" + now + "')"
"stats_builds_age", "stats_meta_pkg_state", "stats_build")
],
32: [ # copy stretch packages (includng results) in buster
"""INSERT INTO sources (name, version, suite, architecture, notify_maintainer)
......@@ -632,12 +629,31 @@ schema_updates = {
sr.build_duration, sr.node1, sr.node2, sr.job
FROM buster AS b JOIN sr ON b.name=sr.name
AND b.architecture=sr.architecture""",
"INSERT INTO rb_schema (version, date) VALUES (32, '" + now + "')"
],
33: [ # the message columns. They are not actually needed.
"ALTER TABLE schedule DROP COLUMN message",
"ALTER TABLE stats_build DROP COLUMN schedule_message",
"INSERT INTO rb_schema (version, date) VALUES (33, '" + now + "')"],
],
34: [ # rename status "not for us" to "NFU"
"UPDATE results SET status='NFU' WHERE status='not for us'",
"UPDATE stats_build SET status='NFU' WHERE status='not for us'",
"INSERT INTO rb_schema (version, date) VALUES (34, CURRENT_TIMESTAMP)",
],
35: [ # rename status "unreproducible" to "FTBR"
"UPDATE results SET status='FTBR' WHERE status='unreproducible'",
"UPDATE stats_build SET status='FTBR' WHERE status='unreproducible'",
"ALTER TABLE stats_pkg_state RENAME COLUMN unreproducible to FTBR",
"ALTER TABLE stats_meta_pkg_state RENAME COLUMN unreproducible to FTBR",
"ALTER TABLE stats_builds_per_day RENAME COLUMN unreproducible to FTBR",
"ALTER TABLE stats_builds_age " + \
"RENAME COLUMN oldest_unreproducible to oldest_FTBR",
"INSERT INTO rb_schema (version, date) VALUES (35, CURRENT_TIMESTAMP)",
],
36: [ # rename status "404" to "E404"
"UPDATE results SET status='E404' WHERE status='404'",
"UPDATE stats_build SET status='E404' WHERE status='404'",
"INSERT INTO rb_schema (version, date) VALUES (36, CURRENT_TIMESTAMP)",
],
}
......@@ -682,16 +698,23 @@ def db_update():
if current == last:
return False
if current > last:
print_critiacal_message('The active database schema is higher than' +
print_critical_message('The active database schema is higher than' +
' the last update available.\nPlease check!')
sys.exit(1)
log.info('Found schema updates.')
Session = sessionmaker(bind=DB_ENGINE, autocommit=True)
session = Session()
for update in range(current+1, last+1):
log.info('Applying database update #' + str(update) + '. Queries:')
startTime = datetime.now()
with session.begin():
for query in schema_updates[update]:
log.info('\t' + query)
query_db(query)
session.execute(query)
session.execute(
"INSERT INTO rb_schema (version, date) "
"VALUES (:ver, CURRENT_TIMESTAMP)", {'ver': update}
)
log.info(str(len(schema_updates[update])) + ' queries executed in ' +
str(datetime.now() - startTime))
return True
......
bin/reproducible_html_breakages.py
View file @ ca3416d7
......@@ -37,7 +37,7 @@ def unrep_with_dbd_issues():
sources_without_dbd = set()
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status='unreproducible'
WHERE r.status='FTBR'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
......@@ -48,7 +48,7 @@ def unrep_with_dbd_issues():
without_dbd.append((pkg, version, suite, arch))
sources_without_dbd.add(pkg)
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') is '
'unreproducible without diffoscope file.')
'FTBR without diffoscope file.')
else:
log.debug(dbd + ' found.')
data = open(dbd, 'br').read(3)
......@@ -72,7 +72,7 @@ def not_unrep_with_dbd_file():
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != 'unreproducible'
WHERE r.status != 'FTBR'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
......@@ -82,7 +82,7 @@ def not_unrep_with_dbd_file():
if os.access(dbd, os.R_OK):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(dbd + ' exists but ' + suite + '/' + arch + '/' + pkg + ' (' + version + ')'
' is not unreproducible.')
' is not FTBR.')
return bad_pkgs
......@@ -110,7 +110,7 @@ def lack_buildinfo():
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status NOT IN
('blacklisted', 'not for us', 'FTBFS', 'depwait', '404', '')
('blacklisted', 'NFU', 'FTBFS', 'depwait', 'E404', '')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
......@@ -206,7 +206,7 @@ def alien_buildinfo():
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}'
AND s.architecture='{arch}'
AND r.status IN ('reproducible', 'unreproducible')
AND r.status IN ('reproducible', 'FTBR')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
bad_files = []
for root, dirs, files in os.walk(BUILDINFO_PATH):
......@@ -382,10 +382,10 @@ def gen_html():
# link artifacts
html += '<br/> <a href="https://tests.reproducible-builds.org/debian/artifacts/">Artifacts diffoscope crashed</a> on are available for 48h for download.'
html += _gen_packages_html('are marked as unreproducible, but there is no ' +
html += _gen_packages_html('are marked as FTBR, but there is no ' +
'diffoscope output - so probably diffoscope ' +
'crashed:', without_dbd)
html += _gen_packages_html('are marked as unreproducible, but their ' +
html += _gen_packages_html('are marked as FTBR, but their ' +
'diffoscope output does not seem to be an html ' +
'file - so probably diffoscope ran into a ' +
'timeout:', bad_dbd)
......@@ -402,7 +402,7 @@ def gen_html():
html += _gen_files_html('history pages that should not be there and thus have been removed:',
entries=alien_history())
# diffoscope reports where they shouldn't be
html += _gen_packages_html('are not marked as unreproducible, but they ' +
html += _gen_packages_html('are not marked as FTBR, but they ' +
'have a diffoscope file:', not_unrep_with_dbd_file())
# missing files
html += _gen_packages_html('have been built but don\'t have a buildlog:',
......
bin/reproducible_html_dashboard.sh
View file @ ca3416d7
......@@ -22,9 +22,9 @@ touch -d "$(date '+%Y-%m-%d') 00:00 UTC" $DUMMY_FILE
NOTES_GIT_PATH="/var/lib/jenkins/jobs/reproducible_html_notes/workspace"
# variables related to the stats we update
FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested"
FIELDS[0]="datum, reproducible, FTBR, FTBFS, other, untested"
FIELDS[1]="datum"
for i in reproducible unreproducible FTBFS other ; do
for i in reproducible FTBR FTBFS other ; do
for j in $SUITES ; do
FIELDS[1]="${FIELDS[1]}, ${i}_${j}"
done
......@@ -110,14 +110,14 @@ update_suite_arch_stats() {
ALL=$(query_db "SELECT count(name) FROM sources WHERE suite='${SUITE}' AND architecture='$ARCH'")
GOOD=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE';")
GOOAY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)='$DATE';")
BAD=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE';")
BAAY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND date(r.build_date)='$DATE';")
BAD=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBR' AND date(r.build_date)<='$DATE';")
BAAY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBR' AND date(r.build_date)='$DATE';")
UGLY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE';")
UGLDAY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)='$DATE';")
REST=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND s.architecture='$ARCH' AND date(r.build_date)<='$DATE';")
RESDAY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND s.architecture='$ARCH' AND date(r.build_date)='$DATE';")
REST=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'FTBR' AND r.status != 'reproducible') AND s.suite='$SUITE' AND s.architecture='$ARCH' AND date(r.build_date)<='$DATE';")
RESDAY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'FTBR' AND r.status != 'reproducible') AND s.suite='$SUITE' AND s.architecture='$ARCH' AND date(r.build_date)='$DATE';")
OLDESTG=$(query_db "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.status = 'reproducible' AND s.suite='$SUITE' AND s.architecture='$ARCH' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
OLDESTB=$(query_db "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
OLDESTB=$(query_db "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBR' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
OLDESTU=$(query_db "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
# only if we have results…
if [ -n "$OLDESTG" ] ; then
......@@ -192,10 +192,10 @@ gather_suite_arch_stats() {
AMOUNT=$(query_db "SELECT count(*) FROM sources WHERE suite='${SUITE}' AND architecture='$ARCH'")
COUNT_TOTAL=$(query_db "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH'")
COUNT_GOOD=$(query_db "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status='reproducible'")
COUNT_BAD=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible'")
COUNT_BAD=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBR'")
COUNT_UGLY=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS'")
COUNT_SOURCELESS=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = '404'")
COUNT_NOTFORUS=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'not for us'")
COUNT_SOURCELESS=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'E404'")
COUNT_NOTFORUS=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'NFU'")
COUNT_BLACKLISTED=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'blacklisted'")
COUNT_DEPWAIT=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'depwait'")
COUNT_OTHER=$(( $COUNT_SOURCELESS+$COUNT_NOTFORUS+$COUNT_BLACKLISTED+$COUNT_DEPWAIT ))
......@@ -335,10 +335,10 @@ write_build_performance_stats() {
done
write_page "</tr><tr><td class=\"left\">oldest build result in stretch / buster / unstable / experimental</td>"
for ARCH in ${ARCHS} ; do
AGE_UNSTABLE=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_unreproducible), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='unstable' AND architecture='$ARCH' AND datum='$DATE'")
AGE_EXPERIMENTAL=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_unreproducible), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='experimental' AND architecture='$ARCH' AND datum='$DATE'")
AGE_STRETCH=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_unreproducible), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='stretch' AND architecture='$ARCH' AND datum='$DATE'")
AGE_BUSTER=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_unreproducible), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='buster' AND architecture='$ARCH' AND datum='$DATE'")
AGE_UNSTABLE=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_FTBR), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='unstable' AND architecture='$ARCH' AND datum='$DATE'")
AGE_EXPERIMENTAL=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_FTBR), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='experimental' AND architecture='$ARCH' AND datum='$DATE'")
AGE_STRETCH=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_FTBR), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='stretch' AND architecture='$ARCH' AND datum='$DATE'")
AGE_BUSTER=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_FTBR), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='buster' AND architecture='$ARCH' AND datum='$DATE'")
write_page "<td>$AGE_STRETCH / $AGE_BUSTER / $AGE_UNSTABLE / $AGE_EXPERIMENTAL days</td>"
done
write_page "</tr><tr><td class=\"left\">Build jobs configured</td>"
......@@ -414,7 +414,7 @@ write_suite_arch_table() {
set_icon reproducible
write_icon
write_page "reproducible packages</th><th class=\"center\">"
set_icon unreproducible
set_icon FTBR
write_icon
write_page "unreproducible packages</th><th class=\"center\">"
set_icon FTBFS
......@@ -460,7 +460,7 @@ create_suite_arch_stats_page() {
set_icon reproducible
write_icon
write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly in $SUITE/$ARCH.<br />"
set_icon unreproducible
set_icon FTBR
write_icon
write_page "$COUNT_BAD packages ($PERCENT_BAD%) failed to build reproducibly.<br />"
set_icon FTBFS
......@@ -472,7 +472,7 @@ create_suite_arch_stats_page() {
write_page "$COUNT_DEPWAIT ($PERCENT_DEPWAIT%) source packages had build-depends which could not be satisfied.<br />"
fi
if [ $COUNT_SOURCELESS -gt 0 ] ; then
set_icon 404
set_icon E404
write_icon
write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) source packages could not be downloaded.<br />"
fi
......@@ -567,9 +567,9 @@ create_dashboard_page() {
gather_suite_arch_stats
TD_PKG_SID_ISSUES="$TD_PKG_SID_ISSUES<td>$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%</td>"
RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('unreproducible', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('FTBR', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_SID_NOISSUES="$TD_PKG_SID_NOISSUES<td><a href=\"/debian/$SUITE/$ARCH/index_no_notes.html\">$RESULT</a> / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='unreproducible' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBR' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_SID_FTBR="$TD_PKG_SID_FTBR<td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_SID_FTBFS="$TD_PKG_SID_FTBFS<td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
......@@ -577,9 +577,9 @@ create_dashboard_page() {
SUITE="buster"
gather_suite_arch_stats
TD_PKG_BUSTER_ISSUES="$TD_PKG_BUSTER_ISSUES<td>$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%</td>"
RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('unreproducible', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('FTBR', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_BUSTER_NOISSUES="$TD_PKG_BUSTER_NOISSUES<td><a href=\"/debian/$SUITE/$ARCH/index_no_notes.html\">$RESULT</a> / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='unreproducible' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBR' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_BUSTER_FTBR="$TD_PKG_BUSTER_FTBR<td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_BUSTER_FTBFS="$TD_PKG_BUSTER_FTBFS<td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
......
bin/reproducible_html_dd_list.py
View file @ ca3416d7
......@@ -40,7 +40,7 @@ for suite in SUITES:
continue
query = "SELECT s.name " + \
"FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \
"WHERE r.status='unreproducible' AND s.suite='{suite}'"
"WHERE r.status='FTBR' AND s.suite='{suite}'"
try:
pkgs = [x[0] for x in query_db(query.format(suite=suite))]
except IndexError:
......
bin/reproducible_html_indexes.py
View file @ ca3416d7
......@@ -143,14 +143,14 @@ queries = {
),
"FTBR_all":
select_sources.where(
results.c.status == 'unreproducible',
results.c.status == 'FTBR',
).order_by(
desc(results.c.build_date)
),
"FTBR_last24h":
select_sources.where(
and_(
results.c.status == 'unreproducible',
results.c.status == 'FTBR',
results.c.build_date > timespan_date_map[24],
)
).order_by(
......@@ -159,7 +159,7 @@ queries = {
"FTBR_last48h":
select_sources.where(
and_(
results.c.status == 'unreproducible',
results.c.status == 'FTBR',
results.c.build_date > timespan_date_map[48],
)
).order_by(
......@@ -167,7 +167,7 @@ queries = {
),
"FTBR_all_abc":
select_sources.where(
results.c.status == 'unreproducible',
results.c.status == 'FTBR',
).order_by(
sources.c.name
),
......@@ -235,15 +235,15 @@ queries = {
).order_by(
desc(results.c.build_date)
),
"404_all":
"E404_all":
select_sources.where(
results.c.status == '404',
results.c.status == 'E404',
).order_by(
desc(results.c.build_date)
),
"404_all_abc":
"E404_all_abc":
select_sources.where(
results.c.status == '404',
results.c.status == 'E404',
).order_by(
sources.c.name
),
......@@ -281,7 +281,7 @@ queries = {
"not_for_us_all":
select_sources.where(
and_(
results.c.status == 'not for us',
results.c.status == 'NFU',
)
).order_by(
......@@ -368,8 +368,8 @@ pages = {
'title': 'Packages in {suite}/{arch} where the sources failed to download',
'body': [
{
'icon_status': '404',
'query': '404_all',
'icon_status': 'E404',
'query': 'E404_all',
'text': Template('$tot ($percent%) packages where the sources failed to download in $suite/$arch:')
}
]
......@@ -388,7 +388,7 @@ pages = {
'title': 'Packages in {suite}/{arch} which should not be build on "{arch}"',
'body': [
{
'icon_status': 'not_for_us',
'icon_status': 'NFU',
'query': 'not_for_us_all',
'text': Template('$tot ($percent%) packages which should not be build in $suite/$arch:')
}
......@@ -420,15 +420,15 @@ pages = {
'text': Template('$tot packages ($percent%) failed to build from source in total $suite/$arch:')
},
{
'icon_status': 'not_for_us',
'icon_status': 'NFU',
'icon_link': '/index_not_for_us.html',
'query': 'not_for_us_all',
'text': Template('$tot ($percent%) packages which should not be build in $suite/$arch:')
},
{
'icon_status': '404',
'icon_status': 'E404',
'icon_link': '/index_404.html',
'query': '404_all_abc',
'query': 'E404_all_abc',
'text': Template('$tot ($percent%) source packages could not be downloaded in $suite/$arch:')
},
{
......@@ -541,7 +541,7 @@ pages = {
'body': [
{
'icon_status': 'FTBR',
'db_status': 'unreproducible',
'db_status': 'FTBR',
'icon_link': '/index_FTBR.html',
'query': 'notes',
'nosuite': True,
......@@ -563,8 +563,8 @@ pages = {
'text': Template('$tot ($percent%) source packages failed to satisfy their build-dependencies, ordered by build date:')
},
{
'icon_status': 'not_for_us',
'db_status': 'not for us',
'icon_status': 'NFU',
'db_status': 'NFU',
'icon_link': '/index_not_for_us.html',
'query': 'notes',
'nosuite': True,
......@@ -593,11 +593,11 @@ pages = {
'notes_hint': True,
'title': 'Packages without notes',
'header': '<p>There are {tot} faulty packages without notes in {suite}/{arch}.{hint}</p>',
'header_query': "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('unreproducible', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='{suite}' AND s.architecture='{arch}') AS tmp",
'header_query': "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('FTBR', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='{suite}' AND s.architecture='{arch}') AS tmp",
'body': [
{
'icon_status': 'FTBR',
'db_status': 'unreproducible',
'db_status': 'FTBR',
'icon_link': '/index_FTBR.html',
'query': 'no_notes',
'text': Template('$tot unreproducible packages in $suite/$arch, ordered by build date:')
......@@ -628,7 +628,7 @@ pages = {
'body': [
{
'icon_status': 'FTBR',
'db_status': 'unreproducible',
'db_status': 'FTBR',
'icon_link': '/index_FTBR.html',
'query': 'notification',
'text': Template('$tot unreproducible packages in $suite/$arch:'),
......
bin/reproducible_html_live_status.py
View file @ ca3416d7
......@@ -49,7 +49,7 @@ def generate_schedule(arch):
func.coalesce(func.avg(cast(stats_build.c.build_duration, Integer)), 0)
]).where(
and_(
stats_build.c.status.in_(('reproducible', 'unreproducible')),
stats_build.c.status.in_(('reproducible', 'FTBR')),
stats_build.c.name == sources.c.name,
stats_build.c.suite == sources.c.suite,
stats_build.c.architecture == sources.c.architecture,
......@@ -108,7 +108,7 @@ def generate_live_status_table(arch):
func.coalesce(func.avg(cast(stats_build.c.build_duration, Integer)), 0)
]).where(
and_(
stats_build.c.status.in_(('reproducible', 'unreproducible')),
stats_build.c.status.in_(('reproducible', 'FTBR')),
stats_build.c.name == sources.c.name,
stats_build.c.suite == sources.c.suite,
stats_build.c.architecture == sources.c.architecture,
......
bin/reproducible_html_notes.py
View file @ ca3416d7
......@@ -188,7 +188,7 @@ def load_notes():
"FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \
"WHERE s.name='{pkg}' AND r.status != ''"
query = query.format(pkg=package)
result = query_db(query)[0]
query_db(query)[0] # just discard this result, we only care of its success
except IndexError:
log.warning("This query produces no results: " + query)
log.warning("This means there is no tested package with the name " + package + ".")
......@@ -314,7 +314,7 @@ def gen_html_issue(issue, suite):
)
try:
arch = 'amd64'
for status in ['unreproducible', 'FTBFS', 'not for us', 'blacklisted', 'reproducible', 'depwait']:
for status in ['FTBR', 'FTBFS', 'NFU', 'blacklisted', 'reproducible', 'depwait']:
pkgs = query_db(sql.where(sources.c.name.in_(issues_count[issue]))\
.params({'suite': suite, 'arch': arch, 'status': status}))
pkgs = [p[0] for p in pkgs]
......@@ -327,8 +327,8 @@ def gen_html_issue(issue, suite):
affected += tab*5 + '<code>\n'
pkgs_popcon = issues_popcon_annotate(pkgs)
try:
for pkg, popcon, is_popular in sorted(pkgs_popcon, key=lambda x: x[0] in bugs):
affected += tab*6 + Package(pkg).html_link(suite, arch, bugs, popcon, is_popular)
for pkg, popc_num, is_popular in sorted(pkgs_popcon, key=lambda x: x[0] in bugs):
affected += tab*6 + Package(pkg).html_link(suite, arch, bugs, popc_num, is_popular)
except ValueError:
pass
affected += tab*5 + '</code>\n'
......
bin/reproducible_html_packages.py
View file @ ca3416d7
......@@ -112,7 +112,7 @@ def get_and_clean_dbd_links(package, eversion, suite, arch, status):
'dbd_uri': links['dbd_uri'],
}
else:
if status == 'unreproducible' and not args.ignore_missing_files:
if status == 'FTBR' and not args.ignore_missing_files:
log.critical(DISTRO_URL + '/' + suite + '/' + arch + '/' + package +
' is unreproducible, but without diffoscope output.')
# if there are no diffoscope results, we want to remove the old package
......@@ -158,7 +158,7 @@ def gen_suitearch_details(package, version, suite, arch, status, spokenstatus,
context['buildinfo_uri'] = build.buildinfo.url
default_view = default_view if default_view else build.buildinfo.url
elif not args.ignore_missing_files and status not in \
('untested', 'blacklisted', 'FTBFS', 'not_for_us', 'depwait', '404'):
('untested', 'blacklisted', 'FTBFS', 'NFU', 'depwait', '404'):
log.critical('buildinfo not detected at ' + build.buildinfo.path)
# Get rbuild, build2 and build diffs context
......
bin/reproducible_html_pkg_sets.py
View file @ ca3416d7
......@@ -83,7 +83,7 @@ def gather_meta_stats(suite, arch, pkgset_name):
stats['good'] = [t[0] for t in good]
stats['count_good'] = len(stats['good'])
bad = query_db(root_query + "AND r.status = 'unreproducible'" +
bad = query_db(root_query + "AND r.status = 'FTBR'" +
"ORDER BY r.build_date;")
stats['bad'] = [t[0] for t in bad]
stats['count_bad'] = len(stats['bad'])
......@@ -94,7 +94,7 @@ def gather_meta_stats(suite, arch, pkgset_name):
stats['count_ugly'] = len(stats['ugly'])
rest = query_db(root_query + "AND (r.status != 'FTBFS' AND " +
"r.status != 'unreproducible' AND " +
"r.status != 'FTBR' AND " +
"r.status != 'reproducible') ORDER BY r.build_date;")
stats['rest'] = [t[0] for t in rest]
stats['count_rest'] = len(stats['rest'])
......@@ -234,7 +234,7 @@ def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name):
pkgset_context['status_details'] = []
status_cutename_descriptions = [
('unreproducible', 'bad', 'failed to build reproducibly'),
('FTBR', 'bad', 'failed to build reproducibly'),
('FTBFS', 'ugly', 'failed to build from source'),
('rest', 'rest',
'are either in depwait state, blacklisted, not for us, or cannot be downloaded'),
......@@ -244,7 +244,7 @@ def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name):
for (status, cutename, description) in status_cutename_descriptions:
icon_html = ''
if status == 'rest':
for s in ['depwait', 'blacklisted', 'not_for_us', '404']:
for s in ['depwait', 'blacklisted', 'NFU', 'E404']:
s, icon, spokenstatus = get_status_icon(s)
icon_html += gen_status_link_icon(s, None, icon, suite, arch)
else:
......@@ -259,7 +259,7 @@ def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name):
'status_percent': stats["percent_" + cutename],
}
if (status in ('reproducible', 'unreproducible') or
if (status in ('reproducible', 'FTBR') or
stats["count_" + cutename] != 0):
pkgset_context['status_details'].append(details_context)
......@@ -284,7 +284,7 @@ def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name):
def create_pkgset_graph(png_file, suite, arch, pkgset_name):
table = "stats_meta_pkg_state"
columns = ["datum", "reproducible", "unreproducible", "FTBFS", "other"]
columns = ["datum", "reproducible", "FTBR", "FTBFS", "other"]
where = "WHERE suite = '%s' AND architecture = '%s' AND meta_pkg = '%s'" % \
(suite, arch, pkgset_name)
if arch == 'i386':
......
bin/reproducible_json.py
View file @ ca3416d7
......@@ -34,7 +34,7 @@ log.info('Creating json dump of current reproducible status')
# filter_query is defined in reproducible_common.py and excludes some FTBFS issues
query = "SELECT s.name, r.version, s.suite, s.architecture, r.status, r.build_date " + \
"FROM results AS r JOIN sources AS s ON r.package_id = s.id "+ \
"WHERE status != '' AND status NOT IN ('not for us', '404', 'blacklisted' ) AND (( status != 'FTBFS' ) OR " \
"WHERE status != '' AND status NOT IN ('NFU', 'E404', 'blacklisted' ) AND (( status != 'FTBFS' ) OR " \
" ( status = 'FTBFS' and r.package_id NOT IN (SELECT n.package_id FROM NOTES AS n WHERE " + filter_query + " )))"
result = sorted(query_db(query))
......@@ -88,8 +88,8 @@ for row in result:
else:
if 'FTBFS' in [status1, status2]:
newstatus = 'FTBFS'
elif 'unreproducible' in [status1, status2]:
newstatus = 'unreproducible'
elif 'FTBR' in [status1, status2]:
newstatus = 'FTBR'
elif 'reproducible' in [status1, status2]:
newstatus = 'reproducible'
else:
......
bin/reproducible_scheduler.py
View file @ ca3416d7
......@@ -64,7 +64,7 @@ else:
So, the 3rd step happens only when there are more than 350 packages queued up.
LIMITS_404 defines how many packages with status 404 are rescheduled at max.
LIMITS_E404 defines how many packages with status E404 are rescheduled at max.
"""
# only old packages older than this will be rescheduled
......@@ -204,8 +204,8 @@ LIMITS = {
}
}
}
# maximum amount of packages with status 404 which will be rescheduled
LIMIT_404 = 255
# maximum amount of packages with status E404 which will be rescheduled
LIMIT_E404 = 255
class Limit:
......@@ -540,14 +540,14 @@ def query_old_versions(suite, arch, limit):
print_schedule_result(suite, arch, criteria, packages)
return packages
def query_404_versions(suite, arch, limit):
criteria = """tested at least 12h ago, status 404,
def query_e404_versions(suite, arch, limit):
criteria = """tested at least 12h ago, status E404,
sorted by last build date"""
date = (datetime.now()-timedelta(days=0.5)).strftime('%Y-%m-%d %H:%M')
query = """SELECT s.id, s.name, max(r.build_date) max_date
FROM sources AS s JOIN results AS r ON s.id = r.package_id
WHERE s.suite='{suite}' AND s.architecture='{arch}'
AND r.status = '404'
AND r.status = 'E404'
AND r.build_date < '{date}'
AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
GROUP BY s.id, s.name
......@@ -657,18 +657,18 @@ def schedule_old_versions(arch, total):
msg = ''
return packages, msg
def schedule_404_versions(arch, total):
def schedule_e404_versions(arch, total):
packages = {}
for suite in SUITES:
log.info('Requesting 404 packages in %s/%s...',
log.info('Requesting E404 packages in %s/%s...',
suite, arch)
packages[suite] = query_404_versions(suite, arch, LIMIT_404)
packages[suite] = query_e404_versions(suite, arch, LIMIT_E404)
log.info('Received ' + str(len(packages[suite])) +
' 404 packages in ' + suite + '/' + arch + ' to schedule.')
' E404 packages in ' + suite + '/' + arch + ' to schedule.')
log.info('--------------------------------------------------------------')
msg = add_up_numbers(packages, arch)
if msg != '0':
msg += ' with status \'404\''
msg += ' with status \'E404\''
else:
msg = ''
return packages, msg
......@@ -692,7 +692,7 @@ def scheduler(arch):
old_ftbfs, msg_old_ftbfs = empty_pkgs, ''
old_depwait, msg_old_depwait = empty_pkgs, ''
old, msg_old = empty_pkgs, ''
four04, msg_404 = empty_pkgs, ''
four04, msg_e404 = empty_pkgs, ''
else:
log.info(str(total) + ' packages already scheduled' +
', scheduling some more...')
......@@ -700,7 +700,7 @@ def scheduler(arch):
new, msg_new = schedule_new_versions(arch, total+len(untested))
old_ftbfs, msg_old_ftbfs = schedule_old_ftbfs_versions(arch, total+len(untested)+len(new))
old_depwait, msg_old_depwait = schedule_old_depwait_versions(arch, total+len(untested)+len(new)+len(old_ftbfs))
four04, msg_404 = schedule_404_versions(arch, total+len(untested)+len(new)+len(old_ftbfs)+len(old_depwait))
four04, msg_e404 = schedule_e404_versions(arch, total+len(untested)+len(new)+len(old_ftbfs)+len(old_depwait))
old, msg_old = schedule_old_versions(arch, total+len(untested)+len(new)+len(old_ftbfs)+len(old_depwait)+len(four04))
now_queued_here = {}
......@@ -734,8 +734,8 @@ def scheduler(arch):
message += msg_untested + ', '
if msg_new:
message += msg_new + ', '
if msg_404:
message += msg_404 + ', '
if msg_e404:
message += msg_e404 + ', '
if msg_old_ftbfs:
message += msg_old_ftbfs + ', '
if msg_old_depwait:
......