reproducible_html_dashboard.sh 36.4 KB
Newer Older
1
#!/bin/bash
2
# vim: set noexpandtab:
3

4
# Copyright 2014-2018 Holger Levsen <holger@layer-acht.org>
5
#         © 2015 Mattia Rizzolo <mattia@mapreri.org>
6 7
# released under the GPLv=2

8
DEBUG=false
Holger Levsen's avatar
Holger Levsen committed
9 10 11
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

Holger Levsen's avatar
Holger Levsen committed
12 13
# common code defining db access
. /srv/jenkins/bin/reproducible_common.sh
14

15
#
16
# init some variables
17
#
Holger Levsen's avatar
Holger Levsen committed
18
# we only do stats up until yesterday... we also could do today too but not update the db yet...
19 20
DATE=$(date -u -d "1 day ago" '+%Y-%m-%d')
FORCE_DATE=$(date -u -d "3 days ago" '+%Y-%m-%d')
21 22
DUMMY_FILE=$(mktemp -t reproducible-dashboard-XXXXXXXX)
touch -d "$(date '+%Y-%m-%d') 00:00 UTC" $DUMMY_FILE
23
NOTES_GIT_PATH="/var/lib/jenkins/jobs/reproducible_html_notes/workspace"
24
COMMA_VAR=""
25

Holger Levsen's avatar
Holger Levsen committed
26
# variables related to the stats we update
27
FIELDS[0]="datum, reproducible, FTBR, FTBFS, other, untested"
28
FIELDS[1]="datum"
29
for i in reproducible FTBR FTBFS other ; do
30 31 32 33
	for j in $SUITES ; do
		FIELDS[1]="${FIELDS[1]}, ${i}_${j}"
	done
done
34
FIELDS[2]="datum, oldest"
35
FIELDS[3]="datum "
36 37 38 39 40 41
for TAG in $USERTAGS ; do
	# for this table (#3) bugs with ftbfs tags are ignored _now_…
	if [ "$TAG" = "ftbfs" ] ; then
		continue
	fi
	FIELDS[3]="${FIELDS[3]}, open_$TAG, done_$TAG"
42
done
43 44 45
# …and added at the end (so they are not ignored but rather sorted this way)
# Also note how FIELDS is only used for reading data, not writing.
FIELDS[3]="${FIELDS[3]}, open_ftbfs, done_ftbfs"
46 47
FIELDS[4]="datum, packages_with_notes"
FIELDS[5]="datum, known_issues"
48 49 50 51 52 53 54 55 56
FIELDS[7]="datum, done_bugs, open_bugs"
SUM_DONE="(0"
SUM_OPEN="(0"
for TAG in $USERTAGS ; do
	SUM_DONE="$SUM_DONE+done_$TAG"
	SUM_OPEN="$SUM_OPEN+open_$TAG"
done
SUM_DONE="$SUM_DONE)"
SUM_OPEN="$SUM_OPEN)"
57
FIELDS[8]="datum "
58
for STATE in open_ done_ ; do
59 60 61 62 63 64
	for TAG in $USERTAGS ; do
		if [ "$TAG" = "ftbfs" ] ; then
			continue
		fi
		FIELDS[8]="${FIELDS[8]}, ${STATE}$TAG"
	done
65
	# ftbfs bugs are excluded from 8+9
66 67 68 69 70
done
FIELDS[9]="datum, done_bugs, open_bugs"
REPRODUCIBLE_DONE="(0"
REPRODUCIBLE_OPEN="(0"
for TAG in $USERTAGS ; do
71
	# for this table (#9) bugs with ftbfs tags are ignored.
72 73 74 75 76 77 78 79
	if [ "$TAG" = "ftbfs" ] ; then
		continue
	fi
	REPRODUCIBLE_DONE="$REPRODUCIBLE_DONE+done_$TAG"
	REPRODUCIBLE_OPEN="$REPRODUCIBLE_OPEN+open_$TAG"
done
REPRODUCIBLE_DONE="$REPRODUCIBLE_DONE)"
REPRODUCIBLE_OPEN="$REPRODUCIBLE_OPEN)"
80
COLOR[0]=5
81
COLOR[1]=16
82
COLOR[2]=1
83
COLOR[3]=32
84 85
COLOR[4]=1
COLOR[5]=1
86
COLOR[7]=2
87 88 89
COLOR[8]=30
COLOR[9]=2
MAINLABEL[3]="Bugs (with all usertags) for user reproducible-builds@lists.alioth.debian.org"
90 91
MAINLABEL[4]="Packages which have notes"
MAINLABEL[5]="Identified issues"
92 93 94
MAINLABEL[7]="Open and closed bugs (with all usertags)"
MAINLABEL[8]="Bugs (with all usertags except 'ftbfs') for user reproducible-builds@lists.alioth.debian.org"
MAINLABEL[9]="Open and closed bugs (with all usertags except tagged 'ftbfs')"
95 96 97 98 99 100
YLABEL[0]="Amount (total)"
YLABEL[1]="Amount (per day)"
YLABEL[2]="Age in days"
YLABEL[3]="Amount of bugs"
YLABEL[4]="Amount of packages"
YLABEL[5]="Amount of issues"
101
YLABEL[7]="Amount of bugs open / closed"
102 103
YLABEL[8]="Amount of bugs"
YLABEL[9]="Amount of bugs open / closed"
104 105

#
106
# update package + build stats
107
#
108
update_suite_arch_stats() {
109
	RESULT=$(query_db "SELECT datum,suite from ${TABLE[0]} WHERE datum = '$DATE' AND suite = '$SUITE' AND architecture = '$ARCH'")
110
	if [ -z $RESULT ] ; then
111
		echo "Updating packages and builds stats for $SUITE/$ARCH in $DATE."
112 113 114
		ALL=$(query_db "SELECT count(name) FROM sources WHERE suite='${SUITE}' AND architecture='$ARCH'")
		GOOD=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE';")
		GOOAY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)='$DATE';")
115 116
		BAD=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBR' AND date(r.build_date)<='$DATE';")
		BAAY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id  WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBR' AND date(r.build_date)='$DATE';")
117 118
		UGLY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id  WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE';")
		UGLDAY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id  WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)='$DATE';")
119 120
		REST=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'FTBR' AND r.status != 'reproducible') AND s.suite='$SUITE' AND s.architecture='$ARCH' AND date(r.build_date)<='$DATE';")
		RESDAY=$(query_db "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'FTBR' AND r.status != 'reproducible') AND s.suite='$SUITE' AND s.architecture='$ARCH' AND date(r.build_date)='$DATE';")
121
		OLDESTG=$(query_db "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.status = 'reproducible' AND s.suite='$SUITE' AND s.architecture='$ARCH' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
122
		OLDESTB=$(query_db "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBR' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
123
		OLDESTU=$(query_db "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;")
124 125 126 127 128 129 130 131 132
		# only if we have results…
		if [ -n "$OLDESTG" ] ; then
			DIFFG=$(query_db "SELECT (date '$DATE' - date '$OLDESTG');")
			if [ -z $DIFFG ] ; then DIFFG=0 ; fi
			DIFFB=$(query_db "SELECT (date '$DATE' - date '$OLDESTB');")
			if [ -z $DIFFB ] ; then DIFFB=0 ; fi
			DIFFU=$(query_db "SELECT (date '$DATE' - date '$OLDESTU');")
			if [ -z $DIFFU ] ; then DIFFU=0 ; fi
		fi
133 134 135 136 137
		let "TOTAL=GOOD+BAD+UGLY+REST" || true # let FOO=0+0 returns error in bash...
		if [ "$ALL" != "$TOTAL" ] ; then
			let "UNTESTED=ALL-TOTAL"
		else
			UNTESTED=0
138
		fi
139 140 141 142 143
		if [ -n "$OLDESTG" ] ; then
			query_db "INSERT INTO ${TABLE[0]} VALUES ('$DATE', '$SUITE', '$ARCH', $UNTESTED, $GOOD, $BAD, $UGLY, $REST)"
			query_db "INSERT INTO ${TABLE[1]} VALUES ('$DATE', '$SUITE', '$ARCH', $GOOAY, $BAAY, $UGLDAY, $RESDAY)"
			query_db "INSERT INTO ${TABLE[2]} VALUES ('$DATE', '$SUITE', '$ARCH', '$DIFFG', '$DIFFB', '$DIFFU')"
		fi
144
		# we do 3 later and 6 is special anyway...
145
		for i in 0 1 2 4 5 ; do
146
			PREFIX=""
147
			if [ $i -eq 0 ] || [ $i -eq 2 ] ; then
Holger Levsen's avatar
Holger Levsen committed
148
				PREFIX=$SUITE/$ARCH
149 150
			fi
			# force regeneration of the image if it exists
151
			if [ -f $DEBIAN_BASE/$PREFIX/${TABLE[$i]}.png ] ; then
152
				echo "Touching $PREFIX/${TABLE[$i]}.png..."
153
				touch -d "$FORCE_DATE 00:00 UTC" $DEBIAN_BASE/$PREFIX/${TABLE[$i]}.png
154 155 156 157
			fi
		done
	fi
}
158 159

#
160
# update notes stats
161
#
162
update_notes_stats() {
163
	NOTES=$(query_db "SELECT COUNT(package_id) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite='unstable' AND s.architecture='amd64'")
164
	ISSUES=$(query_db "SELECT COUNT(name) FROM issues")
165 166
	# the following is a hack to workaround the bad sql db design which is the issue_s_ column in the notes table...
	# it assumes we don't have packages with more than 7 issues. (we have one with 6...)
167
	COUNT_ISSUES=$(query_db "SELECT \
168
		(SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite='unstable' AND s.architecture='amd64' AND n.issues = '[]') \
169
		+ \
170
		(SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite='unstable' AND s.architecture='amd64' AND n.issues != '[]' AND n.issues NOT LIKE '%,%') \
171
		+ \
172
		(SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite='unstable' AND s.architecture='amd64' AND n.issues LIKE '%,%') \
173
		+ \
174
		(SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite='unstable' AND s.architecture='amd64' AND n.issues LIKE '%,%,%') \
175
		+ \
176
		(SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite='unstable' AND s.architecture='amd64' AND n.issues LIKE '%,%,%,%') \
177
		+ \
178
		(SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite='unstable' AND s.architecture='amd64' AND n.issues LIKE '%,%,%,%,%') \
179
		+ \
180
		(SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite='unstable' AND s.architecture='amd64' AND n.issues LIKE '%,%,%,%,%,%') \
181
		")
182
	RESULT=$(query_db "SELECT datum from ${TABLE[4]} WHERE datum = '$DATE'")
183 184
	if [ -z $RESULT ] ; then
		echo "Updating notes stats for $DATE."
185 186
		query_db "INSERT INTO ${TABLE[4]} VALUES ('$DATE', '$NOTES')"
		query_db "INSERT INTO ${TABLE[5]} VALUES ('$DATE', '$ISSUES')"
187
	fi
188
}
189

190
#
191
# gather suite/arch stats
192
#
193
gather_suite_arch_stats() {
194 195 196
	AMOUNT=$(query_db "SELECT count(*) FROM sources WHERE suite='${SUITE}' AND architecture='$ARCH'")
	COUNT_TOTAL=$(query_db "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH'")
	COUNT_GOOD=$(query_db "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status='reproducible'")
197
	COUNT_BAD=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBR'")
198
	COUNT_UGLY=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS'")
199
	COUNT_TIMEOUT=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'timeout'")
200
	COUNT_SOURCELESS=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'E404'")
201
	COUNT_NOTFORUS=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'NFU'")
202 203
	COUNT_BLACKLISTED=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'blacklisted'")
	COUNT_DEPWAIT=$(query_db "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'depwait'")
204
	COUNT_OTHER=$(( $COUNT_SOURCELESS+$COUNT_NOTFORUS+$COUNT_BLACKLISTED+$COUNT_DEPWAIT+$COUNT_TIMEOUT ))
205
	PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc)
206 207 208
	PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc || echo 0)
	PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc || echo 0)
	PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc || echo 0)
209
	PERCENT_TIMEOUT=$(echo "scale=1 ; ($COUNT_TIMEOUT*100/$COUNT_TOTAL)" | bc || echo 0)
210 211 212
	PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc || echo 0)
	PERCENT_DEPWAIT=$(echo "scale=1 ; ($COUNT_DEPWAIT*100/$COUNT_TOTAL)" | bc || echo 0)
	PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc || echo 0)
213
	PERCENT_BLACKLISTED=$(echo "scale=1 ; ($COUNT_BLACKLISTED*100/$COUNT_TOTAL)" | bc || echo 0)
214
	PERCENT_OTHER=$(echo "scale=1 ; ($COUNT_OTHER*100/$COUNT_TOTAL)" | bc || echo 0)
215 216
}

217
#
218
# update bug stats
219
#
220
update_bug_stats() {
221
	RESULT=$(query_db "SELECT * from ${TABLE[3]} WHERE datum = '$DATE'")
222 223 224 225
	if [ -z $RESULT ] ; then
		echo "Updating bug stats for $DATE."
		declare -a DONE
		declare -a OPEN
226
		GOT_BTS_RESULTS=false
227
		SQL="INSERT INTO ${TABLE[3]} VALUES ('$DATE' "
228 229 230 231 232 233 234 235 236 237
		for TAG in $USERTAGS ; do
			OPEN[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:open status:forwarded 2>/dev/null|wc -l)
			DONE[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:done archive:both 2>/dev/null|wc -l)
			# test if both values are integers
			if ! ( [[ ${DONE[$TAG]} =~ ^-?[0-9]+$ ]] && [[ ${OPEN[$TAG]} =~ ^-?[0-9]+$ ]] ) ; then
				echo "Non-integers value detected, exiting."
				echo "Usertag: $TAG"
				echo "Open: ${OPEN[$TAG]}"
				echo "Done: ${DONE[$TAG]}"
				exit 1
Holger Levsen's avatar
Holger Levsen committed
238
			elif [ ! "${DONE[$TAG]}" = "0" ] || [ ! "${OPEN[$TAG]}" = "0" ] ; then
239
				GOT_BTS_RESULTS=true
240 241 242 243 244
			fi
			SQL="$SQL, ${OPEN[$TAG]}, ${DONE[$TAG]}"
		done
		SQL="$SQL)"
		echo $SQL
245
		if $GOT_BTS_RESULTS ; then
246
			echo "Updating database with bug stats for $DATE."
247
			query_db "$SQL"
248
			# force regeneration of the image
249 250 251
			local i=0
			for i in 3 7 8 9 ; do
				echo "Touching ${TABLE[$i]}.png..."
252
				touch -d "$FORCE_DATE 00:00 UTC" $DEBIAN_BASE/${TABLE[$i]}.png
253
			done
254
		fi
255 256
	fi
}
257

258 259 260
#
# gather bugs stats and generate html table
#
261
write_usertag_table() {
262
	RESULT=$(query_db "SELECT ${FIELDS[3]} from ${TABLE[3]} WHERE datum = '$DATE'")
Holger Levsen's avatar
Holger Levsen committed
263
	if [ ! -z "$RESULT" ] ; then
264
		COUNT=0
265
		TOPEN=0 ; TDONE=0 ; TTOTAL=0
266 267 268 269
		for FIELD in $(echo ${FIELDS[3]} | tr -d ,) ; do
			let "COUNT+=1"
			VALUE=$(echo $RESULT | cut -d "|" -f$COUNT)
			if [ $COUNT -eq 1 ] ; then
270
				write_page "<table class=\"main\" id=\"usertagged-bugs\"><tr><th>Usertagged bugs</th><th>Open</th><th>Done</th><th>Total</th></tr>"
271
			elif [ $((COUNT%2)) -eq 0 ] ; then
Holger Levsen's avatar
Holger Levsen committed
272
				write_page "<tr><td><a href=\"https://bugs.debian.org/cgi-bin/pkgreport.cgi?tag=${FIELD:5};users=reproducible-builds@lists.alioth.debian.org&amp;archive=both\">${FIELD:5}</a></td><td>$VALUE</td>"
273
				TOTAL=$VALUE
274
				let "TOPEN=TOPEN+VALUE" || TOPEN=0
275
			else
276
				write_page "<td>$VALUE</td>"
277
				let "TOTAL=TOTAL+VALUE" || true # let FOO=0+0 returns error in bash...
278
				let "TDONE=TDONE+VALUE"
279
				write_page "<td>$TOTAL</td></tr>"
280
				let "TTOTAL=TTOTAL+TOTAL"
281 282
			fi
		done
283
		# now subtract the ftbfs bugs again (=the last fields from the result)
284
		# as those others are the ones we really care about
285 286 287
		let "CCOUNT=COUNT-1"
		let "REPRODUCIBLE_TOPEN=TOPEN-$(echo $RESULT | cut -d "|" -f$CCOUNT)"
		let "REPRODUCIBLE_TDONE=TDONE-$(echo $RESULT | cut -d "|" -f$COUNT)"
288 289 290
		let "REPRODUCIBLE_TTOTAL=REPRODUCIBLE_TOPEN+REPRODUCIBLE_TDONE"
		write_page "<tr><td>Sum of <a href=\"https://wiki.debian.org/ReproducibleBuilds/Contribute#How_to_report_bugs\">bugs with usertags related to reproducible builds</a>, excluding those tagged 'ftbfs'</td><td>$REPRODUCIBLE_TOPEN</td><td>$REPRODUCIBLE_TDONE</td><td>$REPRODUCIBLE_TTOTAL</td></tr>"
		write_page "<tr><td>Sum of all bugs with usertags related to reproducible builds</td><td>$TOPEN</td><td>$TDONE</td><td>$TTOTAL</td></tr>"
291
		write_page "<tr><td colspan=\"4\" class=\"left\">Stats are from $DATE.<br />The sums of usertags shown are not equivalent to the sum of bugs as a single bug can have several tags.</td></tr>"
292 293 294 295
		write_page "</table>"
	fi
}

Holger Levsen's avatar
Holger Levsen committed
296
#
297
# write build performance stats
Holger Levsen's avatar
Holger Levsen committed
298
#
299 300 301 302 303 304 305 306
_average_builds_per_day() {
	local TIMESPAN_RAW="$1"
	local TIMESPAN_VERBOSE="$2"
	local MIN_DAYS="${3-0}"
	write_page "<tr><td class=\"left\">packages tested on average per day in the last $TIMESPAN_VERBOSE</td>"
	for ARCH in ${ARCHS} ; do
		local OLDEST_BUILD="$(query_db "SELECT build_date FROM stats_build WHERE architecture='$ARCH' ORDER BY build_date ASC LIMIT 1")"
		local DAY_DIFFS="$(( ($(date -d "$DATE" +%s) - $(date -d "$OLDEST_BUILD" +%s)) / (60*60*24) ))"
307
		local DISCLAIMER=""
308
		local TIMESPAN="$TIMESPAN_RAW"
309
		if [ $DAY_DIFFS -ge $MIN_DAYS ]; then
310 311
			if [ $DAY_DIFFS -lt $TIMESPAN ]; then
				# this is a new architecture, there are fewer days to compare to.
312
				DISCLAIMER=" <span style=\"font-size: 0.8em;\">(last $DAY_DIFFS days)</span>"
313 314
				TIMESPAN=$DAY_DIFFS
			fi
315 316
			# find stats for since the day before $TIMESPAN_RAW days ago,
			# since no stats exist for today yet.
317
			local TIMESPAN="$(echo $TIMESPAN-1|bc)"
318 319 320
			local TIMESPAN_DATE=$(date '+%Y-%m-%d %H:%M' -d "- $TIMESPAN days")

			RESULT=$(query_db "SELECT COUNT(r.build_date) FROM stats_build AS r WHERE r.build_date > '$TIMESPAN_DATE' AND r.architecture='$ARCH'")
321
			RESULT="$(echo $RESULT/$TIMESPAN|bc)"
322
		else
323
			# very new arch with too few results to care about stats
324 325
			RESULT="&nbsp;"
		fi
326
		write_page "<td>${RESULT}${DISCLAIMER}</td>"
327 328 329
	done
	write_page "</tr>"
}
330
write_build_performance_stats() {
331
	local ARCH
332 333 334 335
	local lockfile="/var/lib/jenkins/NO-RB-BUILDERS-PLEASE"
	if [ -f "$lockfile" ]; then
		write_page "<p><strong>Warning</strong>: lock file <code>$lockfile</code> present, the builders are shutting down due to administrator request.</p>"
	fi
336
	write_page "<table class=\"main\"><tr><th>Architecture build statistics</th>"
337 338 339
	for ARCH in ${ARCHS} ; do
		write_page " <th>$ARCH</th>"
	done
340
	write_page "</tr><tr><td class=\"left\">oldest build result in stretch / buster / unstable / experimental</td>"
341
	for ARCH in ${ARCHS} ; do
342 343 344 345
		AGE_UNSTABLE=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_FTBR), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='unstable' AND architecture='$ARCH' AND datum='$DATE'")
		AGE_EXPERIMENTAL=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_FTBR), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='experimental' AND architecture='$ARCH' AND datum='$DATE'")
		AGE_STRETCH=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_FTBR), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='stretch' AND architecture='$ARCH' AND datum='$DATE'")
		AGE_BUSTER=$(query_db "SELECT CAST(greatest(max(oldest_reproducible), max(oldest_FTBR), max(oldest_FTBFS)) AS INTEGER) FROM ${TABLE[2]} WHERE suite='buster' AND architecture='$ARCH' AND datum='$DATE'")
346
		write_page "<td>$AGE_STRETCH / $AGE_BUSTER / $AGE_UNSTABLE / $AGE_EXPERIMENTAL days</td>"
347
	done
348 349
	write_page "</tr><tr><td class=\"left\">Build jobs configured</td>"
	for ARCH in ${ARCHS} ; do
350
		write_page "<td>$(grep NODE1= /srv/jenkins/bin/reproducible_build_service.sh | grep -v \# | grep -c $ARCH)</td>"
351
	done
352
	write_page "</tr><tr><td class=\"left\">Build jobs currently running remotely</td>"
353
	for ARCH in ${ARCHS} ; do
354
		write_page "<td>$(ps fax|grep reproducible_build.sh|grep ssh|grep -c $ARCH)</td>"
355
	done
356 357
	write_page "</tr><tr><td class=\"left\">Build jobs currently down due to remote node problems</td>"
	for ARCH in ${ARCHS} ; do
358
		write_page "<td>$(ps fax|egrep -B 1 'sleep .*\.1337m'|grep -c $ARCH)</td>"
359
	done
360 361 362 363
	write_page "</tr><tr><td class=\"left\">Build jobs currently down due to local problems</td>"
	for ARCH in ${ARCHS} ; do
		write_page "<td>$(rgrep -l 'Failed to get properties: Connection timed out' $DEBIAN_BASE/build_service/${ARCH}_*/worker.log | wc -l)</td>"
	done
364 365 366 367
	write_page "</tr><tr><td class=\"left\">Build jobs currently running diffoscope</td>"
	for ARCH in ${ARCHS} ; do
		write_page "<td>$(ps fax|grep "diffoscope --html /srv/reproducible-results/rbuild-debian"|grep -c $ARCH)</td>"
	done
368
	write_page "</tr><tr><td class=\"left\">average test duration (on $DATE)</td>"
369
	for ARCH in ${ARCHS} ; do
370
		RESULT=$(query_db "SELECT COALESCE(CAST(AVG(r.build_duration) AS INTEGER), 0) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.build_duration!='0' AND r.build_date LIKE '%$DATE%' AND s.architecture='$ARCH'")
371 372
		MIN=$(echo $RESULT/60|bc)
		SEC=$(echo "$RESULT-($MIN*60)"|bc)
373
		write_page "<td>$MIN min., $SEC sec.</td>"
374
	done
375

376 377
	local TIMESPAN_VERBOSE="4 weeks"
	local TIMESPAN_RAW="28"
378
	# Find stats for 28 days since yesterday, no stats exist for today
379

380
	local TIMESPAN="$(echo $TIMESPAN_RAW-1|bc)"
381 382
	local TIMESPAN_DATE=$(date '+%Y-%m-%d %H:%M' -d "- $TIMESPAN days")

383
	write_page "</tr><tr><td class=\"left\">average test duration (in the last $TIMESPAN_VERBOSE)</td>"
384
	for ARCH in ${ARCHS} ; do
385
		RESULT=$(query_db "SELECT COALESCE(CAST(AVG(r.build_duration) AS INTEGER), 0) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.build_duration!='0' AND r.build_date > '$TIMESPAN_DATE' AND s.architecture='$ARCH'")
386 387
		MIN=$(echo $RESULT/60|bc)
		SEC=$(echo "$RESULT-($MIN*60)"|bc)
388
		write_page "<td>$MIN min., $SEC sec.</td>"
389
	done
390

391
	write_page "</tr><tr><td class=\"left\">packages tested on $DATE</td>"
392
	for ARCH in ${ARCHS} ; do
393
		RESULT=$(query_db "SELECT COUNT(r.build_date) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.build_date LIKE '%$DATE%' AND s.architecture='$ARCH'")
394 395
		write_page "<td>$RESULT</td>"
	done
396
	write_page "</tr><tr><td class=\"left\">packages tested in the last 24h</td>"
397
	for ARCH in ${ARCHS} ; do
398
		RESULT=$(query_db "SELECT COUNT(r.build_date) FROM stats_build AS r WHERE r.build_date > '$(date '+%Y-%m-%d %H:%M' -d '-1 days')' AND r.architecture='$ARCH'")
399 400
		write_page "<td>$RESULT</td>"
	done
401
	write_page "</tr>"
402

403
	_average_builds_per_day "7" "week"
404
	_average_builds_per_day "$TIMESPAN_RAW" "$TIMESPAN_VERBOSE"
405
	_average_builds_per_day "91" "3 months" "30"
406

407
	write_page "</table>"
Holger Levsen's avatar
Holger Levsen committed
408 409
}

410
#
411
# write suite/arch table
412
#
413 414 415
write_suite_arch_table() {
	local SUITE=""
	local ARCH=""
416
	write_page "<p>"
417
	write_page "<table class=\"main\"><tr><th class=\"left\">suite</th><th class=\"center\">all source packages</th><th class=\"center\">"
418 419
	set_icon reproducible
	write_icon
420
	write_page "reproducible packages</th><th class=\"center\">"
421
	set_icon FTBR
422
	write_icon
423
	write_page "unreproducible packages</th><th class=\"center\">"
424 425
	set_icon FTBFS
	write_icon
426
	write_page "packages failing to build</th><th class=\"center\">"
427 428 429
	set_icon timeout
	write_icon
	write_page "packages timing out</th><th class=\"center\">"
430 431
	set_icon depwait
	write_icon
432
	write_page "packages in depwait state</th><th class=\"center\">"
433
	set_icon NFU
434
	write_icon
435
	write_page "not for this architecture</th><th class=\"center\">"
436 437 438
	set_icon blacklisted
	write_icon
	write_page "blacklisted</th></tr>"
439
	for SUITE in $SUITES ; do
440 441
		for ARCH in ${ARCHS} ; do
			gather_suite_arch_stats
442
			write_page "<tr><td class=\"left\"><a href=\"/debian/$SUITE/$ARCH\">$SUITE/$ARCH</a></td><td>$AMOUNT"
443 444 445
			if [ $(echo $PERCENT_TOTAL/1|bc) -lt 99 ] ; then
				write_page "<span style=\"font-size:0.8em;\">($PERCENT_TOTAL% tested)</span>"
			fi
446
			write_page "</td><td>$COUNT_GOOD / $PERCENT_GOOD%</td><td>$COUNT_BAD / $PERCENT_BAD%</td><td>$COUNT_UGLY / $PERCENT_UGLY%</td><td>$COUNT_TIMEOUT / $PERCENT_TIMEOUT%</td><td>$COUNT_DEPWAIT / $PERCENT_DEPWAIT%</td><td>$COUNT_NOTFORUS / $PERCENT_NOTFORUS%</td><td>$COUNT_BLACKLISTED / $PERCENT_BLACKLISTED%</td></tr>"
447
		done
448 449 450 451 452
	done
        write_page "</table>"
	write_page "</p><p style=\"clear:both;\">"
}

453 454 455
#
# create suite stats page
#
456
create_suite_arch_stats_page() {
457 458
	VIEW=suite_arch_stats
	PAGE=index_suite_${ARCH}_stats.html
459 460
	MAINLABEL[0]="Reproducibility status for packages in '$SUITE' for '$ARCH'"
	MAINLABEL[2]="Age in days of oldest reproducible build result in '$SUITE' for '$ARCH'"
Holger Levsen's avatar
Holger Levsen committed
461
	echo "$(date -u) - starting to write $PAGE page."
462
	write_page_header $VIEW "Overview of reproducible builds for packages in $SUITE for $ARCH"
463
	if [ $(echo $PERCENT_TOTAL/1|bc) -lt 100 ] ; then
464
		write_page "<p>$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE/$ARCH.</p>"
465 466 467
	fi
	write_page "<p>"
	set_icon reproducible
468
	write_icon
469
	write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly in $SUITE/$ARCH.<br />"
470
	set_icon FTBR
471
	write_icon
472
	write_page "$COUNT_BAD packages ($PERCENT_BAD%) failed to build reproducibly.<br />"
473 474
	set_icon FTBFS
	write_icon
475
	write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source.<br /></p>"
476 477 478 479
	write_page "<p>"
	set_icon timeout
	write_icon
	write_page "$COUNT_TIMEOUT packages ($PERCENT_TIMEOUT%) timed out during the build.<br />"
480 481 482 483 484 485
	set_icon depwait
	write_icon
	write_page "$COUNT_DEPWAIT ($PERCENT_DEPWAIT%) source packages had build-depends which could not be satisfied.<br />"
	set_icon E404
	write_icon
	write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) source packages could not be downloaded.<br />"
486
	set_icon NFU
487
	write_icon
488
	if [ "$ARCH" = "armhf" ] || [ "$ARCH" = "arm64" ]; then
489 490
		ARMSPECIALARCH=" 'any-arm',"
	fi
491
	write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', '$ARCH', 'linux-any', 'linux-$ARCH'$ARMSPECIALARCH nor 'any-$ARCH' will not be built here.<br />"
492 493
	set_icon blacklisted
	write_icon
494 495
	write_page "$COUNT_BLACKLISTED ($PERCENT_BLACKLISTED%) packages have been blacklisted on $SUITE/$ARCH.<br />"
	write_page "</p><p>"
496
	write_page " <a href=\"/debian/$SUITE/$ARCH/${TABLE[0]}.png\"><img src=\"/debian/$SUITE/$ARCH/${TABLE[0]}.png\" alt=\"${MAINLABEL[0]}\"></a>"
497
	for i in 0 2 ; do
498
		# recreate png once a day
499
		if [ ! -f $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[$i]}.png ] || [ $DUMMY_FILE -nt $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[$i]}.png ] ; then
Holger Levsen's avatar
Holger Levsen committed
500
			create_debian_png_from_table $i $SUITE/$ARCH/${TABLE[$i]}.png
501 502
		fi
	done
503
	write_page "</p>"
504 505 506
	if [ "$SUITE" != "experimental" ] ; then
		write_meta_pkg_graphs_links
	fi
507
	write_page_footer
508
	publish_page debian/$SUITE
509 510
}

511 512 513 514 515
write_meta_pkg_graphs_links () {
	write_page "<p style=\"clear:both;\"><center>"
	for i in $(seq 1 ${#META_PKGSET[@]}) ; do
		THUMB=${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png
		LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'"
516
		write_page "<a href=\"/debian/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"  title=\"$LABEL\"><img src=\"/debian/$SUITE/$ARCH/$THUMB\" class=\"metaoverview\" alt=\"$LABEL\"></a>"
517 518 519 520
	done
	write_page "</center></p>"
}

521 522 523 524
write_global_graph() {
	write_page " <a href=\"/debian/${TABLE[$i]}.png\"><img src=\"/debian/${TABLE[$i]}.png\" class="halfview" alt=\"${MAINLABEL[$i]}\"></a>"
	# redo pngs once a day
	if [ ! -f $DEBIAN_BASE/${TABLE[$i]}.png ] || [ $DUMMY_FILE -nt $DEBIAN_BASE/${TABLE[$i]}.png ] ; then
Holger Levsen's avatar
Holger Levsen committed
525
		create_debian_png_from_table $i ${TABLE[$i]}.png
526 527 528
	fi
}

529
#
530
# create dashboard page
531
#
532 533
create_dashboard_page() {
	VIEW=dashboard
534
	PAGE=index_${VIEW}.html
535 536
	SUITE="unstable"
	ARCH="amd64"
Holger Levsen's avatar
Holger Levsen committed
537
	echo "$(date -u) - starting to write $PAGE page."
538
	write_page_header $VIEW "Overview of various statistics about reproducible builds"
539
	write_suite_arch_table
540
	# write suite graphs
541 542
	for ARCH in ${ARCHS} ; do
		for SUITE in $SUITES ; do
543
			write_page " <a href=\"/debian/$SUITE/$ARCH\"><img src=\"/debian/$SUITE/$ARCH/${TABLE[0]}.png\" class=\"overview\" alt=\"$SUITE/$ARCH stats\"></a>"
544
		done
545
		SUITE="unstable"
546 547 548
		if [ "$ARCH" = "amd64" ] ; then
			write_meta_pkg_graphs_links
		fi
549
	done
550
	write_page "</p>"
551
	# write inventory table
552
	write_page "<p><table class=\"main\"><tr><th class=\"left\">Various reproducibility statistics</th><th class=\"center\">source based</th>"
553
	AC=0
554
	for ARCH in ${ARCHS} ; do
555
		write_page "<th class=\"center\">$ARCH</th>"
556
		let AC+=1
557
	done
558 559
	write_page "</tr>"
	ARCH="amd64"
560
	write_page "<tr><td class=\"left\">identified <a href=\"/debian/index_issues.html\">distinct and categorized issues</a></td><td>$ISSUES</td><td colspan=\"$AC\"></td></tr>"
561 562
	write_page "<tr><td class=\"left\">total number of identified issues in packages</td><td>$COUNT_ISSUES</td><td colspan=\"$AC\"></td></tr>"
	write_page "<tr><td class=\"left\">packages with notes about these issues</td><td>$NOTES</td><td colspan=\"$AC\"></td></tr>"
563

564 565 566 567
	local TD_PKG_SID_NOISSUES="<tr><td class=\"left\">packages in unstable with issues but without identified ones</td><td></td>"
	local TD_PKG_SID_FTBR="<tr><td class=\"left\">&nbsp;&nbsp;- unreproducible ones</a></td><td></td>"
	local TD_PKG_SID_FTBFS="<tr><td class=\"left\">&nbsp;&nbsp;- failing to build</a></td><td></td>"
	local TD_PKG_SID_ISSUES="<tr><td class=\"left\">packages in unstable which need to be fixed</td><td></td>"
568 569 570 571
	local TD_PKG_BUSTER_NOISSUES="<tr><td class=\"left\">packages in buster with issues but without identified ones</td><td></td>"
	local TD_PKG_BUSTER_FTBR="<tr><td class=\"left\">&nbsp;&nbsp;- unreproducible ones</a></td><td></td>"
	local TD_PKG_BUSTER_FTBFS="<tr><td class=\"left\">&nbsp;&nbsp;- failing to build</a></td><td></td>"
	local TD_PKG_BUSTER_ISSUES="<tr><td class=\"left\">packages in buster which need to be fixed</td><td></td>"
572 573 574
	for ARCH in ${ARCHS} ; do
		SUITE="unstable"
		gather_suite_arch_stats
575 576
		TD_PKG_SID_ISSUES="$TD_PKG_SID_ISSUES<td>$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%</td>"

577
		RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('FTBR', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
578
		TD_PKG_SID_NOISSUES="$TD_PKG_SID_NOISSUES<td><a href=\"/debian/$SUITE/$ARCH/index_no_notes.html\">$RESULT</a> / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
579
		RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBR' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
580
		TD_PKG_SID_FTBR="$TD_PKG_SID_FTBR<td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
581
		RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
582 583
		TD_PKG_SID_FTBFS="$TD_PKG_SID_FTBFS<td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"

584
		SUITE="buster"
585
		gather_suite_arch_stats
586
		TD_PKG_BUSTER_ISSUES="$TD_PKG_BUSTER_ISSUES<td>$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%</td>"
587
		RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('FTBR', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
588
		TD_PKG_BUSTER_NOISSUES="$TD_PKG_BUSTER_NOISSUES<td><a href=\"/debian/$SUITE/$ARCH/index_no_notes.html\">$RESULT</a> / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
589
		RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBR' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
590
		TD_PKG_BUSTER_FTBR="$TD_PKG_BUSTER_FTBR<td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
591
		RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
592
		TD_PKG_BUSTER_FTBFS="$TD_PKG_BUSTER_FTBFS<td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
593
	done
594 595 596 597
	write_page "$TD_PKG_SID_NOISSUES</tr>"
	write_page "$TD_PKG_SID_FTBR</tr>"
	write_page "$TD_PKG_SID_FTBFS</tr>"
	write_page "$TD_PKG_SID_ISSUES</tr>"
598 599 600 601
	write_page "$TD_PKG_BUSTER_NOISSUES</tr>"
	write_page "$TD_PKG_BUSTER_FTBR</tr>"
	write_page "$TD_PKG_BUSTER_FTBFS</tr>"
	write_page "$TD_PKG_BUSTER_ISSUES</tr>"
602 603
	ARCH="amd64"
	SUITE="unstable"
604

605 606 607
	# in the following two write_page() calls we use the same
	# insane grep to filter people who committed with several
	# usernames…
608
	if [ -f ${NOTES_GIT_PATH}/packages.yml ] && [ -f ${NOTES_GIT_PATH}/issues.yml ] ; then
609
		write_page "<tr><td class=\"left\">committers to <a href=\"https://salsa.debian.org/reproducible-builds/reproducible-notes\" target=\"_parent\">notes.git</a> (in the last three months)</td><td>$(cd ${NOTES_GIT_PATH} ; git log --since="3 months ago"|grep Author|sort -u | \
610 611
				grep -v alexis@passoire.fr | grep -v christoph.berg@credativ.de | grep -v d.s@daniel.shahaf.name | grep -v dhole@openmailbox.com | grep -v jelmer@jelmer.uk | grep -v mattia@mapreri.org | grep -v micha@lenk.info | grep -v mail@sandroknauss.de | grep -v sanvila@unex.es | \
				wc -l)</td><td colspan=\"$AC\"></td></tr>"
612
		write_page "<tr><td class=\"left\">committers to notes.git (in total)</td><td>$(cd ${NOTES_GIT_PATH} ; git log |grep Author|sort -u | \
613 614
				grep -v alexis@passoire.fr | grep -v christoph.berg@credativ.de | grep -v d.s@daniel.shahaf.name | grep -v dhole@openmailbox.com | grep -v jelmer@jelmer.uk | grep -v mattia@mapreri.org | grep -v micha@lenk.info | grep -v mail@sandroknauss.de | grep -v sanvila@unex.es | \
				wc -l)</td><td colspan=\"$AC\"></td></tr>"
615
	fi
616
	RESULT=$(cat /srv/reproducible-results/modified_in_sid.txt || echo "unknown")	# written by reproducible_html_repository_comparison.sh
617
	write_page "<tr><td class=\"left\">packages <a href=\"/debian/index_repositories.html\">modified in our toolchain</a> (in unstable)</td><td>$(echo $RESULT)</td><td colspan=\"$AC\"></td></tr>"
618 619 620 621
	if ! diff /srv/reproducible-results/modified_in_sid.txt /srv/reproducible-results/modified_in_exp.txt ; then
		RESULT=$(cat /srv/reproducible-results/modified_in_exp.txt || echo "unknown")	# written by reproducible_html_repository_comparison.sh
		write_page "<tr><td class=\"left\">&nbsp;&nbsp;- (in experimental)</td><td>$(echo $RESULT)</td><td colspan=\"$AC\"></td></tr>"
	fi
622 623
	RESULT=$(cat /srv/reproducible-results/binnmus_needed.txt || echo "unknown")	# written by reproducible_html_repository_comparison.sh
	if [ "$RESULT" != "0" ] ; then
624
		write_page "<tr><td class=\"left\">&nbsp;&nbsp;- which need to be build on some archs</td><td>$(echo $RESULT)</td><td colspan=\"$AC\"></td></tr>"
625
	fi
626
	write_page "</table>"
627 628 629 630
	write_page "<p style=\"clear:both;\">"
	# show issue graphs
	for i in 4 5 ; do
		write_global_graph
631
	done
632
	write_page "</p>"
633 634
	# the end
	write_page_footer
635
	cp $PAGE $DEBIAN_BASE/reproducible.html
636
	publish_page debian
637 638
}

639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660
#
# create bugs page
#
create_bugs_page() {
	VIEW=bugs
	PAGE=index_${VIEW}.html
	ARCH="amd64"
	SUITE="unstable"
	echo "$(date -u) - starting to write $PAGE page."
	write_page_header $VIEW "Bugs filed"
	# write bugs with usertags table
	write_usertag_table
	write_page "<p style=\"clear:both;\">"
	# show bug graphs
	for i in 8 9 3 7 ; do
		write_global_graph
	done
	write_page "</p>"
	write_page_footer
	publish_page debian
}

661 662 663 664 665 666 667 668 669

comma_comma_and(){
	case $1 in
		amd64|i386)	COMMA_VAR=", " ;;
		arm64)		COMMA_VAR=" and " ;;
		*)		COMMA_VAR="" ;;
	esac
}

670 671 672 673 674 675
#
# create performance page
#
create_performance_page() {
	VIEW=performance
	PAGE=index_${VIEW}.html
676 677
	ARCH="amd64"
	SUITE="unstable"
678
	echo "$(date -u) - starting to write $PAGE page."
679
	write_page_header $VIEW "Build network performance stats"
680
	# arch performance stats
681
	write_page "<p style=\"clear:both;\">"
682
	for ARCH in ${ARCHS} ; do
683
		MAINLABEL[1]="Amount of packages built each day on '$ARCH'"
684
		write_page " <a href=\"/debian/${TABLE[1]}_$ARCH.png\"><img src=\"/debian/${TABLE[1]}_$ARCH.png\" class=\"overview\" alt=\"${MAINLABEL[1]}\"></a>"
685
		if [ ! -f $DEBIAN_BASE/${TABLE[1]}_$ARCH.png ] || [ $DUMMY_FILE -nt $DEBIAN_BASE/${TABLE[1]}_$ARCH.png ] ; then
Holger Levsen's avatar
Holger Levsen committed
686
				create_debian_png_from_table 1 ${TABLE[1]}_$ARCH.png
687 688
		fi
	done
689
	write_page "<p style=\"clear:both;\">"
690
	write_build_performance_stats
691
	# write suite builds age graphs
692
	write_page "</p><p style=\"clear:both;\">"
693 694
	for ARCH in ${ARCHS} ; do
		for SUITE in $SUITES ; do
695 696 697 698 699
			if [ $SUITE = "stretch" ] ; then
				continue
			else
				write_page " <a href=\"/debian/$SUITE/$ARCH/${TABLE[2]}.png\"><img src=\"/debian/$SUITE/$ARCH/${TABLE[2]}.png\" class=\"tripleview\" alt=\"age of oldest reproducible build result in $SUITE/$ARCH\"></a>"
			fi
700 701 702
		done
		write_page "</p><p style=\"clear:both;\">"
	done
703
	# the end
704 705 706 707 708 709 710 711
	write_page "Daily <a href=\"https://jenkins.debian.net/view/reproducible/job/reproducible_html_nodes_info/lastBuild/console\">individual build node performance stats</a> are available as well as oldest results for "
	for ARCH in ${ARCHS} ; do
		comma_comma_and ${ARCH}
		write_page "<a href=\"/debian/index_${ARCH}_oldies.html\">$ARCH</a>$COMMA_VAR"
	done
	write_page ".</p>"
	write_page "<p>And finally there are also graphs about the oldest build in stretch for "
	SUITE="stretch"
712
	for ARCH in ${ARCHS} ; do
713 714
		comma_comma_and ${ARCH}
		write_page "<a href=\"/debian/$SUITE/$ARCH/${TABLE[2]}.png\">$ARCH</a>$COMMA_VAR"
715 716
	done
	write_page ".</p>"
717
	write_page_footer
718
	publish_page debian
719 720
}

721 722 723 724 725 726
#
# create variations page
#
create_variations_page() {
	VIEW=variations
	PAGE=index_${VIEW}.html
727 728
	ARCH="amd64"
	SUITE="unstable"
729 730 731 732 733 734 735
	echo "$(date -u) - starting to write $PAGE page."
	write_page_header $VIEW "Variations introduced when testing Debian packages"
	# explain setup
	write_variation_table debian
	write_page "<p style=\"clear:both;\">"
	write_page "</p>"
	write_page_footer
736
	publish_page debian
737 738
}

739 740 741
#
# main
#
742
SUITE="unstable"
743 744
update_bug_stats
update_notes_stats
745 746
for ARCH in ${ARCHS} ; do
	for SUITE in $SUITES ; do
747
		update_suite_arch_stats
748
		gather_suite_arch_stats
749
		create_suite_arch_stats_page
750
	done
751
done
752 753
create_performance_page
create_variations_page
754
create_bugs_page
755
create_dashboard_page
756
rm -f $DUMMY_FILE >/dev/null