reproducible_common.sh 56.3 KB
Newer Older
Holger Levsen's avatar
Holger Levsen committed
1
#!/bin/bash
2
# vim: set noexpandtab:
Holger Levsen's avatar
Holger Levsen committed
3

4
# Copyright 2014-2019 Holger Levsen <holger@layer-acht.org>
5
#         © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
Holger Levsen's avatar
Holger Levsen committed
6 7
# released under the GPLv=2
#
8 9 10
# included by all reproducible_*.sh scripts, so be quiet
set +x

11 12
# postgres database definitions
export PGDATABASE=reproducibledb
Holger Levsen's avatar
Holger Levsen committed
13

14 15
# query reproducible database
query_db() {
16
	psql -t --no-align -c "$@" || exit 1
17 18
}

19 20
# query reproducible database, output to csv format
query_to_csv() {
21
	psql -c "COPY ($@) to STDOUT with csv DELIMITER ','" || exit 1
22 23
}

24
# common variables
25
BASE="/var/lib/jenkins/userContent/reproducible"
26
REPRODUCIBLE_URL=https://tests.reproducible-builds.org
27
REPRODUCIBLE_DOT_ORG_URL=https://reproducible-builds.org
28 29
# shop trailing slash
JENKINS_URL=${JENKINS_URL:0:-1}
Holger Levsen's avatar
Holger Levsen committed
30
DBDSUITE="unstable"
31
BIN_PATH=/srv/jenkins/bin
32
TEMPLATE_PATH=/srv/jenkins/mustache-templates/reproducible
33 34
CHPATH=/srv/reproducible-results/chdist
mkdir -p "$CHPATH"
35

36
# Debian suites being tested
37
SUITES="stretch buster unstable experimental"
38
# Debian architectures being tested
39
ARCHS="amd64 i386 arm64 armhf"
Holger Levsen's avatar
Holger Levsen committed
40

41
# define Debian build nodes in use
42
. /srv/jenkins/bin/jenkins_node_definitions.sh
43
MAINNODE="jenkins" # used by reproducible_maintenance.sh only
44
JENKINS_OFFLINE_LIST="/var/lib/jenkins/offline_nodes"
45 46

# variables on the nodes we are interested in
47
BUILD_ENV_VARS="ARCH NUM_CPU CPU_MODEL DATETIME KERNEL" # these also needs to be defined in bin/reproducible_info.sh
48

49 50 51 52 53 54
# common settings for Debian
DEBIAN_URL=https://tests.reproducible-builds.org/debian
DEBIAN_DASHBOARD_URI=/debian/reproducible.html
DEBIAN_BASE="/var/lib/jenkins/userContent/reproducible/debian"
mkdir -p "$DEBIAN_BASE"

55
# existing usertags in the Debian BTS
56
USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness buildinfo cpu signatures environment umask ftbfs locale"
57

58 59 60 61 62
# common settings for testing alpine
ALPINE_REPOS="main community"
ALPINE_PKGS=/srv/reproducible-results/alpine_pkgs
ALPINE_BASE="$BASE/alpine"

63
# common settings for testing Arch Linux
64
ARCHLINUX_REPOS="core extra multilib community"
65
ARCHLINUX_PKGS=/srv/reproducible-results/archlinux_pkgs
66
ARCHBASE=$BASE/archlinux
67

68
# common settings for testing rpm based distros
69
RPM_BUILD_NODE=osuosl-build171-amd64
70
RPM_PKGS=/srv/reproducible-results/rpm_pkgs
71

Holger Levsen's avatar
Holger Levsen committed
72
# number of cores to be used
73
NUM_CPU=$(nproc)
Holger Levsen's avatar
Holger Levsen committed
74

75 76 77
# diffoscope memory limit in kilobytes
DIFFOSCOPE_VIRT_LIMIT=$((10*1024*1024))

78
# we only this array for html creation but we cannot declare them in a function
79
declare -A SPOKENTARGET
80

81 82
# to hold reproducible temporary files/directories without polluting /tmp
TEMPDIR="/tmp/reproducible"
83
mkdir -p "$TEMPDIR"
84

85 86
# create subdirs for suites
for i in $SUITES ; do
87
	mkdir -p "$DEBIAN_BASE/$i"
88 89
done

90
# table names and image names
Holger Levsen's avatar
Holger Levsen committed
91 92 93 94 95 96 97 98
TABLE[0]=stats_pkg_state
TABLE[1]=stats_builds_per_day
TABLE[2]=stats_builds_age
TABLE[3]=stats_bugs
TABLE[4]=stats_notes
TABLE[5]=stats_issues
TABLE[6]=stats_meta_pkg_state
TABLE[7]=stats_bugs_state
99 100
TABLE[8]=stats_bugs_sin_ftbfs
TABLE[9]=stats_bugs_sin_ftbfs_state
Holger Levsen's avatar
Holger Levsen committed
101

102
# package sets defined in meta_pkgsets.csv
103
# csv file columns: (pkgset_group, pkgset_name)
104
colindex=0
105
while IFS=, read col1 col2
106
do
107
	let colindex+=1
108
	META_PKGSET[$colindex]=$col2
109
done < $BIN_PATH/reproducible_pkgsets.csv
110

111 112
# mustache templates
PAGE_FOOTER_TEMPLATE=$TEMPLATE_PATH/default_page_footer.mustache
113 114 115
PROJECT_LINKS_TEMPLATE=$TEMPLATE_PATH/project_links.mustache
MAIN_NAVIGATION_TEMPLATE=$TEMPLATE_PATH/main_navigation.mustache

116 117 118 119
# be loud again if DEBUG
if $DEBUG ; then
	set -x
fi
120

121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
# some cmomon logging functions
log_info () {
	_log "I:" "$@"
}

log_error () {
	_log "E:" "$@"
}

log_warning () {
	_log "W:" "$@"
}

log_file () {
	cat $@ | tee -a $RBUILDLOG
}

_log () {
	local prefix="$1"
	shift 1
	echo -e "$(date -u)  $prefix $*" | tee -a $RBUILDLOG
}

144 145 146 147 148
# sleep 1-23 secs to randomize start times
delay_start() {
	/bin/sleep $(echo "scale=1 ; $(shuf -i 1-230 -n 1)/10" | bc )
}

149
schedule_packages() {
150 151 152 153 154
	LC_USER="$REQUESTER" \
	LOCAL_CALL="true" \
	/srv/jenkins/bin/reproducible_remote_scheduler.py \
		--message "$REASON" \
		--no-notify \
155
		--suite "$SUITE" \
156
		--architecture "$ARCH" \
157
		$@
158 159
}

160 161 162 163 164 165 166
set_icon() {
	# icons taken from tango-icon-theme (0.8.90-5)
	# licenced under http://creativecommons.org/licenses/publicdomain/
	STATE_TARGET_NAME="$1"
	case "$1" in
		reproducible)		ICON=weather-clear.png
					;;
167
		FTBR)		ICON=weather-showers-scattered.png
168
					STATE_TARGET_NAME="FTBR"
169 170 171
					;;
		FTBFS)			ICON=weather-storm.png
					;;
172
		timeout)	ICON=Current_event_clock.png ;;
173 174
		depwait)		ICON=weather-snow.png
					;;
175
		E404)			ICON=weather-severe-alert.png
176
					;;
177
		NFU)		ICON=weather-few-clouds-night.png
178
					STATE_TARGET_NAME="NFU"
179 180 181 182 183 184 185 186 187
					;;
		blacklisted)		ICON=error.png
					;;
		*)			ICON=""
	esac
}

write_icon() {
	# ICON and STATE_TARGET_NAME are set by set_icon()
188
	write_page "<a href=\"/debian/$SUITE/$ARCH/index_${STATE_TARGET_NAME}.html\" target=\"_parent\"><img src=\"/static/$ICON\" alt=\"${STATE_TARGET_NAME} icon\" /></a>"
189 190 191
}

write_page_header() {
192
	# this is really quite uncomprehensible and should be killed
193
	# the solution is to write all HTML pages with python…
194
	rm -f $PAGE
195
	MAINVIEW="dashboard"
196 197
	write_page "<!DOCTYPE html><html><head>"
	write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />"
198
	write_page "<meta name=\"viewport\" content=\"width=device-width\" />"
199
	write_page "<link href=\"/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />"
200
	write_page "<title>$2</title></head>"
201
	if [ "$1" != "$MAINVIEW" ] ; then
202
		write_page "<body class=\"wrapper\">"
203
	else
204
		write_page "<body class=\"wrapper\" onload=\"selectSearch()\">"
205
	fi
206 207 208 209 210 211 212 213 214 215 216 217

	# Build context for the main_navigation mustache template.

	# Do not show package set links for "experimental" pages
	if [ "$SUITE" != "experimental" ] ; then
		# no pkg_sets are tested in experimental
		include_pkgset_link="\"include_pkgset_link\" : \"true\""
	else
		include_pkgset_link=''
	fi

	# Used to highlight the link for the current page
218 219 220 221 222 223 224
	if [ "$1" = "dashboard" ] \
		|| [ "$1" = "performance" ] \
		|| [ "$1" = "repositories" ] \
		|| [ "$1" = "variations" ] \
		|| [ "$1" = "suite_arch_stats" ] \
		|| [ "$1" = "bugs" ] \
		|| [ "$1" = "nodes_health" ] \
225
		|| [ "$1" = "job_health" ] \
226 227
		|| [ "$1" = "nodes_weekly_graphs" ] \
		|| [ "$1" = "nodes_daily_graphs" ] ; then
228 229 230 231 232 233
		displayed_page="\"$1\": \"true\""
	else
		displayed_page=''
	fi

	# Create json for suite links (a list of objects)
234
	suite_links="\"suite_nav\": { \"suite_list\": ["
235
	comma=0
236 237
	for s in $SUITES ; do
		if [ "$s" = "$SUITE" ] ; then
238
			class="current"
239 240
		else
			class=''
241
		fi
242
		uri="/debian/${s}/index_suite_${ARCH}_stats.html"
Holger Levsen's avatar
Holger Levsen committed
243
		if [ $comma = 1 ] ; then
244
			suite_links+=", {\"s\": \"${s}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
245
		else
246
			suite_links+="{\"s\": \"${s}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
247
			comma=1
248
		fi
249
	done
250
	suite_links+="]}"
251 252

	# Create json for arch links (a list of objects)
253
	arch_links="\"arch_nav\": {\"arch_list\": ["
254
	comma=0
255 256
	for a in ${ARCHS} ; do
		if [ "$a" = "$ARCH" ] ; then
257
			class="current"
258
		else
259
			class=''
260
		fi
261
		uri="/debian/$SUITE/index_suite_${a}_stats.html"
Holger Levsen's avatar
Holger Levsen committed
262
		if [ $comma = 1 ] ; then
263
			arch_links+=", {\"a\": \"${a}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
264
		else
265
			arch_links+="{\"a\": \"${a}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
266
			comma=1
267
		fi
268
	done
269
	arch_links+="]}"
270 271 272 273 274 275

	# finally, the completely formed JSON context
	context=$(printf '{
		"arch" : "%s",
		"suite" : "%s",
		"page_title" : "%s",
276
		"debian_uri" : "%s",
277 278
		%s,
		%s
279
	' "$ARCH" "$SUITE" "$2" "$DEBIAN_DASHBOARD_URI" "$arch_links" "$suite_links")
280 281 282 283 284 285 286 287 288 289 290
	if [[ ! -z $displayed_page ]] ; then
		context+=", $displayed_page"
	fi
	if [[ ! -z $include_pkgset_link ]] ; then
		context+=", $include_pkgset_link"
	fi
	context+="}"

	write_page "<header class=\"head\">"
	write_page "$(pystache3 $MAIN_NAVIGATION_TEMPLATE "$context")"
	write_page "$(pystache3 $PROJECT_LINKS_TEMPLATE "{}")"
291
	write_page "</header>"
292

293
	write_page "<div class=\"mainbody\">"
294
	write_page "<h2>$2</h2>"
295
	if [ "$1" = "$MAINVIEW" ] ; then
296
		write_page "<ul>"
297
		write_page "   Please also visit the more general website <li><a href=\"https://reproducible-builds.org\">Reproducible-builds.org</a></li> where <em>reproducible builds</em> are explained in more detail than just <em>bit by bit identical rebuilds to enable verifcation of the sources used to build</em>."
298 299 300 301
		write_page "   We think that reproducible builds should become the norm, so we wrote <li><a href=\"https://reproducible-builds.org/howto\">How to make your software reproducible</a></li>."
		write_page "   Also aimed at the free software world at large, is the first specification we have written: the <li><a href=\"https://reproducible-builds.org/specs/source-date-epoch/\">SOURCE_DATE_EPOCH specification</a></li>."
		write_page "</ul>"
		write_page "<ul>"
Chris Lamb's avatar
Chris Lamb committed
302
		write_page "   These pages are showing the <em>potential</em> of <li><a href=\"https://wiki.debian.org/ReproducibleBuilds\" target=\"_blank\">reproducible builds of Debian packages</a></li>."
303 304 305 306
		write_page "   The results shown were obtained by <a href=\"$JENKINS_URL/view/reproducible\">several jobs</a> running on"
		write_page "   <a href=\"$JENKINS_URL/userContent/about.html#_reproducible_builds_jobs\">jenkins.debian.net</a>."
		write_page "   Thanks to <a href=\"https://www.profitbricks.co.uk\">Profitbricks</a> for donating the virtual machines this is running on!"
		write_page "</ul>"
307
		LATEST=$(query_db "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id = s.id WHERE r.status = 'FTBR' AND s.suite = 'unstable' AND s.architecture = 'amd64' AND s.id NOT IN (SELECT package_id FROM notes) ORDER BY build_date DESC LIMIT 23"|sort -R|head -1)
308
		write_page "<form action=\"$REPRODUCIBLE_URL/redirect\" method=\"GET\">$REPRODUCIBLE_URL/"
309
		write_page "<input type=\"text\" name=\"SrcPkg\" placeholder=\"Type my friend..\" value=\"$LATEST\" />"
310
		write_page "<input type=\"submit\" value=\"submit source package name\" />"
311
		write_page "</form>"
312 313 314 315 316
		write_page "<ul>"
		write_page "   We are reachable via IRC (<code>#debian-reproducible</code> and <code>#reproducible-builds</code> on OFTC),"
		write_page "   or <a href="mailto:reproducible-builds@lists.alioth.debian.org">email</a>,"
		write_page "   and we care about free software in general,"
		write_page "   so whether you are an upstream developer or working on another distribution, or have any other feedback - we'd love to hear from you!"
317 318 319 320 321
		write_page "   Besides Debian we are also testing "
		write_page "   <li><a href=\"/coreboot/\">coreboot</a></li>,"
		write_page "   <li><a href=\"/openwrt/\">OpenWrt</a></li>, "
		write_page "   <li><a href=\"/netbsd/\">NetBSD</a></li>, "
		write_page "   <li><a href=\"/freebsd/\">FreeBSD</a></li>, "
322
		write_page "   and <li><a href=\"/archlinux/\">Arch Linux</a></li> "
323
		write_page "   though not as thoroughly as Debian yet. "
324 325 326
		write_page "   <li><a href=\"http://rb.zq1.de/\">openSUSE</a></li>, "
		write_page "   <li><a href=\"https://r13y.com/\">NixOS</a></li> and "
		write_page "   <li><a href=\"https://verification.f-droid.org/\">F-Droid</a></li> are also being tested, though elsewhere."
327
		write_page "   As far as we know, the <a href=\"https://www.gnu.org/software/guix/manual/en/html_node/Invoking-guix-challenge.html\">Guix challenge</a> is not yet run systematically anywhere."
328
		write_page "   Testing of "
329
		write_page "   <a href=\"/rpms/fedora-23.html\">Fedora</a> "
330 331
		write_page "   has sadly been suspended for now. "
		write_page " We can test more projects, if <em>you</em> contribute!"
332
		write_page "</ul>"
333
	fi
334 335
}

336
write_page_intro() {
337
	write_page "       <p><em>Reproducible builds</em> enable anyone to reproduce bit by bit identical binary packages from a given source, so that anyone can verify that a given binary derived from the source it was said to be derived."
338 339
	write_page "         There is more information about <a href=\"https://wiki.debian.org/ReproducibleBuilds\">reproducible builds on the Debian wiki</a> and on <a href=\"https://reproducible-builds.org\">https://reproducible-builds.org</a>."
	write_page "         These pages explain in more depth why this is useful, what common issues exist and which workarounds and solutions are known."
340
	write_page "        </p>"
341
	local BUILD_ENVIRONMENT=" in a Debian environment"
342
	local BRANCH="master"
Holger Levsen's avatar
Holger Levsen committed
343
	if [ "$1" = "coreboot" ] ; then
344
		write_page "        <p><em>Reproducible Coreboot</em> is an effort to apply this to coreboot. Thus each coreboot.rom is build twice (without payloads), with a few variations added and then those two ROMs are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
345 346
		local PROJECTNAME="$1"
		local PROJECTURL="https://review.coreboot.org/p/coreboot.git"
347
	elif [ "$1" = "OpenWrt" ] ; then
348
		local PROJECTNAME="$1"
349
		local PROJECTURL="https://github.com/openwrt/openwrt.git"
350
		write_page "        <p><em>Reproducible $PROJECTNAME</em> is an effort to apply this to $PROJECTNAME. Thus each $PROJECTNAME target is build twice, with a few variations added and then the resulting images and packages from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. $PROJECTNAME generates many different types of raw <code>.bin</code> files, and diffoscope does not know how to parse these. Thus the resulting diffoscope output is not nearly as clear as it could be - hopefully this limitation will be overcome eventually, but in the meanwhile the input components (uImage kernel file, rootfs.tar.gz, and/or rootfs squashfs) can be inspected. Also please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
351
	elif [ "$1" = "NetBSD" ] ; then
352
		write_page "        <p><em>Reproducible NetBSD</em> is an effort to apply this to NetBSD. Thus each NetBSD target is build twice, with a few variations added and then the resulting files from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
353
		local PROJECTNAME="netbsd"
354
		local PROJECTURL="https://github.com/NetBSD/src"
355
	elif [ "$1" = "FreeBSD" ] ; then
356
		write_page "        <p><em>Reproducible FreeBSD</em> is an effort to apply this to FreeBSD. Thus FreeBSD is build twice, with a few variations added and then the resulting filesystems from the two builds are put into a compressed tar archive, which is finally compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
357 358
		local PROJECTNAME="freebsd"
		local PROJECTURL="https://github.com/freebsd/freebsd.git"
359 360
		local BUILD_ENVIRONMENT=", which via ssh triggers a build on a FreeBSD 11.2 system"
		local BRANCH="master"
361 362
	elif [ "$1" = "Arch Linux" ] ; then
		local PROJECTNAME="Arch Linux"
363 364 365
		write_page "        <p><em>Reproducible $PROJECTNAME</em> is an effort to apply this to $PROJECTNAME. Thus $PROJECTNAME packages are build twice, with a few variations added and then the resulting packages from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>."
		write_page "   Please note that this is still at an early stage. Also there are more variations expected to be seen in the wild."
		write_page "Missing bits for <em>testing</em> Arch Linux:<ul>"
366
		write_page " <li>cross references to <a href=\"https://tests.reproducible-builds.org/debian/index_issues.html\">Debian notes</a> - and having Arch Linux specific notes.</li>"
367
		write_page "</ul></p>"
368
		write_page "<p>Missing bits for Arch Linux:<ul>"
369
		write_page " <li>code needs to be written to compare the packages built twice here against newly built packages from the Official Arch Linux repositories.</li>"
370 371
		write_page " <li>user tools, for users to verify all of this easily.</li>"
		write_page "</ul></p>"
372
		write_page "<p>If you want to help out or discuss reproducible builds in $PROJECTNAME, please join #archlinux-reproducible on freenode.</p>"
373 374 375 376 377 378
	elif [ "$1" = "fedora-23" ] ; then
		local PROJECTNAME="Fedora 23"
		write_page "        <p><em>Reproducible $PROJECTNAME</em> is a (currently somewhat stalled) effort to apply this to $PROJECTNAME, which is rather obvious with 23… <br/> $PROJECTNAME packages are build twice, with a few variations added and then the resulting packages from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
	fi
	if [ "$1" != "Arch Linux" ] && [ "$1" != "fedora-23" ] ; then
		local SMALLPROJECTNAME="$(echo $PROJECTNAME|tr '[:upper:]' '[:lower:]')"
379
		write_page "       <p>There is a weekly run <a href=\"https://jenkins.debian.net/view/reproducible/job/reproducible_$SMALLPROJECTNAME/\">jenkins job</a> to test the <code>$BRANCH</code> branch of <a href=\"$PROJECTURL\">$PROJECTNAME.git</a>. The jenkins job is running <a href=\"https://salsa.debian.org/qa/jenkins.debian.net/tree/master/bin/reproducible_$SMALLPROJECTNAME.sh\">reproducible_$SMALLPROJECTNAME.sh</a>$BUILD_ENVIRONMENT and this script is solely responsible for creating this page. Feel invited to join <code>#reproducible-builds</code> (on irc.oftc.net) to request job runs whenever sensible. Patches and other <a href=\"mailto:reproducible-builds@lists.alioth.debian.org\">feedback</a> are very much appreciated - if you want to help, please start by looking at the <a href=\"https://jenkins.debian.net/userContent/todo.html#_reproducible_$(echo $1|tr '[:upper:]' '[:lower:]')\">ToDo list for $1</a>, you might find something easy to contribute."
380 381
		write_page "       <br />Thanks to <a href=\"https://www.profitbricks.co.uk\">Profitbricks</a> for donating the virtual machines this is running on!</p>"
	elif [ "$1" = "fedora-23" ] ; then
382
		write_page "       <p><img src=\"/userContent/static/weather-storm.png\"> TODO: explain $PROJECTNAME test setup here.</p>"
383
	fi
384 385
}

386
write_page_footer() {
387
	if [ "$1" = "coreboot" ] ; then
388
		other_distro_details='The <a href=\"http://www.coreboot.org\">Coreboot</a> logo is Copyright © 2008 by Konsult Stuge and coresystems GmbH and can freely be used to refer to the Coreboot project.'
389
	elif [ "$1" = "NetBSD" ] ; then
390
		other_distro_details="NetBSD® is a registered trademark of The NetBSD Foundation, Inc."
391
	elif [ "$1" = "FreeBSD" ] ; then
392
		other_distro_details="FreeBSD is a registered trademark of The FreeBSD Foundation. The FreeBSD logo and The Power to Serve are trademarks of The FreeBSD Foundation."
393
	elif [ "$1" = "Arch Linux" ] ; then
394
		other_distro_details='The <a href=\"https://www.archlinux.org\">Arch Linux</a> name and logo are recognized trademarks. Some rights reserved. The registered trademark Linux® is used pursuant to a sublicense from LMI, the exclusive licensee of Linus Torvalds, owner of the mark on a world-wide basis.'
395
	elif [ "$1" = "fedora-23" ] ; then
396
			other_distro_details="Fedora is sponsored by Red Hat. © 2017 Red Hat, Inc. and others."
397 398
	else
		other_distro_details=''
399
	fi
400 401 402 403 404 405 406 407 408 409 410
	now=$(date +'%Y-%m-%d %H:%M %Z')

	# The context for pystache3 CLI must be json
	context=$(printf '{
		"job_url" : "%s",
		"job_name" : "%s",
		"date" : "%s",
		"other_distro_details" : "%s"
	}' "${JOB_URL:-""}" "${JOB_NAME:-""}" "$now" "$other_distro_details")

	write_page "$(pystache3 $PAGE_FOOTER_TEMPLATE "$context")"
411
	write_page "</div>"
412
	write_page "</body></html>"
413
 }
414

415
write_variation_table() {
416
	write_page "<p style=\"clear:both;\">"
417 418 419 420
	if [ "$1" = "fedora-23" ] ; then
		write_page "There are no variations introduced in the $1 builds yet. Stay tuned.</p>"
		return
	fi
421
	write_page "<table class=\"main\" id=\"variation\"><tr><th>variation</th><th width=\"40%\">first build</th><th width=\"40%\">second build</th></tr>"
422
	if [ "$1" = "debian" ] ; then
423 424
		write_page "<tr><td>hostname</td><td>one of:"
		for a in ${ARCHS} ; do
425
			local COMMA=""
426 427
			local ARCH_NODES=""
			write_page "<br />&nbsp;&nbsp;"
428 429
			for i in $(echo $BUILD_NODES | sed -s 's# #\n#g' | sort -u) ; do
				if [ "$(echo $i | grep $a)" ] ; then
430 431 432
					echo -n "$COMMA ${ARCH_NODES}$(echo $i | cut -d '.' -f1 | sed -s 's# ##g')" >> $PAGE
					if [ -z $COMMA ] ; then
						COMMA=","
433
					fi
434 435 436 437
				fi
			done
		done
		write_page "</td><td>i-capture-the-hostname</td></tr>"
438
		write_page "<tr><td>domainname</td><td>$(hostname -d)</td><td>i-capture-the-domainname</td></tr>"
439
	else
440
		if [ "$1" != "Arch Linux" ] || [ "$1" != "OpenWrt" ] ; then
441
			write_page "<tr><td>hostname</td><td> osuosl-build169-amd64 or osuosl-build170-amd64</td><td>the other one</td></tr>"
442 443 444
		else
			write_page "<tr><td>hostname</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		fi
445
		write_page "<tr><td>domainname</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
446
	fi
447
	if [ "$1" != "FreeBSD" ] && [ "$1" != "Arch Linux" ] && [ "$1" != "fedora-23" ] ; then
448 449
		write_page "<tr><td>env CAPTURE_ENVIRONMENT</td><td><em>not set</em></td><td>CAPTURE_ENVIRONMENT=\"I capture the environment\"</td></tr>"
	fi
450
	write_page "<tr><td>env TZ</td><td>TZ=\"/usr/share/zoneinfo/Etc/GMT+12\"</td><td>TZ=\"/usr/share/zoneinfo/Etc/GMT-14\"</td></tr>"
451
	if [ "$1" = "debian" ]  ; then
452 453 454
		write_page "<tr><td>env LANG</td><td>LANG=\"C\"</td><td>on amd64: LANG=\"fr_CH.UTF-8\"<br />on i386: LANG=\"de_CH.UTF-8\"<br />on arm64: LANG=\"nl_BE.UTF-8\"<br />on armhf: LANG=\"it_CH.UTF-8\"</td></tr>"
		write_page "<tr><td>env LANGUAGE</td><td>LANGUAGE=\"en_US:en\"</td><td>on amd64: LANGUAGE=\"fr_CH:fr\"<br />on i386: LANGUAGE=\"de_CH:de\"<br />on arm64: LANGUAGE=\"nl_BE:nl\"<br />on armhf: LANGUAGE=\"it_CH:it\"</td></tr>"
		write_page "<tr><td>env LC_ALL</td><td><em>not set</em></td><td>on amd64: LC_ALL=\"fr_CH.UTF-8\"<br />on i386: LC_ALL=\"de_CH.UTF-8\"<br />on arm64: LC_ALL=\"nl_BE.UTF-8\"<br />on armhf: LC_ALL=\"it_CH.UTF-8\"</td></tr>"
455
	elif [ "$1" = "Arch Linux" ]  ; then
456 457
		write_page "<tr><td>env LANG</td><td><em>LANG=\"en_US.UTF-8\"</em></td><td>LANG=\"fr_CH.UTF-8\"</td></tr>"
		write_page "<tr><td>env LC_ALL</td><td><em>LANG=\"en_US.UTF-8\"</em></td><td>LC_ALL=\"fr_CH.UTF-8\"</td></tr>"
458
		write_page "<tr><td>the build path</td><td colspan=\"2\">is not yet varied between rebuilds of Arch Linux</td></tr>"
459
	else
460
		write_page "<tr><td>env LANG</td><td>LANG=\"en_GB.UTF-8\"</td><td>LANG=\"fr_CH.UTF-8\"</td></tr>"
461
		write_page "<tr><td>env LC_ALL</td><td><em>not set</em></td><td>LC_ALL=\"fr_CH.UTF-8\"</td></tr>"
462
	fi
463
	if [ "$1" != "FreeBSD" ] && [ "$1" != "Arch Linux" ]  ; then
464
		write_page "<tr><td>env PATH</td><td>PATH=\"/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:\"</td><td>PATH=\"/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path\"</td></tr>"
465 466
	elif [ "$1" = "Arch Linux" ]  ; then
		write_page "<tr><td>env PATH</td><td colspan=\"2\">is set to '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' because that's what <i>makechrootpkg</i> is using</td>"
467
	else
468
		write_page "<tr><td>env PATH</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
469
	fi
470 471
	if [ "$1" = "debian" ] ; then
		write_page "<tr><td>env BUILDUSERID</td><td>BUILDUSERID=\"1111\"</td><td>BUILDUSERID=\"2222\"</td></tr>"
472 473
		write_page "<tr><td>env BUILDUSERNAME</td><td>BUILDUSERNAME=\"pbuilder1\"</td><td>BUILDUSERNAME=\"pbuilder2\"</td></tr>"
		write_page "<tr><td>env USER</td><td>USER=\"pbuilder1\"</td><td>USER=\"pbuilder2\"</td></tr>"
474
		write_page "<tr><td>env HOME</td><td>HOME=\"/nonexistent/first-build\"</td><td>HOME=\"/nonexistent/second-build\"</td></tr>"
475
		write_page "<tr><td>niceness</td><td>10</td><td>11</td></tr>"
476 477
		write_page "<tr><td>uid</td><td>uid=1111</td><td>uid=2222</td></tr>"
		write_page "<tr><td>gid</td><td>gid=1111</td><td>gid=2222</td></tr>"
478
		write_page "<tr><td>/bin/sh</td><td>/bin/dash</td><td>/bin/bash</td></tr>"
479
		write_page "<tr><td><em><a href=\"https://wiki.debian.org/UsrMerge\">usrmerge</a></em> package installed</td><td>no</td><td>yes</td></tr>"
480
		write_page "<tr><td>build path</td><td>/build/1st/\$pkg-\$ver <em>(not varied for stretch/buster)</em></td><td>/build/\$pkg-\$ver/2nd <em>(not varied for stretch/buster)</em></td></tr>"
481
		write_page "<tr><td>user's login shell</td><td>/bin/sh</td><td>/bin/bash</td></tr>"
482
		write_page "<tr><td>user's <a href="https://en.wikipedia.org/wiki/Gecos_field">GECOS</a></td><td>first user,first room,first work-phone,first home-phone,first other</td><td>second user,second room,second work-phone,second home-phone,second other</td></tr>"
483
		write_page "<tr><td>env DEB_BUILD_OPTIONS</td><td>DEB_BUILD_OPTIONS=\"parallel=XXX\"<br />&nbsp;&nbsp;XXX on amd64: 16 or 15<br />&nbsp;&nbsp;XXX on i386: 10 or 9<br />&nbsp;&nbsp;XXX on armhf: 8, 4 or 2</td><td>DEB_BUILD_OPTIONS=\"parallel=YYY\"<br />&nbsp;&nbsp;YYY on amd64: 16 or 15 (!= the first build)<br />&nbsp;&nbsp;YYY on i386: 10 or 9 (!= the first build)<br />&nbsp;&nbsp;YYY is the same as XXX on arm64<br />&nbsp;&nbsp;YYY on armhf: 8, 4, or 2 (not varied systematically)</td></tr>"
484
		write_page "<tr><td>UTS namespace</td><td><em>shared with the host</em></td><td><em>modified using</em> /usr/bin/unshare --uts</td></tr>"
485 486 487 488
	elif [ "$1" = "Arch Linux" ]  ; then
		write_page "<tr><td>env USER</td><td>jenkins</td><td>build 2</td></tr>"
		write_page "<tr><td>user/uid</td><td>jenkins/103</td><td>build2/1235</td></tr>"
		write_page "<tr><td>group/gid</td><td>jenkins/105</td><td>build2/1235</td></tr>"
489
	else
490 491 492
		write_page "<tr><td>env USER</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		write_page "<tr><td>uid</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		write_page "<tr><td>gid</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
493 494 495 496 497
		if [ "$1" != "FreeBSD" ] ; then
			write_page "<tr><td>UTS namespace</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		fi
	fi
	if [ "$1" != "FreeBSD" ] ; then
498
		if [ "$1" = "debian" ] ; then
499 500 501 502 503
			write_page "<tr><td>kernel version</td></td><td>"
			for a in ${ARCHS} ; do
				write_page "<br />on $a one of:"
				write_page "$(cat /srv/reproducible-results/node-information/*$a* | grep KERNEL | cut -d '=' -f2- | sort -u | tr '\n' '\0' | xargs -0 -n1 echo '<br />&nbsp;&nbsp;')"
			done
504
			write_page "</td>"
505
			write_page "<td>on amd64 systematically varied, on armhf not systematically, on i386 and arm64 not at all<br />"
506 507 508 509
			for a in ${ARCHS} ; do
				write_page "<br />on $a one of:"
				write_page "$(cat /srv/reproducible-results/node-information/*$a* | grep KERNEL | cut -d '=' -f2- | sort -u | tr '\n' '\0' | xargs -0 -n1 echo '<br />&nbsp;&nbsp;')"
			done
510
			write_page "</td></tr>"
511
		elif [ "$1" != "Arch Linux" ]  ; then
512
			write_page "<tr><td>kernel version, modified using /usr/bin/linux64 --uname-2.6</td><td>$(uname -sr)</td><td>$(/usr/bin/linux64 --uname-2.6 uname -sr)</td></tr>"
513
		else
514 515 516
			write_page "<tr><td>kernel version</td>"
			write_page "$(cat /srv/reproducible-results/node-information/osuosl-build169* | grep KERNEL | cut -d '=' -f2- | sort -u | tr '\n' '\0' | xargs -0 -n1 echo '<br />&nbsp;&nbsp;')"
			write_page "<td colspan=\"2\"> is currently not varied between rebuilds of $1.</td></tr>"
517
		fi
518 519
		if [ "$1" != "OpenWrt" ] ; then
			write_page "<tr><td>umask</td><td>0022<td>0002</td></tr>"
520
		else
521
			write_page "<tr><td>umask</td><td colspan=\"2\">is always set to 0022 by the OpenWrt build system.</td></tr>"
522
		fi
523 524 525
	else
		write_page "<tr><td>FreeBSD kernel version</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		write_page "<tr><td>umask</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td><tr>"
526
	fi
527 528
	local TODAY=$(date +'%Y-%m-%d')
	local FUTURE=$(date --date="${TODAY}+398 days" +'%Y-%m-%d')
529
	if [ "$1" = "debian" ] ; then
530
		write_page "<tr><td>CPU type</td><td>one of: $(cat /srv/reproducible-results/node-information/* | grep CPU_MODEL | cut -d '=' -f2- | sort -u | tr '\n' '\0' | xargs -0 -n1 echo '<br />&nbsp;&nbsp;')</td><td>on i386: systematically varied (AMD or Intel CPU with different names & features)<br />on amd64: same for both builds<br />on arm64: always the same<br />on armhf: sometimes varied (depending on the build job), but only the minor CPU revision</td></tr>"
531
		write_page "<tr><td>year, month, date</td><td>today (${TODAY}) or (on amd64, i386 and arm64 only) also: $FUTURE</td><td>on amd64, i386 and arm64: varied (398 days difference)<br />on armhf: same for both builds (currently, work in progress)</td></tr>"
532 533
	else
		write_page "<tr><td>CPU type</td><td>$(cat /proc/cpuinfo|grep 'model name'|head -1|cut -d ":" -f2-)</td><td>same for both builds</td></tr>"
534 535 536 537 538
		if [ "$1" = "Arch Linux" ]; then
			write_page "<tr><td>/bin/sh</td><td>/bin/dash</td><td>/bin/bash</td></tr>"
		else
			write_page "<tr><td>/bin/sh</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		fi
539
		if [ "$1" != "FreeBSD" ] && [ "$1" != "Arch Linux" ] ; then
540
			write_page "<tr><td>year, month, date</td><td>today (${TODAY})</td><td>same for both builds (currently, work in progress)</td></tr>"
541
		elif [ "$1" = "Arch Linux" ] ; then
542
			write_page "<tr><td>year, month, date</td><td>osuosl-build169-amd64: today (${TODAY}) or osuosl-build170-amd64: 398 days in the future ($FUTURE)</td><td>the other one</td></tr>"
543
		else
544
			write_page "<tr><td>year, month, date</td><td>osuosl-build171-amd64: today (${TODAY}) or osuosl-build172-amd64: 398 days in the future ($FUTURE)</td><td>the other one</td></tr>"
545
		fi
546
	fi
547
	if [ "$1" != "FreeBSD" ] ; then
548
		if [ "$1" = "debian" ] ; then
549
			write_page "<tr><td>hour, minute</td><td>at least the minute will probably vary between two builds anyway...</td><td>on amd64, i386 and arm64 the \"future builds\" additionally run 6h and 23min ahead</td></tr>"
550
		        write_page "<tr><td>filesystem</td><td>tmpfs</td><td><em>temporarily not</em> varied using <a href=\"https://tracker.debian.org/disorderfs\">disorderfs</a> (<a href=\"https://sources.debian.org/src/disorderfs/sid/disorderfs.1.txt/\">manpage</a>)</td></tr>"
551
		else
552
			write_page "<tr><td>hour, minute</td><td>hour and minute will probably vary between two builds...</td><td>the future system actually runs 398 days, 6 hours and 23 minutes ahead...</td></tr>"
553 554
			write_page "<tr><td>Filesystem</td><td>tmpfs</td><td>same for both builds (currently, this could be varied using <a href=\"https://tracker.debian.org/disorderfs\">disorderfs</a>)</td></tr>"
		fi
555
	else
556
		write_page "<tr><td>year, month, date</td><td>today ($TODAY)</td><td>the 2nd build is done with the build node set 1 year, 1 month and 1 day in the future</td></tr>"
557
		write_page "<tr><td>hour, minute</td><td>hour and minute will vary between two builds</td><td>additionally the \"future build\" also runs 6h and 23min ahead</td></tr>"
558
		write_page "<tr><td>filesystem of the build directory</td><td>ufs</td><td>same for both builds</td></tr>"
559
	fi
560
	if [ "$1" = "debian" ] ; then
Chris Lamb's avatar
Chris Lamb committed
561
		write_page "<tr><td><em>everything else...</em></td><td colspan=\"2\">is likely the same. So far, this is just about the <em>potential</em> of <a href=\"https://wiki.debian.org/ReproducibleBuilds\">reproducible builds of Debian</a> - there will be more variations in the wild.</td></tr>"
562 563 564 565 566 567
	else
		write_page "<tr><td><em>everything else...</em></td><td colspan=\"2\">is likely the same. There will be more variations in the wild.</td></tr>"
	fi
	write_page "</table></p>"
}

568
publish_page() {
569 570 571 572
	if [ "$1" = "" ] ; then
		TARGET=$PAGE
	else
		TARGET=$1/$PAGE
573
	fi
574
	echo "$(date -u) - $(cp -v $PAGE $BASE/$TARGET)"
575
	rm $PAGE
576
	echo "$(date -u) - enjoy $REPRODUCIBLE_URL/$TARGET"
577 578
}

579
gen_package_html() {
580
	cd /srv/jenkins/bin
581
	python3 -c "import reproducible_html_packages as rep
582 583
from rblib.models import Package
pkg = Package('$1', no_notes=True)
584
rep.gen_packages_html([pkg], no_clean=True)" || echo "Warning: cannot update HTML pages for $1"
585
	cd - > /dev/null
586 587
}

588 589 590 591 592 593
calculate_build_duration() {
	END=$(date +'%s')
	DURATION=$(( $END - $START ))
}

print_out_duration() {
594 595 596
	if [ -z "$DURATION" ]; then
		return
	fi
597 598 599
	local HOUR=$(echo "$DURATION/3600"|bc)
	local MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc)
	local SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc)
Holger Levsen's avatar
Holger Levsen committed
600
	echo "$(date -u) - total duration: ${HOUR}h ${MIN}m ${SEC}s." | tee -a ${RBUILDLOG}
601 602
}

603
irc_message() {
604 605
	local CHANNEL="$1"
	shift
606
	local MESSAGE="$@"
607
	echo "Sending '$MESSAGE' to $CHANNEL now."
608
	kgb-client --conf /srv/jenkins/kgb/$CHANNEL.conf --relay-msg "$MESSAGE" || echo "$(date -u) - couldn't send message to $CHANNEL, continuing anyway." # don't fail the whole job
609 610
}

611
call_diffoscope() {
612 613 614 615
	mkdir -p $TMPDIR/$1/$(dirname $2)
	local TMPLOG=(mktemp --tmpdir=$TMPDIR)
	local msg=""
	set +e
616
	# remember to also modify the retry diffoscope call 15 lines below
617 618
	( ulimit -v "$DIFFOSCOPE_VIRT_LIMIT"
	  timeout "$TIMEOUT" nice schroot \
619
		--directory $TMPDIR \
620 621
		-c source:jenkins-reproducible-${DBDSUITE}-diffoscope \
		diffoscope -- \
622 623 624 625 626
			--html $TMPDIR/$1/$2.html \
			$TMPDIR/b1/$1/$2 \
			$TMPDIR/b2/$1/$2 2>&1 \
	) 2>&1 >> $TMPLOG
	RESULT=$?
Holger Levsen's avatar
Holger Levsen committed
627
	LOG_RESULT=$(grep '^E: 15binfmt: update-binfmts: unable to open' $TMPLOG || true)
Holger Levsen's avatar
Holger Levsen committed
628
	if [ ! -z "$LOG_RESULT" ] ; then
629
		rm -f $TMPLOG $TMPDIR/$1/$2.html
Holger Levsen's avatar
Holger Levsen committed
630
		echo "$(date -u) - schroot jenkins-reproducible-${DBDSUITE}-diffoscope not available, will sleep 2min and retry."
631 632
		sleep 2m
		# remember to also modify the retry diffoscope call 15 lines above
633 634
		( ulimit -v "$DIFFOSCOPE_VIRT_LIMIT"
		  timeout "$TIMEOUT" nice schroot \
635 636 637 638 639 640 641 642 643
			--directory $TMPDIR \
			-c source:jenkins-reproducible-${DBDSUITE}-diffoscope \
			diffoscope -- \
				--html $TMPDIR/$1/$2.html \
				$TMPDIR/b1/$1/$2 \
				$TMPDIR/b2/$1/$2 2>&1 \
			) 2>&1 >> $TMPLOG
		RESULT=$?
	fi
644 645 646 647 648 649 650 651
	if ! "$DEBUG" ; then set +x ; fi
	set -e
	cat $TMPLOG # print dbd output
	rm -f $TMPLOG
	case $RESULT in
		0)	echo "$(date -u) - $1/$2 is reproducible, yay!"
			;;
		1)
652
			echo "$(date -u) - $DIFFOSCOPE found issues, please investigate $1/$2"
653 654
			;;
		2)
655
			msg="$(date -u) - $DIFFOSCOPE had trouble comparing the two builds. Please investigate $1/$2"
656 657 658
			;;
		124)
			if [ ! -s $TMPDIR/$1.html ] ; then
659
				msg="$(date -u) - $DIFFOSCOPE produced no output for $1/$2 and was killed after running into timeout after ${TIMEOUT}..."
660
			else
661
				msg="$DIFFOSCOPE was killed after running into timeout after $TIMEOUT, but there is still $TMPDIR/$1/$2.html"
662 663 664
			fi
			;;
		*)
665 666 667 668 669
			# Process killed by signal exits with 128+${signal number}.
			# 31 = SIGSYS = maximum signal number in signal(7)
			if (( $RESULT > 128 )) && (( $RESULT <= 128+31 )); then
				RESULT="$RESULT (SIG$(kill -l $(($RESULT - 128))))"
			fi
670
			msg="$(date -u) - Something weird happened, $DIFFOSCOPE on $1/$2 exited with $RESULT and I don't know how to handle it."
671 672 673 674 675 676
			;;
	esac
	if [ ! -z "$msg" ] ; then
		echo $msg | tee -a $TMPDIR/$1/$2.html
	fi
}
677 678 679

get_filesize() {
		local BYTESIZE="$(du -h -b $1 | cut -f1)"
680 681
		# numbers below 16384K are understood and more meaningful than 16M...
		if [ $BYTESIZE -gt 16777216 ] ; then
682 683 684 685
			SIZE="$(echo $BYTESIZE/1048576|bc)M"
		elif [ $BYTESIZE -gt 1024 ] ; then
			SIZE="$(echo $BYTESIZE/1024|bc)K"
		else
686
			SIZE="$BYTESIZE bytes"
687 688
		fi
}
689

690
cleanup_pkg_files() {
691 692 693 694
	rm -vf $DEBIAN_BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_*.rbuild.log{,.gz}
	rm -vf $DEBIAN_BASE/logs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.build?.log{,.gz}
	rm -vf $DEBIAN_BASE/dbd/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.html
	rm -vf $DEBIAN_BASE/dbdtxt/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.txt{,.gz}
695
	rm -vf $DEBIAN_BASE/dbdjson/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.json{,.gz}
696 697
	rm -vf $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}/${SRCPACKAGE}_*.buildinfo
	rm -vf $DEBIAN_BASE/logdiffs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diff{,.gz}
698 699
}

700 701 702 703 704 705 706 707 708 709 710 711 712
handle_race_condition() {
	local RESULT=$(query_db "SELECT job FROM schedule WHERE package_id='$SRCPKGID'")
	local msg="Package ${SRCPACKAGE} (id=$SRCPKGID) in ${SUITE} on ${ARCH} is probably already building at $RESULT, while this is $BUILD_URL.\n"
	log_warning "$msg"
	printf "$(date -u) - $msg" >> /var/log/jenkins/reproducible-race-conditions.log
	log_warning "Terminating this build quickly and nicely..."
	if [ $SAVE_ARTIFACTS -eq 1 ] ; then
		SAVE_ARTIFACTS=0
		if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
	fi
	exit 0
}

713 714 715 716 717 718 719 720
unregister_build() {
	# unregister this build so it will immeditiatly tried again
	if [ -n "$SRCPKGID" ] ; then
		query_db "UPDATE schedule SET date_build_started = NULL, job = NULL WHERE package_id=$SRCPKGID"
	fi
	NOTIFY=""
}

721 722 723 724 725 726 727 728
handle_remote_error() {
	MESSAGE="${BUILD_URL}console.log got remote error $1"
	echo "$(date -u ) - $MESSAGE" | tee -a /var/log/jenkins/reproducible-remote-error.log
	echo "Sleeping 5m before aborting the job."
	sleep 5m
	exit 0
}

729
#
Holger Levsen's avatar
Holger Levsen committed
730
# create the png (and query the db to populate a csv file...) for Debian
731
#
Holger Levsen's avatar
Holger Levsen committed
732
create_debian_png_from_table() {
733 734 735 736 737 738
	echo "Checking whether to update $2..."
	# $1 = id of the stats table
	# $2 = image file name
	echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv
	# prepare query
	WHERE_EXTRA="WHERE suite = '$SUITE'"
739 740 741
	if [ "$ARCH" = "armhf" ] ; then
		# armhf was only build since 2015-08-30
		WHERE2_EXTRA="WHERE s.datum >= '2015-08-30'"
742 743 744
	elif [ "$ARCH" = "i386" ] ; then
		# i386 was only build since 2016-03-28
		WHERE2_EXTRA="WHERE s.datum >= '2016-03-28'"
745 746 747
	elif [ "$ARCH" = "arm64" ] ; then
		# arm63 was only build since 2016-12-23
		WHERE2_EXTRA="WHERE s.datum >= '2016-12-23'"
748 749 750
	else
		WHERE2_EXTRA=""
	fi
751 752
	if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] || [ $1 -eq 8 ] ; then
		# TABLE[3+4+5] don't have a suite column: (and TABLE[8] (and 9) is faked, based on 3)
753 754
		WHERE_EXTRA=""
	fi
755 756
	if [ $1 -eq 0 ] || [ $1 -eq 2 ] ; then
		# TABLE[0+2] have a architecture column:
757
		WHERE_EXTRA="$WHERE_EXTRA AND architecture = '$ARCH'"
758 759
		if [ "$ARCH" = "armhf" ]  ; then
			if [ $1 -eq 2 ] ; then
760
				# unstable/armhf was only build since 2015-08-30 (and experimental/armhf since 2015-12-19 and stretch/armhf since 2016-01-01)
761 762
				WHERE_EXTRA="$WHERE_EXTRA AND datum >= '2015-08-30'"
			fi
763 764 765 766 767
		elif [ "$ARCH" = "i386" ]  ; then
			if [ $1 -eq 2 ] ; then
				# i386 was only build since 2016-03-28
				WHERE_EXTRA="$WHERE_EXTRA AND datum >= '2016-03-28'"
			fi
768 769 770 771 772
		elif [ "$ARCH" = "arm64" ]  ; then
			if [ $1 -eq 2 ] ; then
				# arm64 was only build since 2016-12-23
				WHERE_EXTRA="$WHERE_EXTRA AND datum >= '2016-12-23'"
			fi
773
		fi
774
		# stretch/amd64 was only build since...
775 776 777
		# WHERE2_EXTRA="WHERE s.datum >= '2015-03-08'"
		# experimental/amd64 was only build since...
		# WHERE2_EXTRA="WHERE s.datum >= '2015-02-28'"
778
	fi
779 780 781
	# run query
	if [ $1 -eq 1 ] ; then
		# not sure if it's worth to generate the following query...
782
		WHERE_EXTRA="AND architecture='$ARCH'"
783

784
		# This query became much more obnoxious when gaining
785 786
		# compatibility with postgres
		query_to_csv "SELECT stats.datum,
787
			 COALESCE(reproducible_stretch,0) AS reproducible_stretch,
788
			 COALESCE(reproducible_buster,0) AS reproducible_buster,
789 790
			 COALESCE(reproducible_unstable,0) AS reproducible_unstable,
			 COALESCE(reproducible_experimental,0) AS reproducible_experimental,
791 792 793 794
			 COALESCE(FTBR_stretch,0) AS FTBR_stretch,
			 COALESCE(FTBR_buster,0) AS FTBR_buster,
			 COALESCE(FTBR_unstable,0) AS FTBR_unstable,
			 COALESCE(FTBR_experimental,0) AS FTBR_experimental,
795
			 COALESCE(FTBFS_stretch,0) AS FTBFS_stretch,
796
			 COALESCE(FTBFS_buster,0) AS FTBFS_buster,
797 798
			 COALESCE(FTBFS_unstable,0) AS FTBFS_unstable,
			 COALESCE(FTBFS_experimental,0) AS FTBFS_experimental,
799
			 COALESCE(other_stretch,0) AS other_stretch,
800
			 COALESCE(other_buster,0) AS other_buster,
801 802 803
			 COALESCE(other_unstable,0) AS other_unstable,
			 COALESCE(other_experimental,0) AS other_experimental
			FROM (SELECT s.datum,
804
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA),0) AS reproducible_stretch,
805
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA),0) AS reproducible_buster,
806 807
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS reproducible_unstable,
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS reproducible_experimental,
808 809 810 811
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBR_stretch,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBR_buster,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBR_unstable,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBR_experimental,
812
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBFS_stretch,
813
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBFS_buster,
814 815
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBFS_unstable,
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBFS_experimental,
816
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS other_stretch,
817
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS other_buster,
818 819
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS other_unstable,
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS other_experimental
820 821
			 FROM stats_builds_per_day AS s $WHERE2_EXTRA GROUP BY s.datum) as stats
			ORDER BY datum" >> ${TABLE[$1]}.csv
822
	elif [ $1 -eq 2 ] ; then
823
		# just make a graph of the oldest reproducible build (ignore FTBFS and FTBR)
824
		query_to_csv "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
825
	elif [ $1 -eq 7 ] ; then
826
		query_to_csv "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv
827
	elif [ $1 -eq 8 ] ; then
828
		query_to_csv "SELECT ${FIELDS[$1]} from ${TABLE[3]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
829
	elif [ $1 -eq 9 ] ; then
830
		query_to_csv "SELECT datum, $REPRODUCIBLE_DONE, $REPRODUCIBLE_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv
831
	else
832
		query_to_csv "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
833 834 835 836 837 838
	fi
	# this is a gross hack: normally we take the number of colors a table should have...
	#  for the builds_age table we only want one color, but different ones, so this hack:
	COLORS=${COLOR[$1]}
	if [ $1 -eq 2 ] ; then
		case "$SUITE" in
839
			stretch)	COLORS=40 ;;
840 841 842
			buster)		COLORS=41 ;;
			unstable)	COLORS=42 ;;
			experimental)	COLORS=43 ;;
843 844
		esac
	fi
845 846
	local WIDTH=1920
	local HEIGHT=960
847 848 849 850 851 852
	# only generate graph if the query returned data
	if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then
		echo "Updating $2..."
		DIR=$(dirname $2)
		mkdir -p $DIR
		echo "Generating $2."
853
		/srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" $WIDTH $HEIGHT
854
		mv $2 $DEBIAN_BASE/$DIR
855 856
		[ "$DIR" = "." ] || rmdir $(dirname $2)
	# create empty dummy png if there havent been any results ever
857
	elif [ ! -f $DEBIAN_BASE/$DIR/$(basename $2) ] ; then
858 859 860 861
		DIR=$(dirname $2)
		mkdir -p $DIR
		echo "Creating $2 dummy."
		convert -size 1920x960 xc:#aaaaaa -depth 8 $2
862
		mv $2 $DEBIAN_BASE/$DIR
863 864 865 866
		[ "$DIR" = "." ] || rmdir $(dirname $2)
	fi
	rm ${TABLE[$1]}.csv
}
867

868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965
#
# create the png (and query the db to populate a csv file...) for Arch Linux
#
create_archlinux_png_from_table() {
	echo "Checking whether to update $2..."
	# $1 = id of the stats table
	# $2 = image file name
	echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv
	# prepare query
	WHERE_EXTRA="WHERE suite = '$SUITE'"
	if [ $1 -eq 0 ] || [ $1 -eq 2 ] ; then
		# TABLE[0+2] have a architecture column:
		WHERE_EXTRA="$WHERE_EXTRA AND architecture = '$ARCH'"
	fi
	# run query
	if [ $1 -eq 1 ] ; then
		# not sure if it's worth to generate the following query...
		WHERE_EXTRA="AND architecture='$ARCH'"

		# This query became much more obnoxious when gaining
		# compatibility with postgres
		query_to_csv "SELECT stats.datum,
			 COALESCE(reproducible_stretch,0) AS reproducible_stretch,
			 COALESCE(reproducible_buster,0) AS reproducible_buster,
			 COALESCE(reproducible_unstable,0) AS reproducible_unstable,
			 COALESCE(reproducible_experimental,0) AS reproducible_experimental,
			 COALESCE(FTBR_stretch,0) AS FTBR_stretch,
			 COALESCE(FTBR_buster,0) AS FTBR_buster,
			 COALESCE(FTBR_unstable,0) AS FTBR_unstable,
			 COALESCE(FTBR_experimental,0) AS FTBR_experimental,
			 COALESCE(FTBFS_stretch,0) AS FTBFS_stretch,
			 COALESCE(FTBFS_buster,0) AS FTBFS_buster,
			 COALESCE(FTBFS_unstable,0) AS FTBFS_unstable,
			 COALESCE(FTBFS_experimental,0) AS FTBFS_experimental,
			 COALESCE(other_stretch,0) AS other_stretch,
			 COALESCE(other_buster,0) AS other_buster,
			 COALESCE(other_unstable,0) AS other_unstable,
			 COALESCE(other_experimental,0) AS other_experimental
			FROM (SELECT s.datum,
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA),0) AS reproducible_stretch,
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA),0) AS reproducible_buster,
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS reproducible_unstable,
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS reproducible_experimental,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBR_stretch,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBR_buster,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBR_unstable,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBR_experimental,
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBFS_stretch,
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBFS_buster,
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBFS_unstable,
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBFS_experimental,
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS other_stretch,
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS other_buster,
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS other_unstable,
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS other_experimental
			 FROM stats_builds_per_day AS s GROUP BY s.datum) as stats
			ORDER BY datum" >> ${TABLE[$1]}.csv
	elif [ $1 -eq 2 ] ; then
		# just make a graph of the oldest reproducible build (ignore FTBFS and FTBR)
		query_to_csv "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
	else
		query_to_csv "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
	fi
	# this is a gross hack: normally we take the number of colors a table should have...
	#  for the builds_age table we only want one color, but different ones, so this hack:
	COLORS=${COLOR[$1]}
	if [ $1 -eq 2 ] ; then
		case "$SUITE" in
			stretch)	COLORS=40 ;;
			buster)		COLORS=41 ;;
			unstable)	COLORS=42 ;;
			experimental)	COLORS=43 ;;
		esac
	fi
	local WIDTH=1920
	local HEIGHT=960
	# only generate graph if the query returned data
	if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then
		echo "Updating $2..."
		DIR=$(dirname $2)
		mkdir -p $DIR
		echo "Generating $2."
		/srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" $WIDTH $HEIGHT
		mv $2 $ARCHBASE/$DIR
		[ "$DIR" = "." ] || rmdir $(dirname $2)
	# create empty dummy png if there havent been any results ever
	elif [ ! -f $ARCHBASE/$DIR/$(basename $2) ] ; then
		DIR=$(dirname $2)
		mkdir -p $DIR
		echo "Creating $2 dummy."
		convert -size 1920x960 xc:#aaaaaa -depth 8 $2
		mv $2 $ARCHBASE/$DIR
		[ "$DIR" = "." ] || rmdir $(dirname $2)
	fi
	rm ${TABLE[$1]}.csv
}


966 967 968 969 970
find_in_buildlogs() {
    egrep -q "$1" $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null
}

include_icon(){
971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988
	local STATE=$1
	local TEXT=$2
	local ALT=${STATE%%_*}
	local PNG=
	ALT=${ALT,,}
	case $STATE in
		BLACKLISTED)
			PNG=error;;
		DEPWAIT_*)
			PNG=weather-snow;;
		404_*)
			PNG=weather-severe-alert;;
		FTBFS_*)
			PNG=weather-storm;;
		FTBR_*)
			PNG=weather-showers-scattered ALT=unreproducible ;;
		GOOD)
			PNG=weather-clear ALT=reproducible ;;
989 990 991 992 993 994
	esac
	echo "       <img src=\"/userContent/static/$PNG.png\" alt=\"$ALT icon\" /> $TEXT" >> $HTML_BUFFER
}

create_pkg_html() {
	local ARCHLINUX_PKG_PATH=$ARCHBASE/$REPOSITORY/$SRCPACKAGE
995 996 997
	local HTML_BUFFER=$(mktemp -t archlinuxrb-html-XXXXXXXX)
	local buffer_message
	local STATE
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015

	# clear files from previous builds
	cd "$ARCHLINUX_PKG_PATH"
	for file in build1.log build2.log build1.version build2.version *BUILDINFO.txt *.html; do
		if [ -f $file ] && [ pkg.build_duration -nt $file ] ; then
			rm $file
			echo "$ARCHLINUX_PKG_PATH/$file older than $ARCHLINUX_PKG_PATH/pkg.build_duration, thus deleting it."
		fi
	done

	echo "     <tr>" >> $HTML_BUFFER
	echo "      <td>$REPOSITORY</td>" >> $HTML_BUFFER
	echo "      <td>$SRCPACKAGE</td>" >> $HTML_BUFFER
	echo "      <td>$VERSION</td>" >> $HTML_BUFFER
	echo "      <td>" >> $HTML_BUFFER
	#
	#
	if [ -z "$(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
1016 1017
		# this horrible if elif elif elif elif...  monster should be replaced
		# by using pacman's exit code which is possible since sometime in 2018
1018

1019
		# check different states and figure out what the page should look like
1020
		if find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then
1021 1022
			STATE=DEPWAIT_0
			buffer_message='could not resolve dependencies as there are conflicts'
1023 1024
		elif find_in_buildlogs '==> ERROR: (Could not resolve all dependencies|.pacman. failed to install missing dependencies)'; then
			if find_in_buildlogs 'error: failed to init transaction \(unable to lock database\)'; then
1025 1026
				STATE=DEPWAIT_2
				buffer_message='pacman could not lock database'
1027
			else
1028 1029
				STATE=DEPWAIT_1
				buffer_message='could not resolve dependencies'
1030 1031
			fi
		elif find_in_buildlogs '^error: unknown package: '; then
1032 1033
			STATE=404_0
			buffer_message='unknown package'
1034 1035 1036
		elif find_in_buildlogs '(==> ERROR: Failure while downloading|==> ERROR: One or more PGP signatures could not be verified|==> ERROR: One or more files did not pass the validity check|==> ERROR: Integrity checks \(.*\) differ in size from the source array|==> ERROR: Failure while branching|==> ERROR: Failure while creating working copy|Failed to source PKGBUILD.*PKGBUILD)'; then
			REASON="download failed"
			EXTRA_REASON=""
1037
			STATE=404_0
1038
			if find_in_buildlogs 'FAILED \(unknown public key'; then
1039
				STATE=404_6
1040 1041
				EXTRA_REASON="to verify source with PGP due to unknown public key"
			elif find_in_buildlogs 'The requested URL returned error: 403'; then
1042
				STATE=404_2
1043 1044
				EXTRA_REASON="with 403 - forbidden"
			elif find_in_buildlogs 'The requested URL returned error: 500'; then
1045
				STATE=404_4
1046 1047
				EXTRA_REASON="with 500 - internal server error"
			elif find_in_buildlogs 'The requested URL returned error: 503'; then
1048
				STATE=404_5
1049 1050
				EXTRA_REASON="with 503 - service unavailable"
			elif find_in_buildlogs '==> ERROR: One or more PGP signatures could not be verified'; then
1051
				STATE=404_7
1052 1053
				EXTRA_REASON="to verify source with PGP signatures"
			elif find_in_buildlogs '(SSL certificate problem: unable to get local issuer certificate|^bzr: ERROR: .SSL: CERTIFICATE_VERIFY_FAILED)'; then
1054
				STATE=404_1
1055 1056
				EXTRA_REASON="with SSL problem"
			elif find_in_buildlogs '==> ERROR: One or more files did not pass the validity check'; then
1057
				STATE=404_8
1058 1059
				REASON="downloaded ok but failed to verify source"
			elif find_in_buildlogs '==> ERROR: Integrity checks \(.*\) differ in size from the source array'; then
1060
				STATE=404_9
1061 1062
				REASON="Integrity checks differ in size from the source array"
			elif find_in_buildlogs 'The requested URL returned error: 404'; then
1063
				STATE=404_3
1064 1065
				EXTRA_REASON="with 404 - file not found"
			elif find_in_buildlogs 'fatal: the remote end hung up unexpectedly'; then
1066
				STATE=404_A
1067 1068
				EXTRA_REASON="could not clone git repository"
			elif find_in_buildlogs 'The requested URL returned error: 504'; then
1069
				STATE=404_B
1070 1071
				EXTRA_REASON="with 504 - gateway timeout"
			elif find_in_buildlogs '==> ERROR: Failure while downloading .* git repo'; then
1072
				STATE=404_C
1073 1074
				EXTRA_REASON="from git repo"
			fi
1075
			buffer_message="$REASON $EXTRA_REASON"
1076
		elif find_in_buildlogs '==> ERROR: (install file .* does not exist or is not a regular file|The download program wget is not installed)'; then
1077 1078
			STATE=FTBFS_0
			buffer_message='failed to build, requirements not met'
1079
		elif find_in_buildlogs '==> ERROR: A failure occurred in check'; then
1080 1081
			STATE=FTBFS_1
			buffer_message='failed to build while running tests'
1082
		elif find_in_buildlogs '==> ERROR: (An unknown error has occurred|A failure occurred in (build|package|prepare))'; then
1083 1084
			STATE=FTBFS_2
			buffer_message='failed to build'
1085
		elif find_in_buildlogs 'makepkg was killed by timeout after'; then
1086 1087
			STATE=FTBFS_3
			buffer_message='failed to build, killed by timeout'
1088
		elif find_in_buildlogs '==> ERROR: .* contains invalid characters:'; then
1089 1090
			STATE=FTBFS_4
			buffer_message='failed to build, pkg relations contain invalid characters'
1091
		else
1092
			STATE=$(query_db "SELECT r.status FROM results AS r
1093 1094 1095
				JOIN sources as s on s.id=r.package_id
				WHERE s.architecture='x86_64'
				AND s.name='$SRCPACKAGE'
1096
				AND s.suite='archlinux_$REPOSITORY';")
1097 1098 1099 1100 1101 1102
			if [ "$STATE" = "BLACKLISTED" ] ; then
				buffer_message='blacklisted'