reproducible_common.sh 65 KB
Newer Older
Holger Levsen's avatar
Holger Levsen committed
1
#!/bin/bash
2
# vim: set noexpandtab:
Holger Levsen's avatar
Holger Levsen committed
3

4
# Copyright 2014-2019 Holger Levsen <holger@layer-acht.org>
5
#         © 2015-2018 Mattia Rizzolo <mattia@mapreri.org>
Holger Levsen's avatar
Holger Levsen committed
6 7
# released under the GPLv=2
#
8 9 10
# included by all reproducible_*.sh scripts, so be quiet
set +x

11 12
# postgres database definitions
export PGDATABASE=reproducibledb
Holger Levsen's avatar
Holger Levsen committed
13

14 15
# query reproducible database
query_db() {
16
	psql -t --no-align -c "$@" || exit 1
17 18
}

19 20
# query reproducible database, output to csv format
query_to_csv() {
21
	psql -c "COPY ($@) to STDOUT with csv DELIMITER ','" || exit 1
22 23
}

24
# common variables
25
BASE="/var/lib/jenkins/userContent/reproducible"
26
REPRODUCIBLE_URL=https://tests.reproducible-builds.org
27
REPRODUCIBLE_DOT_ORG_URL=https://reproducible-builds.org
28 29
# shop trailing slash
JENKINS_URL=${JENKINS_URL:0:-1}
Holger Levsen's avatar
Holger Levsen committed
30
DBDSUITE="unstable"
31
BIN_PATH=/srv/jenkins/bin
32
TEMPLATE_PATH=/srv/jenkins/mustache-templates/reproducible
33 34
CHPATH=/srv/reproducible-results/chdist
mkdir -p "$CHPATH"
35

36
# Debian suites being tested
37
SUITES="stretch buster unstable experimental"
38
# Debian architectures being tested
39
ARCHS="amd64 i386 arm64 armhf"
Holger Levsen's avatar
Holger Levsen committed
40

41
# define Debian build nodes in use
42
. /srv/jenkins/bin/jenkins_node_definitions.sh
43
MAINNODE="jenkins" # used by reproducible_maintenance.sh only
44
JENKINS_OFFLINE_LIST="/var/lib/jenkins/offline_nodes"
45 46

# variables on the nodes we are interested in
47
BUILD_ENV_VARS="ARCH NUM_CPU CPU_MODEL DATETIME KERNEL" # these also needs to be defined in bin/reproducible_info.sh
48

49 50 51 52 53 54
# common settings for Debian
DEBIAN_URL=https://tests.reproducible-builds.org/debian
DEBIAN_DASHBOARD_URI=/debian/reproducible.html
DEBIAN_BASE="/var/lib/jenkins/userContent/reproducible/debian"
mkdir -p "$DEBIAN_BASE"

55
# existing usertags in the Debian BTS
56
USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness buildinfo cpu signatures environment umask ftbfs locale"
57

58 59 60 61 62
# common settings for testing alpine
ALPINE_REPOS="main community"
ALPINE_PKGS=/srv/reproducible-results/alpine_pkgs
ALPINE_BASE="$BASE/alpine"

63
# common settings for testing Arch Linux
64
ARCHLINUX_REPOS="core extra multilib community"
65
ARCHLINUX_PKGS=/srv/reproducible-results/archlinux_pkgs
66
ARCHBASE=$BASE/archlinux
67

68
# common settings for testing rpm based distros
69
RPM_BUILD_NODE=osuosl-build171-amd64
70
RPM_PKGS=/srv/reproducible-results/rpm_pkgs
71

Holger Levsen's avatar
Holger Levsen committed
72
# number of cores to be used
73
NUM_CPU=$(nproc)
Holger Levsen's avatar
Holger Levsen committed
74

75 76 77
# diffoscope memory limit in kilobytes
DIFFOSCOPE_VIRT_LIMIT=$((10*1024*1024))

78
# we only this array for html creation but we cannot declare them in a function
79
declare -A SPOKENTARGET
80

81 82
# to hold reproducible temporary files/directories without polluting /tmp
TEMPDIR="/tmp/reproducible"
83
mkdir -p "$TEMPDIR"
84

85 86
# create subdirs for suites
for i in $SUITES ; do
87
	mkdir -p "$DEBIAN_BASE/$i"
88 89
done

90
# table names and image names
Holger Levsen's avatar
Holger Levsen committed
91 92 93 94 95 96 97 98
TABLE[0]=stats_pkg_state
TABLE[1]=stats_builds_per_day
TABLE[2]=stats_builds_age
TABLE[3]=stats_bugs
TABLE[4]=stats_notes
TABLE[5]=stats_issues
TABLE[6]=stats_meta_pkg_state
TABLE[7]=stats_bugs_state
99 100
TABLE[8]=stats_bugs_sin_ftbfs
TABLE[9]=stats_bugs_sin_ftbfs_state
Holger Levsen's avatar
Holger Levsen committed
101

102
# package sets defined in meta_pkgsets.csv
103
# csv file columns: (pkgset_group, pkgset_name)
104
colindex=0
105
while IFS=, read col1 col2
106
do
107
	let colindex+=1
108
	META_PKGSET[$colindex]=$col2
109
done < $BIN_PATH/reproducible_pkgsets.csv
110

111 112
# mustache templates
PAGE_FOOTER_TEMPLATE=$TEMPLATE_PATH/default_page_footer.mustache
113 114 115
PROJECT_LINKS_TEMPLATE=$TEMPLATE_PATH/project_links.mustache
MAIN_NAVIGATION_TEMPLATE=$TEMPLATE_PATH/main_navigation.mustache

116 117 118 119
# be loud again if DEBUG
if $DEBUG ; then
	set -x
fi
120

121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
# some cmomon logging functions
log_info () {
	_log "I:" "$@"
}

log_error () {
	_log "E:" "$@"
}

log_warning () {
	_log "W:" "$@"
}

log_file () {
	cat $@ | tee -a $RBUILDLOG
}

_log () {
	local prefix="$1"
	shift 1
	echo -e "$(date -u)  $prefix $*" | tee -a $RBUILDLOG
}

144 145 146 147 148
# sleep 1-23 secs to randomize start times
delay_start() {
	/bin/sleep $(echo "scale=1 ; $(shuf -i 1-230 -n 1)/10" | bc )
}

149
schedule_packages() {
150 151 152 153 154
	LC_USER="$REQUESTER" \
	LOCAL_CALL="true" \
	/srv/jenkins/bin/reproducible_remote_scheduler.py \
		--message "$REASON" \
		--no-notify \
155
		--suite "$SUITE" \
156
		--architecture "$ARCH" \
157
		$@
158 159
}

160 161 162 163 164 165 166
set_icon() {
	# icons taken from tango-icon-theme (0.8.90-5)
	# licenced under http://creativecommons.org/licenses/publicdomain/
	STATE_TARGET_NAME="$1"
	case "$1" in
		reproducible)		ICON=weather-clear.png
					;;
167
		FTBR)		ICON=weather-showers-scattered.png
168
					STATE_TARGET_NAME="FTBR"
169 170 171
					;;
		FTBFS)			ICON=weather-storm.png
					;;
172
		timeout)	ICON=Current_event_clock.png ;;
173 174
		depwait)		ICON=weather-snow.png
					;;
175
		E404)			ICON=weather-severe-alert.png
176
					;;
177
		NFU)		ICON=weather-few-clouds-night.png
178
					STATE_TARGET_NAME="NFU"
179 180 181 182 183 184 185 186 187
					;;
		blacklisted)		ICON=error.png
					;;
		*)			ICON=""
	esac
}

write_icon() {
	# ICON and STATE_TARGET_NAME are set by set_icon()
188
	write_page "<a href=\"/debian/$SUITE/$ARCH/index_${STATE_TARGET_NAME}.html\" target=\"_parent\"><img src=\"/static/$ICON\" alt=\"${STATE_TARGET_NAME} icon\" /></a>"
189 190 191
}

write_page_header() {
192
	# this is really quite uncomprehensible and should be killed
193
	# the solution is to write all HTML pages with python…
194
	rm -f $PAGE
195
	MAINVIEW="dashboard"
196 197
	write_page "<!DOCTYPE html><html><head>"
	write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />"
198
	write_page "<meta name=\"viewport\" content=\"width=device-width\" />"
199
	write_page "<link href=\"/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />"
200
	write_page "<title>$2</title></head>"
201
	if [ "$1" != "$MAINVIEW" ] ; then
202
		write_page "<body class=\"wrapper\">"
203
	else
204
		write_page "<body class=\"wrapper\" onload=\"selectSearch()\">"
205
	fi
206 207 208 209 210 211 212 213 214 215 216 217

	# Build context for the main_navigation mustache template.

	# Do not show package set links for "experimental" pages
	if [ "$SUITE" != "experimental" ] ; then
		# no pkg_sets are tested in experimental
		include_pkgset_link="\"include_pkgset_link\" : \"true\""
	else
		include_pkgset_link=''
	fi

	# Used to highlight the link for the current page
218 219 220 221 222 223 224
	if [ "$1" = "dashboard" ] \
		|| [ "$1" = "performance" ] \
		|| [ "$1" = "repositories" ] \
		|| [ "$1" = "variations" ] \
		|| [ "$1" = "suite_arch_stats" ] \
		|| [ "$1" = "bugs" ] \
		|| [ "$1" = "nodes_health" ] \
225
		|| [ "$1" = "job_health" ] \
226 227
		|| [ "$1" = "nodes_weekly_graphs" ] \
		|| [ "$1" = "nodes_daily_graphs" ] ; then
228 229 230 231 232 233
		displayed_page="\"$1\": \"true\""
	else
		displayed_page=''
	fi

	# Create json for suite links (a list of objects)
234
	suite_links="\"suite_nav\": { \"suite_list\": ["
235
	comma=0
236 237
	for s in $SUITES ; do
		if [ "$s" = "$SUITE" ] ; then
238
			class="current"
239 240
		else
			class=''
241
		fi
242
		uri="/debian/${s}/index_suite_${ARCH}_stats.html"
Holger Levsen's avatar
Holger Levsen committed
243
		if [ $comma = 1 ] ; then
244
			suite_links+=", {\"s\": \"${s}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
245
		else
246
			suite_links+="{\"s\": \"${s}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
247
			comma=1
248
		fi
249
	done
250
	suite_links+="]}"
251 252

	# Create json for arch links (a list of objects)
253
	arch_links="\"arch_nav\": {\"arch_list\": ["
254
	comma=0
255 256
	for a in ${ARCHS} ; do
		if [ "$a" = "$ARCH" ] ; then
257
			class="current"
258
		else
259
			class=''
260
		fi
261
		uri="/debian/$SUITE/index_suite_${a}_stats.html"
Holger Levsen's avatar
Holger Levsen committed
262
		if [ $comma = 1 ] ; then
263
			arch_links+=", {\"a\": \"${a}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
264
		else
265
			arch_links+="{\"a\": \"${a}\", \"class\": \"$class\", \"uri\": \"$uri\"}"
266
			comma=1
267
		fi
268
	done
269
	arch_links+="]}"
270 271 272 273 274 275

	# finally, the completely formed JSON context
	context=$(printf '{
		"arch" : "%s",
		"suite" : "%s",
		"page_title" : "%s",
276
		"debian_uri" : "%s",
277 278
		%s,
		%s
279
	' "$ARCH" "$SUITE" "$2" "$DEBIAN_DASHBOARD_URI" "$arch_links" "$suite_links")
280 281 282 283 284 285 286 287 288 289 290
	if [[ ! -z $displayed_page ]] ; then
		context+=", $displayed_page"
	fi
	if [[ ! -z $include_pkgset_link ]] ; then
		context+=", $include_pkgset_link"
	fi
	context+="}"

	write_page "<header class=\"head\">"
	write_page "$(pystache3 $MAIN_NAVIGATION_TEMPLATE "$context")"
	write_page "$(pystache3 $PROJECT_LINKS_TEMPLATE "{}")"
291
	write_page "</header>"
292

293
	write_page "<div class=\"mainbody\">"
294
	write_page "<h2>$2</h2>"
295
	if [ "$1" = "$MAINVIEW" ] ; then
296
		write_page "<ul>"
297
		write_page "   Please also visit the more general website <li><a href=\"https://reproducible-builds.org\">Reproducible-builds.org</a></li> where <em>reproducible builds</em> are explained in more detail than just <em>bit by bit identical rebuilds to enable verifcation of the sources used to build</em>."
298 299 300 301
		write_page "   We think that reproducible builds should become the norm, so we wrote <li><a href=\"https://reproducible-builds.org/howto\">How to make your software reproducible</a></li>."
		write_page "   Also aimed at the free software world at large, is the first specification we have written: the <li><a href=\"https://reproducible-builds.org/specs/source-date-epoch/\">SOURCE_DATE_EPOCH specification</a></li>."
		write_page "</ul>"
		write_page "<ul>"
Chris Lamb's avatar
Chris Lamb committed
302
		write_page "   These pages are showing the <em>potential</em> of <li><a href=\"https://wiki.debian.org/ReproducibleBuilds\" target=\"_blank\">reproducible builds of Debian packages</a></li>."
303 304 305 306
		write_page "   The results shown were obtained by <a href=\"$JENKINS_URL/view/reproducible\">several jobs</a> running on"
		write_page "   <a href=\"$JENKINS_URL/userContent/about.html#_reproducible_builds_jobs\">jenkins.debian.net</a>."
		write_page "   Thanks to <a href=\"https://www.profitbricks.co.uk\">Profitbricks</a> for donating the virtual machines this is running on!"
		write_page "</ul>"
307
		LATEST=$(query_db "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id = s.id WHERE r.status = 'FTBR' AND s.suite = 'unstable' AND s.architecture = 'amd64' AND s.id NOT IN (SELECT package_id FROM notes) ORDER BY build_date DESC LIMIT 23"|sort -R|head -1)
308
		write_page "<form action=\"$REPRODUCIBLE_URL/redirect\" method=\"GET\">$REPRODUCIBLE_URL/"
309
		write_page "<input type=\"text\" name=\"SrcPkg\" placeholder=\"Type my friend..\" value=\"$LATEST\" />"
310
		write_page "<input type=\"submit\" value=\"submit source package name\" />"
311
		write_page "</form>"
312 313 314 315 316
		write_page "<ul>"
		write_page "   We are reachable via IRC (<code>#debian-reproducible</code> and <code>#reproducible-builds</code> on OFTC),"
		write_page "   or <a href="mailto:reproducible-builds@lists.alioth.debian.org">email</a>,"
		write_page "   and we care about free software in general,"
		write_page "   so whether you are an upstream developer or working on another distribution, or have any other feedback - we'd love to hear from you!"
317 318 319 320 321
		write_page "   Besides Debian we are also testing "
		write_page "   <li><a href=\"/coreboot/\">coreboot</a></li>,"
		write_page "   <li><a href=\"/openwrt/\">OpenWrt</a></li>, "
		write_page "   <li><a href=\"/netbsd/\">NetBSD</a></li>, "
		write_page "   <li><a href=\"/freebsd/\">FreeBSD</a></li>, "
322
		write_page "   and <li><a href=\"/archlinux/\">Arch Linux</a></li> "
323
		write_page "   though not as thoroughly as Debian yet. "
324 325 326
		write_page "   <li><a href=\"http://rb.zq1.de/\">openSUSE</a></li>, "
		write_page "   <li><a href=\"https://r13y.com/\">NixOS</a></li> and "
		write_page "   <li><a href=\"https://verification.f-droid.org/\">F-Droid</a></li> are also being tested, though elsewhere."
327
		write_page "   As far as we know, the <a href=\"https://www.gnu.org/software/guix/manual/en/html_node/Invoking-guix-challenge.html\">Guix challenge</a> is not yet run systematically anywhere."
328
		write_page "   Testing of "
329
		write_page "   <a href=\"/rpms/fedora-23.html\">Fedora</a> "
330 331
		write_page "   has sadly been suspended for now. "
		write_page " We can test more projects, if <em>you</em> contribute!"
332
		write_page "</ul>"
333
	fi
334 335
}

336
write_page_intro() {
337
	write_page "       <p><em>Reproducible builds</em> enable anyone to reproduce bit by bit identical binary packages from a given source, so that anyone can verify that a given binary derived from the source it was said to be derived."
338 339
	write_page "         There is more information about <a href=\"https://wiki.debian.org/ReproducibleBuilds\">reproducible builds on the Debian wiki</a> and on <a href=\"https://reproducible-builds.org\">https://reproducible-builds.org</a>."
	write_page "         These pages explain in more depth why this is useful, what common issues exist and which workarounds and solutions are known."
340
	write_page "        </p>"
341
	local BUILD_ENVIRONMENT=" in a Debian environment"
342
	local BRANCH="master"
Holger Levsen's avatar
Holger Levsen committed
343
	if [ "$1" = "coreboot" ] ; then
344
		write_page "        <p><em>Reproducible Coreboot</em> is an effort to apply this to coreboot. Thus each coreboot.rom is build twice (without payloads), with a few variations added and then those two ROMs are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
345 346
		local PROJECTNAME="$1"
		local PROJECTURL="https://review.coreboot.org/p/coreboot.git"
347
	elif [ "$1" = "OpenWrt" ] ; then
348
		local PROJECTNAME="$1"
349
		local PROJECTURL="https://github.com/openwrt/openwrt.git"
350
		write_page "        <p><em>Reproducible $PROJECTNAME</em> is an effort to apply this to $PROJECTNAME. Thus each $PROJECTNAME target is build twice, with a few variations added and then the resulting images and packages from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. $PROJECTNAME generates many different types of raw <code>.bin</code> files, and diffoscope does not know how to parse these. Thus the resulting diffoscope output is not nearly as clear as it could be - hopefully this limitation will be overcome eventually, but in the meanwhile the input components (uImage kernel file, rootfs.tar.gz, and/or rootfs squashfs) can be inspected. Also please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
351
	elif [ "$1" = "NetBSD" ] ; then
352
		write_page "        <p><em>Reproducible NetBSD</em> is an effort to apply this to NetBSD. Thus each NetBSD target is build twice, with a few variations added and then the resulting files from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
353
		local PROJECTNAME="netbsd"
354
		local PROJECTURL="https://github.com/NetBSD/src"
355
	elif [ "$1" = "FreeBSD" ] ; then
356
		write_page "        <p><em>Reproducible FreeBSD</em> is an effort to apply this to FreeBSD. Thus FreeBSD is build twice, with a few variations added and then the resulting filesystems from the two builds are put into a compressed tar archive, which is finally compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
357 358
		local PROJECTNAME="freebsd"
		local PROJECTURL="https://github.com/freebsd/freebsd.git"
359 360
		local BUILD_ENVIRONMENT=", which via ssh triggers a build on a FreeBSD 11.2 system"
		local BRANCH="master"
361 362 363 364 365 366 367 368 369 370 371 372
	elif [ "$1" = "alpine" ] ; then
		local PROJECTNAME="alpine"
		write_page "        <p><em>Reproducible $PROJECTNAME</em> is an effort to apply this to $PROJECTNAME. Thus $PROJECTNAME packages are build twice, with a few variations added and then the resulting packages from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>."
		write_page "   Please note that this is still at an early stage. Also there are more variations expected to be seen in the wild."
		write_page "Missing bits for <em>testing</em> alpine:<ul>"
		write_page " <li>cross references to <a href=\"https://tests.reproducible-builds.org/debian/index_issues.html\">Debian notes</a> - and having alpine specific notes.</li>"
		write_page "</ul></p>"
		write_page "<p>Missing bits for alpine:<ul>"
		write_page " <li>code needs to be written to compare the packages built twice here against newly built packages from the Official alpine repositories.</li>"
		write_page " <li>user tools, for users to verify all of this easily.</li>"
		write_page "</ul></p>"
		write_page "<p>If you want to help out or discuss reproducible builds in $PROJECTNAME, please join #alpine-reproducible on freenode.</p>"
373 374
	elif [ "$1" = "Arch Linux" ] ; then
		local PROJECTNAME="Arch Linux"
375 376 377
		write_page "        <p><em>Reproducible $PROJECTNAME</em> is an effort to apply this to $PROJECTNAME. Thus $PROJECTNAME packages are build twice, with a few variations added and then the resulting packages from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>."
		write_page "   Please note that this is still at an early stage. Also there are more variations expected to be seen in the wild."
		write_page "Missing bits for <em>testing</em> Arch Linux:<ul>"
378
		write_page " <li>cross references to <a href=\"https://tests.reproducible-builds.org/debian/index_issues.html\">Debian notes</a> - and having Arch Linux specific notes.</li>"
379
		write_page "</ul></p>"
380
		write_page "<p>Missing bits for Arch Linux:<ul>"
381
		write_page " <li>code needs to be written to compare the packages built twice here against newly built packages from the Official Arch Linux repositories.</li>"
382 383
		write_page " <li>user tools, for users to verify all of this easily.</li>"
		write_page "</ul></p>"
384
		write_page "<p>If you want to help out or discuss reproducible builds in $PROJECTNAME, please join #archlinux-reproducible on freenode.</p>"
385 386 387 388 389 390
	elif [ "$1" = "fedora-23" ] ; then
		local PROJECTNAME="Fedora 23"
		write_page "        <p><em>Reproducible $PROJECTNAME</em> is a (currently somewhat stalled) effort to apply this to $PROJECTNAME, which is rather obvious with 23… <br/> $PROJECTNAME packages are build twice, with a few variations added and then the resulting packages from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
	fi
	if [ "$1" != "Arch Linux" ] && [ "$1" != "fedora-23" ] ; then
		local SMALLPROJECTNAME="$(echo $PROJECTNAME|tr '[:upper:]' '[:lower:]')"
391
		write_page "       <p>There is a weekly run <a href=\"https://jenkins.debian.net/view/reproducible/job/reproducible_$SMALLPROJECTNAME/\">jenkins job</a> to test the <code>$BRANCH</code> branch of <a href=\"$PROJECTURL\">$PROJECTNAME.git</a>. The jenkins job is running <a href=\"https://salsa.debian.org/qa/jenkins.debian.net/tree/master/bin/reproducible_$SMALLPROJECTNAME.sh\">reproducible_$SMALLPROJECTNAME.sh</a>$BUILD_ENVIRONMENT and this script is solely responsible for creating this page. Feel invited to join <code>#reproducible-builds</code> (on irc.oftc.net) to request job runs whenever sensible. Patches and other <a href=\"mailto:reproducible-builds@lists.alioth.debian.org\">feedback</a> are very much appreciated - if you want to help, please start by looking at the <a href=\"https://jenkins.debian.net/userContent/todo.html#_reproducible_$(echo $1|tr '[:upper:]' '[:lower:]')\">ToDo list for $1</a>, you might find something easy to contribute."
392 393
		write_page "       <br />Thanks to <a href=\"https://www.profitbricks.co.uk\">Profitbricks</a> for donating the virtual machines this is running on!</p>"
	elif [ "$1" = "fedora-23" ] ; then
394
		write_page "       <p><img src=\"/userContent/static/weather-storm.png\"> TODO: explain $PROJECTNAME test setup here.</p>"
395
	fi
396 397
}

398
write_page_footer() {
399
	if [ "$1" = "coreboot" ] ; then
400
		other_distro_details='The <a href=\"http://www.coreboot.org\">Coreboot</a> logo is Copyright © 2008 by Konsult Stuge and coresystems GmbH and can freely be used to refer to the Coreboot project.'
401
	elif [ "$1" = "NetBSD" ] ; then
402
		other_distro_details="NetBSD® is a registered trademark of The NetBSD Foundation, Inc."
403
	elif [ "$1" = "FreeBSD" ] ; then
404
		other_distro_details="FreeBSD is a registered trademark of The FreeBSD Foundation. The FreeBSD logo and The Power to Serve are trademarks of The FreeBSD Foundation."
405
	elif [ "$1" = "Arch Linux" ] ; then
406
		other_distro_details='The <a href=\"https://www.archlinux.org\">Arch Linux</a> name and logo are recognized trademarks. Some rights reserved. The registered trademark Linux® is used pursuant to a sublicense from LMI, the exclusive licensee of Linus Torvalds, owner of the mark on a world-wide basis.'
407
	elif [ "$1" = "fedora-23" ] ; then
408
			other_distro_details="Fedora is sponsored by Red Hat. © 2017 Red Hat, Inc. and others."
409 410
	else
		other_distro_details=''
411
	fi
412 413 414 415 416 417 418 419 420 421 422
	now=$(date +'%Y-%m-%d %H:%M %Z')

	# The context for pystache3 CLI must be json
	context=$(printf '{
		"job_url" : "%s",
		"job_name" : "%s",
		"date" : "%s",
		"other_distro_details" : "%s"
	}' "${JOB_URL:-""}" "${JOB_NAME:-""}" "$now" "$other_distro_details")

	write_page "$(pystache3 $PAGE_FOOTER_TEMPLATE "$context")"
423
	write_page "</div>"
424
	write_page "</body></html>"
425
 }
426

427
write_variation_table() {
428
	write_page "<p style=\"clear:both;\">"
429 430 431 432
	if [ "$1" = "fedora-23" ] ; then
		write_page "There are no variations introduced in the $1 builds yet. Stay tuned.</p>"
		return
	fi
433
	write_page "<table class=\"main\" id=\"variation\"><tr><th>variation</th><th width=\"40%\">first build</th><th width=\"40%\">second build</th></tr>"
434
	if [ "$1" = "debian" ] ; then
435 436
		write_page "<tr><td>hostname</td><td>one of:"
		for a in ${ARCHS} ; do
437
			local COMMA=""
438 439
			local ARCH_NODES=""
			write_page "<br />&nbsp;&nbsp;"
440 441
			for i in $(echo $BUILD_NODES | sed -s 's# #\n#g' | sort -u) ; do
				if [ "$(echo $i | grep $a)" ] ; then
442 443 444
					echo -n "$COMMA ${ARCH_NODES}$(echo $i | cut -d '.' -f1 | sed -s 's# ##g')" >> $PAGE
					if [ -z $COMMA ] ; then
						COMMA=","
445
					fi
446 447 448 449
				fi
			done
		done
		write_page "</td><td>i-capture-the-hostname</td></tr>"
450
		write_page "<tr><td>domainname</td><td>$(hostname -d)</td><td>i-capture-the-domainname</td></tr>"
451
	else
452
		if [ "$1" != "Arch Linux" ] || [ "$1" != "OpenWrt" ] ; then
453
			write_page "<tr><td>hostname</td><td> osuosl-build169-amd64 or osuosl-build170-amd64</td><td>the other one</td></tr>"
454 455 456
		else
			write_page "<tr><td>hostname</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		fi
457
		write_page "<tr><td>domainname</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
458
	fi
459
	if [ "$1" != "FreeBSD" ] && [ "$1" != "Arch Linux" ] && [ "$1" != "fedora-23" ] ; then
460 461
		write_page "<tr><td>env CAPTURE_ENVIRONMENT</td><td><em>not set</em></td><td>CAPTURE_ENVIRONMENT=\"I capture the environment\"</td></tr>"
	fi
462
	write_page "<tr><td>env TZ</td><td>TZ=\"/usr/share/zoneinfo/Etc/GMT+12\"</td><td>TZ=\"/usr/share/zoneinfo/Etc/GMT-14\"</td></tr>"
463
	if [ "$1" = "debian" ]  ; then
464 465 466
		write_page "<tr><td>env LANG</td><td>LANG=\"C\"</td><td>on amd64: LANG=\"fr_CH.UTF-8\"<br />on i386: LANG=\"de_CH.UTF-8\"<br />on arm64: LANG=\"nl_BE.UTF-8\"<br />on armhf: LANG=\"it_CH.UTF-8\"</td></tr>"
		write_page "<tr><td>env LANGUAGE</td><td>LANGUAGE=\"en_US:en\"</td><td>on amd64: LANGUAGE=\"fr_CH:fr\"<br />on i386: LANGUAGE=\"de_CH:de\"<br />on arm64: LANGUAGE=\"nl_BE:nl\"<br />on armhf: LANGUAGE=\"it_CH:it\"</td></tr>"
		write_page "<tr><td>env LC_ALL</td><td><em>not set</em></td><td>on amd64: LC_ALL=\"fr_CH.UTF-8\"<br />on i386: LC_ALL=\"de_CH.UTF-8\"<br />on arm64: LC_ALL=\"nl_BE.UTF-8\"<br />on armhf: LC_ALL=\"it_CH.UTF-8\"</td></tr>"
467
	elif [ "$1" = "Arch Linux" ]  ; then
468 469
		write_page "<tr><td>env LANG</td><td><em>LANG=\"en_US.UTF-8\"</em></td><td>LANG=\"fr_CH.UTF-8\"</td></tr>"
		write_page "<tr><td>env LC_ALL</td><td><em>LANG=\"en_US.UTF-8\"</em></td><td>LC_ALL=\"fr_CH.UTF-8\"</td></tr>"
470
		write_page "<tr><td>the build path</td><td colspan=\"2\">is not yet varied between rebuilds of Arch Linux</td></tr>"
471
	else
472
		write_page "<tr><td>env LANG</td><td>LANG=\"en_GB.UTF-8\"</td><td>LANG=\"fr_CH.UTF-8\"</td></tr>"
473
		write_page "<tr><td>env LC_ALL</td><td><em>not set</em></td><td>LC_ALL=\"fr_CH.UTF-8\"</td></tr>"
474
	fi
475
	if [ "$1" != "FreeBSD" ] && [ "$1" != "Arch Linux" ]  ; then
476
		write_page "<tr><td>env PATH</td><td>PATH=\"/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:\"</td><td>PATH=\"/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path\"</td></tr>"
477 478
	elif [ "$1" = "Arch Linux" ]  ; then
		write_page "<tr><td>env PATH</td><td colspan=\"2\">is set to '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' because that's what <i>makechrootpkg</i> is using</td>"
479
	else
480
		write_page "<tr><td>env PATH</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
481
	fi
482 483
	if [ "$1" = "debian" ] ; then
		write_page "<tr><td>env BUILDUSERID</td><td>BUILDUSERID=\"1111\"</td><td>BUILDUSERID=\"2222\"</td></tr>"
484 485
		write_page "<tr><td>env BUILDUSERNAME</td><td>BUILDUSERNAME=\"pbuilder1\"</td><td>BUILDUSERNAME=\"pbuilder2\"</td></tr>"
		write_page "<tr><td>env USER</td><td>USER=\"pbuilder1\"</td><td>USER=\"pbuilder2\"</td></tr>"
486
		write_page "<tr><td>env HOME</td><td>HOME=\"/nonexistent/first-build\"</td><td>HOME=\"/nonexistent/second-build\"</td></tr>"
487
		write_page "<tr><td>niceness</td><td>10</td><td>11</td></tr>"
488 489
		write_page "<tr><td>uid</td><td>uid=1111</td><td>uid=2222</td></tr>"
		write_page "<tr><td>gid</td><td>gid=1111</td><td>gid=2222</td></tr>"
490
		write_page "<tr><td>/bin/sh</td><td>/bin/dash</td><td>/bin/bash</td></tr>"
491
		write_page "<tr><td><em><a href=\"https://wiki.debian.org/UsrMerge\">usrmerge</a></em> package installed</td><td>no</td><td>yes</td></tr>"
492
		write_page "<tr><td>build path</td><td>/build/1st/\$pkg-\$ver <em>(not varied for stretch/buster)</em></td><td>/build/\$pkg-\$ver/2nd <em>(not varied for stretch/buster)</em></td></tr>"
493
		write_page "<tr><td>user's login shell</td><td>/bin/sh</td><td>/bin/bash</td></tr>"
494
		write_page "<tr><td>user's <a href="https://en.wikipedia.org/wiki/Gecos_field">GECOS</a></td><td>first user,first room,first work-phone,first home-phone,first other</td><td>second user,second room,second work-phone,second home-phone,second other</td></tr>"
495
		write_page "<tr><td>env DEB_BUILD_OPTIONS</td><td>DEB_BUILD_OPTIONS=\"parallel=XXX\"<br />&nbsp;&nbsp;XXX on amd64: 16 or 15<br />&nbsp;&nbsp;XXX on i386: 10 or 9<br />&nbsp;&nbsp;XXX on armhf: 8, 4 or 2</td><td>DEB_BUILD_OPTIONS=\"parallel=YYY\"<br />&nbsp;&nbsp;YYY on amd64: 16 or 15 (!= the first build)<br />&nbsp;&nbsp;YYY on i386: 10 or 9 (!= the first build)<br />&nbsp;&nbsp;YYY is the same as XXX on arm64<br />&nbsp;&nbsp;YYY on armhf: 8, 4, or 2 (not varied systematically)</td></tr>"
496
		write_page "<tr><td>UTS namespace</td><td><em>shared with the host</em></td><td><em>modified using</em> /usr/bin/unshare --uts</td></tr>"
497 498 499 500
	elif [ "$1" = "Arch Linux" ]  ; then
		write_page "<tr><td>env USER</td><td>jenkins</td><td>build 2</td></tr>"
		write_page "<tr><td>user/uid</td><td>jenkins/103</td><td>build2/1235</td></tr>"
		write_page "<tr><td>group/gid</td><td>jenkins/105</td><td>build2/1235</td></tr>"
501
	else
502 503 504
		write_page "<tr><td>env USER</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		write_page "<tr><td>uid</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		write_page "<tr><td>gid</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
505 506 507 508 509
		if [ "$1" != "FreeBSD" ] ; then
			write_page "<tr><td>UTS namespace</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		fi
	fi
	if [ "$1" != "FreeBSD" ] ; then
510
		if [ "$1" = "debian" ] ; then
511 512 513 514 515
			write_page "<tr><td>kernel version</td></td><td>"
			for a in ${ARCHS} ; do
				write_page "<br />on $a one of:"
				write_page "$(cat /srv/reproducible-results/node-information/*$a* | grep KERNEL | cut -d '=' -f2- | sort -u | tr '\n' '\0' | xargs -0 -n1 echo '<br />&nbsp;&nbsp;')"
			done
516
			write_page "</td>"
517
			write_page "<td>on amd64 systematically varied, on armhf not systematically, on i386 and arm64 not at all<br />"
518 519 520 521
			for a in ${ARCHS} ; do
				write_page "<br />on $a one of:"
				write_page "$(cat /srv/reproducible-results/node-information/*$a* | grep KERNEL | cut -d '=' -f2- | sort -u | tr '\n' '\0' | xargs -0 -n1 echo '<br />&nbsp;&nbsp;')"
			done
522
			write_page "</td></tr>"
523
		elif [ "$1" != "Arch Linux" ]  ; then
524
			write_page "<tr><td>kernel version, modified using /usr/bin/linux64 --uname-2.6</td><td>$(uname -sr)</td><td>$(/usr/bin/linux64 --uname-2.6 uname -sr)</td></tr>"
525
		else
526 527 528
			write_page "<tr><td>kernel version</td>"
			write_page "$(cat /srv/reproducible-results/node-information/osuosl-build169* | grep KERNEL | cut -d '=' -f2- | sort -u | tr '\n' '\0' | xargs -0 -n1 echo '<br />&nbsp;&nbsp;')"
			write_page "<td colspan=\"2\"> is currently not varied between rebuilds of $1.</td></tr>"
529
		fi
530 531
		if [ "$1" != "OpenWrt" ] ; then
			write_page "<tr><td>umask</td><td>0022<td>0002</td></tr>"
532
		else
533
			write_page "<tr><td>umask</td><td colspan=\"2\">is always set to 0022 by the OpenWrt build system.</td></tr>"
534
		fi
535 536 537
	else
		write_page "<tr><td>FreeBSD kernel version</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		write_page "<tr><td>umask</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td><tr>"
538
	fi
539 540
	local TODAY=$(date +'%Y-%m-%d')
	local FUTURE=$(date --date="${TODAY}+398 days" +'%Y-%m-%d')
541
	if [ "$1" = "debian" ] ; then
542
		write_page "<tr><td>CPU type</td><td>one of: $(cat /srv/reproducible-results/node-information/* | grep CPU_MODEL | cut -d '=' -f2- | sort -u | tr '\n' '\0' | xargs -0 -n1 echo '<br />&nbsp;&nbsp;')</td><td>on i386: systematically varied (AMD or Intel CPU with different names & features)<br />on amd64: same for both builds<br />on arm64: always the same<br />on armhf: sometimes varied (depending on the build job), but only the minor CPU revision</td></tr>"
543
		write_page "<tr><td>year, month, date</td><td>today (${TODAY}) or (on amd64, i386 and arm64 only) also: $FUTURE</td><td>on amd64, i386 and arm64: varied (398 days difference)<br />on armhf: same for both builds (currently, work in progress)</td></tr>"
544 545
	else
		write_page "<tr><td>CPU type</td><td>$(cat /proc/cpuinfo|grep 'model name'|head -1|cut -d ":" -f2-)</td><td>same for both builds</td></tr>"
546 547 548 549 550
		if [ "$1" = "Arch Linux" ]; then
			write_page "<tr><td>/bin/sh</td><td>/bin/dash</td><td>/bin/bash</td></tr>"
		else
			write_page "<tr><td>/bin/sh</td><td colspan=\"2\"> is not yet varied between rebuilds of $1.</td></tr>"
		fi
551
		if [ "$1" != "FreeBSD" ] && [ "$1" != "Arch Linux" ] ; then
552
			write_page "<tr><td>year, month, date</td><td>today (${TODAY})</td><td>same for both builds (currently, work in progress)</td></tr>"
553
		elif [ "$1" = "Arch Linux" ] ; then
554
			write_page "<tr><td>year, month, date</td><td>osuosl-build169-amd64: today (${TODAY}) or osuosl-build170-amd64: 398 days in the future ($FUTURE)</td><td>the other one</td></tr>"
555
		else
556
			write_page "<tr><td>year, month, date</td><td>osuosl-build171-amd64: today (${TODAY}) or osuosl-build172-amd64: 398 days in the future ($FUTURE)</td><td>the other one</td></tr>"
557
		fi
558
	fi
559
	if [ "$1" != "FreeBSD" ] ; then
560
		if [ "$1" = "debian" ] ; then
561
			write_page "<tr><td>hour, minute</td><td>at least the minute will probably vary between two builds anyway...</td><td>on amd64, i386 and arm64 the \"future builds\" additionally run 6h and 23min ahead</td></tr>"
562
		        write_page "<tr><td>filesystem</td><td>tmpfs</td><td><em>temporarily not</em> varied using <a href=\"https://tracker.debian.org/disorderfs\">disorderfs</a> (<a href=\"https://sources.debian.org/src/disorderfs/sid/disorderfs.1.txt/\">manpage</a>)</td></tr>"
563
		else
564
			write_page "<tr><td>hour, minute</td><td>hour and minute will probably vary between two builds...</td><td>the future system actually runs 398 days, 6 hours and 23 minutes ahead...</td></tr>"
565 566
			write_page "<tr><td>Filesystem</td><td>tmpfs</td><td>same for both builds (currently, this could be varied using <a href=\"https://tracker.debian.org/disorderfs\">disorderfs</a>)</td></tr>"
		fi
567
	else
568
		write_page "<tr><td>year, month, date</td><td>today ($TODAY)</td><td>the 2nd build is done with the build node set 1 year, 1 month and 1 day in the future</td></tr>"
569
		write_page "<tr><td>hour, minute</td><td>hour and minute will vary between two builds</td><td>additionally the \"future build\" also runs 6h and 23min ahead</td></tr>"
570
		write_page "<tr><td>filesystem of the build directory</td><td>ufs</td><td>same for both builds</td></tr>"
571
	fi
572
	if [ "$1" = "debian" ] ; then
Chris Lamb's avatar
Chris Lamb committed
573
		write_page "<tr><td><em>everything else...</em></td><td colspan=\"2\">is likely the same. So far, this is just about the <em>potential</em> of <a href=\"https://wiki.debian.org/ReproducibleBuilds\">reproducible builds of Debian</a> - there will be more variations in the wild.</td></tr>"
574 575 576 577 578 579
	else
		write_page "<tr><td><em>everything else...</em></td><td colspan=\"2\">is likely the same. There will be more variations in the wild.</td></tr>"
	fi
	write_page "</table></p>"
}

580
publish_page() {
581 582 583 584
	if [ "$1" = "" ] ; then
		TARGET=$PAGE
	else
		TARGET=$1/$PAGE
585
	fi
586
	echo "$(date -u) - $(cp -v $PAGE $BASE/$TARGET)"
587
	rm $PAGE
588
	echo "$(date -u) - enjoy $REPRODUCIBLE_URL/$TARGET"
589 590
}

591
gen_package_html() {
592
	cd /srv/jenkins/bin
593
	python3 -c "import reproducible_html_packages as rep
594 595
from rblib.models import Package
pkg = Package('$1', no_notes=True)
596
rep.gen_packages_html([pkg], no_clean=True)" || echo "Warning: cannot update HTML pages for $1"
597
	cd - > /dev/null
598 599
}

600 601 602 603 604 605
calculate_build_duration() {
	END=$(date +'%s')
	DURATION=$(( $END - $START ))
}

print_out_duration() {
606 607 608
	if [ -z "$DURATION" ]; then
		return
	fi
609 610 611
	local HOUR=$(echo "$DURATION/3600"|bc)
	local MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc)
	local SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc)
Holger Levsen's avatar
Holger Levsen committed
612
	echo "$(date -u) - total duration: ${HOUR}h ${MIN}m ${SEC}s." | tee -a ${RBUILDLOG}
613 614
}

615
irc_message() {
616 617
	local CHANNEL="$1"
	shift
618
	local MESSAGE="$@"
619
	echo "Sending '$MESSAGE' to $CHANNEL now."
620
	kgb-client --conf /srv/jenkins/kgb/$CHANNEL.conf --relay-msg "$MESSAGE" || echo "$(date -u) - couldn't send message to $CHANNEL, continuing anyway." # don't fail the whole job
621 622
}

623
call_diffoscope() {
624 625 626 627
	mkdir -p $TMPDIR/$1/$(dirname $2)
	local TMPLOG=(mktemp --tmpdir=$TMPDIR)
	local msg=""
	set +e
628
	# remember to also modify the retry diffoscope call 15 lines below
629 630
	( ulimit -v "$DIFFOSCOPE_VIRT_LIMIT"
	  timeout "$TIMEOUT" nice schroot \
631
		--directory $TMPDIR \
632 633
		-c source:jenkins-reproducible-${DBDSUITE}-diffoscope \
		diffoscope -- \
634 635 636 637 638
			--html $TMPDIR/$1/$2.html \
			$TMPDIR/b1/$1/$2 \
			$TMPDIR/b2/$1/$2 2>&1 \
	) 2>&1 >> $TMPLOG
	RESULT=$?
Holger Levsen's avatar
Holger Levsen committed
639
	LOG_RESULT=$(grep '^E: 15binfmt: update-binfmts: unable to open' $TMPLOG || true)
Holger Levsen's avatar
Holger Levsen committed
640
	if [ ! -z "$LOG_RESULT" ] ; then
641
		rm -f $TMPLOG $TMPDIR/$1/$2.html
Holger Levsen's avatar
Holger Levsen committed
642
		echo "$(date -u) - schroot jenkins-reproducible-${DBDSUITE}-diffoscope not available, will sleep 2min and retry."
643 644
		sleep 2m
		# remember to also modify the retry diffoscope call 15 lines above
645 646
		( ulimit -v "$DIFFOSCOPE_VIRT_LIMIT"
		  timeout "$TIMEOUT" nice schroot \
647 648 649 650 651 652 653 654 655
			--directory $TMPDIR \
			-c source:jenkins-reproducible-${DBDSUITE}-diffoscope \
			diffoscope -- \
				--html $TMPDIR/$1/$2.html \
				$TMPDIR/b1/$1/$2 \
				$TMPDIR/b2/$1/$2 2>&1 \
			) 2>&1 >> $TMPLOG
		RESULT=$?
	fi
656 657 658 659 660 661 662 663
	if ! "$DEBUG" ; then set +x ; fi
	set -e
	cat $TMPLOG # print dbd output
	rm -f $TMPLOG
	case $RESULT in
		0)	echo "$(date -u) - $1/$2 is reproducible, yay!"
			;;
		1)
664
			echo "$(date -u) - $DIFFOSCOPE found issues, please investigate $1/$2"
665 666
			;;
		2)
667
			msg="$(date -u) - $DIFFOSCOPE had trouble comparing the two builds. Please investigate $1/$2"
668 669 670
			;;
		124)
			if [ ! -s $TMPDIR/$1.html ] ; then
671
				msg="$(date -u) - $DIFFOSCOPE produced no output for $1/$2 and was killed after running into timeout after ${TIMEOUT}..."
672
			else
673
				msg="$DIFFOSCOPE was killed after running into timeout after $TIMEOUT, but there is still $TMPDIR/$1/$2.html"
674 675 676
			fi
			;;
		*)
677 678 679 680 681
			# Process killed by signal exits with 128+${signal number}.
			# 31 = SIGSYS = maximum signal number in signal(7)
			if (( $RESULT > 128 )) && (( $RESULT <= 128+31 )); then
				RESULT="$RESULT (SIG$(kill -l $(($RESULT - 128))))"
			fi
682
			msg="$(date -u) - Something weird happened, $DIFFOSCOPE on $1/$2 exited with $RESULT and I don't know how to handle it."
683 684 685 686 687 688
			;;
	esac
	if [ ! -z "$msg" ] ; then
		echo $msg | tee -a $TMPDIR/$1/$2.html
	fi
}
689 690 691

get_filesize() {
		local BYTESIZE="$(du -h -b $1 | cut -f1)"
692 693
		# numbers below 16384K are understood and more meaningful than 16M...
		if [ $BYTESIZE -gt 16777216 ] ; then
694 695 696 697
			SIZE="$(echo $BYTESIZE/1048576|bc)M"
		elif [ $BYTESIZE -gt 1024 ] ; then
			SIZE="$(echo $BYTESIZE/1024|bc)K"
		else
698
			SIZE="$BYTESIZE bytes"
699 700
		fi
}
701

702
cleanup_pkg_files() {
703 704 705 706
	rm -vf $DEBIAN_BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_*.rbuild.log{,.gz}
	rm -vf $DEBIAN_BASE/logs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.build?.log{,.gz}
	rm -vf $DEBIAN_BASE/dbd/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.html
	rm -vf $DEBIAN_BASE/dbdtxt/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.txt{,.gz}
707
	rm -vf $DEBIAN_BASE/dbdjson/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.json{,.gz}
708 709
	rm -vf $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}/${SRCPACKAGE}_*.buildinfo
	rm -vf $DEBIAN_BASE/logdiffs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diff{,.gz}
710 711
}

712 713 714 715 716 717 718 719 720 721 722 723 724
handle_race_condition() {
	local RESULT=$(query_db "SELECT job FROM schedule WHERE package_id='$SRCPKGID'")
	local msg="Package ${SRCPACKAGE} (id=$SRCPKGID) in ${SUITE} on ${ARCH} is probably already building at $RESULT, while this is $BUILD_URL.\n"
	log_warning "$msg"
	printf "$(date -u) - $msg" >> /var/log/jenkins/reproducible-race-conditions.log
	log_warning "Terminating this build quickly and nicely..."
	if [ $SAVE_ARTIFACTS -eq 1 ] ; then
		SAVE_ARTIFACTS=0
		if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
	fi
	exit 0
}

725 726 727 728 729 730 731 732
unregister_build() {
	# unregister this build so it will immeditiatly tried again
	if [ -n "$SRCPKGID" ] ; then
		query_db "UPDATE schedule SET date_build_started = NULL, job = NULL WHERE package_id=$SRCPKGID"
	fi
	NOTIFY=""
}

733 734 735 736 737 738 739 740
handle_remote_error() {
	MESSAGE="${BUILD_URL}console.log got remote error $1"
	echo "$(date -u ) - $MESSAGE" | tee -a /var/log/jenkins/reproducible-remote-error.log
	echo "Sleeping 5m before aborting the job."
	sleep 5m
	exit 0
}

741
#
Holger Levsen's avatar
Holger Levsen committed
742
# create the png (and query the db to populate a csv file...) for Debian
743
#
Holger Levsen's avatar
Holger Levsen committed
744
create_debian_png_from_table() {
745 746 747 748 749 750
	echo "Checking whether to update $2..."
	# $1 = id of the stats table
	# $2 = image file name
	echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv
	# prepare query
	WHERE_EXTRA="WHERE suite = '$SUITE'"
751 752 753
	if [ "$ARCH" = "armhf" ] ; then
		# armhf was only build since 2015-08-30
		WHERE2_EXTRA="WHERE s.datum >= '2015-08-30'"
754 755 756
	elif [ "$ARCH" = "i386" ] ; then
		# i386 was only build since 2016-03-28
		WHERE2_EXTRA="WHERE s.datum >= '2016-03-28'"
757 758 759
	elif [ "$ARCH" = "arm64" ] ; then
		# arm63 was only build since 2016-12-23
		WHERE2_EXTRA="WHERE s.datum >= '2016-12-23'"
760 761 762
	else
		WHERE2_EXTRA=""
	fi
763 764
	if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] || [ $1 -eq 8 ] ; then
		# TABLE[3+4+5] don't have a suite column: (and TABLE[8] (and 9) is faked, based on 3)
765 766
		WHERE_EXTRA=""
	fi
767 768
	if [ $1 -eq 0 ] || [ $1 -eq 2 ] ; then
		# TABLE[0+2] have a architecture column:
769
		WHERE_EXTRA="$WHERE_EXTRA AND architecture = '$ARCH'"
770 771
		if [ "$ARCH" = "armhf" ]  ; then
			if [ $1 -eq 2 ] ; then
772
				# unstable/armhf was only build since 2015-08-30 (and experimental/armhf since 2015-12-19 and stretch/armhf since 2016-01-01)
773 774
				WHERE_EXTRA="$WHERE_EXTRA AND datum >= '2015-08-30'"
			fi
775 776 777 778 779
		elif [ "$ARCH" = "i386" ]  ; then
			if [ $1 -eq 2 ] ; then
				# i386 was only build since 2016-03-28
				WHERE_EXTRA="$WHERE_EXTRA AND datum >= '2016-03-28'"
			fi
780 781 782 783 784
		elif [ "$ARCH" = "arm64" ]  ; then
			if [ $1 -eq 2 ] ; then
				# arm64 was only build since 2016-12-23
				WHERE_EXTRA="$WHERE_EXTRA AND datum >= '2016-12-23'"
			fi
785
		fi
786
		# stretch/amd64 was only build since...
787 788 789
		# WHERE2_EXTRA="WHERE s.datum >= '2015-03-08'"
		# experimental/amd64 was only build since...
		# WHERE2_EXTRA="WHERE s.datum >= '2015-02-28'"
790
	fi
791 792 793
	# run query
	if [ $1 -eq 1 ] ; then
		# not sure if it's worth to generate the following query...
794
		WHERE_EXTRA="AND architecture='$ARCH'"
795

796
		# This query became much more obnoxious when gaining
797 798
		# compatibility with postgres
		query_to_csv "SELECT stats.datum,
799
			 COALESCE(reproducible_stretch,0) AS reproducible_stretch,
800
			 COALESCE(reproducible_buster,0) AS reproducible_buster,
801 802
			 COALESCE(reproducible_unstable,0) AS reproducible_unstable,
			 COALESCE(reproducible_experimental,0) AS reproducible_experimental,
803 804 805 806
			 COALESCE(FTBR_stretch,0) AS FTBR_stretch,
			 COALESCE(FTBR_buster,0) AS FTBR_buster,
			 COALESCE(FTBR_unstable,0) AS FTBR_unstable,
			 COALESCE(FTBR_experimental,0) AS FTBR_experimental,
807
			 COALESCE(FTBFS_stretch,0) AS FTBFS_stretch,
808
			 COALESCE(FTBFS_buster,0) AS FTBFS_buster,
809 810
			 COALESCE(FTBFS_unstable,0) AS FTBFS_unstable,
			 COALESCE(FTBFS_experimental,0) AS FTBFS_experimental,
811
			 COALESCE(other_stretch,0) AS other_stretch,
812
			 COALESCE(other_buster,0) AS other_buster,
813 814 815
			 COALESCE(other_unstable,0) AS other_unstable,
			 COALESCE(other_experimental,0) AS other_experimental
			FROM (SELECT s.datum,
816
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA),0) AS reproducible_stretch,
817
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA),0) AS reproducible_buster,
818 819
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS reproducible_unstable,
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS reproducible_experimental,
820 821 822 823
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBR_stretch,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBR_buster,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBR_unstable,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBR_experimental,
824
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBFS_stretch,
825
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBFS_buster,
826 827
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBFS_unstable,
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBFS_experimental,
828
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS other_stretch,
829
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS other_buster,
830 831
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS other_unstable,
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS other_experimental
832 833
			 FROM stats_builds_per_day AS s $WHERE2_EXTRA GROUP BY s.datum) as stats
			ORDER BY datum" >> ${TABLE[$1]}.csv
834
	elif [ $1 -eq 2 ] ; then
835
		# just make a graph of the oldest reproducible build (ignore FTBFS and FTBR)
836
		query_to_csv "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
837
	elif [ $1 -eq 7 ] ; then
838
		query_to_csv "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv
839
	elif [ $1 -eq 8 ] ; then
840
		query_to_csv "SELECT ${FIELDS[$1]} from ${TABLE[3]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
841
	elif [ $1 -eq 9 ] ; then
842
		query_to_csv "SELECT datum, $REPRODUCIBLE_DONE, $REPRODUCIBLE_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv
843
	else
844
		query_to_csv "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
845 846 847 848 849 850
	fi
	# this is a gross hack: normally we take the number of colors a table should have...
	#  for the builds_age table we only want one color, but different ones, so this hack:
	COLORS=${COLOR[$1]}
	if [ $1 -eq 2 ] ; then
		case "$SUITE" in
851
			stretch)	COLORS=40 ;;
852 853 854
			buster)		COLORS=41 ;;
			unstable)	COLORS=42 ;;
			experimental)	COLORS=43 ;;
855 856
		esac
	fi
857 858
	local WIDTH=1920
	local HEIGHT=960
859 860 861 862 863 864
	# only generate graph if the query returned data
	if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then
		echo "Updating $2..."
		DIR=$(dirname $2)
		mkdir -p $DIR
		echo "Generating $2."
865
		/srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" $WIDTH $HEIGHT
866
		mv $2 $DEBIAN_BASE/$DIR
867 868
		[ "$DIR" = "." ] || rmdir $(dirname $2)
	# create empty dummy png if there havent been any results ever
869
	elif [ ! -f $DEBIAN_BASE/$DIR/$(basename $2) ] ; then
870 871 872 873
		DIR=$(dirname $2)
		mkdir -p $DIR
		echo "Creating $2 dummy."
		convert -size 1920x960 xc:#aaaaaa -depth 8 $2
874
		mv $2 $DEBIAN_BASE/$DIR
875 876 877 878
		[ "$DIR" = "." ] || rmdir $(dirname $2)
	fi
	rm ${TABLE[$1]}.csv
}
879

880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977
#
# create the png (and query the db to populate a csv file...) for Arch Linux
#
create_archlinux_png_from_table() {
	echo "Checking whether to update $2..."
	# $1 = id of the stats table
	# $2 = image file name
	echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv
	# prepare query
	WHERE_EXTRA="WHERE suite = '$SUITE'"
	if [ $1 -eq 0 ] || [ $1 -eq 2 ] ; then
		# TABLE[0+2] have a architecture column:
		WHERE_EXTRA="$WHERE_EXTRA AND architecture = '$ARCH'"
	fi
	# run query
	if [ $1 -eq 1 ] ; then
		# not sure if it's worth to generate the following query...
		WHERE_EXTRA="AND architecture='$ARCH'"

		# This query became much more obnoxious when gaining
		# compatibility with postgres
		query_to_csv "SELECT stats.datum,
			 COALESCE(reproducible_stretch,0) AS reproducible_stretch,
			 COALESCE(reproducible_buster,0) AS reproducible_buster,
			 COALESCE(reproducible_unstable,0) AS reproducible_unstable,
			 COALESCE(reproducible_experimental,0) AS reproducible_experimental,
			 COALESCE(FTBR_stretch,0) AS FTBR_stretch,
			 COALESCE(FTBR_buster,0) AS FTBR_buster,
			 COALESCE(FTBR_unstable,0) AS FTBR_unstable,
			 COALESCE(FTBR_experimental,0) AS FTBR_experimental,
			 COALESCE(FTBFS_stretch,0) AS FTBFS_stretch,
			 COALESCE(FTBFS_buster,0) AS FTBFS_buster,
			 COALESCE(FTBFS_unstable,0) AS FTBFS_unstable,
			 COALESCE(FTBFS_experimental,0) AS FTBFS_experimental,
			 COALESCE(other_stretch,0) AS other_stretch,
			 COALESCE(other_buster,0) AS other_buster,
			 COALESCE(other_unstable,0) AS other_unstable,
			 COALESCE(other_experimental,0) AS other_experimental
			FROM (SELECT s.datum,
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA),0) AS reproducible_stretch,
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA),0) AS reproducible_buster,
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS reproducible_unstable,
			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS reproducible_experimental,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBR_stretch,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBR_buster,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBR_unstable,
			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBR_experimental,
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBFS_stretch,
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBFS_buster,
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBFS_unstable,
			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBFS_experimental,
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS other_stretch,
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS other_buster,
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS other_unstable,
			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS other_experimental
			 FROM stats_builds_per_day AS s GROUP BY s.datum) as stats
			ORDER BY datum" >> ${TABLE[$1]}.csv
	elif [ $1 -eq 2 ] ; then
		# just make a graph of the oldest reproducible build (ignore FTBFS and FTBR)
		query_to_csv "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
	else
		query_to_csv "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
	fi
	# this is a gross hack: normally we take the number of colors a table should have...
	#  for the builds_age table we only want one color, but different ones, so this hack:
	COLORS=${COLOR[$1]}
	if [ $1 -eq 2 ] ; then
		case "$SUITE" in
			stretch)	COLORS=40 ;;
			buster)		COLORS=41 ;;
			unstable)	COLORS=42 ;;
			experimental)	COLORS=43 ;;
		esac
	fi
	local WIDTH=1920
	local HEIGHT=960
	# only generate graph if the query returned data
	if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then
		echo "Updating $2..."
		DIR=$(dirname $2)
		mkdir -p $DIR
		echo "Generating $2."
		/srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" $WIDTH $HEIGHT
		mv $2 $ARCHBASE/$DIR
		[ "$DIR" = "." ] || rmdir $(dirname $2)
	# create empty dummy png if there havent been any results ever
	elif [ ! -f $ARCHBASE/$DIR/$(basename $2) ] ; then
		DIR=$(dirname $2)
		mkdir -p $DIR
		echo "Creating $2 dummy."
		convert -size 1920x960 xc:#aaaaaa -depth 8 $2
		mv $2 $ARCHBASE/$DIR
		[ "$DIR" = "." ] || rmdir $(dirname $2)
	fi
	rm ${TABLE[$1]}.csv
}


978 979 980 981 982
find_in_buildlogs() {
    egrep -q "$1" $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null
}

include_icon(){
983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
	local STATE=$1
	local TEXT=$2
	local ALT=${STATE%%_*}
	local PNG=
	ALT=${ALT,,}
	case $STATE in
		BLACKLISTED)
			PNG=error;;
		DEPWAIT_*)
			PNG=weather-snow;;
		404_*)
			PNG=weather-severe-alert;;
		FTBFS_*)
			PNG=weather-storm;;
		FTBR_*)
			PNG=weather-showers-scattered ALT=unreproducible ;;
999
		reproducible)
1000
			PNG=weather-clear ALT=reproducible ;;
1001 1002 1003 1004 1005 1006
	esac
	echo "       <img src=\"/userContent/static/$PNG.png\" alt=\"$ALT icon\" /> $TEXT" >> $HTML_BUFFER
}

create_pkg_html() {
	local ARCHLINUX_PKG_PATH=$ARCHBASE/$REPOSITORY/$SRCPACKAGE
1007 1008 1009
	local HTML_BUFFER=$(mktemp -t archlinuxrb-html-XXXXXXXX)
	local buffer_message
	local STATE
1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027

	# clear files from previous builds
	cd "$ARCHLINUX_PKG_PATH"
	for file in build1.log build2.log build1.version build2.version *BUILDINFO.txt *.html; do
		if [ -f $file ] && [ pkg.build_duration -nt $file ] ; then
			rm $file
			echo "$ARCHLINUX_PKG_PATH/$file older than $ARCHLINUX_PKG_PATH/pkg.build_duration, thus deleting it."
		fi
	done

	echo "     <tr>" >> $HTML_BUFFER
	echo "      <td>$REPOSITORY</td>" >> $HTML_BUFFER
	echo "      <td>$SRCPACKAGE</td>" >> $HTML_BUFFER
	echo "      <td>$VERSION</td>" >> $HTML_BUFFER
	echo "      <td>" >> $HTML_BUFFER
	#
	#
	if [ -z "$(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
1028 1029
		# this horrible if elif elif elif elif...  monster should be replaced
		# by using pacman's exit code which is possible since sometime in 2018
1030

1031
		# check different states and figure out what the page should look like
1032
		if find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then
1033 1034
			STATE=DEPWAIT_0
			buffer_message='could not resolve dependencies as there are conflicts'
1035 1036
		elif find_in_buildlogs '==> ERROR: (Could not resolve all dependencies|.pacman. failed to install missing dependencies)'; then
			if find_in_buildlogs 'error: failed to init transaction \(unable to lock database\)'; then
1037 1038
				STATE=DEPWAIT_2
				buffer_message='pacman could not lock database'
1039
			else
1040 1041
				STATE=DEPWAIT_1
				buffer_message='could not resolve dependencies'
1042 1043
			fi
		elif find_in_buildlogs '^error: unknown package: '; then
1044 1045
			STATE=404_0
			buffer_message='unknown package'
1046 1047 1048
		elif find_in_buildlogs '(==> ERROR: Failure while downloading|==> ERROR: One or more PGP signatures could not be verified|==> ERROR: One or more files did not pass the validity check|==> ERROR: Integrity checks \(.*\) differ in size from the source array|==> ERROR: Failure while branching|==> ERROR: Failure while creating working copy|Failed to source PKGBUILD.*PKGBUILD)'; then
			REASON="download failed"
			EXTRA_REASON=""
1049
			STATE=404_0
1050
			if find_in_buildlogs 'FAILED \(unknown public key'; then
1051
				STATE=404_6
1052 1053
				EXTRA_REASON="to verify source with PGP due to unknown public key"
			elif find_in_buildlogs 'The requested URL returned error: 403'; then
1054
				STATE=404_2
1055 1056
				EXTRA_REASON="with 403 - forbidden"
			elif find_in_buildlogs 'The requested URL returned error: 500'; then
1057
				STATE=404_4
1058 1059
				EXTRA_REASON="with 500 - internal server error"
			elif find_in_buildlogs 'The requested URL returned error: 503'; then
1060
				STATE=404_5
1061 1062
				EXTRA_REASON="with 503 - service unavailable"
			elif find_in_buildlogs '==> ERROR: One or more PGP signatures could not be verified'; then
1063
				STATE=404_7
1064 1065
				EXTRA_REASON="to verify source with PGP signatures"
			elif find_in_buildlogs '(SSL certificate problem: unable to get local issuer certificate|^bzr: ERROR: .SSL: CERTIFICATE_VERIFY_FAILED)'; then
1066
				STATE=404_1
1067 1068
				EXTRA_REASON="with SSL problem"
			elif find_in_buildlogs '==> ERROR: One or more files did not pass the validity check'; then
1069
				STATE=404_8
1070 1071
				REASON="downloaded ok but failed to verify source"
			elif find_in_buildlogs '==> ERROR: Integrity checks \(.*\) differ in size from the source array'; then
1072
				STATE=404_9
1073 1074
				REASON="Integrity checks differ in size from the source array"
			elif find_in_buildlogs 'The requested URL returned error: 404'; then
1075
				STATE=404_3
1076 1077
				EXTRA_REASON="with 404 - file not found"
			elif find_in_buildlogs 'fatal: the remote end hung up unexpectedly'; then
1078
				STATE=404_A
1079 1080
				EXTRA_REASON="could not clone git repository"
			elif find_in_buildlogs 'The requested URL returned error: 504'; then
1081
				STATE=404_B
1082 1083
				EXTRA_REASON="with 504 - gateway timeout"
			elif find_in_buildlogs '==> ERROR: Failure while downloading .* git repo'; then
1084
				STATE=404_C
1085 1086
				EXTRA_REASON="from git repo"
			fi
1087
			buffer_message="$REASON $EXTRA_REASON"
1088
		elif find_in_buildlogs '==> ERROR: (install file .* does not exist or is not a regular file|The download program wget is not installed)'; then
1089 1090
			STATE=FTBFS_0
			buffer_message='failed to build, requirements not met'
1091
		elif find_in_buildlogs '==> ERROR: A failure occurred in check'; then
1092 1093
			STATE=FTBFS_1
			buffer_message='failed to build while running tests'
1094
		elif find_in_buildlogs '==> ERROR: (An unknown error has occurred|A failure occurred in (build|package|prepare))'; then
1095 1096
			STATE=FTBFS_2
			buffer_message='failed to build'
1097
		elif find_in_buildlogs 'makepkg was killed by timeout after'; then
1098 1099
			STATE=FTBFS_3
			buffer_message='failed to build, killed by timeout'
1100
		elif find_in_buildlogs '==> ERROR: .* contains invalid characters:'; then
1101 1102
			STATE=FTBFS_4
			buffer_message='failed to build, pkg relations contain invalid characters'
1103
		else