reproducible_build.sh 36 KB
Newer Older
1
#!/bin/bash
2
# vim: set noexpandtab:
3

Holger Levsen's avatar
Holger Levsen committed
4
# Copyright 2014-2019 Holger Levsen <holger@layer-acht.org>
5
#         © 2015-2018 Mattia Rizzolo <mattia@debian.org>
6 7
# released under the GPLv=2

8 9 10 11 12 13
# Exit status of this script:
# - when run in remote mode:
#   - 2 → the build failed
#   - 3 → the build timeouted
#   - 404 → failed to download the sources

14
DEBUG=false
15 16 17
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

Holger Levsen's avatar
Holger Levsen committed
18 19
# common code defining db access
. /srv/jenkins/bin/reproducible_common.sh
20

21 22
set -e

23
exit_early_if_debian_is_broken() {
24
	# debian is fine, thanks
25 26 27
	if false && [ "$ARCH" = "armhf" ] ; then
		echo "Temporarily stopping the builds on armhf due to #827724… sleeping 12h now…"
		for i in $(seq 1 12) ; do
28
			sleep 1h
Holger Levsen's avatar
Holger Levsen committed
29
			echo "one hour passed…."
30 31 32 33 34
		done
		exit 0
	fi
}

35
create_results_dirs() {
36 37
	mkdir -vp $DEBIAN_BASE/dbd/${SUITE}/${ARCH}
	mkdir -vp $DEBIAN_BASE/dbdtxt/${SUITE}/${ARCH}
38
	mkdir -vp $DEBIAN_BASE/dbdjson/${SUITE}/${ARCH}
39 40 41 42
	mkdir -vp $DEBIAN_BASE/logs/${SUITE}/${ARCH}
	mkdir -vp $DEBIAN_BASE/logdiffs/${SUITE}/${ARCH}
	mkdir -vp $DEBIAN_BASE/rbuild/${SUITE}/${ARCH}
	mkdir -vp $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}
43
}
44

45
save_artifacts() {
46
		local random=$(head /dev/urandom | tr -cd '[:alnum:]'| head -c5)
47
		local ARTIFACTS="artifacts/r00t-me/${SRCPACKAGE}_${SUITE}_${ARCH}_tmp-${random}"
48
		local URL="$DEBIAN_URL/$ARTIFACTS/"
49 50
		local HEADER="$DEBIAN_BASE/$ARTIFACTS/.HEADER.html"
		mkdir -p $DEBIAN_BASE/$ARTIFACTS
51
		cp -r $TMPDIR/* $DEBIAN_BASE/$ARTIFACTS/ || true
52
		local msg="Artifacts from this build have been preserved. They will be available for 24h only, so download them now.\n"
53
		msg="${msg}WARNING: You shouldn't trust packages downloaded from this host, they can contain malware or the worst of your fears, packaged nicely in debian format.\n"
54
		msg="${msg}If you are aware of this and just want to use these artifacts to investigate why $DIFFOSCOPE had issues, you can download the artifacts from the following location: $URL\n"
55
		log_info "$msg"
56 57
		echo "<p>" > $HEADER
		printf "$msg" | sed 's#$#<br />#g' >> $HEADER
58
		echo "Package page: <a href=\"$DEBIAN_URL/${SUITE}/${ARCH}/${SRCPACKAGE}\">$DEBIAN_URL/${SUITE}/${ARCH}/${SRCPACKAGE}</a><br />" >> $HEADER
59
		echo "</p>" >> $HEADER
60 61
		chmod 644 $HEADER
		# irc message
62 63
		if [ ! -z "$NOTIFY" ] ; then
			local MESSAGE="Artifacts for ${SRCPACKAGE}, $STATUS in ${SUITE}/${ARCH}: $URL"
64 65
			if [ "$NOTIFY" = "diffoscope_err" ] ; then
				irc_message debian-reproducible-changes "$DEBIAN_URL/$SUITE/$ARCH/$SRCPACKAGE $STATUS and $DIFFOSCOPE failed"
66 67 68 69 70 71 72
				irc_message debian-reproducible-changes "$MESSAGE (error running $DIFFOSCOPE)"
				MESSAGE="$MESSAGE (error running $DIFFOSCOPE)"
			else
				# somebody explicitly asked for artifacts, so give them the artifacts
				irc_message debian-reproducible "$MESSAGE"
			fi
		fi
73 74
}

75
notification() {
76
	if [ "$SAVE_ARTIFACTS" = "1" ] ; then
77
		save_artifacts  # this will also notify IRC as needed
78
	else
79
		case "$NOTIFY" in  # the diffoscope_err case is handled by save_artifacts()
80
			''|0) ;;
81 82 83
			1|2)
				irc_message debian-reproducible "$DEBIAN_URL/$SUITE/$ARCH/$SRCPACKAGE done: $STATUS"
				;;
84 85
			diffoscope_timeout)
				irc_message debian-reproducible-changes "$DEBIAN_URL/$SUITE/$ARCH/$SRCPACKAGE $STATUS and $DIFFOSCOPE timed out"
86 87 88
				;;
			*)
				# a weird value of $NOTIFY that we don't know about
89
				irc_message debian-reproducible-changes "$DEBIAN_URL/$SUITE/$ARCH/$SRCPACKAGE done: '$STATUS' debug: '$NOTIFY'"
90 91
				;;
		esac
92
	fi
93
	# final compress, the RBUILDLOG is already in place.
94
	[ ! -f $RBUILDLOG ] || gzip -9fvn $RBUILDLOG
95 96 97 98 99 100 101 102
	# XXX quite ugly: this is just needed to update the sizes of the
	# compressed files in the html. It's cheap and quite safe so, *shrugs*...
	gen_package_html $SRCPACKAGE
}

cleanup_all() {
	echo "Starting cleanup."
	cd  # move out of $TMPDIR, if we are still inside
103
	if [ "$MODE" = "master" ] ; then
104 105 106
		notification
		# the TMPDIR in the remote nodes is removed by a later ssh call
		# from master after the artifacts have been copied out.
107
		rm -r $TMPDIR || true
108
	fi
109
	echo "All cleanup done."
110
	print_out_duration
111 112
}

113
update_db_and_html() {
114 115
	#
	# save everything as status of this package in the db
116
	#
117
	STATUS="$@"
118 119
	local OLD_STATUS=$(query_db "SELECT status FROM results WHERE package_id='${SRCPKGID}'" || \
			   query_db "SELECT status FROM results WHERE package_id='${SRCPKGID}'")
120
	# irc+mail notifications for changing status in unstable and experimental
121
	if [ "$SUITE" = "unstable" ] || [ "$SUITE" = "experimental" ] ; then
122
		if ([ "$OLD_STATUS" = "reproducible" ] && ( [ "$STATUS" = "FTBR" ] || [ "$STATUS" = "FTBFS" ] || [ "$STATUS" = "timeout" ])) || \
123
			([ "$OLD_STATUS" = "FTBR" ] && [ "$STATUS" = "FTBFS" ] ); then
124
			MESSAGE="${DEBIAN_URL}/${SUITE}/${ARCH}/${SRCPACKAGE} : ${OLD_STATUS}${STATUS}"
125
			log_info "$MESSAGE"
126
			irc_message debian-reproducible-changes "$MESSAGE"
127
		fi
128 129
		if [ "$OLD_STATUS" != "$STATUS" ] && [ "$NOTIFY_MAINTAINER" -eq 1 ] && \
		  [ "$OLD_STATUS" != "depwait" ] && [ "$STATUS" != "depwait" ] && \
130
		  [ "$OLD_STATUS" != "E404" ] && [ "$STATUS" != "E404" ]; then
131 132
			# spool notifications and mail them once a day
			mkdir -p /srv/reproducible-results/notification-emails
133
			echo "$(date -u +'%Y-%m-%d %H:%M') $DEBIAN_URL/$SUITE/$ARCH/$SRCPACKAGE changed from $OLD_STATUS -> $STATUS" >> /srv/reproducible-results/notification-emails/$SRCPACKAGE
134
		fi
135
	fi
136
	# Insert or update existing entry in results table
137
	query_db "INSERT INTO results (package_id, version, status, build_date, build_duration, node1, node2, job) VALUES ('$SRCPKGID', '$VERSION', '$STATUS', '$DATE', '$DURATION', '$NODE1', '$NODE2', '$JOB') ON CONFLICT (package_id) DO UPDATE SET version='$VERSION', status='$STATUS', build_date='$DATE', build_duration='$DURATION', node1='$NODE1', node2='$NODE2', job='$JOB' WHERE results.package_id='$SRCPKGID'"
138
	query_db "INSERT INTO stats_build (name, version, suite, architecture, status, build_date, build_duration, node1, node2, job) VALUES ('$SRCPACKAGE', '$VERSION', '$SUITE', '$ARCH', '$STATUS', '$DATE', '$DURATION', '$NODE1', '$NODE2', '$JOB')"
139
	# unmark build since it's properly finished
140
	query_db "DELETE FROM schedule WHERE package_id='$SRCPKGID';"
141
	gen_package_html $SRCPACKAGE
142
	echo
143
	echo "$(date -u) - successfully updated the database and updated $DEBIAN_URL/rb-pkg/${SUITE}/${ARCH}/$SRCPACKAGE.html"
144
	echo
145 146
}

Holger Levsen's avatar
Holger Levsen committed
147 148
update_rbuildlog() {
	chmod 644 $RBUILDLOG
149 150
	mv $RBUILDLOG $DEBIAN_BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log
	RBUILDLOG=$DEBIAN_BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log
Holger Levsen's avatar
Holger Levsen committed
151 152
}

153
diff_copy_buildlogs() {
154
	local DIFF="$DEBIAN_BASE/logdiffs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.diff"
155 156 157
	if [ -f b1/build.log ] ; then
		if [ -f b2/build.log ] ; then
			printf "Diff of the two buildlogs:\n\n--\n" | tee -a $DIFF
158 159
			LOCALDIFFTIMEOUT="30m"
			timeout $LOCALDIFFTIMEOUT diff -u b1/build.log b2/build.log | tee -a $DIFF
160 161
			if [ ${PIPESTATUS[0]} -eq 0 ] ; then
				echo "The two build logs are identical! \o/" | tee -a $DIFF
162 163
			elif [ ${PIPESTATUS[0]} -eq 124 ] ; then
				echo "Diffing the two build logs ran into timeout after $LOCALDIFFTIMEOUT, sorry." | tee -a $DIFF
164
			fi
165
			echo -e "\nCompressing the 2nd log..."
166
			gzip -9vn $DIFF
167 168
			gzip -9cvn b2/build.log > $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build2.log.gz
			chmod 644 $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build2.log.gz
169
		elif [ "${1:-}" = "ftbfs" ] ; then
170
			log_warning "No second build log, what happened?"
171
		fi
172
		echo "Compressing the 1st log..."
173 174
		gzip -9cvn b1/build.log > $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build1.log.gz
		chmod 644 $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build1.log.gz
175
	else
176
		log_error "No first build log, not even looking for the second"
177 178 179
	fi
}

180
handle_E404() {
181 182 183
	log_warning "Download of ${SRCPACKAGE} sources from ${SUITE} failed."
	ls -l ${SRCPACKAGE}* | log_file -
	log_warning "Maybe there was a network problem, or ${SRCPACKAGE} is not a source package in ${SUITE}, or it was removed or renamed. Please investigate. Sleeping 30m as this should not happen."
184
	DURATION=0
Holger Levsen's avatar
Holger Levsen committed
185
	update_rbuildlog
186
	update_db_and_html "E404"
187 188
	if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=0 ; fi
	if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
189
	sleep 30m
190
	exit 0 # RBUILDLOG and SAVE_ARTIFACTS and NOTIFY are used in cleanup_all called at exit
191 192
}

193
handle_depwait() {
194 195 196
	log_warning "Downloading the build dependencies failed"
	log_warning "Maybe there was a network problem, or the build dependencies are currently uninstallable; consider filing a bug in the last case."
	log_warning "Network problems are automatically rescheduled after some hours."
197 198 199 200
	calculate_build_duration
	update_db_and_html "depwait"
	if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=0 ; fi
	if [ -n "$NOTIFY" ] ; then NOTIFY="depwait" ; fi
201
	exit 0
202 203
}

204
handle_NFU() {
205
	# a list of valid architecture for this package should be passed to this function
206
	log_info "Package ${SRCPACKAGE} (${VERSION}) shall only be build on \"$(echo "$@" | xargs echo )\" and thus was skipped."
207
	DURATION=0
Holger Levsen's avatar
Holger Levsen committed
208
	update_rbuildlog
209
	update_db_and_html "NFU"
210 211
	if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=0 ; fi
	if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
212
	exit 0 # RBUILDLOG and SAVE_ARTIFACTS and NOTIFY are used in cleanup_all called at exit
213 214
}

215 216 217 218 219 220 221 222 223 224 225 226
handle_timeout() {
	echo "${SRCPACKAGE} build timed out"
	cleanup_pkg_files
	diff_copy_buildlogs
	update_rbuildlog
	calculate_build_duration
	update_db_and_html "timeout"
	if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=0 ; fi
	if [ ! -z "$NOTIFY" ] ; then NOTIFY="timeout" ; fi
	exit 0
}

227 228
handle_ftbfs() {
	echo "${SRCPACKAGE} failed to build from source."
229
	cleanup_pkg_files
230
	diff_copy_buildlogs ftbfs
231 232
	update_rbuildlog
	local BUILD NEEDLE
233
	for BUILD in "1" "2"; do
234 235
		local nodevar="NODE$BUILD"
		local node=""
236
		# eval is needed here, because we want the value of $node1 or $node2
237
		eval node=\$$nodevar
238
		if [ ! -f "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ] ; then
239 240
			continue
		fi
241
		if zgrep -q -F "E: pbuilder-satisfydepends failed." "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then
242 243
			handle_depwait
		fi
244 245 246 247 248 249 250 251 252
		for NEEDLE in \
			'^tar:.*Cannot write: No space left on device' \
			'fatal error: error writing to .* No space left on device' \
			'./configure: line .* printf: write error: No space left on device' \
			'cat: write error: No space left on device' '^dpkg-deb.*No space left on device' \
			'^cp: (erreur|impossible).*No space left on device' '^tee: .* No space left on device' \
			'^zip I/O error: No space left on device' \
			'^mkdir .*: No space left on device' \
			'exceeds available storage space.*\(No space left on device\)$' \
253
			'^dpkg-source: error: cannot create directory .* No space left on device$' \
254
			'Requested size .* exceeds available storage space .*\(No space left on device\)$' \
255
			; do
256
			if zgrep -q -e "$NEEDLE" "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then
257
				handle_enospace $node
258 259
			fi
		done
260
		# notify about unkown diskspace issues where we are not 100% sure yet those are diskspace issues
261 262
		# ignore syslinux, clisp, klibc and glibc which are false positives…
		if zgrep -e "No space left on device" "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" && [ "$SRCPACKAGE" != "syslinux" ] && [ "$SRCPACKAGE" != "clisp" ] && [ "$SRCPACKAGE" != "klibc" ] && [ "$SRCPACKAGE" != "glibc" ]; then
263
			MESSAGE="${BUILD_URL}console.log for ${SRCPACKAGE} (ftbfs in $SUITE/$ARCH) _probably_ had a diskspace issue on $node. Please check, tune handle_ftbfs() and reschedule the package."
264
			echo $MESSAGE | tee -a /var/log/jenkins/reproducible-diskspace-issues.log
265
			irc_message debian-reproducible "$MESSAGE"
266
		fi
267
	done
268
	calculate_build_duration
269
	update_db_and_html "FTBFS"
270 271
	if [ $SAVE_ARTIFACTS -eq 1 ] ; then SAVE_ARTIFACTS=0 ; fi
	if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
272
	exit 0
273 274
}

275
handle_ftbr() {
276 277
	# a ftbr explaination message could be passed
	local FTBRmessage="$@"
278
	log_error "${SRCPACKAGE} failed to build reproducibly in ${SUITE} on ${ARCH}."
279
	cp b1/${BUILDINFO} $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}/ > /dev/null 2>&1 || true  # will fail if there is no .buildinfo
280
	if [ ! -z "$FTRmessage" ] ; then
281
		log_error "${FTBRmessage}."
282
	fi
283
	if [ -f ./${DBDREPORT} ] ; then
284
		mv ./${DBDREPORT} $DEBIAN_BASE/dbd/${SUITE}/${ARCH}/
285
	else
286
		log_warning "$DIFFOSCOPE produced no output (which is strange)."
287
	fi
288
	if [ -f ./$DBDTXT ] ; then
289 290
		mv ./$DBDTXT $DEBIAN_BASE/dbdtxt/$SUITE/$ARCH/
		gzip -9n $DEBIAN_BASE/dbdtxt/$SUITE/$ARCH/$DBDTXT
291
	fi
292 293 294 295
	if [ -f ./$DBDJSON ] ; then
		mv ./$DBDJSON $DEBIAN_BASE/dbdjson/$SUITE/$ARCH/
		gzip -9n $DEBIAN_BASE/dbdjson/$SUITE/$ARCH/$DBDJSON
	fi
296
	calculate_build_duration
297
	update_db_and_html "FTBR"
298 299
}

300 301
handle_reproducible() {
	if [ ! -f ./${DBDREPORT} ] && [ -f b1/${BUILDINFO} ] ; then
302
		cp b1/${BUILDINFO} $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}/ > /dev/null 2>&1
303
		figlet ${SRCPACKAGE}
304
		log_info "$DIFFOSCOPE found no differences in the changes files, and a .buildinfo file also exists."
305
		log_info "${SRCPACKAGE} from $SUITE built successfully and reproducibly on ${ARCH}."
306
		calculate_build_duration
307
		update_db_and_html "reproducible"
308
	elif [ -f ./$DBDREPORT ] ; then
309
		log_warning "Diffoscope claims the build is reproducible, but there is a diffoscope file. Please investigate."
310 311
		handle_ftbr
	elif [ ! -f b1/$BUILDINFO ] ; then
312
		log_warning "Diffoscope claims the build is reproducible, but there is no .buildinfo file. Please investigate."
313 314 315 316
		handle_ftbr
	fi
}

317
handle_env_changes() {
318
	unregister_build
319
	MESSAGE="$(date -u ) - ${BUILD_URL}console.log encountered a problem: $1"
320
	echo -e "$MESSAGE" | tee -a /var/log/jenkins/reproducible-env-changes.log
321 322 323 324
	# no need to slow down
	exit 0
}

325 326
handle_enospace() {
	unregister_build
327
	MESSAGE="${BUILD_URL}console.log hit diskspace issues with $SRCPACKAGE on $SUITE/$ARCH on $1, sleeping 60m."
328
	echo "$MESSAGE"
329
	echo "$MESSAGE" | mail -s "$JOB on $1 ran into diskspace problems" qa-jenkins-scm@lists.alioth.debian.org
330 331
	echo "Sleeping 60m before aborting the job."
	sleep 60m
332 333 334
	exit 0
}

335
dbd_timeout() {
336
	local msg="$DIFFOSCOPE was killed after running into timeout after $1"
337
	if [ ! -s ./${DBDREPORT} ] ; then
338
		echo "$(date -u) - $DIFFOSCOPE produced no output and was killed after running into timeout after ${1}..." >> ${DBDREPORT}
339
	else
340
		msg="$msg, but there is still $DEBIAN_URL/dbd/$SUITE/$ARCH/$DDBREPORT"
341
	fi
342
	SAVE_ARTIFACTS=0
343
	NOTIFY="diffoscope_timeout"
344
	handle_ftbr "$msg"
345 346
}

347
call_diffoscope_on_changes_files() {
348 349
	# filter lines describing .buildinfo files from .changes file
	sed -i -e '/^ [a-f0-9]\{32,64\} .*\.buildinfo$/d' b{1,2}/$CHANGES
Holger Levsen's avatar
Holger Levsen committed
350
	local TMPLOG=$(mktemp --tmpdir=$TMPDIR)
351
	local TIMEOUT="120m"  # note that below there is another instance of this + 5 minutes
352 353
	DBDSUITE=$SUITE
	if [ "$SUITE" = "experimental" ] ; then
Holger Levsen's avatar
Holger Levsen committed
354
		# there is no extra diffoscope-schroot for experimental ( because we specical case ghc enough already )
355 356
		DBDSUITE="unstable"
	fi
357
	# diffoscope temporary files are going to end up in this
358
	local TEMP=$(mktemp --tmpdir=$TMPDIR -d dbd-tmp-XXXXXXX)
359 360
	local session="$(schroot --begin-session -c "chroot:jenkins-reproducible-$DBDSUITE-diffoscope")"
	DIFFOSCOPE="$(schroot --directory $TMPDIR --run-session -c "$session" diffoscope -- --version 2>&1 || true)"
361
	log_info "$DIFFOSCOPE will be used to compare the two builds:"
362
	set +e
363
	set -x
364
	( timeout 125m nice schroot \
365
		--directory $TMPDIR \
366 367
		--run-session \
		-c "$session" \
368
		-- sh -c "export TMPDIR=$TEMP ; timeout $TIMEOUT diffoscope \
369
			--html $TMPDIR/${DBDREPORT} \
370
			--text $TMPDIR/$DBDTXT \
371
			--json $TMPDIR/$DBDJSON \
372
			--profile=- \
373 374
			$TMPDIR/b1/${CHANGES} \
			$TMPDIR/b2/${CHANGES}" \
375
	2>&1 ) >> $TMPLOG
376
	RESULT=$?
377
	if ! "$DEBUG" ; then set +x ; fi
378
	set -e
379
	log_file $TMPLOG  # print dbd output
380
	schroot --end-session -c "$session"
381
	rm $TMPLOG
382
	case $RESULT in
383 384
		0)
			handle_reproducible
385
			;;
386
		1)
387
			handle_ftbr "$DIFFOSCOPE found issues, please check $DEBIAN_URL/dbd/${SUITE}/${ARCH}/${DBDREPORT}"
388 389
			;;
		2)
390
			SAVE_ARTIFACTS=1
391
			NOTIFY="diffoscope_err"
392
			handle_ftbr "$DIFFOSCOPE had trouble comparing the two builds. Please investigate $DEBIAN_URL/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log"
393 394
			;;
		124)
395
			dbd_timeout $TIMEOUT
396
			;;
397
		*)
398 399 400 401 402
			# Process killed by signal exits with 128+${signal number}.
			# 31 = SIGSYS = maximum signal number in signal(7)
			if (( $RESULT > 128 )) && (( $RESULT <= 128+31 )); then
				RESULT="$RESULT (SIG$(kill -l $(($RESULT - 128))))"
			fi
403
			local MSG_PART1="Something weird happened, $DIFFOSCOPE exited with $RESULT and I don't know how to handle it."
404 405
			handle_ftbr "$MSG_PART1"
			irc_message debian-reproducible-changes "$MSG_PART1 Please check $RBUILDLOG and $DEBIAN_URL/$SUITE/$ARCH/$SRCPACKAGE"
406
			;;
407
	esac
408 409
}

410
choose_package() {
411
	DISTROID=$(query_db "SELECT id FROM distributions WHERE name='debian'")
412
	local RESULT=$(query_db "
413
		SELECT s.suite, s.id, s.name, s.version, sch.save_artifacts, sch.notify, s.notify_maintainer, sch.date_scheduled
414
		FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id
415
		WHERE sch.date_build_started is NULL
416
		AND s.distribution=$DISTROID
417
		AND s.architecture='$ARCH'
418
		ORDER BY date_scheduled LIMIT 5"|sort -R|head -1)
419 420 421 422 423
	if [ -z "$RESULT" ] ; then
		echo "No packages scheduled, sleeping 30m."
		sleep 30m
		exit 0
	fi
424 425 426
	SUITE=$(echo $RESULT|cut -d "|" -f1)
	SRCPKGID=$(echo $RESULT|cut -d "|" -f2)
	SRCPACKAGE=$(echo $RESULT|cut -d "|" -f3)
427
	VERSION=$(echo $RESULT|cut -d "|" -f4)
428
	SAVE_ARTIFACTS=$(echo $RESULT|cut -d "|" -f5)
429
	NOTIFY=$(echo $RESULT|cut -d "|" -f6)
430
	NOTIFY_MAINTAINER=$(echo $RESULT|cut -d "|" -f7)
431
	# remove previous build attempts which didnt finish correctly:
432
	JOB_PREFIX="${JOB_NAME#reproducible_builder_}/"
433
	BAD_BUILDS=$(mktemp --tmpdir=$TMPDIR)
434
	query_db "SELECT package_id, date_build_started, job FROM schedule WHERE job LIKE '${JOB_PREFIX}%'" > $BAD_BUILDS
435
	if [ -s "$BAD_BUILDS" ] ; then
436 437
		local STALELOG=/var/log/jenkins/reproducible-stale-builds.log
		# reproducible-stale-builds.log is mailed once a day by reproducible_maintenance.sh
438
		echo -n "$(date -u) - stale builds found, cleaning db from these: " | tee -a $STALELOG
439
		cat $BAD_BUILDS | tee -a $STALELOG
440
		query_db "UPDATE schedule SET date_build_started = NULL, job = NULL WHERE job LIKE '${JOB_PREFIX}%'"
441
	fi
442
	rm -f $BAD_BUILDS
443
	# mark build attempt, first test if none else marked a build attempt recently
444
	echo "ok, let's check if $SRCPACKAGE is building anywhere yet…"
445
	RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID'")
446
	if [ -z "$RESULT" ] ; then
447
		echo "ok, $SRCPACKAGE is not building anywhere…"
448
		# try to update the schedule with our build attempt, then check no else did it, if so, abort
449 450
		query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND date_build_started IS NULL"
		RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB'")
451
		if [ -z "$RESULT" ] ; then
452
			echo "hm, seems $SRCPACKAGE is building somewhere… failed to update the schedule table with our build ($SRCPKGID, $DATE, $JOB)."
453 454
			handle_race_condition
		fi
455
	else
456
		echo "hm, seems $SRCPACKAGE is building somewhere… schedule table now listed it as building somewhere else."
457
		handle_race_condition
458
	fi
459 460 461 462 463 464
	local ANNOUNCE=""
	if [ $SAVE_ARTIFACTS -eq 1 ] ; then
		ANNOUNCE="Artifacts will be preserved."
	fi
	create_results_dirs
	echo "============================================================================="
465
	echo "Initialising reproducibly build of ${SRCPACKAGE} in ${SUITE} on ${ARCH} on $(hostname) now. $ANNOUNCE"
466 467
	echo "1st build will be done on $NODE1."
	echo "2nd build will be done on $NODE2."
468 469 470 471 472 473
	echo "============================================================================="
	# force debug mode for certain packages
	case $SRCPACKAGE in
		xxxxxxx)
			export DEBUG=true
			set -x
474
			irc_message debian-reproducible "$SRCPACKAGE/$SUITE/$ARCH started building at ${BUILD_URL}console.log"
475 476 477
			;;
		*)      ;;
	esac
478
	if [ "$NOTIFY" = "2" ] ; then
479
		irc_message debian-reproducible "$SRCPACKAGE/$SUITE/$ARCH started building at ${BUILD_URL}console.log"
480 481
	elif [ "$NOTIFY" = "0" ] ; then  # the build script has a different idea of notify than the scheduler,
		NOTIFY=''                  # the scheduler uses integers, build.sh uses strings.
482
	fi
483
	log_info "starting to build ${SRCPACKAGE}/${SUITE}/${ARCH} on $(hostname) on '$DATE'"
484
	log_info "The jenkins build log is/was available at ${BUILD_URL}console.log"
485
}
486

487
download_source() {
488
	log_info "Downloading source for ${SUITE}/${SRCPACKAGE}=${VERSION}"
489
	set +e
490
	local TMPLOG=$(mktemp --tmpdir=$TMPDIR)
491
	if [ "$MODE" != "master" ] ; then
492
		chdist --data-dir="$CHPATH" apt-get "$SUITE-$ARCH" --download-only --only-source source ${SRCPACKAGE}=${VERSION} 2>&1 | tee ${TMPLOG}
493
	else
494
		# the build master only needs to the the .dsc file
495
		chdist --data-dir="$CHPATH" apt-get "$SUITE-$(dpkg --print-architecture)" --download-only --only-source --print-uris source ${SRCPACKAGE}=${VERSION} | grep \.dsc|cut -d " " -f1|xargs -r wget --timeout=180 --tries=3 2>&1 | tee ${TMPLOG}
496
	fi
497 498
	local ENGLISH_RESULT=$(egrep 'E: (Unable to find a source package for|Failed to fetch.*(Unable to connect to|Connection failed|Size mismatch|Cannot initiate the connection to|Bad Gateway|Service Unavailable))' ${TMPLOG})
	local FRENCH_RESULT=$(egrep 'E: (Unable to find a source package for|impossible de récupérer.*(Unable to connect to|Échec de la connexion|Size mismatch|Cannot initiate the connection to|Bad Gateway|Service Unavailable))' ${TMPLOG})
499
	PARSED_RESULT="${ENGLISH_RESULT}${FRENCH_RESULT}"
500
	log_file ${TMPLOG}
501 502 503 504 505
	rm ${TMPLOG}
	set -e
}

download_again_if_needed() {
506
	if [ "$(ls ${SRCPACKAGE}_${EVERSION}.dsc 2> /dev/null)" = "" ] || [ ! -z "$PARSED_RESULT" ] ; then
507 508
		# sometimes apt-get cannot download a package for whatever reason.
		# if so, wait some time and try again. only if that fails, give up.
509
		log_error "Download of ${SRCPACKAGE}=${VERSION} sources (for ${SUITE}) failed."
510
		ls -l ${SRCPACKAGE}* | log_file -
511
		log_error "Sleeping 5m before re-trying..."
512
		sleep 5m
513
		download_source
514
	fi
515 516 517 518
}

get_source_package() {
	PARSED_RESULT=""
519 520 521
	EVERSION="$(echo $VERSION | cut -d ':' -f2)"  # EPOCH_FREE_VERSION is too long
	DBDREPORT="${SRCPACKAGE}_${EVERSION}.diffoscope.html"
	DBDTXT="${SRCPACKAGE}_${EVERSION}.diffoscope.txt"
522
	DBDJSON="${SRCPACKAGE}_${EVERSION}.diffoscope.json"
523 524 525
	BUILDINFO="${SRCPACKAGE}_${EVERSION}_${ARCH}.buildinfo"
	BUILDINFO_SIGNED="${BUILDINFO}.asc"

526 527 528 529
	download_source
	download_again_if_needed
	download_again_if_needed
	download_again_if_needed # yes, this is called three times. this should really not happen
530
	if [ "$(ls ${SRCPACKAGE}_${EVERSION}.dsc 2> /dev/null)" = "" ] || [ ! -z "$PARSED_RESULT" ] ; then
531
		if [ "$MODE" = "master" ] ; then
532
			handle_E404
533
		else
534
			exit 404
535
		fi
536
	fi
537 538
}

539
check_suitability() {
540 541
	log_info "Checking whether the package is not for us"

542 543
	local SUITABLE=false
	local ARCHITECTURES=$(grep "^Architecture: " ${SRCPACKAGE}_*.dsc| cut -d " " -f2- | sed -s "s# #\n#g" | sort -u)
544 545 546 547 548 549

	# packages that are *only* arch:all can be tried on any arch
	if [ "$ARCHITECTURES" = "all" ]; then
		ARCHITECTURES="any"
	fi

550
	for arch in ${ARCHITECTURES} ; do
551
		if [ "$arch" = "any" ] || [ "$arch" = "$ARCH" ] || [ "$arch" = "linux-any" ] || [ "$arch" = "linux-$ARCH" ] || [ "$arch" = "any-$ARCH" ] ; then
552
			SUITABLE=true
553 554
			break
		fi
555
		# special case arm…
556
		if ( [ "$ARCH" = "armhf" ] || [ "$ARCH" = "arm64" ] ) && [ "$arch" = "any-arm" ] ; then
557 558 559 560
			SUITABLE=true
			break
		fi

561
	done
562
	if ! $SUITABLE ; then handle_NFU $ARCHITECTURES ; fi
563 564
}

565
first_build() {
566
	echo "============================================================================="
567
	echo "Building ${SRCPACKAGE} in ${SUITE} on ${ARCH} on $(hostname) now."
568 569
	echo "Date:     $(date)"
	echo "Date UTC: $(date -u)"
570
	echo "============================================================================="
571
	local TMPCFG=$(mktemp -t pbuilderrc_XXXX --tmpdir=$TMPDIR)
572 573
	cat > "$TMPCFG" << EOF
BUILDUSERID=1111
574
export BUILDUSERNAME=pbuilder1
575
export BUILDUSERGECOS="first user,first room,first work-phone,first home-phone,first other"
576
# pbuilder sets HOME to the value of BUILD_HOME…
577
BUILD_HOME=/nonexistent/first-build
578
export DEB_BUILD_OPTIONS="buildinfo=+all reproducible=+all parallel=$NUM_CPU"
579
export TZ="/usr/share/zoneinfo/Etc/GMT+12"
580 581 582
export LANG="C"
unset LC_ALL
export LANGUAGE="en_US:en"
583
EOF
584 585
	# build path is only varied on unstable and experimental
	if [ "${SUITE}" = "unstable" ] || [ "$SUITE" = "experimental" ]; then
586 587 588
		echo "BUILDDIR=/build/1st" >> "$TMPCFG"
	else
		echo "BUILDDIR=/build" >> "$TMPCFG"
589
	fi
590
	if [ "$SRCPACKAGE" = "debian-installer" -o "$SRCPACKAGE" = "debian-installer-netboot-images" ] ; then
591
		# d-i needs to access to a debian-archive.  this is not possible in
592
		# pbuilder, so grant complete network access to it.
593 594
		echo "USENETWORK=yes" >> "$TMPCFG"
	fi
595
	set +e
596
	# remember to change the sudoers setting if you change the following command
597
	sudo timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice \
598
	  /usr/sbin/pbuilder --build \
599
		--configfile $TMPCFG \
600
		--debbuildopts "-b --buildinfo-id=${ARCH}" \
601
		--basetgz /var/cache/pbuilder/$SUITE-reproducible-base.tgz \
602
		--buildresult $TMPDIR/b1 \
603 604
		--logfile b1/build.log \
		${SRCPACKAGE}_${EVERSION}.dsc
605 606
	local PRESULT=$?
	set -e
607
	if ! "$DEBUG" ; then set +x ; fi
Holger Levsen's avatar
Holger Levsen committed
608
	rm $TMPCFG
609 610
	case $PRESULT in
		124)
611
			echo "$(date -u) - pbuilder was killed by timeout after 18h." | tee -a b1/build.log
612 613 614 615 616 617
			exit 3
			;;
		1)  # FTBFS, for whatever reason.
			exit 2
			;;
	esac
Holger Levsen's avatar
Holger Levsen committed
618 619
}

620
second_build() {
621
	echo "============================================================================="
622
	echo "Re-Building ${SRCPACKAGE} in ${SUITE} on ${ARCH} on $(hostname) now."
623 624 625
	echo "Date:     $(date)"
	echo "Date UTC: $(date -u)"
	echo "============================================================================="
626 627
	set -x
	local TMPCFG=$(mktemp -t pbuilderrc_XXXX --tmpdir=$TMPDIR)
628
	NEW_NUM_CPU=$NUM_CPU	# on amd64+i386 we vary this based on node choices (by design), on armhf only sometimes.
629 630
	# differ locale+language depending on the architecture (mostly for readability by different people…)
	case $ARCH in
631
		armhf)	locale=it_CH
632 633
			language=it
			;;
634 635 636
		arm64)	locale=nl_BE
			language=nl
			;;
637 638 639 640 641 642 643
		i386)	locale=de_CH
			language=de
			;;
		*)	locale=fr_CH
			language=fr
			;;
	esac
644 645
	cat > "$TMPCFG" << EOF
BUILDUSERID=2222
646
export BUILDUSERNAME=pbuilder2
647
export BUILDUSERGECOS="second user,second room,second work-phone,second home-phone,second other"
648
# pbuilder sets HOME to the value of BUILD_HOME…
649
BUILD_HOME=/nonexistent/second-build
650
export DEB_BUILD_OPTIONS="buildinfo=+all reproducible=+all parallel=$NUM_CPU"
651
export TZ="/usr/share/zoneinfo/Etc/GMT-14"
652 653 654
export LANG="$locale.UTF-8"
export LC_ALL="$locale.UTF-8"
export LANGUAGE="$locale:$language"
655 656
umask 0002
EOF
657 658
	# build path is only varied on unstable and experimental
	if [ "${SUITE}" = "unstable" ] || [ "$SUITE" = "experimental" ]; then
659
		local src_dir_name="$(perl -mDpkg::Source::Package -e '$_ = Dpkg::Source::Package->new(filename => $ARGV[0])->get_basename; s/_/-/g; print' -- "${SRCPACKAGE}_${EVERSION}.dsc")"
660
		echo "BUILDDIR=/build/$src_dir_name" >> "$TMPCFG"
661
		echo "BUILDSUBDIR=2nd" >> "$TMPCFG"
662 663
	else
		echo "BUILDDIR=/build" >> "$TMPCFG"
664
	fi
665
	if [ "$SRCPACKAGE" = "debian-installer" -o "$SRCPACKAGE" = "debian-installer-netboot-images" ] ; then
666
		# d-i needs to access to a debian-archive.  this is not possible in
667
		# pbuilder, so grant complete network access to it.
668 669
		echo "USENETWORK=yes" >> "$TMPCFG"
	fi
670 671 672 673 674 675 676 677

	local pbuilder_options=()
	case "${SUITE}" in
		(unstable|experimental|buster)
			pbuilder_options+=(--extrapackages usrmerge)
			;;
	esac

678
	set +e
679
	# remember to change the sudoers setting if you change the following command
680 681
	# (the 2nd build gets a longer timeout trying to make sure the first build
	# aint wasted when then 2nd happens on a highly loaded node)
682
	sudo timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 \
683 684 685 686
		/usr/bin/unshare --uts -- \
		/usr/sbin/pbuilder --build \
			--configfile $TMPCFG \
			--hookdir /etc/pbuilder/rebuild-hooks \
687
			--debbuildopts "-b --buildinfo-id=${ARCH}" \
688
			--basetgz /var/cache/pbuilder/$SUITE-reproducible-base.tgz \
689
			--buildresult $TMPDIR/b2 \
690
			--logfile b2/build.log \
691
			"${pbuilder_options[@]}" \
692
			${SRCPACKAGE}_${EVERSION}.dsc
693
	local PRESULT=$?
694
	set -e
695
	if ! "$DEBUG" ; then set +x ; fi
696
	rm $TMPCFG
697 698 699 700 701 702 703 704 705 706
	case $PRESULT in
		124)
			echo "$(date -u) - pbuilder was killed by timeout after 24h." | tee -a b2/build.log
			exit 3
			;;
		1)  # FTBFS, for whatever reason.
			exit 2
			;;
	esac

707 708
}

709
check_node_is_up() {
710 711
	# this actually tests three things:
	# - the node is not listed in ~/offline_nodes
712 713
	# - ssh login works
	# - /tmp is not mounted in read-only mode
714
	local NODE=$1
715
	local SLEEPTIME=$2
716
	set +e
717 718 719 720 721 722 723 724
	echo "$(date -u) - checking ~/offline_nodes if $NODE is marked as down."
	if grep -q $NODE ~/offline_nodes >/dev/null 2>&1 ; then
		echo "$(date -u) - $NODE seems to marked as down, sleeping ${SLEEPTIME}min before aborting this job."
		unregister_build
		sleep ${SLEEPTIME}.1337m
		exit 0
	fi
	echo "$(date -u) - checking via ssh if $NODE is up."
725
	ssh -o "BatchMode = yes" $NODE 'rm -v $(mktemp --tmpdir=/tmp read-only-fs-test-XXXXXX)'
726 727
	RESULT=$?
	# abort job if host is down
Holger Levsen's avatar
Holger Levsen committed
728
	if [ $RESULT -ne 0 ] ; then
729
		echo "$(date -u) - $NODE seems to be down, sleeping ${SLEEPTIME}min before aborting this job."
730
		unregister_build
731
		sleep ${SLEEPTIME}.1337m
732
		exit 0
733
	fi
734 735 736
	set -e
}

737 738
check_nodes_are_up() {
	local SLEEPTIME=30
Holger Levsen's avatar
Holger Levsen committed
739
	get_node_information $NODE1
740
	check_node_is_up $NODE1 $SLEEPTIME
Holger Levsen's avatar
Holger Levsen committed
741
	get_node_information $NODE2
742
	check_node_is_up $NODE2 $SLEEPTIME
743 744
}

745 746 747
remote_build() {
	local BUILDNR=$1
	local NODE=$2
748
	log_info "Preparing to do remote build '$BUILDNR' on $NODE."
Holger Levsen's avatar
Holger Levsen committed
749
	get_node_information $NODE
750 751 752
	# sleep 15min if first node is down
	# but 1h if the 2nd node is down
	local SLEEPTIME=$(echo "$BUILDNR*$BUILDNR*15"|bc)
753
	check_node_is_up $NODE $SLEEPTIME
754
	set +e
755
	ssh -o "BatchMode = yes" $NODE /srv/jenkins/bin/reproducible_build.sh $BUILDNR ${SRCPACKAGE} ${SUITE} ${TMPDIR} "$VERSION"
756
	local BUILD_RESULT=$?
757
	rsync -e "ssh -o 'BatchMode = yes'" -r $NODE:$TMPDIR/b$BUILDNR $TMPDIR/
758 759
	local RSYNC_RESULT=$?
	if [ $RSYNC_RESULT -ne 0 ] ; then
760
		log_warning "rsync from $NODE failed, sleeping 2m before re-trying..."
761
		sleep 2m
762
		rsync -e "ssh -o 'BatchMode = yes'" -r $NODE:$TMPDIR/b$BUILDNR $TMPDIR/
763 764
		local RSYNC_RESULT=$?
		if [ $RSYNC_RESULT -ne 0 ] ; then
765
			unregister_build
766
			handle_remote_error "when rsyncing remote build #$BUILDNR results from $NODE"
767
		fi
768
	fi
769
	ls -lR $TMPDIR
770
	log_info "Deleting \$TMPDIR on $NODE."
771
	ssh -o "BatchMode = yes" $NODE "rm -r $TMPDIR"
772
	set -e
773
	if [ $BUILDNR -eq 1 ] ; then
774
		log_file $TMPDIR/b1/build.log
775
	fi
776 777 778 779
	case $BUILD_RESULT in
		148)  # 404-256=148... (ssh 'really' only 'supports' exit codes below 255...)
			handle_E404
			;;
780
		2)
781 782
			handle_ftbfs
			;;
783 784 785
		3)
			handle_timeout
			;;
786 787 788
		0)  # build succcessfully completed
			;;
		*)
789
			unregister_build
790 791 792
			handle_remote_error "with exit code $RESULT from $NODE for build #$BUILDNR for ${SRCPACKAGE} on ${SUITE}/${ARCH}"
			;;
	esac
793 794
}

795
check_installed_build_depends() {
796 797
	local TMPFILE1=$(mktemp --tmpdir=$TMPDIR)
	local TMPFILE2=$(mktemp --tmpdir=$TMPDIR)
798 799
	grep-dctrl -s Installed-Build-Depends -n ${SRCPACKAGE} ./b1/$BUILDINFO > $TMPFILE1
	grep-dctrl -s Installed-Build-Depends -n ${SRCPACKAGE} ./b2/$BUILDINFO > $TMPFILE2
800 801 802 803 804
	set +e
	diff $TMPFILE1 $TMPFILE2
	RESULT=$?
	set -e
	if [ $RESULT -eq 1 ] ; then
805
		printf "$(date -u) - $BUILDINFO in ${SUITE} on ${ARCH} varies, probably due to mirror updates. Doing the first build again, please check ${BUILD_URL}console.log for now...\n" >> /var/log/jenkins/reproducible-env-changes.log
806 807
		echo
		echo "============================================================================="
808
		echo "$(date -u) - The installed build depends vary according to the two .buildinfo files, probably due to mirror updates. Doing the first build on $NODE1 again."
809 810
		echo "============================================================================="
		echo
811
		remote_build 1 $NODE1
812
		grep-dctrl -s Installed-Build-Depends -n ${SRCPACKAGE} ./b1/$BUILDINFO > $TMPFILE1
813 814 815
		set +e
		diff $TMPFILE1 $TMPFILE2
		RESULT=$?
816
		rm $TMPFILE1 $TMPFILE2
817 818
		set -e
		if [ $RESULT -eq 1 ] ; then
819
			handle_env_changes "different packages were installed in the 1st+2nd builds and also in the 2nd+3rd build.\n$(ls -l ./b1/$BUILDINFO) on $NODE1\n$(ls -l ./b2/$BUILDINFO) on $NODE2\n"
820 821
		fi
	fi
822
	rm -f $TMPFILE1 $TMPFILE2
823 824
}

825
sign_buildinfo() {
826
	local BUILDPATH="./$1"
827 828 829
	log_info "Signing $BUILDPATH/$BUILDINFO as $BUILDINFO_SIGNED"
	gpg --output=$BUILDPATH/$BUILDINFO_SIGNED --clearsign $BUILDPATH/$BUILDINFO || log_error "Could not sign $PWD/$BUILDPATH/$BUILDINFO"
	log_info "Signed $BUILDPATH/$BUILDINFO as $BUILDPATH/$BUILDINFO_SIGNED"
830 831
}

832
share_buildinfo() {
833
	# Submit the -buildinfo files to third-party archives:
834
	log_info "Submitting .buildinfo files to external archives:"
835

836 837 838 839 840 841 842
	# Disabled as that mail address bounces.
	## buildinfo.kfreebsd.eu administered by Steven Chamberlain <steven@pyro.eu.org>
	#for X in b1 b2
	#do
	#	mail -s "${X}/$BUILDINFO_SIGNED" submit@buildinfo.kfreebsd.eu < ./${X}/$BUILDINFO_SIGNED || log_error "Could not submit ${X}/$BUILDINFO_SIGNED to submit@buildinfo.kfreebsd.eu."
	#done
	#log_info "Done submitting .buildinfo files to submit@buildinfo.kfreebsd.eu."
843

844
	# buildinfo.debian.net administred by Chris Lamb <lamby@debian.org>
845
	local TMPFILE=$(mktemp --tmpdir=$TMPDIR)
846 847
	for X in b1 b2
	do
848 849 850 851
		log_info "Submitting $(du -h ${X}/$BUILDINFO_SIGNED)"
		curl -s -X PUT --max-time 30 --data-binary @- "https://buildinfo.debian.net/api/submit" < ./${X}/$BUILDINFO_SIGNED > $TMPFILE || log_error "Could not submit buildinfo from ${X} to http://buildinfo.debian.net/api/submit"
		cat $TMPFILE
		if grep -q "500 Internal Server Error" $TMPFILE ; then
852
			MESSAGE="$(date -u ) - ${BUILD_URL}console.log got error code 500 from buildinfo.debian.net for $(du -h ${X}/$BUILDINFO_SIGNED)"
853
			echo -e "$MESSAGE" | tee -a /var/log/jenkins/reproducible-submit2buildinfo.debian.net.log
854 855
		fi
		rm $TMPFILE
856
	done
857
	log_info "Done submitting .buildinfo files to http://buildinfo.debian.net/api/submit."
858

859
	log_info "Done submitting .buildinfo files."
860
	log_info "Removing signed $BUILDINFO_SIGNED files:"
861
	rm -vf ./b1/$BUILDINFO_SIGNED ./b2/$BUILDINFO_SIGNED | tee -a "$RBUILDLOG"
862 863
}

Holger Levsen's avatar
Holger Levsen committed
864
build_rebuild() {
865
	CHANGES="${SRCPACKAGE}_${EVERSION}_${ARCH}.changes" # changes file with expected version
Holger Levsen's avatar
Holger Levsen committed
866
	mkdir b1 b2
867
	log_info "Starting 1st build on remote node $NODE1."
868
	remote_build 1 $NODE1
869 870
	log_info "1st build successful. Starting 2nd build on remote node $NODE2."
	remote_build 2 $NODE2
871
	# at this point, both builds were fine, i.e., they did not FTBFS.
872 873
	log_info "$CHANGES:"
	log_file b1/$CHANGES
874 875 876 877 878 879 880
	check_installed_build_depends
	# save the build:
	cleanup_pkg_files
	diff_copy_buildlogs
	update_rbuildlog
	call_diffoscope_on_changes_files  # defines DIFFOSCOPE, update_db_and_html defines STATUS
	share_buildinfo
881 882
}

883
#
Holger Levsen's avatar
Holger Levsen committed
884
# below is what controls the world
885 886
#

887 888
mkdir -p /srv/reproducible-results/rbuild-debian
TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results/rbuild-debian -d)  # where everything actually happens
889 890 891
trap cleanup_all INT TERM EXIT
cd $TMPDIR

892
DATE=$(date -u +'%Y-%m-%d %H:%M')
893
START=$(date +'%s')
894
RBUILDLOG=$(mktemp --tmpdir=$TMPDIR)
895
JOB="${JOB_NAME#reproducible_builder_}/${BUILD_ID}"
896

897
#
898
# determine mode
899
#
900
if [ "$1" = "" ] ; then
901 902
	echo "Error, needs at least one parameter."
	exit 1
903 904 905 906
elif [ "$1" = "1" ] || [ "$1" = "2" ] ; then
	MODE="$1"
	SRCPACKAGE="$2"
	SUITE="$3"
907
	ARCH="$(dpkg --print-architecture)"
908 909
	SAVE_ARTIFACTS="0"
	TMPDIR="$4"
910
	VERSION="$5"
911 912
	[ -d $TMPDIR ] || mkdir -p $TMPDIR
	cd $TMPDIR
913
	get_source_package
914 915 916 917 918 919
	mkdir b$MODE
	if [ "$MODE" = "1" ] ; then
		first_build
	else
		second_build
	fi
920
	sign_buildinfo "b$MODE"
921
	echo "$(date -u) - build #$MODE for $SRCPACKAGE/$SUITE/$ARCH on $HOSTNAME done."
922 923
	exit 0
elif [ "$2" != "" ] ; then
924
	MODE="master"
Holger Levsen's avatar
Holger Levsen committed
925 926
	NODE1="$(echo $1 | cut -d ':' -f1).debian.net"
	NODE2