reproducible_build_archlinux_pkg.sh 12 KB
Newer Older
1
2
3
4
5
6
7
8
9
#!/bin/bash

# Copyright 2015 Holger Levsen <holger@layer-acht.org>
# released under the GPLv=2

DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

10
11
12
# common code
. /srv/jenkins/bin/reproducible_common.sh

13
14
15
set -e

cleanup_all() {
16
	cd
17
18
	# delete session if it still exists
	if [ "$MODE" != "master" ] ; then
19
		schroot --end-session -c archlinux-$SRCPACKAGE-$(basename $TMPDIR) > /dev/null 2>&1 || true
20
	fi
21
	# delete makepkg build dir
22
23
24
	if [ ! -z $SRCPACKAGE ] && [ -d /tmp/$SRCPACKAGE-$(basename $TMPDIR) ] ; then
		rm -r /tmp/$SRCPACKAGE-$(basename $TMPDIR)
	fi
25
26
27
28
	# delete main work dir (only on master)
	if [ "$MODE" = "master" ] ; then
		rm $TMPDIR -r
		echo "$(date -u) - $TMPDIR deleted."
29
	fi
30
	rm -f $DUMMY > /dev/null || true
31
32
33
34
35
36
37
38
39
40
41
}

handle_remote_error() {
	MESSAGE="${BUILD_URL}console got remote error $1"
	echo "$(date -u ) - $MESSAGE" | tee -a /var/log/jenkins/reproducible-remote-error.log
	echo "Sleeping 5m before aborting the job."
	sleep 5m
	exec /srv/jenkins/bin/abort.sh
	exit 0
}

42
43
update_archlinux_repositories() {
	# every 2 days we check for new archlinux packages in all tested repositories
44
	touch -d "$(date -d '2 days ago' '+%Y-%m-%d') 00:00 UTC" $DUMMY
45
	local NEED_UPDATE=false
46
	local REPO
47
	for REPO in $ARCHLINUX_REPOS ; do
48
		if [ ! -f ${ARCHLINUX_PKGS}_$REPO ] || [ $DUMMY -nt ${ARCHLINUX_PKGS}_$REPO ] ; then
49
			NEED_UPDATE=true
50
51
		fi
	done
52
53
54
55
56
57
	if $NEED_UPDATE ; then
		local SESSION="archlinux-scheduler-$RANDOM"
		schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
		for REPO in $ARCHLINUX_REPOS ; do
			if [ ! -f ${ARCHLINUX_PKGS}_$REPO ] || [ $DUMMY -nt ${ARCHLINUX_PKGS}_$REPO ] ; then
				echo "$(date -u ) - updating list of available packages in repository '$REPO'."
58
				schroot --run-session -c $SESSION --directory /var/abs/$REPO -- ls -1|sort -u|xargs echo > ${ARCHLINUX_PKGS}_$REPO
59
60
61
62
63
64
65
66
				echo "$(date -u ) - these packages in repository '$REPO' are known to us:"
				cat ${ARCHLINUX_PKGS}_$REPO
			fi
		done
		schroot --end-session -c $SESSION
	else
		echo "$(date -u ) - repositories recent enough, no update needed."
	fi
67
	rm $DUMMY > /dev/null
68
69
70
71
72
73
}

choose_package() {
	echo "$(date -u ) - choosing package to be build."
	update_archlinux_repositories
	local REPO
74
	local PKG
75
	for REPO in $(echo $ARCHLINUX_REPOS | sed -s "s# #\n#g" | sort -R | xargs echo ); do
76
		case $REPO in
77
			core)		MIN_AGE=7
78
					;;
79
			extra|multilib)	MIN_AGE=28
80
					;;
81
82
83
			community)	MIN_AGE=42
					;;
			*)		MIN_AGE=365	# should never happen…
84
					;;
85
		esac
86
		touch -d "$(date -d '$MIN_AGE days ago' '+%Y-%m-%d') 00:00 UTC" $DUMMY
87
		for PKG in $(sort -R ${ARCHLINUX_PKGS}_$REPO) ; do
88
			# build package if it has never build or at least $MIN_AGE days ago
89
			if [ ! -d $BASE/archlinux/$REPO/$PKG ] || [ $DUMMY -nt $BASE/archlinux/$REPO/$PKG ] ; then
90
				REPOSITORY=$REPO
91
92
93
94
95
				SRCPACKAGE=$PKG
				echo "$(date -u ) - building package $PKG from '$REPOSITORY' now..."
				# very simple locking…
				mkdir -p $BASE/archlinux/$REPOSITORY/$PKG
				touch $BASE/archlinux/$REPOSITORY/$PKG
96
				# break out of the loop (and then out of the next loop too...)
97
98
99
				break
			fi
		done
100
101
102
103
		# if we broke out of the previous loop we have choosen a package
		if [ ! -z "$SRCPACKAGE" ] ; then
			break
		fi
104
	done
105
	rm $DUMMY > /dev/null
106
107
108
109
110
111
112
113
114
115
116
	if [ -z $SRCPACKAGE ] ; then
		echo "$(date -u ) - no package found to be build, sleeping 6h."
		for i in $(seq 1 12) ; do
			sleep 30m
			echo "$(date -u ) - still sleeping..."
		done
		echo "$(date -u ) - exiting cleanly now."
		exit 0
	fi
}

117
118
first_build() {
	echo "============================================================================="
119
120
121
122
	echo "Building for Arch Linux on $(hostname -f) now."
	echo "Source package: ${SRCPACKAGE}"
	echo "Repository:     $REPOSITORY"
	echo "Date:           $(date -u)"
123
124
	echo "============================================================================="
	set -x
125
	local SESSION="archlinux-$SRCPACKAGE-$(basename $TMPDIR)"
126
	local BUILDDIR="/tmp/$SRCPACKAGE-$(basename $TMPDIR)"
127
	local LOG=$TMPDIR/b1/$SRCPACKAGE/build1.log
128
	schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
129
	echo "MAKEFLAGS=-j$NUM_CPU" | schroot --run-session -c $SESSION --directory /tmp -u root -- tee -a /etc/makepkg.conf
130
	schroot --run-session -c $SESSION --directory /tmp -- mkdir $BUILDDIR
131
	schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/$REPOSITORY/$SRCPACKAGE $BUILDDIR/
132
133
	# just set timezone in the 1st build
	echo 'export TZ="/usr/share/zoneinfo/Etc/GMT+12"' | schroot --run-session -c $SESSION --directory /tmp -- tee -a /var/lib/jenkins/.bashrc
134
135
	# nicely run makepkg with a timeout of $TIMEOUT hours
	timeout -k $TIMEOUT.1h ${TIMEOUT}h /usr/bin/ionice -c 3 /usr/bin/nice \
136
		schroot --run-session -c $SESSION --directory $BUILDDIR/$SRCPACKAGE -- bash -l -c 'makepkg --syncdeps --noconfirm 2>&1' | tee -a $LOG
137
138
	PRESULT=${PIPESTATUS[0]}
	if [ $PRESULT -eq 124 ] ; then
139
		echo "$(date -u) - makepkg was killed by timeout after ${TIMEOUT}h." | tee -a $LOG
140
	fi
141
142
143
	schroot --end-session -c $SESSION | tee -a $LOG
	PRESULT=${PIPESTATUS[0]}
	if [ $PRESULT -ne 0 ] ; then
144
145
146
		echo "$(date -u) - could not end schroot session, maybe some processes are still running? Sleeping 60 seconds and trying again…" | tee -a $LOG
		sleep 60
		schroot --end-session -f -c $SESSION | tee -a $LOG
147
148
		P2RESULT=${PIPESTATUS[0]}
		if [ $P2RESULT -ne 0 ] ; then
149
150
151
152
153
			echo "$(date -u) - could not end schroot session even with force. Sleeping 10 seconds and trying once more…" | tee -a $LOG
			sleep 10
			schroot --end-session -f -c $SESSION | tee -a $LOG
			P3RESULT=${PIPESTATUS[0]}
			if [ $P3RESULT -ne 0 ] ; then
154
155
156
157
158
159
160
				if [ -n "$(grep 'ERROR: One or more PGP signatures could not be verified' $LOG)" ] ; then
					# abort only
					exit 42
				else
					# fail with notification
					exit 23
				fi
161
			fi
162
163
		fi
	fi
164
165
166
167
168
	if ! "$DEBUG" ; then set +x ; fi
}

second_build() {
	echo "============================================================================="
169
170
171
172
	echo "Re-Building for Arch Linux on $(hostname -f) now."
	echo "Source package: ${SRCPACKAGE}"
	echo "Repository:     $REPOSITORY"
	echo "Date:           $(date -u)"
173
174
	echo "============================================================================="
	set -x
175
	local SESSION="archlinux-$SRCPACKAGE-$(basename $TMPDIR)"
176
	local BUILDDIR="/tmp/$SRCPACKAGE-$(basename $TMPDIR)"
177
	local LOG=$TMPDIR/b2/$SRCPACKAGE/build2.log
178
	NEW_NUM_CPU=$(echo $NUM_CPU-1|bc)
179
	schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
180
	echo "MAKEFLAGS=-j$NEW_NUM_CPU" | schroot --run-session -c $SESSION --directory /tmp -u root -- tee -a /etc/makepkg.conf
181
	schroot --run-session -c $SESSION --directory /tmp -- mkdir $BUILDDIR
182
	schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/$REPOSITORY/$SRCPACKAGE $BUILDDIR/
183
	# add more variations in the 2nd build: TZ, LANG, LC_ALL, umask
Holger Levsen's avatar
Holger Levsen committed
184
185
186
187
188
189
	schroot --run-session -c $SESSION --directory /tmp -- tee -a /var/lib/jenkins/.bashrc <<-__END__
	export TZ="/usr/share/zoneinfo/Etc/GMT-14"
	export LANG="fr_CH.UTF-8"
	export LC_ALL="fr_CH.UTF-8"
	umask 0002
	__END__
190
191
	# nicely run makepkg with a timeout of $TIMEOUT hours
	timeout -k $TIMEOUT.1h ${TIMEOUT}h /usr/bin/ionice -c 3 /usr/bin/nice \
192
		schroot --run-session -c $SESSION --directory $BUILDDIR/$SRCPACKAGE -- bash -l -c 'makepkg --syncdeps --noconfirm 2>&1' | tee -a $LOG
193
194
	PRESULT=${PIPESTATUS[0]}
	if [ $PRESULT -eq 124 ] ; then
195
		echo "$(date -u) - makepkg was killed by timeout after ${TIMEOUT}h." | tee -a $LOG
196
	fi
197
198
199
	schroot --end-session -c $SESSION | tee -a $LOG
	PRESULT=${PIPESTATUS[0]}
	if [ $PRESULT -ne 0 ] ; then
200
201
202
		echo "$(date -u) - could not end schroot session, maybe some processes are still running? Sleeping 60 seconds and trying again…" | tee -a $LOG
		sleep 60
		schroot --end-session -f -c $SESSION | tee -a $LOG
203
204
		P2RESULT=${PIPESTATUS[0]}
		if [ $P2RESULT -ne 0 ] ; then
205
206
207
208
209
210
211
			echo "$(date -u) - could not end schroot session even with force. Sleeping 10 seconds and trying once more…" | tee -a $LOG
			sleep 10
			schroot --end-session -f -c $SESSION | tee -a $LOG
			P3RESULT=${PIPESTATUS[0]}
			if [ $P3RESULT -ne 0 ] ; then
				exit 23
			fi
212
213
		fi
	fi
214
215
216
217
218
	if ! "$DEBUG" ; then set +x ; fi
}

remote_build() {
	local BUILDNR=$1
219
	local NODE=$ARCHLINUX_BUILD_NODE
220
	local FQDN=$NODE.debian.net
221
222
	local PORT=22
	set +e
223
	ssh -o "Batchmode = yes" -p $PORT $FQDN /bin/true
224
225
226
227
228
229
230
231
	RESULT=$?
	# abort job if host is down
	if [ $RESULT -ne 0 ] ; then
		SLEEPTIME=$(echo "$BUILDNR*$BUILDNR*5"|bc)
		echo "$(date -u) - $NODE seems to be down, sleeping ${SLEEPTIME}min before aborting this job."
		sleep ${SLEEPTIME}m
		exec /srv/jenkins/bin/abort.sh
	fi
232
	ssh -o "Batchmode = yes" -p $PORT $FQDN /srv/jenkins/bin/reproducible_build_archlinux_pkg.sh $BUILDNR $BUILDER_JOB $REPOSITORY ${SRCPACKAGE} ${TMPDIR}
233
234
	RESULT=$?
	if [ $RESULT -ne 0 ] ; then
235
		ssh -o "Batchmode = yes" -p $PORT $FQDN "rm -r $TMPDIR" || true
236
		if [ $RESULT -eq 23 ] ; then
237
			echo "$(date -u) - remote job could not end schroot session properly and sent error 23 so we could abort silently."
238
			exec /srv/jenkins/bin/abort.sh
239
		elif [ $RESULT -eq 42 ] ; then
240
241
			echo "$($date -u) - sigh, failure after not being able to verify pgp signatures. work to debug why ahead."
			exec /srv/jenkins/bin/abort.sh
242
		else
243
			# FIXME: atm this is never reached…
244
245
			handle_remote_error "with exit code $RESULT from $NODE for build #$BUILDNR for ${SRCPACKAGE} from $REPOSITORY"
		fi
246
	fi
247
	rsync -e "ssh -o 'Batchmode = yes' -p $PORT" -r $FQDN:$TMPDIR/b$BUILDNR $TMPDIR/
248
249
250
251
	RESULT=$?
	if [ $RESULT -ne 0 ] ; then
		echo "$(date -u ) - rsync from $NODE failed, sleeping 2m before re-trying..."
		sleep 2m
252
		rsync -e "ssh -o 'Batchmode = yes' -p $PORT" -r $FQDN:$TMPDIR/b$BUILDNR $TMPDIR/
253
254
255
256
257
		RESULT=$?
		if [ $RESULT -ne 0 ] ; then
			handle_remote_error "when rsyncing remote build #$BUILDNR results from $NODE"
		fi
	fi
258
	ls -lR $TMPDIR
259
	ssh -o "Batchmode = yes" -p $PORT $FQDN "rm -r $TMPDIR"
260
261
262
263
264
265
	set -e
}

#
# below is what controls the world
#
266
TIMEOUT=8	# maximum time in hours for a single build
267
268
DATE=$(date -u +'%Y-%m-%d %H:%M')
START=$(date +'%s')
269
DUMMY=$(mktemp -t archlinux-dummy-XXXXXXXX)
270
trap cleanup_all INT TERM EXIT
271
272
273
274
275

#
# determine mode
#
if [ "$1" = "" ] ; then
276
	MODE="master"
277
278
	BUILDER_JOB="${JOB_NAME#reproducible_builder_archlinux}"
	TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t rbuild-archlinux$BUILDER_JOB-XXXXXXXX)  # where everything actually happens
279
	cd $TMPDIR
280
281
elif [ "$1" = "1" ] || [ "$1" = "2" ] ; then
	MODE="$1"
282
283
284
285
	BUILDER_JOB="$2"
	REPOSITORY="$3"
	SRCPACKAGE="$4"
	TMPDIR="$5"
286
287
	[ -d $TMPDIR ] || mkdir -p $TMPDIR
	cd $TMPDIR
288
	mkdir -p b$MODE/$SRCPACKAGE
289
290
291
292
293
	if [ "$MODE" = "1" ] ; then
		first_build
	else
		second_build
	fi
294
	# preserve results and delete build directory
295
	mv -v /tmp/$SRCPACKAGE-$(basename $TMPDIR)/$SRCPACKAGE/*.pkg.tar.xz $TMPDIR/b$MODE/$SRCPACKAGE/ || ls /tmp/$SRCPACKAGE-$(basename $TMPDIR)/$SRCPACKAGE/
296
	rm -r /tmp/$SRCPACKAGE-$(basename $TMPDIR)/
297
298
299
300
301
302
303
	echo "$(date -u) - build #$MODE for $SRCPACKAGE on $HOSTNAME done."
	exit 0
fi

#
# main - only used in master-mode
#
304
delay_start # randomize start times
305
306
# first, we need to choose a package from a repository…
REPOSITORY=""
307
SRCPACKAGE=""
308
choose_package
309
# build package twice
Holger Levsen's avatar
Holger Levsen committed
310
311
mkdir b1 b2
remote_build 1
312
# only do the 2nd build if the 1st produced results
313
if [ ! -z "$(ls $TMPDIR/b1/$SRCPACKAGE/*.pkg.tar.xz 2>/dev/null|| true)" ] ; then
Holger Levsen's avatar
Holger Levsen committed
314
315
316
317
318
319
320
	remote_build 2
	# run diffoscope on the results
	TIMEOUT="30m"
	DIFFOSCOPE="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)"
	echo "$(date -u) - Running $DIFFOSCOPE now..."
	cd $TMPDIR/b1/$SRCPACKAGE
	for ARTIFACT in *.pkg.tar.xz ; do
Holger Levsen's avatar
Holger Levsen committed
321
		[ -f $ARTIFACT ] || continue
Holger Levsen's avatar
Holger Levsen committed
322
323
324
		call_diffoscope $SRCPACKAGE $ARTIFACT
		# publish page
		if [ -f $TMPDIR/$SRCPACKAGE/$ARTIFACT.html ] ; then
325
			cp $TMPDIR/$SRCPACKAGE/$ARTIFACT.html $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/
Holger Levsen's avatar
Holger Levsen committed
326
327
328
		fi
	done
fi
329
330
# publish logs
cd $TMPDIR/b1/$SRCPACKAGE
331
332
333
cp build1.log $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/
[ ! -f $TMPDIR/b2/$SRCPACKAGE/build2.log ] || cp $TMPDIR/b2/$SRCPACKAGE/build2.log $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/
echo "$(date -u) - $REPRODUCIBLE_URL/archlinux/$REPOSITORY/$SRCPACKAGE/ updated."
334

335
cd
336
337
338
cleanup_all
trap - INT TERM EXIT