reproducible_build_rpm.sh 8.74 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
#!/bin/bash

# Copyright 2015 Holger Levsen <holger@layer-acht.org>
# released under the GPLv=2

DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

# common code
. /srv/jenkins/bin/reproducible_common.sh

set -e

cleanup_all() {
	cd
17
	# delete mock result dir
18 19 20 21 22 23 24 25 26 27 28
	if [ ! -z $SRCPACKAGE ] && [ -d /tmp/$SRCPACKAGE-$(basename $TMPDIR) ] ; then
		rm -r /tmp/$SRCPACKAGE-$(basename $TMPDIR)
	fi
	# delete main work dir (only on master)
	if [ "$MODE" = "master" ] ; then
		rm $TMPDIR -r
		echo "$(date -u) - $TMPDIR deleted."
	fi
	rm -f $DUMMY > /dev/null || true
}

29
update_mock() {
30
	echo "$(date -u ) - checking whether to update mock and yum for $RELEASE ($ARCH) on $HOSTNAME."
31
	local STAMP="${RPM_STAMPS}-$RELEASE-$ARCH"
32
	touch -d "$(date -u -d "6 hours ago" '+%Y-%m-%d %H:%M') UTC" $DUMMY
33 34
	if [ ! -f $STAMP ] || [ $DUMMY -nt $STAMP ] ; then
		echo "$(date -u ) - updating mock for $RELEASE ($ARCH) on $HOSTNAME now..."
35
		mock -r $RELEASE-$ARCH --uniqueext=$UNIQUEEXT --resultdir=. --cleanup-after -v --update 2>&1
36
		echo "$(date -u ) - mock updated."
37
		yum -v --releasever=23 check-update # FIXME: dont hard-code releasever here.
38
		echo "$(date -u ) - yum updated."
39 40
		touch $STAMP
	else
41
		echo "$(date -u ) - mock and yum not updated, last update was at $(date -u --date=@$(stat -c %Y $STAMP) +'%F %T %Z')"
42
	fi
43
	rm $DUMMY > /dev/null
44
}
45 46 47 48

download_package() {
	echo "$(date -u ) - downloading ${SRCPACKAGE} for $RELEASE now."
	yumdownloader --source ${SRCPACKAGE}
49
	SRC_RPM="$(ls *.src.rpm)"
50 51 52
}

choose_package() {
53 54
	local MIN_AGE=7
	touch -d "$(date -u -d "$MIN_AGE days ago" '+%Y-%m-%d %H:%M') UTC" $DUMMY
55 56
	if [ ! -f ${RPM_PKGS}_$RELEASE ] || [ $DUMMY -nt ${RPM_PKGS}_$RELEASE ] ; then
		echo "$(date -u ) - updating list of available packages for $RELEASE"
57 58
		SEARCHTERMS="apache2 awesome bash fedora firefox gcc gnome gnu gpg ipa kde linux mock openssl pgp redhat rpm ssh system-config systemd xfce xorg yum"
		echo "$(date -u ) - for now, instead of building everything, only packages matching these searchterms are build: $SEARCHTERMS"
59
		local i=""
60 61 62 63
		# http://fedoraproject.org/wiki/Packaging:NamingGuidelines describes the rpm naming scheme
		# the awk command removes the last two "columns" seperated by "-"
		# so system-config-printer-1.5.7-5.fc23.src.rpm becomes system-config-printer
		( for i in $SEARCHTERMS ; do repoquery --qf "%{sourcerpm}" "*$i*" | awk 'NF{NF-=2}1' FS='-' OFS='-' ; done ) | sort -u > ${RPM_PKGS}_$RELEASE
64 65 66 67
		cat ${RPM_PKGS}_$RELEASE
	fi
	echo "$(date -u ) - choosing package to be build."
	local PKG=""
68
	for PKG in $(sort -R ${RPM_PKGS}_$RELEASE) ; do
69
		# build package if it has never build or at least $MIN_AGE days ago
70
		if [ ! -d $BASE/rpms/$RELEASE/$ARCH/$PKG ] || [ $DUMMY -nt $BASE/rpms/$RELEASE/$ARCH/$PKG ] ; then
71 72
			SRCPACKAGE=$PKG
			echo "$(date -u ) - building package $PKG from '$RELEASE' on '$ARCH' now..."
73
			# very simple locking…
74 75
			mkdir -p $BASE/rpms/$RELEASE/$ARCH/$PKG
			touch $BASE/rpms/$RELEASE/$ARCH/$PKG
76 77 78 79
			# break out of the loop and then out of this function too,
			# to build this package…
			break
		fi
80
	done
81
	rm $DUMMY > /dev/null
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
	if [ -z $SRCPACKAGE ] ; then
		echo "$(date -u ) - no package found to be build, sleeping 6h."
		for i in $(seq 1 12) ; do
			sleep 30m
			echo "$(date -u ) - still sleeping..."
		done
		echo "$(date -u ) - exiting cleanly now."
		exit 0
	fi
}

first_build() {
	echo "============================================================================="
	echo "Building for $RELEASE ($ARCH) on $(hostname -f) now."
	echo "Source package: ${SRCPACKAGE}"
	echo "Date:           $(date -u)"
	echo "============================================================================="
	set -x
100
	update_mock
101
	download_package
102
	local RESULTDIR="/tmp/$SRCPACKAGE-$(basename $TMPDIR)"
103
	local LOG=$TMPDIR/b1/$SRCPACKAGE/build1.log
104 105
	# nicely run mock with a timeout of $TIMEOUT hours
	timeout -k $TIMEOUT.1h ${TIMEOUT}h /usr/bin/ionice -c 3 /usr/bin/nice \
106
		mock -r $RELEASE-$ARCH --uniqueext=$UNIQUEEXT --resultdir=$RESULTDIR --cleanup-after -v --rebuild $SRC_RPM 2>&1 | tee -a $LOG
107 108
	PRESULT=${PIPESTATUS[0]}
	if [ $PRESULT -eq 124 ] ; then
109
		echo "$(date -u) - mock was killed by timeout after ${TIMEOUT}h." | tee -a $LOG
110 111 112 113 114 115 116 117 118 119 120
	fi
	if ! "$DEBUG" ; then set +x ; fi
}

second_build() {
	echo "============================================================================="
	echo "Re-Building for $RELEASE ($ARCH) on $(hostname -f) now."
	echo "Source package: ${SRCPACKAGE}"
	echo "Date:           $(date -u)"
	echo "============================================================================="
	set -x
121
	update_mock
122
	download_package
123
	local RESULTDIR="/tmp/$SRCPACKAGE-$(basename $TMPDIR)"
124 125
	local LOG=$TMPDIR/b2/$SRCPACKAGE/build2.log
	# NEW_NUM_CPU=$(echo $NUM_CPU-1|bc)
126 127
        # nicely run mock with a timeout of $TIMEOUT hours
        timeout -k $TIMEOUT.1h ${TIMEOUT}h /usr/bin/ionice -c 3 /usr/bin/nice \
128
		mock -r $RELEASE-$ARCH --uniqueext=$UNIQUEEXT --resultdir=$RESULTDIR --cleanup-after -v --rebuild $SRC_RPM 2>&1 | tee -a $LOG
129 130
	PRESULT=${PIPESTATUS[0]}
	if [ $PRESULT -eq 124 ] ; then
131
		echo "$(date -u) - mock was killed by timeout after ${TIMEOUT}h." | tee -a $LOG
132 133 134 135 136 137 138 139 140 141
	fi
	if ! "$DEBUG" ; then set +x ; fi
}

remote_build() {
	local BUILDNR=$1
	local NODE=$RPM_BUILD_NODE
	local FQDN=$NODE.debian.net
	local PORT=22
	set +e
142
	ssh -o "Batchmode = yes" -p $PORT $FQDN /bin/true
143 144 145 146 147 148 149 150
	RESULT=$?
	# abort job if host is down
	if [ $RESULT -ne 0 ] ; then
		SLEEPTIME=$(echo "$BUILDNR*$BUILDNR*5"|bc)
		echo "$(date -u) - $NODE seems to be down, sleeping ${SLEEPTIME}min before aborting this job."
		sleep ${SLEEPTIME}m
		exec /srv/jenkins/bin/abort.sh
	fi
151
	ssh -o "Batchmode = yes" -p $PORT $FQDN /srv/jenkins/bin/reproducible_build_rpm.sh $BUILDNR $RELEASE $ARCH $UNIQUEEXT ${SRCPACKAGE} ${TMPDIR}
152 153
	RESULT=$?
	if [ $RESULT -ne 0 ] ; then
154
		ssh -o "Batchmode = yes" -p $PORT $FQDN "rm -r $TMPDIR" || true
155 156
		handle_remote_error "with exit code $RESULT from $NODE for build #$BUILDNR for ${SRCPACKAGE} from $RELEASE ($ARCH)"
	fi
157
	rsync -e "ssh -o 'Batchmode = yes' -p $PORT" -r $FQDN:$TMPDIR/b$BUILDNR $TMPDIR/
158 159 160 161
	RESULT=$?
	if [ $RESULT -ne 0 ] ; then
		echo "$(date -u ) - rsync from $NODE failed, sleeping 2m before re-trying..."
		sleep 2m
162
		rsync -e "ssh -o 'Batchmode = yes' -p $PORT" -r $FQDN:$TMPDIR/b$BUILDNR $TMPDIR/
163 164 165 166 167
		RESULT=$?
		if [ $RESULT -ne 0 ] ; then
			handle_remote_error "when rsyncing remote build #$BUILDNR results from $NODE"
		fi
	fi
168
	ls -lR $TMPDIR
169
	ssh -o "Batchmode = yes" -p $PORT $FQDN "rm -r $TMPDIR"
170 171 172 173 174 175
	set -e
}

#
# below is what controls the world
#
176
TIMEOUT=8	# maximum time in hours for a single build
177 178 179
DATE=$(date -u +'%Y-%m-%d %H:%M')
START=$(date +'%s')
DUMMY=$(mktemp -t rpm-dummy-XXXXXXXX)
180
RPM_STAMPS=/srv/reproducible-results/.rpm_stamp
181
trap cleanup_all INT TERM EXIT
182 183 184 185 186 187 188 189

#
# determine mode
#
if [ "$1" = "1" ] || [ "$1" = "2" ] ; then
	MODE="$1"
	RELEASE="$2"
	ARCH="$3"
190
	UNIQUEEXT="$4"
191 192
	SRCPACKAGE="$5"
	TMPDIR="$6"
193 194 195 196 197 198 199 200 201
	[ -d $TMPDIR ] || mkdir -p $TMPDIR
	cd $TMPDIR
	mkdir -p b$MODE/$SRCPACKAGE
	if [ "$MODE" = "1" ] ; then
		first_build
	else
		second_build
	fi
	# preserve results and delete build directory
202
	mv -v /tmp/$SRCPACKAGE-$(basename $TMPDIR)/*.rpm $TMPDIR/b$MODE/$SRCPACKAGE/ || ls /tmp/$SRCPACKAGE-$(basename $TMPDIR)/
203 204 205 206 207
	rm -r /tmp/$SRCPACKAGE-$(basename $TMPDIR)/
	echo "$(date -u) - build #$MODE for $SRCPACKAGE on $HOSTNAME done."
	exit 0
fi
MODE="master"
208 209
TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t rbuild-rpm-XXXXXXXX)  # where everything actually happens
cd $TMPDIR
210 211 212 213 214

#
# main - only used in master-mode
#
delay_start # randomize start times
215
# first, we need to choose a package…
216 217
RELEASE="$1"
ARCH="$2"
218
UNIQUEEXT="mock${JOB_NAME#reproducible_builder_${RELEASE}_$ARCH}"
219 220
SRCPACKAGE=""	# package name
SRC_RPM=""	# src rpm file name
221
#update_mock # FIXME: we dont have to run mock on the main node yet, but we will need at least have to update yum there…
222 223 224 225 226 227 228 229 230
choose_package
# build package twice
mkdir b1 b2
remote_build 1
# only do the 2nd build if the 1st produced results
if [ ! -z "$(ls $TMPDIR/b1/$SRCPACKAGE/*.rpm 2>/dev/null|| true)" ] ; then
	remote_build 2
	# run diffoscope on the results
	TIMEOUT="30m"
231
	DIFFOSCOPE="$(schroot --directory /tmp -c chroot:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)"
232 233 234 235 236 237 238
	echo "$(date -u) - Running $DIFFOSCOPE now..."
	cd $TMPDIR/b1/$SRCPACKAGE
	for ARTIFACT in *.rpm ; do
		[ -f $ARTIFACT ] || continue
		call_diffoscope $SRCPACKAGE $ARTIFACT
		# publish page
		if [ -f $TMPDIR/$SRCPACKAGE/$ARTIFACT.html ] ; then
239
			cp $TMPDIR/$SRCPACKAGE/$ARTIFACT.html $BASE/rpms/$RELEASE/$ARCH/$SRCPACKAGE/
240 241 242 243 244
		fi
	done
fi
# publish logs
cd $TMPDIR/b1/$SRCPACKAGE
245 246 247
cp build1.log $BASE/rpms/$RELEASE/$ARCH/$SRCPACKAGE/
[ ! -f $TMPDIR/b2/$SRCPACKAGE/build2.log ] || cp $TMPDIR/b2/$SRCPACKAGE/build2.log $BASE/rpms/$RELEASE/$ARCH/$SRCPACKAGE/
echo "$(date -u) - $REPRODUCIBLE_URL/rpms/$RELEASE/$ARCH/$SRCPACKAGE/ updated."
248 249 250 251

cd
cleanup_all
trap - INT TERM EXIT