reproducible_setup_pbuilder.sh 7.13 KB
Newer Older
1
#!/bin/bash
2
# vim: set noexpandtab:
3

Holger Levsen's avatar
Holger Levsen committed
4
# Copyright 2014-2019 Holger Levsen <holger@layer-acht.org>
5
#           ©    2018 Mattia Rizzolo <mattia@debian.org>
6 7
# released under the GPLv=2

8
DEBUG=false
9 10 11
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

Holger Levsen's avatar
Holger Levsen committed
12 13
# common code defining db access
. /srv/jenkins/bin/reproducible_common.sh
14

15 16
# support different suites
if [ -z "$1" ] ; then
17
	SUITE="unstable"
18 19 20 21
else
	SUITE="$1"
fi

22 23 24
#
# create script to configure a pbuilder chroot
#
Holger Levsen's avatar
Holger Levsen committed
25
create_customized_tmpfile() {
26 27
	TMPFILE=$1
	shift
28
	cat >> $TMPFILE <<- EOF
29
#
30 31 32
# this script is run within the pbuilder environment to further customize initially
#
echo
33 34
echo "Preseeding man-db/auto-update to false"
echo "man-db man-db/auto-update boolean false" | debconf-set-selections
35
echo
36 37 38
echo "Configuring dpkg to not fsync()"
echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02speedup
echo
39
EOF
40
	. /srv/jenkins/bin/jenkins_node_definitions.sh
Holger Levsen's avatar
Holger Levsen committed
41
	get_node_information "$HOSTNAME"
42 43 44
	if "$NODE_RUN_IN_THE_FUTURE" ; then
		cat >> $TMPFILE <<- EOF
			echo "Configuring APT to ignore the Release file expiration"
45
			sed -i 's,^deb ,deb [check-valid-until=no] ,g' /etc/apt/sources.list
46 47 48 49
			echo
		EOF
	fi

50 51 52 53 54 55 56 57
}

create_setup_our_repo_tmpfile() {
	TMPFILE=$1
	shift
	cat >> $TMPFILE <<- EOF
#
# this script is run within the pbuilder environment to further customize once more
58
#
59
echo "Configure the chroot to use the reproducible team experimental archive..."
60 61 62 63 64 65 66 67 68 69 70 71 72 73
echo "-----BEGIN PGP PUBLIC KEY BLOCK-----

mQINBFQsy/gBEADKGF55qQpXxpTn7E0Vvqho82/HFB/yT9N2wD8TkrejhJ1I6hfJ
zFXD9fSi8WnNpLc6IjcaepuvvO4cpIQ8620lIuONQZU84sof8nAO0LDoMp/QdN3j
VViXRXQtoUmTAzlOBNpyb8UctAoSzPVgO3jU1Ngr1LWi36hQPvQWSYPNmbsDkGVE
unB0p8DCN88Yq4z2lDdlHgFIy0IDNixuRp/vBouuvKnpe9zyOkijV83Een0XSUsZ
jmoksFzLzjChlS5fAL3FjtLO5XJGng46dibySWwYx2ragsrNUUSkqTTmU7bOVu9a
zlnQNGR09kJRM77UoET5iSXXroK7xQ26UJkhorW2lXE5nQ97QqX7igWp2u0G74RB
e6y3JqH9W8nV+BHuaCVmW0/j+V/l7T3XGAcbjZw1A4w5kj8YGzv3BpztXxqyHQsy
piewXLTBn8dvgDqd1DLXI5gGxC3KGGZbC7v0rQlu2N6OWg2QRbcVKqlE5HeZxmGV
vwGQs/vcChc3BuxJegw/bnP+y0Ys5tsVLw+kkxM5wbpqhWw+hgOlGHKpJLNpmBxn
T+o84iUWTzpvHgHiw6ShJK50AxSbNzDWdbo7p6e0EPHG4Gj41bwO4zVzmQrFz//D
txVBvoATTZYMLF5owdCO+rO6s/xuC3s04pk7GpmDmi/G51oiz7hIhxJyhQARAQAB
tC5EZWJpYW4gUmVwcm9kdWNpYmxlIEJ1aWxkcyBBcmNoaXZlIFNpZ25pbmcgS2V5
74 75 76 77 78 79 80 81 82 83 84 85 86 87
iQJUBBMBCAA+AhsDBQsJCAcDBRUKCQgLBRYDAgEAAh4BAheAFiEESbZXRzbQtjfM
NwHqXbfKZ+pZox8FAlxG+gsFCQ29yJMACgkQXbfKZ+pZox/oKhAArNl6txTTDzjh
9DG5qywijR4ydUOuoLZBsvoiltzaTXZVlRdHm3JDU2gpQcZfgWzsGBiN9f1/a9uJ
teg93n5BlcBa+FEazcdWd9fssOkkphOMpv15y92G3nqfuhHnK/vhI5tP4lC4bGBi
MCoCLWULD86rPNYZxdr4KuY6RpvbrM7kj4PDaHwWH9EGvfBdqvrbjfG7e4KULl1D
SfeCxXV5bIVKxlyL8dLwKoyHe9Mp+jXGG3ZdyISprGPTIvSrpHzWIKuToJc4gYdJ
FHG9jRsJ+tBO5qW8GQ5NsthJJJ3YH3RQwGLLDdV065/DhHTzMvE5KgSkn2eN36Zl
gCLuT/qlpUkxmoBcqph9jLm/f1Mu9uo9psM/+n+aRscGRoRxtfcEpn+jXgumQ39a
S3EbiwsTVFqWr1FaCJBkT6biTgHH3oUj+Q9aq7ymZAWOvZ3WeAjRbfkYq9TjgNx9
LLAt784kuLfFlUv2/jE+pIXrCXp7RfWHq+UbIMLAtHXP8je4G8Sl8m0jmfaYmXL+
4fCGLQn6VMu/h4iQo0Kr1XpLWcvE3s6fK2GVKm8awbJhtn+xH3BIS4M9TmVz0934
PmJ9QvvxltW0X/5bhEJaaQRN2HuoiC1hItak9E8GfUIuDng24KGl9mgsgU4thO1F
TqlQavDrP+hwePoKd4P3Wvj50NVjQL4=
=1Wlp
88
-----END PGP PUBLIC KEY BLOCK-----" > /etc/apt/trusted.gpg.d/reproducible.asc
89
echo 'deb http://tests.reproducible-builds.org/debian/repository/debian/ ./' > /etc/apt/sources.list.d/reproducible.list
90 91 92
echo "Package: *
Pin: release o=reproducible
Pin-Priority: 1001" > /etc/apt/preferences.d/reproducible
93
echo
Holger Levsen's avatar
Holger Levsen committed
94
apt-get update
95
apt-get -y upgrade
96
apt-get install -y $@
97
echo
98 99
apt-cache policy
echo
100
dpkg -l
101
echo
102
for i in \$(dpkg -l |grep ^ii |awk -F' ' '{print \$2}'); do   apt-cache madison "\$i" | head -1 | grep reproducible-builds.org || true  ; done
103
echo
104
EOF
Holger Levsen's avatar
Holger Levsen committed
105 106
}

107

108
#
Holger Levsen's avatar
Holger Levsen committed
109
# setup pbuilder for reproducible builds
110
#
111
setup_pbuilder() {
112
	SUITE=$1
113
	shift
114 115
	NAME=$1
	shift
116
	PACKAGES="$@"						# from our repo
117
	EXTRA_PACKAGES="locales-all fakeroot disorderfs"	# from sid
Holger Levsen's avatar
Holger Levsen committed
118
	echo "$(date -u) - creating /var/cache/pbuilder/${NAME}.tgz now..."
119 120
	TMPFILE=$(mktemp --tmpdir=$TEMPDIR pbuilder-XXXXXXXXX)
	LOG=$(mktemp --tmpdir=$TEMPDIR pbuilder-XXXXXXXX)
Holger Levsen's avatar
Holger Levsen committed
121
	if [ "$SUITE" = "experimental" ] ; then
122
		SUITE=unstable
123 124
		echo "echo 'deb $MIRROR experimental main' > /etc/apt/sources.list.d/experimental.list" > ${TMPFILE}
		echo "echo 'deb-src $MIRROR experimental main' >> /etc/apt/sources.list.d/experimental.list" >> ${TMPFILE}
125
	fi
126
	# use host apt proxy configuration for pbuilder too
127 128 129 130
	if [ ! -z "$http_proxy" ] ; then
		echo "echo '$(cat /etc/apt/apt.conf.d/80proxy)' > /etc/apt/apt.conf.d/80proxy" >> ${TMPFILE}
		pbuilder_http_proxy="--http-proxy $http_proxy"
	fi
131
	# setup base.tgz
132
	sudo pbuilder --create $pbuilder_http_proxy --basetgz /var/cache/pbuilder/${NAME}-new.tgz --distribution $SUITE --debootstrapopts --no-merged-usr --extrapackages "$EXTRA_PACKAGES" --loglevel D
133

134 135 136 137 138 139 140 141
	# customize pbuilder
	create_customized_tmpfile ${TMPFILE}
	if [ "$DEBUG" = "true" ] ; then
		cat "$TMPFILE"
	fi
	sudo pbuilder --execute $pbuilder_http_proxy --save-after-exec --basetgz /var/cache/pbuilder/${NAME}-new.tgz -- ${TMPFILE} | tee ${LOG}
	rm ${TMPFILE}

142 143
	# add repo only for experimental and unstable - keep stretch/buster "real" (and sid progressive!)
	if [ "$SUITE" = "unstable" ] || [ "$SUITE" = "experimental" ]; then
144
		# apply further customisations, eg. install $PACKAGES from our repo
145
		create_setup_our_repo_tmpfile ${TMPFILE} "${PACKAGES}"
146 147 148 149 150 151 152 153 154 155
		if [ "$DEBUG" = "true" ] ; then
			cat "$TMPFILE"
		fi
		sudo pbuilder --execute $pbuilder_http_proxy --save-after-exec --basetgz /var/cache/pbuilder/${NAME}-new.tgz -- ${TMPFILE} | tee ${LOG}
		rm ${TMPFILE}
		if [ ! -z "$PACKAGES" ] ; then
			# finally, confirm things are as they should be
			echo
			echo "Now let's see whether the correct packages where installed..."
			for PKG in ${PACKAGES} ; do
156
				egrep "http://tests.reproducible-builds.org/debian/repository/debian(/|) ./ Packages" ${LOG} \
157 158 159 160
					| grep -v grep | grep "${PKG} " \
					|| ( echo ; echo "Package ${PKG} is not installed at all or probably rather not in our version, so removing the chroot and exiting now." ; sudo rm -v /var/cache/pbuilder/${NAME}-new.tgz ; rm $LOG ; exit 1 )
			done
		fi
161
	fi
162

163
	sudo mv /var/cache/pbuilder/${NAME}-new.tgz /var/cache/pbuilder/${NAME}.tgz
164 165
	# create stamp file to record initial creation date minus some hours so the file will be older than 24h when checked in <24h...
	touch -d "$(date -u -d '6 hours ago' '+%Y-%m-%d %H:%M')" /var/log/jenkins/${NAME}.tgz.stamp
166
	rm ${LOG}
Holger Levsen's avatar
Holger Levsen committed
167 168
}

169 170 171 172 173
#
# main
#
BASETGZ=/var/cache/pbuilder/$SUITE-reproducible-base.tgz
STAMP=/var/log/jenkins/$SUITE-reproducible-base.tgz.stamp
174

175 176
if [ -f "$STAMP" ] ; then
	if [ -f "$STAMP" -a $(stat -c %Y "$STAMP") -gt $(date +%s) ]; then
177
		if [ $(stat -c %Y "$STAMP") -gt $(date +%s -d "+ 6 months") ]; then
178 179 180 181 182 183
			echo "Warning: stamp file is too far in the future, assuming something is wrong and deleting it"
			rm -v "$STAMP"
		else
			echo "stamp file has a timestamp from the future."
			exit 1
		fi
184
	fi
185 186
fi

Holger Levsen's avatar
Holger Levsen committed
187
OLDSTAMP=$(find $STAMP -mtime +1 -exec ls -lad {} \; || echo "nostamp")
188 189 190 191 192 193
if [ -n "$OLDSTAMP" ] || [ ! -f $BASETGZ ] || [ ! -f $STAMP ] ; then
	if [ ! -f $BASETGZ ] ; then
		echo "No $BASETGZ exists, creating a new one..."
	else
		echo "$BASETGZ outdated, creating a new one..."
	fi
194
	setup_pbuilder $SUITE $SUITE-reproducible-base # list packages which must be installed from our repo here
195
else
Holger Levsen's avatar
Holger Levsen committed
196
	echo "$BASETGZ not old enough, doing nothing..."
197 198
fi
echo