reproducible_debstrap.sh 4.81 KB
Newer Older
1
2
3
4
5
6
#!/bin/bash
# vim: set noexpandtab:

# Copyright 2021-2022 Holger Levsen <holger@layer-acht.org>
# released under the GPLv2

7
DEBUG=false
8
9
10
11
12
13
14
15
16
17
18
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

# common code for tests.reproducible-builds.org
. /srv/jenkins/bin/reproducible_common.sh
set -e
set -o pipefail # see eg http://petereisentraut.blogspot.com/2010/11/pipefail.html

cleanup() {
	local RESULT=$1
	output_echo "Cleanup ${RESULT}"
19
	# Cleanup the workspace and results directory
20
21
22
23
24
25
26
27
	if [ ! -z "$BUILDDIR" ]; then
		output_echo "Removing ${BUILDDIR}"
		sudo rm -rf --one-file-system ${BUILDDIR}
	fi
	if [ ! -z "$RESULTSDIR" ]; then
		output_echo "Removing ${RESULTSDIR}"
		rm -rf --one-file-system ${RESULTSDIR}
	fi
28
29
}

30
cleanup_unreproducible_file(){
31
32
	local TYPE=$1
	local FILE=$2
33
34
	if sudo file -E $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}/$FILE >/dev/null 2>&1 ; then
		output_echo "Warning: modifying $TOOL result, deleting unreproducible $TYPE $FILE"
35
36
		sudo rm -rf --one-file-system $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}/$FILE
	else
37
		sudo file $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}/$FILE
38
39
40
		output_echo "Warning: shall remove $FILE but it does not exist. Maybe $TOOL was improved."
	fi
}
41
42

# Init some variables
43
44
export TOOL="$1"
export SUITE="$2"
45
output_echo "About to bootstrap $SUITE using $TOOL version $(dpkg-query  --showformat='${Version}' --show $TOOL)."
46
47
48
49
50
51
52
53
54
55

if [ "$SUITE" != "unstable" ] ; then
	CODENAME=$SUITE
else
	CODENAME=sid
fi
S_D_E_DATE="$(grep $CODENAME /usr/share/distro-info/debian.csv | awk  -F',' '{print $4}')"
export SOURCE_DATE_EPOCH="$(date +%s -d $S_D_E_DATE)"
output_echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH that is $S_D_E_DATE which is when $SUITE was created."

56
57
export BUILDDIR=$(mktemp --tmpdir=/srv/workspace/ -d -t ${TOOL}-${SUITE}.XXXXXXXX)
export RESULTSDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t ${TOOL}-${SUITE}.XXXXXXXX) # accessible in schroots, used to compare results
58
59
60
61
62
63
64

# Cleanup if something goes wrong
trap cleanup INT TERM EXIT

# Randomize start time
delay_start

65
# Actual run ${TOOL} twice
66
67
68
69
70
for LOOP in "first" "second" ; do
	case $LOOP in
		first)	SUBDIR=b1
			ACTION="Verbosely running"
			REALTOOL=$TOOL
71
			;;
72
73
74
75
76
		second)	SUBDIR=b2
			ACTION="Running"
			case $TOOL in
				mmdebstrap)	REALTOOL="mmdebstrap -v" ;;
				debootstrap)	REALTOOL="debootstrap --verbose" ;;
77
				cdebootstrap)	REALTOOL="cdebootstrap --verbose" ;;
78
			esac
79
			;;
80
81
	esac
	output_echo "$ACTION ${TOOL} $SUITE for the $LOOP run."
82
	mkdir -p $BUILDDIR/$SUBDIR/${TOOL}
83
84
	case ${TOOL} in
		mmdebstrap)	sudo $REALTOOL $SUITE > $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}.tar
85
				# josch> | h01ger: should you ever try --variant=standard, remember that that is not reproducible in stable because of #864082, #1004557, #1004558
86
				;;
87
88
		debootstrap|cdebootstrap)
				sudo $REALTOOL $SUITE $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}
89
				# work around unreproducible artifacts
90
				BAD_FILES="/etc/machine-id /var/cache/ldconfig/aux-cache"
91
				case ${TOOL} in
92
					debootstrap)	BAD_LOGFILES="/var/log/dpkg.log /var/log/alternatives.log /var/log/bootstrap.log"
93
							;;
94
					cdebootstrap)	BAD_LOGFILES="/var/log/dpkg.log /var/log/alternatives.log /var/log/bootstrap.log /var/log/apt/history.log /var/log/apt/term.log"
95
96
							;;
				esac
97
98
				for i in $BAD_LOGFILES ; do
					cleanup_unreproducible_file logfile "$i"
99
				done
100
101
				for i in $BAD_FILES ; do
					cleanup_unreproducible_file file "$i"
102
				done
103
				sudo tar --mtime="@$SOURCE_DATE_EPOCH" --clamp-mtime -C $BUILDDIR/$SUBDIR/${TOOL}/ -cf $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}.tar ${SUITE}
104
105
				sudo rm -rf --one-file-system $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}
				;;
106
		*)		output_echo "Failure: ${TOOL} is unsupported."
107
108
109
				exit 1
				;;
	esac
110
	mv $BUILDDIR/$SUBDIR $RESULTSDIR/ 1>/dev/null
111
done
112

113
output_echo "Done running ${TOOL} twice."
114

115
# show sha256sum results
116
sha256sum $RESULTSDIR/b1/${TOOL}/${SUITE}.tar $RESULTSDIR/b2/${TOOL}/${SUITE}.tar
117

118
# show human readable results
119
120
if diff $RESULTSDIR/b1/${TOOL}/${SUITE}.tar $RESULTSDIR/b2/${TOOL}/${SUITE}.tar ; then
	output_echo "Success: ${TOOL} of $SUITE is reproducible today."
121
else
122
	output_echo "Warning: ${TOOL} of $SUITE is not reproducible."
123
124
125
126
127
	# Run diffoscope on the images
	output_echo "Calling diffoscope on the results."
	TIMEOUT="240m"
	DIFFOSCOPE="$(schroot --directory /tmp -c chroot:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)"
	TMPDIR=${RESULTSDIR}
128
129
	#call_diffoscope ${TOOL} ${SUITE}.tar
	# the previous, temporarily disabled line is only useful if we also make the .html file visible...
130
	schroot --directory /tmp -c chroot:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --restructured-text $RESULTSDIR/${TOOL}_${SUITE}.txt $RESULTSDIR/b1/${TOOL}/${SUITE}.tar $RESULTSDIR/b2/${TOOL}/${SUITE}.tar || true # diffoscope will exi with error...
131
	cat $RESULTSDIR/${TOOL}_${SUITE}.txt
132
fi
133
134
135
136
137
138
139

cleanup success
# Turn off the trap
trap - INT TERM EXIT

# We reached the end, return with PASS
exit 0