reproducible_debstrap.sh

#!/bin/bash
# vim: set noexpandtab:

# Copyright 2021-2022 Holger Levsen <holger@layer-acht.org>
# released under the GPLv2

DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

# common code for tests.reproducible-builds.org
. /srv/jenkins/bin/reproducible_common.sh
set -e
set -o pipefail # see eg http://petereisentraut.blogspot.com/2010/11/pipefail.html

cleanup() {
	local RESULT=$1
	output_echo "Cleanup ${RESULT}"
	# Cleanup the workspace and results directory
	if [ ! -z "$BUILDDIR" ]; then
		output_echo "Removing ${BUILDDIR}"
		sudo rm -rf --one-file-system ${BUILDDIR}
	fi
	if [ ! -z "$RESULTSDIR" ]; then
		output_echo "Removing ${RESULTSDIR}"
		rm -rf --one-file-system ${RESULTSDIR}
	fi
}

cleanup_unreproducible_file(){
	local TYPE=$1
	local FILE=$2
	if sudo file -E $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}/$FILE >/dev/null 2>&1 ; then
		output_echo "Warning: modifying $TOOL result, deleting unreproducible $TYPE $FILE"
		sudo rm -rf --one-file-system $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}/$FILE
	else
		sudo file $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}/$FILE
		output_echo "Warning: shall remove $FILE but it does not exist. Maybe $TOOL was improved."
	fi
}

# Init some variables
export TOOL="$1"
export SUITE="$2"
output_echo "About to bootstrap $SUITE using $TOOL version $(dpkg-query  --showformat='${Version}' --show $TOOL)."

if [ "$SUITE" != "unstable" ] ; then
	CODENAME=$SUITE
else
	CODENAME=sid
fi

export BUILDDIR=$(mktemp --tmpdir=/srv/workspace/ -d -t ${TOOL}-${SUITE}.XXXXXXXX)
export RESULTSDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t ${TOOL}-${SUITE}.XXXXXXXX) # accessible in schroots, used to compare results

# Cleanup if something goes wrong
trap cleanup INT TERM EXIT

# Randomize start time
delay_start

# SOURCE_DATE_EPOCH needs to be set to allow clamping file timestamps newer than this.
S_D_E_DATE="$(grep $CODENAME /usr/share/distro-info/debian.csv | awk  -F',' '{print $4}')"
export SOURCE_DATE_EPOCH="$(date +%s -d $S_D_E_DATE)"
output_echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH that is $S_D_E_DATE which is when $SUITE was created."

# Actual run ${TOOL} twice
for LOOP in "first" "second" ; do
	case $LOOP in
		first)	SUBDIR=b1
			ACTION="Verbosely running"
			REALTOOL=$TOOL
			;;
		second)	SUBDIR=b2
			ACTION="Running"
			case $TOOL in
				mmdebstrap)	REALTOOL="mmdebstrap -v" ;;
				debootstrap)	REALTOOL="debootstrap --verbose" ;;
				cdebootstrap)	REALTOOL="cdebootstrap --verbose" ;;
			esac
			;;
	esac
	output_echo "$ACTION ${TOOL} $SUITE for the $LOOP run."
	mkdir -p $BUILDDIR/$SUBDIR/${TOOL}
	case ${TOOL} in
		mmdebstrap)	sudo $REALTOOL $SUITE > $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}.tar
				# josch> | h01ger: should you ever try --variant=standard, remember that that is not reproducible in stable because of #864082, #1004557, #1004558
				;;
		debootstrap|cdebootstrap)
				sudo $REALTOOL $SUITE $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}
				# work around unreproducible artifacts
				BAD_FILES="/etc/machine-id /var/cache/ldconfig/aux-cache"
				case ${TOOL} in
					debootstrap)	BAD_LOGFILES="/var/log/dpkg.log /var/log/alternatives.log /var/log/bootstrap.log"
							;;
					cdebootstrap)	BAD_LOGFILES="/var/log/dpkg.log /var/log/alternatives.log /var/log/bootstrap.log /var/log/apt/history.log /var/log/apt/term.log"
							;;
				esac
				for i in $BAD_LOGFILES ; do
					cleanup_unreproducible_file logfile "$i"
				done
				for i in $BAD_FILES ; do
					cleanup_unreproducible_file file "$i"
				done
				sudo tar --mtime="@$SOURCE_DATE_EPOCH" --clamp-mtime -C $BUILDDIR/$SUBDIR/${TOOL}/ -cf $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}.tar ${SUITE}
				sudo rm -rf --one-file-system $BUILDDIR/$SUBDIR/${TOOL}/${SUITE}
				;;
		*)		output_echo "Failure: ${TOOL} is unsupported."
				exit 1
				;;
	esac
	mv $BUILDDIR/$SUBDIR $RESULTSDIR/ 1>/dev/null
done

output_echo "Done running ${TOOL} twice."

# show sha256sum results
sha256sum $RESULTSDIR/b1/${TOOL}/${SUITE}.tar $RESULTSDIR/b2/${TOOL}/${SUITE}.tar

# show human readable results
if diff $RESULTSDIR/b1/${TOOL}/${SUITE}.tar $RESULTSDIR/b2/${TOOL}/${SUITE}.tar ; then
	output_echo "Success: ${TOOL} of $SUITE is reproducible today."
else
	output_echo "Warning: ${TOOL} of $SUITE is not reproducible."
	# Run diffoscope on the images
	output_echo "Calling diffoscope on the results."
	TIMEOUT="240m"
	DIFFOSCOPE="$(schroot --directory /tmp -c chroot:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)"
	TMPDIR=${RESULTSDIR}
	#call_diffoscope ${TOOL} ${SUITE}.tar
	# the previous, temporarily disabled line is only useful if we also make the .html file visible...
	schroot --directory /tmp -c chroot:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --restructured-text $RESULTSDIR/${TOOL}_${SUITE}.txt $RESULTSDIR/b1/${TOOL}/${SUITE}.tar $RESULTSDIR/b2/${TOOL}/${SUITE}.tar || true # diffoscope will exi with error...
	cat $RESULTSDIR/${TOOL}_${SUITE}.txt
fi

cleanup success
# Turn off the trap
trap - INT TERM EXIT

# We reached the end, return with PASS
exit 0