reproducible_debian_live_build.sh

#!/bin/bash
# vim: set noexpandtab:

# Copyright 2021-2022 Holger Levsen <holger@layer-acht.org>
# Copyright 2021-2022 Roland Clobus <rclobus@rclobus.nl>
# released under the GPLv2

# Coding convention: enforced by 'shfmt'

DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

# common code for tests.reproducible-builds.org
. /srv/jenkins/bin/reproducible_common.sh
set -e
set -o pipefail # see eg http://petereisentraut.blogspot.com/2010/11/pipefail.html

# Copy a single file to the web server
#
# Argument 1 = description (one word)
# Argument 2 = absolute path of a file that will be published on the web server
# Argument 3 = filename for the published file
publish_file() {
	local DESCRIPTION=$1
	local ORIGINAL_FILE=$2
	local PUBLISHED_NAME
	PUBLISHED_NAME=$(basename "$3")

	ssh jenkins@jenkins.debian.net "${DESCRIPTION}" "${ORIGINAL_FILE}" "${PUBLISHED_NAME}"
}

cleanup() {
	local RESULT=$1
	output_echo "Publishing results."

	if [ "${RESULT}" == "success" ]; then
		output_echo "Info: no differences found."

		# Upload the ISO file and its summary to the web server
		ISONAME=${CONFIGURATION}-${DEBIAN_VERSION}.iso
		publish_file ISOfile ${RESULTSDIR}/b1/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso ${ISONAME}
		publish_file Summary ${RESULTSDIR}/summary_build1.txt ${CONFIGURATION}-${DEBIAN_VERSION}.txt

		# Invoke openQA
		DESKTOP=${CONFIGURATION}
		if [ ${DESKTOP} == "smallest-build" ]; then
			DESKTOP="text"
		elif [ ${DESKTOP} == "standard" ]; then
			DESKTOP="text"
		fi
		CHECKSUM=$(grep "Checksum:" ${RESULTSDIR}/summary_build1.txt | cut -f 2 -d " ")
		#openqa-cli api -X POST isos ISO=${SNAPSHOT_TIMESTAMP}_${DEBIAN_VERSION}_${CONFIGURATION}.iso DISTRI=debian VERSION=${DEBIAN_VERSION}_${CONFIGURATION} FLAVOR=live-build ARCH=x86_64 BUILD=:${CONFIGURATION}_${DEBIAN_VERSION}_${SNAPSHOT_TIMESTAMP} DESKTOP=${DESKTOP} CHECKSUM=${CHECKSUM} TIMESTAMP=${SNAPSHOT_TIMESTAMP} ISO_URL=https://tests.reproducible-builds.org/debian_live_build/${ISONAME} --odn --apikey ${OPENQA_APIKEY} --apisecret ${OPENQA_APISECRET}
	else
		if [ -f "${RESULTSDIR}/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso.html" ]; then
			# Publish the output of diffoscope, there are differences
			publish_file DiffoscopeOutput ${RESULTSDIR}/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso.html ${CONFIGURATION}-${DEBIAN_VERSION}.html
		else
			output_echo "Error: Something went wrong."
			printenv >environment.txt
			publish_file Environment ${PWD}/environment.txt ${CONFIGURATION}-${DEBIAN_VERSION}.txt
		fi
	fi

	output_echo "Cleanup ${RESULT}"
	# Cleanup the workspace
	if [ ! -z "${BUILDDIR}" ]; then
		sudo rm -rf --one-file-system ${BUILDDIR}
	fi
	# Cleanup the results
	if [ ! -z "${RESULTSDIR}" ]; then
		rm -rf --one-file-system ${RESULTSDIR}
	fi
}

#
# main: follow https://wiki.debian.org/ReproducibleInstalls/LiveImages
#

# Argument 1 = image type
export CONFIGURATION="$1"

# Argument 2 = Debian version
export DEBIAN_VERSION="$2"

# Two arguments are required
if [ -z "${CONFIGURATION}" -o -z "${DEBIAN_VERSION}" ]; then
	output_echo "Error: Bad command line arguments."
	exit 1
fi

# Configuration option: Select the server for the repository
# Allowed values:
#  auto (default): Detect the best available server
#  snapshot (preferred): Use a snapshot server
#  archive (fallback): Use deb.debian.org if a snapshot server is not available
#
# Note that 'archive' allows only for a time window of about 6 hours to test for reproducibility
export REPOSERVER=auto

if [ "${REPOSERVER}" == "auto" ]; then
	set +e # We are interested in the return value
	wget --quiet --no-hsts http://snapshot.notset.fr/mr/timestamp/debian/latest --output-document latest
	RETURNVALUE=$?
	rm -f latest
	set -e
	if [ ${RETURNVALUE} -eq 0 ]; then
		export REPOSERVER=snapshot
	else
		# The snapshot server is not available
		output_echo "Info: The snapshot server is not available, using deb.debian.org instead."
		export REPOSERVER=archive
	fi
fi

# Cleanup if something goes wrong
trap cleanup INT TERM EXIT

if $DEBUG; then
	export WGET_OPTIONS=
else
	export WGET_OPTIONS=--quiet
fi

# Randomize start time
delay_start

# Generate and use an isolated workspace
export PROJECTNAME="live-build"
mkdir -p /srv/workspace/live-build
export BUILDDIR=$(mktemp --tmpdir=/srv/workspace/live-build -d -t ${CONFIGURATION}-${DEBIAN_VERSION}.XXXXXXXX)
cd ${BUILDDIR}
export RESULTSDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t ${PROJECTNAME}-${CONFIGURATION}-${DEBIAN_VERSION}-XXXXXXXX) # accessible in schroots, used to compare results

# Fetch the rebuild script (and nothing else)
git clone https://salsa.debian.org/live-team/live-build.git rebuild_script --no-checkout --depth 1
cd rebuild_script
git checkout HEAD test/rebuild.sh
cd ..

# First build
output_echo "Running the rebuild script for the 1st build."
set +e # We are interested in the return value
# The 3rd argument is provided to allow for local testing of this script.
# - If SNAPSHOT_TIMESTAMP is already set, use that value
# - If SNAPSHOT_TIMESTAMP is not set, use REPOSERVER
export SNAPSHOT_TIMESTAMP="${SNAPSHOT_TIMESTAMP:-${REPOSERVER}}"
rebuild_script/test/rebuild.sh "$1" "$2" "${SNAPSHOT_TIMESTAMP}"
RETURNVALUE=$?
grep --quiet --no-messages "Build result: 0" summary.txt
BUILD_OK_FOUND=$?
set -e

if [ ${RETURNVALUE} -eq 99 -a "${SNAPSHOT_TIMESTAMP}" == "archive" ]; then
	# The archive was updated while building. Retry
	output_echo "Info: during the first build the archive was updated, now trying again."
	rebuild_script/test/rebuild.sh "$1" "$2" "${SNAPSHOT_TIMESTAMP}"
elif [ ${RETURNVALUE} -ne 0 -o ${BUILD_OK_FOUND} -ne 0 ]; then
	# Something went wrong. Perhaps an alternative timestamp is proposed
	SNAPSHOT_TIMESTAMP=$(grep --no-messages "Alternative timestamp:" summary.txt | cut -f 3 -d " ")
	if [ -z ${SNAPSHOT_TIMESTAMP} ]; then
		output_echo "Error: the image could not be built, no alternative was proposed."
		exit 1
	fi
	output_echo "Warning: the build failed with ${RETURNVALUE}. The latest snapshot might not be complete (yet). Now trying again using the previous snapshot instead."
	rebuild_script/test/rebuild.sh "$1" "$2" "${SNAPSHOT_TIMESTAMP}"
else
	# Determine the value of SNAPSHOT_TIMESTAMP for the second build
	# If 'archive' is used for the first build, use it for the second build too and hope that the same timestamp will still be available
	if [ "${SNAPSHOT_TIMESTAMP}" != "archive" ]; then
		export SNAPSHOT_TIMESTAMP=$(grep --no-messages "Snapshot timestamp:" summary.txt | cut -f 3 -d " ")
	fi
fi

# Move the image away
mkdir -p ${RESULTSDIR}/b1/${PROJECTNAME}/${CONFIGURATION}
mv live-image-amd64.hybrid.iso ${RESULTSDIR}/b1/${PROJECTNAME}/${CONFIGURATION}
mv summary.txt ${RESULTSDIR}/summary_build1.txt

# Second build
output_echo "Running the rebuild script for the 2nd build."
set +e # We are interested in the return value
rebuild_script/test/rebuild.sh "$1" "$2" "${SNAPSHOT_TIMESTAMP}"
RETURNVALUE=$?
set -e

if [ "${SNAPSHOT_TIMESTAMP}" == "archive" ]; then
	if [ ${RETURNVALUE} -eq 99 ]; then
		# The archive was updated while building. The content might not be consistent.
		# Discard the first build and retry
		output_echo "Info: during the second build the archive was updated, now trying again."
		mv live-image-amd64.hybrid.iso ${RESULTSDIR}/b1/${PROJECTNAME}/${CONFIGURATION}
		mv summary.txt ${RESULTSDIR}/summary_build1.txt
		rebuild_script/test/rebuild.sh "$1" "$2" "${SNAPSHOT_TIMESTAMP}"
	elif [ $(stat ${RESULTSDIR}/b1/${PROJECTNAME}/${CONFIGURATION}/live-image-amd64.hybrid.iso live-image-amd64.hybrid.iso | grep Modify: | uniq | wc -l) -ne 1 ]; then
		# The timestamps are different. Discard the first build and retry
		output_echo "Info: between the builds the archive was updated, now trying again."
		mv live-image-amd64.hybrid.iso ${RESULTSDIR}/b1/${PROJECTNAME}/${CONFIGURATION}
		mv summary.txt ${RESULTSDIR}/summary_build1.txt
		rebuild_script/test/rebuild.sh "$1" "$2" "${SNAPSHOT_TIMESTAMP}"
	fi
fi

# Move the image away
mkdir -p ${RESULTSDIR}/b2/${PROJECTNAME}/${CONFIGURATION}
mv live-image-amd64.hybrid.iso ${RESULTSDIR}/b2/${PROJECTNAME}/${CONFIGURATION}
mv summary.txt ${RESULTSDIR}/summary_build2.txt

# Clean up
output_echo "Running lb clean after the 2nd build."
sudo lb clean --purge

# The workspace is no longer required
cd ..

# Run diffoscope on the images
output_echo "Calling diffoscope on the results."
TIMEOUT="240m"
DIFFOSCOPE="$(schroot --directory /tmp -c chroot:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)"
TMPDIR=${RESULTSDIR}
call_diffoscope ${PROJECTNAME} ${CONFIGURATION}/live-image-amd64.hybrid.iso

cleanup success

# Turn off the trap
trap - INT TERM EXIT

# We reached the end, return with PASS
exit 0