bin/reproducible_compare_Debian_sha1sums.sh

 #!/bin/bash
 
-# as posted by Vagrant on https://lists.reproducible-builds.org/pipermail/rb-general/2018-October/001239.html
-
 # Copyright 2019 Holger Levsen <holger@layer-acht.org>
 # released under the GPLv=2+
+#
+# based on an idea by Vagrant Cascadian <vagrant@debian.org>
+# see https://lists.reproducible-builds.org/pipermail/rb-general/2018-October/001239.html
 
 DEBUG=false
 . /srv/jenkins/bin/common-functions.sh
@@ -31,6 +32,7 @@ echo 'this is an early prototype...'
 echo
 echo
 RELEASE=buster
+MODE="$1"
 
 bdn_url="https://buildinfo.debian.net/api/v1/buildinfos/checksums/sha1"
 log=$(mktemp --tmpdir=$TMPDIR sha1-comp-XXXXXXX)
@@ -39,10 +41,14 @@ SHA1DIR=/srv/reproducible-results/debian-sha1
 mkdir -p $SHA1DIR
 cd $SHA1DIR
 
-# downloading (and keeping) all the packages is also too much, but let's prototype this... (and improve later)
 PACKAGES=$(mktemp --tmpdir=$TMPDIR sha1-comp-XXXXXXX)
 schroot --directory  $SHA1DIR -c chroot:jenkins-reproducible-${RELEASE}-diffoscope cat /var/lib/apt/lists/cdn-fastly.deb.debian.org_debian_dists_${RELEASE}_main_binary-amd64_Packages > $PACKAGES
-packages="$(grep ^Package: $PACKAGES| awk '{print $2}' | sort | xargs echo)"
+case MODE in
+	random)		SORT="sort -R";;
+	reverse)	SORT="sort -r" ;;
+	*)		SORT="sort" ;;
+esac
+packages="$(grep ^Package: $PACKAGES| awk '{print $2}' | $SORT | xargs echo)"
 
 reproducible_packages=
 unreproducible_packages=
@@ -73,9 +79,19 @@ cleanup_all() {
 
 trap cleanup_all INT TERM EXIT
 
+rm -f *.lock	# this is a tiny bit hackish, but also an elegant way to get rid of old locks...
+		# (locks are held for 30s only anyway and there is an 3/60000th chance of a race condition only anyway)
+
 for package in $packages ; do
 	cd $SHA1DIR
 	echo
+	LOCK="$SHA1DIR/${package}.lock"
+	if [ -e $LOCK ] ; then
+		echo "$(date -u) - skipping locked package $package"
+		continue
+	else
+		touch $LOCK
+	fi
 	echo "$(date -u) - checking whether we have seen the .deb for $package before"
 	version=$(grep-dctrl -X -P ${package} -s version -n $PACKAGES)
 	arch=$(grep-dctrl -X -P ${package} -s Architecture -n $PACKAGES)
@@ -106,6 +122,7 @@ for package in $packages ; do
 	else
 		echo "$(date -u) - UNREPRODUCIBLE: $package_file: $SHA1SUM_PKG on ftp.debian.org, but nowhere else."
 	fi
+	rm -f $LOCK
 done | tee $log
 
 cleanup_all

job-cfg/reproducible.yaml

@@ -584,6 +584,14 @@
                 - 'compare_Debian_sha1sums':
                     my_description: 'Compare sha1sums of Debian binary packages with rebuild ones'
                     my_timed: ''
+                - 'compare_Debian_sha1sums_reverse_runner':
+                    my_description: 'Compare sha1sums of Debian binary packages (in reverse order) with rebuild ones'
+                    my_timed: ''
+                    my_shell: '/srv/jenkins/bin/reproducible_compare_Debian_sha1sums.sh reverse'
+                - 'compare_Debian_sha1sums_random_runner':
+                    my_description: 'Compare sha1sums of Debian binary packages (in random order) with rebuild ones'
+                    my_timed: ''
+                    my_shell: '/srv/jenkins/bin/reproducible_compare_Debian_sha1sums.sh random'
             my_shellext: ".sh"
             my_shell: '/srv/jenkins/bin/reproducible_{my_task}{my_shellext}'
             my_hname: ''