bin/reproducible_compare_Debian_sha1sums.sh

@@ -23,7 +23,6 @@ set -e
 # - show results in 'normal pages' 
 # - store date when a package was last reproduced... (and constantly do that...)
 # - throw away results (if none has been|which have not) signed with a tests.r-b.o key
-# - json files from buildinfo.d.n are never re-downloaded
 # - rebuilder:
 #   - run on osuosl173
 #   - loop randomly through unreproducible packages first, then reproducible ones. do one attempt only.
@@ -51,16 +50,20 @@ bdn_url="https://buildinfo.debian.net/api/v1/buildinfos/checksums/sha1"
 log=$(mktemp --tmpdir=$TMPDIR sha1-log-XXXXXXX)
 echo "$(date -u) - logfile used is $log"
 
+FORCE_DATE=$(date -u -d "14 days ago" '+%Y-%m-%d')
+DUMMY_FILE=$(mktemp --tmpdir=$TMPDIR sha1-date-XXXXXXX)
+touch -d "$(date '+%Y-%m-%d') 00:00 UTC" $DUMMY_FILE
+
 SHA1DIR=/srv/reproducible-results/debian-sha1
 mkdir -p $SHA1DIR
 
 PACKAGES=$(mktemp --tmpdir=$TMPDIR sha1-pkgs-XXXXXXX)
 schroot --directory  $SHA1DIR -c chroot:jenkins-reproducible-${RELEASE}-diffoscope cat /var/lib/apt/lists/cdn-fastly.deb.debian.org_debian_dists_${RELEASE}_main_binary-amd64_Packages > $PACKAGES
 case "$MODE" in
-	random)		SORT="sort -R";;
-	reverse)	SORT="sort -r" ;;
-	forward)	SORT="sort" ;;
-	*)		SORT="sort" ; MODE="results" ; RESULTS=$(mktemp --tmpdir=$TMPDIR sha1-results-XXXXXXX) ; find $SHA1DIR -name "*REPRODUCIBLE.buster" > $RESULTS ;;
+	random)		SORT="sort -u -R";;
+	reverse)	SORT="sort -u -r" ;;
+	forward)	SORT="sort -u" ;;
+	*)		SORT="sort -u" ; MODE="results" ; RESULTS=$(mktemp --tmpdir=$TMPDIR sha1-results-XXXXXXX) ; find $SHA1DIR -name "*REPRODUCIBLE.buster" > $RESULTS ;;
 esac
 packages="$(grep ^Package: $PACKAGES| awk '{print $2}' | $SORT | xargs echo)"
 
@@ -84,32 +87,21 @@ cleanup_all() {
 		percent_unknown=$(echo "scale=4 ; $unknown_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
 		percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
 		percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+		percent_binnmu_repro=$(echo "scale=4 ; $reproducible_binnmu / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+		percent_binnmu_unrepro=$(echo "scale=4 ; $unreproducible_binnmu / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+		percent_arch_all_repro=$(echo "scale=4 ; $reproducible_arch_all / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+		percent_arch_all_unrepro=$(echo "scale=4 ; $unreproducible_arch_all / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+		percent_arch_amd64_repro=$(echo "scale=4 ; $reproducible_arch_amd64 / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
+		percent_arch_amd64_unrepro=$(echo "scale=4 ; $unreproducible_arch_amd64 / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
 		echo "-------------------------------------------------------------"
-		echo "unknown packages: $unknown_count: $unknown_packages"
-		echo
-		echo "reproducible packages: $reproducible_count: $reproducible_packages"
-		echo
-		echo "unreproducible packages: $unreproducible_count: $unreproducible_packages"
 		echo
-		echo "ftp.debian.org package reproducibility statistics including packages (currently) in an unknown state"
+		echo "ftp.debian.org package reproducibility statistics"
 		echo "-------------------------------------------------------------"
 		echo "packages in unknown reproducibility state in $RELEASE/amd64: $unknown_count: ($percent_unknown%)"
 		echo "reproducible packages in $RELEASE/amd64: $reproducible_count: ($percent_repro%)"
 		echo "unreproducible packages in $RELEASE/amd64: $unreproducible_count: ($percent_unrepro%)"
 		echo "total number of packages in $RELEASE/amd64: $(echo $reproducible_count+$unreproducible_count+$unknown_count | bc)"
 		echo
-		percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
-		percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
-		percent_binnmu_repro=$(echo "scale=4 ; $reproducible_binnmu / ($reproducible_count+$unreproducible_count) * 100" | bc)
-		percent_binnmu_unrepro=$(echo "scale=4 ; $unreproducible_binnmu / ($reproducible_count+$unreproducible_count) * 100" | bc)
-		percent_arch_all_repro=$(echo "scale=4 ; $reproducible_arch_all / ($reproducible_count+$unreproducible_count) * 100" | bc)
-		percent_arch_all_unrepro=$(echo "scale=4 ; $unreproducible_arch_all / ($reproducible_count+$unreproducible_count) * 100" | bc)
-		percent_arch_amd64_repro=$(echo "scale=4 ; $reproducible_arch_amd64 / ($reproducible_count+$unreproducible_count) * 100" | bc)
-		percent_arch_amd64_unrepro=$(echo "scale=4 ; $unreproducible_arch_amd64 / ($reproducible_count+$unreproducible_count) * 100" | bc)
-		echo "ftp.debian.org package reproducibility statistics of packages in known states only"
-		echo "-------------------------------------------------------------"
-		echo "reproducible packages in $RELEASE/amd64: $reproducible_count: ($percent_repro%)"
-		echo "unreproducible packages in $RELEASE/amd64: $unreproducible_count: ($percent_unrepro%)"
 		echo
 		echo "reproducible binNMUs in $RELEASE/amd64: $reproducible_binnmu: ($percent_binnmu_repro%)"
 		echo "unreproducible binNMU in $RELEASE/amd64: $unreproducible_binnmu: ($percent_binnmu_unrepro%)"
@@ -167,10 +159,10 @@ for package in $packages ; do
 	else
 		touch $LOCK
 	fi
-	version=$(grep-dctrl -X -P ${package} -s version -n $PACKAGES)
-	arch=$(grep-dctrl -X -P ${package} -s Architecture -n $PACKAGES)
+	version=$(grep-dctrl -X -P ${package} -s Version -n $PACKAGES | head -1)
+	arch=$(grep-dctrl -X -P ${package} -s Architecture -n $PACKAGES | head -1)
 	package_file="${package}_$(echo $version | sed 's#:#%3a#')_${arch}.deb"
-	pool_dir="$SHA1DIR/$(dirname $(grep-dctrl -X -P ${package} -s Filename -n $PACKAGES))"
+	pool_dir="$SHA1DIR/$(dirname $(grep-dctrl -X -P ${package} -s Filename -n $PACKAGES | head -1))"
 	mkdir -p $pool_dir
 	cd $pool_dir
 	if [ ! -e ${package_file}.sha1output ] ; then
@@ -184,7 +176,8 @@ for package in $packages ; do
 			rm ${package}_*REPRODUCIBLE.$RELEASE
 		fi
 	fi
-	if [ ! -e ${package_file}.json ]; then
+	# download .json if non existing or older than $DUMMY_FILE
+	if [ ! -e ${package_file}.json ] || [ ${package_file}.json -ot $DUMMY_FILE ]; then
 		echo "$(date -u) - downloading .json for ${package_file} (${SHA1SUM_PKG}) from buildinfo.debian.net"
 		wget --quiet -O ${package_file}.json ${bdn_url}/${SHA1SUM_PKG} || echo "WARNING: failed to download ${bdn_url}/${SHA1SUM_PKG}"
 		count=$(fmt ${package_file}.json | grep -c '\.buildinfo' || true)

job-cfg/reproducible.yaml

@@ -606,7 +606,6 @@
                     my_timed: '56 * * * *'
                     my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
                     my_hname: 'osuosl173'
-                    my_disabled: true
                 - 'compare_Debian_sha1sums_random_runner_1':
                     my_description: 'Fetch .buildinfo files of Debian binary packages (in random order)'
                     my_timed: '56 * * * *'