Skip to content
Commits on Source (3)
Show whitespace changes
Inline Side-by-side
bin/reproducible_compare_Debian_sha1sums.sh
View file @ 9b6fd8e7
......@@ -61,6 +61,7 @@ case "$MODE" in
reverse) SORT="sort -r" ;;
forward) SORT="sort" ;;
*) SORT="sort" ; MODE="results" ; RESULTS=$(mktemp --tmpdir=$TMPDIR sha1-results-XXXXXXX) ; find $SHA1DIR -name "*REPRODUCIBLE.buster" > $RESULTS
JSONS=$(mktemp --tmpdir=$TMPDIR sha1-results-XXXXXXX) ; find $SHA1DIR -name "*.json" > $JSONS ;;
esac
packages="$(grep ^Package: $PACKAGES| awk '{print $2}' | $SORT | xargs echo)"
......@@ -69,24 +70,39 @@ unreproducible_packages=
cleanup_all() {
if [ "$MODE" = "results" ]; then
unknown_packages=$(awk '/ UNKNOWN: /{print $9}' $log)
unknown_count=$(echo $unknown_packages | wc -w)
reproducible_packages=$(awk '/ REPRODUCIBLE: /{print $9}' $log)
reproducible_count=$(echo $reproducible_packages | wc -w)
unreproducible_packages=$(awk '/ UNREPRODUCIBLE: /{print $9}' $log)
unreproducible_count=$(echo $unreproducible_packages | wc -w)
percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
percent_unknown=$(echo "scale=4 ; $unknown_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count+$unknown_count) * 100" | bc)
echo "-------------------------------------------------------------"
echo "unknown packages: $unknown_count: $unknown_packages"
echo
echo "reproducible packages: $reproducible_count: $reproducible_packages"
echo
echo "unreproducible packages: $unreproducible_count: $unreproducible_packages"
echo
echo "Statistics including packages (currently) in an unknown state"
echo "-------------------------------------------------------------"
echo "unknown packages in $RELEASE/amd64: $unknown_count: ($percent_unknown%)"
echo "reproducible packages in $RELEASE/amd64: $reproducible_count: ($percent_repro%)"
echo "unreproducible packages in $RELEASE/amd64: $unreproducible_count: ($percent_unrepro%)"
echo
percent_repro=$(echo "scale=4 ; $reproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
percent_unrepro=$(echo "scale=4 ; $unreproducible_count / ($reproducible_count+$unreproducible_count) * 100" | bc)
echo "Statistics of packages in known state only"
echo "-------------------------------------------------------------"
echo "reproducible packages in $RELEASE/amd64: $reproducible_count: ($percent_repro%)"
echo "unreproducible packages in $RELEASE/amd64: $unreproducible_count: ($percent_unrepro%)"
echo
echo
echo "$(du -sch $SHA1DIR)"
echo
rm $RESULTS
rm $RESULTS $JSONS
fi
rm $log $PACKAGES
}
......@@ -96,32 +112,53 @@ trap cleanup_all INT TERM EXIT
rm -f $SHA1DIR/*.lock # this is a tiny bit hackish, but also an elegant way to get rid of old locks...
# (locks are held for 30s only anyway and there is an 3/60000th chance of a race condition only anyway)
for package in $packages ; do
if [ "$MODE" = "results" ] ; then
for package in $packages ; do
result=$(grep "/${package}_" $RESULTS || true)
if [ -n "$result" ] ; then
if $(echo $result | grep -q UNREPRODUCIBLE) ; then
package_file=$(echo $result | sed 's#\.deb\.UNREPRODUCIBLE\.buster$#.deb#' )
count=1
SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
package_file=$(basename $package_file)
echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
#package_file=$(echo $result | sed 's#\.deb\.UNREPRODUCIBLE\.buster$#.deb#' )
#count=1
#SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
#package_file=$(basename $package_file)
#echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
echo "$(date -u) - UNREPRODUCIBLE: $package"
else
package_file=$(echo $result | sed 's#\.deb\.REPRODUCIBLE\.buster$#.deb#' )
count=$(cat ${package_file}.REPRODUCIBLE.$RELEASE)
#package_file=$(echo $result | sed 's#\.deb\.REPRODUCIBLE\.buster$#.deb#' )
#count=$(cat ${package_file}.REPRODUCIBLE.$RELEASE)
#SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
#package_file=$(basename $package_file)
#echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
echo "$(date -u) - REPRODUCIBLE: $package"
fi
continue
fi
json=$(grep "/${package}_" $JSONS || true)
if [ -n "$json" ] ; then
package_file=$(echo $json | sed 's#\.deb\.json$#.deb#' )
count=$(fmt ${package_file}.json | grep -c '\.buildinfo' || true)
SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
package_file=$(basename $package_file)
if [ "${count}" -ge 2 ]; then
echo $count > ${package_file}.REPRODUCIBLE.$RELEASE
echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
else
echo 1 > ${package_file}.UNREPRODUCIBLE.$RELEASE
echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
fi
continue
fi
echo "$(date -u) - UNKNOWN: $package"
done | tee $log
exit
fi
# only used by the runners
for package in $packages ; do
LOCK="$SHA1DIR/${package}.lock"
if [ -e $LOCK ] ; then
echo "$(date -u) - skipping locked package $package"
continue
elif [ ! "$MODE" = "results" ] ; then
# MODE=results is read-only
else
touch $LOCK
fi
version=$(grep-dctrl -X -P ${package} -s version -n $PACKAGES)
......@@ -130,29 +167,6 @@ for package in $packages ; do
pool_dir="$SHA1DIR/$(dirname $(grep-dctrl -X -P ${package} -s Filename -n $PACKAGES))"
mkdir -p $pool_dir
cd $pool_dir
if [ "$MODE" = "results" ] ; then
# this code block can be removed once all packages with existing results have been processed once...
if [ -e ${package_file}.REPRODUCIBLE.$RELEASE ] ; then
count=$(cat ${package_file}.REPRODUCIBLE.$RELEASE)
SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
elif [ -e ${package_file}.UNREPRODUCIBLE.$RELEASE ] ; then
count=1
SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
elif [ -e ${package_file}.json ] ; then
count=$(fmt ${package_file}.json | grep -c '\.buildinfo' || true)
SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
if [ "${count}" -ge 2 ]; then
echo $count > ${package_file}.REPRODUCIBLE.$RELEASE
echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
else
echo 1 > ${package_file}.UNREPRODUCIBLE.$RELEASE
echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
fi
fi
continue
fi
if [ ! -e ${package_file}.sha1output ] ; then
echo -n "$(date -u) - downloading... "
( schroot --directory $pool_dir -c chroot:jenkins-reproducible-${RELEASE}-diffoscope apt-get download ${package}/${RELEASE} 2>&1 |xargs echo ) || continue
......@@ -180,7 +194,7 @@ for package in $packages ; do
echo "$(date -u) - not updating data about ${package_file}"
fi
rm -f $LOCK
done | tee $log
done | tee -a $log
cleanup_all
trap - INT TERM EXIT