reproducible: properly check and handle the schroot (under|over)lays

I'm not really sure any of that is actually needed, besides laeving the management to schroot, but here we are

Signed-off-by: Mattia Rizzolo <mattia@debian.org>

bin/reproducible_maintenance.sh

@@ -420,36 +420,38 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then
 			fi
 		done
 	fi
-	# TODO: rewrite the below to not mess with schroot internals so happily.
-	#       (i.e., it should verify first )hat those files aren't referenced in sessions files first of all)
-	#for path in session union/underlay union/overlay ; do
-	#	dir=/var/lib/schroot/$path/
-	#	if [ "$path" = "session" ] ; then
-	#		FILETYPE=f
-	#		RM=rm
-	#	else
-	#		FILETYPE=d
-	#		RM=rmdir
-	#	fi
-	#	OLDSTUFF=$(find "$dir" -name "jenkins-reproducible-*-diffoscope-*" -mindepth 1 -maxdepth 1 -type $FILETYPE -mtime +1 -exec ls -lad {} \; || true)
-	#	if [ ! -z "$OLDSTUFF" ]; then
-	#		echo
-	#		echo "data in $dir older than 2 days found, which will be deleted:"
-	#		echo "$OLDSTUFF" | sed "s#$dir#./#g"
-
-	#		echo
-	#		find "$dir" -name "jenkins-reproducible-*-diffoscope-*" -mindepth 1 -maxdepth 1 -type $FILETYPE -mtime +1 -exec sudo $RM -v {} \; || true
-	#		OLDSTUFF=$(find "$dir" -name "jenkins-reproducible-*-diffoscope-*" -mindepth 1 -maxdepth 1 -type $FILETYPE -mtime +1 -exec ls -lad {} \; || true)
-	#		if [ ! -z "$OLDSTUFF" ]; then
-	#			echo
-	#			echo "Warning: Tried, but failed to delete these in $dir:"
-	#			echo "$OLDSTUFF" | sed "s#$dir#./#g"
-	#			echo "Manual cleanup needed."
-	#		fi
-	#		echo
-	#		DIRTY=true
-	#	fi
-	#done
+	for path in underlay overlay ; do
+		# give these directories an extra day over the above sessions, in case they have a slightly older mtime.
+		mapfile -t OLDSTUFF < <(find "/var/lib/schroot/union/$path" -name "jenkins-reproducible-*-diffoscope-*" -maxdepth 1 -type d -mtime +2 -exec ls -lad {} \;)
+		if [ ${#OLDSTUFF[@]} -ne 0 ]; then
+			echo
+			echo "Found old schroot (from diffoscope) $path, which will be investigated now:"
+			echo "${OLDSTUFF[@]}"
+			echo
+			pushd /var/lib/schroot/session
+			for dir_ls in "${OLDSTUFF[@]}"; do
+				# don't use `basename`,etc because dir_ls contains the whole output from ls -l, not just the path
+				dir_path="/${dir_ls#*/}"
+				mapfile -t matches < <(grep -lF "union-overlay-directory=$dir_path")
+				if [ ${#matches[@]} -eq 0 ]; then
+					echo
+					echo "This $path is not referenced by any existing schroot, deleting it now."
+					set -x
+					echo sudo rm -rf --one-file-system "$dir_path"
+					if ! "$DEBUG" ; then set +x ; fi
+				fi
+			done
+		fi
+		mapfile -t OLDSTUFF < <(find "/var/lib/schroot/union/$path" -name "jenkins-reproducible-*-diffoscope-*" -maxdepth 1 -type d -mtime +2 -exec ls -lad {} \;)
+		if [ ${#OLDSTUFF[@]} -ne 0 ]; then
+			echo
+			echo "Warning: Tried, but failed to delete these in /var/lib/schroot/union/$dir:"
+			echo "${OLDSTUFF[@]}"
+			echo "Manual cleanup needed."
+			echo
+			DIRTY=true
+		fi
+	done
 fi
 
 # cleanup old schroots

hosts/common/etc/sudoers.d/jenkins

@@ -23,6 +23,7 @@ jenkins ALL=  \
 	/bin/rm -rf --one-file-system /schroots/*, \
 	/bin/rm -rf --one-file-system /srv/workspace/pbuilder/*, \
 	/bin/rm -rf --one-file-system /var/lib/sbuild/build/*, \
+	/bin/rm -rf --one-file-system /var/lib/schroot/union/*, \
 	/bin/rm -rf --one-file-system /tmp/*, \
 	/bin/rm -rv --one-file-system /tmp/*, \
 	/bin/mv /chroots/* /schroots/*, \