reproducible_maintenance.sh 25.9 KB
Newer Older
1
#!/bin/bash
2
# vim: set noexpandtab:
3

4
# Copyright 2014-2019 Holger Levsen <holger@layer-acht.org>
5
#         © 2015-2018 Mattia Rizzolo <mattia@debian.org>
6 7
# released under the GPLv=2

8
DEBUG=false
9 10 11 12 13 14
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

# common code defining db access
. /srv/jenkins/bin/reproducible_common.sh

15
DIRTY=false
16
REP_RESULTS=/srv/reproducible-results
17

18 19 20

# query reproducible database, print output
query_to_print() {
21
	printf "$(psql -c "$@")"
22 23
}

24
# backup db
25
if [ "$HOSTNAME" = "$MAINNODE" ] ; then
26
	echo "$(date -u) - backup db and update public copy."
27 28 29 30 31 32
	# prepare backup
	mkdir -p $REP_RESULTS/backup

	# keep 30 days and the 1st of the month
	DAY=(date -d "30 day ago" '+%d')
	DATE=$(date -d "30 day ago" '+%Y-%m-%d')
33 34 35
	BACKUPFILE="$REP_RESULTS/backup/reproducible_$DATE.sql.xz"
	if [ "$DAY" != "01" ] &&  [ -f "$BACKUPFILE" ] ; then
		rm -f "$BACKUPFILE"
36 37
	fi

38
	# Make a daily backup of database
39
	DATE=$(date '+%Y-%m-%d')
40 41 42
	BACKUPFILE="$REP_RESULTS/backup/reproducible_$DATE.sql"
	if [ ! -f $BACKUPFILE.xz ] ; then
		# make the backup
43
		DATE=$(date '+%Y-%m-%d')
44
		pg_dump -x -O $PGDATABASE > "$BACKUPFILE"
45
		xz "$BACKUPFILE"
46

47 48
		# make the backup public
		ln -s -f "$BACKUPFILE.xz" $BASE/reproducible.sql.xz
49 50

		# recreate documentation of database
51
		postgresql_autodoc -d $PGDATABASE -t html -f "$BASE/reproducibledb"
52
	fi
53 54
fi

55 56 57
#
# we fail hard
#
58
set -e
59

60 61 62 63 64 65 66 67 68 69 70
#
# find too large files in /var/log
#
echo "$(date -u) - Looking for too large files in /var/log/"
TOOBIG=$(find /var/log -size +8G -exec ls -lah {} \; 2>/dev/null || true)
if [ ! -z "$TOOBIG" ] ; then
	echo
	echo "$(date -u) - Warning: too large files found in /var/log:"
	echo "$TOOBIG"
	echo
	DIRTY=true
71
	if [ -n "$(find /var/log -size +32G 2> >(grep -v 'Permission denied'))" ] ; then
72 73 74 75 76
		echo "$(date -u) - Error, more than 32gb is just wrong..."
		exit 1
	fi
fi

77 78 79
#
# delete old temp directories
#
80
echo "$(date -u) - Deleting temp directories in $REP_RESULTS/rbuild-debian, older than 3 days."
81
OLDSTUFF=$(find $REP_RESULTS/rbuild-debian -maxdepth 1 -type d -mtime +2 -name "tmp.*" -exec ls -lad {} \; 2>/dev/null|| true)
82 83 84
if [ ! -z "$OLDSTUFF" ] ; then
	echo
	echo "Old temp directories found in $REP_RESULTS/rbuild-debian"
85
	find $REP_RESULTS/rbuild-debian -maxdepth 1 -type d -mtime +2 -name "tmp.*" -exec rm -rv --one-file-system {} \; || true
86 87 88 89 90
	echo "These old directories have been deleted."
	echo
	DIRTY=true
fi

91
#
92
# delete old temp directories in /tmp (probably only useful on osuosl171+172)
93 94
#
echo "$(date -u) - Deleting temporary directories in /tmp, older than 3 days."
95
OLDSTUFF=$(find /tmp -maxdepth 1 -type d -mtime +2 -regextype egrep -regex '/tmp/(tmp.*|Test.*|usession-release.*|.*test.*)' -exec ls -lad {} \; || true)
96 97 98
if [ ! -z "$OLDSTUFF" ] ; then
	echo
	echo "Old temp directories found in /tmp"
99
	find /tmp -maxdepth 1 -type d -mtime +2 -regextype egrep -regex '/tmp/(tmp.*|Test.*|usession-release.*|.*test.*)' -exec sudo rm -rv --one-file-system {} \; || true
100 101 102 103 104
	echo "These old directories have been deleted."
	echo
	DIRTY=true
fi

105 106 107 108 109 110 111 112 113 114
#
# delete old pbuilder build directories
#
if [ -d /srv/workspace/pbuilder/ ] ; then
	echo "$(date -u) - Deleting pbuilder build directories, older than 3 days."
	OLDSTUFF=$(find /srv/workspace/pbuilder/ -maxdepth 2 -regex '.*/[0-9]+' -type d -mtime +2 -exec ls -lad {} \; || true)
	if [ ! -z "$OLDSTUFF" ] ; then
		echo
		echo "Old pbuilder build directories found in /srv/workspace/pbuilder/"
		echo -n "$OLDSTUFF"
115
		( find /srv/workspace/pbuilder/ -maxdepth 2 -regex '.*/[0-9]+' -type d -mtime +2 -exec sudo rm -rf --one-file-system {} \; ) || true
116 117 118 119 120
		echo
		DIRTY=true
	fi
fi

121 122 123 124 125
#
# delete old chroot-installation directories (not related to reproducible builds)
#
if [ -d /srv/workspace/chroots/ ] ; then
	echo "$(date -u) - Deleting chroots build directories, older than 7 days."
126
	OLDSTUFF=$(find /srv/workspace/chroots/ -maxdepth 2 -name 'chroot-installation*' -type d -mtime +6 -exec ls -lad {} \; || true)
127 128 129 130
	if [ ! -z "$OLDSTUFF" ] ; then
		echo
		echo "Old chroot-installation directories found in /srv/workspace/chroots/"
		echo -n "$OLDSTUFF"
131
		( find /srv/workspace/chroots/ -maxdepth 2 -name 'chroot-installation*' -type d -mtime +6 -exec sudo rm -rf --one-file-system {} \; ) || true
132 133 134 135 136
		echo
		DIRTY=true
	fi
fi

137 138 139 140 141 142
#
# check for working proxy
#
echo "$(date -u) - testing whether the proxy works..."
curl http://www.debian.org > /dev/null
if [ $? -ne 0 ] ; then
143 144
	echo "Error: curl http://www.debian.org failed, probably the proxy is down for $HOSTNAME"
	exit 1
145 146
fi

147 148 149 150 151 152
if [ "$HOSTNAME" = "$MAINNODE" ] ; then
	#
	# find nodes with problems and temporarily turn them offline
	#
	echo "$(date -u) - Looking for unhealthy nodes."
	cd ~/jobs
153
	DUMMY_FILE=$(mktemp --tmpdir=$TMPDIR maintenance-XXXXXXX)
154
	SICK=""
155 156
	for i in reproducible_node_health_check_* reproducible_maintenance_* ; do
		case $i in
157 158 159 160
			reproducible_node_health_check_amd64_jenkins|reproducible_maintenance_amd64_jenkins)
				echo "Skipping $i..."
				continue
				;;
161 162 163
			reproducible_node_health_check_*)
				NODE_ALIAS=$(echo $i | cut -d '_' -f6)
				NODE_ARCH=$(echo $i | cut -d '_' -f5)
164 165
				FORCE_DATE=$(date -u -d "2 hour ago" '+%Y-%m-%d %H:%M')
				MAXDIFF=8
166 167 168 169 170 171 172 173 174
				;;
			reproducible_maintenance_*)
				NODE_ALIAS=$(echo $i | cut -d '_' -f4)
				NODE_ARCH=$(echo $i | cut -d '_' -f3)
				FORCE_DATE=$(date -u -d "5 hour ago" '+%Y-%m-%d %H:%M')
				MAXDIFF=2
				;;
		esac
		touch -d "$FORCE_DATE" $DUMMY_FILE
175 176 177 178 179 180
		case $NODE_ARCH in
			amd64)	NODE="profitbricks-build${NODE_ALIAS#profitbricks}-amd64.debian.net" ;;
			i386)	NODE="profitbricks-build${NODE_ALIAS#profitbricks}-i386.debian.net" ;;
			arm64)	NODE="codethink-sled${NODE_ALIAS#codethink}-arm64.debian.net" ;;
			armhf)	NODE="${NODE_ALIAS}-armhf-rb.debian.net" ;;
		esac
181
		case "$NODE" in
182
			profitbricks-build9-amd64.debian.net|profitbricks-build10-amd64.debian.net)
183 184
				# pb9 and pb10 are not used for r-b and sometimes are too busy
				# to run healthcheck / maintenance jobs
185 186 187 188
				echo "Skipping ${NODE}..."
				continue
				;;
		esac
189 190
		cd $i/builds
		LAST=$(ls -rt1 | tail -1)
191
		GOOD=$(basename $(readlink -f lastSuccessfulBuild))
Holger Levsen's avatar
Holger Levsen committed
192
		if [ "$LAST" = "$GOOD" ] ; then
193 194 195 196 197 198
			DIFF=0
		else
			let DIFF=$LAST-$GOOD || DIFF=-1
		fi
		if [ $DIFF -eq -1 ] ; then
			echo "Problems analysing $i build logs, ignoring $NODE."
199
		# either the diff is greater than $MAXDIFF (=the last $MAXDIFF job runs failed)
200
		# or the last successful run is older than an hour (=a job is still running/hanging)
201
		elif [ $DIFF -gt $MAXDIFF ] || [ $LAST -ot $DUMMY_FILE ] ; then
202
			echo -n "$i job has issues since more than an hour"
203 204 205 206 207
			if grep -q $NODE ~/offline_nodes >/dev/null 2>&1 ; then
				echo " and $NODE already marked as offline, good."
			else
				echo $NODE >> ~/offline_nodes
				echo " so $NODE has (temporarily) been marked as offline now."
208
				SICK="$SICK $NODE"
209 210 211 212 213 214
			fi
		else
			echo "$NODE is doing fine, good."
		fi
		cd ../..
	done
215
	if [ -n "$SICK" ] ; then
216
		SICK=$(echo "$SICK" | sed 's#.debian.net##g' | sed 's#-rb##g' | sed 's# ##' )
217
		if echo "$SICK" | grep -q ' ' 2>/dev/null ; then
218
			SICK=$(echo "$SICK" | sed 's# # and #g')
219
			MESSAGE="$SICK have health problems and have temporarily been marked as offline."
220
		else
221
			MESSAGE="$SICK has health problems and has temporarily been marked as offline."
222
		fi
223
		irc_message debian-reproducible "$MESSAGE To make this permanent, edit jenkins-home/offline_nodes in git."
224
	fi
225
	rm -f $DUMMY_FILE
226 227
fi

228
echo "$(date -u) - updating the schroots and pbuilder now..."
Holger Levsen's avatar
Holger Levsen committed
229
# use host architecture (only)
230
ARCH=$(dpkg --print-architecture)
Holger Levsen's avatar
Holger Levsen committed
231 232 233 234
# use host apt proxy configuration for pbuilder
if [ ! -z "$http_proxy" ] ; then
	pbuilder_http_proxy="--http-proxy $http_proxy"
fi
235
for s in $SUITES ; do
236 237 238 239
	if [ "${HOSTNAME:0:6}" = "osuosl" ] ; then
		# osuosl nodes are not used to do Debian rebuilds
		continue
	fi
240 241 242 243 244 245 246 247 248 249 250
	#
	# chdist update
	#
	distname="$s-$ARCH"
	echo "$(date -u) - updating the $s/$ARCH chdist now."
	if [ ! -d "$CHPATH/$distname" ]; then
		echo "$(date -u) - chdist not existing, creating one now..."
		if ! chdist --data-dir="$CHPATH" --arch="$ARCH" create "$distname" "$MIRROR" "$s" main ; then
			echo "Error: failed to create the $s/$ARCH chdist."
			exit 1
		fi
251
		. /srv/jenkins/bin/jenkins_node_definitions.sh
Holger Levsen's avatar
Holger Levsen committed
252
		get_node_information "$HOSTNAME"
253 254 255 256
		if "$NODE_RUN_IN_THE_FUTURE" ; then
			echo "This node is reported to run in the future, configuring APT to ignore the Release file expiration..."
			echo 'Acquire::Check-Valid-Until "false";' > "$CHPATH/$distname/etc/apt/apt.conf.d/398future"
		fi
257 258 259 260 261
	fi
	if ! chdist --data-dir="$CHPATH" apt-get "$distname" update ; then
		echo "Warning: failed to update the $s/$ARCH chdist."
		DIRTY=true
	fi
262 263 264
	#
	# schroot update
	#
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
	#echo "$(date -u) - updating the $s/$ARCH schroot now."
	#for i in 1 2 3 4 ; do
	#	[ ! -d $SCHROOT_BASE/reproducible-$s ] || schroot --directory /root -u root -c source:jenkins-reproducible-$s -- apt-get update
	#	RESULT=$?
	#	if [ $RESULT -eq 1 ] ; then
	#		# sleep 61-120 secs
	#		echo "Sleeping some time... (to workaround network problems like 'Hash Sum mismatch'...)"
	#		/bin/sleep $(echo "scale=1 ; ($(shuf -i 1-600 -n 1)/10)+60" | bc )
	#		echo "$(date -u) - Retrying to update the $s/$ARCH schroot."
	#	elif [ $RESULT -eq 0 ] ; then
	#		break
	#	fi
	#done
	#if [ $RESULT -eq 1 ] ; then
	#	echo "Warning: failed to update the $s/$ARCH schroot."
	#	DIRTY=true
	#fi
282 283 284
	#
	# pbuilder update
	#
285
	# pbuilder aint used on jenkins anymore
286
	if [ "$HOSTNAME" = "$MAINNODE" ] ; then
287 288 289 290
		continue
	else
		echo "$(date -u) - updating pbuilder for $s/$ARCH now."
	fi
291
	for i in 1 2 3 4 ; do
292
		[ ! -f /var/cache/pbuilder/$s-reproducible-base.tgz ] || sudo pbuilder --update $pbuilder_http_proxy --basetgz /var/cache/pbuilder/$s-reproducible-base.tgz
293 294
		RESULT=$?
		if [ $RESULT -eq 1 ] ; then
295
			# sleep 61-120 secs
296
			echo "Sleeping some time... (to workaround network problems like 'Hash Sum mismatch'...)"
297
			/bin/sleep $(echo "scale=1 ; ($(shuf -i 1-600 -n 1)/10)+60" | bc )
298 299
			echo "$(date -u) - Retrying to update pbuilder for $s/$ARCH."
		elif [ $RESULT -eq 0 ] ; then
300
			break
301 302 303 304 305 306 307 308 309
		fi
	done
	if [ $RESULT -eq 1 ] ; then
		echo "Warning: failed to update pbuilder for $s/$ARCH."
		DIRTY=true
	fi
done
set -e

310
# for Arch Linux
311
set +e
312
case $HOSTNAME in
313
	osuosl-build169*|osuosl-build170*|jenkins)
314 315 316 317
		echo "$(date -u) - updating Arch Linux schroot now."
		schroot --directory /tmp -c source:jenkins-reproducible-archlinux -u root -- pacman -Syu --noconfirm
		RESULT=$?
		if [ $RESULT -eq 1 ] ; then
318
			echo "Warning: failed to update Arch Linux schroot."
319 320
			echo "Let's see if /var/lib/pacman/db.lck exists in the schroot."
			schroot --directory /tmp -c source:jenkins-reproducible-archlinux -u root -- ls /var/lib/pacman/db.lck
321
			DIRTY=true
322 323 324 325 326 327
		else
			echo "$(date -u) - updating Arch Linux schroot done."
		fi
		;;
	*)	;;
esac
328
set -e
329

330
# delete build services logfiles
331 332
if [ "$HOSTNAME" = "$MAINNODE" ] ; then
	if [ -d /var/lib/jenkins/userContent/reproducible/debian/build_service/ ] ; then
333 334
		echo "$(date -u) - Deleting logfiles from build services directories, older than a day."
		OLDSTUFF=$(find /var/lib/jenkins/userContent/reproducible/debian/build_service/ -maxdepth 2 -regex '.*/[0-9]+' -type d -mtime +0 -exec ls -lad {} \; || true)
335 336 337 338
		if [ ! -z "$OLDSTUFF" ] ; then
			echo
			echo "Old logfiles cleaned in /var/lib/jenkins/userContent/reproducible/debian/build_service/"
			echo -n "$OLDSTUFF"
339 340 341 342
			# we make sure to actually only delete console.log.gz older than a day
			# other stuff we only delete after two days (in case a build is running more than 24h...)
			find /var/lib/jenkins/userContent/reproducible/debian/build_service/ -maxdepth 2 -regex '.*/[0-9]+' -type d -mtime +0 -name console.log.gz -exec rm -rf --one-file-system {} \; || true
			find /var/lib/jenkins/userContent/reproducible/debian/build_service/ -maxdepth 2 -regex '.*/[0-9]+' -type d -mtime +1 -exec rm -rf --one-file-system {} \; || true
343 344
			echo
		fi
345 346 347
	fi
fi

348
# remove too old schroot sessions
349
echo "$(date -u) - Removing schroot sessions older than 3 days."
350
dir=/var/lib/schroot/unpack/
351
OLDSTUFF=$(find "$dir" -mindepth 1 -maxdepth 1 -type d -mtime +2 -exec ls -lad {} \;)
352 353
if [ ! -z "$OLDSTUFF" ]; then
	echo
354
	echo "schroot sessions older than 3 days found, which will be deleted:"
355 356 357 358 359 360
	echo "$OLDSTUFF"
	echo
	for s in $(find "$dir" -mindepth 1 -maxdepth 1 -type d -mtime +2 -print0 | xargs -0 -r basename -a); do
		echo "$(date -u) - removing schroot session $s..."
		schroot -c "$s" --end-session
	done
361
	OLDSTUFF=$(find "$dir" -mindepth 1 -maxdepth 1 -type d -mtime +2 -exec ls -lad {} \;)
362 363
	if [ ! -z "$OLDSTUFF" ]; then
		echo
364
		echo "Warning: Tried, but failed to delete these:"
365 366 367 368 369 370
		echo "$OLDSTUFF"
		echo "Manual cleanup needed"
	fi
	echo
	DIRTY=true
fi
371

372
# find old schroots
373
echo "$(date -u) - Removing schroots older than 3 days."
374
regex="/schroots/(reproducible-.+-[0-9]{1,5}|schroot-install-.+)"
375
OLDSTUFF=$(find /schroots/ -maxdepth 1 -type d -regextype posix-extended -regex "$regex" -mtime +2 -exec ls -lad {} \; || true)
376 377
if [ ! -z "$OLDSTUFF" ] ; then
	echo
378
	echo "schroots older than 3 days found in /schroots, which will be deleted:"
379
	find /schroots/ -maxdepth 1 -type d -regextype posix-extended -regex "$regex" -mtime +2 -exec sudo rm -rf --one-file-system {} \; || true
380
	echo "$OLDSTUFF"
381
	OLDSTUFF=$(find /schroots/ -maxdepth 1 -type d -regextype posix-extended -regex "$regex" -mtime +2 -exec ls -lad {} \; || true)
382 383 384 385 386 387
	if [ ! -z "$OLDSTUFF" ] ; then
		echo
		echo "Warning: Tried, but failed to delete these:"
		echo "$OLDSTUFF"
		echo "Manual cleanup needed!"
	fi
388
	echo
389
	DIRTY=true
390 391
fi

392 393
# find very old schroots
echo "$(date -u) - Detecting schroots older than 1 month"
394
# the reproducible-archlinux schroot is ignored because its ment to be long living
395
OLDSTUFF=$(find /schroots/ -mindepth 1 -maxdepth 1 -mtime +30 -exec ls -lad {} \; | grep -v reproducible-archlinux | true)
396 397 398 399 400 401 402 403 404 405
if [ ! -z "$OLDSTUFF" ]; then
	echo
	echo "Warning: schroots older than 1 month found in /schroot:"
	echo "$OLDSTUFF"
	echo
	echo "Manual cleanup needed!"
	echo
	DIRTY=true
fi

406
if [ "$HOSTNAME" = "$MAINNODE" ] ; then
407 408 409 410 411 412
	#
	# find failed builds due to network problems and reschedule them
	#
	# only grep through the last 5h (300 minutes) of builds...
	# (ignore "*None.rbuild.log" because these are build which were just started)
	# this job runs every 4h
413
	echo "$(date -u) - Rescheduling failed builds due to network issues."
414
	FAILED_BUILDS=$(find $DEBIAN_BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: Failed to fetch.*(Unable to connect to|Connection failed|Size mismatch|Cannot initiate the connection to|Bad Gateway|Service Unavailable)' {} \; || true)
415 416 417 418 419 420
	if [ ! -z "$FAILED_BUILDS" ] ; then
		echo
		echo "The following builds have failed due to network problems and will be rescheduled now:"
		echo "$FAILED_BUILDS"
		echo
		echo "Rescheduling packages: "
421 422
		REQUESTER="jenkins maintenance job"
		REASON="maintenance reschedule: reschedule builds which failed due to network errors"
423 424 425
		for SUITE in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f9 | sort -u) ; do
			for ARCH in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f10 | sort -u) ; do
				CANDIDATES=$(for PKG in $(echo $FAILED_BUILDS | sed "s# #\n#g" | grep "/$SUITE/$ARCH/" | cut -d "/" -f11 | cut -d "_" -f1) ; do echo "$PKG" ; done)
426 427 428 429 430
				# double check those builds actually failed
				TO_SCHEDULE=""
				for pkg in $CANDIDATES ; do
					QUERY="SELECT s.name FROM sources AS s JOIN results AS r ON r.package_id=s.id
						   WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND (r.status='FTBFS' OR r.status='depwait') AND s.name='$pkg'"
431
					TO_SCHEDULE=${TO_SCHEDULE:+"$TO_SCHEDULE "}$(query_db "$QUERY")
432 433
				done
				schedule_packages $TO_SCHEDULE
434
			done
435
		done
436 437 438
		DIRTY=true
	fi

439 440 441 442 443 444 445
	#
	# find failed builds due to diffoscope schroot problems and reschedule them
	#
	# only grep through the last 5h (300 minutes) of builds...
	# (ignore "*None.rbuild.log" because these are build which were just started)
	# this job runs every 4h
	echo "$(date -u) - Rescheduling failed builds due to diffoscope schroot issues."
446
	FAILED_BUILDS=$(find $DEBIAN_BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -F 'E: 10mount: error: Directory' {} \; || true)
447 448 449 450 451 452 453 454
	if [ ! -z "$FAILED_BUILDS" ] ; then
		echo
		echo "Warning: The following builds have failed due to diffoscope schroot problems and will be rescheduled now:"
		echo "$FAILED_BUILDS"
		echo
		echo "Rescheduling packages: "
		REQUESTER="jenkins maintenance job"
		REASON="maintenance reschedule: reschedule builds which failed due to diffoscope schroot errors"
455 456 457
		for SUITE in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f9 | sort -u) ; do
			for ARCH in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f10 | sort -u) ; do
				CANDIDATES=$(echo $FAILED_BUILDS | sed "s# #\n#g" | grep "/$SUITE/$ARCH/" | cut -d "/" -f11 | cut -d "_" -f1 | xargs)
458
				if [ ! -z "$CANDIDATES" ]; then
459
					schedule_packages $CANDIDATES
460
				fi
461 462 463 464 465
			done
		done
		DIRTY=true
	fi

466 467 468
	#
	# find packages which build didnt end correctly
	#
469
	echo "$(date -u) - Rescheduling builds which didn't end correctly."
470
	DATE=$(date '+%Y-%m-%d %H:%M' -d "-2 days")
471 472 473 474
	QUERY="
		SELECT s.id, s.name, p.date_scheduled, p.date_build_started
			FROM schedule AS p JOIN sources AS s ON p.package_id=s.id
			WHERE p.date_scheduled != ''
475
			AND p.date_build_started IS NOT NULL
476
			AND p.date_build_started < '$DATE'
477 478 479
			ORDER BY p.date_scheduled
		"
	PACKAGES=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXXX)
480
	query_db "$QUERY" > $PACKAGES 2> /dev/null || echo "Warning: SQL query '$QUERY' failed."
481 482
	if grep -q '|' $PACKAGES ; then
		echo
483
		echo "Packages found where the build was started more than 48h ago:"
484
		query_to_print "$QUERY" 2> /dev/null || echo "Warning: SQL query '$QUERY' failed."
485 486
		echo
		for PKG in $(cat $PACKAGES | cut -d "|" -f1) ; do
487 488
			echo "query_db \"UPDATE schedule SET date_build_started = NULL, job = NULL WHERE package_id = '$PKG';\""
			query_db "UPDATE schedule SET date_build_started = NULL, job = NULL WHERE package_id = '$PKG';"
489
		done
490
		echo "Packages have been rescheduled."
491 492 493 494 495 496 497 498
		echo
		DIRTY=true
	fi
	rm $PACKAGES

	#
	# find packages which have been removed from the archive
	#
499
	echo "$(date -u) - Looking for packages which have been removed from the archive."
500 501 502
	PACKAGES=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXX)
	QUERY="SELECT name, suite, architecture FROM removed_packages
			LIMIT 25"
503
	query_db "$QUERY" > $PACKAGES 2> /dev/null || echo "Warning: SQL query '$QUERY' failed."
504 505 506 507 508
	if grep -q '|' $PACKAGES ; then
		DIRTY=true
		echo
		echo "Found files relative to old packages, no more in the archive:"
		echo "Removing these removed packages from database:"
509
		query_to_print "$QUERY" 2> /dev/null || echo "Warning: SQL query '$QUERY' failed."
510 511 512 513 514 515 516
		echo
		for pkg in $(cat $PACKAGES) ; do
			PKGNAME=$(echo "$pkg" | cut -d '|' -f 1)
			SUITE=$(echo "$pkg" | cut -d '|' -f 2)
			ARCH=$(echo "$pkg" | cut -d '|' -f 3)
			QUERY="DELETE FROM removed_packages
				WHERE name='$PKGNAME' AND suite='$SUITE' AND architecture='$ARCH'"
517
			query_db "$QUERY"
518
			cd $DEBIAN_BASE
519
			find rb-pkg/$SUITE/$ARCH rbuild/$SUITE/$ARCH dbd/$SUITE/$ARCH dbdtxt/$SUITE/$ARCH buildinfo/$SUITE/$ARCH logs/$SUITE/$ARCH logdiffs/$SUITE/$ARCH -name "${PKGNAME}_*" | xargs -r rm -v || echo "Warning: couldn't delete old files from ${PKGNAME} in $SUITE/$ARCH"
520 521 522 523 524 525
		done
		cd - > /dev/null
	fi
	rm $PACKAGES

	#
526
	# delete jenkins html logs from reproducible_builder_(fedora|archlinux)* jobs as they are mostly redundant
527 528 529 530 531 532 533 534
	# (they only provide the extended value of parsed console output, which we dont need here.)
	#
	OLDSTUFF=$(find /var/lib/jenkins/jobs/reproducible_builder_* -maxdepth 3 -mtime +0 -name log_content.html  -exec rm -v {} \; | wc -l)
	if [ ! -z "$OLDSTUFF" ] ; then
		echo
		echo "Removed $OLDSTUFF jenkins html logs."
		echo
	fi
535

536 537
fi

538
# find+terminate processes which should not be there
539
echo "$(date -u) - Looking for processes which should not be there."
540 541 542
HAYSTACK=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXX)
RESULT=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXX)
TOKILL=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXX)
543
PBUIDS="1234 1111 2222"
544
ps axo pid,user,size,pcpu,cmd > $HAYSTACK
545
for i in $PBUIDS ; do
546
	for PROCESS in $(pgrep -u $i -P 1 || true) ; do
547
		# faked-sysv comes and goes...
548
		grep ^$PROCESS $HAYSTACK | grep -v faked-sysv >> $RESULT 2> /dev/null || true
549
	done
550 551
done
if [ -s $RESULT ] ; then
552
	for PROCESS in $(cat $RESULT | cut -d " " -f1 | grep -v ^UID | xargs echo) ; do
553
		AGE=$(ps -p $PROCESS -o etimes= || echo 0)
554 555
		# a single build may take day, so... (first build: 18h, 2nd: 24h)
		if [ $AGE -gt $(( 24*60*60 )) ] ; then
556
			echo "$PROCESS" >> $TOKILL
557 558
		fi
	done
559 560
	if [ -s $TOKILL ] ; then
		DIRTY=true
561
		PSCALL=""
562
		echo
563
		echo "Info: processes found which should not be there, killing them now:"
564 565 566 567
		for PROCESS in $(cat $TOKILL) ; do
			PSCALL=${PSCALL:+"$PSCALL,"}"$PROCESS"
		done
		ps -F -p $PSCALL
568
		echo
569
		for PROCESS in $(cat $TOKILL) ; do
570 571
			sudo kill -9 $PROCESS 2>&1
			echo "'kill -9 $PROCESS' done."
572 573 574
		done
		echo
	fi
575
fi
576
rm $HAYSTACK $RESULT $TOKILL
577 578
# There are naughty processes spawning childs and leaving them to their grandparents
PSCALL=""
579
for i in $PBUIDS ; do
580 581
	for p in $(pgrep -u $i) ; do
		AGE=$(ps -p $p -o etimes= || echo 0)
582 583
		# let's be generous and consider 26 hours here...
		if [ $AGE -gt $(( 26*60*60 )) ] ; then
584
			sudo kill -9 $p 2>&1 || (echo "Could not kill:" ; ps -F -p "$p")
585
			sleep 2
586 587 588 589 590
			# check it's gone
			AGE=$(ps -p $p -o etimes= || echo 0)
			if [ $AGE -gt $(( 14*60*60 )) ] ; then
				PSCALL=${PSCALL:+"$PSCALL,"}"$p"
			fi
591 592 593 594
		fi
	done
done
if [ ! -z "$PSCALL" ] ; then
595
	echo -e "Warning: processes found which should not be there and which could not be killed. Please fix up manually:"
596 597 598
	ps -F -p "$PSCALL"
	echo
fi
599

600
# find builds which should not be there
601 602 603 604 605 606 607 608
# (not on i386 as we start builds differently here… work in progress)
if [ "$ARCH" != "i386" ] ; then
	RESULTS=$(pgrep -f reproducible_build.sh --parent 1 || true)
	if [ ! -z "$RESULTS" ] ; then
		DIRTY=true
		echo "Warning: found reproducible_build.sh processes which have pid 1 as parent (and not sshd), thus something went wrong… please investigate."
		echo -e "$RESULTS"
	fi
609
fi
610

611 612
# remove artifacts older than a day
echo "$(date -u) - Checking for artifacts older than a day."
613
ARTIFACTS=$(find $DEBIAN_BASE/artifacts/r00t-me/* -maxdepth 1 -type d -mtime +1 -exec ls -lad {} \; 2>/dev/null|| true)
614 615
if [ ! -z "$ARTIFACTS" ] ; then
	echo
616
	echo "Removed old artifacts:"
617
	find $DEBIAN_BASE/artifacts/r00t-me/* -maxdepth 1 -type d -mtime +1 -exec rm -rv --one-file-system {} \; || true
618 619 620
	echo
fi

621
# find + chmod files with bad permissions
622
echo "$(date -u) - Checking for files with bad permissions."
623 624
# automatically fix rbuild files with wrong permissions...
# (we know it happens (very rarely) but... shrugs.)
625
[ ! -d $DEBIAN_BASE/rbuild ] || find $DEBIAN_BASE/rbuild ! -perm 644 -type f -exec chmod -v 644 {} \; 2>/dev/null|| true
626
BADPERMS=$(find $DEBIAN_BASE/{buildinfo,dbd,artifacts,stretch,buster,unstable,experimental,rb-pkg} ! -perm 644 -type f 2>/dev/null|| true)
627
if [ ! -z "$BADPERMS" ] ; then
628 629 630 631 632 633 634 635
    DIRTY=true
    echo
    echo "Warning: Found files with bad permissions (!=644):"
    echo "Please fix permission manually"
    echo "$BADPERMS" | xargs echo chmod -v 644
    echo
fi

636
# daily mails
637
if [ "$HOSTNAME" = "$MAINNODE" ] && [ $(date -u +%H) -eq 0 ]  ; then
638
	# once a day, send mail about builder problems
639
	files_to_mail=(
640 641
		/var/log/jenkins/reproducible-builder-errors.log
		/var/log/jenkins/reproducible-stale-builds.log
642
		/var/log/jenkins/reproducible-archlinux-stale-builds.log
643 644 645
		/var/log/jenkins/reproducible-race-conditions.log
		/var/log/jenkins/reproducible-diskspace-issues.log
		/var/log/jenkins/reproducible-remote-error.log
646
		/var/log/jenkins/reproducible-scheduler.log
647 648 649
		/var/log/jenkins/reproducible-env-changes.log
		/var/log/jenkins/reproducible-submit2buildinfo.debian.net.log
		/var/log/postgresql/postgresql-9.6-main.log
650 651
	)
	for PROBLEM in "${files_to_mail[@]}" ; do
652 653
		if [ -s $PROBLEM ] ; then
			TMPFILE=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXXX)
654
			if [ "$(dirname $PROBLEM)" = "/var/log/jenkins" ] ; then
655 656 657 658 659
				if [ "$(basename $PROBLEM)" = "reproducible-diskspace-issues.log" ]; then
					echo "diskspace issues should always be investigated." > $TMPFILE
				fi
				if grep -q https $PROBLEM ; then
					echo "$(grep -c https $PROBLEM) entries found:"
660
					if [ "$(basename $PROBLEM)" != "reproducible-remote-error.log" ] && [ "$(basename $PROBLEM)" != "reproducible-race-conditions.log" ] ; then
661 662 663 664
						OTHERPROJECTS=""
					else
						OTHERPROJECTS="archlinux fedora"
					fi
665
					echo "$(grep -c https $PROBLEM || echo 0) entries found:" >> $TMPFILE
666
					for a in $ARCHS $OTHERPROJECTS; do
667
						echo "- $(grep https $PROBLEM|grep -c ${a}_) from $a." >> $TMPFILE
668 669 670 671 672 673 674 675
					done
				elif grep -q 'stale builds found' $PROBLEM ; then
					echo "$(grep -c 'stale builds found' $PROBLEM || echo 0) entries found:" >> $TMPFILE
					for a in $ARCHS ; do
							echo "- $(grep -c ${a}_ $PROBLEM) from $a." >> $TMPFILE
					done
				fi
				echo >> $TMPFILE
676
				# maybe we should use logrotate for our jenkins logs too…
677 678
				cat $PROBLEM >> $TMPFILE
				rm $PROBLEM
679
			else
680 681
				# regular logfile, logrotate is used (and the file ain't owned by jenkins)
				# only care for yesterday's entries:
682 683
				( grep $(date -u -d "1 day ago" '+%Y-%m-%d') $PROBLEM || echo "no problems yesterday…" ) > $TMPFILE
			fi
684 685
			# send mail if we found issues
			if [ -s $TMPFILE ] && ! grep -q "no problems yesterday…" $TMPFILE ; then
686 687 688
				if [ "$(basename $PROBLEM)" = "reproducible-submit2buildinfo.debian.net.log" ]; then
					CC="-c lamby@debian.org"
				fi
689
				cat $TMPFILE | mail -s "$(basename $PROBLEM) found" ${CC:-} qa-jenkins-scm@lists.alioth.debian.org
690
				CC=""
691
			fi
692 693 694
			rm -f $TMPFILE
		fi
	done
695 696 697 698
	# once a day, send notifications to package maintainers
	cd /srv/reproducible-results/notification-emails
	for NOTE in $(find . -type f) ; do
			TMPFILE=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXXX)
699
			PKG=$(basename $NOTE)
700
			mv $NOTE $TMPFILE
701
			cat $TMPFILE | mail -s "$PKG: status change on tests.reproducible-builds.org/debian" \
702
				-a "From: Reproducible builds folks <reproducible-builds@lists.alioth.debian.org>" \
703
				-a "X-Reproducible-Builds-Pkg: $PKG" \
704
				 $PKG@packages.debian.org
705 706
			rm -f $TMPFILE
	done
707 708
fi

709
if ! $DIRTY ; then
710
	echo "$(date -u ) - Everything seems to be fine."
711 712
	echo
fi
713

714
echo "$(date -u) - the end."