common-functions.sh

#!/bin/bash
# vim: set noexpandtab:

# Copyright 2014-2022 Holger Levsen <holger@layer-acht.org>
#         © 2018      Mattia Rizzolo <mattia@mapreri.org>
# released under the GPLv2

common_cleanup() {
	echo "$(date -u) - $(basename $0) stopped running as $TTT, removing."
	rm -f $TTT
}

abort_if_bug_is_still_open() {
	local TMPFILE=$(mktemp --tmpdir=/tmp jenkins-bugcheck-XXXXXXX)
	echo "$(date -u) - checking bug #$1 status."
	bts status $1 fields:done > $TMPFILE || true
	# if we get a valid response…
	if [ ! -z "$(grep done $TMPFILE)" ] ; then
		# if the bug is not done (by some email address containing a @)
		if [ -z "$(grep "@" $TMPFILE)" ] ; then
			rm $TMPFILE
			echo
			echo
			echo "########################################################################"
			echo "#                                                                      #"
			echo "#   https://bugs.debian.org/$1 is still open, aborting this job.   #"
			echo "#                                                                      #"
			echo "########################################################################"
			echo
			echo
			echo "Warning: aborting the job because of bug because #$1"
			echo
			echo "After having fixed the cause for #$1, remove the check from common_init()"
			echo "in bin/common-functions.sh and re-run the job. Close #$1 after the"
			echo "problem is indeed fixed."
			echo
			exec /srv/jenkins/bin/abort.sh
			exit 0
		fi
	fi
	rm $TMPFILE
}

#
# run ourself with the same parameter as we are running
# but run a copy from /tmp so that the source can be updated
# (Running shell scripts fail weirdly when overwritten when running,
#  this hack makes it possible to overwrite long running scripts
#  anytime...)
#
common_init() {
# some sensible default
if [ "${LC_ALL+x}" != "x" ] ; then
	export LC_ALL=C.UTF-8
fi
# check whether this script has been started from /tmp already
if [ "${0:0:5}" != "/tmp/" ] ; then
	# check that we are not root
	if [ $(id -u) -eq 0 ] ; then
		echo "Do not run this as root."
		exit 1
	fi
	# - for remote jobs we need to check against $SSH_ORIGINAL_COMMAND
	# - for local jobs this would be $JOB_NAME
	if [ "${JOB_NAME+x}" = "x" ] ; then
		WHOAREWE=$JOB_NAME
	else
		# $JOB_NAME is undefined
		WHOAREWE=${SSH_ORIGINAL_COMMAND/%\ */}
	fi
	# abort certain jobs if we know they will fail due to certain bugs…
	case $WHOAREWE in
		#chroot-installation_*_install_design-desktop-*)
		#	for BLOCKER in 869155 867695 ; do
		#		abort_if_bug_is_still_open $BLOCKER
		#	done ;;
		chroot-installation_buster_install_design*)
			# technically these two bugs dont affect design-desktop
			# but just a depends of it, however I don't think it's likely
			# design-desktop will enter buster without these two bugs being fixed
			abort_if_bug_is_still_open 890754 ;;
		chroot-installation_stretch_install_education-desktop-gnome_upgrade_to_buster|chroot-installation_stretch_install_education-desktop-xfce_upgrade_to_buster|chroot-installation_stretch_install_education-networked_upgrade_to_buster)
			abort_if_bug_is_still_open 928429 ;;
		chroot-installation_buster_install_parl-desktop-world*)
			abort_if_bug_is_still_open 945930 ;;
		*) ;;
	esac
	# mktemp some place for us...
	TTT="$(mktemp --tmpdir=/tmp jenkins-script-XXXXXXXX)"
	if [ -z "$TTT" ] ; then
		echo "Failed to create tmpfile, aborting. (Probably due to read-only filesystem…)"
		exit 1
	fi
	# prepare cleanup
	trap common_cleanup INT TERM EXIT
	# cp $0 to /tmp and run it from there
	cp $0 $TTT
	chmod +x $TTT
	echo "===================================================================================="
	echo "$(date -u) - running $0 (for job $WHOAREWE) on $(hostname), called using \"$@\" as arguments."
	echo "$(date -u) - actually running \"$(basename $0)\" (md5sum $(md5sum $0|cut -d ' ' -f1)) as \"$TTT\""
	echo
	echo "$ git clone https://salsa.debian.org/qa/jenkins.debian.net.git ; more CONTRIBUTING"
	echo
	# this is the "hack": call ourself as a copy in /tmp again
	$TTT "$@"
	exit $?
	# cleanup is done automatically via trap
else
	# this directory resides on tmpfs, so it might be gone after reboots...
	mkdir -p /srv/workspace/chroots

	if [ "${MIRROR+x}" != "x" ] ; then
		case $HOSTNAME in
			jenkins|ionos*|osuosl*|snapshot)
				export MIRROR=http://cdn-fastly.deb.debian.org/debian ;;
			cbxi4*|wbq0|ff*|jt?1*|virt32[a-z]|virt64[a-z])
				export MIRROR=http://cdn-fastly.deb.debian.org/debian ;;
			codethink*)
				export MIRROR=http://cdn-fastly.deb.debian.org/debian ;;
			spectrum|warren)  # warren is mattia's laptop
				export MIRROR=none ;;
			*)
				echo "unsupported host, exiting." ; exit 1 ;;
		esac
	fi
	# force http_proxy as we want it
	case $HOSTNAME in
		jenkins|ionos1-a*|ionos2*|ionos9*|ionos11*|ionos12*)
			# IONOS datacenter in karlsruhe uses ionos1 as proxy:
			export http_proxy="http://78.137.99.97:3128" ;;
		ionos3*|ionos5*|ionos6*|ionos7*|ionos10*|ionos15*|ionos16*)
			# IONOS datacenter in frankfurt uses ionos10 as proxy:
			export http_proxy="http://85.184.249.68:3128" ;;
		osuosl*)
			# all nodes at OSUOSL use themself as proxy:
			export http_proxy="http://127.0.0.1:3128" ;;
		codethink*)
			export http_proxy="http://192.168.101.16:3128" ;;
		cbxi4*|wbq0|ff*|jt?1*|virt64[a-z]|virt32[a-z])
			export http_proxy="http://10.0.0.15:3142/" ;;
		spectrum)
			export http_proxy="http://127.0.0.1:3128" ;;
		snapshot) : ;;
		warren) : ;;  # warren is mattia's laptop
		*)
			echo "unsupported host, exiting." ; exit 1 ;;
	esac
	if [ "${CHROOT_BASE+x}" != "x" ] ; then
		export CHROOT_BASE=/chroots
	fi
	if [ "${SCHROOT_BASE+x}" != "x" ] ; then
		export SCHROOT_BASE=/schroots
	fi
	if [ ! -d "$SCHROOT_BASE" ]; then
		echo "Directory $SCHROOT_BASE does not exist, aborting."
		exit 1
	fi
	# use these settings in the scripts in the (s)chroots too
	export SCRIPT_HEADER="#!/bin/bash
	if $DEBUG ; then
		set -x
	fi
	set -e
	export DEBIAN_FRONTEND=noninteractive
	export LC_ALL=$LC_ALL
	export http_proxy=$http_proxy
	export MIRROR=$MIRROR"
	# be more verbose, maybe
	if $DEBUG ; then
		export
		set -x
	fi
	set -e
fi
}

publish_changes_to_userContent() {
	echo "Extracting contents from .deb files..."
	CHANGES=$1
	CHANNEL=$2
	SRCPKG="$(basename $CHANGES | cut -d "_" -f1)"
	if [ -z "$SRCPKG" ] ; then
		echo '$SRCPKG is empty, exiting with error.'
		exit 1
	fi
	VERSION=$(basename $CHANGES | cut -d "_" -f2)
	TARGET="/var/lib/jenkins/userContent/$SRCPKG"
	NEW_CONTENT=$(mktemp -d -t new-content-XXXXXXXX)
	for DEB in $(dcmd --deb $CHANGES) ; do
		dpkg --extract $DEB ${NEW_CONTENT} 2>/dev/null
	done
	rm -rf $TARGET
	mkdir $TARGET
	mv ${NEW_CONTENT}/usr/share/doc/${SRCPKG}* $TARGET/
	rm -r ${NEW_CONTENT}
	if [ "${3+x}" != "x" ] ; then
		touch "$TARGET/${VERSION}"
		FROM=""
	else
		touch "$TARGET/${VERSION}_$3"
		FROM=" from $3"
	fi
	MESSAGE="https://jenkins.debian.net/userContent/$SRCPKG/ has been updated${FROM}."
	echo
	echo $MESSAGE
	echo
	if [ "${CHANNEL+x}" != "x" ] ; then
		kgb-client --conf /srv/jenkins/kgb/$CHANNEL.conf --relay-msg "$MESSAGE"
	fi
}

write_page() {
	echo "$1" >> $PAGE
}

output_echo() {
	set +x
	echo "###########################################################################################"
	echo
	echo -e "$(date -u) - $1"
	echo
	if $DEBUG; then
		set -x
	fi
}

jenkins_zombie_check() {
	#
	# sometimes deleted jobs come back as zombies
	# and we dont know why and when that happens,
	# so just report those zombies here.
	#
	# this has last happened on 2021-08-09, with a jenkins.deb
	# upgrade on 2021-07-28, thus likely unrelated. what seems
	# related however is that I issued a reboot (via running 
	# /sbin/reboot) right before the zombies appeared...
	#
	ZOMBIES="$(ls -1d /var/lib/jenkins/jobs/* | egrep 'strip-nondeterminism|reprotest|reproducible_(builder_(amd64|i386|armhf|arm64)|setup_(pbuilder|schroot)_testing)|chroot-installation_wheezy|aptdpkg|stretch_install_education-thin-client-server|jessie_multiarch_versionskew|dpkg_stretch_find_trigger_cycles|sid_install_education-services|buster_install_education-services|lvc|chroot-installation_stretch_.*_upgrade_to_sid|chroot-installation_buster_.*_upgrade_to_sid|piuparts_.*_jessie|udd_stretch|d-i_pu-build|debsums-tests_stretch|debian-archive-keyring-tests_stretch|chroot-installation_jessie|chroot-installation_.*education-lang-|kirkwoot|rebootstrap_.*_gcc[5-9]($|_)|rebootstrap_.*_gcc1[01]($|_)|brcm47xx|rebootstrap_kfreebsd|diffoscope_from_git_|disorderfs_from_git_master|diffoscope_pypi|diffoscope_freebsd|diffoscope_netbsd|diffoscope_macports|diffoscope_archlinux|openwrt-target-ath97|profitbricks|pool_buildinfos_suites|g-i-installation|reproducible_compare_Debian_sha1sums|bbx15|cb3a|ff2a|ff2b|jtk1a|jtk1b|odxu4a|odxu4b|odu3a|opi2a|opi2c|p64b|p64c|ar71xx|reproducible_debian_live_build$|chroot-installation_stretch|chroot-installation_bullseye*upgrade_to_sid|rebuilder_prototype|osuosl169' || true)"
	if [ ! -z "$ZOMBIES" ] ; then
		DIRTY=true
		figlet 'zombies!!!'
		echo "Warning: rise of the jenkins job zombies has started again, these jobs should not exist:"
		for z in $ZOMBIES ; do

			echo $(basename $z)
		done
		echo
	fi
}

jenkins_logsize_check() {
	#
	# /var/log/jenkins/jenkins.log sometimes grows very fast
	# and we don't yet know why, so let's monitor this for now.
	JENKINSLOG="$(find /var/log/jenkins -name jenkins.log -size +42G)"
	if [ ! -z "$JENKINSLOG" ] ; then
		figlet 'jenkins.log size'
		echo "Warning: jenkins.log is larger than 42G, please fix, erroring out now."
		exit 1
	else
		JENKINSLOG="$(find /var/log/jenkins -name jenkins.log -size +23G)"
		if [ ! -z "$JENKINSLOG" ] ; then
			DIRTY=true
			figlet 'jenkins.log size'
			echo "Warning: jenkins.log is larger than 23G, please do something…"
		fi
	fi
}

jenkins_bugs_check() {
	jenkins_zombie_check
	jenkins_logsize_check
}