common-functions.sh 8.29 KB
Newer Older
1 2
#!/bin/bash

3
# Copyright 2014-2017 Holger Levsen <holger@layer-acht.org>
4 5
# released under the GPLv=2

6
common_cleanup() {
Holger Levsen's avatar
Holger Levsen committed
7
	echo "$(date -u) - $0 stopped running as $TTT, which will now be removed."
8 9 10
	rm -f $TTT
}

11 12 13 14 15 16 17 18
abort_if_bug_is_still_open() {
	local TMPFILE=$(mktemp --tmpdir=/tmp jenkins-bugcheck-XXXXXXX)
	bts status $1 fields:done > $TMPFILE || true
	# if we get a valid response…
	if [ ! -z "$(grep done $TMPFILE)" ] ; then
		# if the bug is not done (by some email address containing a @)
		if [ -z "$(grep "@" $TMPFILE)" ] ; then
			rm $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
19 20
			echo
			echo
21 22
			echo "########################################################################"
			echo "#                                                                      #"
Holger Levsen's avatar
Holger Levsen committed
23
			echo "#   https://bugs.debian.org/$1 is still open, aborting this job.   #"
24 25 26 27
			echo "#                                                                      #"
			echo "########################################################################"
			echo
			echo
28
			echo "Warning: aborting the job because of bug because #$1"
29
			echo
30 31 32 33
			echo "After having fixed the cause for #$1, remove the check from common_init()"
			echo "in bin/common-functions.sh and re-run the job. Close #$1 after the"
			echo "problem is indeed fixed."
			echo
34 35 36 37 38 39 40
			exec /srv/jenkins/bin/abort.sh
			exit 0
		fi
	fi
	rm $TMPFILE
}

41 42 43 44 45 46 47
#
# run ourself with the same parameter as we are running
# but run a copy from /tmp so that the source can be updated
# (Running shell scripts fail weirdly when overwritten when running,
#  this hack makes it possible to overwrite long running scripts
#  anytime...)
#
48 49 50
common_init() {
# check whether this script has been started from /tmp already
if [ "${0:0:5}" != "/tmp/" ] ; then
51 52 53 54 55
	# check that we are not root
	if [ $(id -u) -eq 0 ] ; then
		echo "Do not run this as root."
		exit 1
	fi
56
	# abort certain jobs if we know they will fail due to certain bugs…
Holger Levsen's avatar
Holger Levsen committed
57
	BLOCKER=848422
58
	case $JOB_NAME in
59 60 61 62
		#chroot-installation_*_install_design-desktop-*)
		#	for BLOCKER in 869155 867695 ; do
		#		abort_if_bug_is_still_open $BLOCKER
		#	done ;;
63
		chroot-installation_buster_install_parl-desktop*)
64
			for BLOCKER in 886676 ; do
65 66
				abort_if_bug_is_still_open $BLOCKER
			done ;;
67 68
		#dpkg_*_find_trigger_cycles)
		#	abort_if_bug_is_still_open 874504 ;;
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
		#chroot-installation_stretch_install_full_desktop_upgrade_to_buster)
		#	abort_if_bug_is_still_open 877970 ;;
		#chroot-installation_stretch_install_developer_upgrade_to_buster)
		#	abort_if_bug_is_still_open 877970 ;;
		#chroot-installation_stretch_install_education-workstation_upgrade_to_buster)
		#	abort_if_bug_is_still_open 877970 ;;
		#chroot-installation_stretch_install_education-standalone_upgrade_to_buster)
		#	abort_if_bug_is_still_open 877970 ;;
		#chroot-installation_stretch_install_education-roaming-workstation_upgrade_to_buster)
		#	abort_if_bug_is_still_open 877970 ;;
		#chroot-installation_stretch_install_education-ltsp-server_upgrade_to_buster)
		#	abort_if_bug_is_still_open 877970 ;;
		#chroot-installation_stretch_install_education-desktop-other_upgrade_to_buster)
		#	abort_if_bug_is_still_open 877970 ;;
		#chroot-installation_stretch_install_developer_upgrade_to_buster)
		#	abort_if_bug_is_still_open 877970 ;;
85
		#lintian-tests_sid)
Holger Levsen's avatar
Holger Levsen committed
86
		#	abort_if_bug_is_still_open $BLOCKER ;;
87
		#haskell-package-plan)
Holger Levsen's avatar
Holger Levsen committed
88
		#	abort_if_bug_is_still_open $BLOCKER ;;
89
		#edu-packages_sid*)
Holger Levsen's avatar
Holger Levsen committed
90
		#	abort_if_bug_is_still_open $BLOCKER ;;
91
		#reproducible_*_from_git_master)
Holger Levsen's avatar
Holger Levsen committed
92
		#	abort_if_bug_is_still_open $BLOCKER ;;
93
		#chroot-installation_sid_install_education*)
Holger Levsen's avatar
Holger Levsen committed
94
		#	abort_if_bug_is_still_open $BLOCKER ;;
95
		#chroot-installation_stretch_install_education-*_upgrade_to_sid)
Holger Levsen's avatar
Holger Levsen committed
96
		#	abort_if_bug_is_still_open $BLOCKER ;;
97 98
		*) ;;
	esac
99
	# mktemp some place for us...
100
	TTT=$(mktemp --tmpdir=/tmp jenkins-script-XXXXXXXX)
101 102 103 104
	if [ -z "$TTT" ] ; then
		echo "Failed to create tmpfile, aborting. (Probably due to read-only filesystem…)"
		exit 1
	fi
105 106 107 108 109
	# prepare cleanup
	trap common_cleanup INT TERM EXIT
	# cp $0 to /tmp and run it from there
	cp $0 $TTT
	chmod +x $TTT
110 111
	echo "===================================================================================="
	echo
112
	echo "$(date -u) - running $0 ($JOB_NAME) on $(hostname) now."
113
	echo
114
	echo "To learn to understand this, git clone https://anonscm.debian.org/git/qa/jenkins.debian.net.git"
115
	echo "and then have a look at the files README, INSTALL, CONTRIBUTING and maybe TODO."
116
	echo
117 118 119
	echo "This invocation of this script, which is located in bin/$(basename $0),"
	echo "has been called using \"$@\" as arguments." 
	echo
120
	echo "Please send feedback about jenkins to qa-jenkins-dev@lists.alioth.debian.org,"
121
	echo "or as a bug against the 'jenkins.debian.org' pseudo-package,"
122
	echo "feedback about specific job results should go to their respective lists and/or the BTS."
123
	echo 
124
	echo "===================================================================================="
125
	echo "$(date -u) - start running \"$0\" (md5sum $(md5sum $0|cut -d ' ' -f1)) as \"$TTT\" on $(hostname)."
Holger Levsen's avatar
Holger Levsen committed
126
	echo
127
	# this is the "hack": call ourself as a copy in /tmp again
128
	$TTT "$@"
Holger Levsen's avatar
Holger Levsen committed
129 130
	exit $?
	# cleanup is done automatically via trap
131
else
132 133
	# this directory resides on tmpfs, so it might be gone after reboots...
	mkdir -p /srv/workspace/chroots
134 135
	# default settings used for the jenkins.debian.net environment
	if [ -z "$LC_ALL" ]; then
136
		export LC_ALL=C.UTF-8
137
	fi
138

139
	if [ -z "$MIRROR" ]; then
140
		case $HOSTNAME in
141
			jenkins|jenkins-test-vm|profitbricks-build*)
142
				export MIRROR=http://deb.debian.org/debian ;;
143
			bbx15|bpi0|cb3*|cbxi4*|hb0|wbq0|odxu4*|odu3*|odc*|wbd0|rpi2*|ff*|ff4*|opi2*|jt?1*|p64*)
144
				export MIRROR=http://deb.debian.org/debian ;;
145
			codethink*)
146
				export MIRROR=http://deb.debian.org/debian ;;
147 148 149 150
			spectrum)
				export MIRROR=none ;;
			*)
				echo "unsupported host, exiting." ; exit 1 ;;
151
		esac
152 153
	fi
	if [ -z "$http_proxy" ]; then
154
		case $HOSTNAME in
155
			jenkins|jenkins-test-vm|profitbricks-build*|codethink*)
156
				export http_proxy="http://localhost:3128" ;;
157
			bbx15|bpi0|cb3*|cbxi4*|hb0|wbq0|odxu4*|odu3*|odc*|wbd0|rpi2*|ff*|ff4*|opi2*|jt?1*|p64*)
158 159 160 161 162
				export http_proxy="http://10.0.0.15:8000/" ;;
			spectrum)
				export MIRROR=none ;;
			*)
				echo "unsupported host, exiting." ; exit 1 ;;
163
		esac
164 165 166 167 168 169 170
	fi
	if [ -z "$CHROOT_BASE" ]; then
		export CHROOT_BASE=/chroots
	fi
	if [ -z "$SCHROOT_BASE" ]; then
		export SCHROOT_BASE=/schroots
	fi
Holger Levsen's avatar
Holger Levsen committed
171 172 173 174
	if [ ! -d "$SCHROOT_BASE" ]; then
		echo "Directory $SCHROOT_BASE does not exist, aborting."
		exit 1
	fi
175 176
	# use these settings in the scripts in the (s)chroots too
	export SCRIPT_HEADER="#!/bin/bash
177 178 179
	if $DEBUG ; then
		set -x
	fi
180 181 182 183 184
	set -e
	export DEBIAN_FRONTEND=noninteractive
	export LC_ALL=$LC_ALL
	export http_proxy=$http_proxy
	export MIRROR=$MIRROR"
185 186 187 188 189
	# be more verbose, maybe
	if $DEBUG ; then
		export
		set -x
	fi
190 191 192 193
	set -e
fi
}

194
publish_changes_to_userContent() {
195 196 197 198 199 200 201 202 203
	echo "Extracting contents from .deb files..."
	CHANGES=$1
	CHANNEL=$2
	SRCPKG=$(basename $CHANGES | cut -d "_" -f1)
	if [ -z "$SRCPKG" ] ; then
		exit 1
	fi
	VERSION=$(basename $CHANGES | cut -d "_" -f2)
	TARGET="/var/lib/jenkins/userContent/$SRCPKG"
204
	NEW_CONTENT=$(mktemp -d -t new-content-XXXXXXXX)
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
	for DEB in $(dcmd --deb $CHANGES) ; do
		dpkg --extract $DEB ${NEW_CONTENT} 2>/dev/null
	done
	rm -rf $TARGET
	mkdir $TARGET
	mv ${NEW_CONTENT}/usr/share/doc/${SRCPKG}-* $TARGET/
	rm -r ${NEW_CONTENT}
	if [ -z "$3" ] ; then
		touch "$TARGET/${VERSION}"
		FROM=""
	else
		touch "$TARGET/${VERSION}_$3"
		FROM=" from $3"
	fi
	MESSAGE="https://jenkins.debian.net/userContent/$SRCPKG/ has been updated${FROM}."
	echo
	echo $MESSAGE
	echo
	if [ ! -z "$CHANNEL" ] ; then
		kgb-client --conf /srv/jenkins/kgb/$CHANNEL.conf --relay-msg "$MESSAGE"
	fi
}

228 229 230 231 232
cleanup_schroot_sessions() {
	echo
	local RESULT=""
	for loop in $(seq 0 40) ; do
		# first, check if no process using "schroot" is running, if thats the case, loop through all schroot sessions:
233
		# arch sessions are ignored, because they are handled properly
234
		pgrep -f "schroot --directory" || for i in $(schroot --all-sessions -l |grep -v "session:archlinux"||true) ; do
235
			# then, check that schroot is still not run, and then delete the session
236 237 238
			if [ -z $i ] ; then
				continue
			fi
239 240
			pgrep -f "schroot --directory" || schroot -e -c $i
		done
241
		RESULT=$(schroot --all-sessions -l|grep -v "session:archlinux"||true)
242 243 244 245 246 247 248 249 250 251 252
		if [ -z "$RESULT" ] ; then
			echo "No schroot sessions in use atm..."
			echo
			break
		fi
		echo "$(date -u) - schroot session cleanup loop $loop"
		sleep 15
	done
	echo
}