deploy_jdn 10 KB
Newer Older
1
#!/bin/bash
2
# vim: set noexpandtab:
3
#
4
# Copyright 2014-2019 Holger Levsen <holger@layer-acht.org>
5
#           ©    2018 Mattia Rizzolo <mattia@debian.org>
6 7 8 9 10 11 12 13
#
# released under the GPLv=2
#
# deployment helper for jenkins.debian.net and build nodes
# (misses freebsd node)

# disclaimers:
#   this script grew over the years…
14
#   the code is horrible and was always meant to be a quick local hack and never to be published
15 16 17 18
#   use at your own risk. it might eat your cats. you have been warned.
#   the HOSTS lists below is yet another code duplication…

# useful things:
19
# ./deploy_jdn all		- deploy on all nodes (and handle 398 days hosts properly)
20
# ./deploy_jdn $host		- deploy on $host and jenkins (and handle 398 days hosts properly)
21 22 23 24
# ./deploy_jdn all $foo		- run "$foo" on all nodes (and handle 398 days hosts properly)
# ./deploy_jdn 			- deploy on jenkins only
# ./deploy_jdn jenkins		- deploy on jenkins only
# ./deploy_jdn jenkins pb10	- deploy on jenkins and pb10
Holger Levsen's avatar
Holger Levsen committed
25 26
# ./deploy_jdn jenkins o167	- deploy on jenkins and osuosl167
# ./deploy_jdn jenkins c9	- deploy on jenkins and codethink-sled9
27
# ./deploy_jdn jenkins 10	- deploy on jenkins and pb10
28 29
# ./deploy_jdn jenkins 5 6	- deploy on jenkins and pb6 and pb6
# ./deploy_jdn jenkins amd64	- deploy on jenkins and all amd64 nodes
30 31 32
# ./deploy_jdn upgrade		- run "apt-get update && upgrade && clean" everywhere
# ./deploy_jdn upgradey		- run "apt-get upgrade -y" everywhere
# ./deploy_jdn rmstamp		- delete stamp files everywhere
33 34 35


START=$(date +'%s')
36
GIT_REPO="https://salsa.debian.org/qa/jenkins.debian.net.git"
37
HOSTS=(
38
codethink-sled9-arm64.debian.net
39 40 41 42 43 44 45
codethink-sled10-arm64.debian.net
codethink-sled11-arm64.debian.net
codethink-sled12-arm64.debian.net
codethink-sled13-arm64.debian.net
codethink-sled14-arm64.debian.net
codethink-sled15-arm64.debian.net
codethink-sled16-arm64.debian.net
46
profitbricks-build2-i386.debian.net
47 48 49 50
profitbricks-build6-i386.debian.net
profitbricks-build12-i386.debian.net
profitbricks-build16-i386.debian.net
profitbricks-build1-amd64.debian.net
51 52 53
profitbricks-build5-amd64.debian.net
profitbricks-build9-amd64.debian.net
profitbricks-build10-amd64.debian.net
54 55
profitbricks-build11-amd64.debian.net
profitbricks-build15-amd64.debian.net
56 57 58 59 60 61 62 63
osuosl-build167-amd64.debian.net
osuosl-build168-amd64.debian.net
osuosl-build169-amd64.debian.net
osuosl-build170-amd64.debian.net
osuosl-build171-amd64.debian.net
osuosl-build172-amd64.debian.net
osuosl-build173-amd64.debian.net
osuosl-build174-amd64.debian.net
64
bbx15-armhf-rb.debian.net
Holger Levsen's avatar
Holger Levsen committed
65
cb3a-armhf-rb.debian.net
66 67
cbxi4a-armhf-rb.debian.net
cbxi4b-armhf-rb.debian.net
Holger Levsen's avatar
Holger Levsen committed
68
cbxi4pro0-armhf-rb.debian.net
69 70 71
ff2a-armhf-rb.debian.net
ff2b-armhf-rb.debian.net
ff4a-armhf-rb.debian.net
72
ff64a-armhf-rb.debian.net
73
jtk1a-armhf-rb.debian.net
74
jtk1b-armhf-rb.debian.net
75
jtx1a-armhf-rb.debian.net
76
jtx1b-armhf-rb.debian.net
77
jtx1c-armhf-rb.debian.net
Holger Levsen's avatar
Holger Levsen committed
78
odu3a-armhf-rb.debian.net
79
odxu4a-armhf-rb.debian.net
Holger Levsen's avatar
Holger Levsen committed
80 81
odxu4b-armhf-rb.debian.net
odxu4c-armhf-rb.debian.net
82 83 84
opi2a-armhf-rb.debian.net
opi2b-armhf-rb.debian.net
opi2c-armhf-rb.debian.net
85 86
p64b-armhf-rb.debian.net
p64c-armhf-rb.debian.net
Holger Levsen's avatar
Holger Levsen committed
87
wbq0-armhf-rb.debian.net
88
root@jenkins.debian.net
89
)
Holger Levsen's avatar
Holger Levsen committed
90

91
ALL_HOSTS=("${HOSTS[@]}")
92

93 94
node_in_the_future () {
	case "$1" in
95
		profitbricks-build5-amd64*|profitbricks-build6-i386*|profitbricks-build15-amd64*|profitbricks-build16-i386*) true ;;
96
		codethink-sled9*|codethink-sled11*|codethink-sled13*|codethink-sled15*) true ;;
Mattia Rizzolo's avatar
Mattia Rizzolo committed
97
		osuosl-build170*|osuosl-build172*) true ;;
98 99 100 101
		*) false ;;
	esac
}

102 103 104
echo
echo -n "$(date) - "
reset_clock=true
105
if [ "$1" = "all" ] ; then
106
	echo -n "Running j.d.n.git updates on ${HOSTS[@]} now"
107 108
	# reset_clock can be false as update_jdn.sh sets the time
	reset_clock=false
109
	shift
Holger Levsen's avatar
Holger Levsen committed
110
	if [ ! -z "$1" ] ; then
111
		real_command="$@"
112
		echo -n "Running '$real_command' on ${HOSTS[@]} now."
113 114
		real_command="$@ && echo '__reallyreally=ok__'"
	fi
115 116 117 118 119 120 121
elif [ "$1" = "upgrade" ] ; then
	real_command="export LANG=C && sudo apt-get update && sudo apt-get upgrade && sudo apt-get dist-upgrade && sudo apt-get clean"
	shift
elif [ "$1" = "upgradey" ] ; then
	real_command="export LANG=C && sudo apt-get update && sudo apt-get -y upgrade && sudo apt-get -y dist-upgrade && sudo apt-get clean"
	shift
elif [ "$1" = "rmstamp" ] ; then
122
	real_command="sudo rm -f /var/log/jenkins/*stamp && echo '__reallyreally=ok__'"
123 124 125 126 127 128
	reset_clock=false
	shift
elif [ "$1" = "check" ] ; then
	real_command="pgrep -l -a -f _build.sh"
	reset_clock=false
	shift
129
elif [ "$1" = "" ] ; then
130 131
	HOSTS=(root@jenkins.debian.net)
	echo -n "Running j.d.n.git updates on ${HOSTS[@]} now"
132
elif [ "$1" = "jenkins" ] ; then
133
	HOSTS=(root@jenkins.debian.net)
134
	shift
135 136 137 138 139 140 141 142 143 144
	for i in "$@" ; do
		case "$i" in
			1|pb1)	 HOSTS+=(profitbricks-build1-amd64.debian.net) ;;
			2|pb2)	 HOSTS+=(profitbricks-build2-i386.debian.net) ;;
			5|pb5)	 HOSTS+=(profitbricks-build5-amd64.debian.net) ;;
			6|pb6)	 HOSTS+=(profitbricks-build6-i386.debian.net) ;;
			9|pb9)	 HOSTS+=(profitbricks-build9-amd64.debian.net) ;;
			10|pb10) HOSTS+=(profitbricks-build10-amd64.debian.net) ;;
			11|pb11) HOSTS+=(profitbricks-build11-amd64.debian.net) ;;
			15|pb15) HOSTS+=(profitbricks-build15-amd64.debian.net) ;;
145 146 147 148 149 150 151 152
			o167)		HOSTS+=(osuosl-build167-amd64.debian.net) ;;
			o168)		HOSTS+=(osuosl-build168-amd64.debian.net) ;;
			o169)		HOSTS+=(osuosl-build169-amd64.debian.net) ;;
			o170)		HOSTS+=(osuosl-build170-amd64.debian.net) ;;
			o171)		HOSTS+=(osuosl-build171-amd64.debian.net) ;;
			o172)		HOSTS+=(osuosl-build172-amd64.debian.net) ;;
			o173)		HOSTS+=(osuosl-build173-amd64.debian.net) ;;
			o174)		HOSTS+=(osuosl-build174-amd64.debian.net) ;;
153 154 155 156 157 158 159 160 161 162
			c9|cs9)		HOSTS+=(codethink-sled9-arm64.debian.net) ;;
			c10|cs10)	HOSTS+=(codethink-sled10-arm64.debian.net) ;;
			c11|cs11)	HOSTS+=(codethink-sled11-arm64.debian.net) ;;
			c12|cs12)	HOSTS+=(codethink-sled12-arm64.debian.net) ;;
			c13|cs13)	HOSTS+=(codethink-sled13-arm64.debian.net) ;;
			c14|cs14)	HOSTS+=(codethink-sled14-arm64.debian.net) ;;
			c15|cs15)	HOSTS+=(codethink-sled15-arm64.debian.net) ;;
			c16|cs16)	HOSTS+=(codethink-sled16-arm64.debian.net) ;;
			armhf|amd64|i386|arm64)	 HOSTS+=($(echo "${ALL_HOSTS[@]}" | sed -e 's# #\n#g' | grep "$i")) ;;
			*) 	if ping -c 1 "$i" ; then HOSTS+=("$i") ; fi ;;
163 164
		esac
	done
165
	echo -n "Running j.d.n.git updates on ${HOSTS[@]} now"
166
fi
167 168 169
BG=""

get_arch_color() {
170
	case "$1" in
171
		*amd64*)		BG=lightgreen ;;
172 173
		*i386*)			BG=lightblue ;;
		*arm64*)		BG=orange ;;
174 175
		*armhf*)		BG=lightyellow ;;
		*jenkins.debian.*)	BG=yellow ;;
176
		*)				BG=white ;;
177 178
	esac
}
179

180 181 182 183 184
LOG="$(mktemp -u)"
# reverse the array
STSOH=()
for i in "${HOSTS[@]}" ; do
	STSOH=("$i" "${STSOH[@]}")
185
done
186 187 188
HOSTS=("${STSOH[@]}")

for i in "${HOSTS[@]}" ; do
189
	if [ -z "$real_command" ]; then
190 191
		if node_in_the_future "$i"; then GITOPTS="-c http.sslVerify=false" ; fi
		# real command, for running manually: cd ~jenkins-adm/jenkins.debian.net/ ; sudo -u jenkins-adm git pull ; ./update_jdn.sh
Mattia Rizzolo's avatar
Mattia Rizzolo committed
192
		read -r -d '' remote_command <<-EOF
193 194 195 196 197
			set -e
			export LANG=C
			cd ~jenkins-adm
			if [ ! -d jenkins.debian.net ]; then
				[ -x /usr/bin/git ] || sudo apt-get install -y git
198
				sudo -u jenkins-adm git ${GITOPTS:-} clone $GIT_REPO
199 200 201
				cd jenkins.debian.net
			else
				cd jenkins.debian.net
202
				sudo -u jenkins-adm git ${GITOPTS:-} pull $GIT_REPO
203 204 205
			fi
			./update_jdn.sh 2>&1 | sudo tee -a /var/log/jenkins/update_jdn.log
		EOF
206 207
	else
		remote_command="$real_command"
208 209
	fi

210 211
	echo -n "."
	if $reset_clock ; then
212 213
		if node_in_the_future "$i" ; then
			#  set correct future date
214
			case "$i" in
215 216
				osuosl*)	NTP_SERVER=time.osuosl.org ;;
				*)		NTP_SERVER=de.pool.ntp.org ;;
217 218
			esac
			remote_command="sudo ntpdate -b $NTP_SERVER && $remote_command && sudo date --set=\"+398 days +6 hours + 23 minutes\" && echo '__$(echo $i|cut -d '.' -f1)=ok__'"
219
		fi
220
	fi
221
	get_arch_color $i
Mattia Rizzolo's avatar
Mattia Rizzolo committed
222
	xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -o 'BatchMode = yes' -t $i '$remote_command' | tee $LOG.$i ; sleep 2 ; touch $LOG.$i.done" &
223
	unset GITOPTS
224 225 226
done
sleep 3
COUNTER=0
227
for i in "${HOSTS[@]}" ; do
228 229 230 231 232 233 234 235 236 237 238 239 240 241
	while ! test -f $LOG.$i.done ; do
		let COUNTER+=1
		sleep 1
		echo -n "."
		if [ $COUNTER -eq 42 ] ; then
			echo 
			echo -n "$LOG.$i.done still doesnt exist, how strange…"
			COUNTER=0
			continue
		fi
	done
done
echo

242 243 244 245
JENKINS_OFFLINE_LIST="$(dirname $0)/jenkins-home/offline_nodes"
echo
echo $JENKINS_OFFLINE_LIST
echo
246
OFFLINE=""
247
PROBLEMS=""
248
for i in "${HOSTS[@]}" ; do
249 250
	HNAME1=$(echo $i | cut -d "@" -f2 | cut -d "." -f1|cut -d "-" -f1)	# pb nodes (h01ger)
	HNAME2=$(echo $i | cut -d "@" -f2 | cut -d "." -f1)			# non -armhf ones (vagrant)
251
	TAIL=$(tail -1 $LOG.$i 2>/dev/null)
252
	if [ "$i" = "root@jenkins.debian.net" ] ; then
253 254 255 256 257
		if ! ( [[ "$TAIL" =~ "__$HNAME1=ok__" ]] || [[ "$TAIL" =~ "__$HNAME2=ok__" ]] || [[ "$TAIL" =~ "__reallyreally=ok__" ]] || [[ "$TAIL" =~ "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." ]] ) ; then
			echo "Problems on $i:"
		fi
		cat $LOG.$i
		rm $LOG.$i $LOG.$i.done > /dev/null
258
	elif [[ "$TAIL" =~ "__$HNAME1=ok__" ]] || [[ "$TAIL" =~ "__$HNAME2=ok__" ]] || [[ "$TAIL" =~ "__reallyreally=ok__" ]] || [[ "$TAIL" =~ "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." ]] || [ ! -f $LOG.$i ] ; then
259 260 261
		rm $LOG.$i $LOG.$i.done > /dev/null
	else
		echo "Problems on $i: $TAIL"
262
		KNOWN_OFFLINE=false
263 264
		if [ -f "$JENKINS_OFFLINE_LIST" ]; then
			if grep -q "$i" "$JENKINS_OFFLINE_LIST"; then
265
				KNOWN_OFFLINE=true
266 267 268 269 270 271
				if [ -z "$OFFLINE" ] ; then
					OFFLINE=" $i"
				else
					OFFLINE=" $i\n$OFFLINE"
				fi
			fi
272 273 274 275 276 277 278
		fi
		if ! $KNOWN_OFFLINE ; then
			if [ -z "$PROBLEMS" ] ; then
				PROBLEMS=" $i"
			else
				PROBLEMS=" $i\n$PROBLEMS"
			fi
279
		fi
280 281
		get_arch_color $i
		xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "less +G $LOG.$i ; rm $LOG.$i $LOG.$i.done" &
282 283 284 285
	fi
done
echo

286
echo "$(echo "${HOSTS[@]}" | sed -s "s# #\n#g" | wc -l) hosts updated."
287 288 289 290 291
if [ ! -z "$PROBLEMS" ] ; then
	echo "Problems on:"
	echo -e "$PROBLEMS"
	echo
fi
292 293 294 295 296
if [ ! -z "$OFFLINE" ] ; then
	echo "Offline nodes with unsurprising problems encountered:"
	echo -e "$OFFLINE"
	echo
fi
297 298 299 300 301 302 303 304
END=$(date +'%s')
DURATION=$(( $END - $START ))
HOUR=$(echo "$DURATION/3600"|bc)
MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc)
SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc)
echo "$(date) - total duration: ${HOUR}h ${MIN}m ${SEC}s."
echo