update_jdn.sh 25.9 KB
Newer Older
1
#!/bin/bash
2
# vim: set noexpandtab:
3
# Copyright 2012-2019 Holger Levsen <holger@layer-acht.org>
4
#         ©      2018 Mattia Rizzolo <mattia@debian.org>
5
6
# released under the GPLv=2

Holger Levsen's avatar
Holger Levsen committed
7
# puppet / salt / ansible / fai / chef / deployme.app - disclaimer
Holger Levsen's avatar
Holger Levsen committed
8
9
10
11
12
13
14
15
16
17
18
# (IOW: this script has been grown in almost 500 commits and it shows…)
#
# yes, we know… and: "it" should probably still be done.
#
# It just unclear, how/what, and what we have actually mostly works.
#
# Switching to jenkins.debian.org is probably an opportunity
# to write (refactor this into) *yet another deployment script*
# (interacting with the DSA machine setup which is in puppet…),
# thus obsoleting this script gradually, though this is used on
# 47 hosts currently (of which quite some were initially installed
Edward Betts's avatar
Edward Betts committed
19
# manually…)
Holger Levsen's avatar
Holger Levsen committed
20
21
22
#
# so, yes, patches welcome. saying this is crap alone is not helpful,
# nor is just suggesting some new or old technology. patches most welcome!
23
24
#
# that said, there's a new one: init_node ;)
Holger Levsen's avatar
Holger Levsen committed
25

26
27
set -e

Holger Levsen's avatar
Holger Levsen committed
28
# prevent failure on first run...
Holger Levsen's avatar
Holger Levsen committed
29
[ ! -f /srv/jenkins/bin/common-functions.sh ] || . /srv/jenkins/bin/common-functions.sh
Holger Levsen's avatar
Holger Levsen committed
30

31
BASEDIR="$(dirname "$(readlink -e $0)")"
32
STAMP=/var/log/jenkins/update-jenkins.stamp
33
34
# The $@ below means that command line args get passed on to j-j-b
# which allows one to specify --flush-cache or --ignore-cache
35
JJB="jenkins-jobs $@"
36
DPKG_ARCH="$(dpkg --print-architecture)"
Holger Levsen's avatar
Holger Levsen committed
37

38
# so we can later run some commands only if $0 has been updated…
Philip Hands's avatar
Philip Hands committed
39
if [ -f $STAMP ] && [ $STAMP -nt $BASEDIR/$0 ] ; then
Holger Levsen's avatar
Holger Levsen committed
40
	UP2DATE=true
Holger Levsen's avatar
Holger Levsen committed
41
	echo $HOSTNAME is up2date.
Philip Hands's avatar
Philip Hands committed
42
43
else
	UP2DATE=false
Holger Levsen's avatar
Holger Levsen committed
44
	echo $HOSTNAME needs to be updated.
45
46
47
fi


Holger Levsen's avatar
Holger Levsen committed
48
explain() {
Holger Levsen's avatar
Holger Levsen committed
49
	echo "$HOSTNAME: $1"
Holger Levsen's avatar
Holger Levsen committed
50
51
}

52
53
54
set_correct_date() {
		# set correct date
		sudo service ntp stop || true
55
		sudo ntpdate -b $1
56
}
57

58
disable_dsa_check_packages() {
59
	# disable check for outdated packages as someday in the future
60
	# packages from security.d.o will appear outdated always…
61
62
	echo -e "#!/bin/sh\n# disabled dsa-check by update_jdn.sh\nexit 0" | sudo tee /usr/local/bin/dsa-check-packages
	sudo chmod a+rx /usr/local/bin/dsa-check-packages
63
64
}

65
echo "--------------------------------------------"
Holger Levsen's avatar
Holger Levsen committed
66
explain "$(date) - begin deployment update."
67

68
69
70
71
#
# temporarily test to check which hosts don't use systemd
#
if [ -z "$(dpkg -l|grep systemd-sysv||true)" ] ; then 
72
	echo "no systemd-sysv installed on $(hostname), please enter to continue…"
73
74
75
	read
fi

76
# some nodes need special treatment…
77
case $HOSTNAME in
78
	profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386)
Holger Levsen's avatar
Holger Levsen committed
79
		# set correct date
80
		set_correct_date de.pool.ntp.org
81
		;;
82
	codethink-sled9*|codethink-sled11*|codethink-sled13*|codethink-sled15*)
83
		# set correct date
84
		set_correct_date de.pool.ntp.org
Holger Levsen's avatar
Holger Levsen committed
85
		;;
86
	osuosl-build170-amd64|osuosl-build172-amd64)
87
		# set correct date
88
		set_correct_date time.osuosl.org
89
		;;
90
91
92
	*)	;;
esac

93
94
95
96
97
98
99
100
101
102
103
# ubuntu decided to change kernel perms in the middle of LTS…
case $HOSTNAME in
	codethink-sled*)
		# fixup perms
		sudo chmod +r /boot/vmlinuz-*
		;;
	*)	;;
esac



104
105
106
#
# set up users and groups
#
107
declare -A user_host_groups u_shell
108
sudo_groups='jenkins,jenkins-adm,sudo,adm'
109
110
111

# if there's a need for host groups, a case statement on $HOSTNAME here that sets $GROUPNAME, say, should do the trick
# then you can define user_host_groups['phil','lvm_group']=... below
112
# and add checks for the GROUP version where ever the HOSTNAME is checked in the following code
113
114
115
116
117

user_host_groups['helmut','*']="$sudo_groups"
user_host_groups['holger','*']="$sudo_groups"
user_host_groups['holger','jenkins']="reproducible,${user_host_groups['holger','*']}"
user_host_groups['mattia','*']="$sudo_groups"
118
user_host_groups['mattia','jenkins']="reproducible,${user_host_groups['mattia','*']}"
119
user_host_groups['phil','jenkins-test-vm']="$sudo_groups,libvirt,libvirt-qemu"
120
user_host_groups['phil','jenkins']="$sudo_groups"
121
user_host_groups['lunar','jenkins']='reproducible'
122
123
user_host_groups['lynxis','osuosl-build171-amd64']="$sudo_groups"
user_host_groups['lynxis','osuosl-build172-amd64']="$sudo_groups"
124
user_host_groups['lynxis','jenkins']="jenkins"
125
user_host_groups['hans','osuosl-build168-amd64']="$sudo_groups"
Mattia Rizzolo's avatar
Mattia Rizzolo committed
126
user_host_groups['vagrant','*']="$sudo_groups"
127
user_host_groups['bernhard','osuosl-build174-amd64']="$sudo_groups"
128
129

u_shell['mattia']='/bin/zsh'
130
u_shell['lynxis']='/usr/bin/fish'
131
u_shell['jenkins-adm']='/bin/bash'
132

133
134
135
# get the users out of the user_host_groups array's index
users=$(for i in ${!user_host_groups[@]}; do echo ${i%,*} ; done | sort -u)

136
( $UP2DATE && [ -z "$(find authorized_keys -newer $0)" ] ) || for user in ${users}; do
137
	# -v is a bashism to check for set variables, used here to see if this user is active on this host
138
	if [ ! -v user_host_groups["$user","$HOSTNAME"] ] && [ ! -v user_host_groups["$user",'*'] ] && [ ! -v user_host_groups["$user","$DPKG_ARCH"] ] ; then
Holger Levsen's avatar
Holger Levsen committed
139
140
		continue
	fi
141

142
143
144
145
146
147
	# create the user
	if ! getent passwd $user > /dev/null ; then
		# adduser, defaulting to /bin/bash as shell
		sudo adduser --gecos "" --shell "${u_shell[$user]:-/bin/bash}" --disabled-password $user
	fi
	# add groups: first try the specific host, or if unset fall-back to default '*' setting
148
	for h in "$HOSTNAME" "$DPKG_ARCH" '*' ; do
149
		if [ -v user_host_groups["$user","$h"] ] ; then
Holger Levsen's avatar
Holger Levsen committed
150
			sudo usermod -G "${user_host_groups["$user","$h"]}" $user
151
			break
152
		fi
153
154
155
156
157
	done
	# add the user's keys (if any)
	if ls authorized_keys/${user}@*.pub >/dev/null 2>&1 ; then
		[ -d /var/lib/misc/userkeys ] || sudo mkdir -p /var/lib/misc/userkeys
		cat authorized_keys/${user}@*.pub | sudo tee /var/lib/misc/userkeys/${user} > /dev/null
158
	fi
159
160
done

161
162
163
164
165
sudo mkdir -p /srv/workspace
[ -d /srv/schroots ] || sudo mkdir -p /srv/schroots
[ -h /chroots ] || sudo ln -s /srv/workspace/chroots /chroots
[ -h /schroots ] || sudo ln -s /srv/schroots /schroots

166
167
# prepare tmpfs on some hosts
case $HOSTNAME in
Mattia Rizzolo's avatar
Mattia Rizzolo committed
168
169
170
171
	jenkins)
		TMPFSSIZE=100
		TMPSIZE=15
		;;
Holger Levsen's avatar
Holger Levsen committed
172
	profitbricks-build9-amd64)
173
		TMPFSSIZE=40
Mattia Rizzolo's avatar
Mattia Rizzolo committed
174
175
		TMPSIZE=8
		;;
Holger Levsen's avatar
Holger Levsen committed
176
	profitbricks-build*)
Mattia Rizzolo's avatar
Mattia Rizzolo committed
177
178
179
		TMPFSSIZE=200
		TMPSIZE=15
		;;
180
181
182
183
	codethink*)
		TMPFSSIZE=100
		TMPSIZE=15
		;;
184
185
186
187
	osuosl*)
		TMPFSSIZE=400
		TMPSIZE=50
		;;
188
189
190
	*) ;;
esac
case $HOSTNAME in
191
192
193
194
195
196
	profitbricks-build*i386)
		if ! grep -q '/srv/workspace' /etc/fstab; then
			echo "Warning: you need to manually create a /srv/workspace partition on i386 nodes, exiting."
			exit 1
		fi
		;;
197
	jenkins|profitbricks-build*amd64|codethink*|osuosl*)
198
		if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
199
			echo "tmpfs		/srv/workspace	tmpfs	defaults,size=${TMPFSSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null  
200
		fi
Mattia Rizzolo's avatar
Mattia Rizzolo committed
201
202
203
		if ! grep -q '^tmpfs\s\+/tmp\s' /etc/fstab; then
			echo "tmpfs		/tmp	tmpfs	defaults,size=${TMPSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null
		fi
204
205
		if ! mountpoint -q /srv/workspace; then
			if test -z "$(ls -A /srv/workspace)"; then
206
				sudo mount /srv/workspace
207
			else
208
				explain "WARNING: mountpoint /srv/workspace is non-empty."
209
210
211
212
213
			fi
		fi
		;;
	*) ;;
esac
214

215
# make sure needed directories exists - some directories will not be needed on all hosts...
216
for directory in /schroots /srv/reproducible-results /srv/d-i /srv/udebs /var/log/jenkins/ /srv/jenkins /srv/jenkins/pseudo-hosts /srv/workspace/chroots ; do
217
218
219
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
	fi
Holger Levsen's avatar
Holger Levsen committed
220
	sudo chown jenkins.jenkins $directory
221
done
Holger Levsen's avatar
Holger Levsen committed
222
for directory in /srv/jenkins ; do
223
224
225
226
227
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
		sudo chown jenkins-adm.jenkins-adm $directory
	fi
done
228

229
if ! test -h /chroots; then
Holger Levsen's avatar
Holger Levsen committed
230
	sudo rmdir /chroots || sudo rm -f /chroots # do not recurse
231
	if test -e /chroots; then
Holger Levsen's avatar
Holger Levsen committed
232
		explain "/chroots could not be cleared."
233
	else
234
		sudo ln -s /srv/workspace/chroots /chroots
235
236
237
	fi
fi

238
239
240
241
242
# only on Debian systems
if [ -f /etc/debian_version ] ; then
	#
	# install packages we need
	#
243
	if [ $BASEDIR/$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then
Holger Levsen's avatar
Holger Levsen committed
244
245
		DEBS=" 
			bash-completion 
246
247
			bc
			bsd-mailx
248
249
			curl
			debian-archive-keyring
250
			cdebootstrap-
251
252
			devscripts
			eatmydata
253
			etckeeper
254
			figlet
Holger Levsen's avatar
Holger Levsen committed
255
			git
Holger Levsen's avatar
Holger Levsen committed
256
			gnupg
257
			haveged
Holger Levsen's avatar
Holger Levsen committed
258
			htop
259
			less
260
			locales-all
Mattia Rizzolo's avatar
Mattia Rizzolo committed
261
			lsof
262
263
			molly-guard
			moreutils
Holger Levsen's avatar
Holger Levsen committed
264
			needrestart
265
			netcat-traditional
266
267
			ntp
			ntpdate
268
			pigz 
269
			postfix
270
			procmail
271
			psmisc
272
			python3-psycopg2 
273
			python3-yaml
Holger Levsen's avatar
Holger Levsen committed
274
			schroot 
275
276
			screen
			slay
277
			stunnel
Holger Levsen's avatar
Holger Levsen committed
278
279
			subversion 
			subversion-tools 
280
			systemd-sysv
Holger Levsen's avatar
Holger Levsen committed
281
282
283
			sudo 
			unzip 
			vim 
284
			zsh
285
			"
286
		case $HOSTNAME in
287
288
289
			codethink*)
				# install debootstrap from stretch-backports on ubuntu nodes as since 20180927 debootstrap 1.0.78+nmu1ubuntu1.6 cannot install sid anymore
				DEBS="$DEBS
290
291
292
293
294
295
296
				debootstrap/stretch-backports
				munin-node/stretch-backports
				munin-plugins-core/stretch-backports
				munin-plugins-extra/stretch-backports
				pbuilder/stretch-backports
				" ;;
		esac
297
		case $HOSTNAME in
298
			# needed for rebuilding Debian (using .buildinfo files)
299
			osuosl-build173-amd64) DEBS="$DEBS libdpkg-perl libwww-mechanize-perl sbuild" ;;
300
301
			# needed for opensuse openbuildservice
			osuosl-build174-amd64) DEBS="$DEBS osc obs-build qemu-kvm libjson-perl libxml-simple-perl bsdtar" ;;
302
303
			*) ;;
		esac
304
		# install squid / apache2 on a few nodes only
305
		case $HOSTNAME in
306
			profitbricks-build1-a*|profitbricks-build10*|codethink-sled16*|osuosl-build167*) DEBS="$DEBS
307
				squid" ;;
308
309
			profitbricks-build7-a*) DEBS="$DEBS
				apache2" ;;
310
311
312
313
314
			*) ;;
		esac
		# notifications are only done from a view nodes
		case $HOSTNAME in
			jenkins|jenkins-test-vm|profitbricks-build*) DEBS="$DEBS
315
316
				kgb-client
				python3-yaml" ;;
317
318
			*) ;;
		esac
319
320
		# needed to run the 2nd reproducible builds nodes in the future...
		case $HOSTNAME in
321
			profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386) DEBS="$DEBS ntpdate" ;;
322
			codethink-sled9*|codethink-sled11*|codethink-sled13*|codethink-sled15*) DEBS="$DEBS ntpdate" ;;
323
			osuosl-build170-amd64|osuosl-build172-amd64) DEBS="$DEBS ntpdate" ;;
324
325
			*) ;;
		esac
326
		# needed to run coreboot/openwrt/netbsd/fedora jobs
327
		case $HOSTNAME in
328
		osuosl-build171-amd64|osuosl-build172-amd64) DEBS="$DEBS
Holger Levsen's avatar
Holger Levsen committed
329
				bison
330
				ca-certificates
Holger Levsen's avatar
Holger Levsen committed
331
332
333
				cmake
				diffutils
				findutils
334
				fish
Holger Levsen's avatar
Holger Levsen committed
335
336
337
338
				flex
				g++
				gawk
				gcc
339
				gnat
Holger Levsen's avatar
Holger Levsen committed
340
341
342
343
344
345
346
347
				grep
				libc6-dev
				libncurses5-dev
				libssl-dev
				locales-all
				kgb-client
				m4
				make
348
349
				python3-clint
				python3-git
350
				python3-pystache
351
				python3-requests
Holger Levsen's avatar
Holger Levsen committed
352
353
				python3-yaml
				subversion
354
				tree
Holger Levsen's avatar
Holger Levsen committed
355
356
				unzip
				util-linux
357
358
				zlib1g-dev
				"
359
360
361
			;;
			*) ;;
		esac
362
363
		# needed to run fdroid jobs
		case $HOSTNAME in
364
			osuosl-build168-amd64) DEBS="$DEBS
365
				androguard
366
				android-sdk
367
368
				bzr
				git-svn
369
				fdroidserver
370
				linux-headers-amd64
371
				mercurial
372
				python3-asn1crypto
373
				python3-babel
374
				python3-mwclient
375
				python3-setuptools
376
				subversion
377
				vagrant
378
				virtualbox virtualbox-dkms
379
				"
380
			;;
381
			*) ;;
382
		esac
383
384
385
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			# for phil only
			DEBS="$DEBS postfix-pcre"
386
387
			# only needed on the main node
		elif [ "$HOSTNAME" = "jenkins" ] ; then
388
			DEBS="$DEBS ffmpeg python3-popcon dose-extra"
389
390
391
392
		fi
		# mock is needed to build fedora
		if [ "$HOSTNAME" = "osuosl-build171-amd64" ] || [ "$HOSTNAME" = "osuosl-build172-amd64" ] || [ "$HOSTNAME" = "jenkins" ] ; then
			DEBS="$DEBS mock"
393
		fi
394
		# only on main node
395
		if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
Holger Levsen's avatar
Holger Levsen committed
396
397
398
399
			MASTERDEBS=" 
				apache2 
				apt-file 
				apt-listchanges 
400
				asciidoc
Holger Levsen's avatar
Holger Levsen committed
401
				binfmt-support 
402
403
				bison
				botch
Holger Levsen's avatar
Holger Levsen committed
404
405
406
407
408
409
410
				build-essential 
				cmake 
				cron-apt 
				csvtool 
				dnsmasq-base 
				dstat 
				figlet 
Holger Levsen's avatar
Holger Levsen committed
411
				flex
Holger Levsen's avatar
Holger Levsen committed
412
				gawk 
413
				ghc
414
				git-lfs
415
				git-notifier 
Holger Levsen's avatar
Holger Levsen committed
416
417
418
419
				gocr 
				graphviz 
				iasl 
				imagemagick 
420
421
				ip2host
				jekyll
422
				jenkins-job-builder
423
				jq
424
				kgb-client
Holger Levsen's avatar
Holger Levsen committed
425
426
427
428
429
430
431
432
433
				libcap2-bin 
				libfile-touch-perl 
				libguestfs-tools 
				libjson-rpc-perl 
				libsoap-lite-perl 
				libxslt1-dev 
				moreutils 
				mr 
				mtr-tiny 
434
				munin
Holger Levsen's avatar
Holger Levsen committed
435
				ntp 
436
				obfs4proxy
Holger Levsen's avatar
Holger Levsen committed
437
438
				openbios-ppc 
				openbios-sparc 
439
				openjdk-11-jre-headless
440
				pandoc
441
				postgresql
442
				postgresql-autodoc
443
				postgresql-client 
Holger Levsen's avatar
Holger Levsen committed
444
445
446
				poxml 
				procmail 
				python3-debian 
447
				python3-pystache
448
				python3-requests
449
				python3-rpy2 
450
451
452
				python3-sqlalchemy
				python3-xdg
				python3-yaml
Holger Levsen's avatar
Holger Levsen committed
453
454
455
				python-arpy 
				python-hachoir-metadata 
				python-lzma 
456
				python-pil
Holger Levsen's avatar
Holger Levsen committed
457
458
459
460
461
462
463
464
465
				python-pip 
				python-setuptools 
				python-twisted 
				python-yaml 
				qemu 
				qemu-kvm 
				qemu-system-x86 
				qemu-user-static 
				radvd 
466
467
				ruby-rspec
				rustc
Holger Levsen's avatar
Holger Levsen committed
468
469
470
471
				seabios 
				shorewall 
				shorewall6 
				sqlite3 
472
				syslinux
473
				systemd
474
				thin-provisioning-tools
475
				tor
Holger Levsen's avatar
Holger Levsen committed
476
				vncsnapshot 
477
478
				vnstat
				whohas
Holger Levsen's avatar
Holger Levsen committed
479
				x11-apps 
480
				xtightvncviewer
481
				xvfb
482
				xvkbd
483
484
				zutils
				"
485
486
487
		else
			MASTERDEBS=""
		fi
488
		$UP2DATE || sudo apt-get update
489
		$UP2DATE || sudo apt-get install $DEBS $MASTERDEBS
490
491
492
493
494
495
		# for varying kernels:
		# - we use bpo kernels on pb-build5+15 (and the default amd64 kernel on pb-build6+16-i386)
		# - we also use the bpo kernel on osuosl-build172 (but not osuosl-build171)
		# - this is done as a seperate step as bpo kernels are frequently uninstallable when upgraded on bpo
		if [ "$HOSTNAME" = "profitbricks-build5-amd64" ] || [ "$HOSTNAME" = "profitbricks-build15-amd64" ] \
			|| [ "$HOSTNAME" = "osuosl-build172-amd64" ] ; then
496
			sudo apt install linux-image-amd64/buster-backports || true # backport kernels are frequently uninstallable...
497
			:
498
499
500
501
502
503
		elif [ "$HOSTNAME" = "profitbricks-build6-i386" ] || [ "$HOSTNAME" = "profitbricks-build16-i386" ] \
			|| [ "$HOSTNAME" = "profitbricks-build2-i386" ] || [ "$HOSTNAME" = "profitbricks-build12-i386" ] ; then
			# we dont vary the kernel on i386 atm, see #875990 + #876035
			sudo apt install linux-image-amd64:amd64
		elif [ "$HOSTNAME" = "osuosl-build169-amd64" ] || [ "$HOSTNAME" = "osuosl-build170-amd64" ] ; then
			# Arch Linux builds latest stuff which sometimes (eg, currentlt Qt) needs newer kernel to build...
504
			sudo apt install linux-image-amd64/buster-backports || true # backport kernels are frequently uninstallable...
505
			:
506
		fi
Edward Betts's avatar
Edward Betts committed
507
		# don't (re-)install pbuilder if it's on hold
508
		if [ "$(dpkg-query -W -f='${db:Status-Abbrev}\n' pbuilder)" != "hi " ] ; then
509
			$UP2DATE || sudo apt-get install pbuilder
510
		fi
511
		# remove unattended-upgrades if it's installed
512
		if [ "$(dpkg-query -W -f='${db:Status-Abbrev}\n' unattended-upgrades 2>/dev/null || true)" = "ii "  ] ; then
513
			 sudo apt-get -y purge unattended-upgrades
514
		fi
515
		sudo apt-get clean
Holger Levsen's avatar
Holger Levsen committed
516
		explain "packages installed."
517
	else
Holger Levsen's avatar
Holger Levsen committed
518
		explain "no new packages to be installed."
519
	fi
520
fi
521

522
#
Holger Levsen's avatar
Holger Levsen committed
523
# deploy package configuration in /etc and /usr
524
#
Holger Levsen's avatar
Holger Levsen committed
525
cd $BASEDIR
526
527
528
529
530
531
532
533
534
for h in common common-amd64 common-i386 common-arm64 common-armhf "$HOSTNAME" ; do
	# $HOSTNAME has precedence over common-$DPKG_ARCH over common
	case $h in
		common-amd64) [ $DPKG_ARCH = "amd64" ] || continue ;;
		common-i386)  [ $DPKG_ARCH = "i386" ] || continue ;;
		common-arm64) [ $DPKG_ARCH = "arm64" ] || continue ;;
		common-armhf) [ $DPKG_ARCH = "armhf" ] || continue ;;
		*) ;;
	esac
535
536
	if [ -d "hosts/$h/etc/sudoers.d/" ]; then
		for f in "hosts/$h/etc/sudoers.d/"* ; do
537
			/usr/sbin/visudo -c -f "$f" > /dev/null
538
539
540
		done
	fi
	for d in etc usr ; do
Mattia Rizzolo's avatar
Mattia Rizzolo committed
541
		if [ -d "hosts/$h/$d" ]; then
542
			sudo cp --preserve=mode,timestamps -r "hosts/$h/$d/"* "/$d"
543
544
545
		fi
	done
done
546
547
# we ship one or two service files…
sudo systemctl daemon-reload &
548
549
550
551
552

#
# more configuration than a simple cp can do
#
sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins
553
sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm
Holger Levsen's avatar
Holger Levsen committed
554
[ -f /etc/mailname ] || ( echo $HOSTNAME.debian.net | sudo tee /etc/mailname )
555

556
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "profitbricks-build7-amd64" ] ; then
557
	if ! $UP2DATE || [ $BASEDIR/hosts/$HOSTNAME/etc/apache2 -nt $STAMP ]  ; then
558
559
560
561
562
563
564
565
566
		if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
			sudo a2enmod proxy
			sudo a2enmod proxy_http
			sudo a2enmod rewrite
			sudo a2enmod ssl
			sudo a2enmod headers
			sudo a2enmod macro
			sudo a2enmod filter
		fi
567
568
569
		if [ "$HOSTNAME" = "jenkins" ] ; then
			sudo a2ensite -q jenkins.debian.net
			sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.debian.net.conf
570
			sudo a2enconf -q munin
571
572
573
574
		else # "$HOSTNAME" = "profitbricks-build7-amd64"
			sudo a2ensite -q buildinfos.debian.net
			sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/buildinfos.debian.net.conf
		fi
575
576
577
		# for reproducible.d.n url rewriting:
		[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
		sudo service apache2 reload
578
579
580
	fi
fi

581
if ! $UP2DATE || [ $BASEDIR/hosts/$HOSTNAME/etc/munin -nt $STAMP ] ; then
582
	cd /etc/munin/plugins
583
	sudo rm -f postfix_* open_inodes interrupts irqstats threads proc_pri vmstat if_err_* exim_* netstat fw_forwarded_local fw_packets forks open_files users nfs* iostat_ios ntp* df_abs 2>/dev/null
584
	case $HOSTNAME in
585
			profitbricks-build1-a*|profitbricks-build10*|codethink-sled16*|osuosl-build167*) [ -L /etc/munin/plugins/squid_cache ] || for i in squid_cache squid_objectsize squid_requests squid_traffic ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
586
587
			*)	;;
	esac
Holger Levsen's avatar
Holger Levsen committed
588
589
590
591
	case $HOSTNAME in
			jenkins) [ -L /etc/munin/plugins/postfix_mailstats ] || for i in postfix_mailstats postfix_mailvolume postfix_mailqueue ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
			*)	;;
	esac
592
593
594
	if [ "$HOSTNAME" != "jenkins" ] && [ -L /etc/munin/plugins/iostat ] ; then
		sudo rm /etc/munin/plugins/iostat
	fi
595
	if ( [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "profitbricks-build7-amd64" ] ) && [ ! -L /etc/munin/plugins/apache_accesses ] ; then
596
		for i in apache_accesses apache_volume ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done
597
		sudo ln -s /usr/share/munin/plugins/loggrep jenkins_oom
598
	fi
599
600
	# this is a hack to work around (rare) problems with restarting munin-node...
	sudo service munin-node restart || sudo service munin-node restart || sudo service munin-node restart
601
fi
602
603
604
605

# add some users to groups after packages have been installed
if ! $UP2DATE ; then
	case $HOSTNAME in
606
		jenkins|osuosl-build169-amd64|osuosl-build170-amd64)
607
608
609
			sudo addgroup --system --gid 300 abuild
			sudo adduser jenkins abuild
			;;
610
611
612
613
614
		osuosl-build173-amd64)		sudo adduser jenkins sbuild ;;
		*) 				;;
	esac
fi
# finally
Holger Levsen's avatar
Holger Levsen committed
615
explain "packages configured."
Holger Levsen's avatar
Holger Levsen committed
616
617
618
619

#
# install the heart of jenkins.debian.net
#
Holger Levsen's avatar
Holger Levsen committed
620
cd $BASEDIR
621
[ -d /srv/jenkins/features ] && sudo rm -rf /srv/jenkins/features
622
# check for bash syntax *before* actually deploying anything
623
shopt -s nullglob
624
for f in bin/*.sh bin/**/*.sh ; do bash -n "$f" ; done
625
shopt -u nullglob
626
for dir in bin logparse mustache-templates ; do
627
628
	sudo mkdir -p /srv/jenkins/$dir
	sudo rsync -rpt --delete $dir/ /srv/jenkins/$dir/
Holger Levsen's avatar
Holger Levsen committed
629
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
630
done
631
632
HOST_JOBS="hosts/$HOSTNAME/job-cfg"
if [ -e "$HOST_JOBS" ] ; then
633
	sudo -u jenkins-adm rsync -rpt --copy-links --delete "$HOST_JOBS/" /srv/jenkins/job-cfg/
634
else
Philip Hands's avatar
Philip Hands committed
635
636
	# tidying up ... assuming that we don't want clutter on peripheral servers
	[ -d /srv/jenkins/job-cfg ] && sudo rm -rf /srv/jenkins/job-cfg
637
638
fi

639

640
sudo mkdir -p -m 700 /var/lib/jenkins/.ssh
641
sudo chown jenkins.jenkins /var/lib/jenkins/.ssh
642
if [ "$HOSTNAME" = "jenkins" ] ; then
643
644
	sudo -u jenkins install -m 600 jenkins-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
	sudo -u jenkins cp jenkins-home/procmailrc /var/lib/jenkins/.procmailrc
645
	sudo -u jenkins cp jenkins-home/offline_nodes /var/lib/jenkins/offline_nodes
646
647
648
else
	sudo cp jenkins-nodes-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
fi
649
if [ -f jenkins-nodes-home/authorized_keys.$HOSTNAME ] ; then
650
	cat jenkins-nodes-home/authorized_keys.$HOSTNAME | sudo tee -a /var/lib/jenkins/.ssh/authorized_keys
651
fi
652
sudo -u jenkins cp jenkins-home/gitconfig /var/lib/jenkins/.gitconfig
653
654
sudo -u jenkins cp jenkins-home/ssh_config.in /var/lib/jenkins/.ssh/config
nodes/gen_ssh_config | sudo -u jenkins tee -a /var/lib/jenkins/.ssh/config > /dev/null
655
nodes/gen_known_host_file | sudo tee /etc/ssh/ssh_known_hosts > /dev/null
Holger Levsen's avatar
Holger Levsen committed
656
explain "scripts and configurations for jenkins updated."
657

658
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
659
	sudo cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
660
	TMPFILE=$(mktemp)
661
	git log | grep ^Author| cut -d " " -f2-|sort -u -f > $TMPFILE
662
	echo "----" >> $TMPFILE
663
	sudo tee /var/lib/jenkins/userContent/THANKS > /dev/null < THANKS.head
664
665
	# samuel, lunar, jelle, josch and phil committed with several committers, only display one
	grep -v -e "samuel.thibault@ens-lyon.org" -e Lunar -e "j.schauer@email.de" -e "mattia@mapreri.org" -e "phil@jenkins-test-vm" -e "jelle@vdwaa.nl" $TMPFILE | sudo tee -a /var/lib/jenkins/userContent/THANKS > /dev/null
666
	rm $TMPFILE
667
	TMPDIR=$(mktemp -d -t update-jdn-XXXXXXXX)
668
	sudo cp -pr userContent $TMPDIR/
Holger Levsen's avatar
Holger Levsen committed
669
670
671
	sudo chown -R jenkins.jenkins $TMPDIR
	sudo cp -pr $TMPDIR/userContent  /var/lib/jenkins/
	sudo rm -r $TMPDIR > /dev/null
672
673
674
675
676
677
678
679
680
	cd /var/lib/jenkins/userContent/
	ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
	[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
	[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
	[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
	[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
	diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
	mv THANKS .THANKS
	rm TODO README INSTALL CONTRIBUTING
681
	sudo chown jenkins.jenkins /var/lib/jenkins/userContent/*html
Holger Levsen's avatar
Holger Levsen committed
682
	explain "user content for jenkins updated."
683
fi
684

685
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
686
687
688
689
690
691
	#
	# run jenkins-job-builder to update jobs if needed
	#     (using sudo because /etc/jenkins_jobs is root:root 700)
	#
	cd /srv/jenkins/job-cfg
	for metaconfig in *.yaml.py ; do
Holger Levsen's avatar
Holger Levsen committed
692
693
694
695
696
697
		if [ -f $metaconfig ] ; then
			TMPFILE=$(sudo -u jenkins-adm mktemp)
			./$metaconfig | sudo -u jenkins-adm tee "$TMPFILE" >/dev/null
			if ! sudo -u jenkins-adm cmp -s ${metaconfig%.py} "$TMPFILE" ; then
				sudo -u jenkins-adm mv "$TMPFILE" "${metaconfig%.py}"
			fi
698
699
700
		fi
	done
	for config in *.yaml ; do
701
702
703
704
705
706
707
		# do update, if
		# no stamp file exist or
		# no .py file exists and config is newer than stamp or
		# a .py file exists and .py file is newer than stamp
		if [ ! -f $STAMP ] || \
		 ( [ ! -f $config.py ] && [ $config -nt $STAMP ] ) || \
		 ( [ -f $config.py ] && [ $config.py -nt $STAMP ] ) ; then
708
			echo "$config has changed, executing updates."
709
			$JJB update $config
710
711
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
712
	explain "jenkins jobs updated."
713
fi
714

715
716
717
#
# generate the kgb-client configurations
#
718
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "osuosl-build168-amd64" ] || [ "$HOSTNAME" = "osuosl-build171-amd64" ] || [ "$HOSTNAME" = "osuosl-build172-amd64" ] || [ "$HOSTNAME" = "profitbricks-build2-i386" ] || [ "$HOSTNAME" = "profitbricks-build12-i386" ] ; then
719
720
721
	cd $BASEDIR
	KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
	if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
Philip Hands's avatar
Philip Hands committed
722
		# the last condition is to assure the files are owned by the right user/team
723
		if [ "$KGB_SECRETS" -nt $STAMP ] || [ "deploy_kgb.py" -nt "$STAMP" ] || [ ! -f $STAMP ] ; then
Philip Hands's avatar
Philip Hands committed
724
725
726
727
			sudo -u jenkins-adm "./deploy_kgb.py"
		else
			explain "kgb-client configuration unchanged, nothing to do."
		fi
728
	else
Philip Hands's avatar
Philip Hands committed
729
730
731
		figlet -f banner Warning
		echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
		echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
Holger Levsen's avatar
Holger Levsen committed
732
733
		echo "/srv/jenkins/kgb should be mode 755 and owned by jenkins-adm:root."
		echo "/srv/jenkins/kgb/client-status should be mode 755 and owned by jenkins:jenkins."
734
	fi
735
	KGB_STATUS="/srv/jenkins/kgb/client-status"
736
737
	sudo mkdir -p $KGB_STATUS
	sudo chown jenkins:jenkins $KGB_STATUS
738
739
fi

740
741
742
#
# Create GPG key for jenkins user if they do not already exist (eg. to sign .buildinfo files)
#
743
if sudo -H -u jenkins gpg --with-colons --fixed-list-mode --list-secret-keys | cut -d: -f1 | grep -qsFx 'sec' >/dev/null 2>&1 ; then
Holger Levsen's avatar
Holger Levsen committed
744
	: # not generating GPG key as one already exists for jenkins user
745
else
Holger Levsen's avatar
Holger Levsen committed
746
	explain "$(date) - Generating GPG key for jenkins user."
747

748
	sudo -H -u jenkins gpg --no-tty --batch --gen-key <<EOF
749
750
751
Key-Type: RSA
Key-Length: 4096
Key-Usage: sign
752
Name-Real: $HOSTNAME
753
754
755
756
757
758
Name-Comment: Automatically generated key for signing .buildinfo files
Expire-Date: 0
%no-ask-passphrase
%no-protection
%commit
EOF
759

760
	GPG_KEY_ID="$(sudo -H -u jenkins gpg --with-colons --fixed-list-mode --list-secret-keys | grep '^sec' | cut -d: -f5 | tail -n1)"
761
762
763
764
765
766

	if [ "$GPG_KEY_ID" = "" ]
	then
		explain "$(date) - Generated GPG key but could not parse key ID"
	else
		explain "$(date) - Generated GPG key $GPG_KEY_ID - submitting to keyserver"
767
		sudo -H -u jenkins gpg --send-keys $GPG_KEY_ID
768
	fi
769
770
fi

Holger Levsen's avatar
Holger Levsen committed
771
772
773
774
775
#
# There's always some work left...
#	echo FIXME is ignored so check-jobs scripts can output templates requiring manual work
#
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
776
	TMPFILE=$(mktemp)
777
	rgrep FI[X]ME $BASEDIR/* | grep -v $BASEDIR/TODO | grep -v echo > $TMPFILE || true
Holger Levsen's avatar
Holger Levsen committed
778
779
	if [ -s $TMPFILE ] ; then
		echo
780
		cat $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
781
782
		echo
	fi
783
	rm -f $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
784
785
fi

786
#
787
# almost finally…
788
#
789
sudo touch $STAMP	# so on the next run, only configs newer than this file will be updated
Holger Levsen's avatar
Holger Levsen committed
790
explain "$(date) - finished deployment."
791

792
793
794
795
#
# some final checks only for the jenkins
#
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
796
	jenkins_bugs_check
Holger Levsen's avatar
Holger Levsen committed
797
	explain "$(date) - done checking for known jenkins bugs."
798
799
fi

Holger Levsen's avatar
Holger Levsen committed
800
#
801
# finally!
Holger Levsen's avatar
Holger Levsen committed
802
#
803
case $HOSTNAME in
804
	# set time back to the future
805
	profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386)
806
		disable_dsa_check_packages
807
808
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
809
	codethink-sled9*|codethink-sled11*|codethink-sled13*|codethink-sled15*)
810
		disable_dsa_check_packages
811
812
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
813
	osuosl-build170-amd64|osuosl-build172-amd64)
814
815
816
		disable_dsa_check_packages
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
817
	jenkins)
818
		# notify irc on updates of jenkins.d.n
819
820
		MESSAGE="jenkins.d.n updated to $(cd $BASEDIR ; git describe --always)."
		kgb-client --conf /srv/jenkins/kgb/debian-qa.conf --relay-msg "$MESSAGE"
Holger Levsen's avatar
Holger Levsen committed
821
		;;
822
823
	*)	;;
esac
824
825
826
827

echo
figlet ok
echo
Holger Levsen's avatar
Holger Levsen committed
828
829
echo "__$HOSTNAME=ok__"