update_jdn.sh 27.6 KB
Newer Older
1
#!/bin/bash
2
# vim: set noexpandtab:
3
# Copyright 2012-2020 Holger Levsen <holger@layer-acht.org>
4
#         ©      2018 Mattia Rizzolo <mattia@debian.org>
Holger Levsen's avatar
Holger Levsen committed
5
# released under the GPLv2
6

Holger Levsen's avatar
Holger Levsen committed
7
# puppet / salt / ansible / fai / chef / deployme.app - disclaimer
Holger Levsen's avatar
Holger Levsen committed
8
9
10
11
12
13
14
15
16
17
18
# (IOW: this script has been grown in almost 500 commits and it shows…)
#
# yes, we know… and: "it" should probably still be done.
#
# It just unclear, how/what, and what we have actually mostly works.
#
# Switching to jenkins.debian.org is probably an opportunity
# to write (refactor this into) *yet another deployment script*
# (interacting with the DSA machine setup which is in puppet…),
# thus obsoleting this script gradually, though this is used on
# 47 hosts currently (of which quite some were initially installed
Edward Betts's avatar
Edward Betts committed
19
# manually…)
Holger Levsen's avatar
Holger Levsen committed
20
21
22
#
# so, yes, patches welcome. saying this is crap alone is not helpful,
# nor is just suggesting some new or old technology. patches most welcome!
23
24
#
# that said, there's a new one: init_node ;)
Holger Levsen's avatar
Holger Levsen committed
25

26
27
set -e

Holger Levsen's avatar
Holger Levsen committed
28
# prevent failure on first run...
Holger Levsen's avatar
Holger Levsen committed
29
[ ! -f /srv/jenkins/bin/common-functions.sh ] || . /srv/jenkins/bin/common-functions.sh
Holger Levsen's avatar
Holger Levsen committed
30

31
BASEDIR="$(dirname "$(readlink -e $0)")"
32
STAMP=/var/log/jenkins/update-jenkins.stamp
33
34
# The $@ below means that command line args get passed on to j-j-b
# which allows one to specify --flush-cache or --ignore-cache
35
JJB="jenkins-jobs $@"
36
DPKG_ARCH="$(dpkg --print-architecture)"
Holger Levsen's avatar
Holger Levsen committed
37

38
# so we can later run some commands only if $0 has been updated…
Philip Hands's avatar
Philip Hands committed
39
if [ -f $STAMP ] && [ $STAMP -nt $BASEDIR/$0 ] ; then
Holger Levsen's avatar
Holger Levsen committed
40
	UP2DATE=true
Holger Levsen's avatar
Holger Levsen committed
41
	echo $HOSTNAME is up2date.
Philip Hands's avatar
Philip Hands committed
42
43
else
	UP2DATE=false
Holger Levsen's avatar
Holger Levsen committed
44
	echo $HOSTNAME needs to be updated.
45
46
47
fi


Holger Levsen's avatar
Holger Levsen committed
48
explain() {
Holger Levsen's avatar
Holger Levsen committed
49
	echo "$HOSTNAME: $1"
Holger Levsen's avatar
Holger Levsen committed
50
51
}

52
53
54
set_correct_date() {
		# set correct date
		sudo service ntp stop || true
55
		sudo ntpdate -b $1
56
}
57

58
disable_dsa_check_packages() {
59
	# disable check for outdated packages as someday in the future
60
	# packages from security.d.o will appear outdated always…
61
62
	echo -e "#!/bin/sh\n# disabled dsa-check by update_jdn.sh\nexit 0" | sudo tee /usr/local/bin/dsa-check-packages
	sudo chmod a+rx /usr/local/bin/dsa-check-packages
63
64
}

65
echo "--------------------------------------------"
Holger Levsen's avatar
Holger Levsen committed
66
explain "$(date) - begin deployment update."
67

68
69
70
71
#
# temporarily test to check which hosts don't use systemd
#
if [ -z "$(dpkg -l|grep systemd-sysv||true)" ] ; then 
72
	echo "no systemd-sysv installed on $(hostname), please enter to continue…"
73
74
75
	read
fi

76
# some nodes need special treatment…
77
case $HOSTNAME in
78
	profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386)
Holger Levsen's avatar
Holger Levsen committed
79
		# set correct date
80
		set_correct_date de.pool.ntp.org
81
		;;
82
	codethink-sled9*|codethink-sled11*|codethink-sled13*|codethink-sled15*)
83
		# set correct date
84
		set_correct_date de.pool.ntp.org
Holger Levsen's avatar
Holger Levsen committed
85
		;;
86
	osuosl-build170-amd64|osuosl-build172-amd64)
87
		# set correct date
88
		set_correct_date time.osuosl.org
89
		;;
90
91
92
	*)	;;
esac

93
94
95
96
97
98
99
100
101
102
103
# ubuntu decided to change kernel perms in the middle of LTS…
case $HOSTNAME in
	codethink-sled*)
		# fixup perms
		sudo chmod +r /boot/vmlinuz-*
		;;
	*)	;;
esac



104
105
106
#
# set up users and groups
#
107
declare -A user_host_groups u_shell
108
sudo_groups='jenkins,jenkins-adm,sudo,adm'
109
110
111

# if there's a need for host groups, a case statement on $HOSTNAME here that sets $GROUPNAME, say, should do the trick
# then you can define user_host_groups['phil','lvm_group']=... below
112
# and add checks for the GROUP version where ever the HOSTNAME is checked in the following code
113
114
115
116
117

user_host_groups['helmut','*']="$sudo_groups"
user_host_groups['holger','*']="$sudo_groups"
user_host_groups['holger','jenkins']="reproducible,${user_host_groups['holger','*']}"
user_host_groups['mattia','*']="$sudo_groups"
118
user_host_groups['mattia','jenkins']="reproducible,${user_host_groups['mattia','*']}"
119
user_host_groups['phil','jenkins-test-vm']="$sudo_groups,libvirt,libvirt-qemu"
120
user_host_groups['phil','jenkins']="$sudo_groups"
121
user_host_groups['lunar','jenkins']='reproducible'
122
123
user_host_groups['lynxis','osuosl-build171-amd64']="$sudo_groups"
user_host_groups['lynxis','osuosl-build172-amd64']="$sudo_groups"
124
user_host_groups['lynxis','jenkins']="jenkins"
125
126
user_host_groups['kpcyrd','osuosl-build169-amd64']="$sudo_groups"
user_host_groups['kpcyrd','osuosl-build170-amd64']="$sudo_groups"
127
user_host_groups['kpcyrd','jenkins']="jenkins"
128
user_host_groups['bubu','osuosl-build168-amd64']="$sudo_groups"
129
user_host_groups['hans','osuosl-build168-amd64']="$sudo_groups"
Mattia Rizzolo's avatar
Mattia Rizzolo committed
130
user_host_groups['vagrant','*']="$sudo_groups"
131
user_host_groups['bernhard','osuosl-build174-amd64']="$sudo_groups"
132
user_host_groups['santiago','profitbricks-build3-amd64']="$sudo_groups"
133
134

u_shell['mattia']='/bin/zsh'
135
u_shell['lynxis']='/usr/bin/fish'
136

137
138
139
# get the users out of the user_host_groups array's index
users=$(for i in ${!user_host_groups[@]}; do echo ${i%,*} ; done | sort -u)

140
141
( $UP2DATE && [ -z "$(find $BASEDIR/authorized_keys -newer $0)" ] ) || for user in ${users}; do
	cd $BASEDIR
142
	# -v is a bashism to check for set variables, used here to see if this user is active on this host
143
	if [ ! -v user_host_groups["$user","$HOSTNAME"] ] && [ ! -v user_host_groups["$user",'*'] ] && [ ! -v user_host_groups["$user","$DPKG_ARCH"] ] ; then
Holger Levsen's avatar
Holger Levsen committed
144
145
		continue
	fi
146

147
148
149
150
151
152
	# create the user
	if ! getent passwd $user > /dev/null ; then
		# adduser, defaulting to /bin/bash as shell
		sudo adduser --gecos "" --shell "${u_shell[$user]:-/bin/bash}" --disabled-password $user
	fi
	# add groups: first try the specific host, or if unset fall-back to default '*' setting
153
	for h in "$HOSTNAME" "$DPKG_ARCH" '*' ; do
154
		if [ -v user_host_groups["$user","$h"] ] ; then
Holger Levsen's avatar
Holger Levsen committed
155
			sudo usermod -G "${user_host_groups["$user","$h"]}" $user
156
			break
157
		fi
158
159
160
161
162
	done
	# add the user's keys (if any)
	if ls authorized_keys/${user}@*.pub >/dev/null 2>&1 ; then
		[ -d /var/lib/misc/userkeys ] || sudo mkdir -p /var/lib/misc/userkeys
		cat authorized_keys/${user}@*.pub | sudo tee /var/lib/misc/userkeys/${user} > /dev/null
163
	fi
164
165
done

166
167
# configure sbuild:
case $HOSTNAME in
Jelmer Vernooij's avatar
Jelmer Vernooij committed
168
	profitbricks-build7-a*|osuosl-build167*) sudo sbuild-adduser jenkins ;;
169
170
171
	*) ;;
esac

172
173
174
175
176
sudo mkdir -p /srv/workspace
[ -d /srv/schroots ] || sudo mkdir -p /srv/schroots
[ -h /chroots ] || sudo ln -s /srv/workspace/chroots /chroots
[ -h /schroots ] || sudo ln -s /srv/schroots /schroots

177
178
# prepare tmpfs on some hosts
case $HOSTNAME in
Mattia Rizzolo's avatar
Mattia Rizzolo committed
179
180
181
182
	jenkins)
		TMPFSSIZE=100
		TMPSIZE=15
		;;
Holger Levsen's avatar
Holger Levsen committed
183
	profitbricks-build9-amd64)
184
		TMPFSSIZE=40
Mattia Rizzolo's avatar
Mattia Rizzolo committed
185
186
		TMPSIZE=8
		;;
Holger Levsen's avatar
Holger Levsen committed
187
	profitbricks-build*)
Mattia Rizzolo's avatar
Mattia Rizzolo committed
188
189
190
		TMPFSSIZE=200
		TMPSIZE=15
		;;
191
192
193
194
	codethink*)
		TMPFSSIZE=100
		TMPSIZE=15
		;;
195
196
197
198
	osuosl*)
		TMPFSSIZE=400
		TMPSIZE=50
		;;
199
200
201
	*) ;;
esac
case $HOSTNAME in
202
203
204
205
206
207
	profitbricks-build*i386)
		if ! grep -q '/srv/workspace' /etc/fstab; then
			echo "Warning: you need to manually create a /srv/workspace partition on i386 nodes, exiting."
			exit 1
		fi
		;;
208
	jenkins|profitbricks-build*amd64|codethink*|osuosl*)
209
		if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
210
			echo "tmpfs		/srv/workspace	tmpfs	defaults,size=${TMPFSSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null  
211
		fi
Mattia Rizzolo's avatar
Mattia Rizzolo committed
212
213
214
		if ! grep -q '^tmpfs\s\+/tmp\s' /etc/fstab; then
			echo "tmpfs		/tmp	tmpfs	defaults,size=${TMPSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null
		fi
215
216
		if ! mountpoint -q /srv/workspace; then
			if test -z "$(ls -A /srv/workspace)"; then
217
				sudo mount /srv/workspace
218
			else
219
				explain "WARNING: mountpoint /srv/workspace is non-empty."
220
221
222
223
224
			fi
		fi
		;;
	*) ;;
esac
225

226
# make sure needed directories exists - some directories will not be needed on all hosts...
227
for directory in /schroots /srv/reproducible-results /srv/d-i /srv/udebs /var/log/jenkins/ /srv/jenkins /srv/jenkins/pseudo-hosts /srv/workspace/chroots ; do
228
229
230
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
	fi
Holger Levsen's avatar
Holger Levsen committed
231
	sudo chown jenkins.jenkins $directory
232
done
Holger Levsen's avatar
Holger Levsen committed
233
for directory in /srv/jenkins ; do
234
235
236
237
238
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
		sudo chown jenkins-adm.jenkins-adm $directory
	fi
done
239

240
if ! test -h /chroots; then
Holger Levsen's avatar
Holger Levsen committed
241
	sudo rmdir /chroots || sudo rm -f /chroots # do not recurse
242
	if test -e /chroots; then
Holger Levsen's avatar
Holger Levsen committed
243
		explain "/chroots could not be cleared."
244
	else
245
		sudo ln -s /srv/workspace/chroots /chroots
246
247
248
	fi
fi

249
250
251
252
253
# only on Debian systems
if [ -f /etc/debian_version ] ; then
	#
	# install packages we need
	#
254
	if [ $BASEDIR/$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then
255
		DEBS="
Holger Levsen's avatar
Holger Levsen committed
256
			bash-completion 
257
258
			bc
			bsd-mailx
259
260
			curl
			debian-archive-keyring
261
			cdebootstrap-
262
			debootstrap
263
			devscripts
264
			eatmydata
265
			etckeeper
266
			figlet
Holger Levsen's avatar
Holger Levsen committed
267
			git
Holger Levsen's avatar
Holger Levsen committed
268
			gnupg
269
			haveged
Holger Levsen's avatar
Holger Levsen committed
270
			htop
271
			less
272
			locales-all
Mattia Rizzolo's avatar
Mattia Rizzolo committed
273
			lsof
274
275
			molly-guard
			moreutils
Holger Levsen's avatar
Holger Levsen committed
276
			mosh
277
278
279
			munin-node
			munin-plugins-core
			munin-plugins-extra
Holger Levsen's avatar
Holger Levsen committed
280
			needrestart
281
			netcat-traditional
282
283
			ntp
			ntpdate
284
			pbuilder
285
			pigz 
286
			postfix
287
			procmail
288
			psmisc
289
			python3-psycopg2 
290
			python3-yaml
Holger Levsen's avatar
Holger Levsen committed
291
			schroot 
292
293
			screen
			slay
294
			stunnel
Holger Levsen's avatar
Holger Levsen committed
295
296
297
			subversion 
			subversion-tools 
			sudo 
298
			systemd-sysv
299
			tmux
Holger Levsen's avatar
Holger Levsen committed
300
301
			unzip 
			vim 
302
			zsh
303
			"
304
		case $HOSTNAME in
305
306
307
			codethink*)
				# install debootstrap from stretch-backports on ubuntu nodes as since 20180927 debootstrap 1.0.78+nmu1ubuntu1.6 cannot install sid anymore
				DEBS="$DEBS
308
309
310
311
312
313
				debootstrap/stretch-backports
				munin-node/stretch-backports
				munin-plugins-core/stretch-backports
				munin-plugins-extra/stretch-backports
				pbuilder/stretch-backports
				" ;;
314
			*)	# not available in Ubuntu 16.04:
315
316
317
318
319
				DEBS="$DEBS
				ripgrep
				mmdebstrap
				devscripts/buster-backports
				" ;;
320
		esac
321
		case $HOSTNAME in
322
			# needed for rebuilding Debian (using .buildinfo files)
323
			osuosl-build173-amd64) DEBS="$DEBS libdpkg-perl libwww-mechanize-perl sbuild" ;;
324
325
			# needed for opensuse openbuildservice
			osuosl-build174-amd64) DEBS="$DEBS osc obs-build qemu-kvm libjson-perl libxml-simple-perl bsdtar" ;;
326
327
			*) ;;
		esac
328
329
		# install stuff for bremner's builtin-pho stuff
		# and building html pages there too
330
		# and for the rebuilder 'thing' too
331
332
333
334
		case $HOSTNAME in
			profitbricks-build7-a*) DEBS="$DEBS
				postgresql
				postgresql-11-debversion
335
				python3-pystache
336
				python3-apt
337
338
				python3-psycopg2
				sbuild" ;;
339
340
			*) ;;
		esac
341
		# install squid / apache2 on a few nodes only
342
		case $HOSTNAME in
343
			profitbricks-build1-a*|profitbricks-build10*|codethink-sled16*|osuosl-build*) DEBS="$DEBS
344
				squid" ;;
345
346
			profitbricks-build7-a*) DEBS="$DEBS
				apache2" ;;
347
348
			*) ;;
		esac
Jelmer Vernooij's avatar
Jelmer Vernooij committed
349
350
351
		# install lintian-brush dependencies for debian-janitor
		case $HOSTNAME in
			osuosl-build167*) DEBS="$DEBS
352
				cython3
Jelmer Vernooij's avatar
Jelmer Vernooij committed
353
354
				devscripts
				python3-aiohttp
355
				python3-all-dev
356
				python3-bs4
357
358
				python3-certifi
				python3-configobj
Jelmer Vernooij's avatar
Jelmer Vernooij committed
359
360
361
				python3-debian
				python3-dulwich
				python3-distro-info
362
				python3-fastimport
Jelmer Vernooij's avatar
Jelmer Vernooij committed
363
				python3-iniparse
364
				python3-launchpadlib
Jelmer Vernooij's avatar
Jelmer Vernooij committed
365
				python3-levenshtein
366
				python3-paramiko
367
				python3-patiencediff
Jelmer Vernooij's avatar
Jelmer Vernooij committed
368
				python3-pkginfo
369
				python3-pyinotify
Jelmer Vernooij's avatar
Jelmer Vernooij committed
370
				python3-ruamel.yaml
371
372
373
374
				python3-setuptools
				python3-six
				python3-subunit
				python3-testtools
375
				python3-toml
Holger Levsen's avatar
Holger Levsen committed
376
				python3-urllib3
377
				python3-yaml
Holger Levsen's avatar
Holger Levsen committed
378
				sbuild
Jelmer Vernooij's avatar
Jelmer Vernooij committed
379
380
381
382
383
384
				gnome-pkg-tools
				postgresql-server-dev-all
				lintian
				dos2unix
				gpg
				libdebhelper-perl
385
				quilt
Jelmer Vernooij's avatar
Jelmer Vernooij committed
386
				" ;;
387
				# python3-asyncpg is not available on buster
Jelmer Vernooij's avatar
Jelmer Vernooij committed
388
		esac
389
390
391
		# notifications are only done from a view nodes
		case $HOSTNAME in
			jenkins|jenkins-test-vm|profitbricks-build*) DEBS="$DEBS
392
393
				kgb-client
				python3-yaml" ;;
394
395
			*) ;;
		esac
396
397
		# needed to run the 2nd reproducible builds nodes in the future...
		case $HOSTNAME in
398
			profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386) DEBS="$DEBS ntpdate" ;;
399
			codethink-sled9*|codethink-sled11*|codethink-sled13*|codethink-sled15*) DEBS="$DEBS ntpdate" ;;
400
			osuosl-build170-amd64|osuosl-build172-amd64) DEBS="$DEBS ntpdate" ;;
401
402
			*) ;;
		esac
403
		# needed to run coreboot/openwrt/netbsd/fedora jobs
404
		case $HOSTNAME in
405
		osuosl-build171-amd64|osuosl-build172-amd64) DEBS="$DEBS
Holger Levsen's avatar
Holger Levsen committed
406
				bison
407
				ca-certificates
Holger Levsen's avatar
Holger Levsen committed
408
409
410
				cmake
				diffutils
				findutils
411
				fish
Holger Levsen's avatar
Holger Levsen committed
412
413
414
415
				flex
				g++
				gawk
				gcc
416
				gnat
Holger Levsen's avatar
Holger Levsen committed
417
418
419
420
421
422
423
424
				grep
				libc6-dev
				libncurses5-dev
				libssl-dev
				locales-all
				kgb-client
				m4
				make
425
426
				python3-clint
				python3-git
427
				python3-pystache
428
				python3-requests
Holger Levsen's avatar
Holger Levsen committed
429
430
				python3-yaml
				subversion
431
				tree
Holger Levsen's avatar
Holger Levsen committed
432
433
				unzip
				util-linux
434
435
				zlib1g-dev
				"
436
437
438
			;;
			*) ;;
		esac
439
440
		# needed to run fdroid jobs
		case $HOSTNAME in
441
			osuosl-build168-amd64) DEBS="$DEBS
442
				androguard
443
				android-sdk
444
445
				bzr
				git-svn
446
				fdroidserver
447
				linux-headers-amd64
448
				mercurial
449
				python3-asn1crypto
450
				python3-babel
451
				python3-mwclient
452
				python3-setuptools
453
				subversion
454
				vagrant
455
				virtualbox virtualbox-dkms
456
				"
457
			;;
458
			*) ;;
459
		esac
460
461
462
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			# for phil only
			DEBS="$DEBS postfix-pcre"
463
464
			# only needed on the main node
		elif [ "$HOSTNAME" = "jenkins" ] ; then
465
			DEBS="$DEBS ffmpeg python3-popcon dose-extra"
466
467
468
469
		fi
		# mock is needed to build fedora
		if [ "$HOSTNAME" = "osuosl-build171-amd64" ] || [ "$HOSTNAME" = "osuosl-build172-amd64" ] || [ "$HOSTNAME" = "jenkins" ] ; then
			DEBS="$DEBS mock"
470
		fi
471
		# only on main node
472
		if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
Holger Levsen's avatar
Holger Levsen committed
473
474
475
476
			MASTERDEBS=" 
				apache2 
				apt-file 
				apt-listchanges 
477
				asciidoc
Holger Levsen's avatar
Holger Levsen committed
478
				binfmt-support 
479
480
				bison
				botch
Holger Levsen's avatar
Holger Levsen committed
481
482
483
484
485
486
487
				build-essential 
				cmake 
				cron-apt 
				csvtool 
				dnsmasq-base 
				dstat 
				figlet 
Holger Levsen's avatar
Holger Levsen committed
488
				flex
Holger Levsen's avatar
Holger Levsen committed
489
				gawk 
490
				ghc
491
				git-lfs
492
				git-notifier 
Holger Levsen's avatar
Holger Levsen committed
493
494
495
496
				gocr 
				graphviz 
				iasl 
				imagemagick 
497
498
				ip2host
				jekyll
499
				jenkins-job-builder
500
				jq
501
				kgb-client
Holger Levsen's avatar
Holger Levsen committed
502
503
504
505
506
507
508
509
510
				libcap2-bin 
				libfile-touch-perl 
				libguestfs-tools 
				libjson-rpc-perl 
				libsoap-lite-perl 
				libxslt1-dev 
				moreutils 
				mr 
				mtr-tiny 
511
				munin
Holger Levsen's avatar
Holger Levsen committed
512
				ntp 
513
				obfs4proxy
Holger Levsen's avatar
Holger Levsen committed
514
515
				openbios-ppc 
				openbios-sparc 
516
				openjdk-11-jre-headless
517
				pandoc
518
				postgresql
519
				postgresql-autodoc
520
				postgresql-client 
Holger Levsen's avatar
Holger Levsen committed
521
522
523
				poxml 
				procmail 
				python3-debian 
524
				python3-pystache
525
				python3-requests
526
				python3-rpy2 
527
528
529
				python3-sqlalchemy
				python3-xdg
				python3-yaml
Holger Levsen's avatar
Holger Levsen committed
530
531
532
				python-arpy 
				python-hachoir-metadata 
				python-lzma 
533
				python-pil
Holger Levsen's avatar
Holger Levsen committed
534
535
536
537
538
539
540
541
				python-pip 
				python-setuptools 
				python-twisted 
				python-yaml 
				qemu 
				qemu-kvm 
				qemu-system-x86 
				qemu-user-static 
542
543
				radvd
				ruby-jekyll-redirect-from
544
545
				ruby-rspec
				rustc
Holger Levsen's avatar
Holger Levsen committed
546
547
548
549
				seabios 
				shorewall 
				shorewall6 
				sqlite3 
550
				syslinux
551
				systemd
552
				thin-provisioning-tools
553
				tor
Holger Levsen's avatar
Holger Levsen committed
554
				vncsnapshot 
555
556
				vnstat
				whohas
Holger Levsen's avatar
Holger Levsen committed
557
				x11-apps 
558
				xtightvncviewer
559
				xvfb
560
				xvkbd
561
562
				zutils
				"
563
564
565
		else
			MASTERDEBS=""
		fi
566
		$UP2DATE || sudo apt-get update
567
		$UP2DATE || sudo apt-get install $DEBS $MASTERDEBS
568
569
570
571
572
573
		# for varying kernels:
		# - we use bpo kernels on pb-build5+15 (and the default amd64 kernel on pb-build6+16-i386)
		# - we also use the bpo kernel on osuosl-build172 (but not osuosl-build171)
		# - this is done as a seperate step as bpo kernels are frequently uninstallable when upgraded on bpo
		if [ "$HOSTNAME" = "profitbricks-build5-amd64" ] || [ "$HOSTNAME" = "profitbricks-build15-amd64" ] \
			|| [ "$HOSTNAME" = "osuosl-build172-amd64" ] ; then
574
			sudo apt install linux-image-amd64/buster-backports || true # backport kernels are frequently uninstallable...
575
			:
576
577
578
579
		elif [ "$HOSTNAME" = "profitbricks-build6-i386" ] || [ "$HOSTNAME" = "profitbricks-build16-i386" ] \
			|| [ "$HOSTNAME" = "profitbricks-build2-i386" ] || [ "$HOSTNAME" = "profitbricks-build12-i386" ] ; then
			# we dont vary the kernel on i386 atm, see #875990 + #876035
			sudo apt install linux-image-amd64:amd64
580
		elif [ "$HOSTNAME" = "osuosl-build169-amd64" ] || [ "$HOSTNAME" = "osuosl-build170-amd64" ] || [ "$HOSTNAME" = "osuosl-build173-amd64" ]; then
581
			# Arch Linux builds latest stuff which sometimes (eg, currentlt Qt) needs newer kernel to build...
582
			sudo apt install linux-image-amd64/buster-backports || true # backport kernels are frequently uninstallable...
583
			:
584
		fi
Edward Betts's avatar
Edward Betts committed
585
		# don't (re-)install pbuilder if it's on hold
586
		if [ "$(dpkg-query -W -f='${db:Status-Abbrev}\n' pbuilder)" != "hi " ] ; then
587
			$UP2DATE || sudo apt-get install pbuilder
588
		fi
589
		# remove unattended-upgrades if it's installed
590
		if [ "$(dpkg-query -W -f='${db:Status-Abbrev}\n' unattended-upgrades 2>/dev/null || true)" = "ii "  ] ; then
591
			 sudo apt-get -y purge unattended-upgrades
592
		fi
593
		sudo apt-get clean
Holger Levsen's avatar
Holger Levsen committed
594
		explain "packages installed."
595
	else
Holger Levsen's avatar
Holger Levsen committed
596
		explain "no new packages to be installed."
597
	fi
598
fi
599

Holger Levsen's avatar
Holger Levsen committed
600
601
602
# harmless but has a service which can fail and thus can make maintenance jobs unstable
sudo apt-get remove munin-async

603
#
Holger Levsen's avatar
Holger Levsen committed
604
# deploy package configuration in /etc and /usr
605
#
Holger Levsen's avatar
Holger Levsen committed
606
cd $BASEDIR
607
608
609
610
611
612
613
614
615
for h in common common-amd64 common-i386 common-arm64 common-armhf "$HOSTNAME" ; do
	# $HOSTNAME has precedence over common-$DPKG_ARCH over common
	case $h in
		common-amd64) [ $DPKG_ARCH = "amd64" ] || continue ;;
		common-i386)  [ $DPKG_ARCH = "i386" ] || continue ;;
		common-arm64) [ $DPKG_ARCH = "arm64" ] || continue ;;
		common-armhf) [ $DPKG_ARCH = "armhf" ] || continue ;;
		*) ;;
	esac
616
617
	if [ -d "hosts/$h/etc/sudoers.d/" ]; then
		for f in "hosts/$h/etc/sudoers.d/"* ; do
618
			/usr/sbin/visudo -c -f "$f" > /dev/null
619
620
621
		done
	fi
	for d in etc usr ; do
Mattia Rizzolo's avatar
Mattia Rizzolo committed
622
		if [ -d "hosts/$h/$d" ]; then
623
			sudo cp --preserve=mode,timestamps -r "hosts/$h/$d/"* "/$d"
624
625
626
		fi
	done
done
627
628
# we ship one or two service files…
sudo systemctl daemon-reload &
629
630
631
632
633

#
# more configuration than a simple cp can do
#
sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins
634
sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm
Holger Levsen's avatar
Holger Levsen committed
635
[ -f /etc/mailname ] || ( echo $HOSTNAME.debian.net | sudo tee /etc/mailname )
636

637
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "profitbricks-build7-amd64" ] ; then
638
	if ! $UP2DATE || [ $BASEDIR/hosts/$HOSTNAME/etc/apache2 -nt $STAMP ]  ; then
639
640
641
642
643
644
645
646
647
		if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
			sudo a2enmod proxy
			sudo a2enmod proxy_http
			sudo a2enmod rewrite
			sudo a2enmod ssl
			sudo a2enmod headers
			sudo a2enmod macro
			sudo a2enmod filter
		fi
648
649
650
		if [ "$HOSTNAME" = "jenkins" ] ; then
			sudo a2ensite -q jenkins.debian.net
			sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.debian.net.conf
651
			sudo a2enconf -q munin
652
653
654
655
		else # "$HOSTNAME" = "profitbricks-build7-amd64"
			sudo a2ensite -q buildinfos.debian.net
			sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/buildinfos.debian.net.conf
		fi
656
657
658
		# for reproducible.d.n url rewriting:
		[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
		sudo service apache2 reload
659
660
661
	fi
fi

662
if ! $UP2DATE || [ $BASEDIR/hosts/$HOSTNAME/etc/munin -nt $STAMP ] ; then
663
	cd /etc/munin/plugins
664
	sudo rm -f postfix_* open_inodes interrupts irqstats threads proc_pri vmstat if_err_* exim_* netstat fw_forwarded_local fw_packets forks open_files users nfs* iostat_ios ntp* df_abs 2>/dev/null
665
	case $HOSTNAME in
666
			profitbricks-build1-a*|profitbricks-build10*|codethink-sled16*|osuosl-build*) [ -L /etc/munin/plugins/squid_cache ] || for i in squid_cache squid_objectsize squid_requests squid_traffic ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
667
668
			*)	;;
	esac
Holger Levsen's avatar
Holger Levsen committed
669
670
671
672
	case $HOSTNAME in
			jenkins) [ -L /etc/munin/plugins/postfix_mailstats ] || for i in postfix_mailstats postfix_mailvolume postfix_mailqueue ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
			*)	;;
	esac
673
674
675
	if [ "$HOSTNAME" != "jenkins" ] && [ -L /etc/munin/plugins/iostat ] ; then
		sudo rm /etc/munin/plugins/iostat
	fi
676
	if ( [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "profitbricks-build7-amd64" ] ) && [ ! -L /etc/munin/plugins/apache_accesses ] ; then
677
		for i in apache_accesses apache_volume ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done
678
		sudo ln -s /usr/share/munin/plugins/loggrep jenkins_oom
679
	fi
680
681
	# this is a hack to work around (rare) problems with restarting munin-node...
	sudo service munin-node restart || sudo service munin-node restart || sudo service munin-node restart
682
fi
683
684
685
686

# add some users to groups after packages have been installed
if ! $UP2DATE ; then
	case $HOSTNAME in
Mattia Rizzolo's avatar
Mattia Rizzolo committed
687
		jenkins|osuosl-build169-amd64|osuosl-build170-amd64|osuosl-build173-amd64)
688
			# for building Archlinux
689
690
691
			sudo addgroup --system --gid 300 abuild
			sudo adduser jenkins abuild
			;;
692
693
694
695
696
		osuosl-build173-amd64)		sudo adduser jenkins sbuild ;;
		*) 				;;
	esac
fi
# finally
Holger Levsen's avatar
Holger Levsen committed
697
explain "packages configured."
Holger Levsen's avatar
Holger Levsen committed
698
699
700
701

#
# install the heart of jenkins.debian.net
#
Holger Levsen's avatar
Holger Levsen committed
702
cd $BASEDIR
703
[ -d /srv/jenkins/features ] && sudo rm -rf /srv/jenkins/features
704
# check for bash syntax *before* actually deploying anything
705
shopt -s nullglob
706
for f in bin/*.sh bin/**/*.sh ; do bash -n "$f" ; done
707
shopt -u nullglob
708
for dir in bin logparse mustache-templates ; do
709
710
	sudo mkdir -p /srv/jenkins/$dir
	sudo rsync -rpt --delete $dir/ /srv/jenkins/$dir/
Holger Levsen's avatar
Holger Levsen committed
711
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
712
done
713
714
HOST_JOBS="hosts/$HOSTNAME/job-cfg"
if [ -e "$HOST_JOBS" ] ; then
715
	sudo -u jenkins-adm rsync -rpt --copy-links --delete "$HOST_JOBS/" /srv/jenkins/job-cfg/
716
else
Philip Hands's avatar
Philip Hands committed
717
718
	# tidying up ... assuming that we don't want clutter on peripheral servers
	[ -d /srv/jenkins/job-cfg ] && sudo rm -rf /srv/jenkins/job-cfg
719
720
fi

721

722
sudo mkdir -p -m 700 /var/lib/jenkins/.ssh
723
sudo chown jenkins.jenkins /var/lib/jenkins/.ssh
724
if [ "$HOSTNAME" = "jenkins" ] ; then
725
726
	sudo -u jenkins install -m 600 jenkins-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
	sudo -u jenkins cp jenkins-home/procmailrc /var/lib/jenkins/.procmailrc
727
	sudo -u jenkins cp jenkins-home/offline_nodes /var/lib/jenkins/offline_nodes
728
729
730
else
	sudo cp jenkins-nodes-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
fi
731
if [ -f jenkins-nodes-home/authorized_keys.$HOSTNAME ] ; then
732
	cat jenkins-nodes-home/authorized_keys.$HOSTNAME | sudo tee -a /var/lib/jenkins/.ssh/authorized_keys
733
fi
734
sudo -u jenkins cp jenkins-home/gitconfig /var/lib/jenkins/.gitconfig
735
736
sudo -u jenkins cp jenkins-home/ssh_config.in /var/lib/jenkins/.ssh/config
nodes/gen_ssh_config | sudo -u jenkins tee -a /var/lib/jenkins/.ssh/config > /dev/null
737
nodes/gen_known_host_file | sudo tee /etc/ssh/ssh_known_hosts > /dev/null
Holger Levsen's avatar
Holger Levsen committed
738
explain "scripts and configurations for jenkins updated."
739

740
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
741
	sudo cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
742
	TMPFILE=$(mktemp)
743
	git log | grep ^Author| cut -d " " -f2-|sort -u -f > $TMPFILE
744
	echo "----" >> $TMPFILE
745
	sudo tee /var/lib/jenkins/userContent/THANKS > /dev/null < THANKS.head
746
747
	# samuel, lunar, jelle, josch and phil committed with several committers, only display one
	grep -v -e "samuel.thibault@ens-lyon.org" -e Lunar -e "j.schauer@email.de" -e "mattia@mapreri.org" -e "phil@jenkins-test-vm" -e "jelle@vdwaa.nl" $TMPFILE | sudo tee -a /var/lib/jenkins/userContent/THANKS > /dev/null
748
	rm $TMPFILE
749
	TMPDIR=$(mktemp -d -t update-jdn-XXXXXXXX)
750
	sudo cp -pr userContent $TMPDIR/
Holger Levsen's avatar
Holger Levsen committed
751
752
753
	sudo chown -R jenkins.jenkins $TMPDIR
	sudo cp -pr $TMPDIR/userContent  /var/lib/jenkins/
	sudo rm -r $TMPDIR > /dev/null
754
755
756
757
758
759
760
761
762
	cd /var/lib/jenkins/userContent/
	ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
	[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
	[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
	[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
	[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
	diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
	mv THANKS .THANKS
	rm TODO README INSTALL CONTRIBUTING
763
	sudo chown jenkins.jenkins /var/lib/jenkins/userContent/*html
Holger Levsen's avatar
Holger Levsen committed
764
	explain "user content for jenkins updated."
765
fi
766

767
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
768
769
770
771
772
773
	#
	# run jenkins-job-builder to update jobs if needed
	#     (using sudo because /etc/jenkins_jobs is root:root 700)
	#
	cd /srv/jenkins/job-cfg
	for metaconfig in *.yaml.py ; do
Holger Levsen's avatar
Holger Levsen committed
774
775
776
777
778
779
		if [ -f $metaconfig ] ; then
			TMPFILE=$(sudo -u jenkins-adm mktemp)
			./$metaconfig | sudo -u jenkins-adm tee "$TMPFILE" >/dev/null
			if ! sudo -u jenkins-adm cmp -s ${metaconfig%.py} "$TMPFILE" ; then
				sudo -u jenkins-adm mv "$TMPFILE" "${metaconfig%.py}"
			fi
780
781
782
		fi
	done
	for config in *.yaml ; do
783
784
785
786
787
788
789
		# do update, if
		# no stamp file exist or
		# no .py file exists and config is newer than stamp or
		# a .py file exists and .py file is newer than stamp
		if [ ! -f $STAMP ] || \
		 ( [ ! -f $config.py ] && [ $config -nt $STAMP ] ) || \
		 ( [ -f $config.py ] && [ $config.py -nt $STAMP ] ) ; then
790
			echo "$config has changed, executing updates."
791
			$JJB update $config # FIXME: we should use sudo and run $JJB as jenkins or jenkins-adm
792
793
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
794
	explain "jenkins jobs updated."
795
fi
796

797
798
799
#
# generate the kgb-client configurations
#
800
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "osuosl-build168-amd64" ] || [ "$HOSTNAME" = "osuosl-build171-amd64" ] || [ "$HOSTNAME" = "osuosl-build172-amd64" ] || [ "$HOSTNAME" = "profitbricks-build2-i386" ] || [ "$HOSTNAME" = "profitbricks-build12-i386" ] ; then
801
802
803
	cd $BASEDIR
	KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
	if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
Philip Hands's avatar
Philip Hands committed
804
		# the last condition is to assure the files are owned by the right user/team
805
806
		if [ "$KGB_SECRETS" -nt $STAMP ] || [ "/srv/jenkins/bin/deploy_kgb.py" -nt "$STAMP" ] || [ ! -f $STAMP ] ; then
			sudo -u jenkins-adm "/srv/jenkins/bin/deploy_kgb.py"
Philip Hands's avatar
Philip Hands committed
807
808
809
		else
			explain "kgb-client configuration unchanged, nothing to do."
		fi
810
	else
Philip Hands's avatar
Philip Hands committed
811
812
813
		figlet -f banner Warning
		echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
		echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
Holger Levsen's avatar
Holger Levsen committed
814
815
		echo "/srv/jenkins/kgb should be mode 755 and owned by jenkins-adm:root."
		echo "/srv/jenkins/kgb/client-status should be mode 755 and owned by jenkins:jenkins."
816
	fi
817
	KGB_STATUS="/srv/jenkins/kgb/client-status"
818
819
	sudo mkdir -p $KGB_STATUS
	sudo chown jenkins:jenkins $KGB_STATUS
820
821
fi

822
823
824
#
# Create GPG key for jenkins user if they do not already exist (eg. to sign .buildinfo files)
#
825
if sudo -H -u jenkins gpg --with-colons --fixed-list-mode --list-secret-keys | cut -d: -f1 | grep -qsFx 'sec' >/dev/null 2>&1 ; then
Holger Levsen's avatar
Holger Levsen committed
826
	: # not generating GPG key as one already exists for jenkins user
827
else
Holger Levsen's avatar
Holger Levsen committed
828
	explain "$(date) - Generating GPG key for jenkins user."
829

830
	sudo -H -u jenkins gpg --no-tty --batch --gen-key <<EOF
831
832
833
Key-Type: RSA
Key-Length: 4096
Key-Usage: sign
834
Name-Real: $HOSTNAME
835
836
837
838
839
840
Name-Comment: Automatically generated key for signing .buildinfo files
Expire-Date: 0
%no-ask-passphrase
%no-protection
%commit
EOF
841

842
	GPG_KEY_ID="$(sudo -H -u jenkins gpg --with-colons --fixed-list-mode --list-secret-keys | grep '^sec' | cut -d: -f5 | tail -n1)"
843
844
845
846
847
848

	if [ "$GPG_KEY_ID" = "" ]
	then
		explain "$(date) - Generated GPG key but could not parse key ID"
	else
		explain "$(date) - Generated GPG key $GPG_KEY_ID - submitting to keyserver"
849
		sudo -H -u jenkins gpg --send-keys $GPG_KEY_ID
850
	fi
851
852
fi

853
#
854
# almost finally…
855
#
856
sudo touch $STAMP	# so on the next run, only configs newer than this file will be updated
Holger Levsen's avatar
Holger Levsen committed
857
explain "$(date) - finished deployment."
858

859
860
861
862
#
# some final checks only for the jenkins
#
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
863
	jenkins_bugs_check
Holger Levsen's avatar
Holger Levsen committed
864
	explain "$(date) - done checking for known jenkins bugs."
865
866
fi

Holger Levsen's avatar
Holger Levsen committed
867
#
868
# finally!
Holger Levsen's avatar
Holger Levsen committed
869
#
870
case $HOSTNAME in
871
	# set time back to the future
872
	profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386)
873
		disable_dsa_check_packages
874
875
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
876
	codethink-sled9*|codethink-sled11*|codethink-sled13*|codethink-sled15*)
877
		disable_dsa_check_packages
878
879
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
880
	osuosl-build170-amd64|osuosl-build172-amd64)
881
882
883
		disable_dsa_check_packages
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
884
	jenkins)
885
		# notify irc on updates of jenkins.d.n
886
887
		MESSAGE="jenkins.d.n updated to $(cd $BASEDIR ; git describe --always)."
		kgb-client --conf /srv/jenkins/kgb/debian-qa.conf --relay-msg "$MESSAGE"
Holger Levsen's avatar
Holger Levsen committed
888
		;;
889
890
	*)	;;
esac
891
892
893
894

echo
figlet ok
echo
Holger Levsen's avatar
Holger Levsen committed
895
896
echo "__$HOSTNAME=ok__"