update_jdn.sh 29.5 KB
Newer Older
1
#!/bin/bash
2
# vim: set noexpandtab:
3
# Copyright 2012-2021 Holger Levsen <holger@layer-acht.org>
4
#         ©      2018 Mattia Rizzolo <mattia@debian.org>
Holger Levsen's avatar
Holger Levsen committed
5
# released under the GPLv2
6

Holger Levsen's avatar
Holger Levsen committed
7
# puppet / salt / ansible / fai / chef / deployme.app - disclaimer
Holger Levsen's avatar
Holger Levsen committed
8
9
10
11
12
13
14
15
16
17
18
# (IOW: this script has been grown in almost 500 commits and it shows…)
#
# yes, we know… and: "it" should probably still be done.
#
# It just unclear, how/what, and what we have actually mostly works.
#
# Switching to jenkins.debian.org is probably an opportunity
# to write (refactor this into) *yet another deployment script*
# (interacting with the DSA machine setup which is in puppet…),
# thus obsoleting this script gradually, though this is used on
# 47 hosts currently (of which quite some were initially installed
Edward Betts's avatar
Edward Betts committed
19
# manually…)
Holger Levsen's avatar
Holger Levsen committed
20
21
22
#
# so, yes, patches welcome. saying this is crap alone is not helpful,
# nor is just suggesting some new or old technology. patches most welcome!
23
24
#
# that said, there's a new one: init_node ;)
Holger Levsen's avatar
Holger Levsen committed
25

26
27
set -e

Holger Levsen's avatar
Holger Levsen committed
28
# prevent failure on first run...
Holger Levsen's avatar
Holger Levsen committed
29
[ ! -f /srv/jenkins/bin/common-functions.sh ] || . /srv/jenkins/bin/common-functions.sh
Holger Levsen's avatar
Holger Levsen committed
30

31
BASEDIR="$(dirname "$(readlink -e $0)")"
32
STAMP=/var/log/jenkins/update-jenkins.stamp
33
34
# The $@ below means that command line args get passed on to j-j-b
# which allows one to specify --flush-cache or --ignore-cache
35
JJB="jenkins-jobs $@"
36
DPKG_ARCH="$(dpkg --print-architecture)"
Holger Levsen's avatar
Holger Levsen committed
37

38
# so we can later run some commands only if $0 has been updated…
Philip Hands's avatar
Philip Hands committed
39
if [ -f $STAMP ] && [ $STAMP -nt $BASEDIR/$0 ] ; then
Holger Levsen's avatar
Holger Levsen committed
40
	UP2DATE=true
Holger Levsen's avatar
Holger Levsen committed
41
	echo $HOSTNAME is up2date.
Philip Hands's avatar
Philip Hands committed
42
43
else
	UP2DATE=false
Holger Levsen's avatar
Holger Levsen committed
44
	echo $HOSTNAME needs to be updated.
45
46
47
fi


Holger Levsen's avatar
Holger Levsen committed
48
explain() {
Holger Levsen's avatar
Holger Levsen committed
49
	echo "$HOSTNAME: $1"
Holger Levsen's avatar
Holger Levsen committed
50
51
}

52
53
54
set_correct_date() {
		# set correct date
		sudo service ntp stop || true
55
		sudo ntpdate -b $1
56
}
57

58
disable_dsa_check_packages() {
59
	# disable check for outdated packages as someday in the future
60
	# packages from security.d.o will appear outdated always…
61
62
	echo -e "#!/bin/sh\n# disabled dsa-check by update_jdn.sh\nexit 0" | sudo tee /usr/local/bin/dsa-check-packages
	sudo chmod a+rx /usr/local/bin/dsa-check-packages
63
64
}

65
echo "--------------------------------------------"
Holger Levsen's avatar
Holger Levsen committed
66
explain "$(date) - begin deployment update."
67

68
69
70
71
#
# temporarily test to check which hosts don't use systemd
#
if [ -z "$(dpkg -l|grep systemd-sysv||true)" ] ; then 
72
	echo "no systemd-sysv installed on $(hostname), please enter to continue…"
73
74
75
	read
fi

76
# some nodes need special treatment…
77
case $HOSTNAME in
78
	ionos5-amd64|ionos6-i386|ionos15-amd64|ionos16-i386)
Holger Levsen's avatar
Holger Levsen committed
79
		# set correct date
80
		set_correct_date de.pool.ntp.org
81
		;;
82
	codethink9*|codethink11*|codethink13*|codethink15*)
83
		# set correct date
84
		set_correct_date de.pool.ntp.org
Holger Levsen's avatar
Holger Levsen committed
85
		;;
86
	osuosl170-amd64|osuosl172-amd64)
87
		# set correct date
88
		set_correct_date time.osuosl.org
89
		;;
90
91
92
	*)	;;
esac

93
94
# ubuntu decided to change kernel perms in the middle of LTS…
case $HOSTNAME in
95
	codethink*)
96
97
98
99
100
101
102
103
		# fixup perms
		sudo chmod +r /boot/vmlinuz-*
		;;
	*)	;;
esac



104
105
106
#
# set up users and groups
#
107
declare -A user_host_groups u_shell
108
sudo_groups='jenkins,jenkins-adm,sudo,adm'
109
110
111

# if there's a need for host groups, a case statement on $HOSTNAME here that sets $GROUPNAME, say, should do the trick
# then you can define user_host_groups['phil','lvm_group']=... below
112
# and add checks for the GROUP version where ever the HOSTNAME is checked in the following code
113
114
115
116
117

user_host_groups['helmut','*']="$sudo_groups"
user_host_groups['holger','*']="$sudo_groups"
user_host_groups['holger','jenkins']="reproducible,${user_host_groups['holger','*']}"
user_host_groups['mattia','*']="$sudo_groups"
118
user_host_groups['mattia','jenkins']="reproducible,${user_host_groups['mattia','*']}"
119
user_host_groups['phil','osuosl167-amd64']="$sudo_groups"
120
user_host_groups['jelmer','osuosl167-amd64']="$sudo_groups"
121
user_host_groups['phil','jenkins']="$sudo_groups"
122
user_host_groups['lunar','jenkins']='reproducible'
123
124
user_host_groups['lynxis','osuosl171-amd64']="$sudo_groups"
user_host_groups['lynxis','osuosl172-amd64']="$sudo_groups"
125
user_host_groups['lynxis','jenkins']="jenkins"
126
127
user_host_groups['kpcyrd','osuosl169-amd64']="$sudo_groups"
user_host_groups['kpcyrd','osuosl170-amd64']="$sudo_groups"
128
user_host_groups['kpcyrd','jenkins']="jenkins"
129
130
user_host_groups['bubu','osuosl168-amd64']="$sudo_groups"
user_host_groups['hans','osuosl168-amd64']="$sudo_groups"
131
user_host_groups['flx','osuosl168-amd64']="$sudo_groups"
132
user_host_groups['jspricke','osuosl168-amd64']="$sudo_groups"
Mattia Rizzolo's avatar
Mattia Rizzolo committed
133
user_host_groups['vagrant','*']="$sudo_groups"
134
user_host_groups['bernhard','osuosl174-amd64']="$sudo_groups"
135
user_host_groups['santiago','ionos3-amd64']="$sudo_groups"
136
user_host_groups['fpierret','snapshot']="$sudo_groups"
137
138

u_shell['mattia']='/bin/zsh'
139
u_shell['lynxis']='/usr/bin/fish'
140

141
142
143
# get the users out of the user_host_groups array's index
users=$(for i in ${!user_host_groups[@]}; do echo ${i%,*} ; done | sort -u)

144
145
( $UP2DATE && [ -z "$(find $BASEDIR/authorized_keys -newer $0)" ] ) || for user in ${users}; do
	cd $BASEDIR
146
	# -v is a bashism to check for set variables, used here to see if this user is active on this host
147
	if [ ! -v user_host_groups["$user","$HOSTNAME"] ] && [ ! -v user_host_groups["$user",'*'] ] && [ ! -v user_host_groups["$user","$DPKG_ARCH"] ] ; then
Holger Levsen's avatar
Holger Levsen committed
148
149
		continue
	fi
150

151
152
	# create the letsencrypt group and user in jenkins if needed
	# doing it here instead of using the array above as we have to use --system
153
	if [ "$HOSTNAME" = jenkins ] || [ "$HOSTNAME" = "ionos7-amd64" ]; then
154
155
156
		if ! getent group letsencrypt > /dev/null ; then
			sudo addgroup --system letsencrypt
		fi
157
		if ! getent passwd letsencrypt > /dev/null ; then
158
159
160
161
162
163
164
165
166
167
168
			sudo adduser --system \
				--gecos "Let's Encrypt role account" \
				--shell /usr/sbin/nologin \
				--disabled-login \
				--home /var/lib/dehydrated \
				--no-create-home \
				--ingroup letsencrypt \
				letsencrypt
		fi
	fi

169
170
171
172
173
174
	# create the user
	if ! getent passwd $user > /dev/null ; then
		# adduser, defaulting to /bin/bash as shell
		sudo adduser --gecos "" --shell "${u_shell[$user]:-/bin/bash}" --disabled-password $user
	fi
	# add groups: first try the specific host, or if unset fall-back to default '*' setting
175
	for h in "$HOSTNAME" "$DPKG_ARCH" '*' ; do
176
		if [ -v user_host_groups["$user","$h"] ] ; then
Holger Levsen's avatar
Holger Levsen committed
177
			sudo usermod -G "${user_host_groups["$user","$h"]}" $user
178
			break
179
		fi
180
181
182
183
184
	done
	# add the user's keys (if any)
	if ls authorized_keys/${user}@*.pub >/dev/null 2>&1 ; then
		[ -d /var/lib/misc/userkeys ] || sudo mkdir -p /var/lib/misc/userkeys
		cat authorized_keys/${user}@*.pub | sudo tee /var/lib/misc/userkeys/${user} > /dev/null
185
	fi
186
187
done

188
189
# configure sbuild:
case $HOSTNAME in
190
	ionos7-a*|osuosl167*) sudo sbuild-adduser jenkins ;;
191
192
193
	*) ;;
esac

194
195
196
197
198
sudo mkdir -p /srv/workspace
[ -d /srv/schroots ] || sudo mkdir -p /srv/schroots
[ -h /chroots ] || sudo ln -s /srv/workspace/chroots /chroots
[ -h /schroots ] || sudo ln -s /srv/schroots /schroots

199
200
# prepare tmpfs on some hosts
case $HOSTNAME in
Mattia Rizzolo's avatar
Mattia Rizzolo committed
201
202
203
204
	jenkins)
		TMPFSSIZE=100
		TMPSIZE=15
		;;
205
	ionos9-amd64)
206
		TMPFSSIZE=40
Mattia Rizzolo's avatar
Mattia Rizzolo committed
207
208
		TMPSIZE=8
		;;
209
	ionos*)
Mattia Rizzolo's avatar
Mattia Rizzolo committed
210
211
212
		TMPFSSIZE=200
		TMPSIZE=15
		;;
213
214
215
216
	codethink*)
		TMPFSSIZE=100
		TMPSIZE=15
		;;
217
218
219
220
	osuosl*)
		TMPFSSIZE=400
		TMPSIZE=50
		;;
221
222
223
	*) ;;
esac
case $HOSTNAME in
224
	ionos*i386)
225
226
227
228
229
		if ! grep -q '/srv/workspace' /etc/fstab; then
			echo "Warning: you need to manually create a /srv/workspace partition on i386 nodes, exiting."
			exit 1
		fi
		;;
230
	jenkins|ionos*amd64|codethink*|osuosl*)
231
		if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
232
			echo "tmpfs		/srv/workspace	tmpfs	defaults,size=${TMPFSSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null  
233
		fi
Mattia Rizzolo's avatar
Mattia Rizzolo committed
234
235
236
		if ! grep -q '^tmpfs\s\+/tmp\s' /etc/fstab; then
			echo "tmpfs		/tmp	tmpfs	defaults,size=${TMPSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null
		fi
237
238
		if ! mountpoint -q /srv/workspace; then
			if test -z "$(ls -A /srv/workspace)"; then
239
				sudo mount /srv/workspace
240
			else
241
				explain "WARNING: mountpoint /srv/workspace is non-empty."
242
243
244
245
246
			fi
		fi
		;;
	*) ;;
esac
247

248
# make sure needed directories exists - some directories will not be needed on all hosts...
249
for directory in /schroots /srv/reproducible-results /srv/d-i /srv/udebs /var/log/jenkins/ /srv/jenkins /srv/jenkins/pseudo-hosts /srv/workspace/chroots ; do
250
251
252
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
	fi
Holger Levsen's avatar
Holger Levsen committed
253
	sudo chown jenkins.jenkins $directory
254
done
Holger Levsen's avatar
Holger Levsen committed
255
for directory in /srv/jenkins ; do
256
257
258
259
260
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
		sudo chown jenkins-adm.jenkins-adm $directory
	fi
done
261

262
if ! test -h /chroots; then
Holger Levsen's avatar
Holger Levsen committed
263
	sudo rmdir /chroots || sudo rm -f /chroots # do not recurse
264
	if test -e /chroots; then
Holger Levsen's avatar
Holger Levsen committed
265
		explain "/chroots could not be cleared."
266
	else
267
		sudo ln -s /srv/workspace/chroots /chroots
268
269
270
	fi
fi

271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
#
# deploy package configuration in /etc and /usr
#
cd $BASEDIR
for h in common common-amd64 common-i386 common-arm64 common-armhf "$HOSTNAME" ; do
	# $HOSTNAME has precedence over common-$DPKG_ARCH over common
	case $h in
		common-amd64) [ $DPKG_ARCH = "amd64" ] || continue ;;
		common-i386)  [ $DPKG_ARCH = "i386" ] || continue ;;
		common-arm64) [ $DPKG_ARCH = "arm64" ] || continue ;;
		common-armhf) [ $DPKG_ARCH = "armhf" ] || continue ;;
		*) ;;
	esac
	if [ -d "hosts/$h/etc/sudoers.d/" ]; then
		for f in "hosts/$h/etc/sudoers.d/"* ; do
			/usr/sbin/visudo -c -f "$f" > /dev/null
		done
	fi
	for d in etc usr ; do
		if [ -d "hosts/$h/$d" ]; then
			sudo cp --preserve=mode,timestamps -r "hosts/$h/$d/"* "/$d"
		fi
	done
done
295
296
# We really don't need the man database here, save time.
$UP2DATE || echo "man-db man-db/auto-update boolean false" | sudo debconf-set-selections
297
298
299
300
301


#
# install packages we need
#
302
# only on Debian-based systems
303
if [ -f /etc/debian_version ] ; then
304
	if [ $BASEDIR/$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then
305
		DEBS="
306
			apt-utils
Holger Levsen's avatar
Holger Levsen committed
307
			bash-completion 
308
309
			bc
			bsd-mailx
310
311
			curl
			debian-archive-keyring
312
			dstat
313
			cdebootstrap-
314
			debootstrap
315
			devscripts
316
			eatmydata
317
			etckeeper
318
			figlet
Holger Levsen's avatar
Holger Levsen committed
319
			git
Holger Levsen's avatar
Holger Levsen committed
320
			gnupg
321
			haveged
Holger Levsen's avatar
Holger Levsen committed
322
			htop
323
			less
324
			liquidprompt
325
			locales-all
Mattia Rizzolo's avatar
Mattia Rizzolo committed
326
			lsof
327
328
			molly-guard
			moreutils
Holger Levsen's avatar
Holger Levsen committed
329
			mosh
330
331
332
			munin-node
			munin-plugins-core
			munin-plugins-extra
Holger Levsen's avatar
Holger Levsen committed
333
			needrestart
334
			netcat-traditional
335
336
			ntp
			ntpdate
337
			pbuilder
338
			pigz 
339
			postfix
340
			procmail
341
			psmisc
342
			python3-psycopg2 
343
			python3-yaml
Holger Levsen's avatar
Holger Levsen committed
344
			schroot 
345
346
			screen
			slay
347
			stunnel
Holger Levsen's avatar
Holger Levsen committed
348
349
350
			subversion 
			subversion-tools 
			sudo 
351
			systemd-sysv
352
			tmux
Holger Levsen's avatar
Holger Levsen committed
353
354
			unzip 
			vim 
355
			zsh
356
			"
357
		case $HOSTNAME in
358
			codethink*)
359
360
				# the apt_preferences(5) file will cause some packages to come
				# from Debian backports
361
362
				# munin-node from buster-backports requires init-system-helpers>=1.54
				# which is available in bionic-backports, we only need to convince apt
363
364
365
366
				DEBS="$DEBS
				init-system-helpers/bionic-backports
				" ;;
			*)	# packages to be installed on all Debian systems but which are not available in Ubuntu 18.04:
367
				DEBS="$DEBS
368
				fasttrack-archive-keyring
369
370
				ripgrep
				mmdebstrap
371
372
				" ;;
		esac
373
		# special case bullseye-backports:
374
		case $HOSTNAME in
375
			X)
376
				DEBS="$DEBS
377
				devscripts/bullseye-backports
378
				" ;;
379
			*)	;;
380
		esac
381
		case $HOSTNAME in
382
			# needed for rebuilding Debian (using .buildinfo files)
383
384
			osuosl173-amd64) DEBS="$DEBS libdpkg-perl libwww-mechanize-perl sbuild" 
					 DEBS="$DEBS live-build" ;; # needed for live-build
385
			# needed for opensuse openbuildservice
386
			osuosl174-amd64) DEBS="$DEBS osc obs-build qemu-kvm libarchive-tools libjson-perl libxml-simple-perl" ;;
387
388
			*) ;;
		esac
389
390
		# install stuff for bremner's builtin-pho stuff
		# and building html pages there too
391
		# and for the rebuilder 'thing' too
392
		case $HOSTNAME in
393
			ionos7-a*) DEBS="$DEBS
394
				postgresql
395
				postgresql-13-debversion
396
				python3-pystache
397
				python3-apt
398
399
				python3-psycopg2
				sbuild" ;;
400
401
			*) ;;
		esac
402
		# install squid / apache2 on a few nodes only
403
		case $HOSTNAME in
404
			ionos1-a*|ionos10*|codethink16*|osuosl*) DEBS="$DEBS
405
				squid" ;;
406
			ionos7-a*) DEBS="$DEBS
407
408
				dehydrated
				dehydrated-apache2
409
				apache2" ;;
410
411
			*) ;;
		esac
Jelmer Vernooij's avatar
Jelmer Vernooij committed
412
413
		# install lintian-brush dependencies for debian-janitor
		case $HOSTNAME in
414
			# TODO(jelmer): Add python3-tr once it hits backports
415
			osuosl167*) DEBS="$DEBS
416
				cargo
417
				cython3
Jelmer Vernooij's avatar
Jelmer Vernooij committed
418
419
				devscripts
				python3-aiohttp
420
				python3-all-dev
421
422
				dpkg-source-gitarchive
				python3-asyncpg
423
				python3-bs4
424
425
				python3-certifi
				python3-configobj
Jelmer Vernooij's avatar
Jelmer Vernooij committed
426
				python3-debian
427
				python3-docutils
Jelmer Vernooij's avatar
Jelmer Vernooij committed
428
429
				python3-dulwich
				python3-distro-info
430
				python3-fastimport
Jelmer Vernooij's avatar
Jelmer Vernooij committed
431
				python3-iniparse
432
				python3-jinja2
433
				python3-launchpadlib
Jelmer Vernooij's avatar
Jelmer Vernooij committed
434
				python3-levenshtein
435
				python3-lz4
436
				python3-markdown
437
				python3-merge3
438
				python3-paramiko
439
				python3-patiencediff
Jelmer Vernooij's avatar
Jelmer Vernooij committed
440
				python3-pkginfo
441
				python3-prometheus-client
442
				python3-pyinotify
443
				python3-requirement-parser
Jelmer Vernooij's avatar
Jelmer Vernooij committed
444
				python3-ruamel.yaml
445
				python3-semver
446
447
448
449
				python3-setuptools
				python3-six
				python3-subunit
				python3-testtools
450
				python3-toml
451
				python3-tomlkit
Holger Levsen's avatar
Holger Levsen committed
452
				python3-urllib3
453
				python3-yaml
454
				rustc
Holger Levsen's avatar
Holger Levsen committed
455
				sbuild
Jelmer Vernooij's avatar
Jelmer Vernooij committed
456
457
458
459
460
				gnome-pkg-tools
				postgresql-server-dev-all
				lintian
				dos2unix
				gpg
461
				libdebhelper-perl
462
				quilt
463
				debcargo
Jelmer Vernooij's avatar
Jelmer Vernooij committed
464
				cme
465
				debhelper
Jelmer Vernooij's avatar
Jelmer Vernooij committed
466
				decopy
Jelmer Vernooij's avatar
Jelmer Vernooij committed
467
468
				" ;;
		esac
469
		# notifications are only done from a few nodes
470
		case $HOSTNAME in
471
			jenkins|ionos*|osuosl167-amd64) DEBS="$DEBS
472
473
				kgb-client
				python3-yaml" ;;
474
475
			*) ;;
		esac
476
477
		# needed to run the 2nd reproducible builds nodes in the future...
		case $HOSTNAME in
478
			ionos5-amd64|ionos6-i386|ionos15-amd64|ionos16-i386) DEBS="$DEBS ntpdate" ;;
479
			codethink9*|codethink11*|codethink13*|codethink15*) DEBS="$DEBS ntpdate" ;;
480
			osuosl170-amd64|osuosl172-amd64) DEBS="$DEBS ntpdate" ;;
481
482
			*) ;;
		esac
483
		# needed to run coreboot/openwrt/netbsd/fedora jobs
484
		case $HOSTNAME in
485
		osuosl171-amd64|osuosl172-amd64) DEBS="$DEBS
Holger Levsen's avatar
Holger Levsen committed
486
				bison
487
				ca-certificates
Holger Levsen's avatar
Holger Levsen committed
488
489
490
				cmake
				diffutils
				findutils
491
				fish
Holger Levsen's avatar
Holger Levsen committed
492
493
494
495
				flex
				g++
				gawk
				gcc
496
				gnat
Holger Levsen's avatar
Holger Levsen committed
497
498
499
500
501
502
503
504
				grep
				libc6-dev
				libncurses5-dev
				libssl-dev
				locales-all
				kgb-client
				m4
				make
505
506
				python3-clint
				python3-git
507
				python3-pystache
508
				python3-requests
Holger Levsen's avatar
Holger Levsen committed
509
510
				python3-yaml
				subversion
511
				tree
Holger Levsen's avatar
Holger Levsen committed
512
513
				unzip
				util-linux
514
515
				zlib1g-dev
				"
516
517
518
			;;
			*) ;;
		esac
519
520
		# needed to run fdroid jobs
		case $HOSTNAME in
521
			osuosl168-amd64) DEBS="$DEBS
522
				androguard
523
				android-sdk
524
525
				bzr
				git-svn
526
				fdroidserver
527
				linux-headers-amd64
528
				mercurial
529
				python3-asn1crypto
530
				python3-babel
531
				python3-mwclient
532
				python3-setuptools
533
				subversion
534
				vagrant
Mattia Rizzolo's avatar
Mattia Rizzolo committed
535
				virtualbox/bullseye-fasttrack virtualbox-dkms/bullseye-fasttrack
536
				"
537
				;;
538
			*) ;;
539
		esac
540
		if [ "$HOSTNAME" = "osuosl167-amd64" ]; then
541
542
			# for phil only
			DEBS="$DEBS postfix-pcre"
543
			# OpenQA worker -- currently installed with help from:
Philip Hands's avatar
Philip Hands committed
544
545
546
547
			#   hosts/osuosl167-amd64/etc/apt/sources.list.d/philh-openqa.list
			DEBS="$DEBS
				openqa-worker
				"
548
549
			# only needed on the main node
		elif [ "$HOSTNAME" = "jenkins" ] ; then
550
			DEBS="$DEBS ffmpeg python3-popcon dose-extra"
551
		fi
552
553
554
		# packages needed for snapshot.reproducible-builds.org
		case $HOSTNAME in
			snapshot*)	DEBS="$DEBS
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
					nginx-full
					postgresql-13
					postgresql-plpython3-13
					python3-dateutil
					python3-debian
					python3-flask
					python3-flask-caching
					python3-flask-sqlalchemy
					python3-httpx
					python3-psycopg2
					python3-sqlalchemy
					python3-sqlalchemy
					python3-tenacity
					uwsgi
					uwsgi-plugin-python3
570
571
					xfsprogs
					" ;;
572
					*) ;;
573
		esac
574
		# mock is needed to build fedora
575
576
577
		#if [ "$HOSTNAME" = "osuosl171-amd64" ] || [ "$HOSTNAME" = "osuosl172-amd64" ] || [ "$HOSTNAME" = "jenkins" ] ; then
		#	DEBS="$DEBS mock"
		#fi
578
		# only on main node
579
		if [ "$HOSTNAME" = "jenkins" ] ; then
580
581
			# required by _db_backup, but not available in bullseye.  https://bugs.debian.org/970870
			#postgresql-autodoc
Holger Levsen's avatar
Holger Levsen committed
582
583
584
585
			MASTERDEBS=" 
				apache2 
				apt-file 
				apt-listchanges 
586
				asciidoc
Holger Levsen's avatar
Holger Levsen committed
587
				binfmt-support 
588
589
				bison
				botch
Holger Levsen's avatar
Holger Levsen committed
590
591
592
593
				build-essential 
				cmake 
				cron-apt 
				csvtool 
594
595
				dehydrated
				dehydrated-apache2
Holger Levsen's avatar
Holger Levsen committed
596
597
				dnsmasq-base 
				figlet 
Holger Levsen's avatar
Holger Levsen committed
598
				flex
Holger Levsen's avatar
Holger Levsen committed
599
				gawk 
600
				ghc
601
				git-lfs
Holger Levsen's avatar
Holger Levsen committed
602
603
604
605
				gocr 
				graphviz 
				iasl 
				imagemagick 
606
607
				ip2host
				jekyll
608
				jenkins-job-builder
609
				jq
610
				kgb-client
Holger Levsen's avatar
Holger Levsen committed
611
				libcap2-bin 
612
				libarchive-tools
Holger Levsen's avatar
Holger Levsen committed
613
614
615
616
617
618
619
620
				libfile-touch-perl 
				libguestfs-tools 
				libjson-rpc-perl 
				libsoap-lite-perl 
				libxslt1-dev 
				moreutils 
				mr 
				mtr-tiny 
621
				munin
Holger Levsen's avatar
Holger Levsen committed
622
				ntp 
623
				obfs4proxy
Holger Levsen's avatar
Holger Levsen committed
624
625
				openbios-ppc 
				openbios-sparc 
626
				openjdk-11-jre-headless
627
				pandoc
628
				postgresql
629
				postgresql-client 
Holger Levsen's avatar
Holger Levsen committed
630
631
632
				poxml 
				procmail 
				python3-debian 
633
				python3-pystache
634
				python3-requests
635
				python3-rpy2 
636
637
638
				python3-sqlalchemy
				python3-xdg
				python3-yaml
Holger Levsen's avatar
Holger Levsen committed
639
640
641
642
				qemu 
				qemu-kvm 
				qemu-system-x86 
				qemu-user-static 
643
				radvd
644
				ruby-jekyll-polyglot
645
				ruby-jekyll-redirect-from
646
647
				ruby-rspec
				rustc
Holger Levsen's avatar
Holger Levsen committed
648
649
650
651
				seabios 
				shorewall 
				shorewall6 
				sqlite3 
652
				syslinux
653
				systemd
654
				thin-provisioning-tools
655
				tor
Holger Levsen's avatar
Holger Levsen committed
656
				vncsnapshot 
657
658
				vnstat
				whohas
Holger Levsen's avatar
Holger Levsen committed
659
				x11-apps 
660
				xtightvncviewer
661
				xvfb
662
				xvkbd
663
664
				zutils
				"
665
666
667
		else
			MASTERDEBS=""
		fi
668
		$UP2DATE || sudo apt-get update
669
		$UP2DATE || sudo apt-get install $DEBS $MASTERDEBS
670
		# for varying kernels:
671
		# - we use bpo kernels on ionos5+15 (and the default amd64 kernel on ionos6+16-i386)
672
		# - we also use the bpo kernel on osuosl172 (but not osuosl171)
673
		# - this is done as a seperate step as bpo kernels are frequently uninstallable when upgraded on bpo
674
		if [ "$HOSTNAME" = "ionos5-amd64" ] || [ "$HOSTNAME" = "ionos15-amd64" ] \
675
			|| [ "$HOSTNAME" = "osuosl172-amd64" ] ; then
676
			sudo apt install linux-image-amd64/bullseye-backports || true # backport kernels are frequently uninstallable...
677
			:
678
679
680
681
682
683
		elif [ "$HOSTNAME" = "ionos6-i386" ] || [ "$HOSTNAME" = "ionos16-i386" ] ; then
			# run with the amd64 kernel in these i386 nodes
			sudo apt install linux-image-amd64 linux-image-686-pae-
		elif [ "$HOSTNAME" = "ionos2-i386" ] || [ "$HOSTNAME" = "ionos12-i386" ] ; then
			# run with the i386 kernel in these i386 nodes
			sudo apt install linux-image-686-pae linux-image-amd64-
684
		elif [ "$HOSTNAME" = "osuosl169-amd64" ] || [ "$HOSTNAME" = "osuosl170-amd64" ] || [ "$HOSTNAME" = "osuosl173-amd64" ]; then
685
			# Arch Linux builds latest stuff which sometimes (eg, currentlt Qt) needs newer kernel to build...
686
			sudo apt install linux-image-amd64/bullseye-backports || true # backport kernels are frequently uninstallable...
687
			:
688
		fi
Edward Betts's avatar
Edward Betts committed
689
		# don't (re-)install pbuilder if it's on hold
690
		if [ "$(dpkg-query -W -f='${db:Status-Abbrev}\n' pbuilder)" != "hi " ] ; then
691
			$UP2DATE || sudo apt-get install pbuilder
692
		fi
693
		# remove unattended-upgrades if it's installed
694
		if [ "$(dpkg-query -W -f='${db:Status-Abbrev}\n' unattended-upgrades 2>/dev/null || true)" = "ii "  ] ; then
695
			 sudo apt-get -y purge unattended-upgrades
696
		fi
697
		sudo apt-get clean
Holger Levsen's avatar
Holger Levsen committed
698
		explain "packages installed."
699
	else
Holger Levsen's avatar
Holger Levsen committed
700
		explain "no new packages to be installed."
701
	fi
702
fi
703

704
# remove munin-async (if it's installed which happens during setup as it's a recommends)
Holger Levsen's avatar
Holger Levsen committed
705
# harmless but has a service which can fail and thus can make maintenance jobs unstable
706
( dpkg -l munin-async 2>/dev/null && sudo apt-get remove munin-async ) || true
Holger Levsen's avatar
Holger Levsen committed
707

708
# we ship one or two service files…
709
sudo systemctl daemon-reload
710
711
712
713
714

#
# more configuration than a simple cp can do
#
sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins
715
sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm
Holger Levsen's avatar
Holger Levsen committed
716
[ -f /etc/mailname ] || ( echo $HOSTNAME.debian.net | sudo tee /etc/mailname )
717

718
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "ionos7-amd64" ]; then
719
720
721
722
723
724
725
	for path in /var/lib/dehydrated /var/lib/dehydrated/acme-challenges; do
		if ! dpkg-statoverride --list "$path" > /dev/null; then
			sudo dpkg-statoverride --update --add letsencrypt letsencrypt 755 "$path"
		fi
	done
fi

726
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "ionos7-amd64" ] ; then
727
	if ! $UP2DATE || [ $BASEDIR/hosts/$HOSTNAME/etc/apache2 -nt $STAMP ]  ; then
728
729
730
731
732
733
734
735
736
		if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
			sudo a2enmod proxy
			sudo a2enmod proxy_http
			sudo a2enmod rewrite
			sudo a2enmod ssl
			sudo a2enmod headers
			sudo a2enmod macro
			sudo a2enmod filter
		fi
737
738
739
740
741
742
743
744
745
746
747
		case "$HOSTNAME" in
			jenkins)
				sudo a2ensite -q jenkins.debian.net
				sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.debian.net.conf
				sudo a2enconf -q munin
				;;
			ionos7-amd64)
				sudo a2ensite -q buildinfos.debian.net
				sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/buildinfos.debian.net.conf
				;;
		esac
748
749
750
		# for reproducible.d.n url rewriting:
		[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
		sudo service apache2 reload
751
752
753
	fi
fi

754
if ! $UP2DATE || [ $BASEDIR/hosts/$HOSTNAME/etc/munin -nt $STAMP ] ; then
755
	cd /etc/munin/plugins
756
	sudo rm -f postfix_* open_inodes interrupts irqstats threads proc_pri vmstat if_err_* exim_* netstat fw_forwarded_local fw_packets forks open_files users nfs* iostat_ios ntp* df_abs 2>/dev/null
757
	case $HOSTNAME in
758
			ionos1-a*|ionos10*|codethink16*|osuosl*) [ -L /etc/munin/plugins/squid_cache ] || for i in squid_cache squid_objectsize squid_requests squid_traffic ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
759
760
			*)	;;
	esac
Holger Levsen's avatar
Holger Levsen committed
761
762
763
764
	case $HOSTNAME in
			jenkins) [ -L /etc/munin/plugins/postfix_mailstats ] || for i in postfix_mailstats postfix_mailvolume postfix_mailqueue ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
			*)	;;
	esac
765
766
767
	if [ "$HOSTNAME" != "jenkins" ] && [ -L /etc/munin/plugins/iostat ] ; then
		sudo rm /etc/munin/plugins/iostat
	fi
768
	if ( [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "ionos7-amd64" ] ) && [ ! -L /etc/munin/plugins/apache_accesses ] ; then
769
		for i in apache_accesses apache_volume ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done
770
		sudo ln -s /usr/share/munin/plugins/loggrep jenkins_oom
771
	fi
772
773
	# this is a hack to work around (rare) problems with restarting munin-node...
	sudo service munin-node restart || sudo service munin-node restart || sudo service munin-node restart
774
fi
775
776
777
778

# add some users to groups after packages have been installed
if ! $UP2DATE ; then
	case $HOSTNAME in
779
		jenkins|osuosl169-amd64|osuosl170-amd64|osuosl173-amd64)
780
			# for building Archlinux
781
782
783
			sudo addgroup --system --gid 300 abuild
			sudo adduser jenkins abuild
			;;
784
		osuosl173-amd64)		sudo adduser jenkins sbuild ;;
785
786
787
788
		*) 				;;
	esac
fi
# finally
Holger Levsen's avatar
Holger Levsen committed
789
explain "packages configured."
Holger Levsen's avatar
Holger Levsen committed
790
791
792
793

#
# install the heart of jenkins.debian.net
#
Holger Levsen's avatar
Holger Levsen committed
794
cd $BASEDIR
795
[ -d /srv/jenkins/features ] && sudo rm -rf /srv/jenkins/features
796
# check for bash syntax *before* actually deploying anything
797
shopt -s nullglob
798
for f in bin/*.sh bin/**/*.sh ; do bash -n "$f" ; done
799
shopt -u nullglob
800
for dir in bin logparse mustache-templates ; do
801
802
	sudo mkdir -p /srv/jenkins/$dir
	sudo rsync -rpt --delete $dir/ /srv/jenkins/$dir/
Holger Levsen's avatar
Holger Levsen committed
803
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
804
done
805
806
HOST_JOBS="hosts/$HOSTNAME/job-cfg"
if [ -e "$HOST_JOBS" ] ; then
807
	sudo -u jenkins-adm rsync -rpt --copy-links --delete "$HOST_JOBS/" /srv/jenkins/job-cfg/
808
else
Philip Hands's avatar
Philip Hands committed
809
810
	# tidying up ... assuming that we don't want clutter on peripheral servers
	[ -d /srv/jenkins/job-cfg ] && sudo rm -rf /srv/jenkins/job-cfg
811
812
fi

813

814
sudo mkdir -p -m 700 /var/lib/jenkins/.ssh
815
sudo chown jenkins.jenkins /var/lib/jenkins/.ssh
816
if [ "$HOSTNAME" = "jenkins" ] ; then
817
818
	sudo -u jenkins install -m 600 jenkins-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
	sudo -u jenkins cp jenkins-home/procmailrc /var/lib/jenkins/.procmailrc
819
	sudo -u jenkins cp jenkins-home/offline_nodes /var/lib/jenkins/offline_nodes
820
821
822
else
	sudo cp jenkins-nodes-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
fi
823
if [ -f jenkins-nodes-home/authorized_keys.$HOSTNAME ] ; then
824
	cat jenkins-nodes-home/authorized_keys.$HOSTNAME | sudo tee -a /var/lib/jenkins/.ssh/authorized_keys
825
fi
826
sudo -u jenkins cp jenkins-home/gitconfig /var/lib/jenkins/.gitconfig
827
828
sudo -u jenkins cp jenkins-home/ssh_config.in /var/lib/jenkins/.ssh/config
nodes/gen_ssh_config | sudo -u jenkins tee -a /var/lib/jenkins/.ssh/config > /dev/null
829
nodes/gen_known_host_file | sudo tee /etc/ssh/ssh_known_hosts > /dev/null
Holger Levsen's avatar
Holger Levsen committed
830
explain "scripts and configurations for jenkins updated."
831

832
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
833
	sudo cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
834
	TMPFILE=$(mktemp)
835
	git log | grep ^Author| cut -d " " -f2-|sort -u -f > $TMPFILE
836
	echo "----" >> $TMPFILE
837
	sudo tee /var/lib/jenkins/userContent/THANKS > /dev/null < THANKS.head
Holger Levsen's avatar
Holger Levsen committed
838
	# several people committed with several committers, only display one
839
	DUPLICATES="samuel.thibault@ens-lyon.org|Jérémy Bobbio|j.schauer@email.de|mattia@mapreri.org|phil@jenkins-test-vm|jelle@vdwaa.nl|<kpcyrd>|vagrant@debian.org"
Holger Levsen's avatar
Holger Levsen committed
840
	grep -E -v "$DUPLICATES" $TMPFILE | sudo tee -a /var/lib/jenkins/userContent/THANKS > /dev/null
841
	rm $TMPFILE
842
	TMPDIR=$(mktemp -d -t update-jdn-XXXXXXXX)
843
	sudo cp -pr userContent $TMPDIR/
Holger Levsen's avatar
Holger Levsen committed
844
845
846
	sudo chown -R jenkins.jenkins $TMPDIR
	sudo cp -pr $TMPDIR/userContent  /var/lib/jenkins/
	sudo rm -r $TMPDIR > /dev/null
847
848
849
850
851
852
853
854
855
	cd /var/lib/jenkins/userContent/
	ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
	[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
	[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
	[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
	[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
	diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
	mv THANKS .THANKS
	rm TODO README INSTALL CONTRIBUTING
856
	sudo chown jenkins.jenkins /var/lib/jenkins/userContent/*html
Holger Levsen's avatar
Holger Levsen committed
857
	explain "user content for jenkins updated."
858
fi
859

860
if [ "$HOSTNAME" = "jenkins" ] ; then
861
862
863
864
865
866
	#
	# run jenkins-job-builder to update jobs if needed
	#     (using sudo because /etc/jenkins_jobs is root:root 700)
	#
	cd /srv/jenkins/job-cfg
	for metaconfig in *.yaml.py ; do
Holger Levsen's avatar
Holger Levsen committed
867
868
869
870
871
872
		if [ -f $metaconfig ] ; then
			TMPFILE=$(sudo -u jenkins-adm mktemp)
			./$metaconfig | sudo -u jenkins-adm tee "$TMPFILE" >/dev/null
			if ! sudo -u jenkins-adm cmp -s ${metaconfig%.py} "$TMPFILE" ; then
				sudo -u jenkins-adm mv "$TMPFILE" "${metaconfig%.py}"
			fi
873
874
875
		fi
	done
	for config in *.yaml ; do
876
877
878
879
880
881
882
		# do update, if
		# no stamp file exist or
		# no .py file exists and config is newer than stamp or
		# a .py file exists and .py file is newer than stamp
		if [ ! -f $STAMP ] || \
		 ( [ ! -f $config.py ] && [ $config -nt $STAMP ] ) || \
		 ( [ -f $config.py ] && [ $config.py -nt $STAMP ] ) ; then
883
			echo "$config has changed, executing updates."
884
			$JJB update $config # FIXME: we should use sudo and run $JJB as jenkins or jenkins-adm
885
886
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
887
	explain "jenkins jobs updated."
888
fi
889

890
891
892
#
# generate the kgb-client configurations
#
893
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "osuosl168-amd64" ] || [ "$HOSTNAME" = "osuosl171-amd64" ] || [ "$HOSTNAME" = "osuosl172-amd64" ] || [ "$HOSTNAME" = "ionos2-i386" ] || [ "$HOSTNAME" = "ionos12-i386" ] ; then
894
895
896
	cd $BASEDIR
	KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
	if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
Philip Hands's avatar
Philip Hands committed
897
		# the last condition is to assure the files are owned by the right user/team
898
899
		if [ "$KGB_SECRETS" -nt $STAMP ] || [ "/srv/jenkins/bin/deploy_kgb.py" -nt "$STAMP" ] || [ ! -f $STAMP ] ; then
			sudo -u jenkins-adm "/srv/jenkins/bin/deploy_kgb.py"
Philip Hands's avatar
Philip Hands committed
900
901
902
		else
			explain "kgb-client configuration unchanged, nothing to do."
		fi
903
	else
Philip Hands's avatar
Philip Hands committed
904
905
906
		figlet -f banner Warning
		echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
		echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
Holger Levsen's avatar
Holger Levsen committed
907
908
		echo "/srv/jenkins/kgb should be mode 755 and owned by jenkins-adm:root."
		echo "/srv/jenkins/kgb/client-status should be mode 755 and owned by jenkins:jenkins."
909
	fi
910
	KGB_STATUS="/srv/jenkins/kgb/client-status"
911
912
	sudo mkdir -p $KGB_STATUS
	sudo chown jenkins:jenkins $KGB_STATUS
913
914
fi

915
916
917
#
# Create GPG key for jenkins user if they do not already exist (eg. to sign .buildinfo files)
#
918
if sudo -H -u jenkins gpg --with-colons --fixed-list-mode --list-secret-keys | cut -d: -f1 | grep -qsFx 'sec' >/dev/null 2>&1 ; then
Holger Levsen's avatar
Holger Levsen committed
919
	: # not generating GPG key as one already exists for jenkins user
920
else
Holger Levsen's avatar
Holger Levsen committed
921
	explain "$(date) - Generating GPG key for jenkins user."
922

923
	sudo -H -u jenkins gpg --no-tty --batch --gen-key <<EOF
924
925
926
Key-Type: RSA
Key-Length: 4096
Key-Usage: sign
927
Name-Real: $HOSTNAME
928
929
930
931
932
933
Name-Comment: Automatically generated key for signing .buildinfo files
Expire-Date: 0
%no-ask-passphrase
%no-protection
%commit
EOF
934

935
	GPG_KEY_ID="$(sudo -H -u jenkins gpg --with-colons --fixed-list-mode --list-secret-keys | grep '^sec' | cut -d: -f5 | tail -n1)"
936
937
938
939
940
941

	if [ "$GPG_KEY_ID" = "" ]
	then
		explain "$(date) - Generated GPG key but could not parse key ID"
	else
		explain "$(date) - Generated GPG key $GPG_KEY_ID - submitting to keyserver"
942
		sudo -H -u jenkins gpg --send-keys $GPG_KEY_ID
943
	fi
944
945
fi

946

947
#
948
# almost finally…
949
#
950
sudo touch $STAMP	# so on the next run, only configs newer than this file will be updated
Holger Levsen's avatar
Holger Levsen committed
951
explain "$(date) - finished deployment."
952

953
954
955
956
#
# some final checks only for the jenkins
#
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
957
	jenkins_bugs_check
Holger Levsen's avatar
Holger Levsen committed
958
	explain "$(date) - done checking for known jenkins bugs."
959
960
fi

Holger Levsen's avatar
Holger Levsen committed
961
#
962
# finally!
Holger Levsen's avatar
Holger Levsen committed
963
#
964
case $HOSTNAME in
965
	# set time back to the future
966
	ionos5-amd64|ionos6-i386|ionos15-amd64|ionos16-i386)
967
		disable_dsa_check_packages
968
969
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
970
	codethink9*|codethink11*|codethink13*|codethink15*)
971
		disable_dsa_check_packages
972
973
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
974
	osuosl170-amd64|osuosl172-amd64)
975
976
977
		disable_dsa_check_packages
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
978
	jenkins)
979
		# notify irc on updates of jenkins.d.n
980
981
		MESSAGE="jenkins.d.n updated to $(cd $BASEDIR ; git describe --always)."
		kgb-client --conf /srv/jenkins/kgb/debian-qa.conf --relay-msg "$MESSAGE"
Holger Levsen's avatar
Holger Levsen committed
982
		;;
983
984
	*)	;;
esac
985
986
987
988

echo
figlet ok
echo
Holger Levsen's avatar
Holger Levsen committed
989
990
echo "__$HOSTNAME=ok__"