update_jdn.sh 22.9 KB
Newer Older
1
#!/bin/bash
2
# Copyright 2012-2016 Holger Levsen <holger@layer-acht.org>
3
4
# released under the GPLv=2

5
6
set -e

7
BASEDIR="$(dirname "$(readlink -e $0)")"
8
9
PVNAME=/dev/vdb      # LVM physical volume for jobs
VGNAME=jenkins01     # LVM volume group
10
STAMP=/var/log/jenkins/update-jenkins.stamp
11
TMPFILE=$(mktemp)
12
13
14
# The $@ below means that command line args get passed on to j-j-b
# which allows one to specify --flush-cache or --ignore-cache
JJB="jenkins-job-builder $@"
15
DPKG_ARCH="$(dpkg --print-architecture)"
Holger Levsen's avatar
Holger Levsen committed
16

17
# so we can later run some commands only if $0 has been updated…
Philip Hands's avatar
Philip Hands committed
18
if [ -f $STAMP ] && [ $STAMP -nt $BASEDIR/$0 ] ; then
Holger Levsen's avatar
Holger Levsen committed
19
	UP2DATE=true
Philip Hands's avatar
Philip Hands committed
20
21
else
	UP2DATE=false
22
23
24
fi


Holger Levsen's avatar
Holger Levsen committed
25
explain() {
Holger Levsen's avatar
Holger Levsen committed
26
	echo "$HOSTNAME: $1"
Holger Levsen's avatar
Holger Levsen committed
27
28
}

29
30


31
echo "--------------------------------------------"
Holger Levsen's avatar
Holger Levsen committed
32
explain "$(date) - begin deployment update."
33

34
# some nodes need special treatment…
35
case $HOSTNAME in
36
	profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386)
Holger Levsen's avatar
Holger Levsen committed
37
38
39
		# set correct date
		sudo service ntp stop
		sudo ntpdate -b de.pool.ntp.org
40
41
42
43
		;;
	*)	;;
esac

44
45
46
#
# set up users and groups
#
47
if ! getent passwd jenkins > /dev/null ; then
48
	sudo addgroup --system jenkins
49
	sudo adduser --system --shell /bin/bash --home /var/lib/jenkins --ingroup jenkins --disabled-login jenkins
50
fi
Holger Levsen's avatar
Holger Levsen committed
51
if ! getent group jenkins-adm > /dev/null ; then
52
53
	sudo addgroup --system jenkins-adm
fi
54
if ! getent passwd jenkins-adm > /dev/null  ; then
Holger Levsen's avatar
Holger Levsen committed
55
	sudo adduser --system --shell /bin/bash --home /home/jenkins-adm --ingroup jenkins-adm --disabled-login jenkins-adm
Holger Levsen's avatar
Holger Levsen committed
56
	sudo usermod -G jenkins jenkins-adm
57
fi
Mattia Rizzolo's avatar
Mattia Rizzolo committed
58
if [ ! -d /home/jenkins-adm ]; then
Mattia Rizzolo's avatar
Mattia Rizzolo committed
59
60
61
    sudo mkdir /home/jenkins-adm
    sudo chown jenkins-adm.jenkins-adm /home/jenkins-adm
fi
62

63
64

declare -A user_host_groups u_shell
65
66

sudo_groups='jenkins,jenkins-adm,sudo,adm'
67
68
69
70
71
72
73
74
75

# if there's a need for host groups, a case statement on $HOSTNAME here that sets $GROUPNAME, say, should do the trick
# then you can define user_host_groups['phil','lvm_group']=... below
# and add checks for the GROUP version whereever the HOSTNAME is checked in the following code

user_host_groups['helmut','*']="$sudo_groups"
user_host_groups['holger','*']="$sudo_groups"
user_host_groups['holger','jenkins']="reproducible,${user_host_groups['holger','*']}"
user_host_groups['mattia','*']="$sudo_groups"
76
user_host_groups['mattia','jenkins']="reproducible,${user_host_groups['mattia','*']}"
77
user_host_groups['phil','jenkins-test-vm']="$sudo_groups,libvirt,libvirt-qemu"
Philip Hands's avatar
Philip Hands committed
78
user_host_groups['phil','profitbricks-build10-amd64']="$sudo_groups"
79
user_host_groups['phil','jenkins']="$sudo_groups"
80
user_host_groups['lunar','jenkins']='reproducible'
81
82
user_host_groups['lynxis','profitbricks-build3-amd64']="$sudo_groups"
user_host_groups['lynxis','profitbricks-build4-amd64']="$sudo_groups"
83
84
user_host_groups['vagrant','armhf']="$sudo_groups"
user_host_groups['vagrant','arm64']="$sudo_groups"
85

86
87

u_shell['mattia']='/bin/zsh'
88
u_shell['mattia']='/bin/fish'
89
u_shell['jenkins-adm']='/bin/bash'
90

91
92
93
# get the users out of the user_host_groups array's index
users=$(for i in ${!user_host_groups[@]}; do echo ${i%,*} ; done | sort -u)

94
( $UP2DATE && [ -z $(find authorized_keys -newer $0) ] ) || for user in ${users}; do
95
	# -v is a bashism to check for set variables, used here to see if this user is active on this host
96
	[ -v user_host_groups["$user","$HOSTNAME"] -o -v user_host_groups["$user",'*'] -o -v user_host_groups["$user", "$DPKG_ARCH"]  ] || continue
97

98
99
100
101
102
103
	# create the user
	if ! getent passwd $user > /dev/null ; then
		# adduser, defaulting to /bin/bash as shell
		sudo adduser --gecos "" --shell "${u_shell[$user]:-/bin/bash}" --disabled-password $user
	fi
	# add groups: first try the specific host, or if unset fall-back to default '*' setting
104
	for h in "$HOSTNAME" "$DPKG_ARCH" '*' ; do
105
		if [ -v user_host_groups["$user","$h"] ] ; then
Holger Levsen's avatar
Holger Levsen committed
106
			sudo usermod -G "${user_host_groups["$user","$h"]}" $user
107
			break
108
		fi
109
110
111
112
113
	done
	# add the user's keys (if any)
	if ls authorized_keys/${user}@*.pub >/dev/null 2>&1 ; then
		[ -d /var/lib/misc/userkeys ] || sudo mkdir -p /var/lib/misc/userkeys
		cat authorized_keys/${user}@*.pub | sudo tee /var/lib/misc/userkeys/${user} > /dev/null
114
	fi
115
116
done

117
# change defaults
118
119
120
$UP2DATE || grep -q '^AuthorizedKeysFile' /etc/ssh/sshd_config || {
	sudo sh -c "echo 'AuthorizedKeysFile /var/lib/misc/userkeys/%u %h/.ssh/authorized_keys' >> /etc/ssh/sshd_config"
	sudo service ssh reload
121
}
122
123
124
125
126
# change vagrants manual configuration on some armhf hosts
$UP2DATE || grep -q '/var/lib/misc/userkeys' /etc/ssh/sshd_config || {
	sudo sed -i "s#/var/lib/monkeysphere/authorized_keys/#/var/lib/misc/userkeys/#g" /etc/ssh/sshd_config
	sudo service ssh reload
}
127

128
129
130
131
132
sudo mkdir -p /srv/workspace
[ -d /srv/schroots ] || sudo mkdir -p /srv/schroots
[ -h /chroots ] || sudo ln -s /srv/workspace/chroots /chroots
[ -h /schroots ] || sudo ln -s /srv/schroots /schroots

133
if [ "$HOSTNAME" = "jenkins-test-vm" ] || [ "$HOSTNAME" = "profitbricks-build10-amd64" ] || [ "$HOSTNAME" = "profitbricks-build7-amd64" ] ; then
134
	# jenkins needs access to libvirt
135
	sudo adduser jenkins kvm
136
137
138
	sudo adduser jenkins libvirt
	sudo adduser jenkins libvirt-qemu

139
	# we need a directory for the VM's storage pools
140
	VM_POOL_DIR=/srv/lvc/vm-pools
141
	if [ ! -d $VM_POOL_DIR ] ; then
142
		sudo mkdir -p $VM_POOL_DIR
143
144
145
		sudo chown jenkins:libvirt-qemu $VM_POOL_DIR
		sudo chmod 775 $VM_POOL_DIR
	fi
146
147
148
149
150
151

	# tidy up after ourselves, for a while at least
	OLD_VM_POOL_DIR=/srv/workspace/vm-pools
	if [ -d "$OLD_VM_POOL_DIR" ] ; then
		sudo rm -r "$OLD_VM_POOL_DIR"
	fi
152
fi
153

154
155
# prepare tmpfs on some hosts
case $HOSTNAME in
Mattia Rizzolo's avatar
Mattia Rizzolo committed
156
157
158
159
	jenkins)
		TMPFSSIZE=100
		TMPSIZE=15
		;;
Holger Levsen's avatar
Holger Levsen committed
160
	profitbricks-build9-amd64)
161
		TMPFSSIZE=40
Mattia Rizzolo's avatar
Mattia Rizzolo committed
162
163
		TMPSIZE=8
		;;
Holger Levsen's avatar
Holger Levsen committed
164
	profitbricks-build*)
Mattia Rizzolo's avatar
Mattia Rizzolo committed
165
166
167
		TMPFSSIZE=200
		TMPSIZE=15
		;;
168
169
170
	*) ;;
esac
case $HOSTNAME in
171
172
173
174
175
176
177
	profitbricks-build*i386)
		if ! grep -q '/srv/workspace' /etc/fstab; then
			echo "Warning: you need to manually create a /srv/workspace partition on i386 nodes, exiting."
			exit 1
		fi
		;;
	jenkins|profitbricks-build*amd64)
178
		if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
179
			echo "tmpfs		/srv/workspace	tmpfs	defaults,size=${TMPFSSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null  
180
		fi
Mattia Rizzolo's avatar
Mattia Rizzolo committed
181
182
183
		if ! grep -q '^tmpfs\s\+/tmp\s' /etc/fstab; then
			echo "tmpfs		/tmp	tmpfs	defaults,size=${TMPSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null
		fi
184
185
		if ! mountpoint -q /srv/workspace; then
			if test -z "$(ls -A /srv/workspace)"; then
186
				sudo mount /srv/workspace
187
			else
188
				explain "WARNING: mountpoint /srv/workspace is non-empty."
189
190
191
192
193
			fi
		fi
		;;
	*) ;;
esac
194
case $HOSTNAME in
195
	profitbricks-build10-amd64)
196
197
198
199
200
201
202
203
204
205
206
207
208
209
		[ -d /srv/lvc/vm-pools ] || sudo mkdir -p /srv/lvc/vm-pools
		if ! grep -q '^/dev/vdb\s\+/srv/lvc/vm-pools\s' /etc/fstab; then
			echo "/dev/vdb	/srv/lvc/vm-pools ext4	errors=remount-ro	0	2" | sudo tee -a /etc/fstab >/dev/null  
		fi
		if ! mountpoint -q /srv/lvc/vm-pools; then
			if test -z "$(ls -A /srv/lvc/vm-pools)"; then
				sudo mount /srv/lvc/vm-pools
			else
				explain "WARNING: mountpoint /srv/lvc/vm-pools is non-empty."
			fi
		fi
		;;
	*) ;;
esac
210

211
# make sure needed directories exists - some directories will not be needed on all hosts...
212
for directory in /schroots /srv/reproducible-results /srv/d-i /srv/udebs /srv/live-build /var/log/jenkins/ /srv/jenkins /srv/jenkins/pseudo-hosts /srv/workspace/chroots ; do
213
214
215
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
	fi
Holger Levsen's avatar
Holger Levsen committed
216
	sudo chown jenkins.jenkins $directory
217
done
Holger Levsen's avatar
Holger Levsen committed
218
for directory in /srv/jenkins ; do
219
220
221
222
223
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
		sudo chown jenkins-adm.jenkins-adm $directory
	fi
done
224

225
if ! test -h /chroots; then
Holger Levsen's avatar
Holger Levsen committed
226
	sudo rmdir /chroots || sudo rm -f /chroots # do not recurse
227
	if test -e /chroots; then
Holger Levsen's avatar
Holger Levsen committed
228
		explain "/chroots could not be cleared."
229
	else
230
		sudo ln -s /srv/workspace/chroots /chroots
231
232
233
	fi
fi

234
235
236
237
238
# only on Debian systems
if [ -f /etc/debian_version ] ; then
	#
	# install packages we need
	#
239
	if [ $BASEDIR/$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then
Holger Levsen's avatar
Holger Levsen committed
240
241
		DEBS=" 
			bash-completion 
242
243
			bc
			bsd-mailx
Holger Levsen's avatar
Holger Levsen committed
244
			curl 
245
			debootstrap 
246
247
			devscripts
			eatmydata
248
			etckeeper
249
			figlet
Holger Levsen's avatar
Holger Levsen committed
250
			git
251
			haveged
Holger Levsen's avatar
Holger Levsen committed
252
			htop
253
			less
254
			locales-all
Mattia Rizzolo's avatar
Mattia Rizzolo committed
255
			lsof
256
257
			molly-guard
			moreutils
258
			munin-node
259
260
			munin-plugins-extra
			netcat-traditional
261
262
			ntp
			ntpdate
263
			pigz 
264
			postfix
265
			procmail
266
			psmisc
267
			python3-psycopg2 
Holger Levsen's avatar
Holger Levsen committed
268
			schroot 
269
270
			screen
			slay
271
			stunnel
Holger Levsen's avatar
Holger Levsen committed
272
273
274
275
276
			subversion 
			subversion-tools 
			sudo 
			unzip 
			vim 
277
			zsh
278
			"
279
		# install squid everywhere except on the armhf nodes
280
		case $HOSTNAME in
281
			jenkins|jenkins-test-vm|profitbricks-build*|codethink*) DEBS="$DEBS
282
				squid3
283
284
				kgb-client
				python3-yaml" ;;
285
286
			*) ;;
		esac
287
288
		# needed to run the 2nd reproducible builds nodes in the future...
		case $HOSTNAME in
289
			profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386) DEBS="$DEBS ntpdate" ;;
290
291
			*) ;;
		esac
292
		# needed to run coreboot/openwrt/lede/netbsd/fedora jobs
293
		case $HOSTNAME in
294
			profitbricks-build3-amd64|profitbricks-build4-amd64) DEBS="$DEBS
Holger Levsen's avatar
Holger Levsen committed
295
				bison
296
				ca-certificates
Holger Levsen's avatar
Holger Levsen committed
297
298
299
				cmake
				diffutils
				findutils
300
				fish
Holger Levsen's avatar
Holger Levsen committed
301
302
303
304
305
306
307
308
309
310
311
312
313
314
				flex
				g++
				gawk
				gcc
				git
				grep
				iasl
				libc6-dev
				libncurses5-dev
				libssl-dev
				locales-all
				kgb-client
				m4
				make
315
316
				python3-clint
				python3-git
317
				python3-pystache
318
				python3-requests
Holger Levsen's avatar
Holger Levsen committed
319
320
				python3-yaml
				subversion
321
				sysvinit-core
322
				tree
Holger Levsen's avatar
Holger Levsen committed
323
324
325
				unzip
				util-linux
				zlib1g-dev"
326
327
328
			;;
			*) ;;
		esac
329
330
331
		# needed to run fdroid jobs
		case $HOSTNAME in
			profitbricks-build7-amd64) DEBS="$DEBS
332
				libvirt-clients
333
334
				libvirt-daemon
				libvirt-daemon-system
335
				python3-libvirt
336
				python3-vagrant
337
				qemu-kvm
338
339
340
				vagrant
				vagrant-mutate
				vagrant-libvirt"
341
			;;
342
343
			*) ;;
		esac
344
		# cucumber dependencies (for lvc jobs)
345
346
347
348
349
		case $HOSTNAME in
			profitbricks-build10-amd64|jenkins-test-vm) DEBS="$DEBS
				cucumber
				tesseract-ocr
				i18nspector
Holger Levsen's avatar
Holger Levsen committed
350
				imagemagick
351
352
				libav-tools
				libsikuli-script-java
353
				libvirt-bin
354
355
356
357
				libvirt-dev
				ovmf
				python-jabberbot
				python-potr
358
				python3-yaml
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
				ruby-guestfs
				ruby-libvirt
				ruby-net-irc
				ruby-packetfu
				ruby-rb-inotify
				ruby-rjb
				ruby-test-unit
				tcpdump
				unclutter
				virt-viewer
				x264
				xvfb
				x11vnc"
			   ;;
			*) ;;
		esac
375
376
377
378
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			# for phil only
			DEBS="$DEBS postfix-pcre"
		fi
379
		if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
Holger Levsen's avatar
Holger Levsen committed
380
381
382
383
			MASTERDEBS=" 
				apache2 
				apt-file 
				apt-listchanges 
384
				asciidoc
Holger Levsen's avatar
Holger Levsen committed
385
386
387
388
389
390
391
392
393
394
				binfmt-support 
				bison 
				build-essential 
				calamaris 
				cmake 
				cron-apt 
				csvtool 
				dnsmasq-base 
				dstat 
				figlet 
Holger Levsen's avatar
Holger Levsen committed
395
				flex
Holger Levsen's avatar
Holger Levsen committed
396
				gawk 
397
398
				ghc
				git-notifier 
Holger Levsen's avatar
Holger Levsen committed
399
400
401
402
				gocr 
				graphviz 
				iasl 
				imagemagick 
403
404
				ip2host
				jekyll
405
				kgb-client
Holger Levsen's avatar
Holger Levsen committed
406
407
408
409
410
411
412
413
414
415
				libapache2-mod-macro 
				libcap2-bin 
				libfile-touch-perl 
				libguestfs-tools 
				libjson-rpc-perl 
				libsoap-lite-perl 
				libvirt0 
				libvirt-bin 
				libvpx1 
				libxslt1-dev 
416
				linux-image-amd64
Holger Levsen's avatar
Holger Levsen committed
417
418
419
420
421
422
423
424
				moreutils 
				mr 
				mtr-tiny 
				munin 
				ntp 
				openbios-ppc 
				openbios-sparc 
				openjdk-7-jre 
425
				pandoc
426
				postgresql
Holger Levsen's avatar
Holger Levsen committed
427
428
429
430
				postgresql-client-9.4 
				poxml 
				procmail 
				python3-debian 
431
				python3-pystache
432
433
434
				python3-sqlalchemy
				python3-xdg
				python3-yaml
Holger Levsen's avatar
Holger Levsen committed
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
				python-arpy 
				python-hachoir-metadata 
				python-imaging 
				python-lzma 
				python-pip 
				python-rpy2 
				python-setuptools 
				python-twisted 
				python-yaml 
				qemu 
				qemu-kvm 
				qemu-system-x86 
				qemu-user-static 
				radvd 
				ruby-json 
				ruby-rspec 
				seabios 
				shorewall 
				shorewall6 
				sqlite3 
				syslinux 
456
				tor
Holger Levsen's avatar
Holger Levsen committed
457
				vncsnapshot 
458
459
				vnstat
				whohas
Holger Levsen's avatar
Holger Levsen committed
460
				x11-apps 
461
				xtightvncviewer
462
				xvfb
463
				xvkbd
464
				zutils"
465
466
467
		else
			MASTERDEBS=""
		fi
468
		$UP2DATE || sudo apt-get update
469
		$UP2DATE || sudo apt-get install $DEBS $MASTERDEBS
470
		case $HOSTNAME in
471
472
			codethink*) 	$UP2DATE || sudo apt-get install pbuilder lintian
					;;
473
474
475
476
			*)		$UP2DATE || sudo apt-get install -t jessie-backports \
						pbuilder lintian || echo "this should only fail on the first install"
					;;
		esac
477
		# we need mock from bpo to build current fedora
478
479
		if [ "$HOSTNAME" = "profitbricks-build3-amd64" ] || [ "$HOSTNAME" = "profitbricks-build4-amd64" ] || [ "$HOSTNAME" = "jenkins" ] ; then
			$UP2DATE || sudo apt-get install -t jessie-backports mock \
480
				|| echo "this should only fail on the first install"
481
		fi
482
		# for varying kernels
483
		# we use bpo kernels on pb-build5+6+15+16 (and the default i386 kernel on pb-build2+12-i386)
484
485
		if [ "$HOSTNAME" = "profitbricks-build5-amd64" ] || [ "$HOSTNAME" = "profitbricks-build6-i386" ] || \
		   [ "$HOSTNAME" = "profitbricks-build15-amd64" ] || [ "$HOSTNAME" = "profitbricks-build16-i386" ] ; then
486
			$UP2DATE || sudo apt-get install -t jessie-backports linux-image-amd64 || echo "this should only fail on the first install"
487
		fi
488
		# only needed on the main node
Holger Levsen's avatar
Holger Levsen committed
489
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
490
			$UP2DATE || sudo apt-get install -t jessie-backports jenkins-job-builder || echo "this should only fail on the first install"
Holger Levsen's avatar
Holger Levsen committed
491
		elif [ "$HOSTNAME" = "jenkins" ] ; then
492
			$UP2DATE || sudo apt-get install -t jessie-backports ffmpeg libav-tools python3-popcon jenkins-job-builder dose-extra
493
		fi
Holger Levsen's avatar
Holger Levsen committed
494
		explain "packages installed."
495
	else
Holger Levsen's avatar
Holger Levsen committed
496
		explain "no new packages to be installed."
497
	fi
498
fi
499

500
#
501
# deploy package configuration in /etc and /usr/local/
502
#
Holger Levsen's avatar
Holger Levsen committed
503
cd $BASEDIR
504
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/etc/* /etc
505
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/usr/* /usr/
506
507
508
509
510

#
# more configuration than a simple cp can do
#
sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins
511
sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm
512
513

if [ "$HOSTNAME" = "jenkins" ] ; then
514
515
516
517
518
519
520
521
522
523
524
525
	if [ $BASEDIR/hosts/$HOSTNAME/etc/apache2 -nt $STAMP ] || [ ! -f $STAMP ] ; then
		if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
			sudo a2enmod proxy
			sudo a2enmod proxy_http
			sudo a2enmod rewrite
			sudo a2enmod ssl
			sudo a2enmod headers
			sudo a2enmod macro
			sudo a2enmod filter
		fi
		sudo a2ensite -q jenkins.debian.net
		sudo a2enconf -q munin
526
		sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.debian.net.conf
527
528
529
		# for reproducible.d.n url rewriting:
		[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
		sudo service apache2 reload
530
531
532
	fi
fi

533
if [ $BASEDIR/hosts/$HOSTNAME/etc/munin -nt $STAMP ] || [ ! -f $STAMP ] ; then
534
	cd /etc/munin/plugins
535
	sudo rm -f postfix_* open_inodes interrupts irqstats threads proc_pri vmstat if_err_* exim_* netstat fw_forwarded_local fw_packets forks open_files users nfs* iostat_ios 2>/dev/null
536
	case $HOSTNAME in
Holger Levsen's avatar
Holger Levsen committed
537
			jenkins|profitbricks-build*) [ -L /etc/munin/plugins/squid_cache ] || for i in squid_cache squid_objectsize squid_requests squid_traffic ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
538
539
			*)	;;
	esac
Holger Levsen's avatar
Holger Levsen committed
540
541
542
543
	case $HOSTNAME in
			jenkins) [ -L /etc/munin/plugins/postfix_mailstats ] || for i in postfix_mailstats postfix_mailvolume postfix_mailqueue ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
			*)	;;
	esac
544
545
546
	if [ "$HOSTNAME" != "jenkins" ] && [ -L /etc/munin/plugins/iostat ] ; then
		sudo rm /etc/munin/plugins/iostat
	fi
547
548
	if [ "$HOSTNAME" = "jenkins" ] && [ ! -L /etc/munin/plugins/apache_accesses ] ; then
		for i in apache_accesses apache_volume ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done
549
		sudo ln -s /usr/share/munin/plugins/loggrep jenkins_oom
550
	fi
551
fi
Holger Levsen's avatar
Holger Levsen committed
552
explain "packages configured."
Holger Levsen's avatar
Holger Levsen committed
553
554
555
556

#
# install the heart of jenkins.debian.net
#
Holger Levsen's avatar
Holger Levsen committed
557
cd $BASEDIR
558
[ -d /srv/jenkins/features ] && sudo rm -rf /srv/jenkins/features
559
for dir in bin logparse cucumber live mustache-templates ; do
560
561
	sudo mkdir -p /srv/jenkins/$dir
	sudo rsync -rpt --delete $dir/ /srv/jenkins/$dir/
Holger Levsen's avatar
Holger Levsen committed
562
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
563
done
564
565
HOST_JOBS="hosts/$HOSTNAME/job-cfg"
if [ -e "$HOST_JOBS" ] ; then
566
	sudo rsync -rpt --copy-links --delete "$HOST_JOBS/" /srv/jenkins/job-cfg/
567
568
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
else
Philip Hands's avatar
Philip Hands committed
569
570
	# tidying up ... assuming that we don't want clutter on peripheral servers
	[ -d /srv/jenkins/job-cfg ] && sudo rm -rf /srv/jenkins/job-cfg
571
572
fi

573

Holger Levsen's avatar
Holger Levsen committed
574
sudo mkdir -p /var/lib/jenkins/.ssh
575
576
577
578
579
580
if [ "$HOSTNAME" = "jenkins" ] ; then
	sudo cp jenkins-home/procmailrc /var/lib/jenkins/.procmailrc
	sudo cp jenkins-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
else
	sudo cp jenkins-nodes-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
fi
Holger Levsen's avatar
Holger Levsen committed
581
582
583
sudo chown -R jenkins:jenkins /var/lib/jenkins/.ssh
sudo chmod 700 /var/lib/jenkins/.ssh
sudo chmod 600 /var/lib/jenkins/.ssh/authorized_keys
Holger Levsen's avatar
Holger Levsen committed
584
explain "scripts and configurations for jenkins updated."
585

586
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
587
	sudo cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
588
	git log | grep ^Author| cut -d " " -f2-|sort -u -f > $TMPFILE
589
	echo "----" >> $TMPFILE
590
	sudo tee /var/lib/jenkins/userContent/THANKS > /dev/null < THANKS.head
Holger Levsen's avatar
Holger Levsen committed
591
592
	# samuel, lunar, josch and phil committed with several commiters, only display one
	grep -v -e "samuel.thibault@ens-lyon.org" -e Lunar -e "j.schauer@email.de" -e "mattia@mapreri.org" -e "phil@jenkins-test-vm" $TMPFILE | sudo tee -a /var/lib/jenkins/userContent/THANKS > /dev/null
593
	rm $TMPFILE
594
	TMPDIR=$(mktemp -d -t update-jdn-XXXXXXXX)
595
	sudo cp -pr userContent $TMPDIR/
Holger Levsen's avatar
Holger Levsen committed
596
597
598
	sudo chown -R jenkins.jenkins $TMPDIR
	sudo cp -pr $TMPDIR/userContent  /var/lib/jenkins/
	sudo rm -r $TMPDIR > /dev/null
599
600
601
602
603
604
605
606
607
	cd /var/lib/jenkins/userContent/
	ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
	[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
	[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
	[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
	[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
	diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
	mv THANKS .THANKS
	rm TODO README INSTALL CONTRIBUTING
608
	sudo chown jenkins.jenkins /var/lib/jenkins/userContent/*html
Holger Levsen's avatar
Holger Levsen committed
609
	explain "user content for jenkins updated."
610
fi
611

612
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
613
614
615
616
617
618
619
	#
	# run jenkins-job-builder to update jobs if needed
	#     (using sudo because /etc/jenkins_jobs is root:root 700)
	#
	cd /srv/jenkins/job-cfg
	for metaconfig in *.yaml.py ; do
	# there are both python2 and python3 scripts here
Philip Hands's avatar
Philip Hands committed
620
		[ -e ./$metaconfig ] || continue
621
		./$metaconfig > $TMPFILE
622
623
		if ! sudo -u jenkins-adm cmp -s ${metaconfig%.py} - < $TMPFILE ; then
			sudo -u jenkins-adm tee ${metaconfig%.py} > /dev/null < $TMPFILE
624
625
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
626
	rm -f $TMPFILE
627
	for config in *.yaml ; do
628
629
630
631
632
633
634
		# do update, if
		# no stamp file exist or
		# no .py file exists and config is newer than stamp or
		# a .py file exists and .py file is newer than stamp
		if [ ! -f $STAMP ] || \
		 ( [ ! -f $config.py ] && [ $config -nt $STAMP ] ) || \
		 ( [ -f $config.py ] && [ $config.py -nt $STAMP ] ) ; then
635
			$JJB update $config
636
637
638
639
		else
			echo "$config has not changed, nothing to do."
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
640
	explain "jenkins jobs updated."
641
fi
642

643
#
Holger Levsen's avatar
Holger Levsen committed
644
# configure git for jenkins
645
#
Holger Levsen's avatar
Holger Levsen committed
646
647
648
649
if [ "$(sudo su - jenkins -c 'git config --get user.email')" != "jenkins@jenkins.debian.net" ] ; then
	sudo su - jenkins -c "git config --global user.email jenkins@jenkins.debian.net"
	sudo su - jenkins -c "git config --global user.name Jenkins"
fi
Holger Levsen's avatar
Holger Levsen committed
650

651
652
653
654
655
if [ "$HOSTNAME" = "jenkins" ] ; then
	#
	# creating LVM volume group for jobs
	#
	if [ "$PVNAME" = "" ]; then
Philip Hands's avatar
Philip Hands committed
656
657
658
		figlet -f banner Error
		explain "you must set \$PVNAME to physical volume pathname, exiting."
		exit 1
Holger Levsen's avatar
Holger Levsen committed
659
	elif ! $UP2DATE ; then
Philip Hands's avatar
Philip Hands committed
660
661
662
663
664
665
		if ! sudo pvs $PVNAME >/dev/null 2>&1; then
			sudo pvcreate $PVNAME
		fi
		if ! sudo vgs $VGNAME >/dev/null 2>&1; then
			sudo vgcreate $VGNAME $PVNAME
		fi
666
	fi
667
668
669
670
671
fi

#
# generate the kgb-client configurations
#
672
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "profitbricks-build3-amd64" ] || [ "$HOSTNAME" = "profitbricks-build4-amd64" ] || [ "$HOSTNAME" = "profitbricks-build7-amd64" ] || [ "$HOSTNAME" = "profitbricks-build2-i386" ] || [ "$HOSTNAME" = "profitbricks-build12-i386" ] ; then
673
674
675
	cd $BASEDIR
	KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
	if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
Philip Hands's avatar
Philip Hands committed
676
		# the last condition is to assure the files are owned by the right user/team
677
		if [ "$KGB_SECRETS" -nt $STAMP ] || [ "deploy_kgb.py" -nt "$STAMP" ] || [ ! -f $STAMP ] ; then
Philip Hands's avatar
Philip Hands committed
678
679
680
681
			sudo -u jenkins-adm "./deploy_kgb.py"
		else
			explain "kgb-client configuration unchanged, nothing to do."
		fi
682
	else
Philip Hands's avatar
Philip Hands committed
683
684
685
		figlet -f banner Warning
		echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
		echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
Holger Levsen's avatar
Holger Levsen committed
686
687
		echo "/srv/jenkins/kgb should be mode 755 and owned by jenkins-adm:root."
		echo "/srv/jenkins/kgb/client-status should be mode 755 and owned by jenkins:jenkins."
688
	fi
689
690
fi

691
692
693
#
# Create GPG key for jenkins user if they do not already exist (eg. to sign .buildinfo files)
#
694
if sudo -H -u jenkins gpg --with-colons --fixed-list-mode --list-secret-keys | cut -d: -f1 | grep -qsFx 'sec' >/dev/null 2>&1 ; then
Holger Levsen's avatar
Holger Levsen committed
695
	explain "$(date) - Not generating GPG key as one already exists for jenkins user."
696
else
Holger Levsen's avatar
Holger Levsen committed
697
	explain "$(date) - Generating GPG key for jenkins user."
698

699
	sudo -H -u jenkins gpg --no-tty --batch --gen-key <<EOF
700
701
702
Key-Type: RSA
Key-Length: 4096
Key-Usage: sign
703
Name-Real: $HOSTNAME
704
705
706
707
708
709
Name-Comment: Automatically generated key for signing .buildinfo files
Expire-Date: 0
%no-ask-passphrase
%no-protection
%commit
EOF
710

711
	GPG_KEY_ID="$(sudo -H -u jenkins gpg --with-colons --fixed-list-mode --list-secret-keys | grep '^sec' | cut -d: -f5 | tail -n1)"
712
713
714
715
716
717

	if [ "$GPG_KEY_ID" = "" ]
	then
		explain "$(date) - Generated GPG key but could not parse key ID"
	else
		explain "$(date) - Generated GPG key $GPG_KEY_ID - submitting to keyserver"
718
		sudo -H -u jenkins gpg --send-keys $GPG_KEY_ID
719
	fi
720
721
fi

Holger Levsen's avatar
Holger Levsen committed
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
#
# There's always some work left...
#	echo FIXME is ignored so check-jobs scripts can output templates requiring manual work
#
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
	rgrep FI[X]ME $BASEDIR/* | grep -v echo > $TMPFILE || true
	if [ -s $TMPFILE ] ; then
		echo
		# only show cucumber FIXMEs when deploying on jenkins-test-vm
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			cat $TMPFILE
		else
			cat $TMPFILE | grep -v cucumber
		fi
		echo
	fi
	rm -f $TMPFILE
fi

741
#
742
# almost finally…
743
#
744
sudo touch $STAMP	# so on the next run, only configs newer than this file will be updated
Holger Levsen's avatar
Holger Levsen committed
745
explain "$(date) - finished deployment."
746

747
# finally!
748
case $HOSTNAME in
749
	# set time back to the future
750
	profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386)
751
752
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
753
	jenkins)
754
		# notify irc on updates of jenkins.d.n
755
756
		MESSAGE="jenkins.d.n updated to $(cd $BASEDIR ; git describe --always)."
		kgb-client --conf /srv/jenkins/kgb/debian-qa.conf --relay-msg "$MESSAGE"
Holger Levsen's avatar
Holger Levsen committed
757
		;;
758
759
	*)	;;
esac
760
761
762
763

echo
figlet ok
echo
Holger Levsen's avatar
Holger Levsen committed
764
765
echo "__$HOSTNAME=ok__"