update_jdn.sh 22.6 KB
Newer Older
1
#!/bin/bash
2
# Copyright 2012-2016 Holger Levsen <holger@layer-acht.org>
3
4
# released under the GPLv=2

5
6
set -e

7
BASEDIR="$(dirname "$(readlink -e $0)")"
8
9
PVNAME=/dev/vdb      # LVM physical volume for jobs
VGNAME=jenkins01     # LVM volume group
10
STAMP=/var/log/jenkins/update-jenkins.stamp
11
TMPFILE=$(mktemp)
12
13
14
# The $@ below means that command line args get passed on to j-j-b
# which allows one to specify --flush-cache or --ignore-cache
JJB="jenkins-job-builder $@"
Holger Levsen's avatar
Holger Levsen committed
15

16
# so we can later run some commands only if $0 has been updated…
Philip Hands's avatar
Philip Hands committed
17
if [ -f $STAMP ] && [ $STAMP -nt $BASEDIR/$0 ] ; then
Holger Levsen's avatar
Holger Levsen committed
18
	UP2DATE=true
Philip Hands's avatar
Philip Hands committed
19
20
else
	UP2DATE=false
21
22
23
fi


Holger Levsen's avatar
Holger Levsen committed
24
explain() {
Holger Levsen's avatar
Holger Levsen committed
25
	echo "$HOSTNAME: $1"
Holger Levsen's avatar
Holger Levsen committed
26
27
}

28
29


30
echo "--------------------------------------------"
Holger Levsen's avatar
Holger Levsen committed
31
explain "$(date) - begin deployment update."
32

33
# some nodes need special treatment…
34
case $HOSTNAME in
35
	profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386)
Holger Levsen's avatar
Holger Levsen committed
36
37
38
		# set correct date
		sudo service ntp stop
		sudo ntpdate -b de.pool.ntp.org
39
40
41
42
		;;
	*)	;;
esac

43
44
45
#
# set up users and groups
#
46
if ! getent passwd jenkins > /dev/null ; then
47
	sudo addgroup --system jenkins
48
	sudo adduser --system --shell /bin/bash --home /var/lib/jenkins --ingroup jenkins --disabled-login jenkins
49
fi
Holger Levsen's avatar
Holger Levsen committed
50
if ! getent group jenkins-adm > /dev/null ; then
51
52
	sudo addgroup --system jenkins-adm
fi
53
if ! getent passwd jenkins-adm > /dev/null  ; then
Holger Levsen's avatar
Holger Levsen committed
54
	sudo adduser --system --shell /bin/bash --home /home/jenkins-adm --ingroup jenkins-adm --disabled-login jenkins-adm
Holger Levsen's avatar
Holger Levsen committed
55
	sudo usermod -G jenkins jenkins-adm
56
fi
Mattia Rizzolo's avatar
Mattia Rizzolo committed
57
if [ ! -d /home/jenkins-adm ]; then
Mattia Rizzolo's avatar
Mattia Rizzolo committed
58
59
60
    sudo mkdir /home/jenkins-adm
    sudo chown jenkins-adm.jenkins-adm /home/jenkins-adm
fi
61

62
63

declare -A user_host_groups u_shell
64
65

sudo_groups='jenkins,jenkins-adm,sudo,adm'
66
67
68
69
70
71
72
73
74

# if there's a need for host groups, a case statement on $HOSTNAME here that sets $GROUPNAME, say, should do the trick
# then you can define user_host_groups['phil','lvm_group']=... below
# and add checks for the GROUP version whereever the HOSTNAME is checked in the following code

user_host_groups['helmut','*']="$sudo_groups"
user_host_groups['holger','*']="$sudo_groups"
user_host_groups['holger','jenkins']="reproducible,${user_host_groups['holger','*']}"
user_host_groups['mattia','*']="$sudo_groups"
75
user_host_groups['mattia','jenkins']="reproducible,${user_host_groups['mattia','*']}"
76
user_host_groups['phil','jenkins-test-vm']="$sudo_groups,libvirt,libvirt-qemu"
Philip Hands's avatar
Philip Hands committed
77
user_host_groups['phil','profitbricks-build10-amd64']="$sudo_groups"
78
user_host_groups['phil','jenkins']="$sudo_groups"
79
user_host_groups['lunar','jenkins']='reproducible'
80
81
user_host_groups['lynxis','profitbricks-build3-amd64']="$sudo_groups"
user_host_groups['lynxis','profitbricks-build4-amd64']="$sudo_groups"
82

83
84

u_shell['mattia']='/bin/zsh'
85
u_shell['mattia']='/bin/fish'
86
u_shell['jenkins-adm']='/bin/bash'
87

88
89
90
# get the users out of the user_host_groups array's index
users=$(for i in ${!user_host_groups[@]}; do echo ${i%,*} ; done | sort -u)

91
( $UP2DATE && [ -z $(find authorized_keys -newer $0) ] ) || for user in ${users}; do
92
93
	# -v is a bashism to check for set variables, used here to see if this user is active on this host
	[ -v user_host_groups["$user","$HOSTNAME"] -o -v user_host_groups["$user",'*'] ] || continue
94

95
96
97
98
99
100
101
102
	# create the user
	if ! getent passwd $user > /dev/null ; then
		# adduser, defaulting to /bin/bash as shell
		sudo adduser --gecos "" --shell "${u_shell[$user]:-/bin/bash}" --disabled-password $user
	fi
	# add groups: first try the specific host, or if unset fall-back to default '*' setting
	for h in "$HOSTNAME" '*' ; do
		if [ -v user_host_groups["$user","$h"] ] ; then
Holger Levsen's avatar
Holger Levsen committed
103
			sudo usermod -G "${user_host_groups["$user","$h"]}" $user
104
			break
105
		fi
106
107
108
109
110
	done
	# add the user's keys (if any)
	if ls authorized_keys/${user}@*.pub >/dev/null 2>&1 ; then
		[ -d /var/lib/misc/userkeys ] || sudo mkdir -p /var/lib/misc/userkeys
		cat authorized_keys/${user}@*.pub | sudo tee /var/lib/misc/userkeys/${user} > /dev/null
111
	fi
112
113
done

114
# change defaults
115
116
117
$UP2DATE || grep -q '^AuthorizedKeysFile' /etc/ssh/sshd_config || {
	sudo sh -c "echo 'AuthorizedKeysFile /var/lib/misc/userkeys/%u %h/.ssh/authorized_keys' >> /etc/ssh/sshd_config"
	sudo service ssh reload
118
}
119
120
121
122
123
# change vagrants manual configuration on some armhf hosts
$UP2DATE || grep -q '/var/lib/misc/userkeys' /etc/ssh/sshd_config || {
	sudo sed -i "s#/var/lib/monkeysphere/authorized_keys/#/var/lib/misc/userkeys/#g" /etc/ssh/sshd_config
	sudo service ssh reload
}
124

125
126
127
128
129
sudo mkdir -p /srv/workspace
[ -d /srv/schroots ] || sudo mkdir -p /srv/schroots
[ -h /chroots ] || sudo ln -s /srv/workspace/chroots /chroots
[ -h /schroots ] || sudo ln -s /srv/schroots /schroots

130
if [ "$HOSTNAME" = "jenkins-test-vm" ] || [ "$HOSTNAME" = "profitbricks-build10-amd64" ] || [ "$HOSTNAME" = "profitbricks-build7-amd64" ] ; then
131
	# jenkins needs access to libvirt
132
	sudo adduser jenkins kvm
133
134
135
	sudo adduser jenkins libvirt
	sudo adduser jenkins libvirt-qemu

136
	# we need a directory for the VM's storage pools
137
	VM_POOL_DIR=/srv/lvc/vm-pools
138
	if [ ! -d $VM_POOL_DIR ] ; then
139
		sudo mkdir -p $VM_POOL_DIR
140
141
142
		sudo chown jenkins:libvirt-qemu $VM_POOL_DIR
		sudo chmod 775 $VM_POOL_DIR
	fi
143
144
145
146
147
148

	# tidy up after ourselves, for a while at least
	OLD_VM_POOL_DIR=/srv/workspace/vm-pools
	if [ -d "$OLD_VM_POOL_DIR" ] ; then
		sudo rm -r "$OLD_VM_POOL_DIR"
	fi
149
fi
150

151
152
# prepare tmpfs on some hosts
case $HOSTNAME in
Mattia Rizzolo's avatar
Mattia Rizzolo committed
153
154
155
156
	jenkins)
		TMPFSSIZE=100
		TMPSIZE=15
		;;
Holger Levsen's avatar
Holger Levsen committed
157
	profitbricks-build9-amd64)
158
		TMPFSSIZE=40
Mattia Rizzolo's avatar
Mattia Rizzolo committed
159
160
		TMPSIZE=8
		;;
Holger Levsen's avatar
Holger Levsen committed
161
	profitbricks-build*)
Mattia Rizzolo's avatar
Mattia Rizzolo committed
162
163
164
		TMPFSSIZE=200
		TMPSIZE=15
		;;
165
166
167
	*) ;;
esac
case $HOSTNAME in
168
169
170
171
172
173
174
	profitbricks-build*i386)
		if ! grep -q '/srv/workspace' /etc/fstab; then
			echo "Warning: you need to manually create a /srv/workspace partition on i386 nodes, exiting."
			exit 1
		fi
		;;
	jenkins|profitbricks-build*amd64)
175
		if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
176
			echo "tmpfs		/srv/workspace	tmpfs	defaults,size=${TMPFSSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null  
177
		fi
Mattia Rizzolo's avatar
Mattia Rizzolo committed
178
179
180
		if ! grep -q '^tmpfs\s\+/tmp\s' /etc/fstab; then
			echo "tmpfs		/tmp	tmpfs	defaults,size=${TMPSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null
		fi
181
182
		if ! mountpoint -q /srv/workspace; then
			if test -z "$(ls -A /srv/workspace)"; then
183
				sudo mount /srv/workspace
184
			else
185
				explain "WARNING: mountpoint /srv/workspace is non-empty."
186
187
188
189
190
			fi
		fi
		;;
	*) ;;
esac
191
case $HOSTNAME in
192
	profitbricks-build10-amd64)
193
194
195
196
197
198
199
200
201
202
203
204
205
206
		[ -d /srv/lvc/vm-pools ] || sudo mkdir -p /srv/lvc/vm-pools
		if ! grep -q '^/dev/vdb\s\+/srv/lvc/vm-pools\s' /etc/fstab; then
			echo "/dev/vdb	/srv/lvc/vm-pools ext4	errors=remount-ro	0	2" | sudo tee -a /etc/fstab >/dev/null  
		fi
		if ! mountpoint -q /srv/lvc/vm-pools; then
			if test -z "$(ls -A /srv/lvc/vm-pools)"; then
				sudo mount /srv/lvc/vm-pools
			else
				explain "WARNING: mountpoint /srv/lvc/vm-pools is non-empty."
			fi
		fi
		;;
	*) ;;
esac
207

208
# make sure needed directories exists - some directories will not be needed on all hosts...
209
for directory in /schroots /srv/reproducible-results /srv/d-i /srv/udebs /srv/live-build /var/log/jenkins/ /srv/jenkins /srv/jenkins/pseudo-hosts /srv/workspace/chroots ; do
210
211
212
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
	fi
Holger Levsen's avatar
Holger Levsen committed
213
	sudo chown jenkins.jenkins $directory
214
done
Holger Levsen's avatar
Holger Levsen committed
215
for directory in /srv/jenkins ; do
216
217
218
219
220
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
		sudo chown jenkins-adm.jenkins-adm $directory
	fi
done
221

222
if ! test -h /chroots; then
Holger Levsen's avatar
Holger Levsen committed
223
	sudo rmdir /chroots || sudo rm -f /chroots # do not recurse
224
	if test -e /chroots; then
Holger Levsen's avatar
Holger Levsen committed
225
		explain "/chroots could not be cleared."
226
	else
227
		sudo ln -s /srv/workspace/chroots /chroots
228
229
230
	fi
fi

231
232
233
234
235
# only on Debian systems
if [ -f /etc/debian_version ] ; then
	#
	# install packages we need
	#
236
	if [ $BASEDIR/$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then
Holger Levsen's avatar
Holger Levsen committed
237
238
		DEBS=" 
			bash-completion 
239
240
			bc
			bsd-mailx
Holger Levsen's avatar
Holger Levsen committed
241
			curl 
242
			debootstrap 
243
244
			devscripts
			eatmydata
245
			etckeeper
246
			figlet
Holger Levsen's avatar
Holger Levsen committed
247
			git
248
			haveged
Holger Levsen's avatar
Holger Levsen committed
249
			htop
250
			less
251
			locales-all
Mattia Rizzolo's avatar
Mattia Rizzolo committed
252
			lsof
253
254
			molly-guard
			moreutils
255
			munin-node
256
257
			munin-plugins-extra
			netcat-traditional
258
259
			ntp
			ntpdate
260
			pigz 
261
			postfix
262
			procmail
263
			psmisc
264
			python3-psycopg2 
Holger Levsen's avatar
Holger Levsen committed
265
			schroot 
266
267
			screen
			slay
268
			stunnel
Holger Levsen's avatar
Holger Levsen committed
269
270
271
272
273
			subversion 
			subversion-tools 
			sudo 
			unzip 
			vim 
274
			zsh
275
			"
276
		# install squid everywhere except on the armhf nodes
277
		case $HOSTNAME in
278
			jenkins|jenkins-test-vm|profitbricks-build*) DEBS="$DEBS
279
				squid3
280
281
				kgb-client
				python3-yaml" ;;
282
283
			*) ;;
		esac
284
285
		# needed to run the 2nd reproducible builds nodes in the future...
		case $HOSTNAME in
286
			profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386) DEBS="$DEBS ntpdate" ;;
287
288
			*) ;;
		esac
289
		# needed to run coreboot/openwrt/lede/netbsd/fedora jobs
290
		case $HOSTNAME in
291
			profitbricks-build3-amd64|profitbricks-build4-amd64) DEBS="$DEBS
Holger Levsen's avatar
Holger Levsen committed
292
				bison
293
				ca-certificates
Holger Levsen's avatar
Holger Levsen committed
294
295
296
				cmake
				diffutils
				findutils
297
				fish
Holger Levsen's avatar
Holger Levsen committed
298
299
300
301
302
303
304
305
306
307
308
309
310
311
				flex
				g++
				gawk
				gcc
				git
				grep
				iasl
				libc6-dev
				libncurses5-dev
				libssl-dev
				locales-all
				kgb-client
				m4
				make
312
313
				python3-clint
				python3-git
314
				python3-pystache
315
				python3-requests
Holger Levsen's avatar
Holger Levsen committed
316
317
				python3-yaml
				subversion
318
				sysvinit-core
319
				tree
Holger Levsen's avatar
Holger Levsen committed
320
321
322
				unzip
				util-linux
				zlib1g-dev"
323
324
325
			;;
			*) ;;
		esac
326
327
328
		# needed to run fdroid jobs
		case $HOSTNAME in
			profitbricks-build7-amd64) DEBS="$DEBS
329
				libvirt-clients
330
331
				libvirt-daemon
				libvirt-daemon-system
332
				python3-libvirt
333
				python3-vagrant
334
				qemu-kvm
335
336
337
				vagrant
				vagrant-mutate
				vagrant-libvirt"
338
			;;
339
340
			*) ;;
		esac
341
		# cucumber dependencies (for lvc jobs)
342
343
344
345
346
		case $HOSTNAME in
			profitbricks-build10-amd64|jenkins-test-vm) DEBS="$DEBS
				cucumber
				tesseract-ocr
				i18nspector
Holger Levsen's avatar
Holger Levsen committed
347
				imagemagick
348
349
				libav-tools
				libsikuli-script-java
350
				libvirt-bin
351
352
353
354
				libvirt-dev
				ovmf
				python-jabberbot
				python-potr
355
				python3-yaml
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
				ruby-guestfs
				ruby-libvirt
				ruby-net-irc
				ruby-packetfu
				ruby-rb-inotify
				ruby-rjb
				ruby-test-unit
				tcpdump
				unclutter
				virt-viewer
				x264
				xvfb
				x11vnc"
			   ;;
			*) ;;
		esac
372
373
374
375
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			# for phil only
			DEBS="$DEBS postfix-pcre"
		fi
376
		if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
Holger Levsen's avatar
Holger Levsen committed
377
378
379
380
			MASTERDEBS=" 
				apache2 
				apt-file 
				apt-listchanges 
381
				asciidoc
Holger Levsen's avatar
Holger Levsen committed
382
383
384
385
386
387
388
389
390
391
				binfmt-support 
				bison 
				build-essential 
				calamaris 
				cmake 
				cron-apt 
				csvtool 
				dnsmasq-base 
				dstat 
				figlet 
Holger Levsen's avatar
Holger Levsen committed
392
				flex
Holger Levsen's avatar
Holger Levsen committed
393
				gawk 
394
395
				ghc
				git-notifier 
Holger Levsen's avatar
Holger Levsen committed
396
397
398
399
				gocr 
				graphviz 
				iasl 
				imagemagick 
400
401
				ip2host
				jekyll
402
				kgb-client
Holger Levsen's avatar
Holger Levsen committed
403
404
405
406
407
408
409
410
411
412
				libapache2-mod-macro 
				libcap2-bin 
				libfile-touch-perl 
				libguestfs-tools 
				libjson-rpc-perl 
				libsoap-lite-perl 
				libvirt0 
				libvirt-bin 
				libvpx1 
				libxslt1-dev 
413
				linux-image-amd64
Holger Levsen's avatar
Holger Levsen committed
414
415
416
417
418
419
420
421
				moreutils 
				mr 
				mtr-tiny 
				munin 
				ntp 
				openbios-ppc 
				openbios-sparc 
				openjdk-7-jre 
422
				pandoc
Holger Levsen's avatar
Holger Levsen committed
423
424
425
426
				postgresql-client-9.4 
				poxml 
				procmail 
				python3-debian 
427
				python3-pystache
428
429
430
				python3-sqlalchemy
				python3-xdg
				python3-yaml
Holger Levsen's avatar
Holger Levsen committed
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
				python-arpy 
				python-hachoir-metadata 
				python-imaging 
				python-lzma 
				python-pip 
				python-rpy2 
				python-setuptools 
				python-twisted 
				python-yaml 
				qemu 
				qemu-kvm 
				qemu-system-x86 
				qemu-user-static 
				radvd 
				ruby-json 
				ruby-rspec 
				seabios 
				shorewall 
				shorewall6 
				sqlite3 
				syslinux 
452
				tor
Holger Levsen's avatar
Holger Levsen committed
453
				vncsnapshot 
454
455
				vnstat
				whohas
Holger Levsen's avatar
Holger Levsen committed
456
				x11-apps 
457
				xtightvncviewer
458
				xvfb
459
				xvkbd
460
				zutils"
461
462
463
		else
			MASTERDEBS=""
		fi
464
		$UP2DATE || sudo apt-get update
465
466
		$UP2DATE || sudo apt-get install $DEBS $MASTERDEBS
		$UP2DATE || sudo apt-get install -t jessie-backports \
467
				pbuilder lintian || echo "this should only fail on the first install"
468
		#		botch
469
		# we need mock from bpo to build current fedora
470
471
		if [ "$HOSTNAME" = "profitbricks-build3-amd64" ] || [ "$HOSTNAME" = "profitbricks-build4-amd64" ] || [ "$HOSTNAME" = "jenkins" ] ; then
			$UP2DATE || sudo apt-get install -t jessie-backports mock \
472
				|| echo "this should only fail on the first install"
473
		fi
474
		# for varying kernels
475
		# we use bpo kernels on pb-build5+6+15+16 (and the default i386 kernel on pb-build2+12-i386)
476
477
		if [ "$HOSTNAME" = "profitbricks-build5-amd64" ] || [ "$HOSTNAME" = "profitbricks-build6-i386" ] || \
		   [ "$HOSTNAME" = "profitbricks-build15-amd64" ] || [ "$HOSTNAME" = "profitbricks-build16-i386" ] ; then
478
			$UP2DATE || sudo apt-get install -t jessie-backports linux-image-amd64 || echo "this should only fail on the first install"
479
		fi
480
		# only needed on the main node
Holger Levsen's avatar
Holger Levsen committed
481
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
482
			$UP2DATE || sudo apt-get install -t jessie-backports jenkins-job-builder || echo "this should only fail on the first install"
Holger Levsen's avatar
Holger Levsen committed
483
		elif [ "$HOSTNAME" = "jenkins" ] ; then
484
			$UP2DATE || sudo apt-get install -t jessie-backports ffmpeg libav-tools python3-popcon jenkins-job-builder dose-extra
485
		fi
Holger Levsen's avatar
Holger Levsen committed
486
		explain "packages installed."
487
	else
Holger Levsen's avatar
Holger Levsen committed
488
		explain "no new packages to be installed."
489
	fi
490
fi
491

492
#
493
# deploy package configuration in /etc and /usr/local/
494
#
Holger Levsen's avatar
Holger Levsen committed
495
cd $BASEDIR
496
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/etc/* /etc
497
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/usr/* /usr/
498
499
500
501
502

#
# more configuration than a simple cp can do
#
sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins
503
sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm
504
505

if [ "$HOSTNAME" = "jenkins" ] ; then
506
507
508
509
510
511
512
513
514
515
516
517
	if [ $BASEDIR/hosts/$HOSTNAME/etc/apache2 -nt $STAMP ] || [ ! -f $STAMP ] ; then
		if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
			sudo a2enmod proxy
			sudo a2enmod proxy_http
			sudo a2enmod rewrite
			sudo a2enmod ssl
			sudo a2enmod headers
			sudo a2enmod macro
			sudo a2enmod filter
		fi
		sudo a2ensite -q jenkins.debian.net
		sudo a2enconf -q munin
518
		sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.debian.net.conf
519
520
521
		# for reproducible.d.n url rewriting:
		[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
		sudo service apache2 reload
522
523
524
	fi
fi

525
if [ $BASEDIR/hosts/$HOSTNAME/etc/munin -nt $STAMP ] || [ ! -f $STAMP ] ; then
526
	cd /etc/munin/plugins
527
	sudo rm -f postfix_* open_inodes interrupts irqstats threads proc_pri vmstat if_err_* exim_* netstat fw_forwarded_local fw_packets forks open_files users nfs* iostat_ios 2>/dev/null
528
	case $HOSTNAME in
Holger Levsen's avatar
Holger Levsen committed
529
			jenkins|profitbricks-build*) [ -L /etc/munin/plugins/squid_cache ] || for i in squid_cache squid_objectsize squid_requests squid_traffic ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
530
531
			*)	;;
	esac
Holger Levsen's avatar
Holger Levsen committed
532
533
534
535
	case $HOSTNAME in
			jenkins) [ -L /etc/munin/plugins/postfix_mailstats ] || for i in postfix_mailstats postfix_mailvolume postfix_mailqueue ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
			*)	;;
	esac
536
537
538
	if [ "$HOSTNAME" != "jenkins" ] && [ -L /etc/munin/plugins/iostat ] ; then
		sudo rm /etc/munin/plugins/iostat
	fi
539
540
	if [ "$HOSTNAME" = "jenkins" ] && [ ! -L /etc/munin/plugins/apache_accesses ] ; then
		for i in apache_accesses apache_volume ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done
541
		sudo ln -s /usr/share/munin/plugins/loggrep jenkins_oom
542
	fi
543
fi
Holger Levsen's avatar
Holger Levsen committed
544
explain "packages configured."
Holger Levsen's avatar
Holger Levsen committed
545
546
547
548

#
# install the heart of jenkins.debian.net
#
Holger Levsen's avatar
Holger Levsen committed
549
cd $BASEDIR
550
[ -d /srv/jenkins/features ] && sudo rm -rf /srv/jenkins/features
551
for dir in bin logparse cucumber live mustache-templates ; do
552
553
	sudo mkdir -p /srv/jenkins/$dir
	sudo rsync -rpt --delete $dir/ /srv/jenkins/$dir/
Holger Levsen's avatar
Holger Levsen committed
554
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
555
done
556
557
HOST_JOBS="hosts/$HOSTNAME/job-cfg"
if [ -e "$HOST_JOBS" ] ; then
558
	sudo rsync -rpt --copy-links --delete "$HOST_JOBS/" /srv/jenkins/job-cfg/
559
560
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
else
Philip Hands's avatar
Philip Hands committed
561
562
	# tidying up ... assuming that we don't want clutter on peripheral servers
	[ -d /srv/jenkins/job-cfg ] && sudo rm -rf /srv/jenkins/job-cfg
563
564
fi

565

Holger Levsen's avatar
Holger Levsen committed
566
sudo mkdir -p /var/lib/jenkins/.ssh
567
568
569
570
571
572
if [ "$HOSTNAME" = "jenkins" ] ; then
	sudo cp jenkins-home/procmailrc /var/lib/jenkins/.procmailrc
	sudo cp jenkins-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
else
	sudo cp jenkins-nodes-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
fi
Holger Levsen's avatar
Holger Levsen committed
573
574
575
sudo chown -R jenkins:jenkins /var/lib/jenkins/.ssh
sudo chmod 700 /var/lib/jenkins/.ssh
sudo chmod 600 /var/lib/jenkins/.ssh/authorized_keys
Holger Levsen's avatar
Holger Levsen committed
576
explain "scripts and configurations for jenkins updated."
577

578
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
579
	sudo cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
580
	git log | grep ^Author| cut -d " " -f2-|sort -u -f > $TMPFILE
581
	echo "----" >> $TMPFILE
582
	sudo tee /var/lib/jenkins/userContent/THANKS > /dev/null < THANKS.head
Holger Levsen's avatar
Holger Levsen committed
583
584
	# samuel, lunar, josch and phil committed with several commiters, only display one
	grep -v -e "samuel.thibault@ens-lyon.org" -e Lunar -e "j.schauer@email.de" -e "mattia@mapreri.org" -e "phil@jenkins-test-vm" $TMPFILE | sudo tee -a /var/lib/jenkins/userContent/THANKS > /dev/null
585
	rm $TMPFILE
586
	TMPDIR=$(mktemp -d -t update-jdn-XXXXXXXX)
587
	sudo cp -pr userContent $TMPDIR/
Holger Levsen's avatar
Holger Levsen committed
588
589
590
	sudo chown -R jenkins.jenkins $TMPDIR
	sudo cp -pr $TMPDIR/userContent  /var/lib/jenkins/
	sudo rm -r $TMPDIR > /dev/null
591
592
593
594
595
596
597
598
599
	cd /var/lib/jenkins/userContent/
	ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
	[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
	[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
	[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
	[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
	diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
	mv THANKS .THANKS
	rm TODO README INSTALL CONTRIBUTING
600
	sudo chown jenkins.jenkins /var/lib/jenkins/userContent/*html
Holger Levsen's avatar
Holger Levsen committed
601
	explain "user content for jenkins updated."
602
fi
603

604
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
605
606
607
608
609
610
611
	#
	# run jenkins-job-builder to update jobs if needed
	#     (using sudo because /etc/jenkins_jobs is root:root 700)
	#
	cd /srv/jenkins/job-cfg
	for metaconfig in *.yaml.py ; do
	# there are both python2 and python3 scripts here
Philip Hands's avatar
Philip Hands committed
612
		[ -e ./$metaconfig ] || continue
613
		./$metaconfig > $TMPFILE
614
615
		if ! sudo -u jenkins-adm cmp -s ${metaconfig%.py} - < $TMPFILE ; then
			sudo -u jenkins-adm tee ${metaconfig%.py} > /dev/null < $TMPFILE
616
617
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
618
	rm -f $TMPFILE
619
	for config in *.yaml ; do
620
621
622
623
624
625
626
		# do update, if
		# no stamp file exist or
		# no .py file exists and config is newer than stamp or
		# a .py file exists and .py file is newer than stamp
		if [ ! -f $STAMP ] || \
		 ( [ ! -f $config.py ] && [ $config -nt $STAMP ] ) || \
		 ( [ -f $config.py ] && [ $config.py -nt $STAMP ] ) ; then
627
			$JJB update $config
628
629
630
631
		else
			echo "$config has not changed, nothing to do."
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
632
	explain "jenkins jobs updated."
633
fi
634

635
#
Holger Levsen's avatar
Holger Levsen committed
636
# configure git for jenkins
637
#
Holger Levsen's avatar
Holger Levsen committed
638
639
640
641
if [ "$(sudo su - jenkins -c 'git config --get user.email')" != "jenkins@jenkins.debian.net" ] ; then
	sudo su - jenkins -c "git config --global user.email jenkins@jenkins.debian.net"
	sudo su - jenkins -c "git config --global user.name Jenkins"
fi
Holger Levsen's avatar
Holger Levsen committed
642

643
644
645
646
647
if [ "$HOSTNAME" = "jenkins" ] ; then
	#
	# creating LVM volume group for jobs
	#
	if [ "$PVNAME" = "" ]; then
Philip Hands's avatar
Philip Hands committed
648
649
650
		figlet -f banner Error
		explain "you must set \$PVNAME to physical volume pathname, exiting."
		exit 1
Holger Levsen's avatar
Holger Levsen committed
651
	elif ! $UP2DATE ; then
Philip Hands's avatar
Philip Hands committed
652
653
654
655
656
657
		if ! sudo pvs $PVNAME >/dev/null 2>&1; then
			sudo pvcreate $PVNAME
		fi
		if ! sudo vgs $VGNAME >/dev/null 2>&1; then
			sudo vgcreate $VGNAME $PVNAME
		fi
658
	fi
659
660
661
662
663
fi

#
# generate the kgb-client configurations
#
664
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "profitbricks-build3-amd64" ] || [ "$HOSTNAME" = "profitbricks-build4-amd64" ] || [ "$HOSTNAME" = "profitbricks-build7-amd64" ] || [ "$HOSTNAME" = "profitbricks-build2-i386" ] || [ "$HOSTNAME" = "profitbricks-build12-i386" ] ; then
665
666
667
	cd $BASEDIR
	KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
	if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
Philip Hands's avatar
Philip Hands committed
668
		# the last condition is to assure the files are owned by the right user/team
669
		if [ "$KGB_SECRETS" -nt $STAMP ] || [ "deploy_kgb.py" -nt "$STAMP" ] || [ ! -f $STAMP ] ; then
Philip Hands's avatar
Philip Hands committed
670
671
672
673
			sudo -u jenkins-adm "./deploy_kgb.py"
		else
			explain "kgb-client configuration unchanged, nothing to do."
		fi
674
	else
Philip Hands's avatar
Philip Hands committed
675
676
677
		figlet -f banner Warning
		echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
		echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
Holger Levsen's avatar
Holger Levsen committed
678
679
		echo "/srv/jenkins/kgb should be mode 755 and owned by jenkins-adm:root."
		echo "/srv/jenkins/kgb/client-status should be mode 755 and owned by jenkins:jenkins."
680
	fi
681
682
fi

683
684
685
686
#
# Create GPG key for jenkins user if they do not already exist (eg. to sign .buildinfo files)
#
if sudo -u jenkins gpg --with-colons --fixed-list-mode --list-secret-keys | cut -d: -f1 | grep -qsFx 'sec' >/dev/null 2>&1 ; then
Holger Levsen's avatar
Holger Levsen committed
687
	explain "$(date) - Not generating GPG key as one already exists for jenkins user."
688
else
Holger Levsen's avatar
Holger Levsen committed
689
	explain "$(date) - Generating GPG key for jenkins user."
690

691
	sudo -u jenkins gpg --no-tty --batch --gen-key <<EOF
692
693
694
Key-Type: RSA
Key-Length: 4096
Key-Usage: sign
695
Name-Real: $HOSTNAME
696
697
698
699
700
701
Name-Comment: Automatically generated key for signing .buildinfo files
Expire-Date: 0
%no-ask-passphrase
%no-protection
%commit
EOF
702
703
704
705
706
707
708
709
710
711

	GPG_KEY_ID="$(sudo -u jenkins gpg --with-colons --fixed-list-mode --list-secret-keys | grep '^sec' | cut -d: -f5 | tail -n1)"

	if [ "$GPG_KEY_ID" = "" ]
	then
		explain "$(date) - Generated GPG key but could not parse key ID"
	else
		explain "$(date) - Generated GPG key $GPG_KEY_ID - submitting to keyserver"
		sudo -u jenkins gpg --send-keys $GPG_KEY_ID
	fi
712
713
fi

Holger Levsen's avatar
Holger Levsen committed
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
#
# There's always some work left...
#	echo FIXME is ignored so check-jobs scripts can output templates requiring manual work
#
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
	rgrep FI[X]ME $BASEDIR/* | grep -v echo > $TMPFILE || true
	if [ -s $TMPFILE ] ; then
		echo
		# only show cucumber FIXMEs when deploying on jenkins-test-vm
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			cat $TMPFILE
		else
			cat $TMPFILE | grep -v cucumber
		fi
		echo
	fi
	rm -f $TMPFILE
fi

733
#
734
# almost finally…
735
#
736
sudo touch $STAMP	# so on the next run, only configs newer than this file will be updated
Holger Levsen's avatar
Holger Levsen committed
737
explain "$(date) - finished deployment."
738

739
# finally!
740
case $HOSTNAME in
741
	# set time back to the future
742
	profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386|profitbricks-build15-amd64|profitbricks-build16-i386)
743
744
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
745
	jenkins)
746
		# notify irc on updates of jenkins.d.n
747
748
		MESSAGE="jenkins.d.n updated to $(cd $BASEDIR ; git describe --always)."
		kgb-client --conf /srv/jenkins/kgb/debian-qa.conf --relay-msg "$MESSAGE"
Holger Levsen's avatar
Holger Levsen committed
749
		;;
750
751
	*)	;;
esac
752
753
754
755

echo
figlet ok
echo
Holger Levsen's avatar
Holger Levsen committed
756
757
echo "__$HOSTNAME=ok__"