update_jdn.sh 19.6 KB
Newer Older
1
#!/bin/bash
2
# Copyright 2012-2016 Holger Levsen <holger@layer-acht.org>
3
4
# released under the GPLv=2

5
6
set -e

Holger Levsen's avatar
Holger Levsen committed
7
BASEDIR=$HOME/jenkins.debian.net
8
9
PVNAME=/dev/vdb      # LVM physical volume for jobs
VGNAME=jenkins01     # LVM volume group
10
STAMP=/var/log/jenkins/update-jenkins.stamp
11
TMPFILE=$(mktemp)
12
13
14
# The $@ below means that command line args get passed on to j-j-b
# which allows one to specify --flush-cache or --ignore-cache
JJB="jenkins-job-builder $@"
Holger Levsen's avatar
Holger Levsen committed
15

16
# so we can later run some commands only if $0 has been updated…
Philip Hands's avatar
Philip Hands committed
17
if [ -f $STAMP ] && [ $STAMP -nt $BASEDIR/$0 ] ; then
Holger Levsen's avatar
Holger Levsen committed
18
	UP2DATE=true
Philip Hands's avatar
Philip Hands committed
19
20
else
	UP2DATE=false
21
22
23
fi


Holger Levsen's avatar
Holger Levsen committed
24
explain() {
Holger Levsen's avatar
Holger Levsen committed
25
	echo "$HOSTNAME: $1"
Holger Levsen's avatar
Holger Levsen committed
26
27
}

28
29


30
echo "--------------------------------------------"
Holger Levsen's avatar
Holger Levsen committed
31
explain "$(date) - begin deployment update."
32
33
34

# run update at current date
case $HOSTNAME in
35
	profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386)
36
37
		# set correct date (if ntpdate is installed)
		[ ! -f /usr/sbin/ntpdate ] || sudo ntpdate -b de.pool.ntp.org
38
39
40
41
		;;
	*)	;;
esac

42
43
44
#
# set up users and groups
#
45
if ! getent passwd jenkins > /dev/null ; then
46
	sudo addgroup --system jenkins
47
	sudo adduser --system --shell /bin/bash --home /var/lib/jenkins --ingroup jenkins --disabled-login jenkins
48
fi
Holger Levsen's avatar
Holger Levsen committed
49
if ! getent group jenkins-adm > /dev/null ; then
50
51
	sudo addgroup --system jenkins-adm
fi
52
if ! getent passwd jenkins-adm > /dev/null  ; then
53
	sudo adduser --system --shell /bin/bash --no-create-home --ingroup jenkins-adm --disabled-login --no-create-home jenkins-adm
Holger Levsen's avatar
Holger Levsen committed
54
	sudo usermod -G jenkins jenkins-adm
55
fi
56

57
58

declare -A user_host_groups u_shell
59
60

sudo_groups='jenkins,jenkins-adm,sudo,adm'
61
62
63
64
65
66
67
68
69
70

# if there's a need for host groups, a case statement on $HOSTNAME here that sets $GROUPNAME, say, should do the trick
# then you can define user_host_groups['phil','lvm_group']=... below
# and add checks for the GROUP version whereever the HOSTNAME is checked in the following code

user_host_groups['helmut','*']="$sudo_groups"
user_host_groups['holger','*']="$sudo_groups"
user_host_groups['holger','jenkins']="reproducible,${user_host_groups['holger','*']}"
user_host_groups['mattia','*']="$sudo_groups"
user_host_groups['phil','jenkins-test-vm']="$sudo_groups,libvirt,libvirt-qemu"
Philip Hands's avatar
Philip Hands committed
71
user_host_groups['phil','profitbricks-build10-amd64']="$sudo_groups"
72
73
74
user_host_groups['phil','jenkins']=''
user_host_groups['lunar','jenkins']='reproducible'

75
76
77

u_shell['mattia']='/bin/zsh'

78
79
80
# get the users out of the user_host_groups array's index
users=$(for i in ${!user_host_groups[@]}; do echo ${i%,*} ; done | sort -u)

81
$UP2DATE || for user in ${users}; do
82
83
	# -v is a bashism to check for set variables, used here to see if this user is active on this host
	[ -v user_host_groups["$user","$HOSTNAME"] -o -v user_host_groups["$user",'*'] ] || continue
84

85
86
87
88
89
90
91
92
	# create the user
	if ! getent passwd $user > /dev/null ; then
		# adduser, defaulting to /bin/bash as shell
		sudo adduser --gecos "" --shell "${u_shell[$user]:-/bin/bash}" --disabled-password $user
	fi
	# add groups: first try the specific host, or if unset fall-back to default '*' setting
	for h in "$HOSTNAME" '*' ; do
		if [ -v user_host_groups["$user","$h"] ] ; then
Holger Levsen's avatar
Holger Levsen committed
93
			sudo usermod -G "${user_host_groups["$user","$h"]}" $user
94
			break
95
		fi
96
97
98
99
100
	done
	# add the user's keys (if any)
	if ls authorized_keys/${user}@*.pub >/dev/null 2>&1 ; then
		[ -d /var/lib/misc/userkeys ] || sudo mkdir -p /var/lib/misc/userkeys
		cat authorized_keys/${user}@*.pub | sudo tee /var/lib/misc/userkeys/${user} > /dev/null
101
	fi
102
103
done

104
# change defaults
105
106
107
$UP2DATE || grep -q '^AuthorizedKeysFile' /etc/ssh/sshd_config || {
	sudo sh -c "echo 'AuthorizedKeysFile /var/lib/misc/userkeys/%u %h/.ssh/authorized_keys' >> /etc/ssh/sshd_config"
	sudo service ssh reload
108
}
109
110
111
112
113
# change vagrants manual configuration on some armhf hosts
$UP2DATE || grep -q '/var/lib/misc/userkeys' /etc/ssh/sshd_config || {
	sudo sed -i "s#/var/lib/monkeysphere/authorized_keys/#/var/lib/misc/userkeys/#g" /etc/ssh/sshd_config
	sudo service ssh reload
}
114

115
116
117
118
119
sudo mkdir -p /srv/workspace
[ -d /srv/schroots ] || sudo mkdir -p /srv/schroots
[ -h /chroots ] || sudo ln -s /srv/workspace/chroots /chroots
[ -h /schroots ] || sudo ln -s /srv/schroots /schroots

120
if [ "$HOSTNAME" = "jenkins-test-vm" ] || [ "$HOSTNAME" = "profitbricks-build10-amd64" ] ; then
121
122
123
124
	# jenkins needs access to libvirt
	sudo adduser jenkins libvirt
	sudo adduser jenkins libvirt-qemu

125
	# we need a directory for the VM's storage pools
126
	VM_POOL_DIR=/srv/lvc/vm-pools
127
	if [ ! -d $VM_POOL_DIR ] ; then
128
		sudo mkdir -p $VM_POOL_DIR
129
130
131
		sudo chown jenkins:libvirt-qemu $VM_POOL_DIR
		sudo chmod 775 $VM_POOL_DIR
	fi
132
133
134
135
136
137

	# tidy up after ourselves, for a while at least
	OLD_VM_POOL_DIR=/srv/workspace/vm-pools
	if [ -d "$OLD_VM_POOL_DIR" ] ; then
		sudo rm -r "$OLD_VM_POOL_DIR"
	fi
138
fi
139

140
141
# prepare tmpfs on some hosts
case $HOSTNAME in
Mattia Rizzolo's avatar
Mattia Rizzolo committed
142
143
144
145
	jenkins)
		TMPFSSIZE=100
		TMPSIZE=15
		;;
Holger Levsen's avatar
Holger Levsen committed
146
	profitbricks-build9-amd64)
147
		TMPFSSIZE=40
Mattia Rizzolo's avatar
Mattia Rizzolo committed
148
149
		TMPSIZE=8
		;;
Holger Levsen's avatar
Holger Levsen committed
150
	profitbricks-build*)
Mattia Rizzolo's avatar
Mattia Rizzolo committed
151
152
153
		TMPFSSIZE=200
		TMPSIZE=15
		;;
154
155
156
	*) ;;
esac
case $HOSTNAME in
157
158
159
160
161
162
163
	profitbricks-build*i386)
		if ! grep -q '/srv/workspace' /etc/fstab; then
			echo "Warning: you need to manually create a /srv/workspace partition on i386 nodes, exiting."
			exit 1
		fi
		;;
	jenkins|profitbricks-build*amd64)
164
		if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
165
			echo "tmpfs		/srv/workspace	tmpfs	defaults,size=${TMPFSSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null  
166
		fi
Mattia Rizzolo's avatar
Mattia Rizzolo committed
167
168
169
		if ! grep -q '^tmpfs\s\+/tmp\s' /etc/fstab; then
			echo "tmpfs		/tmp	tmpfs	defaults,size=${TMPSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null
		fi
170
171
		if ! mountpoint -q /srv/workspace; then
			if test -z "$(ls -A /srv/workspace)"; then
172
				sudo mount /srv/workspace
173
			else
174
				explain "WARNING: mountpoint /srv/workspace is non-empty."
175
176
177
178
179
			fi
		fi
		;;
	*) ;;
esac
180

181
# make sure needed directories exists - some directories will not be needed on all hosts...
182
for directory in /schroots /srv/reproducible-results /srv/d-i /srv/udebs /srv/live-build /var/log/jenkins/ /srv/jenkins /srv/jenkins/pseudo-hosts /srv/workspace/chroots ; do
183
184
185
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
	fi
Holger Levsen's avatar
Holger Levsen committed
186
	sudo chown jenkins.jenkins $directory
187
done
Holger Levsen's avatar
Holger Levsen committed
188
for directory in /srv/jenkins ; do
189
190
191
192
193
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
		sudo chown jenkins-adm.jenkins-adm $directory
	fi
done
194

195
if ! test -h /chroots; then
Holger Levsen's avatar
Holger Levsen committed
196
	sudo rmdir /chroots || sudo rm -f /chroots # do not recurse
197
	if test -e /chroots; then
Holger Levsen's avatar
Holger Levsen committed
198
		explain "/chroots could not be cleared."
199
	else
200
		sudo ln -s /srv/workspace/chroots /chroots
201
202
203
	fi
fi

204
205
206
207
208
# only on Debian systems
if [ -f /etc/debian_version ] ; then
	#
	# install packages we need
	#
209
	if [ $BASEDIR/$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then
Holger Levsen's avatar
Holger Levsen committed
210
211
		DEBS=" 
			bash-completion 
212
213
			bc
			bsd-mailx
Holger Levsen's avatar
Holger Levsen committed
214
215
			curl 
			debootstrap 
216
217
			devscripts
			eatmydata
218
			etckeeper
219
			figlet
Holger Levsen's avatar
Holger Levsen committed
220
			git
221
			haveged
Holger Levsen's avatar
Holger Levsen committed
222
			htop
223
			less
224
			locales-all
Mattia Rizzolo's avatar
Mattia Rizzolo committed
225
			lsof
226
227
			molly-guard
			moreutils
228
			munin-node
229
230
			munin-plugins-extra
			netcat-traditional
231
232
			ntp
			ntpdate
233
			pigz 
234
			postfix
235
			procmail
236
			psmisc
237
			python3-psycopg2 
Holger Levsen's avatar
Holger Levsen committed
238
			schroot 
239
240
			screen
			slay
Holger Levsen's avatar
Holger Levsen committed
241
242
243
244
245
			subversion 
			subversion-tools 
			sudo 
			unzip 
			vim 
246
			zsh
247
			"
248
		# install squid everywhere except on the armhf nodes
249
		case $HOSTNAME in
250
			jenkins|jenkins-test-vm|profitbricks-build*) DEBS="$DEBS
251
252
				squid3"
			   ;;
253
254
			*) ;;
		esac
255
256
		# needed to run the 2nd reproducible builds nodes in the future...
		case $HOSTNAME in
257
			profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386) DEBS="$DEBS ntpdate" ;;
258
259
			*) ;;
		esac
lynxis lazus's avatar
lynxis lazus committed
260
		# needed to run coreboot/openwrt/lede/netbsd/fedora/fdroid jobs
261
		case $HOSTNAME in
262
			profitbricks-build3-amd64|profitbricks-build4-amd64) DEBS="$DEBS
Holger Levsen's avatar
Holger Levsen committed
263
				bison
264
				ca-certificates
Holger Levsen's avatar
Holger Levsen committed
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
				cmake
				diffutils
				findutils
				flex
				g++
				gawk
				gcc
				git
				grep
				iasl
				libc6-dev
				libncurses5-dev
				libssl-dev
				locales-all
				kgb-client
				m4
				make
282
283
284
				python3-clint
				python3-git
				python3-requests
Holger Levsen's avatar
Holger Levsen committed
285
286
				python3-yaml
				subversion
287
				sysvinit-core
288
				tree
Holger Levsen's avatar
Holger Levsen committed
289
290
				unzip
				util-linux
291
				vagrant
292
				virtualbox
Holger Levsen's avatar
Holger Levsen committed
293
294
				zlib1g-dev"
			   ;;
295
296
			*) ;;
		esac
297
298
299
300
301
302
		# cucumber dependencies
		case $HOSTNAME in
			profitbricks-build10-amd64|jenkins-test-vm) DEBS="$DEBS
				cucumber
				tesseract-ocr
				i18nspector
Holger Levsen's avatar
Holger Levsen committed
303
				imagemagick
304
305
				libav-tools
				libsikuli-script-java
306
				libvirt-bin
307
308
309
310
				libvirt-dev
				ovmf
				python-jabberbot
				python-potr
311
				python3-yaml
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
				ruby-guestfs
				ruby-libvirt
				ruby-net-irc
				ruby-packetfu
				ruby-rb-inotify
				ruby-rjb
				ruby-test-unit
				tcpdump
				unclutter
				virt-viewer
				x264
				xvfb
				x11vnc"
			   ;;
			*) ;;
		esac
328
329
330
331
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			# for phil only
			DEBS="$DEBS postfix-pcre"
		fi
332
		if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
Holger Levsen's avatar
Holger Levsen committed
333
334
335
336
			MASTERDEBS=" 
				apache2 
				apt-file 
				apt-listchanges 
337
				asciidoc
Holger Levsen's avatar
Holger Levsen committed
338
339
340
341
342
343
344
345
346
347
348
				binfmt-support 
				bison 
				build-essential 
				calamaris 
				cmake 
				cron-apt 
				csvtool 
				dnsmasq-base 
				dose-extra 
				dstat 
				figlet 
Holger Levsen's avatar
Holger Levsen committed
349
				flex
Holger Levsen's avatar
Holger Levsen committed
350
				gawk 
351
352
				ghc
				git-notifier 
Holger Levsen's avatar
Holger Levsen committed
353
354
355
356
				gocr 
				graphviz 
				iasl 
				imagemagick 
357
358
				ip2host
				jekyll
359
				kgb-client
Holger Levsen's avatar
Holger Levsen committed
360
361
362
363
364
365
366
367
368
369
				libapache2-mod-macro 
				libcap2-bin 
				libfile-touch-perl 
				libguestfs-tools 
				libjson-rpc-perl 
				libsoap-lite-perl 
				libvirt0 
				libvirt-bin 
				libvpx1 
				libxslt1-dev 
370
				linux-image-amd64
Holger Levsen's avatar
Holger Levsen committed
371
372
373
374
375
376
377
378
				moreutils 
				mr 
				mtr-tiny 
				munin 
				ntp 
				openbios-ppc 
				openbios-sparc 
				openjdk-7-jre 
379
				pandoc
Holger Levsen's avatar
Holger Levsen committed
380
381
382
383
				postgresql-client-9.4 
				poxml 
				procmail 
				python3-debian 
384
				python3-xdg
Holger Levsen's avatar
Holger Levsen committed
385
				python3-yaml 
386
				python3-pystache
Holger Levsen's avatar
Holger Levsen committed
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
				python-arpy 
				python-hachoir-metadata 
				python-imaging 
				python-lzma 
				python-pip 
				python-rpy2 
				python-setuptools 
				python-twisted 
				python-yaml 
				qemu 
				qemu-kvm 
				qemu-system-x86 
				qemu-user-static 
				radvd 
				ruby-json 
				ruby-rspec 
				seabios 
				shorewall 
				shorewall6 
				sqlite3 
				syslinux 
408
				tor
Holger Levsen's avatar
Holger Levsen committed
409
				vncsnapshot 
410
411
				vnstat
				whohas
Holger Levsen's avatar
Holger Levsen committed
412
				x11-apps 
413
				xtightvncviewer
414
				xvfb
415
				xvkbd
416
				zutils"
417
418
419
		else
			MASTERDEBS=""
		fi
420
		$UP2DATE || sudo apt-get update
421
422
		$UP2DATE || sudo apt-get install $DEBS $MASTERDEBS
		$UP2DATE || sudo apt-get install -t jessie-backports \
423
				pbuilder lintian || echo "this should only fail on the first install"
424
		#		botch
425
		# we need mock from bpo to build current fedora
426
		# we need vagrant from bpo to build fdroid
427
		if [ "$HOSTNAME" = "profitbricks-build3-amd64" ] || [ "$HOSTNAME" = "profitbricks-build4-amd64" ] || [ "$HOSTNAME" = "jenkins" ] ; then
428
			$UP2DATE || sudo apt-get install -t jessie-backports mock python3-vagrant \
429
				|| echo "this should only fail on the first install"
430
		fi
431
		# for varying kernels
432
433
		# we use bpo kernels on pb-build5+6 (and i386 kernel on pb-build2-i386)
		if [ "$HOSTNAME" = "profitbricks-build5-amd64" ] || [ "$HOSTNAME" = "profitbricks-build6-i386" ]; then
434
			$UP2DATE || sudo apt-get install -t jessie-backports linux-image-amd64 || echo "this should only fail on the first install"
435
		fi
436
		# only needed on the main node
Holger Levsen's avatar
Holger Levsen committed
437
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
438
			$UP2DATE || sudo apt-get install -t jessie-backports jenkins-job-builder || echo "this should only fail on the first install"
Holger Levsen's avatar
Holger Levsen committed
439
		elif [ "$HOSTNAME" = "jenkins" ] ; then
440
			$UP2DATE || sudo apt-get install -t jessie-backports ffmpeg libav-tools python3-popcon jenkins-job-builder
441
		fi
Holger Levsen's avatar
Holger Levsen committed
442
		explain "packages installed."
443
	else
Holger Levsen's avatar
Holger Levsen committed
444
		explain "no new packages to be installed."
445
	fi
446
fi
447

448
#
449
# deploy package configuration in /etc and /usr/local/
450
#
Holger Levsen's avatar
Holger Levsen committed
451
cd $BASEDIR
452
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/etc/* /etc
453
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/usr/* /usr/
454
455
456
457
458

#
# more configuration than a simple cp can do
#
sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins
459
sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm
460
461

if [ "$HOSTNAME" = "jenkins" ] ; then
462
463
464
465
466
467
468
469
470
471
472
473
	if [ $BASEDIR/hosts/$HOSTNAME/etc/apache2 -nt $STAMP ] || [ ! -f $STAMP ] ; then
		if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
			sudo a2enmod proxy
			sudo a2enmod proxy_http
			sudo a2enmod rewrite
			sudo a2enmod ssl
			sudo a2enmod headers
			sudo a2enmod macro
			sudo a2enmod filter
		fi
		sudo a2ensite -q jenkins.debian.net
		sudo a2enconf -q munin
474
		sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.debian.net.conf
475
476
477
		# for reproducible.d.n url rewriting:
		[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
		sudo service apache2 reload
478
479
480
	fi
fi

481
if [ $BASEDIR/hosts/$HOSTNAME/etc/munin -nt $STAMP ] || [ ! -f $STAMP ] ; then
482
	cd /etc/munin/plugins
483
	sudo rm -f postfix_* open_inodes interrupts irqstats threads proc_pri vmstat if_err_* exim_* netstat fw_forwarded_local fw_packets forks open_files users nfs* iostat_ios 2>/dev/null
484
	case $HOSTNAME in
Holger Levsen's avatar
Holger Levsen committed
485
			jenkins|profitbricks-build*) [ -L /etc/munin/plugins/squid_cache ] || for i in squid_cache squid_objectsize squid_requests squid_traffic ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
486
487
			*)	;;
	esac
488
489
490
	if [ "$HOSTNAME" != "jenkins" ] && [ -L /etc/munin/plugins/iostat ] ; then
		sudo rm /etc/munin/plugins/iostat
	fi
491
492
	if [ "$HOSTNAME" = "jenkins" ] && [ ! -L /etc/munin/plugins/apache_accesses ] ; then
		for i in apache_accesses apache_volume ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done
493
		sudo ln -s /usr/share/munin/plugins/loggrep jenkins_oom
494
495
	fi
	sudo service munin-node force-reload
496
fi
Holger Levsen's avatar
Holger Levsen committed
497
explain "packages configured."
Holger Levsen's avatar
Holger Levsen committed
498
499
500
501

#
# install the heart of jenkins.debian.net
#
Holger Levsen's avatar
Holger Levsen committed
502
cd $BASEDIR
503
504
[ -d /srv/jenkins/features ] && sudo rm -rf /srv/jenkins/features
for dir in bin logparse cucumber live ; do
505
506
	sudo mkdir -p /srv/jenkins/$dir
	sudo rsync -rpt --delete $dir/ /srv/jenkins/$dir/
Holger Levsen's avatar
Holger Levsen committed
507
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
508
done
509
510
HOST_JOBS="hosts/$HOSTNAME/job-cfg"
if [ -e "$HOST_JOBS" ] ; then
511
	sudo rsync -rpt --copy-links --delete "$HOST_JOBS/" /srv/jenkins/job-cfg/
512
513
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
else
Philip Hands's avatar
Philip Hands committed
514
515
	# tidying up ... assuming that we don't want clutter on peripheral servers
	[ -d /srv/jenkins/job-cfg ] && sudo rm -rf /srv/jenkins/job-cfg
516
517
fi

518

Holger Levsen's avatar
Holger Levsen committed
519
sudo mkdir -p /var/lib/jenkins/.ssh
520
521
522
523
524
525
if [ "$HOSTNAME" = "jenkins" ] ; then
	sudo cp jenkins-home/procmailrc /var/lib/jenkins/.procmailrc
	sudo cp jenkins-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
else
	sudo cp jenkins-nodes-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
fi
Holger Levsen's avatar
Holger Levsen committed
526
527
528
sudo chown -R jenkins:jenkins /var/lib/jenkins/.ssh
sudo chmod 700 /var/lib/jenkins/.ssh
sudo chmod 600 /var/lib/jenkins/.ssh/authorized_keys
Holger Levsen's avatar
Holger Levsen committed
529
explain "scripts and configurations for jenkins updated."
530

531
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
532
	sudo cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
533
	git log | grep ^Author| cut -d " " -f2-|sort -u -f > $TMPFILE
534
	echo "----" >> $TMPFILE
535
	sudo tee /var/lib/jenkins/userContent/THANKS > /dev/null < THANKS.head
Holger Levsen's avatar
Holger Levsen committed
536
537
	# samuel, lunar, josch and phil committed with several commiters, only display one
	grep -v -e "samuel.thibault@ens-lyon.org" -e Lunar -e "j.schauer@email.de" -e "mattia@mapreri.org" -e "phil@jenkins-test-vm" $TMPFILE | sudo tee -a /var/lib/jenkins/userContent/THANKS > /dev/null
538
	rm $TMPFILE
539
	TMPDIR=$(mktemp -d -t update-jdn-XXXXXXXX)
540
	sudo cp -pr userContent $TMPDIR/
Holger Levsen's avatar
Holger Levsen committed
541
542
543
	sudo chown -R jenkins.jenkins $TMPDIR
	sudo cp -pr $TMPDIR/userContent  /var/lib/jenkins/
	sudo rm -r $TMPDIR > /dev/null
544
545
546
547
548
549
550
551
552
	cd /var/lib/jenkins/userContent/
	ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
	[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
	[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
	[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
	[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
	diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
	mv THANKS .THANKS
	rm TODO README INSTALL CONTRIBUTING
553
	sudo chown jenkins.jenkins /var/lib/jenkins/userContent/*html
Holger Levsen's avatar
Holger Levsen committed
554
	explain "user content for jenkins updated."
555
fi
556

557
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
558
559
560
561
562
563
564
	#
	# run jenkins-job-builder to update jobs if needed
	#     (using sudo because /etc/jenkins_jobs is root:root 700)
	#
	cd /srv/jenkins/job-cfg
	for metaconfig in *.yaml.py ; do
	# there are both python2 and python3 scripts here
Philip Hands's avatar
Philip Hands committed
565
		[ -e ./$metaconfig ] || continue
566
		./$metaconfig > $TMPFILE
567
568
		if ! sudo -u jenkins-adm cmp -s ${metaconfig%.py} - < $TMPFILE ; then
			sudo -u jenkins-adm tee ${metaconfig%.py} > /dev/null < $TMPFILE
569
570
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
571
	rm -f $TMPFILE
572
	for config in *.yaml ; do
573
574
575
576
577
578
579
		# do update, if
		# no stamp file exist or
		# no .py file exists and config is newer than stamp or
		# a .py file exists and .py file is newer than stamp
		if [ ! -f $STAMP ] || \
		 ( [ ! -f $config.py ] && [ $config -nt $STAMP ] ) || \
		 ( [ -f $config.py ] && [ $config.py -nt $STAMP ] ) ; then
580
			$JJB update $config
581
582
583
584
		else
			echo "$config has not changed, nothing to do."
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
585
	explain "jenkins jobs updated."
586
fi
587

588
#
Holger Levsen's avatar
Holger Levsen committed
589
# configure git for jenkins
590
#
Holger Levsen's avatar
Holger Levsen committed
591
592
593
594
if [ "$(sudo su - jenkins -c 'git config --get user.email')" != "jenkins@jenkins.debian.net" ] ; then
	sudo su - jenkins -c "git config --global user.email jenkins@jenkins.debian.net"
	sudo su - jenkins -c "git config --global user.name Jenkins"
fi
Holger Levsen's avatar
Holger Levsen committed
595

596
597
598
599
600
if [ "$HOSTNAME" = "jenkins" ] ; then
	#
	# creating LVM volume group for jobs
	#
	if [ "$PVNAME" = "" ]; then
Philip Hands's avatar
Philip Hands committed
601
602
603
		figlet -f banner Error
		explain "you must set \$PVNAME to physical volume pathname, exiting."
		exit 1
Holger Levsen's avatar
Holger Levsen committed
604
	elif ! $UP2DATE ; then
Philip Hands's avatar
Philip Hands committed
605
606
607
608
609
610
		if ! sudo pvs $PVNAME >/dev/null 2>&1; then
			sudo pvcreate $PVNAME
		fi
		if ! sudo vgs $VGNAME >/dev/null 2>&1; then
			sudo vgcreate $VGNAME $PVNAME
		fi
611
	fi
612
613
614
615
616
fi

#
# generate the kgb-client configurations
#
617
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "profitbricks-build3-amd64" ] || [ "$HOSTNAME" = "profitbricks-build4-amd64" ] ; then
618
619
620
	cd $BASEDIR
	KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
	if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
Philip Hands's avatar
Philip Hands committed
621
622
623
624
625
626
		# the last condition is to assure the files are owned by the right user/team
		if [ "$KGB_SECRETS" -nt $STAMP ] || [ ! -f $STAMP ] ; then
			sudo -u jenkins-adm "./deploy_kgb.py"
		else
			explain "kgb-client configuration unchanged, nothing to do."
		fi
627
	else
Philip Hands's avatar
Philip Hands committed
628
629
630
631
632
		figlet -f banner Warning
		echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
		echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
		echo "/srv/jenkins/kbg should be mode 755 and owned by jenkins-adm:root."
		echo "/srv/jenkins/kbg/client-status should be mode 755 and owned by jenkins:jenkins."
633
	fi
634
635
fi

Holger Levsen's avatar
Holger Levsen committed
636
637
638
639
#
# There's always some work left...
#	echo FIXME is ignored so check-jobs scripts can output templates requiring manual work
#
640
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
641
642
643
	rgrep FI[X]ME $BASEDIR/* | grep -v echo > $TMPFILE || true
	if [ -s $TMPFILE ] ; then
		echo
644
645
646
647
648
649
		# only show cucumber FIXMEs when deploying on jenkins-test-vm
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			cat $TMPFILE
		else
			cat $TMPFILE | grep -v cucumber
		fi
650
651
652
		echo
	fi
	rm -f $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
653
fi
Holger Levsen's avatar
Holger Levsen committed
654

655
#
656
# almost finally…
657
#
658
sudo touch $STAMP	# so on the next run, only configs newer than this file will be updated
Holger Levsen's avatar
Holger Levsen committed
659
explain "$(date) - finished deployment."
660

661
# finally!
662
case $HOSTNAME in
663
	# set time back to the future
664
	profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386)
665
666
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
667
	jenkins)
668
		MESSAGE="jenkins.d.n updated to $(cd $BASEDIR ; git describe --always)."
669
		kgb-client --conf /srv/jenkins/kgb/debian-qa.conf --relay-msg "$MESSAGE"
Holger Levsen's avatar
Holger Levsen committed
670
		;;
671
672
	*)	;;
esac
673
674
675
676

echo
figlet ok
echo
Holger Levsen's avatar
Holger Levsen committed
677
678
echo "__$HOSTNAME=ok__"