update_jdn.sh 19.2 KB
Newer Older
1
2
#!/bin/bash

Philip Hands's avatar
set -e    
Philip Hands committed
3
4
set -e

5
# Copyright 2012-2016 Holger Levsen <holger@layer-acht.org>
6
7
# released under the GPLv=2

Holger Levsen's avatar
Holger Levsen committed
8
BASEDIR=$HOME/jenkins.debian.net
9
10
PVNAME=/dev/vdb      # LVM physical volume for jobs
VGNAME=jenkins01     # LVM volume group
11
STAMP=/var/log/jenkins/update-jenkins.stamp
12
TMPFILE=$(mktemp)
13
14
15
# The $@ below means that command line args get passed on to j-j-b
# which allows one to specify --flush-cache or --ignore-cache
JJB="jenkins-job-builder $@"
Holger Levsen's avatar
Holger Levsen committed
16

17
# so we can later run some commands only if $0 has been updated…
Holger Levsen's avatar
Holger Levsen committed
18
if [ ! -f $STAMP ] && [ $BASEDIR/$0 -nt $STAMP ] ; then
Holger Levsen's avatar
Holger Levsen committed
19
	UP2DATE=false
20
else
Holger Levsen's avatar
Holger Levsen committed
21
	UP2DATE=true
22
23
24
fi


Holger Levsen's avatar
Holger Levsen committed
25
explain() {
Holger Levsen's avatar
Holger Levsen committed
26
	echo "$HOSTNAME: $1"
Holger Levsen's avatar
Holger Levsen committed
27
28
}

29
30


31
echo "--------------------------------------------"
Holger Levsen's avatar
Holger Levsen committed
32
explain "$(date) - begin deployment update."
33
34
35

# run update at current date
case $HOSTNAME in
36
	profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386)
37
38
		# set correct date (if ntpdate is installed)
		[ ! -f /usr/sbin/ntpdate ] || sudo ntpdate -b de.pool.ntp.org
39
40
41
42
		;;
	*)	;;
esac

43
44
45
#
# set up users and groups
#
46
if ! getent passwd jenkins > /dev/null ; then
47
	sudo addgroup --system jenkins
48
	sudo adduser --system --shell /bin/bash --home /var/lib/jenkins --ingroup jenkins --disabled-login jenkins
49
fi
Holger Levsen's avatar
Holger Levsen committed
50
if ! getent group jenkins-adm > /dev/null ; then
51
52
	sudo addgroup --system jenkins-adm
fi
53
if ! getent passwd jenkins-adm > /dev/null  ; then
54
	sudo adduser --system --shell /bin/bash --no-create-home --ingroup jenkins-adm --disabled-login --no-create-home jenkins-adm
Holger Levsen's avatar
Holger Levsen committed
55
	sudo usermod -G jenkins jenkins-adm
56
fi
57

58
59

declare -A user_host_groups u_shell
60
61

sudo_groups='jenkins,jenkins-adm,sudo,adm'
62
63
64
65
66
67
68
69
70
71
72
73
74
75

# if there's a need for host groups, a case statement on $HOSTNAME here that sets $GROUPNAME, say, should do the trick
# then you can define user_host_groups['phil','lvm_group']=... below
# and add checks for the GROUP version whereever the HOSTNAME is checked in the following code

user_host_groups['helmut','*']="$sudo_groups"
user_host_groups['holger','*']="$sudo_groups"
user_host_groups['holger','jenkins']="reproducible,${user_host_groups['holger','*']}"
user_host_groups['mattia','*']="$sudo_groups"
user_host_groups['phil','jenkins-test-vm']="$sudo_groups,libvirt,libvirt-qemu"
user_host_groups['phil','profitbricks-build10-amd64']=''
user_host_groups['phil','jenkins']=''
user_host_groups['lunar','jenkins']='reproducible'

76
77
78

u_shell['mattia']='/bin/zsh'

79
80
81
82
83
84
# get the users out of the user_host_groups array's index
users=$(for i in ${!user_host_groups[@]}; do echo ${i%,*} ; done | sort -u)

$UP2DATE || for user in $users ; do
	# -v is a bashism to check for set variables, used here to see if this user is active on this host
	[ -v user_host_groups["$user","$HOSTNAME"] -o -v user_host_groups["$user",'*'] ] || continue
85

86
87
88
89
90
91
92
93
94
95
	# create the user
	if ! getent passwd $user > /dev/null ; then
		# adduser, defaulting to /bin/bash as shell
		sudo adduser --gecos "" --shell "${u_shell[$user]:-/bin/bash}" --disabled-password $user
	fi
	# add groups: first try the specific host, or if unset fall-back to default '*' setting
	for h in "$HOSTNAME" '*' ; do
		if [ -v user_host_groups["$user","$h"] ] ; then
			sudo usermod -G ${user_host_groups["$user","$h"]} $user
			break
96
		fi
97
98
99
100
101
	done
	# add the user's keys (if any)
	if ls authorized_keys/${user}@*.pub >/dev/null 2>&1 ; then
		[ -d /var/lib/misc/userkeys ] || sudo mkdir -p /var/lib/misc/userkeys
		cat authorized_keys/${user}@*.pub | sudo tee /var/lib/misc/userkeys/${user} > /dev/null
102
	fi
103
104
done

105
106
107
$UP2DATE || grep -q '^AuthorizedKeysFile' /etc/ssh/sshd_config || {
	sudo sh -c "echo 'AuthorizedKeysFile /var/lib/misc/userkeys/%u %h/.ssh/authorized_keys' >> /etc/ssh/sshd_config"
	sudo service ssh reload
108
109
}

110
111
112
113
114
sudo mkdir -p /srv/workspace
[ -d /srv/schroots ] || sudo mkdir -p /srv/schroots
[ -h /chroots ] || sudo ln -s /srv/workspace/chroots /chroots
[ -h /schroots ] || sudo ln -s /srv/schroots /schroots

115
if [ "$HOSTNAME" = "jenkins-test-vm" ] || [ "$HOSTNAME" = "profitbricks-build10-amd64" ] ; then
116
117
118
119
	# jenkins needs access to libvirt
	sudo adduser jenkins libvirt
	sudo adduser jenkins libvirt-qemu

120
121
122
123
124
125
126
127
	# we need a directory for the VM's storage pools
	VM_POOL_DIR=/srv/workspace/vm-pools
	if [ ! -d $VM_POOL_DIR ] ; then
		sudo mkdir $VM_POOL_DIR
		sudo chown jenkins:libvirt-qemu $VM_POOL_DIR
		sudo chmod 775 $VM_POOL_DIR
	fi
fi
128

129
130
# prepare tmpfs on some hosts
case $HOSTNAME in
Mattia Rizzolo's avatar
Mattia Rizzolo committed
131
132
133
134
	jenkins)
		TMPFSSIZE=100
		TMPSIZE=15
		;;
Holger Levsen's avatar
Holger Levsen committed
135
	profitbricks-build9-amd64)
Mattia Rizzolo's avatar
Mattia Rizzolo committed
136
137
138
		TMPFSSIZE=32
		TMPSIZE=8
		;;
Holger Levsen's avatar
Holger Levsen committed
139
	profitbricks-build*)
Mattia Rizzolo's avatar
Mattia Rizzolo committed
140
141
142
		TMPFSSIZE=200
		TMPSIZE=15
		;;
143
144
145
	*) ;;
esac
case $HOSTNAME in
146
147
148
149
150
151
152
	profitbricks-build*i386)
		if ! grep -q '/srv/workspace' /etc/fstab; then
			echo "Warning: you need to manually create a /srv/workspace partition on i386 nodes, exiting."
			exit 1
		fi
		;;
	jenkins|profitbricks-build*amd64)
153
		if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
154
			echo "tmpfs		/srv/workspace	tmpfs	defaults,size=${TMPFSSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null  
155
		fi
Mattia Rizzolo's avatar
Mattia Rizzolo committed
156
157
158
		if ! grep -q '^tmpfs\s\+/tmp\s' /etc/fstab; then
			echo "tmpfs		/tmp	tmpfs	defaults,size=${TMPSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null
		fi
159
160
		if ! mountpoint -q /srv/workspace; then
			if test -z "$(ls -A /srv/workspace)"; then
161
				sudo mount /srv/workspace
162
			else
163
				explain "WARNING: mountpoint /srv/workspace is non-empty."
164
165
166
167
168
			fi
		fi
		;;
	*) ;;
esac
169

170
# make sure needed directories exists - some directories will not be needed on all hosts...
171
for directory in /schroots /srv/reproducible-results /srv/d-i /srv/udebs /srv/live-build /var/log/jenkins/ /srv/jenkins /srv/jenkins/pseudo-hosts /srv/workspace/chroots ; do
172
173
174
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
	fi
Holger Levsen's avatar
Holger Levsen committed
175
	sudo chown jenkins.jenkins $directory
176
done
Holger Levsen's avatar
Holger Levsen committed
177
for directory in /srv/jenkins ; do
178
179
180
181
182
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
		sudo chown jenkins-adm.jenkins-adm $directory
	fi
done
183

184
if ! test -h /chroots; then
Holger Levsen's avatar
Holger Levsen committed
185
	sudo rmdir /chroots || sudo rm -f /chroots # do not recurse
186
	if test -e /chroots; then
Holger Levsen's avatar
Holger Levsen committed
187
		explain "/chroots could not be cleared."
188
	else
189
		sudo ln -s /srv/workspace/chroots /chroots
190
191
192
	fi
fi

193
194
195
196
197
# only on Debian systems
if [ -f /etc/debian_version ] ; then
	#
	# install packages we need
	#
198
	if [ $BASEDIR/$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then
Holger Levsen's avatar
Holger Levsen committed
199
200
		DEBS=" 
			bash-completion 
201
202
			bc
			bsd-mailx
Holger Levsen's avatar
Holger Levsen committed
203
204
			curl 
			debootstrap 
205
206
			devscripts
			eatmydata
207
			etckeeper
208
			figlet
Holger Levsen's avatar
Holger Levsen committed
209
			git
210
			haveged
Holger Levsen's avatar
Holger Levsen committed
211
			htop
212
			less
213
			locales-all
214
215
			molly-guard
			moreutils
216
			munin-node
217
218
			munin-plugins-extra
			netcat-traditional
219
			pigz 
220
			postfix
221
			procmail
222
			psmisc
223
			python3-psycopg2 
Holger Levsen's avatar
Holger Levsen committed
224
			schroot 
225
226
			screen
			slay
Holger Levsen's avatar
Holger Levsen committed
227
228
229
230
231
			subversion 
			subversion-tools 
			sudo 
			unzip 
			vim 
232
			zsh
233
			"
234
		# install squid everywhere except on the armhf nodes
235
		case $HOSTNAME in
236
			jenkins|jenkins-test-vm|profitbricks-build*) DEBS="$DEBS
237
238
				squid3"
			   ;;
239
240
			*) ;;
		esac
241
242
		# needed to run the 2nd reproducible builds nodes in the future...
		case $HOSTNAME in
243
			profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386) DEBS="$DEBS ntpdate" ;;
244
245
			*) ;;
		esac
246
		# needed to run coreboot/openwrt/netbsd/fedora/fdroid jobs
247
		case $HOSTNAME in
248
			profitbricks-build3-amd64|profitbricks-build4-amd64) DEBS="$DEBS
Holger Levsen's avatar
Holger Levsen committed
249
				bison
250
				ca-certificates
Holger Levsen's avatar
Holger Levsen committed
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
				cmake
				diffutils
				findutils
				flex
				g++
				gawk
				gcc
				git
				grep
				iasl
				libc6-dev
				libncurses5-dev
				libssl-dev
				locales-all
				kgb-client
				m4
				make
268
269
270
				python3-clint
				python3-git
				python3-requests
Holger Levsen's avatar
Holger Levsen committed
271
272
				python3-yaml
				subversion
273
				sysvinit-core
274
				tree
Holger Levsen's avatar
Holger Levsen committed
275
276
				unzip
				util-linux
277
				vagrant
278
				virtualbox
Holger Levsen's avatar
Holger Levsen committed
279
280
				zlib1g-dev"
			   ;;
281
282
			*) ;;
		esac
283
284
285
286
287
288
		# cucumber dependencies
		case $HOSTNAME in
			profitbricks-build10-amd64|jenkins-test-vm) DEBS="$DEBS
				cucumber
				tesseract-ocr
				i18nspector
Holger Levsen's avatar
Holger Levsen committed
289
				imagemagick
290
291
				libav-tools
				libsikuli-script-java
292
				libvirt-bin
293
294
295
296
				libvirt-dev
				ovmf
				python-jabberbot
				python-potr
297
				python3-yaml
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
				ruby-guestfs
				ruby-libvirt
				ruby-net-irc
				ruby-packetfu
				ruby-rb-inotify
				ruby-rjb
				ruby-test-unit
				tcpdump
				unclutter
				virt-viewer
				x264
				xvfb
				x11vnc"
			   ;;
			*) ;;
		esac
314
315
316
317
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			# for phil only
			DEBS="$DEBS postfix-pcre"
		fi
318
		if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
Holger Levsen's avatar
Holger Levsen committed
319
320
321
322
			MASTERDEBS=" 
				apache2 
				apt-file 
				apt-listchanges 
323
				asciidoc
Holger Levsen's avatar
Holger Levsen committed
324
325
326
327
328
329
330
331
332
333
334
				binfmt-support 
				bison 
				build-essential 
				calamaris 
				cmake 
				cron-apt 
				csvtool 
				dnsmasq-base 
				dose-extra 
				dstat 
				figlet 
Holger Levsen's avatar
Holger Levsen committed
335
				flex
Holger Levsen's avatar
Holger Levsen committed
336
				gawk 
337
338
				ghc
				git-notifier 
Holger Levsen's avatar
Holger Levsen committed
339
340
341
342
				gocr 
				graphviz 
				iasl 
				imagemagick 
343
344
				ip2host
				jekyll
345
				kgb-client
Holger Levsen's avatar
Holger Levsen committed
346
347
348
349
350
351
352
353
354
355
				libapache2-mod-macro 
				libcap2-bin 
				libfile-touch-perl 
				libguestfs-tools 
				libjson-rpc-perl 
				libsoap-lite-perl 
				libvirt0 
				libvirt-bin 
				libvpx1 
				libxslt1-dev 
356
				linux-image-amd64
Holger Levsen's avatar
Holger Levsen committed
357
358
359
360
361
362
363
364
				moreutils 
				mr 
				mtr-tiny 
				munin 
				ntp 
				openbios-ppc 
				openbios-sparc 
				openjdk-7-jre 
365
				pandoc
Holger Levsen's avatar
Holger Levsen committed
366
367
368
369
				postgresql-client-9.4 
				poxml 
				procmail 
				python3-debian 
370
				python3-xdg
Holger Levsen's avatar
Holger Levsen committed
371
				python3-yaml 
372
				python3-pystache
Holger Levsen's avatar
Holger Levsen committed
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
				python-arpy 
				python-hachoir-metadata 
				python-imaging 
				python-lzma 
				python-pip 
				python-rpy2 
				python-setuptools 
				python-twisted 
				python-yaml 
				qemu 
				qemu-kvm 
				qemu-system-x86 
				qemu-user-static 
				radvd 
				ruby-json 
				ruby-rspec 
				seabios 
				shorewall 
				shorewall6 
				sqlite3 
				syslinux 
394
				tor
Holger Levsen's avatar
Holger Levsen committed
395
				vncsnapshot 
396
397
				vnstat
				whohas
Holger Levsen's avatar
Holger Levsen committed
398
				x11-apps 
399
				xtightvncviewer
400
				xvfb
401
				xvkbd
402
				zutils"
403
404
405
		else
			MASTERDEBS=""
		fi
406
		$UP2DATE || sudo apt-get update
407
408
		$UP2DATE || sudo apt-get install $DEBS $MASTERDEBS
		$UP2DATE || sudo apt-get install -t jessie-backports \
409
				pbuilder lintian || echo "this should only fail on the first install"
410
		#		botch
411
		# we need mock from bpo to build current fedora
412
		# we need vagrant from bpo to build fdroid
413
		if [ "$HOSTNAME" = "profitbricks-build3-amd64" ] || [ "$HOSTNAME" = "profitbricks-build4-amd64" ] || [ "$HOSTNAME" = "jenkins" ] ; then
414
			$UP2DATE || sudo apt-get install -t jessie-backports mock python3-vagrant \
415
				|| echo "this should only fail on the first install"
416
		fi
417
		# for varying kernels
418
419
		# we use bpo kernels on pb-build5+6 (and i386 kernel on pb-build2-i386)
		if [ "$HOSTNAME" = "profitbricks-build5-amd64" ] || [ "$HOSTNAME" = "profitbricks-build6-i386" ]; then
420
			$UP2DATE || sudo apt-get install -t jessie-backports linux-image-amd64 || echo "this should only fail on the first install"
421
		fi
422
		# only needed on the main node
Holger Levsen's avatar
Holger Levsen committed
423
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
424
			$UP2DATE || sudo apt-get install -t jessie-backports jenkins-job-builder || echo "this should only fail on the first install"
Holger Levsen's avatar
Holger Levsen committed
425
		elif [ "$HOSTNAME" = "jenkins" ] ; then
426
			$UP2DATE || sudo apt-get install -t jessie-backports ffmpeg libav-tools python3-popcon jenkins-job-builder
427
		fi
Holger Levsen's avatar
Holger Levsen committed
428
		explain "packages installed."
429
	else
Holger Levsen's avatar
Holger Levsen committed
430
		explain "no new packages to be installed."
431
	fi
432
fi
433

434
#
435
# deploy package configuration in /etc and /usr/local/
436
#
Holger Levsen's avatar
Holger Levsen committed
437
cd $BASEDIR
438
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/etc/* /etc
439
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/usr/* /usr/
440
441
442
443
444

#
# more configuration than a simple cp can do
#
sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins
445
sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm
446
447

if [ "$HOSTNAME" = "jenkins" ] ; then
448
449
450
451
452
453
454
455
456
457
458
459
	if [ $BASEDIR/hosts/$HOSTNAME/etc/apache2 -nt $STAMP ] || [ ! -f $STAMP ] ; then
		if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
			sudo a2enmod proxy
			sudo a2enmod proxy_http
			sudo a2enmod rewrite
			sudo a2enmod ssl
			sudo a2enmod headers
			sudo a2enmod macro
			sudo a2enmod filter
		fi
		sudo a2ensite -q jenkins.debian.net
		sudo a2enconf -q munin
460
		sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.debian.net.conf
461
462
463
		# for reproducible.d.n url rewriting:
		[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
		sudo service apache2 reload
464
465
466
	fi
fi

467
if [ $BASEDIR/hosts/$HOSTNAME/etc/munin -nt $STAMP ] || [ ! -f $STAMP ] ; then
468
	cd /etc/munin/plugins
469
	sudo rm -f postfix_* open_inodes interrupts irqstats threads proc_pri vmstat if_err_* exim_* netstat fw_forwarded_local fw_packets forks open_files users nfs* iostat_ios 2>/dev/null
470
	case $HOSTNAME in
Holger Levsen's avatar
Holger Levsen committed
471
			jenkins|profitbricks-build*) [ -L /etc/munin/plugins/squid_cache ] || for i in squid_cache squid_objectsize squid_requests squid_traffic ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
472
473
			*)	;;
	esac
474
475
476
	if [ "$HOSTNAME" != "jenkins" ] && [ -L /etc/munin/plugins/iostat ] ; then
		sudo rm /etc/munin/plugins/iostat
	fi
477
478
	if [ "$HOSTNAME" = "jenkins" ] && [ ! -L /etc/munin/plugins/apache_accesses ] ; then
		for i in apache_accesses apache_volume ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done
479
		sudo ln -s /usr/share/munin/plugins/loggrep jenkins_oom
480
481
	fi
	sudo service munin-node force-reload
482
fi
Holger Levsen's avatar
Holger Levsen committed
483
explain "packages configured."
Holger Levsen's avatar
Holger Levsen committed
484
485
486
487

#
# install the heart of jenkins.debian.net
#
Holger Levsen's avatar
Holger Levsen committed
488
cd $BASEDIR
489
490
[ -d /srv/jenkins/features ] && sudo rm -rf /srv/jenkins/features
for dir in bin logparse cucumber live ; do
491
492
	sudo mkdir -p /srv/jenkins/$dir
	sudo rsync -rpt --delete $dir/ /srv/jenkins/$dir/
Holger Levsen's avatar
Holger Levsen committed
493
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
494
done
495
496
HOST_JOBS="hosts/$HOSTNAME/job-cfg"
if [ -e "$HOST_JOBS" ] ; then
497
	sudo rsync -rpt --copy-links --delete "$HOST_JOBS/" /srv/jenkins/job-cfg/
498
499
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
else
Philip Hands's avatar
Philip Hands committed
500
501
	# tidying up ... assuming that we don't want clutter on peripheral servers
	[ -d /srv/jenkins/job-cfg ] && sudo rm -rf /srv/jenkins/job-cfg
502
503
fi

504

Holger Levsen's avatar
Holger Levsen committed
505
sudo mkdir -p /var/lib/jenkins/.ssh
506
507
508
509
510
511
if [ "$HOSTNAME" = "jenkins" ] ; then
	sudo cp jenkins-home/procmailrc /var/lib/jenkins/.procmailrc
	sudo cp jenkins-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
else
	sudo cp jenkins-nodes-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
fi
Holger Levsen's avatar
Holger Levsen committed
512
513
514
sudo chown -R jenkins:jenkins /var/lib/jenkins/.ssh
sudo chmod 700 /var/lib/jenkins/.ssh
sudo chmod 600 /var/lib/jenkins/.ssh/authorized_keys
Holger Levsen's avatar
Holger Levsen committed
515
explain "scripts and configurations for jenkins updated."
516

517
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
518
	sudo cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
519
	git log | grep ^Author| cut -d " " -f2-|sort -u -f > $TMPFILE
520
	echo "----" >> $TMPFILE
521
	sudo tee /var/lib/jenkins/userContent/THANKS > /dev/null < THANKS.head
Holger Levsen's avatar
Holger Levsen committed
522
523
	# samuel, lunar, josch and phil committed with several commiters, only display one
	grep -v -e "samuel.thibault@ens-lyon.org" -e Lunar -e "j.schauer@email.de" -e "mattia@mapreri.org" -e "phil@jenkins-test-vm" $TMPFILE | sudo tee -a /var/lib/jenkins/userContent/THANKS > /dev/null
524
	rm $TMPFILE
525
	TMPDIR=$(mktemp -d -t update-jdn-XXXXXXXX)
526
	sudo cp -pr userContent $TMPDIR/
Holger Levsen's avatar
Holger Levsen committed
527
528
529
	sudo chown -R jenkins.jenkins $TMPDIR
	sudo cp -pr $TMPDIR/userContent  /var/lib/jenkins/
	sudo rm -r $TMPDIR > /dev/null
530
531
532
533
534
535
536
537
538
	cd /var/lib/jenkins/userContent/
	ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
	[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
	[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
	[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
	[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
	diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
	mv THANKS .THANKS
	rm TODO README INSTALL CONTRIBUTING
539
	sudo chown jenkins.jenkins /var/lib/jenkins/userContent/*html
Holger Levsen's avatar
Holger Levsen committed
540
	explain "user content for jenkins updated."
541
fi
542

543
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
544
545
546
547
548
549
550
	#
	# run jenkins-job-builder to update jobs if needed
	#     (using sudo because /etc/jenkins_jobs is root:root 700)
	#
	cd /srv/jenkins/job-cfg
	for metaconfig in *.yaml.py ; do
	# there are both python2 and python3 scripts here
Philip Hands's avatar
Philip Hands committed
551
		[ -e ./$metaconfig ] || continue
552
		./$metaconfig > $TMPFILE
553
554
		if ! sudo -u jenkins-adm cmp -s ${metaconfig%.py} - < $TMPFILE ; then
			sudo -u jenkins-adm tee ${metaconfig%.py} > /dev/null < $TMPFILE
555
556
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
557
	rm -f $TMPFILE
558
	for config in *.yaml ; do
559
560
561
562
563
564
565
		# do update, if
		# no stamp file exist or
		# no .py file exists and config is newer than stamp or
		# a .py file exists and .py file is newer than stamp
		if [ ! -f $STAMP ] || \
		 ( [ ! -f $config.py ] && [ $config -nt $STAMP ] ) || \
		 ( [ -f $config.py ] && [ $config.py -nt $STAMP ] ) ; then
566
			$JJB update $config
567
568
569
570
		else
			echo "$config has not changed, nothing to do."
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
571
	explain "jenkins jobs updated."
572
fi
573

574
#
Holger Levsen's avatar
Holger Levsen committed
575
# configure git for jenkins
576
#
Holger Levsen's avatar
Holger Levsen committed
577
578
579
580
if [ "$(sudo su - jenkins -c 'git config --get user.email')" != "jenkins@jenkins.debian.net" ] ; then
	sudo su - jenkins -c "git config --global user.email jenkins@jenkins.debian.net"
	sudo su - jenkins -c "git config --global user.name Jenkins"
fi
Holger Levsen's avatar
Holger Levsen committed
581

582
583
584
585
586
if [ "$HOSTNAME" = "jenkins" ] ; then
	#
	# creating LVM volume group for jobs
	#
	if [ "$PVNAME" = "" ]; then
Philip Hands's avatar
Philip Hands committed
587
588
589
		figlet -f banner Error
		explain "you must set \$PVNAME to physical volume pathname, exiting."
		exit 1
Holger Levsen's avatar
Holger Levsen committed
590
	elif ! $UP2DATE ; then
Philip Hands's avatar
Philip Hands committed
591
592
593
594
595
596
		if ! sudo pvs $PVNAME >/dev/null 2>&1; then
			sudo pvcreate $PVNAME
		fi
		if ! sudo vgs $VGNAME >/dev/null 2>&1; then
			sudo vgcreate $VGNAME $PVNAME
		fi
597
	fi
598
599
600
601
602
fi

#
# generate the kgb-client configurations
#
603
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "profitbricks-build3-amd64" ] || [ "$HOSTNAME" = "profitbricks-build4-amd64" ] ; then
604
605
606
	cd $BASEDIR
	KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
	if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
Philip Hands's avatar
Philip Hands committed
607
608
609
610
611
612
		# the last condition is to assure the files are owned by the right user/team
		if [ "$KGB_SECRETS" -nt $STAMP ] || [ ! -f $STAMP ] ; then
			sudo -u jenkins-adm "./deploy_kgb.py"
		else
			explain "kgb-client configuration unchanged, nothing to do."
		fi
613
	else
Philip Hands's avatar
Philip Hands committed
614
615
616
617
618
		figlet -f banner Warning
		echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
		echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
		echo "/srv/jenkins/kbg should be mode 755 and owned by jenkins-adm:root."
		echo "/srv/jenkins/kbg/client-status should be mode 755 and owned by jenkins:jenkins."
619
	fi
620
621
fi

Holger Levsen's avatar
Holger Levsen committed
622
623
624
625
#
# There's always some work left...
#	echo FIXME is ignored so check-jobs scripts can output templates requiring manual work
#
626
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
627
628
629
	rgrep FI[X]ME $BASEDIR/* | grep -v echo > $TMPFILE || true
	if [ -s $TMPFILE ] ; then
		echo
630
631
632
633
634
635
		# only show cucumber FIXMEs when deploying on jenkins-test-vm
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			cat $TMPFILE
		else
			cat $TMPFILE | grep -v cucumber
		fi
636
637
638
		echo
	fi
	rm -f $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
639
fi
Holger Levsen's avatar
Holger Levsen committed
640

641
#
642
# almost finally…
643
#
644
sudo touch $STAMP	# so on the next run, only configs newer than this file will be updated
Holger Levsen's avatar
Holger Levsen committed
645
explain "$(date) - finished deployment."
646

647
# finally!
648
case $HOSTNAME in
649
	# set time back to the future
650
	profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386)
651
652
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
653
	jenkins)
654
		MESSAGE="jenkins.d.n updated to $(cd $BASEDIR ; git describe --always)."
655
		kgb-client --conf /srv/jenkins/kgb/debian-qa.conf --relay-msg "$MESSAGE"
Holger Levsen's avatar
Holger Levsen committed
656
		;;
657
658
	*)	;;
esac
659
660
661
662

echo
figlet ok
echo
Holger Levsen's avatar
Holger Levsen committed
663
664
echo "__$HOSTNAME=ok__"