update_jdn.sh 20.2 KB
Newer Older
1
#!/bin/bash
2
# Copyright 2012-2016 Holger Levsen <holger@layer-acht.org>
3
4
# released under the GPLv=2

5
6
set -e

7
BASEDIR="$(dirname "$(readlink -e $0)")"
8
9
PVNAME=/dev/vdb      # LVM physical volume for jobs
VGNAME=jenkins01     # LVM volume group
10
STAMP=/var/log/jenkins/update-jenkins.stamp
11
TMPFILE=$(mktemp)
12
13
14
# The $@ below means that command line args get passed on to j-j-b
# which allows one to specify --flush-cache or --ignore-cache
JJB="jenkins-job-builder $@"
Holger Levsen's avatar
Holger Levsen committed
15

16
# so we can later run some commands only if $0 has been updated…
Philip Hands's avatar
Philip Hands committed
17
if [ -f $STAMP ] && [ $STAMP -nt $BASEDIR/$0 ] ; then
Holger Levsen's avatar
Holger Levsen committed
18
	UP2DATE=true
Philip Hands's avatar
Philip Hands committed
19
20
else
	UP2DATE=false
21
22
23
fi


Holger Levsen's avatar
Holger Levsen committed
24
explain() {
Holger Levsen's avatar
Holger Levsen committed
25
	echo "$HOSTNAME: $1"
Holger Levsen's avatar
Holger Levsen committed
26
27
}

28
29


30
echo "--------------------------------------------"
Holger Levsen's avatar
Holger Levsen committed
31
explain "$(date) - begin deployment update."
32

33
# some nodes need special treatment…
34
case $HOSTNAME in
35
	profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386)
Holger Levsen's avatar
Holger Levsen committed
36
37
38
		# set correct date
		sudo service ntp stop
		sudo ntpdate -b de.pool.ntp.org
39
40
41
42
		;;
	*)	;;
esac

43
44
45
#
# set up users and groups
#
46
if ! getent passwd jenkins > /dev/null ; then
47
	sudo addgroup --system jenkins
48
	sudo adduser --system --shell /bin/bash --home /var/lib/jenkins --ingroup jenkins --disabled-login jenkins
49
fi
Holger Levsen's avatar
Holger Levsen committed
50
if ! getent group jenkins-adm > /dev/null ; then
51
52
	sudo addgroup --system jenkins-adm
fi
53
if ! getent passwd jenkins-adm > /dev/null  ; then
Holger Levsen's avatar
Holger Levsen committed
54
	sudo adduser --system --shell /bin/bash --home /home/jenkins-adm --ingroup jenkins-adm --disabled-login jenkins-adm
Holger Levsen's avatar
Holger Levsen committed
55
	sudo usermod -G jenkins jenkins-adm
56
fi
Mattia Rizzolo's avatar
Mattia Rizzolo committed
57
if [ ! -d /home/jenkins-adm ]; then
Mattia Rizzolo's avatar
Mattia Rizzolo committed
58
59
60
    sudo mkdir /home/jenkins-adm
    sudo chown jenkins-adm.jenkins-adm /home/jenkins-adm
fi
61

62
63

declare -A user_host_groups u_shell
64
65

sudo_groups='jenkins,jenkins-adm,sudo,adm'
66
67
68
69
70
71
72
73
74

# if there's a need for host groups, a case statement on $HOSTNAME here that sets $GROUPNAME, say, should do the trick
# then you can define user_host_groups['phil','lvm_group']=... below
# and add checks for the GROUP version whereever the HOSTNAME is checked in the following code

user_host_groups['helmut','*']="$sudo_groups"
user_host_groups['holger','*']="$sudo_groups"
user_host_groups['holger','jenkins']="reproducible,${user_host_groups['holger','*']}"
user_host_groups['mattia','*']="$sudo_groups"
75
user_host_groups['mattia','jenkins']="reproducible,${user_host_groups['mattia','*']}"
76
user_host_groups['phil','jenkins-test-vm']="$sudo_groups,libvirt,libvirt-qemu"
Philip Hands's avatar
Philip Hands committed
77
user_host_groups['phil','profitbricks-build10-amd64']="$sudo_groups"
78
79
80
user_host_groups['phil','jenkins']=''
user_host_groups['lunar','jenkins']='reproducible'

81
82

u_shell['mattia']='/bin/zsh'
83
u_shell['jenkins-adm']='/bin/bash'
84

85
86
87
# get the users out of the user_host_groups array's index
users=$(for i in ${!user_host_groups[@]}; do echo ${i%,*} ; done | sort -u)

88
$UP2DATE || for user in ${users}; do
89
90
	# -v is a bashism to check for set variables, used here to see if this user is active on this host
	[ -v user_host_groups["$user","$HOSTNAME"] -o -v user_host_groups["$user",'*'] ] || continue
91

92
93
94
95
96
97
98
99
	# create the user
	if ! getent passwd $user > /dev/null ; then
		# adduser, defaulting to /bin/bash as shell
		sudo adduser --gecos "" --shell "${u_shell[$user]:-/bin/bash}" --disabled-password $user
	fi
	# add groups: first try the specific host, or if unset fall-back to default '*' setting
	for h in "$HOSTNAME" '*' ; do
		if [ -v user_host_groups["$user","$h"] ] ; then
Holger Levsen's avatar
Holger Levsen committed
100
			sudo usermod -G "${user_host_groups["$user","$h"]}" $user
101
			break
102
		fi
103
104
105
106
107
	done
	# add the user's keys (if any)
	if ls authorized_keys/${user}@*.pub >/dev/null 2>&1 ; then
		[ -d /var/lib/misc/userkeys ] || sudo mkdir -p /var/lib/misc/userkeys
		cat authorized_keys/${user}@*.pub | sudo tee /var/lib/misc/userkeys/${user} > /dev/null
108
	fi
109
110
done

111
# change defaults
112
113
114
$UP2DATE || grep -q '^AuthorizedKeysFile' /etc/ssh/sshd_config || {
	sudo sh -c "echo 'AuthorizedKeysFile /var/lib/misc/userkeys/%u %h/.ssh/authorized_keys' >> /etc/ssh/sshd_config"
	sudo service ssh reload
115
}
116
117
118
119
120
# change vagrants manual configuration on some armhf hosts
$UP2DATE || grep -q '/var/lib/misc/userkeys' /etc/ssh/sshd_config || {
	sudo sed -i "s#/var/lib/monkeysphere/authorized_keys/#/var/lib/misc/userkeys/#g" /etc/ssh/sshd_config
	sudo service ssh reload
}
121

122
123
124
125
126
sudo mkdir -p /srv/workspace
[ -d /srv/schroots ] || sudo mkdir -p /srv/schroots
[ -h /chroots ] || sudo ln -s /srv/workspace/chroots /chroots
[ -h /schroots ] || sudo ln -s /srv/schroots /schroots

127
if [ "$HOSTNAME" = "jenkins-test-vm" ] || [ "$HOSTNAME" = "profitbricks-build10-amd64" ] || [ "$HOSTNAME" = "profitbricks-build7-amd64" ] ; then
128
	# jenkins needs access to libvirt
129
	sudo adduser jenkins kvm
130
131
132
	sudo adduser jenkins libvirt
	sudo adduser jenkins libvirt-qemu

133
	# we need a directory for the VM's storage pools
134
	VM_POOL_DIR=/srv/lvc/vm-pools
135
	if [ ! -d $VM_POOL_DIR ] ; then
136
		sudo mkdir -p $VM_POOL_DIR
137
138
139
		sudo chown jenkins:libvirt-qemu $VM_POOL_DIR
		sudo chmod 775 $VM_POOL_DIR
	fi
140
141
142
143
144
145

	# tidy up after ourselves, for a while at least
	OLD_VM_POOL_DIR=/srv/workspace/vm-pools
	if [ -d "$OLD_VM_POOL_DIR" ] ; then
		sudo rm -r "$OLD_VM_POOL_DIR"
	fi
146
fi
147

148
149
# prepare tmpfs on some hosts
case $HOSTNAME in
Mattia Rizzolo's avatar
Mattia Rizzolo committed
150
151
152
153
	jenkins)
		TMPFSSIZE=100
		TMPSIZE=15
		;;
Holger Levsen's avatar
Holger Levsen committed
154
	profitbricks-build9-amd64)
155
		TMPFSSIZE=40
Mattia Rizzolo's avatar
Mattia Rizzolo committed
156
157
		TMPSIZE=8
		;;
Holger Levsen's avatar
Holger Levsen committed
158
	profitbricks-build*)
Mattia Rizzolo's avatar
Mattia Rizzolo committed
159
160
161
		TMPFSSIZE=200
		TMPSIZE=15
		;;
162
163
164
	*) ;;
esac
case $HOSTNAME in
165
166
167
168
169
170
171
	profitbricks-build*i386)
		if ! grep -q '/srv/workspace' /etc/fstab; then
			echo "Warning: you need to manually create a /srv/workspace partition on i386 nodes, exiting."
			exit 1
		fi
		;;
	jenkins|profitbricks-build*amd64)
172
		if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
173
			echo "tmpfs		/srv/workspace	tmpfs	defaults,size=${TMPFSSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null  
174
		fi
Mattia Rizzolo's avatar
Mattia Rizzolo committed
175
176
177
		if ! grep -q '^tmpfs\s\+/tmp\s' /etc/fstab; then
			echo "tmpfs		/tmp	tmpfs	defaults,size=${TMPSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null
		fi
178
179
		if ! mountpoint -q /srv/workspace; then
			if test -z "$(ls -A /srv/workspace)"; then
180
				sudo mount /srv/workspace
181
			else
182
				explain "WARNING: mountpoint /srv/workspace is non-empty."
183
184
185
186
187
			fi
		fi
		;;
	*) ;;
esac
188

189
# make sure needed directories exists - some directories will not be needed on all hosts...
190
for directory in /schroots /srv/reproducible-results /srv/d-i /srv/udebs /srv/live-build /var/log/jenkins/ /srv/jenkins /srv/jenkins/pseudo-hosts /srv/workspace/chroots ; do
191
192
193
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
	fi
Holger Levsen's avatar
Holger Levsen committed
194
	sudo chown jenkins.jenkins $directory
195
done
Holger Levsen's avatar
Holger Levsen committed
196
for directory in /srv/jenkins ; do
197
198
199
200
201
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
		sudo chown jenkins-adm.jenkins-adm $directory
	fi
done
202

203
if ! test -h /chroots; then
Holger Levsen's avatar
Holger Levsen committed
204
	sudo rmdir /chroots || sudo rm -f /chroots # do not recurse
205
	if test -e /chroots; then
Holger Levsen's avatar
Holger Levsen committed
206
		explain "/chroots could not be cleared."
207
	else
208
		sudo ln -s /srv/workspace/chroots /chroots
209
210
211
	fi
fi

212
213
214
215
216
# only on Debian systems
if [ -f /etc/debian_version ] ; then
	#
	# install packages we need
	#
217
	if [ $BASEDIR/$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then
Holger Levsen's avatar
Holger Levsen committed
218
219
		DEBS=" 
			bash-completion 
220
221
			bc
			bsd-mailx
Holger Levsen's avatar
Holger Levsen committed
222
223
			curl 
			debootstrap 
224
225
			devscripts
			eatmydata
226
			etckeeper
227
			figlet
Holger Levsen's avatar
Holger Levsen committed
228
			git
229
			haveged
Holger Levsen's avatar
Holger Levsen committed
230
			htop
231
			less
232
			locales-all
Mattia Rizzolo's avatar
Mattia Rizzolo committed
233
			lsof
234
235
			molly-guard
			moreutils
236
			munin-node
237
238
			munin-plugins-extra
			netcat-traditional
239
240
			ntp
			ntpdate
241
			pigz 
242
			postfix
243
			procmail
244
			psmisc
245
			python3-psycopg2 
Holger Levsen's avatar
Holger Levsen committed
246
			schroot 
247
248
			screen
			slay
249
			stunnel
Holger Levsen's avatar
Holger Levsen committed
250
251
252
253
254
			subversion 
			subversion-tools 
			sudo 
			unzip 
			vim 
255
			zsh
256
			"
257
		# install squid everywhere except on the armhf nodes
258
		case $HOSTNAME in
259
			jenkins|jenkins-test-vm|profitbricks-build*) DEBS="$DEBS
260
261
				squid3"
			   ;;
262
263
			*) ;;
		esac
264
265
		# needed to run the 2nd reproducible builds nodes in the future...
		case $HOSTNAME in
266
			profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386) DEBS="$DEBS ntpdate" ;;
267
268
			*) ;;
		esac
269
		# needed to run coreboot/openwrt/lede/netbsd/fedora jobs
270
		case $HOSTNAME in
271
			profitbricks-build3-amd64|profitbricks-build4-amd64) DEBS="$DEBS
Holger Levsen's avatar
Holger Levsen committed
272
				bison
273
				ca-certificates
Holger Levsen's avatar
Holger Levsen committed
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
				cmake
				diffutils
				findutils
				flex
				g++
				gawk
				gcc
				git
				grep
				iasl
				libc6-dev
				libncurses5-dev
				libssl-dev
				locales-all
				kgb-client
				m4
				make
291
292
293
				python3-clint
				python3-git
				python3-requests
Holger Levsen's avatar
Holger Levsen committed
294
295
				python3-yaml
				subversion
296
				sysvinit-core
297
				tree
Holger Levsen's avatar
Holger Levsen committed
298
299
300
				unzip
				util-linux
				zlib1g-dev"
301
302
303
			;;
			*) ;;
		esac
304
305
306
		# needed to run fdroid jobs
		case $HOSTNAME in
			profitbricks-build7-amd64) DEBS="$DEBS
307
				libvirt-clients
308
309
				libvirt-daemon
				libvirt-daemon-system
310
				python3-libvirt
311
				python3-vagrant
312
				qemu-kvm
313
314
315
				vagrant
				vagrant-mutate
				vagrant-libvirt"
316
			;;
317
318
			*) ;;
		esac
319
		# cucumber dependencies (for lvc jobs)
320
321
322
323
324
		case $HOSTNAME in
			profitbricks-build10-amd64|jenkins-test-vm) DEBS="$DEBS
				cucumber
				tesseract-ocr
				i18nspector
Holger Levsen's avatar
Holger Levsen committed
325
				imagemagick
326
327
				libav-tools
				libsikuli-script-java
328
				libvirt-bin
329
330
331
332
				libvirt-dev
				ovmf
				python-jabberbot
				python-potr
333
				python3-yaml
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
				ruby-guestfs
				ruby-libvirt
				ruby-net-irc
				ruby-packetfu
				ruby-rb-inotify
				ruby-rjb
				ruby-test-unit
				tcpdump
				unclutter
				virt-viewer
				x264
				xvfb
				x11vnc"
			   ;;
			*) ;;
		esac
350
351
352
353
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			# for phil only
			DEBS="$DEBS postfix-pcre"
		fi
354
		if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
Holger Levsen's avatar
Holger Levsen committed
355
356
357
358
			MASTERDEBS=" 
				apache2 
				apt-file 
				apt-listchanges 
359
				asciidoc
Holger Levsen's avatar
Holger Levsen committed
360
361
362
363
364
365
366
367
368
369
370
				binfmt-support 
				bison 
				build-essential 
				calamaris 
				cmake 
				cron-apt 
				csvtool 
				dnsmasq-base 
				dose-extra 
				dstat 
				figlet 
Holger Levsen's avatar
Holger Levsen committed
371
				flex
Holger Levsen's avatar
Holger Levsen committed
372
				gawk 
373
374
				ghc
				git-notifier 
Holger Levsen's avatar
Holger Levsen committed
375
376
377
378
				gocr 
				graphviz 
				iasl 
				imagemagick 
379
380
				ip2host
				jekyll
381
				kgb-client
Holger Levsen's avatar
Holger Levsen committed
382
383
384
385
386
387
388
389
390
391
				libapache2-mod-macro 
				libcap2-bin 
				libfile-touch-perl 
				libguestfs-tools 
				libjson-rpc-perl 
				libsoap-lite-perl 
				libvirt0 
				libvirt-bin 
				libvpx1 
				libxslt1-dev 
392
				linux-image-amd64
Holger Levsen's avatar
Holger Levsen committed
393
394
395
396
397
398
399
400
				moreutils 
				mr 
				mtr-tiny 
				munin 
				ntp 
				openbios-ppc 
				openbios-sparc 
				openjdk-7-jre 
401
				pandoc
Holger Levsen's avatar
Holger Levsen committed
402
403
404
405
				postgresql-client-9.4 
				poxml 
				procmail 
				python3-debian 
406
				python3-pystache
407
408
409
				python3-sqlalchemy
				python3-xdg
				python3-yaml
Holger Levsen's avatar
Holger Levsen committed
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
				python-arpy 
				python-hachoir-metadata 
				python-imaging 
				python-lzma 
				python-pip 
				python-rpy2 
				python-setuptools 
				python-twisted 
				python-yaml 
				qemu 
				qemu-kvm 
				qemu-system-x86 
				qemu-user-static 
				radvd 
				ruby-json 
				ruby-rspec 
				seabios 
				shorewall 
				shorewall6 
				sqlite3 
				syslinux 
431
				tor
Holger Levsen's avatar
Holger Levsen committed
432
				vncsnapshot 
433
434
				vnstat
				whohas
Holger Levsen's avatar
Holger Levsen committed
435
				x11-apps 
436
				xtightvncviewer
437
				xvfb
438
				xvkbd
439
				zutils"
440
441
442
		else
			MASTERDEBS=""
		fi
443
		$UP2DATE || sudo apt-get update
444
445
		$UP2DATE || sudo apt-get install $DEBS $MASTERDEBS
		$UP2DATE || sudo apt-get install -t jessie-backports \
446
				pbuilder lintian || echo "this should only fail on the first install"
447
		#		botch
448
		# we need mock from bpo to build current fedora
449
450
		if [ "$HOSTNAME" = "profitbricks-build3-amd64" ] || [ "$HOSTNAME" = "profitbricks-build4-amd64" ] || [ "$HOSTNAME" = "jenkins" ] ; then
			$UP2DATE || sudo apt-get install -t jessie-backports mock \
451
				|| echo "this should only fail on the first install"
452
		fi
453
		# for varying kernels
454
455
		# we use bpo kernels on pb-build5+6 (and i386 kernel on pb-build2-i386)
		if [ "$HOSTNAME" = "profitbricks-build5-amd64" ] || [ "$HOSTNAME" = "profitbricks-build6-i386" ]; then
456
			$UP2DATE || sudo apt-get install -t jessie-backports linux-image-amd64 || echo "this should only fail on the first install"
457
		fi
458
		# only needed on the main node
Holger Levsen's avatar
Holger Levsen committed
459
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
460
			$UP2DATE || sudo apt-get install -t jessie-backports jenkins-job-builder || echo "this should only fail on the first install"
Holger Levsen's avatar
Holger Levsen committed
461
		elif [ "$HOSTNAME" = "jenkins" ] ; then
462
			$UP2DATE || sudo apt-get install -t jessie-backports ffmpeg libav-tools python3-popcon jenkins-job-builder
463
		fi
Holger Levsen's avatar
Holger Levsen committed
464
		explain "packages installed."
465
	else
Holger Levsen's avatar
Holger Levsen committed
466
		explain "no new packages to be installed."
467
	fi
468
fi
469

470
#
471
# deploy package configuration in /etc and /usr/local/
472
#
Holger Levsen's avatar
Holger Levsen committed
473
cd $BASEDIR
474
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/etc/* /etc
475
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/usr/* /usr/
476
477
478
479
480

#
# more configuration than a simple cp can do
#
sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins
481
sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm
482
483

if [ "$HOSTNAME" = "jenkins" ] ; then
484
485
486
487
488
489
490
491
492
493
494
495
	if [ $BASEDIR/hosts/$HOSTNAME/etc/apache2 -nt $STAMP ] || [ ! -f $STAMP ] ; then
		if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
			sudo a2enmod proxy
			sudo a2enmod proxy_http
			sudo a2enmod rewrite
			sudo a2enmod ssl
			sudo a2enmod headers
			sudo a2enmod macro
			sudo a2enmod filter
		fi
		sudo a2ensite -q jenkins.debian.net
		sudo a2enconf -q munin
496
		sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.debian.net.conf
497
498
499
		# for reproducible.d.n url rewriting:
		[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
		sudo service apache2 reload
500
501
502
	fi
fi

503
if [ $BASEDIR/hosts/$HOSTNAME/etc/munin -nt $STAMP ] || [ ! -f $STAMP ] ; then
504
	cd /etc/munin/plugins
505
	sudo rm -f postfix_* open_inodes interrupts irqstats threads proc_pri vmstat if_err_* exim_* netstat fw_forwarded_local fw_packets forks open_files users nfs* iostat_ios 2>/dev/null
506
	case $HOSTNAME in
Holger Levsen's avatar
Holger Levsen committed
507
			jenkins|profitbricks-build*) [ -L /etc/munin/plugins/squid_cache ] || for i in squid_cache squid_objectsize squid_requests squid_traffic ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
508
509
			*)	;;
	esac
510
511
512
	if [ "$HOSTNAME" != "jenkins" ] && [ -L /etc/munin/plugins/iostat ] ; then
		sudo rm /etc/munin/plugins/iostat
	fi
513
514
	if [ "$HOSTNAME" = "jenkins" ] && [ ! -L /etc/munin/plugins/apache_accesses ] ; then
		for i in apache_accesses apache_volume ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done
515
		sudo ln -s /usr/share/munin/plugins/loggrep jenkins_oom
516
	fi
517
fi
Holger Levsen's avatar
Holger Levsen committed
518
explain "packages configured."
Holger Levsen's avatar
Holger Levsen committed
519
520
521
522

#
# install the heart of jenkins.debian.net
#
Holger Levsen's avatar
Holger Levsen committed
523
cd $BASEDIR
524
[ -d /srv/jenkins/features ] && sudo rm -rf /srv/jenkins/features
525
for dir in bin logparse cucumber live mustache-templates ; do
526
527
	sudo mkdir -p /srv/jenkins/$dir
	sudo rsync -rpt --delete $dir/ /srv/jenkins/$dir/
Holger Levsen's avatar
Holger Levsen committed
528
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
529
done
530
531
HOST_JOBS="hosts/$HOSTNAME/job-cfg"
if [ -e "$HOST_JOBS" ] ; then
532
	sudo rsync -rpt --copy-links --delete "$HOST_JOBS/" /srv/jenkins/job-cfg/
533
534
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
else
Philip Hands's avatar
Philip Hands committed
535
536
	# tidying up ... assuming that we don't want clutter on peripheral servers
	[ -d /srv/jenkins/job-cfg ] && sudo rm -rf /srv/jenkins/job-cfg
537
538
fi

539

Holger Levsen's avatar
Holger Levsen committed
540
sudo mkdir -p /var/lib/jenkins/.ssh
541
542
543
544
545
546
if [ "$HOSTNAME" = "jenkins" ] ; then
	sudo cp jenkins-home/procmailrc /var/lib/jenkins/.procmailrc
	sudo cp jenkins-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
else
	sudo cp jenkins-nodes-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
fi
Holger Levsen's avatar
Holger Levsen committed
547
548
549
sudo chown -R jenkins:jenkins /var/lib/jenkins/.ssh
sudo chmod 700 /var/lib/jenkins/.ssh
sudo chmod 600 /var/lib/jenkins/.ssh/authorized_keys
Holger Levsen's avatar
Holger Levsen committed
550
explain "scripts and configurations for jenkins updated."
551

552
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
553
	sudo cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
554
	git log | grep ^Author| cut -d " " -f2-|sort -u -f > $TMPFILE
555
	echo "----" >> $TMPFILE
556
	sudo tee /var/lib/jenkins/userContent/THANKS > /dev/null < THANKS.head
Holger Levsen's avatar
Holger Levsen committed
557
558
	# samuel, lunar, josch and phil committed with several commiters, only display one
	grep -v -e "samuel.thibault@ens-lyon.org" -e Lunar -e "j.schauer@email.de" -e "mattia@mapreri.org" -e "phil@jenkins-test-vm" $TMPFILE | sudo tee -a /var/lib/jenkins/userContent/THANKS > /dev/null
559
	rm $TMPFILE
560
	TMPDIR=$(mktemp -d -t update-jdn-XXXXXXXX)
561
	sudo cp -pr userContent $TMPDIR/
Holger Levsen's avatar
Holger Levsen committed
562
563
564
	sudo chown -R jenkins.jenkins $TMPDIR
	sudo cp -pr $TMPDIR/userContent  /var/lib/jenkins/
	sudo rm -r $TMPDIR > /dev/null
565
566
567
568
569
570
571
572
573
	cd /var/lib/jenkins/userContent/
	ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
	[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
	[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
	[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
	[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
	diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
	mv THANKS .THANKS
	rm TODO README INSTALL CONTRIBUTING
574
	sudo chown jenkins.jenkins /var/lib/jenkins/userContent/*html
Holger Levsen's avatar
Holger Levsen committed
575
	explain "user content for jenkins updated."
576
fi
577

578
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
579
580
581
582
583
584
585
	#
	# run jenkins-job-builder to update jobs if needed
	#     (using sudo because /etc/jenkins_jobs is root:root 700)
	#
	cd /srv/jenkins/job-cfg
	for metaconfig in *.yaml.py ; do
	# there are both python2 and python3 scripts here
Philip Hands's avatar
Philip Hands committed
586
		[ -e ./$metaconfig ] || continue
587
		./$metaconfig > $TMPFILE
588
589
		if ! sudo -u jenkins-adm cmp -s ${metaconfig%.py} - < $TMPFILE ; then
			sudo -u jenkins-adm tee ${metaconfig%.py} > /dev/null < $TMPFILE
590
591
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
592
	rm -f $TMPFILE
593
	for config in *.yaml ; do
594
595
596
597
598
599
600
		# do update, if
		# no stamp file exist or
		# no .py file exists and config is newer than stamp or
		# a .py file exists and .py file is newer than stamp
		if [ ! -f $STAMP ] || \
		 ( [ ! -f $config.py ] && [ $config -nt $STAMP ] ) || \
		 ( [ -f $config.py ] && [ $config.py -nt $STAMP ] ) ; then
601
			$JJB update $config
602
603
604
605
		else
			echo "$config has not changed, nothing to do."
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
606
	explain "jenkins jobs updated."
607
fi
608

609
#
Holger Levsen's avatar
Holger Levsen committed
610
# configure git for jenkins
611
#
Holger Levsen's avatar
Holger Levsen committed
612
613
614
615
if [ "$(sudo su - jenkins -c 'git config --get user.email')" != "jenkins@jenkins.debian.net" ] ; then
	sudo su - jenkins -c "git config --global user.email jenkins@jenkins.debian.net"
	sudo su - jenkins -c "git config --global user.name Jenkins"
fi
Holger Levsen's avatar
Holger Levsen committed
616

617
618
619
620
621
if [ "$HOSTNAME" = "jenkins" ] ; then
	#
	# creating LVM volume group for jobs
	#
	if [ "$PVNAME" = "" ]; then
Philip Hands's avatar
Philip Hands committed
622
623
624
		figlet -f banner Error
		explain "you must set \$PVNAME to physical volume pathname, exiting."
		exit 1
Holger Levsen's avatar
Holger Levsen committed
625
	elif ! $UP2DATE ; then
Philip Hands's avatar
Philip Hands committed
626
627
628
629
630
631
		if ! sudo pvs $PVNAME >/dev/null 2>&1; then
			sudo pvcreate $PVNAME
		fi
		if ! sudo vgs $VGNAME >/dev/null 2>&1; then
			sudo vgcreate $VGNAME $PVNAME
		fi
632
	fi
633
634
635
636
637
fi

#
# generate the kgb-client configurations
#
638
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "profitbricks-build3-amd64" ] || [ "$HOSTNAME" = "profitbricks-build4-amd64" ] || [ "$HOSTNAME" = "profitbricks-build7-amd64" ] ; then
639
640
641
	cd $BASEDIR
	KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
	if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
Philip Hands's avatar
Philip Hands committed
642
643
644
645
646
647
		# the last condition is to assure the files are owned by the right user/team
		if [ "$KGB_SECRETS" -nt $STAMP ] || [ ! -f $STAMP ] ; then
			sudo -u jenkins-adm "./deploy_kgb.py"
		else
			explain "kgb-client configuration unchanged, nothing to do."
		fi
648
	else
Philip Hands's avatar
Philip Hands committed
649
650
651
		figlet -f banner Warning
		echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
		echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
Holger Levsen's avatar
Holger Levsen committed
652
653
		echo "/srv/jenkins/kgb should be mode 755 and owned by jenkins-adm:root."
		echo "/srv/jenkins/kgb/client-status should be mode 755 and owned by jenkins:jenkins."
654
	fi
655
656
fi

Holger Levsen's avatar
Holger Levsen committed
657
658
659
660
#
# There's always some work left...
#	echo FIXME is ignored so check-jobs scripts can output templates requiring manual work
#
661
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
662
663
664
	rgrep FI[X]ME $BASEDIR/* | grep -v echo > $TMPFILE || true
	if [ -s $TMPFILE ] ; then
		echo
665
666
667
668
669
670
		# only show cucumber FIXMEs when deploying on jenkins-test-vm
		if [ "$HOSTNAME" = "jenkins-test-vm" ] ; then
			cat $TMPFILE
		else
			cat $TMPFILE | grep -v cucumber
		fi
671
672
673
		echo
	fi
	rm -f $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
674
fi
Holger Levsen's avatar
Holger Levsen committed
675

676
#
677
# almost finally…
678
#
679
sudo touch $STAMP	# so on the next run, only configs newer than this file will be updated
Holger Levsen's avatar
Holger Levsen committed
680
explain "$(date) - finished deployment."
681

682
# finally!
683
case $HOSTNAME in
684
	# set time back to the future
685
	profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386)
686
687
		sudo date --set="+398 days +6 hours + 23 minutes"
		;;
688
	jenkins)
689
		# notify irc
690
		MESSAGE="jenkins.d.n updated to $(cd $BASEDIR ; git describe --always)."
691
		kgb-client --conf /srv/jenkins/kgb/debian-qa.conf --relay-msg "$MESSAGE"
Holger Levsen's avatar
Holger Levsen committed
692
		;;
693
694
	*)	;;
esac
695
696
697
698

echo
figlet ok
echo
Holger Levsen's avatar
Holger Levsen committed
699
700
echo "__$HOSTNAME=ok__"