update_jdn.sh 12.5 KB
Newer Older
1
2
#!/bin/bash

3
# Copyright 2012-2015 Holger Levsen <holger@layer-acht.org>
4
5
# released under the GPLv=2

Holger Levsen's avatar
Holger Levsen committed
6
BASEDIR=$HOME/jenkins.debian.net
7
8
PVNAME=/dev/vdb      # LVM physical volume for jobs
VGNAME=jenkins01     # LVM volume group
9
STAMP=/var/log/jenkins/update-jenkins.stamp
10
TMPFILE=$(mktemp)
Holger Levsen's avatar
Holger Levsen committed
11

Holger Levsen's avatar
Holger Levsen committed
12
explain() {
Holger Levsen's avatar
Holger Levsen committed
13
	echo "$HOSTNAME: $1"
Holger Levsen's avatar
Holger Levsen committed
14
15
}

16
echo "--------------------------------------------"
Holger Levsen's avatar
Holger Levsen committed
17
explain "$(date) - begin deployment update."
18
19
20
#
# set up users and groups
#
21
if ! getent passwd jenkins > /dev/null ; then
22
	sudo addgroup --system jenkins
23
	sudo adduser --system --shell /bin/bash --home /var/lib/jenkins --ingroup jenkins --disabled-login jenkins
24
fi
Holger Levsen's avatar
Holger Levsen committed
25
if ! getent group jenkins-adm > /dev/null ; then
26
27
	sudo addgroup --system jenkins-adm
fi
28
if ! getent passwd jenkins-adm > /dev/null  ; then
29
	sudo adduser --system --shell /bin/bash --no-create-home --ingroup jenkins-adm --disabled-login --no-create-home jenkins-adm
Holger Levsen's avatar
Holger Levsen committed
30
	sudo usermod -G jenkins jenkins-adm
31
fi
32
33
34
35
36
for user in helmut holger mattia lunar ; do
	if [ "$user" = "lunar" ] && [ "$HOSTNAME" != "jenkins" ] ; then
		# lunar only wants to configure jekyll
		continue
	fi
37
	if ! getent passwd $user > /dev/null ; then
38
		if [ "$user" = "mattia" ] ; then
39
			usershell=/bin/zsh
40
		else
41
			usershell=/bin/bash
42
		fi
43
		sudo adduser --gecos "" --shell "$usershell" --disabled-password $user
Holger Levsen's avatar
Holger Levsen committed
44
		if [ "$user" = "holger" ] ; then
45
			sudo usermod -G jenkins,jenkins-adm,sudo,adm $user
46
		elif [ "$user" != "lunar" ] ; then
47
48
			sudo usermod -G jenkins,jenkins-adm $user
		fi
49
50
51
	fi
done

52
sudo mkdir -p /srv/workspace
53
[ -d /srv/schroots ] || sudo mkdir -p /srv/schroots
54
[ -h /chroots ] || sudo ln -s /srv/workspace/chroots /chroots
55
[ -h /schroots ] || sudo ln -s /srv/schroots /schroots
56

57
58
59
# prepare tmpfs on some hosts
case $HOSTNAME in
	jenkins)			 	TMPFSSIZE=100 ;;
60
	profitbricks-build4-amd64) 		TMPFSSIZE=32 ;;
61
	profitbricks-build?-amd64) 		TMPFSSIZE=200 ;;
62
63
64
65
66
	*) ;;
esac
case $HOSTNAME in
	jenkins|profitbricks-build?-amd64)
		if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
67
			echo "tmpfs		/srv/workspace	tmpfs	defaults,size=${TMPFSSIZE}g	0	0" | sudo tee -a /etc/fstab >/dev/null  
68
		fi
69
70
		if ! mountpoint -q /srv/workspace; then
			if test -z "$(ls -A /srv/workspace)"; then
71
				sudo mount /srv/workspace
72
			else
73
				explain "WARNING: mountpoint /srv/workspace is non-empty."
74
75
76
77
78
			fi
		fi
		;;
	*) ;;
esac
79

80
# make sure needed directories exists - some directories will not be needed on all hosts...
81
for directory in /schroots /srv/reproducible-results /srv/d-i /srv/live-build /var/log/jenkins/ /srv/jenkins /srv/jenkins/pseudo-hosts /srv/workspace/chroots ; do
82
83
84
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
	fi
Holger Levsen's avatar
Holger Levsen committed
85
	sudo chown jenkins.jenkins $directory
86
done
Holger Levsen's avatar
Holger Levsen committed
87
for directory in /srv/jenkins ; do
88
89
90
91
92
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
		sudo chown jenkins-adm.jenkins-adm $directory
	fi
done
93

94
if ! test -h /chroots; then
Holger Levsen's avatar
Holger Levsen committed
95
	sudo rmdir /chroots || sudo rm -f /chroots # do not recurse
96
	if test -e /chroots; then
Holger Levsen's avatar
Holger Levsen committed
97
		explain "/chroots could not be cleared."
98
	else
99
		sudo ln -s /srv/workspace/chroots /chroots
100
101
102
	fi
fi

103
104
105
106
107
108
# only on Debian systems
if [ -f /etc/debian_version ] ; then
	#
	# install packages we need
	#
	if [ ./$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then
Holger Levsen's avatar
Holger Levsen committed
109
110
		DEBS=" 
			bash-completion 
111
112
			bc
			bsd-mailx
Holger Levsen's avatar
Holger Levsen committed
113
114
115
			curl 
			debootstrap 
			devscripts 
116
			etckeeper
117
			figlet
Holger Levsen's avatar
Holger Levsen committed
118
			git
119
			haveged
Holger Levsen's avatar
Holger Levsen committed
120
			htop
121
			less
122
123
			molly-guard
			moreutils
124
125
			munin-node
			munin-plugins-extra 
126
			pigz 
127
			postfix
128
			procmail
129
			python3-psycopg2 
Holger Levsen's avatar
Holger Levsen committed
130
131
132
133
134
135
136
			schroot 
			screen 
			subversion 
			subversion-tools 
			sudo 
			unzip 
			vim 
137
			zsh
138
			"
139
140
141
142
		case $HOSTNAME in
			jenkins|profitbricks-build?-amd64) DEBS="$DEBS squid3" ;;
			*) ;;
		esac
143
144
		# needed to run the 2nd reproducible builds nodes in the future...
		case $HOSTNAME in
145
			bpi0|hb0|profitbricks-build5-amd64|profitbricks-build6-amd64) DEBS="$DEBS ntpdate" ;;
146
147
			*) ;;
		esac
148
149
		# needed to run coreboot/openwrt/netbsd jobs
		case $HOSTNAME in
Holger Levsen's avatar
Holger Levsen committed
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
			profitbricks-build3-amd64) DEBS="$DEBS
				bison
				cmake
				diffutils
				findutils
				flex
				g++
				gawk
				gcc
				git
				grep
				iasl
				libc6-dev
				libncurses5-dev
				libssl-dev
				locales-all
				kgb-client
				m4
				make
				python3-yaml
				subversion
171
				tree
Holger Levsen's avatar
Holger Levsen committed
172
173
174
175
				unzip
				util-linux
				zlib1g-dev"
			   ;;
176
177
			*) ;;
		esac
178
		if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
			MASTERDEBS=" 
				apache2 
				apt-file 
				apt-listchanges 
				binfmt-support 
				bison 
				build-essential 
				calamaris 
				cmake 
				cron-apt 
				csvtool 
				cucumber 
				dnsmasq-base 
				dose-extra 
				dstat 
				figlet 
				flex 
				gawk 
197
198
				ghc
				git-notifier 
Holger Levsen's avatar
Holger Levsen committed
199
200
201
202
				gocr 
				graphviz 
				iasl 
				imagemagick 
203
204
				ip2host
				jekyll
205
				kgb-client
Holger Levsen's avatar
Holger Levsen committed
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
				libapache2-mod-macro 
				libav-tools 
				libcap2-bin 
				libfile-touch-perl 
				libguestfs-tools 
				libjson-rpc-perl 
				libsikuli-script-java 
				libsoap-lite-perl 
				libvirt0 
				libvirt-bin 
				libvirt-dev 
				libvpx1 
				libxslt1-dev 
				linux-image-amd64 
				mock 
				moreutils 
				mr 
				mtr-tiny 
				munin 
				ntp 
				openbios-ppc 
				openbios-sparc 
				openjdk-7-jre 
229
230
				ovmf
				pandoc
Holger Levsen's avatar
Holger Levsen committed
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
				postgresql-client-9.4 
				poxml 
				procmail 
				python3-debian 
				python3-yaml 
				python-arpy 
				python-hachoir-metadata 
				python-imaging 
				python-lzma 
				python-pip 
				python-rpy2 
				python-setuptools 
				python-twisted 
				python-yaml 
				qemu 
				qemu-kvm 
				qemu-system-x86 
				qemu-user-static 
				radvd 
				ruby-json 
				ruby-libvirt 
				ruby-packetfu 
				ruby-rjb 
				ruby-rspec 
				seabios 
				shorewall 
				shorewall6 
				sqlite3 
				syslinux 
				tcpdump 
				unclutter 
				virt-viewer 
				vncsnapshot 
				vnstat 
				x11-apps 
				x11vnc 
				xtightvncviewer 
				xvfb 
				zutils 
270
271
272
273
				sysvinit-core"
		else
			MASTERDEBS=""
		fi
Holger Levsen's avatar
Holger Levsen committed
274
		sudo apt-get update
Holger Levsen's avatar
Holger Levsen committed
275
		sudo apt-get install $DEBS $MASTERDEBS
276
277
278
		sudo apt-get install -t jessie-backports \
				pbuilder
		#		botch
279
		# to vary kernels
280
		if [ "$HOSTNAME" = "profitbricks-build5-amd64" ] || [ "$HOSTNAME" = "profitbricks-build6-amd64" ]; then
281
282
			sudo apt-get install -t jessie-backports linux-image-amd64
		fi
Holger Levsen's avatar
Holger Levsen committed
283
		explain "packages installed."
284
	else
Holger Levsen's avatar
Holger Levsen committed
285
		explain "no new packages to be installed."
286
	fi
287
fi
288

289
#
290
# deploy package configuration in /etc and /usr/local/
291
#
Holger Levsen's avatar
Holger Levsen committed
292
cd $BASEDIR
293
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/etc/* /etc
294
sudo cp --preserve=mode,timestamps -r hosts/$HOSTNAME/usr/local/* /usr/local/
295
296
297
298
299

#
# more configuration than a simple cp can do
#
sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins
300
sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm
301
302

if [ "$HOSTNAME" = "jenkins" ] ; then
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
	if [ $BASEDIR/hosts/$HOSTNAME/etc/apache2 -nt $STAMP ] || [ ! -f $STAMP ] ; then
		if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
			sudo a2enmod proxy
			sudo a2enmod proxy_http
			sudo a2enmod rewrite
			sudo a2enmod ssl
			sudo a2enmod headers
			sudo a2enmod macro
			sudo a2enmod filter
		fi
		sudo a2ensite -q jenkins.debian.net
		sudo a2enconf -q munin
		sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.conf
		# for reproducible.d.n url rewriting:
		[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
		sudo service apache2 reload
319
320
321
	fi
fi

322
if [ $BASEDIR/hosts/$HOSTNAME/etc/munin -nt $STAMP ] || [ ! -f $STAMP ] ; then
323
	cd /etc/munin/plugins
324
	sudo rm -f postfix_* open_inodes interrupts irqstats threads proc_pri vmstat if_err_* exim_* netstat fw_forwarded_local fw_packets forks open_files users nfs* iostat_ios 2>/dev/null
325
326
327
328
	case $HOSTNAME in
			jenkins|profitbricks-build?-amd64) [ -L /etc/munin/plugins/squid_cache ] || for i in squid_cache squid_objectsize squid_requests squid_traffic ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done ;;
			*)	;;
	esac
329
330
	if [ "$HOSTNAME" = "jenkins" ] && [ ! -L /etc/munin/plugins/apache_accesses ] ; then
		for i in apache_accesses apache_volume ; do sudo ln -s /usr/share/munin/plugins/$i $i ; done
331
		sudo ln -s /usr/share/munin/plugins/loggrep jenkins_oom
332
333
	fi
	sudo service munin-node force-reload
334
fi
Holger Levsen's avatar
Holger Levsen committed
335
explain "packages configured."
Holger Levsen's avatar
Holger Levsen committed
336
337
338
339

#
# install the heart of jenkins.debian.net
#
Holger Levsen's avatar
Holger Levsen committed
340
cd $BASEDIR
341
for dir in bin logparse job-cfg features live ; do
342
	sudo cp --preserve=mode,timestamps -r $dir /srv/jenkins/
Holger Levsen's avatar
Holger Levsen committed
343
	sudo chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
344
done
Holger Levsen's avatar
Holger Levsen committed
345
sudo mkdir -p /var/lib/jenkins/.ssh
346
347
348
349
350
351
if [ "$HOSTNAME" = "jenkins" ] ; then
	sudo cp jenkins-home/procmailrc /var/lib/jenkins/.procmailrc
	sudo cp jenkins-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
else
	sudo cp jenkins-nodes-home/authorized_keys /var/lib/jenkins/.ssh/authorized_keys
fi
Holger Levsen's avatar
Holger Levsen committed
352
353
354
sudo chown -R jenkins:jenkins /var/lib/jenkins/.ssh
sudo chmod 700 /var/lib/jenkins/.ssh
sudo chmod 600 /var/lib/jenkins/.ssh/authorized_keys
Holger Levsen's avatar
Holger Levsen committed
355
explain "scripts and configurations for jenkins updated."
356

357
if [ "$HOSTNAME" = "jenkins" ] ; then
Holger Levsen's avatar
Holger Levsen committed
358
	sudo cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
359
360
	git log | grep ^Author| cut -d " " -f2-|sort -u > $TMPFILE
	echo "----" >> $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
361
	sudo cat THANKS.head > /var/lib/jenkins/userContent/THANKS
362
363
	# samuel, lunar and josch committed with several commiters, only display one
	grep -v "samuel.thibault@ens-lyon.org" $TMPFILE | grep -v Lunar | sudo grep -v "j.schauer@email.de" >> /var/lib/jenkins/userContent/THANKS
364
	rm $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
365
	sudo cp -pr userContent /var/lib/jenkins/
366
367
368
369
370
371
372
373
374
	cd /var/lib/jenkins/userContent/
	ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
	[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
	[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
	[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
	[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
	diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
	mv THANKS .THANKS
	rm TODO README INSTALL CONTRIBUTING
375
	sudo chown -R jenkins.jenkins /var/lib/jenkins/userContent
Holger Levsen's avatar
Holger Levsen committed
376
	explain "user content for jenkins updated."
377
378
379
380
381
382
383
384
385
386
387
388
389

	#
	# run jenkins-job-builder to update jobs if needed
	#     (using sudo because /etc/jenkins_jobs is root:root 700)
	#
	cd /srv/jenkins/job-cfg
	for metaconfig in *.yaml.py ; do
	# there are both python2 and python3 scripts here
		./$metaconfig > $TMPFILE
		if ! $(diff ${metaconfig%.py} $TMPFILE > /dev/null) ; then
			cp $TMPFILE ${metaconfig%.py}
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
390
	rm -f $TMPFILE
391
392
393
394
395
396
397
	for config in *.yaml ; do
		if [ $config -nt $STAMP ] || [ ! -f $STAMP ] ; then
			sudo jenkins-jobs update $config
		else
			echo "$config has not changed, nothing to do."
		fi
	done
Holger Levsen's avatar
Holger Levsen committed
398
	explain "jenkins jobs updated."
399
fi
400

401
#
Holger Levsen's avatar
Holger Levsen committed
402
# configure git for jenkins
403
#
Holger Levsen's avatar
Holger Levsen committed
404
405
406
407
if [ "$(sudo su - jenkins -c 'git config --get user.email')" != "jenkins@jenkins.debian.net" ] ; then
	sudo su - jenkins -c "git config --global user.email jenkins@jenkins.debian.net"
	sudo su - jenkins -c "git config --global user.name Jenkins"
fi
Holger Levsen's avatar
Holger Levsen committed
408

409
410
411
412
413
414
if [ "$HOSTNAME" = "jenkins" ] ; then
	#
	# creating LVM volume group for jobs
	#
	if [ "$PVNAME" = "" ]; then
	    figlet -f banner Error
Holger Levsen's avatar
Holger Levsen committed
415
	    explain "you must set \$PVNAME to physical volume pathname, exiting."
416
417
418
419
420
421
422
423
424
	    exit 1
	else
	    if ! sudo pvs $PVNAME >/dev/null 2>&1; then
	        sudo pvcreate $PVNAME
	    fi
	    if ! sudo vgs $VGNAME >/dev/null 2>&1; then
	        sudo vgcreate $VGNAME $PVNAME
	    fi
	fi
425
426
427
428
429
430
fi

#
# generate the kgb-client configurations
#
if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "profitbricks-build3-amd64" ] ; then
431
432
433
434
435
436
437
438
439
440
	cd $BASEDIR
	KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
	if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
	    # the last condition is to assure the files are owned by the right user/team
	    if [ "$KGB_SECRETS" -nt $STAMP ] || [ ! -f $STAMP ] ; then
	        sudo -u jenkins-adm "./deploy_kgb.py"
	    else
	        explain "kgb-client configuration unchanged, nothing to do."
	    fi
	else
Holger Levsen's avatar
Holger Levsen committed
441
	    figlet -f banner Warning
442
443
444
	    echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
	    echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
	fi
445
446
fi

Holger Levsen's avatar
Holger Levsen committed
447
448
449
450
#
# There's always some work left...
#	echo FIXME is ignored so check-jobs scripts can output templates requiring manual work
#
Holger Levsen's avatar
Holger Levsen committed
451
452
453
454
455
456
457
rgrep FIXME $BASEDIR/* | grep -v "rgrep FIXME" | grep -v echo > $TMPFILE
if [ -s $TMPFILE ] ; then
	echo
	cat $TMPFILE
	echo
fi
rm -f $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
458

459
460
461
#
# finally
#
462
sudo touch $STAMP	# so on the next run, only configs newer than this file will be updated
Holger Levsen's avatar
Holger Levsen committed
463
explain "$(date) - finished deployment."
464