update_jdn.sh 8.81 KB
Newer Older
1
2
#!/bin/bash

3
# Copyright 2012-2015 Holger Levsen <holger@layer-acht.org>
4
5
# released under the GPLv=2

Holger Levsen's avatar
Holger Levsen committed
6
BASEDIR=/root/jenkins.debian.net
7
8
PVNAME=/dev/vdb      # LVM physical volume for jobs
VGNAME=jenkins01     # LVM volume group
Holger Levsen's avatar
Holger Levsen committed
9

Holger Levsen's avatar
Holger Levsen committed
10
11
explain() {
	echo
Holger Levsen's avatar
Holger Levsen committed
12
	echo $1
Holger Levsen's avatar
Holger Levsen committed
13
14
15
	echo
}

16
17
18
#
# set up users and groups
#
19
if ! getent group jenkins-adm > /dev/nnull ; then
20
21
	sudo addgroup --system jenkins-adm
fi
22
23
if ! getent passwd jenkins-adm > /dev/null  ; then
	sudo adduser --system --no-create-home --ingroup jenkins-adm --disabled-login --no-create-home jenkins-adm
24
25
26
	sudo usermod -G jenkins
fi
for user in helmut holger mattia ; do
27
	if ! getent passwd $user > /dev/null ; then
28
29
30
31
32
		sudo adduser --gecos "" $user
		sudo usermod -G jenkins,jenkins-adm
	fi
done

33
34
mkdir -p /srv/workspace

35
if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
36
37
38
39
40
41
42
43
44
45
46
	echo "tmpfs		/srv/workspace	tmpfs	defaults,size=60g	0	0" >> /etc/fstab
fi

if ! mountpoint -q /srv/workspace; then
	if test -z "$(ls -A /srv/workspace)"; then
		mount /srv/workspace
	else
		echo "mountpoint /srv/workspace is non-empty"
	fi
fi

47
# make sure needed directories exists
48
for directory in /schroots /srv/reproducible-results /srv/d-i /srv/live-build ; do
49
50
51
52
53
	if [ ! -d $directory ] ; then
		sudo mkdir $directory
		sudo chown jenkins.jenkins $directory
	fi
done
54
55
sudo mkdir /srv/jenkins
sudo chown jenkins-adm.jenkins-adm /srv/jenkins
56

57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
if ! test -h /chroots; then
	rmdir /chroots || rm -f /chroots # do not recurse
	if test -e /chroots; then
		echo could not clear /chroots
	else
		ln -s /srv/workspace/chroots /chroots
	fi
fi

if ! test -h /var/cache/pbuilder/build; then
	rmdir /var/cache/pbuilder/build || rm -f /var/cache/pbuilder/build
	if test -e /var/cache/pbuilder/build; then
		echo could not clear /var/cache/pbuilder/build
	else
		ln -s /srv/workspace/pbuilder /var/cache/pbuilder/build
	fi
fi

75
76
77
#
# install packages we need
#
78
79
sudo apt-get install	apache2 \
			apt-file \
Holger Levsen's avatar
Holger Levsen committed
80
			apt-listchanges \
81
82
			bash-completion \
			bc \
Holger Levsen's avatar
Holger Levsen committed
83
			build-essential \
84
85
86
			calamaris \
			cron-apt \
			curl \
Holger Levsen's avatar
Holger Levsen committed
87
			debootstrap \
88
89
90
91
92
93
94
			devscripts \
			devscripts \
			dnsmasq-base \
			dose-extra \
			dstat \
			etckeeper \
			ffmpeg \
Holger Levsen's avatar
Holger Levsen committed
95
			figlet \
96
97
98
			gawk \
			ghc \
			gocr \
Holger Levsen's avatar
Holger Levsen committed
99
			graphviz \
100
101
102
103
104
			guestmount \
			haveged \
			imagemagick \
			ip2host \
			less \
Holger Levsen's avatar
Holger Levsen committed
105
			libapache2-mod-macro \
106
107
108
109
110
111
112
113
114
115
116
117
			libav-tools \
			libavcodec-extra-53 \
			libcap2-bin \
			libfile-touch-perl \
			libjson-rpc-perl \
			libsikuli-script-java \
			libsoap-lite-perl \
			libvirt-ruby \
			libvpx1 \
			libxslt1-dev \
			molly-guard \
			moreutils \
Holger Levsen's avatar
Holger Levsen committed
118
			mr \
119
120
121
122
123
			mtr-tiny \
			ntp \
			openjdk-7-jre \
			pigz \
			postgresql-client-9.1 \
Holger Levsen's avatar
Holger Levsen committed
124
			poxml \
125
126
			procmail \
			python-hachoir-metadata \
Holger Levsen's avatar
Holger Levsen committed
127
128
			python-imaging \
			python-lzma \
129
			python-pip \
Holger Levsen's avatar
Holger Levsen committed
130
			python-rpy2 \
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
			python-setuptools \
			python-twisted \
			python-yaml \
			python3-psycopg2 \
			python3-yaml \
			radvd \
			ruby-json \
			ruby-rspec \
			schroot \
			screen \
			shorewall \
			shorewall6 \
			sqlite3 \
			subversion \
			subversion-tools \
			sudo \
			syslinux \
Holger Levsen's avatar
Holger Levsen committed
148
149
			tcpdump \
			unclutter \
150
151
152
153
154
			unzip \
			vim \
			virt-viewer \
			vncsnapshot \
			vnstat \
Holger Levsen's avatar
Holger Levsen committed
155
156
157
			x11-apps \
			x11vnc \
			xtightvncviewer \
158
159
160
			xvfb \
			zutils

Holger Levsen's avatar
Holger Levsen committed
161
sudo apt-get install -t wheezy-backports \
162
163
			binfmt-support \
			cucumber \
Holger Levsen's avatar
Holger Levsen committed
164
			debootstrap \
165
166
167
168
169
170
			libvirt-bin \
			libvirt-dev \
			libvirt0 \
			linux-image-amd64 \
			munin \
			munin-plugins-extra \
171
			python3-debian \
172
			qemu \
Holger Levsen's avatar
Holger Levsen committed
173
174
175
176
			qemu-kvm \
			qemu-system-x86 \
			qemu-user-static \
			ruby-packetfu \
177
			ruby-rjb \
178
179
180
			seabios \
			openbios-ppc \
			openbios-sparc
181

Holger Levsen's avatar
Holger Levsen committed
182
explain "Packages installed."
183

184
echo "Also needs python-arpy from jessie..."
185
echo "Also needs ovmf from jessie..."
186

187
188
189
#
# deploy package configuration in /etc
#
Holger Levsen's avatar
Holger Levsen committed
190
cd $BASEDIR
Holger Levsen's avatar
Holger Levsen committed
191
sudo cp --preserve=mode,timestamps -r etc/* /etc
192
193
194
195
196
197
198

#
# more configuration than a simple cp can do
#
if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
	sudo a2enmod proxy
	sudo a2enmod proxy_http
199
	sudo a2enmod rewrite
Holger Levsen's avatar
use SSL    
Holger Levsen committed
200
	sudo a2enmod ssl
201
	sudo a2enmod headers
202
	sudo a2enmod macro
203
204
fi
sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins
205
sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm
206
sudo ln -sf /etc/apache2/sites-available/jenkins.debian.net /etc/apache2/sites-enabled/000-default
207
sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/000-default
208
209
# for reproducible.d.n url rewriting:
sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
210
sudo service apache2 reload
211
cd /etc/munin/plugins ; sudo rm -f postfix_* open_inodes df_inode interrupts irqstats threads proc_pri vmstat if_err_eth0 fw_forwarded_local fw_packets forks open_files users 2>/dev/null
Holger Levsen's avatar
Holger Levsen committed
212
[ -L apache_accesses ] || for i in apache_accesses apache_volume ; do ln -s /usr/share/munin/plugins/$i $i ; done
Holger Levsen's avatar
Holger Levsen committed
213
explain "Packages configured."
Helmut Grohne's avatar
Helmut Grohne committed
214
sudo service munin-node force-reload
Holger Levsen's avatar
Holger Levsen committed
215
216
217
218

#
# install the heart of jenkins.debian.net
#
Holger Levsen's avatar
Holger Levsen committed
219
cd $BASEDIR
220
221
for dir in bin logparse job-cfg features live ; do
	cp --preserve=mode,timestamps -r $dir /srv/jenkins/
Mattia Rizzolo's avatar
Mattia Rizzolo committed
222
	chown -R jenkins-adm.jenkins-adm /srv/jenkins/$dir
223
done
224
cp procmailrc /var/lib/jenkins/.procmailrc
Holger Levsen's avatar
Holger Levsen committed
225
explain "Jenkins updated."
226
cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
227
228
229
230
TMPFILE=$(mktemp)
git log | grep ^Author| cut -d " " -f2-|sort -u > $TMPFILE
echo "----" >> $TMPFILE
cat THANKS.head $TMPFILE > /var/lib/jenkins/userContent/THANKS
231
rm $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
232
cp -pr userContent /var/lib/jenkins/
233
cd /var/lib/jenkins/userContent/
234
ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
Holger Levsen's avatar
Holger Levsen committed
235
236
237
[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
238
[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
239
240
diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
mv THANKS .THANKS
241
rm TODO README INSTALL CONTRIBUTING
Holger Levsen's avatar
Holger Levsen committed
242
chown -R jenkins.jenkins /var/lib/jenkins/userContent
Holger Levsen's avatar
Holger Levsen committed
243
explain "Updated user content for Jenkins."
244

245
246
247
248
#
# run jenkins-job-builder to update jobs if needed
#     (using sudo because /etc/jenkins_jobs is root:root 700)
#
249
cd /srv/jenkins/job-cfg
250
251
252
for metaconfig in *.yaml.py ; do
	python $metaconfig > ${metaconfig%.py}
done
253
254
255
for config in *.yaml ; do
	sudo jenkins-jobs update $config
done
Holger Levsen's avatar
Holger Levsen committed
256
explain "Jenkins jobs updated."
257

258
259
260
#
# crappy tests for checking that jenkins-job-builder works correctly
#
Holger Levsen's avatar
Holger Levsen committed
261
#wc -m counts one byte too many, so we substract one
262
let DEFINED_MY_TRIGGERS=$(grep -v \# *.yaml | grep my_trigger: | grep -v "my_trigger: ''"|wc -l)+$(grep my_trigger: *.yaml|grep , |xargs -r echo | sed 's/[^,]//g'| wc -m)-1
Holger Levsen's avatar
Holger Levsen committed
263
let DEFINED_DI_TRIGGERS=$(grep "defaults: d-i-manual-html" d-i.yaml|wc -l)
264
#DEFINED_REPRODUCIBLE_TRIGGERS=$(grep "^    defaults: reproducible$" reproducible.yaml|wc -l)
Holger Levsen's avatar
Holger Levsen committed
265
let DEFINED_TRIGGERS=DEFINED_MY_TRIGGERS+DEFINED_DI_TRIGGERS
266
#let DEFINED_TRIGGERS=DEFINED_TRIGGERS+DEFINED_REPRODUCIBLE_TRIGGERS
267
let CONFIGURED_TRIGGERS=$(grep \</childProjects /var/lib/jenkins/jobs/*/config.xml|wc -l)+$(grep  \<childProjects /var/lib/jenkins/jobs/*/config.xml |grep , |xargs -r echo | sed 's/[^,]//g'| wc -m)-1
268
if [ "$DEFINED_TRIGGERS" != "$CONFIGURED_TRIGGERS" ] ; then
Holger Levsen's avatar
Holger Levsen committed
269
	figlet -f banner Warning
Holger Levsen's avatar
Holger Levsen committed
270
	explain "Number of defined triggers ($DEFINED_TRIGGERS) differs from currently configured triggers ($CONFIGURED_TRIGGERS), please investigate."
271
272
fi

273
#
Holger Levsen's avatar
Holger Levsen committed
274
# configure git for jenkins
275
#
Holger Levsen's avatar
Holger Levsen committed
276
277
278
279
if [ "$(sudo su - jenkins -c 'git config --get user.email')" != "jenkins@jenkins.debian.net" ] ; then
	sudo su - jenkins -c "git config --global user.email jenkins@jenkins.debian.net"
	sudo su - jenkins -c "git config --global user.name Jenkins"
fi
Holger Levsen's avatar
Holger Levsen committed
280

Holger Levsen's avatar
Holger Levsen committed
281
#
Holger Levsen's avatar
Holger Levsen committed
282
# configure pbuilder for jenkins user
Holger Levsen's avatar
Holger Levsen committed
283
284
#
sudo chown jenkins /var/cache/pbuilder/result
285

286
287
288
289
#
# creating LVM volume group for jobs
#
if [ "$PVNAME" = "" ]; then
Holger Levsen's avatar
Holger Levsen committed
290
    figlet -f banner Error
291
292
293
294
295
    explain "Set \$PVNAME to physical volume pathname."
    exit 1
else
    if ! sudo pvs $PVNAME >/dev/null 2>&1; then
        sudo pvcreate $PVNAME
Gabriele Giacone's avatar
Gabriele Giacone committed
296
297
    fi
    if ! sudo vgs $VGNAME >/dev/null 2>&1; then
298
299
300
        sudo vgcreate $VGNAME $PVNAME
    fi
fi
Holger Levsen's avatar
Holger Levsen committed
301

302
303
304
305
306
307
308
#
# generate the kgb-client configurations
#
cd $BASEDIR
KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
    # to assure the files are owned by the right user/team
309
    sudo -u jenkins-adm "./deploy_kgb.py"
310
311
312
313
314
else
    echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
    echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
fi

Holger Levsen's avatar
Holger Levsen committed
315
316
317
318
319
320
321
#
# There's always some work left...
#	echo FIXME is ignored so check-jobs scripts can output templates requiring manual work
#
echo
rgrep FIXME $BASEDIR/* | grep -v "rgrep FIXME" | grep -v echo