refactor housekeeping.sh, update README and TODO

README

@@ -22,6 +22,14 @@ Some stats are available using link:http://jenkins.debian.net/munin/jenkins-mont
 
 I love to get feedback on this! Please join #debian-qa on irc.debian.org to report bugs or give other feedback! Or send an email to debian-qa@lists.debian.org if you prefer mail.
 
+=== Setup
+
+See link:http://jenkins.debian.net/userContent/setup.html[INSTALL].
+
+=== ToDo
+
+There is still a lot of work left, check the current link:http://jenkins.debian.net/userContent/todo.html[ToDo list].
+
 == Jobs being run
 
 Remember, this is still pretty much work in progress. If you can think of ways to improve the usefulness of certain jobs, please do give feedback!
@@ -30,7 +38,28 @@ Remember, this is still pretty much work in progress. If you can think of ways t
 
 Installation tests with g-i, the graphical version of d-i, the debian-installer.
 
-FIXME: Describe g-i-installation jobs here
+* g-i-installation_sid_daily_lxde
+** sid installation of LXDE desktop with daily build sid image
+
+* g-i-installation_wheezy_lxde
+** wheezy installation of LXDE desktop with weekly build wheezy image
+
+* Debian Edu instllation of specific profiles and suites
+** g-i-installation_squeeze-test_debian_edu_standalone
+** g-i-installation_wheezy_debian_edu_workstation
+
+=== debian-installer jobs
+
+* d_i_build_* - there is one job for each git repo referred to in http://anonscm.debian.org/viewvc/d-i/trunk/.mrconfig?view=co
+Each job pdebuilds the master branch of its git repo on every git push in a sid environment. (If the architecture(s) specified in debian/control are not amd64,all or any the build exits cleanly.)
+** while these jobs are triggered on commits, the SCM is only polled every 6min to see if there are new commits.
+* d_i_manual - builds the full installation-guide package with pdebuild in sid on every commit to svn://svn.debian.org/svn/d-i/ matching these patterns: 
+'/trunk/manual/debian/.\*', '/trunk/manual/po/.\*', '/trunk/manual/doc/.\*' and '/trunk/manual/scripts/.\*'
+** while this job is triggered on commits, the SCM is only polled every 15min to see if there are new commits.
+* d_i_manual_*_html - builds a language (on wheezy) on every commit of svn/trunk/manual/$LANG with "make languages=$LANG architectures=amd64 formats=html". On successful build, d_i_manual_$lang_pdf is triggered.
+** while these jobs are triggered on commits, the SCM is only polled every 15min to see if there are new commits.
+* d_i_parse_build_logs - parses logs from http://d-i.debian.org/daily-images/build-logs.html daily, to give them a bit more exposure.
+** this job is run daily.
 
 === chroot-installation jobs
 
@@ -74,19 +103,6 @@ Installation tests inside chroot environments.
 ** debootstrap a wheezy chroot and install xfce
 #FIXME: explain full_desktop and developer here, also make this more useful/interesting
 
-=== debian-installer jobs
-
-* d_i_build_* - there is one job for each git repo referred to in http://anonscm.debian.org/viewvc/d-i/trunk/.mrconfig?view=co
-Each job pdebuilds the master branch of its git repo on every git push in a sid environment. (If the architecture(s) specified in debian/control are not amd64,all or any the build exits cleanly.)
-** while these jobs are triggered on commits, the SCM is only polled every 6min to see if there are new commits.
-* d_i_manual - builds the full installation-guide package with pdebuild in sid on every commit to svn://svn.debian.org/svn/d-i/ matching these patterns: 
-'/trunk/manual/debian/.\*', '/trunk/manual/po/.\*', '/trunk/manual/doc/.\*' and '/trunk/manual/scripts/.\*'
-** while this job is triggered on commits, the SCM is only polled every 15min to see if there are new commits.
-* d_i_manual_*_html - builds a language (on wheezy) on every commit of svn/trunk/manual/$LANG with "make languages=$LANG architectures=amd64 formats=html". On successful build, d_i_manual_$lang_pdf is triggered.
-** while these jobs are triggered on commits, the SCM is only polled every 15min to see if there are new commits.
-* d_i_parse_build_logs - parses logs from http://d-i.debian.org/daily-images/build-logs.html daily, to give them a bit more exposure.
-** this job is run daily.
-
 === webcheck jobs
 
 * three jobs for running webcheck against www.debian.org, www.debian.org/News and www.debian.org/security
@@ -94,12 +110,7 @@ Each job pdebuilds the master branch of its git repo on every git push in a sid
 
 These jobs are only run on the first day of each month.
 
-== Setup
-
-See link:http://jenkins.debian.net/userContent/setup.html[INSTALL].
-
-== ToDo
-
-There is still a lot of work left, check the current link:http://jenkins.debian.net/userContent/todo.html[ToDo list].
+== License
 
+* See link:http://anonscm.debian.org/gitweb/?p=users/holger/jenkins.debian.net.git;a=blob;f=LICENSE[LICENSE].
 

TODO

@@ -14,6 +14,7 @@ See link:http://jenkins.debian.net/userContent/about.html["about jenkins.debian.
 
 == General ToDo
 
+* create general and job specific housekeeping jobs
 * keep git+svn urls, and use commit hooks. also better for test setups of this install :)
 * run calamaris, visitors etc
 * proper git repo url, outside users/holger
@@ -67,11 +68,26 @@ See link:http://jenkins.debian.net/userContent/about.html["about jenkins.debian.
 
 === g-i-installation_*:
 
+more lanages
+
+* babelbox.git has a nice langlist
 * $name-preseed.base -> sed .cfg (LANG)
-** pick LANG from predefined list at random
+* for edu: pick LANG from predefined list at random - if last build was not successful or unstable fall back to English
+* for d-i: create jobs for something like Tamil, Traditional Chinese, Japanese, Hebrew, some Arabic and maybe some more Indian languages would be sufficient
+** these jobs would not need to do an install, just booting them in rescue mode is probably enough
+
+post-install
+
 * boot system
 ** scp -r vm:/var/log/ results/log
-** ssh root@vm shutdown
+** login graphically
+** run vim in xterm
+** run emacs in xterm
+** run iceweasel www.debian.org / www.skolelinux.org
+** shutdown as user
+
+general
+
 * use jigdo to download images - but no edu jigdo images.
 * use https://wiki.jenkins-ci.org/display/JENKINS/Image+Gallery+Plugin
 * still havent got http://www.os-autoinst.org/ out of my mind

bin/housekeeping.sh

 #/bin/bash
-# FIXME: make this a general and a specific housekeeping job:
 
 # Copyright 2012 Holger Levsen <holger@layer-acht.org>
 # released under the GPLv=2
@@ -8,38 +7,63 @@
 # default settings
 #
 export LC_ALL=C
+set -e
 
-echo 
-uptime
+check_for_mounted_chroots() {
+	CHROOT_PATTERN="/chroots/$1-*"
+	OUTPUT=$(ls $CHROOT_PATTERN 2>/dev/null)
+	if [ "$OUTPUT" != "" ] ; then
+		figlet "Warning:"
+		echo
+		echo "Probably manual cleanup needed:"
+		echo
+		echo "$ ls -la $CHROOT_PATTERN"
+		# List the processes using the partition
+		echo
+		fuser -mv $CHROOT_TARGET
+		echo $OUTPUT
+		exit 1
+	fi
+}
 
-echo
-df -h
-
-echo
-for DIR in /var/cache/apt/archives/ /var/spool/squid/ /var/cache/pbuilder/build/ /var/lib/jenkins/jobs/ ; do
-	sudo du -sh $DIR 2>/dev/null
-done
-
-echo
-vnstat
+report_disk_usage() {
+	du -schx /var/lib/jenkins/jobs/$1_* |grep total |sed -s "s#total#$1 jobs#"
+	# FIXME: if $2 is given check, that disk usage is below $2 GB
+}
 
+general_housekeeping() {
+	echo
+	uptime
 
-CHROOT_PATTERN="/chroots/chroot-installation-*"
-HOUSE=$(ls $CHROOT_PATTERN 2>/dev/null)
-if [ "$HOUSE" != "" ] ; then
-	figlet "Warning:"
 	echo
-	echo "Probably manual cleanup needed:"
+	df -h
+
 	echo
-	echo "$ ls -la $CHROOT_PATTERN"
-	# List the processes using the partition
+	for DIR in /var/cache/apt/archives/ /var/spool/squid/ /var/cache/pbuilder/build/ /var/lib/jenkins/jobs/ ; do
+		sudo du -shx $DIR 2>/dev/null
+	done
+	JOB_PREFIXES=$(ls -1 /var/lib/jenkins/jobs/|cut -d "_" -f1|sort -f -u)
+	for PREFIX in $JOB_PREFIXES ; do
+		report_disk_usage $PREFIX
+	done
+
 	echo
-	fuser -mv $CHROOT_TARGET
-	echo $HOUSE
-	exit 1
-fi
+	vnstat
+
+	df |grep tmpfs > /dev/null || ( echo ; echo "Warning: no tmpfs mounts in use. Please investigate the host system." ; exit 1 )
+}
 
-df |grep tmpfs > /dev/null || ( echo "Warning: no tmpfs mounts in use. Please investigate the host system." ; exit 1 )
+#
+# if $1 is set, we do housekeeping for a group of jobs, else general
+#
+if [ -n $1 ] ; then
+	report_disk_usage $1
+	if [ "$1" = "chroot-installation" ] ; then
+		check_for_mounted_chroots $1
+	fi
+else
+	general_housekeeping
+fi
 
 echo
 echo "No problems found, all seems good."