reproducible armhf: add two new armhf build nodes & their setup and mainenance...

reproducible armhf: add two new armhf build nodes & their setup and mainenance jobs. Thanks Vagrant.

README

@@ -138,11 +138,12 @@ Installation tests inside chroot environments.
 * Currently, three suites are tested on 'amd64' and 'armhf' architectures: 'testing', 'unstable' and 'experimental'. The tests are done using 'pbuilder' using link:https://wiki.debian.org/ReproducibleBuilds/ExperimentalToolchain[our toolchain] through concurrent builder jobs, 32 for 'amd64' and 42 for 'armhf', which are each constantly testing packages and saving the results of these tests.
 
 ** These builds on remote nodes run on very different hardware: for 'amd64' we are now using four virtual machines, profitbricks-build(1+2+5+6)-amd64, which have 18 or 17 cores and 48gb ram each and are sponsored by link:https://jenkins.debian.net/userContent/thanks.html[Profitbricks].
-** To test 'armhf' we are using 16 small boards donated by vagrant@d.o:
-*** two quad-cores (cbxi4a and cbxi4b) with 4gb ram, 
+** To test 'armhf' we are using 18 small boards donated by vagrant@d.o:
+*** three quad-cores (cbxi4a, cbxi4b and ff4a) with 4gb ram, 
 *** three octo-cores (odxu4, odxu4b and odxu4c) with 2gb ram, 
-*** six quad-cores (wbq0, cbxi4pro0, ff2a, ff2b, opi2a and opi2b) with 2gb ram, 
-*** two quad-cores (rpi2b and rpi2c) with 1gb ram and three dual-cores (bpi0, hb0 and wbd0) with 1gb ram, each. 
+*** seven quad-cores (bbx15, wbq0, cbxi4pro0, ff2a, ff2b, opi2a and opi2b) with 2gb ram, 
+*** two quad-cores (rpi2b and rpi2c) with 1gb ram and
+*** three dual-cores (bpi0, hb0 and wbd0) with 1gb ram, each. 
 ** We would love to have more or more powerful ARM hardware in the future, if you can help, please talk to us!
 
 * Packages to be build are scheduled in the SQLite database via a scheduler job, which runs every hour and if the queue is below a certain threshold schedules four types of packages:

THANKS.head

@@ -12,9 +12,9 @@
  ** 17 cores and 48 GB memory for profitbricks-build6-amd64.debian.net
  ** 3 cores and 6 GB memory for freebsd-jenkins.debian.net (also running on Profitbricks virtual hardware)
  * link:https://qa.debian.org/developer.php?login=vagrant%40debian.org[Vagrant] provides and hosts 16 'armhf' systems:
- ** two quad-cores with 4 GB RAM each,
+ ** three quad-cores with 4 GB RAM each,
  ** three octo-cores with 2 GB RAM each,
- ** six quad-cores with 2 GB RAM each,
+ ** seven quad-cores with 2 GB RAM each,
  ** two quad-cores with 1 GB RAM and
  ** three dual-cores with 1 GB RAM.
  * link:https://letsencrypt.org[Let's encrypt] provides free of charge SSL certificates for jenkins.debian.net, reproducible.debian.net and tests.reproducible-builds.org.

bin/jenkins_node_definitions.sh

@@ -4,12 +4,15 @@
 # released under the GPLv=2
 
 # define Debian build nodes in use
-BUILD_NODES="profitbricks-build1-amd64.debian.net profitbricks-build2-amd64.debian.net profitbricks-build5-amd64.debian.net profitbricks-build6-amd64.debian.net wbq0-armhf-rb.debian.net cbxi4a-armhf-rb.debian.net cbxi4b-armhf-rb.debian.net cbxi4pro0-armhf-rb.debian.net bpi0-armhf-rb.debian.net hb0-armhf-rb.debian.net odxu4-armhf-rb.debian.net wbd0-armhf-rb.debian.net rpi2b-armhf-rb.debian.net rpi2c-armhf-rb.debian.net odxu4b-armhf-rb.debian.net odxu4c-armhf-rb.debian.net ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net opi2a-armhf-rb.debian.net opi2b-armhf-rb.debian.net"
+BUILD_NODES="profitbricks-build1-amd64.debian.net profitbricks-build2-amd64.debian.net profitbricks-build5-amd64.debian.net profitbricks-build6-amd64.debian.net wbq0-armhf-rb.debian.net cbxi4a-armhf-rb.debian.net cbxi4b-armhf-rb.debian.net cbxi4pro0-armhf-rb.debian.net bbx15-armhf-rb.debian.net bpi0-armhf-rb.debian.net hb0-armhf-rb.debian.net odxu4-armhf-rb.debian.net wbd0-armhf-rb.debian.net rpi2b-armhf-rb.debian.net rpi2c-armhf-rb.debian.net odxu4b-armhf-rb.debian.net odxu4c-armhf-rb.debian.net ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net ff4a-armhf-rb.debian.net opi2a-armhf-rb.debian.net opi2b-armhf-rb.debian.net"
 
 # return the ports sshd is listening on
 get_node_ssh_port() {
 	local NODE_NAME=$1
 	case "$NODE_NAME" in
+	  bbx15-armhf-rb.debian.net)
+	    PORT=2242
+	    ;;
 	  bpi0-armhf-rb.debian.net)
 	    PORT=2222
 	    ;;
@@ -52,6 +55,9 @@ get_node_ssh_port() {
 	  ff2b-armhf-rb.debian.net)
 	    PORT=2237
 	    ;;
+	  ff4a-armhf-rb.debian.net)
+	    PORT=2241
+	    ;;
 	  opi2a-armhf-rb.debian.net)
 	    PORT=2236
 	    ;;

hosts/bbx15

+bbx15-armhf-rb
\ No newline at end of file

hosts/bbx15-armhf-rb/etc/apt/apt.conf.d/80proxy

+Acquire::http::Proxy "http://10.0.0.15:8000/";
+

hosts/bbx15-armhf-rb/etc/apt/listchanges.conf

+[apt]
+frontend=mail
+email_address=root
+confirm=0
+save_seen=/var/lib/apt/listchanges.db
+which=both

hosts/bbx15-armhf-rb/etc/apt/sources.list

+deb http://ftp.us.debian.org/debian/ jessie main contrib non-free
+#deb-src http://ftp.us.debian.org/debian/ jessie main contrib non-free
+
+deb http://ftp.us.debian.org/debian/ jessie-updates main contrib non-free
+#deb-src http://ftp.us.debian.org/debian/ jessie-updates main contrib non-free
+
+deb http://security.debian.org/ jessie/updates main contrib non-free
+#deb-src http://security.debian.org/ jessie/updates main contrib non-free
+
+deb http://ftp.us.debian.org/debian/ jessie-backports main contrib non-free
+#deb-src http://ftp.us.debian.org/debian/ jessie-backports main contrib non-free

hosts/bbx15-armhf-rb/etc/cron.d/dsa

+# m h dom mon dow (0|7=sun,1=mon)    command
+
+#
+# cron-jobs for jenkins.debian.net and nodes
+#
+
+MAILTO=root
+
+0 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-running-kernel
+2 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-packages

hosts/bbx15-armhf-rb/etc/munin/munin-node.conf

+#
+# Example config-file for munin-node
+#
+
+log_level 4
+log_file /var/log/munin/munin-node.log
+pid_file /var/run/munin/munin-node.pid
+
+background 1
+setsid 1
+
+user root
+group root
+
+# This is the timeout for the whole transaction.
+# Units are in sec. Default is 15 min
+#
+# global_timeout 900
+
+# This is the timeout for each plugin.
+# Units are in sec. Default is 1 min
+#
+# timeout 60
+
+# Regexps for files to ignore
+ignore_file [\#~]$
+ignore_file DEADJOE$
+ignore_file \.bak$
+ignore_file %$
+ignore_file \.dpkg-(tmp|new|old|dist)$
+ignore_file \.rpm(save|new)$
+ignore_file \.pod$
+
+# Set this if the client doesn't report the correct hostname when
+# telnetting to localhost, port 4949
+#
+#host_name localhost.localdomain
+
+# A list of addresses that are allowed to connect.  This must be a
+# regular expression, since Net::Server does not understand CIDR-style
+# network notation unless the perl module Net::CIDR is installed.  You
+# may repeat the allow line as many times as you'd like
+
+allow ^127\.0\.0\.1$
+allow ^::1$
+
+# If you have installed the Net::CIDR perl module, you can use one or more
+# cidr_allow and cidr_deny address/mask patterns.  A connecting client must
+# match any cidr_allow, and not match any cidr_deny.  Note that a netmask
+# *must* be provided, even if it's /32
+#
+# Example:
+#
+# cidr_allow 127.0.0.1/32
+# cidr_allow 192.0.2.0/24
+# cidr_deny  192.0.2.42/32
+
+# Which address to bind to;
+host *
+# host 127.0.0.1
+
+# And which port
+port 4949
+
+allow ^78\.137\.96\.196
+hostname bbx15-armhf-rb.debian.net

hosts/bbx15-armhf-rb/etc/munin/plugin-conf.d/df

+[df*]
+env.exclude none unknown iso9660 squashfs udf romfs ramfs debugfs devtmpfs sysfs
+env.exclude_re /srv/workspace/pbuilder /run /dev/disk/by /var/lib/schroot/mount /srv/workspace/varlibschroot /dev/shm /sys/fs/cgroup
+env.warning 92
+env.critical 98
+

hosts/bbx15-armhf-rb/etc/munin/plugin-conf.d/diskstats

+[diskstats]
+env.trim_labels yes
+env.include_only /dev/sda
+

hosts/bbx15-armhf-rb/etc/munin/plugin-conf.d/munin-node

+# This file is used to configure how the plugins are invoked.
+# Place in /etc/munin/plugin-conf.d/ or corresponding directory.
+#
+# PLEASE NOTE: Changes in the plugin-conf.d directory are only
+# read at munin-node startup, so restart at any changes.
+#
+# user <user>         # Set the user to run the plugin as.
+# group <group>       # Set the group to run the plugin as.
+# command <command>   # Run <command> instead of the plugin. %c expands to
+#                       what would normally be run.
+# env.<variable> <value> # Sets <variable> in the plugin's environment, see the
+#                       individual plugins to find out which variables they
+#                       care about.
+
+
+[amavis]
+group adm
+env.MUNIN_MKTEMP /bin/mktemp -p /tmp/ $1
+env.amavislog /var/log/mail.info
+
+[apt]
+user root
+
+[courier_mta_mailqueue]
+group daemon
+
+[courier_mta_mailstats]
+group adm
+
+[courier_mta_mailvolume]
+group adm
+
+[cps*]
+user root
+
+[exim_mailqueue]
+group adm, (Debian-exim)
+
+[exim_mailstats]
+group adm, (Debian-exim)
+env.logdir /var/log/exim4/
+env.logname mainlog
+
+[fw_conntrack]
+user root
+
+[fw_forwarded_local]
+user root
+
+[hddtemp_smartctl]
+user root
+
+[hddtemp2]
+user root
+
+[if_*]
+user root
+
+[if_err_*]
+user nobody
+
+[ip_*]
+user root
+
+[ipmi_*]
+user root
+
+[mysql*]
+user root
+env.mysqlopts --defaults-file=/etc/mysql/debian.cnf
+env.mysqluser debian-sys-maint
+env.mysqlconnection DBI:mysql:mysql;mysql_read_default_file=/etc/mysql/debian.cnf
+
+[postfix_mailqueue]
+user postfix
+
+[postfix_mailstats]
+group adm
+
+[postfix_mailvolume]
+group adm
+env.logfile mail.log
+
+[smart_*]
+user root
+
+[vlan*]
+user root
+
+[ejabberd*]
+user ejabberd
+env.statuses available away chat xa
+env.days 1 7 30
+
+[dhcpd3]
+user root
+env.leasefile /var/lib/dhcp3/dhcpd.leases
+env.configfile /etc/dhcp3/dhcpd.conf
+
+[jmx_*]
+env.ip 127.0.0.1
+env.port 5400
+
+[samba]
+user root
+
+[munin_stats]
+user munin
+group munin
+
+[postgres_*]
+user postgres
+env.PGUSER postgres
+env.PGPORT 5432
+
+[fail2ban]
+user root

hosts/bbx15-armhf-rb/etc/munin/plugins/jenkins_reproducible_builds

+#!/bin/sh
+# -*- sh -*-
+
+: << =cut
+
+=head1 NAME
+
+jenkins_reproducible_builds - Plugin to measure number of reproducible builds running
+
+=head1 AUTHOR
+
+Contributed by Holger Levsen
+
+=head1 LICENSE
+
+GPLv2
+
+=head1 MAGIC MARKERS
+
+ #%# family=auto
+ #%# capabilities=autoconf
+
+=cut
+
+. $MUNIN_LIBDIR/plugins/plugin.sh
+
+if [ "$1" = "autoconf" ]; then
+	echo yes
+	exit 0
+fi
+
+JOB_PREFIXES="first second"
+if [ "$1" = "config" ]; then
+	echo 'graph_title Concurrent reproducible builds running'
+	echo 'graph_args --base 1000 -l 0 '
+	echo 'graph_scale no'
+	echo 'graph_total total'
+	echo 'graph_vlabel Concurrent reproducible builds running'
+	echo 'graph_category jenkins'
+	draw=AREA
+	for PREFIX in $JOB_PREFIXES ; do
+		echo "jenkins_reproducible_${PREFIX}_build.label $PREFIX build"
+		echo "jenkins_reproducible_${PREFIX}_build.draw $draw"
+		if [ "$draw" = "AREA" ] ; then draw=STACK ; fi
+	done
+	exit 0
+fi
+
+for PREFIX in $JOB_PREFIXES ; do
+	if [ "$PREFIX" = "first" ] ; then
+		NR=$(pgrep -fc "bin/bash /srv/jenkins/bin/reproducible_build.sh 1")
+	else
+		NR=$(pgrep -fc "bin/bash /srv/jenkins/bin/reproducible_build.sh 2")
+	fi
+	echo "jenkins_reproducible_${PREFIX}_build.value $NR"
+	done

hosts/bbx15-armhf-rb/etc/pbuilder/rebuild-hooks/B01_cleanup

+#!/bin/sh
+
+set -e
+
+BUILDDIR="${BUILDDIR:-/tmp/buildd}"
+
+# exit if we are in the same UTS namespace as init ( != 2nd build )
+[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0
+
+# cease using disorderfs
+if [ -d /tmp/disorderfs ] ; then
+	echo -n "Unmounting /tmp/disorderfs…"
+	fusermount -u "$BUILDDIR"
+	rmdir "$BUILDDIR"
+	mv /tmp/disorderfs "$BUILDDIR"
+	echo " done."
+fi

hosts/bbx15-armhf-rb/etc/pbuilder/rebuild-hooks/C01_cleanup

+#!/bin/sh
+
+set -e
+
+BUILDDIR="${BUILDDIR:-/tmp/buildd}"
+
+echo "debug output: disk usage on $(hostname) at $(date -u)"
+df -h
+echo
+
+# exit if we are in the same UTS namespace as init ( != 2nd build )
+[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0
+
+# cease using disorderfs
+if [ -d /tmp/disorderfs ] ; then
+	echo -n "Unmounting /tmp/disorderfs…"
+	fusermount -u "$BUILDDIR"
+	rmdir "$BUILDDIR"
+	mv /tmp/disorderfs "$BUILDDIR"
+	echo " done."
+fi

hosts/bbx15-armhf-rb/etc/pbuilder/rebuild-hooks/D01_modify_environment

+#!/bin/sh
+
+set -e
+
+BUILDDIR="${BUILDDIR:-/tmp/buildd}"
+
+# exit if we are in the same UTS namespace as init ( != 2nd build )
+[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0
+
+echo "I: Changing host+domainname to test build reproducibility" >&2
+sed -e '/^127.0.0.1/s/$/ i-capture-the-hostname i-capture-the-hostname.i-capture-the-domain/' -i /etc/hosts
+hostname i-capture-the-hostname
+domainname i-capture-the-domain
+echo "I: Adding a custom variable just for the fun of it..." >&2
+export CAPTURE_ENVIRONMENT="I capture the environment"
+
+echo "I: Changing /bin/sh to bash" >&2
+echo "dash dash/sh boolean false" | debconf-set-selections
+DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash
+
+# temporarily disable disorderfs as we have problems properly unmounting it
+exit 0
+
+# use disorderfs
+if [ -x /usr/bin/disorderfs ] ; then
+	echo -n "Moving $BUILDDIR to /tmp/disorderfs and mounting this as $BUILDDIR via the fuse disorderfs…"
+	mknod -m 666 /dev/fuse c 10 229
+	mv "$BUILDDIR" /tmp/disorderfs
+	mkdir "$BUILDDIR"
+	disorderfs --multi-user=yes /tmp/disorderfs "$BUILDDIR"
+	echo " done."
+else
+	echo "Warning: disorderfs not available."
+fi

hosts/bbx15-armhf-rb/etc/pbuilderrc

+# this is your configuration file for pbuilder.
+# the file in /usr/share/pbuilder/pbuilderrc is the default template.
+# /etc/pbuilderrc is the one meant for overwriting defaults in
+# the default template
+#
+# read pbuilderrc.5 document for notes on specific options.
+case $HOSTNAME in 
+	jenkins|profitbricks-build?-amd64)			MIRRORSITE=http://ftp.de.debian.org/debian ;;
+	bpi0|cbxi4*|hb0|wbq0|odxu4*|wbd0|rpi2*|ff2*|opi2*)	MIRRORSITE=http://ftp.us.debian.org/debian ;;
+	*)							echo "unsupported host, exiting." ; exit 1 ;;
+esac
+EXTRAPACKAGES=""	# better list them in bin/reproducible_setup_pbuilder.sh
+APTCACHE=""
+COMPRESSPROG="pigz"
+BUILDPLACE=/srv/workspace/pbuilder  # build in /srv/workspace, which is in tmpfs
+# unset proxy, see #780587
+unset http_proxy
+export -n http_proxy
+# /sys and /proc were added as workarounds for #773767
+# there are software wanting /dev/shm writable, so give them so. There are tons
+# of bug shm related in pbuilder, that's just a workaround for all of them.
+# /run/shm is already mounted as a tmpfs by default.
+BINDMOUNTS="$BINDMOUNTS /sys /dev/shm"
+# set PATH to predictable values, see #780729 and #780725
+PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games"
+
+# used for reproducible builds tests, when doing the 2nd build
+if [ "$(readlink /proc/1/ns/uts)" != "$(readlink /proc/self/ns/uts)" ]; then
+	PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path"
+fi
+

hosts/bbx15-armhf-rb/etc/postfix/main.cf

+# See /usr/share/postfix/main.cf.dist for a commented, more complete version
+
+
+# Debian specific:  Specifying a file name will cause the first
+# line of that file to be used as the name.  The Debian default
+# is /etc/mailname.
+#myorigin = /etc/mailname
+
+smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU)
+biff = no
+
+# appending .domain is the MUA's job.
+append_dot_mydomain = no
+
+# Uncomment the next line to generate "delayed mail" warnings
+#delay_warning_time = 4h
+
+readme_directory = no
+
+# TLS parameters
+smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
+smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
+smtpd_use_tls=yes
+smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
+smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
+
+# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for
+# information on enabling SSL in the smtp client.
+
+myhostname = bbx15-armhf-rb.debian.net
+alias_maps = hash:/etc/aliases
+alias_database = hash:/etc/aliases
+myorigin = /etc/mailname
+mydestination = bbx15-armhf-rb.debian.net, localhost
+relayhost = 
+#mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
+mynetworks = 127.0.0.0/8
+mailbox_size_limit = 0
+recipient_delimiter = +
+inet_interfaces = all
+mailbox_command = /usr/bin/procmail -a "$EXTENSION"

hosts/bbx15-armhf-rb/etc/rc.local

+#!/bin/sh -e
+#
+# rc.local
+#
+# This script is executed at the end of each multiuser runlevel.
+# Make sure that the script will "exit 0" on success or any other
+# value on error.
+#
+# In order to enable or disable this script just change the execution
+# bits.
+#
+# By default this script does nothing.
+
+echo "$(date -u) - system was rebooted" | mail -s "$(hostname -f) rebooted" root
+
+exit 0

hosts/bbx15-armhf-rb/etc/schroot/default/fstab

+# fstab: static file system information for chroots.
+# Note that the mount point will be prefixed by the chroot path
+# (CHROOT_PATH)
+#
+# <file system>	<mount point>	<type>	<options>	<dump>	<pass>
+/proc		/proc		none    rw,bind        0       0
+/sys		/sys		none    rw,bind        0       0
+/dev            /dev            none    rw,bind         0       0
+/dev/pts	/dev/pts	none	rw,bind		0	0
+/home		/home		none	rw,bind		0	0
+/tmp		/tmp		none	rw,bind		0	0
+/srv/reproducible-results	/srv/reproducible-results	none	rw,bind		0	0
+/srv/d-i	/srv/d-i	none	rw,bind		0	0
+/srv/jenkins	/srv/jenkins	none	rw,bind		0	0
+/srv/live-build	/srv/live-build	none	rw,bind		0	0
+
+# It may be desirable to have access to /run, especially if you wish
+# to run additional services in the chroot.  However, note that this
+# may potentially cause undesirable behaviour on upgrades, such as
+# killing services on the host.
+#/run		/run		none	rw,bind		0	0
+#/run/lock	/run/lock	none	rw,bind		0	0
+/dev/shm	/dev/shm	none	rw,bind		0	0
+/run/shm	/run/shm	none	rw,bind		0	0