Holger Levsen · Holger Levsen · Holger Levsen · Andreas Beckmann · Holger Levsen · Andreas Beckmann
--- a/TODO
+++ b/TODO
@@ -6,7 +6,9 @@ wishlist! And see the bottom of this file for wishlist bugs someone once
 expressed…


-Some stuff we might tackle for 0.9x, during the buster development cycle:
+Some stuff we might tackle during the bullseye development cycle:
+
+- switch to python3 (that's actually a must)

 - do an 1.0.0 release, it's about time since 10 years :)

@@ -87,14 +89,10 @@ Some stuff we might tackle for 0.9x, during the buster development cycle:
  (piuparts), adopting apt's naming


-Ideas for 1.0.x and later:
-
- piuparts.conf.pejacevic: maybe use mirror via nfs (faster)
+Ideas for later:

 - maybe compress all logfiles

- piuparts-master: keep track of to whom a reservation was given
-
 - install from git/Makefile: remove the need for /etc/piuparts

 - accept a PIUPARTS_CONF environment variable everywhere to point to a different
@@ -139,10 +137,6 @@ More ideas for even later:
  debugging output for piuparts.py - see
  http://docs.python.org/library/logging.html

- commandline-switches for all programms
-
- move shell cronjobs functionality into master, slave & report
-
 - automated testing of piuparts using an archive of known broken packages:
  - create archive of broken packages to provide test cases for piuparts testing.
  - create emacspeak-broken-dpkg-preconfigure package for broken repo. (then later

--- a/conf/distros.conf
+++ b/conf/distros.conf
@@ -132,6 +132,13 @@ depends = stretch stretch/updates
 target-release = stretch-backports


+[buster/updates]
+uri = http://security.debian.org
+depends = buster
+
+[buster-updates]
+depends = buster
+
 [buster-proposed-updates]
 depends = buster

@@ -139,6 +146,11 @@ depends = buster
 uri = http://deb.debian.org/debian-debug
 depends = buster

+[buster-proposed]
+uri = None
+depends = buster buster/updates buster-updates buster-proposed-updates
+candidates = buster buster/updates buster-proposed-updates
+

 [sid-debug]
 uri = http://deb.debian.org/debian-debug

--- a/custom-scripts/scripts-multi-distro-upgrade/post_distupgrade_base_cleanup_multi-distro-upgrade
+++ b/custom-scripts/scripts-multi-distro-upgrade/post_distupgrade_base_cleanup_multi-distro-upgrade
+#!/bin/sh
+set -e
+
+# Only while creating the reference chroot.
+test "$PIUPARTS_PHASE" = "" || exit 0
+
+
+if	[ "$PIUPARTS_DISTRIBUTION" = "buster" ] || \
+	[ "$PIUPARTS_DISTRIBUTION" = "buster-proposed" ] ; then
+
+	if [ -d /etc/init ]; then
+		rmdir --ignore-fail-on-non-empty -v /etc/init
+	fi
+
+fi
--- a/custom-scripts/scripts/is_testable_uninstallable
+++ b/custom-scripts/scripts/is_testable_uninstallable
@@ -45,6 +45,105 @@ skip_distupgrade()
 #   we usually skip the failing initial install, too, since that is
 #   *not a bug* in the package.

+
+if [ "${PIUPARTS_DISTRIBUTION}" = "lenny" ]; then
+
+	case ${PIUPARTS_OBJECTS%%=*} in
+		blootbot)
+			skip_distupgrade "#502753: fails to install: prompts without using debconf"
+			;;
+		distributed-net)
+			skip_distupgrade "#655922: noninteractive install: postinst loops infinitely"
+			;;
+		dnscache-run)
+			skip_distupgrade "#664848: breaks dns resolution in chroot"
+			;;
+		libpils0|\
+		libpils-dev|\
+		libstonith0|\
+		libstonith-dev)
+			skip_distupgrade "#702055: transitional packages uninstallable in lenny"
+			;;
+		mhc)
+			skip_distupgrade "#696231: fails to install: emacs-package-install failure"
+			;;
+		multex-bin)
+			skip_distupgrade "#668791: fails to install: The format file of 'multex' is NOT built successfully."
+			;;
+		python-milter|\
+		spf-milter-python)
+			skip_always "python-milter contains virus signatures that may get blocked by the firewall"
+			;;
+		runit-run)
+			skip_distupgrade "#542157: fails to install"
+			;;
+		sun-java5-doc|\
+		sun-java6-doc)
+			skip_distupgrade "#709967: fails to install: download location no longer available"
+			;;
+		trn)
+			skip_distupgrade "fails to install: prompts without using debconf"
+			;;
+		ttf-root-installer)
+			skip_distupgrade "#678807: fails to install: ftp:// download location disappeared"
+			;;
+	esac
+
+fi
+
+if [ "${PIUPARTS_DISTRIBUTION}" = "squeeze" ]; then
+
+	case ${PIUPARTS_OBJECTS%%=*} in
+		distributed-net)
+			skip_distupgrade "#655922: noninteractive install: postinst loops infinitely"
+			;;
+		firmware-b43-lpphy-installer)
+			skip_distupgrade "#623703: download location no longer available"
+			;;
+		fossology|\
+		fossology-agents|\
+		fossology-agents-single|\
+		fossology-scheduler|\
+		fossology-web|\
+		fossology-web-single)
+			# removed after squeeze
+			skip_distupgrade "#595593: fails to install"
+			;;
+		mini-buildd-bld|\
+		mini-buildd-rep)
+			skip_distupgrade "#632955, #656746: time limit exceeded during install"
+			fail_test "#632955, #656746: time limit exceeded during install"
+			;;
+	esac
+
+fi
+
+if [ "${PIUPARTS_DISTRIBUTION}" = "wheezy" ]; then
+
+	case ${PIUPARTS_OBJECTS%%=*} in
+		firmware-b43-lpphy-installer)
+			skip_distupgrade "#623703: download location no longer available"
+			;;
+		nautilus-dropbox)
+			skip_distupgrade "#770402: downloads awfully slow, hangs during installation"
+			;;
+	esac
+
+fi
+
+if [ "${PIUPARTS_DISTRIBUTION}" = "jessie" ]; then
+
+	case ${PIUPARTS_OBJECTS%%=*} in
+		steam)
+			skip_distupgrade "#772598: cannot preseed licence acceptance"
+			;;
+		susv4)
+			skip_distupgrade "#900882: download location no longer available"
+			;;
+	esac
+
+fi
+
 case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 	acgvision-agent@squeeze)
 		# removed after squeeze
@@ -55,9 +154,6 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 			skip_distupgrade "arch:all package no longer installable in stretch/i386"
 		fi
 		;;
-	blootbot@lenny)
-		skip_distupgrade "#502753: fails to install: prompts without using debconf"
-		;;
 	bugzilla3@lenny)
 		# fixed in squeeze, removed after squeeze, excluded from squeeze-lts
 		skip_distupgrade "was excluded from squeeze-lts"
@@ -75,13 +171,6 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 		# removed after lenny
 		skip_distupgrade "#523770: must use invoke-rc.d"
 		;;
-	distributed-net@lenny|\
-	distributed-net@squeeze)
-		skip_distupgrade "#655922: noninteractive install: postinst loops infinitely"
-		;;
-	dnscache-run@lenny)
-		skip_distupgrade "#664848: breaks dns resolution in chroot"
-		;;
 	docbookwiki@lenny)
 		# removed after lenny
 		skip_distupgrade "unconditionally uses mysql-client during purge"
@@ -102,15 +191,6 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 		# removed after lenny
 		skip_distupgrade "#574220: package purge (after dependencies removal) fails"
 		;;
-	fossology@squeeze|\
-	fossology-agents@squeeze|\
-	fossology-agents-single@squeeze|\
-	fossology-scheduler@squeeze|\
-	fossology-web@squeeze|\
-	fossology-web-single@squeeze)
-		# removed after squeeze
-		skip_distupgrade "#595593: fails to install"
-		;;
 	gclcvs@squeeze)
 		# removed after squeeze
 		skip_distupgrade "modifies conffiles: /etc/default/gclcvs"
@@ -126,12 +206,6 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 		# removed after lenny
 		skip_distupgrade "modifies conffiles: /etc/security/pam_rsa.conf"
 		;;
-	libpils0@lenny|\
-	libpils-dev@lenny|\
-	libstonith0@lenny|\
-	libstonith-dev@lenny)
-		skip_distupgrade "#702055: transitional packages uninstallable in lenny"
-		;;
 	localepurge@*)
 		skip_distupgrade "requires a full --reinstall to restore disappeared files"
 		;;
@@ -143,20 +217,6 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 		# removed after lenny
 		skip_distupgrade "#574228: package purge (after dependencies removal) fails"
 		;;
-	mhc@lenny)
-		skip_distupgrade "#696231: fails to install: emacs-package-install failure"
-		;;
-	mini-buildd-bld@squeeze|\
-	mini-buildd-rep@squeeze)
-		skip_distupgrade "#632955, #656746: time limit exceeded during install"
-		fail_test "#632955, #656746: time limit exceeded during install"
-		;;
-	multex-bin@lenny)
-		skip_distupgrade "#668791: fails to install: The format file of 'multex' is NOT built successfully."
-		;;
-	nautilus-dropbox@wheezy)
-		skip_distupgrade "#770402: downloads awfully slow, hangs during installation"
-		;;
 	nessusd@lenny|\
 	harden-remoteaudit@lenny)
 		# removed after lenny
@@ -176,10 +236,6 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 		# removed after squeeze
 		skip_distupgrade "#698533: modifies a shipped file: /usr/share/poker-web/currency_one.php"
 		;;
-	python-milter@lenny|\
-	spf-milter-python@lenny)
-		skip_always "virus signatures in python-milter may trigger the firewall"
-		;;
 	python-poker-stats@squeeze)
 		# removed after squeeze
 		skip_distupgrade "#608646: cronjob exits with error after package removal"
@@ -192,28 +248,12 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 		# removed after squeeze
 		skip_distupgrade "fails to purge: rt3.8-rtfm.postrm: ucf: not found"
 		;;
-	runit-run@lenny)
-		skip_distupgrade "#542157: fails to install"
-		;;
 	sdic-eijiro@lenny)
 		skip_distupgrade "requires a dictionary CD"
 		;;
 	sdic-eijiro@*)
 		skip_always "requires a dictionary CD"
 		;;
-	steam@jessie)
-		skip_distupgrade "#772598: cannot preseed licence acceptance"
-		;;
-	sun-java5-doc@lenny|\
-	sun-java6-doc@lenny)
-		skip_distupgrade "#709967: fails to install: download location no longer available"
-		;;
-	trn@lenny)
-		skip_distupgrade "fails to install: prompts without using debconf"
-		;;
-	ttf-root-installer@lenny)
-		skip_always "fails to install: download location disappeared"
-		;;
 	usrmerge@*)
 		skip_always "performs piuparts-unfriendy filesystem layout changes"
 		;;

--- a/debian/changelog
+++ b/debian/changelog
+piuparts (0.98) unstable; urgency=medium
+
+  [ Holger Levsen ]
+  * piuparts-reports.py: add link to states graph for each suite.
+  * instances/piuparts.conf-template.pejacevic: increase sid-nodoc's
+    precedence.
+  * Refactor reschedule_piuparts_tests to use new global configuration
+    variable 'testing-suite'.
+  * generate_daily_report: show failures in sid, $TESTING and testing2sid
+    at the top of the report.
+  * master-bin/prepare_backup: ignore empty submissions.txt for $TESTING-pu
+    and $TESTING-security.
+  * Update TODO for bullseye.
+
+  [ Andreas Beckmann ]
+  * piuparts.py:
+    - If packages were downgraded to restore the reference chroot state
+      reinstall the downgraded-to version again to recover potentially lost
+      files in case of bad interactions of Breaks/Replaces with downgrading.
+  * distros.conf: Add support for buster partial distros.
+  * piuparts.conf:
+    - Use --warn-on-others for [sid-strict] only.
+    - Add sections [buster-security], [buster-pu] and [buster2proposed].
+    - Add sections [stretch2Xbuster] and [stretch2Xbuster-rcmd].
+    - Add section [sid-broken-symlinks], failing on broken symlinks
+  * piuparts-slave.py:
+    - Catch IOError while creating tarballs, will be retried later.
+  * scripts/is_testable_uninstallable:
+    - Refactor handling of known uninstallable packages.
+  * scripts-multi-distro-upgrade/: New custom scripts for distupgrade tests
+    spanning a series of distros.
+  * Add metapackages piuparts-{master,slave}-from-git-deps to ease running
+    piuparts-master and piuparts-slave instances from git.
+
+ -- Holger Levsen <holger@debian.org>  Fri, 01 Mar 2019 11:27:47 +0100
+
 piuparts (0.97) unstable; urgency=medium

  [ Andreas Beckmann ]

--- a/debian/control
+++ b/debian/control
@@ -32,6 +32,7 @@ Package: piuparts
 Architecture: all
 Depends:
 piuparts-common (= ${binary:Version}),
+# keep this list in sync with piuparts-slave-from-git-deps
 debootstrap (>= 1.0.89~),
 debsums (>= 2.2.2~),
 lsb-release,
@@ -62,9 +63,11 @@ Built-Using: ${misc:Built-Using}
 Depends:
 piuparts-common (>= ${source:Version}),
 piuparts-common (<< ${source:Version}+),
+# keep this list in sync with piuparts-master-from-git-deps
 adduser,
 openssh-server,
 python-debianbts,
+ python-setproctitle,
 tango-icon-theme,
 xz-utils,
 python-yaml,
@@ -90,11 +93,42 @@ Description: piuparts master components
 operate. It also contains piuparts-report to generate web pages with the
 testing results as well as documentation.

+Package: piuparts-master-from-git-deps
+Architecture: all
+Depends:
+ git,
+# this list is synced from piuparts-master
+ adduser,
+ openssh-server,
+ python-debianbts,
+ python-setproctitle,
+ tango-icon-theme,
+ xz-utils,
+ python-yaml,
+ python (>= 2.7),
+# selected packages from piuparts-master Recommends
+ apache2 | httpd,
+ python-rpy2,
+# this list is synced from piuparts-common
+ python-apt,
+ python-distro-info,
+ python-lzma,
+ ${misc:Depends},
+Description: dependencies for running piuparts master from git
+ piuparts is meant as a quality assurance tool for people who create .deb
+ packages to test them before they upload them to the Debian package archive.
+ A master/slave piuparts installation is used for testing complete archives
+ as it is done on https://piuparts.debian.org/.
+ .
+ This metapackage provides all the dependencies needed for running a
+ piuparts-master instance (including piuparts-report etc.) from git.
+
 Package: piuparts-slave
 Architecture: all
 Depends:
 piuparts (= ${binary:Version}),
 piuparts-common (= ${binary:Version}),
+# keep this list in sync with piuparts-slave-from-git-deps
 adduser,
 openssh-client,
 screen,
@@ -113,9 +147,41 @@ Description: piuparts slave components
 This package provides the piuparts slave components, which need a piuparts
 master to operate. See the documentation in the piuparts-master package.

+Package: piuparts-slave-from-git-deps
+Architecture: all
+Depends:
+ git,
+# this list is synced from piuparts-slave
+ adduser,
+ openssh-client,
+ screen,
+ sudo,
+ python (>= 2.7),
+# this list is synced from piuparts
+ debootstrap (>= 1.0.89~),
+ debsums (>= 2.2.2~),
+ lsb-release,
+ lsof,
+ mount,
+ python-debian,
+# this list is synced from piuparts-common
+ python-apt,
+ python-distro-info,
+ python-lzma,
+ ${misc:Depends},
+Description: dependencies for running piuparts slave from git
+ piuparts is meant as a quality assurance tool for people who create .deb
+ packages to test them before they upload them to the Debian package archive.
+ A master/slave piuparts installation is used for testing complete archives
+ as it is done on https://piuparts.debian.org/.
+ .
+ This metapackage provides all the dependencies needed for running a
+ piuparts-slave instance from git.
+
 Package: piuparts-common
 Architecture: all
 Depends:
+# keep this list in sync with piuparts-{master,slave}-from-git-deps
 python-apt,
 python-distro-info,
 python-lzma,

--- a/instances/piuparts.conf-template.pejacevic
+++ b/instances/piuparts.conf-template.pejacevic
 [DEFAULT]
 # these are needed always
 flags-base =
-# restrict to problems in the package being tested
-	--warn-on-others
 # default exceptions
 	--scriptsdir /etc/piuparts/scripts
 # pejacevic's slaves are doing everything relevant on a ramdisk anyway
@@ -18,6 +16,8 @@ flags-default =
 # like default flags, but failing on leftover files
 flags-leftovers =
 	%(flags-base)s
+# restrict to problems in the package being tested
+	--warn-on-others
 # perform some additional cleanup
 	--scriptsdir /etc/piuparts/scripts-leftovers

@@ -104,6 +104,46 @@ flags-start-lenny =
 # and removing a shell would remove its entry from /etc/shells
 	-i /etc/shells

+# common flags for testing for broken symlinks
+flags-broken-symlinks =
+	%(flags-default)s
+	--fail-on-broken-symlinks
+	--install-recommends
+	--scriptsdir /etc/piuparts/scripts-broken-symlinks
+#	--fake-essential-packages libjs-sphinxdoc
+# ignore intentionally broken symlinks
+	-i :/usr/lib/libcxx/test/std/experimental/filesystem/Inputs/static_test_env/bad_symlink
+	-i :/usr/share/gocode/src/github.com/Masterminds/glide/testdata/path/x/vendor
+	-i :/usr/share/gocode/src/github.com/constabulary/gb/internal/fileutils/_testdata/copyfile/a/rick
+	-i :/usr/share/gocode/src/github.com/hashicorp/atlas-go/archive/test-fixtures/archive-symlink/link/link
+	-i :/usr/share/gocode/src/github.com/prometheus/procfs/fixtures/26231/exe
+	-i :/usr/share/gocode/src/github.com/prometheus/procfs/fixtures/26231/ns/mnt
+	-i :/usr/share/gocode/src/github.com/prometheus/procfs/fixtures/26231/ns/net
+	-i :/usr/share/gocode/src/github.com/prometheus/procfs/fixtures/26232/cwd
+	-i :/usr/share/gocode/src/github.com/prometheus/procfs/fixtures/26232/root
+# ignore broken symlinks
+	-i :/etc/systemd/system/display-manager.service
+#   civicrm-common, #857601, wontfix, /usr/share/civicrm/packages/OpenFlashChart/open-flash-chart.swf -> ../../../../local/open-flash-chart.swf
+	-i :/usr/share/civicrm/packages/OpenFlashChart/open-flash-chart.swf
+#   musescore-common, #910145, wontfix, /usr/share/mscore-2.3/sound/{sf2,sfz} -> ../../sounds/{sf2,sfz}
+	-i :/usr/share/mscore-2.3/sound/sf2
+	-i :/usr/share/mscore-2.3/sound/sfz
+	-i :/usr/share/mscore-3.0/sound/sf2
+	-i :/usr/share/mscore-3.0/sound/sfz
+#   mender-client, #915680, /var/lib/mender -> /data/mender
+	-i :/var/lib/mender
+# ignore broken symlinks (patterns)
+	-I :/etc/.*/supervise
+	-I /usr/include/python2\.7_d/.*\.h
+	-I /usr/include/python3\..dm/.*\.h
+	-I /usr/include/python3\..dm/internal
+	-I :/usr/lib/jvm/java-.*-openjdk-.*/src\.zip
+	-I :/usr/share/doc/kde/HTML/.*/common
+	-I :/usr/share/man/man1/(.*-)?python.*-config\.1\.gz
+# ignore broken symlinks (third party game data)
+	-I :/usr/lib/openjk/JediAcademy/base/assets.\.pk3
+	-I :/usr/lib/rtcw/main/.*\.(cfg|pk3)
+

 [global]
 basetgz-sections =
@@ -120,13 +160,20 @@ sections =
 	sid
 	sid-strict
 	sid-nodoc
+	sid-broken-symlinks
 	testing2sid
 	stable2sid
 	stable22sid
 	buster
 	buster-rcmd
+	buster-security
+	buster-pu
+#	buster-proposed
+	buster2proposed
 	stretch2buster
+	stretch2Xbuster
 	stretch2buster-rcmd
+	stretch2Xbuster-rcmd
 	stretch2bpo2buster
 	stretch2bpo
 	stretch
@@ -166,6 +213,7 @@ sections =
 	squeeze2squeeze-lts
 	squeeze
 	lenny2squeeze
+testing-suite = buster
 mirror = @MIRROR@
 master-host = pejacevic.debian.org
 master-user = piupartsm
@@ -314,7 +362,7 @@ distro = sid


 [sid-nodoc]
-precedence = 9
+precedence = 7
 description = + Testing without files in /usr/share/doc.
 piuparts-flags =
 	%(flags-default)s
@@ -322,6 +370,15 @@ piuparts-flags =
 distro = sid


+[sid-broken-symlinks]
+precedence = 9
+description = + Failing on broken symlinks.
+json-sections = none
+piuparts-flags =
+	%(flags-broken-symlinks)s
+distro = sid
+
+
 [testing2sid]
 precedence = 2
 piuparts-flags =
@@ -394,6 +451,49 @@ piuparts-flags =
 distro = buster


+[buster-security]
+precedence = 2
+json-sections = none
+piuparts-flags =
+	%(flags-default)s
+	%(flags-start-buster)s
+	%(flags-end-buster)s
+depends-sections = buster
+distro = buster/updates
+
+
+[buster-pu]
+precedence = 3
+json-sections = none
+piuparts-flags =
+	%(flags-default)s
+	%(flags-start-buster)s
+	%(flags-end-buster)s
+depends-sections = buster
+distro = buster-proposed-updates
+
+
+[buster-proposed]
+precedence = 3
+json-sections = none
+piuparts-flags =
+	%(flags-default)s
+	%(flags-start-buster)s
+	%(flags-end-buster)s
+distro = buster-proposed
+
+
+[buster2proposed]
+precedence = 3
+json-sections = none
+piuparts-flags =
+	%(flags-default)s
+	%(flags-start-buster)s
+	%(flags-end-buster)s
+distro = buster
+upgrade-test-distros = buster buster-proposed
+
+
 [stretch2buster]
 precedence = 4
 json-sections = none
@@ -405,6 +505,20 @@ distro = stretch
 upgrade-test-distros = stretch buster


+[stretch2Xbuster]
+precedence = 5
+json-sections = none
+description = + Testing two-stage upgrades.
+piuparts-flags =
+	--upgrade-before-dist-upgrade
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+distro = stretch
+upgrade-test-distros = stretch buster
+exclude-known-problems = packages_have_been_kept_back_error.conf packages_have_been_kept_back_issue.conf
+
+
 [stretch2buster-rcmd]
 precedence = 5
 json-sections = none
@@ -418,6 +532,21 @@ distro = stretch
 upgrade-test-distros = stretch buster


+[stretch2Xbuster-rcmd]
+precedence = 6
+json-sections = none
+description = + Testing two-stage upgrades with --install-recommends.
+piuparts-flags =
+	--install-recommends
+	--upgrade-before-dist-upgrade
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+distro = stretch
+upgrade-test-distros = stretch buster
+exclude-known-problems = packages_have_been_kept_back_error.conf packages_have_been_kept_back_issue.conf
+
+
 [stretch2bpo2buster]
 precedence = 5
 json-sections = none

--- a/instances/piuparts.conf.anbe
+++ b/instances/piuparts.conf.anbe
@@ -4,7 +4,6 @@

 [DEFAULT]
 flags-base =
-	--warn-on-others
 	--no-eatmydata
 	--scriptsdir /etc/piuparts/scripts
 	--allow-database
@@ -13,6 +12,7 @@ flags-default =
 	%(flags-base)s
 	--warn-on-leftovers-after-purge
 flags-leftovers =
+	--warn-on-others
 	%(flags-base)s
 	--scriptsdir /etc/piuparts/scripts-leftovers

@@ -406,6 +406,8 @@ sections =
 	lenny_i386/non-free
 	lenny2squeeze/non-free
 #
+testing-suite = buster
+#
 json-sections = none
 #
 proxy = http://localhost:3128
@@ -2225,6 +2227,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-lenny)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 arch = amd64
 area = main
 components = main
@@ -2240,6 +2243,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-lenny)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = lenny222testing/main lenny222testing/non-free
 arch = amd64
 area = contrib
@@ -2253,6 +2257,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-lenny)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = lenny222testing/main lenny222testing/contrib
 arch = amd64
 area = non-free
@@ -2267,6 +2272,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-lenny)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 arch = i386
 setarch = linux32
 area = main
@@ -2283,6 +2289,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-lenny)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = lenny222testing_i386/main lenny222testing_i386/non-free
 arch = i386
 setarch = linux32
@@ -2297,6 +2304,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-lenny)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = lenny222testing_i386/main lenny222testing_i386/contrib
 arch = i386
 setarch = linux32
@@ -2623,6 +2631,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-squeeze)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 arch = amd64
 area = main
 components = main
@@ -2636,6 +2645,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-squeeze)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = squeeze222testing/main squeeze222testing/non-free
 arch = amd64
 area = contrib
@@ -2649,6 +2659,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-squeeze)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = squeeze222testing/main squeeze222testing/contrib
 arch = amd64
 area = non-free
@@ -2663,6 +2674,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-squeeze)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 arch = i386
 setarch = linux32
 area = main
@@ -2677,6 +2689,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-squeeze)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = squeeze222testing_i386/main squeeze222testing_i386/non-free
 arch = i386
 setarch = linux32
@@ -2691,6 +2704,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-squeeze)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = squeeze222testing_i386/main squeeze222testing_i386/contrib
 arch = i386
 setarch = linux32
@@ -3110,6 +3124,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-wheezy)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 arch = amd64
 area = main
 components = main
@@ -3123,6 +3138,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-wheezy)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = wheezy222testing/main wheezy222testing/non-free
 arch = amd64
 area = contrib
@@ -3136,6 +3152,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-wheezy)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = wheezy222testing/main wheezy222testing/contrib
 arch = amd64
 area = non-free
@@ -3150,6 +3167,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-wheezy)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 arch = i386
 setarch = linux32
 area = main
@@ -3164,6 +3182,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-wheezy)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = wheezy222testing_i386/main wheezy222testing_i386/non-free
 arch = i386
 setarch = linux32
@@ -3178,6 +3197,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-wheezy)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = wheezy222testing_i386/main wheezy222testing_i386/contrib
 arch = i386
 setarch = linux32
@@ -3701,6 +3721,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-jessie)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 arch = amd64
 area = main
 components = main
@@ -3714,6 +3735,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-jessie)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = jessie222testing/main jessie222testing/non-free
 arch = amd64
 area = contrib
@@ -3727,6 +3749,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-jessie)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = jessie222testing/main jessie222testing/contrib
 arch = amd64
 area = non-free
@@ -3741,6 +3764,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-jessie)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 arch = i386
 setarch = linux32
 area = main
@@ -3755,6 +3779,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-jessie)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = jessie222testing_i386/main jessie222testing_i386/non-free
 arch = i386
 setarch = linux32
@@ -3769,6 +3794,7 @@ piuparts-flags =
 	%(flags-default)s
 	%(flags-start-jessie)s
 	%(flags-end-testing)s
+	--scriptsdir /etc/piuparts/scripts-multi-distro-upgrade
 depends-sections = jessie222testing_i386/main jessie222testing_i386/contrib
 arch = i386
 setarch = linux32

--- a/master-bin/generate_daily_report.in
+++ b/master-bin/generate_daily_report.in
 #!/bin/sh

 # Copyright © 2011-2013 Andreas Beckmann (anbe@debian.org)
-# Copyright © 2018 Holger Levsen (holger@layer-acht.org)
+# Copyright © 2018-2019 Holger Levsen (holger@layer-acht.org)
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -24,6 +24,7 @@ get_config_value SECTIONS global sections
 get_config_value HTDOCS global output-directory
 get_config_value URLBASE global urlbase https://piuparts.debian.org
 get_config_value PYTHONPATH global PYTHONPATH ''
+get_config_value TESTING global testing-suite

 export PYTHONPATH

@@ -82,7 +83,10 @@ for SECTION in $SECTIONS ; do
 done | sed s#^$MASTER#$URLBASE# >> $DAILYTMP

 echo "$(cat $DAILYTMP | wc -l) new failures:" >> $DAILYREPORT
-cat $DAILYTMP >> $DAILYREPORT
+# show failures (and show failures in sid, $TESTING and testing2sid first
+cat $DAILYTMP | (egrep "/(sid|$TESTING|testing2sid)/" 2>/dev/null || true) >> $DAILYREPORT
+echo >> $DAILYREPORT
+cat $DAILYTMP | (egrep -v "/(sid|$TESTING|testing2sid)/" 2>/dev/null || true) >> $DAILYREPORT
 rm $DAILYTMP

 echo "" >> $DAILYREPORT

--- a/master-bin/prepare_backup.in
+++ b/master-bin/prepare_backup.in
@@ -32,11 +32,13 @@ get_config_value SECTIONS global sections
 get_config_value MASTER global master-directory
 get_config_value HTDOCS global output-directory
 get_config_value BACKUPDIR global backup-directory ''
-
+get_config_value TESTING global testing-suite

 copy_to_backupdir() {
 	if [ -f $MASTER/$1 ]; then
 		cp $MASTER/$1 $BACKUPDIR/$1
+	elif [ "$(basename $1)" = "submissions.txt" ] && ([ "$(dirname $1)" = "${TESTING}-pu" ] || [ "$(dirname $1)" = "${TESTING}-security" ]) ; then
+	       :	# during development these suites are created but stay empty
 	else
 		echo "Warning: $MASTER/$1 does not exist." # this produces a mail and that's a feature
 	fi

--- a/master-bin/reschedule_piuparts_tests.in
+++ b/master-bin/reschedule_piuparts_tests.in
@@ -2,6 +2,7 @@
 set -e

 # Copyright © 2013-2017 Andreas Beckmann (anbe@debian.org)
+# Copyright © 2018 Holger Levsen (holger@layer-acht.org)
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -21,9 +22,7 @@ set -e

 get_config_value MASTER global master-directory
 get_config_value SECTIONS global sections
-
-
-TESTING="buster"
+get_config_value TESTING global testing-suite

 dryrun=""
 current=""

--- a/piuparts-report.py
+++ b/piuparts-report.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright 2005 Lars Wirzenius (liw@iki.fi)
-# Copyright 2009-2018 Holger Levsen (holger@layer-acht.org)
+# Copyright 2009-2019 Holger Levsen (holger@layer-acht.org)
 # Copyright © 2011-2018 Andreas Beckmann (anbe@debian.org)
 # Copyright 2013 David Steele (dsteele@gmail.com)
 #
@@ -758,6 +758,8 @@ def create_section_navigation(section_names, current_section, doc_root):
                 % (doc_root, current_section)
    tablerows += "<tr><td class=\"contentcell\"><a href=\"%s/%s/source/\">by source package</a></td></tr>\n" \
                 % (doc_root, current_section)
+    tablerows += "<tr><td class=\"contentcell\">states <a href=\"%s/%s/states.png/\">graph</a></td></tr>\n" \
+                 % (doc_root, current_section)
    tablerows += "<tr class=\"titlerow\"><td class=\"alerttitlecell\">all tested suites</td></tr>"
    for section in section_names:
        tablerows += ("<tr class=\"normalrow\"><td class=\"contentcell\"><a href='%s/%s'>%s</a></td></tr>\n") % \

--- a/piuparts-slave.py
+++ b/piuparts-slave.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright 2005 Lars Wirzenius (liw@iki.fi)
-# Copyright © 2011-2017 Andreas Beckmann (anbe@debian.org)
+# Copyright © 2011-2019 Andreas Beckmann (anbe@debian.org)
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -919,6 +919,7 @@ def create_chroot(config, tarball, distro):
                                       time.gmtime()))
            output.write("Executing: " + command2string(command) + "\n\n")
            logging.debug("Executing: " + command2string(command))
+            try:
                p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                for line in p.stdout:
                    output.write(line)
@@ -930,6 +931,10 @@ def create_chroot(config, tarball, distro):
                    os.rename(tarball + ".new", tarball)
                else:
                    logging.error("Tarball creation failed, see %s" % output_name)
+            except IOError:
+                output.write(time.strftime("\nFAIL: %Y-%m-%d %H:%M:%S %Z\n",
+                                           time.gmtime()))
+                logging.error("Tarball creation failed with IOError")


 def create_file(filename, contents):

--- a/piuparts.py
+++ b/piuparts.py
@@ -3,7 +3,7 @@
 #
 # Copyright 2005 Lars Wirzenius (liw@iki.fi)
 # Copyright © 2007-2018 Holger Levsen (holger@layer-acht.org)
-# Copyright © 2010-2018 Andreas Beckmann (anbe@debian.org)
+# Copyright © 2010-2019 Andreas Beckmann (anbe@debian.org)
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -672,7 +672,7 @@ def split_path(pathname):
    parts = []
    while pathname:
        (head, tail) = os.path.split(pathname)
-        # print "split '%s' => '%s' + '%s'" % (pathname, head, tail)
+        # print("split '%s' => '%s' + '%s'" % (pathname, head, tail))
        if tail:
            parts.append(tail)
        elif not head:
@@ -697,18 +697,18 @@ def canonicalize_path(root, pathname, report_links=False):
    one for each symlink encountered.

    """
-    # print "\nCANONICALIZE %s %s" % (root, pathname)
+    # print("\nCANONICALIZE %s %s" % (root, pathname))
    links = []
    seen = []
    parts = split_path(pathname)
-    # print "PARTS ", list(reversed(parts))
+    # print("PARTS ", list(reversed(parts)))
    path = "/"
    while parts:
        tag = "\n".join(parts + [path])
-        # print "TEST '%s' + " % path, list(reversed(parts))
+        # print("TEST '%s' + " % path, list(reversed(parts)))
        if tag in seen or len(seen) > 1024:
            fullpath = os.path.join(path, *reversed(parts))
-            # print "LOOP %s" % fullpath
+            # print("LOOP %s" % fullpath)
            path = fullpath
            logging.error("ELOOP: Too many symbolic links in '%s'" % path)
            break
@@ -724,14 +724,14 @@ def canonicalize_path(root, pathname, report_links=False):
            path = "/"
        elif os.path.islink(rootedpath):
            target = os.readlink(rootedpath)
-            # print "LINK to '%s'" % target
+            # print("LINK to '%s'" % target)
            links.append((newpath, target))
            if os.path.isabs(target):
                path = "/"
            parts.extend(split_path(target))
        else:
            path = newpath
-    # print "FINAL '%s'" % path
+    # print("FINAL '%s'" % path)
    if report_links:
        return links
    return path
@@ -747,7 +747,7 @@ def is_broken_symlink(root, dirpath, filename):

    # The symlink chain, if any, has now been resolved. Does the target
    # exist?
-    # print "EXISTS ", pathname, os.path.exists(pathname)
+    # print("EXISTS ", pathname, os.path.exists(pathname))
    return not os.path.exists(pathname)


@@ -1561,6 +1561,10 @@ class Chroot:
            self.apt_get_install(to_remove=all_to_remove,
                                 to_install=version_qualified,
                                 flags=["--no-install-recommends", "--force-yes"])
+            # reinstall potentially downgraded packages, they may have
+            # missing files in case of unfortunate Breaks+Replaces timing
+            self.apt_get_install(to_install=version_qualified,
+                                 flags=["--no-install-recommends", "--reinstall"])
        else:
            self.remove_packages(all_to_remove)