README_server.txt

@@ -452,8 +452,9 @@ section, too, and will serve as defaults for all other sections
  automatically selected default name) is located. The default is
  '.'.
 
-* "chroot-meta-auto" (global, section) is a file in the section
- directory where the slave will store cached chroot meta data for
+* "chroot-meta-auto" (global, section) is a file in
+ "chroot-meta-directory" (falling back to the section directory)
+ where the slave will store cached chroot meta data for
  the reference target chroot in distupgrade tests. This speeds up
  distupgrade tests since it avoids doing an empty upgrade test to
  generate this data on-the-fly as part of each test. Cached data
@@ -461,6 +462,9 @@ section, too, and will serve as defaults for all other sections
  versions available in the chroot is detected earlier.
  This is not set (and therefore not enabled) by default.
 
+* "chroot-meta-directory" is the directory where "chroot-meta-auto"
+ stores the reference target chroot metadata.
+
 * "upgrade-test-distros" is the space delimited list of
  distributions the slave should use for testing upgrades
  between distributions (i.e., Debian versions). Using "partial"

custom-scripts/scripts-leftovers/pre_remove_preseed_cleanup

@@ -29,6 +29,10 @@ case ${PIUPARTS_OBJECTS%%=*} in
 		log_debug
 		echo "docbookwiki docbookwiki/purge_books boolean true" | debconf-set-selections
 		;;
+	grub-pc)
+		log_debug
+		echo "grub-pc grub-pc/postrm_purge_boot_grub boolean true" | debconf-set-selections
+		;;
 	ifetch-tools)
 		log_debug
 		echo "ifetch-tools ifetch-tools/purge boolean true" | debconf-set-selections

custom-scripts/scripts-multi-distro-upgrade/post_setup_fake-essential_multi-distro-upgrade

+#!/bin/sh
+set -e
+
+log_debug() {
+	echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
+}
+
+case ${PIUPARTS_OBJECTS} in
+	mediamate=None)
+		# removed after lenny
+		#574228: mediamate: package purge (after dependencies removal) fails
+		log_debug
+		apt-get -y install wwwconfig-common
+		;;
+	moodle-book=None|\
+	moodle-debian-edu-theme=None|\
+	moodle=None)
+		# fixed after squeeze, not in stable after squeeze
+		#678027: moodle: fails to purge - command ucf in postrm not found
+		log_debug
+		apt-get -y install ucf
+		;;
+	rt3.6-rtfm=None)
+		# removed after lenny
+		# fails to purge: rt3.6-rtfm.postrm: ucf: not found
+		log_debug
+		apt-get -y install ucf
+		;;
+	rt3.8-rtfm=None)
+		# removed after squeeze
+		# fails to purge: rt3.8-rtfm.postrm: ucf: not found
+		log_debug
+		apt-get -y install ucf
+		;;
+	torrentflux=None)
+		# removed after wheezy
+		#677061: torrentflux: fails to purge - command ucf in postrm not found
+		log_debug
+		apt-get -y install ucf
+		;;
+esac

custom-scripts/scripts-multi-distro-upgrade/pre_remove_exceptions_multi-distro-upgrade

+#!/bin/sh
+set -e
+
+log_debug() {
+	echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
+}
+
+case ${PIUPARTS_OBJECTS} in
+	autotrust=None)
+		# removed after squeeze
+		# logrotate config references /var/log/autotrust/*.log
+		log_debug
+		touch /var/log/autotrust/dummy.log
+		;;
+	dancer-ircd=None)
+		# removed after wheezy
+		# logrotate config references /var/log/dancer-ircd/ircd.log
+		log_debug
+		touch /var/log/dancer-ircd/ircd.log
+		;;
+	dsyslog=None)
+		# removed after jessie
+		# logrotate config references /var/log/messages
+		log_debug
+		touch /var/log/messages
+		;;
+	fossology-scheduler-single=None)
+		# removed after squeeze
+		# logrotate config references /var/log/fossology/*.log
+		log_debug
+		chown root:root /var/log/fossology
+		touch /var/log/fossology/dummy.log
+		;;
+	fspd=None)
+		# removed after lenny
+		# logrotate config references /var/log/fspd/fspd.log, /var/log/fspd/xferlog
+		log_debug
+		touch /var/log/fspd/fspd.log
+		touch /var/log/fspd/xferlog
+		;;
+	gidentd=None)
+		# removed after lenny
+		# logrotate config references /var/log/gidentd.log
+		log_debug
+		touch /var/log/gidentd.log
+		;;
+	interchange=None)
+		# removed after wheezy
+		# logrotate config references /var/log/interchange/debug.log, /var/log/interchange/error.log, /var/log/interchange/makecat.log
+		log_debug
+		chown root:root /var/log/interchange
+		touch /var/log/interchange/debug.log
+		touch /var/log/interchange/error.log
+		touch /var/log/interchange/makecat.log
+		;;
+	mserv=None)
+		# removed after lenny
+		# logrotate config references /var/log/mserv/mserv.log
+		log_debug
+		touch /var/log/mserv/mserv.log
+		;;
+	nws=None)
+		# removed after lenny
+		# logrotate config references /var/log/nws/*.err, /var/log/nws/*.log
+		log_debug
+		touch /var/log/nws/dummy.err
+		touch /var/log/nws/dummy.log
+		;;
+	qwik=None)
+		# removed after lenny
+		# logrotate config references /var/log/qwik/*log
+		log_debug
+		touch /var/log/qwik/dummy.log
+		;;
+	smtpfeed=None)
+		# removed after lenny
+		# logrotate config references /var/log/smtpfeed.log
+		log_debug
+		touch /var/log/smtpfeed.log
+		;;
+esac

custom-scripts/scripts-squeeze/post_setup_squeeze-fake-essential

@@ -19,6 +19,11 @@ case ${PIUPARTS_OBJECTS%%=*} in
 		# allow testing of the fake essential packages
 		exit 0
 		;;
+	mediamate)
+		# removed after lenny
+		#574228: mediamate: package purge (after dependencies removal) fails
+		USED_DURING_PURGE="$USED_DURING_PURGE wwwconfig-common"
+		;;
 	phpbb3)
 		USED_DURING_PURGE="$USED_DURING_PURGE dbconfig-common"
 		;;

custom-scripts/scripts/is_testable_uninstallable

@@ -220,10 +220,6 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 		# removed after lenny
 		skip_distupgrade "modifies conffiles: /var/lib/mailreader/ads/debian/ad.cfg, /etc/mailreader/{main,server}.cfg"
 		;;
-	mediamate@lenny)
-		# removed after lenny
-		skip_distupgrade "#574228: package purge (after dependencies removal) fails"
-		;;
 	nessusd@lenny|\
 	harden-remoteaudit@lenny)
 		# removed after lenny
@@ -247,14 +243,6 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 		# removed after squeeze
 		skip_distupgrade "#608646: cronjob exits with error after package removal"
 		;;
-	rt3.6-rtfm@lenny)
-		# removed after lenny
-		skip_distupgrade "fails to purge: rt3.6-rtfm.postrm: ucf: not found"
-		;;
-	rt3.8-rtfm@squeeze)
-		# removed after squeeze
-		skip_distupgrade "fails to purge: rt3.8-rtfm.postrm: ucf: not found"
-		;;
 	sdic-eijiro@lenny)
 		skip_distupgrade "requires a dictionary CD"
 		;;

custom-scripts/scripts/post_distupgrade_exceptions

@@ -84,11 +84,13 @@ fi
 
 if [ "$PIUPARTS_DISTRIBUTION" = "stretch" ]; then
 
-	# fakeroot:i386 in jessie shipped these with permissions 0775 (#826318)
-	for dir in /usr/share/man/nl /usr/share/man/nl/man1
-	do
-		test ! -d "$dir" || chmod -c g-w "$dir"
-	done
+	# debianutils in jessie shipped /usr/share/man/sl/*/ with mode 0775
+	find /usr/share/man/sl -type d -perm /020 -exec chmod -c g-w {} +
+
+	# fakeroot:i386 in jessie shipped /usr/share/man/**/ with mode 0775 (#826318)
+	if is_installed fakeroot ; then
+		find /usr/share/man -type d -perm /020 -exec chmod -c g-w {} +
+	fi
 
 	# git:i386 in jessie shipped /usr/share/locale/**/ with mode 0775
 	if is_installed git ; then

custom-scripts/scripts/pre_distupgrade_zz_database-server

@@ -56,6 +56,10 @@ case "$PIUPARTS_DISTRIBUTION_NEXT" in
 		CANDIDATES="$CANDIDATES mariadb-server-10.3"
 		CANDIDATES="$CANDIDATES postgresql-11"
 		;;
+	sid)
+		CANDIDATES="$CANDIDATES mariadb-server-10.3"
+		CANDIDATES="$CANDIDATES postgresql-11"
+		;;
 esac
 
 

debian/changelog

@@ -28,6 +28,8 @@ piuparts (0.99) UNRELEASED; urgency=medium
   * piuparts-slave.py:
     - New option "slave-flush-interval" to flush logs more frequently to the
       master. Set to 30 minutes.
+    - New option "chroot-meta-directory" to share reference target chroot
+      metadata between slave instances.
   * scripts/{post_chroot_unpack,pre_distupgrade}_allow_unauthenticated:
     - Disable Check-Valid-Until for wheezy/updates and jessie-backports.
   * scripts/post_distupgrade_exceptions:

instances/piuparts.conf-template.pejacevic

@@ -232,6 +232,7 @@ output-directory = /srv/piuparts.debian.org/htdocs
 backup-directory = /srv/piuparts.debian.org/backup
 tmpdir = /srv/piuparts.debian.org/tmp
 doc-root = /
+chroot-meta-directory = /srv/piuparts.debian.org/slave/refchroot
 chroot-meta-auto = reference-chroot-metadata.dat
 components = main
 arch = @ARCH@

instances/piuparts.conf.anbe

@@ -436,6 +436,7 @@ basetgz-directory = /srv/piuparts/slave/basetgz
 output-directory = /srv/piuparts/htdocs
 backup-directory = /srv/piuparts/backup
 tmpdir = /srv/piuparts/tmp
+chroot-meta-directory = /srv/piuparts/slave/refchroot
 chroot-meta-auto = reference-chroot-metadata.dat
 slave-load-max = 15.5
 idle-sleep = 3600

master-bin/detect_piuparts_issues.in

@@ -54,7 +54,9 @@ add_pattern "E: Sub-process Popen returned an error code \(2\)"
 add_pattern "chroot: failed to run command '.*': Permission denied"
 add_pattern "ERROR: Command failed \(status=(-7|100)\): .* 'apt-cache'"
 add_pattern 'update-binfmts: warning: unable to close /proc/sys/fs/binfmt_misc/register: Invalid argument'
+add_pattern 'USERDEL: USER .* IS CURRENTLY LOGGED IN'
 add_pattern 'userdel: user .* is currently logged in'
+add_pattern 'mount: .* Cannot allocate memory'
 # Disk space issues.
 add_pattern "NO SPACE LEFT ON DEVICE"
 add_pattern "No space left on device"
@@ -68,6 +70,7 @@ add_pattern "(groupadd|useradd): (unable to lock|cannot rewrite) (group|password
 # Database server issues.
 add_pattern 'INVOKE-RC.D: INITSCRIPT (MYSQL|POSTGRESQL|POSTGRESQL-8.3), ACTION "(RE)?START" FAILED.'
 add_pattern 'invoke-rc.d: initscript (mysql|postgresql|postgresql-8.3), action "(re)?start" failed.'
+add_pattern 'There is a MySQL server running, but we failed in our attempts to stop it.'
 add_pattern 'The PostgreSQL server failed to start.'
 add_pattern 'Is another postmaster already running on port 543.'
 add_pattern 'No database found online on port 5432'

master-bin/report_stale_reserved_packages.in

@@ -39,7 +39,7 @@ if [ -s $LOGS ] ; then
 	echo "These packages have been unscheduled." >> $LOG_OUTPUT
 	echo >> $LOG_OUTPUT
 	for package_log in $(cat $LOGS) ; do
-		rm -fv $package_log >> $LOG_OUTPUT
+		mv -fv $package_log $(dirname $package_log)/../recycle/ >> $LOG_OUTPUT
 	done
 	publish_logs $LOG_OUTPUT "" $HTDOCS report_stale_reserved_packages
 fi

piuparts-slave.py

@@ -95,6 +95,7 @@ class Config(piupartslib.conf.Config):
                                          "upgrade-test-distros": None,
                                          "basetgz-directory": ".",
                                          "chroot-meta-auto": None,
+                                         "chroot-meta-directory": None,
                                          "max-reserved": 1,
                                          "debug": "no",
                                          "keep-sources-list": "no",
@@ -449,6 +450,28 @@ class Section:
 
         os.chdir(oldcwd)
 
+    def _get_refchroot_metadata(self):
+        if self._config["chroot-meta-auto"]:
+            if self._config["chroot-meta-directory"]:
+                path = os.path.join(self._config["chroot-meta-directory"], self._config.section)
+                if not os.path.exists(path):
+                    os.makedirs(path)
+                return os.path.join(path, self._config["chroot-meta-auto"])
+            return self._config["chroot-meta-auto"]
+        return None
+
+    def _check_refchroot_metadata(self):
+        refchroot_metadata = self._get_refchroot_metadata()
+        if refchroot_metadata:
+            if os.path.exists(refchroot_metadata):
+                try:
+                    age = time.time() - os.path.getmtime(refchroot_metadata)
+                    if age > 6 * 3600:
+                        os.unlink(refchroot_metadata)
+                        logging.info("Deleting old %s" % refchroot_metadata)
+                except OSError:
+                    pass
+
     def _count_submittable_logs(self):
         files = 0
         subdirs = ["pass", "fail", "untestable"]
@@ -530,13 +553,15 @@ class Section:
                 logging.info("busy")
                 self._error_wait_until = time.time() + 900
             else:
+                if do_processing:
+                    self._check_refchroot_metadata()
                 if self._talk_to_master(fetch=do_processing, recycle=recycle, unreserve=interrupted):
                     if do_processing:
                         if not self._slave.get_reserved():
                             self._idle_wait_until = time.time() + int(self._config["idle-sleep"])
                             if recycle:
                                 self._recycle_wait_until = self._idle_wait_until + 3600
-                        else:
+                if do_processing and self._slave.get_reserved():
                             processed = self._process()
                             if got_sighup and self._slave.get_reserved():
                                 # keep this section at the front of the round-robin runnable queue
@@ -643,15 +668,7 @@ class Section:
         self._check_tarball()
         if not os.path.exists(self._get_tarball()):
             self._error_wait_until = time.time() + 300
-        if self._config["chroot-meta-auto"]:
-            if os.path.exists(self._config["chroot-meta-auto"]):
-                try:
-                    age = time.time() - os.path.getmtime(self._config["chroot-meta-auto"])
-                    if age > 6 * 3600:
-                        os.unlink(self._config["chroot-meta-auto"])
-                        logging.info("Deleting old %s" % self._config["chroot-meta-auto"])
-                except OSError:
-                    pass
+        self._check_refchroot_metadata()
         for package_name, version in self._slave.get_reserved():
             self._throttle_if_overloaded()
             if interrupted or got_sighup:
@@ -713,10 +730,11 @@ class Section:
         if self._config["keep-sources-list"] in ["yes", "true"]:
             command.append("--keep-sources-list")
         if distupgrade and self._config["chroot-meta-auto"]:
-            if not os.path.exists(self._config["chroot-meta-auto"]):
-                command.extend(["-S", self._config["chroot-meta-auto"]])
+            refchroot_metadata = self._get_refchroot_metadata()
+            if not os.path.exists(refchroot_metadata):
+                command.extend(["-S", refchroot_metadata])
             else:
-                command.extend(["-B", self._config["chroot-meta-auto"]])
+                command.extend(["-B", refchroot_metadata])
         command.extend(["--apt", "%s=%s" % (pname, pvers)])
 
         subdir = "fail"
@@ -795,12 +813,13 @@ class Section:
                 output.write(" *** PIUPARTS OUTPUT INCOMPLETE ***\n")
             elif distupgrade and self._config["chroot-meta-auto"]:
                 try:
+                    refchroot_metadata = self._get_refchroot_metadata()
                     if "History of available packages does not match - reference chroot may be outdated" in f:
-                        os.unlink(self._config["chroot-meta-auto"])
-                        logging.info("Deleting outdated %s" % self._config["chroot-meta-auto"])
+                        os.unlink(refchroot_metadata)
+                        logging.info("Deleting outdated %s" % refchroot_metadata)
                     elif "Initial package selections do not match - ignoring loaded reference chroot state" in f:
-                        os.unlink(self._config["chroot-meta-auto"])
-                        logging.info("Deleting mismatching %s" % self._config["chroot-meta-auto"])
+                        os.unlink(refchroot_metadata)
+                        logging.info("Deleting mismatching %s" % refchroot_metadata)
                 except OSError:
                     pass