custom-scripts/scripts-multi-distro-upgrade/pre_remove_exceptions_multi-distro-upgrade

@@ -18,6 +18,13 @@ case ${PIUPARTS_OBJECTS} in
 		log_debug
 		touch /var/log/chef/solr.log
 		;;
+	chef-server-api=None)
+		# removed after wheezy
+		# logrotate config references /var/log/chef/server.log /var/log/chef/merb*log
+		log_debug
+		touch /var/log/chef/server.log
+		touch /var/log/chef/merb-dummy.log
+		;;
 	dancer-ircd=None)
 		# removed after wheezy
 		# logrotate config references /var/log/dancer-ircd/ircd.log

custom-scripts/scripts/is_testable_uninstallable

@@ -34,6 +34,19 @@ skip_distupgrade()
 	fi
 }
 
+# use where a later version might fix the bug and enable dist-upgrade testing
+skip_distupgrade_from()
+{
+	from_ver=$1
+	shift
+	pkg=${PIUPARTS_OBJECTS%%=*}
+	cand_ver=$(apt-cache policy "$pkg" | awk '{if ($1 == "Candidate:") {print $2}}')
+
+	if [ "$from_ver" = "$cand_ver" ]; then
+		skip_distupgrade "$@"
+	fi
+}
+
 
 # * For uninstallable *broken* packages we usually run the normal
 #   installation+purge tests (which will fail and get the package marked
@@ -133,6 +146,27 @@ fi
 if [ "${PIUPARTS_DISTRIBUTION}" = "wheezy" ]; then
 
 	case ${PIUPARTS_OBJECTS%%=*} in
+		firefox-esr-dev|\
+		iceowl-l10n-bn-bd|\
+		iceowl-l10n-pa-in|\
+		iceowl-l10n-ta-lk|\
+		iceweasel-l10n-ak|\
+		iceweasel-l10n-be|\
+		iceweasel-l10n-csb|\
+		iceweasel-l10n-ku|\
+		iceweasel-l10n-lg|\
+		iceweasel-l10n-nso|\
+		iceweasel-l10n-ta-lk|\
+		iceweasel-l10n-zu|\
+		lightning-l10n-bn-bd|\
+		lightning-l10n-pa-in|\
+		lightning-l10n-ta-lk|\
+		thunderbird-dev|\
+		thunderbird-l10n-bn-bd|\
+		thunderbird-l10n-pa-in|\
+		thunderbird-l10n-ta-lk)
+			skip_distupgrade "#928226: uninstallable cruft package"
+			;;
 		firmware-b43-lpphy-installer)
 			skip_distupgrade "#623703: firmware-b43-lpphy-installer: download location no longer available"
 			;;
@@ -146,6 +180,9 @@ if [ "${PIUPARTS_DISTRIBUTION}" = "wheezy" ]; then
 		nautilus-dropbox)
 			skip_distupgrade "#770402: nautilus-dropbox: downloads awfully slow, hangs during installation"
 			;;
+		xul-ext-compactheader)
+			skip_distupgrade "#837387: xul-ext-compactheader: uninstallable in wheezy with icedove (>= 1:45)"
+			;;
 	esac
 
 fi
@@ -153,6 +190,27 @@ fi
 if [ "${PIUPARTS_DISTRIBUTION}" = "jessie" ]; then
 
 	case ${PIUPARTS_OBJECTS%%=*} in
+		enigmail)
+			skip_distupgrade_from "2:1.9.9-1~deb8u1" "#909000: enigmail 2.0 needed after thunderbird 60 upload"
+			;;
+		firefox-esr-dev|\
+		icedove-dev|\
+		icedove-l10n-bn-bd|\
+		icedove-l10n-pa-in|\
+		icedove-l10n-ta-lk|\
+		iceowl-l10n-bn-bd|\
+		iceowl-l10n-pa-in|\
+		iceowl-l10n-ta-lk|\
+		iceweasel-dev|\
+		lightning-l10n-bn-bd|\
+		lightning-l10n-pa-in|\
+		lightning-l10n-ta-lk|\
+		thunderbird-dev|\
+		thunderbird-l10n-bn-bd|\
+		thunderbird-l10n-pa-in|\
+		thunderbird-l10n-ta-lk)
+			skip_distupgrade "#928226: uninstallable cruft package"
+			;;
 		fusionforge-plugin-mediawiki)
 			skip_distupgrade "#929139: fusionforge-plugin-mediawiki: uninstallable cruft package"
 			;;
@@ -160,10 +218,16 @@ if [ "${PIUPARTS_DISTRIBUTION}" = "jessie" ]; then
 			skip_distupgrade "#885441: geoip-database-contrib: download location no longer available"
 			;;
 		steam)
-			skip_distupgrade "#772598: cannot preseed licence acceptance"
+			skip_distupgrade_from "1.0.0.49-1" "#772598: steam: cannot preseed licence acceptance"
 			;;
 		susv4)
-			skip_distupgrade "#900882: download location no longer available"
+			skip_distupgrade_from "7.20161013~deb8u1" "#900882: susv4: download location no longer available"
+			;;
+		xul-ext-compactheader)
+			skip_distupgrade_from "2.0.8-1" "#837387: xul-ext-compactheader: uninstallable in jessie with icedove (>= 1:45)"
+			;;
+		xul-ext-sogo-connector)
+			skip_distupgrade_from "31.0.0-1" "#909313: xul-ext-sogo-connector: not compatible with thunderbird 60"
 			;;
 	esac
 
@@ -236,6 +300,18 @@ if [ "${PIUPARTS_DISTRIBUTION}" = "sid-debug" ]; then
 
 fi
 
+
+if [ "$(uname)" = "Linux" ]; then
+
+	case ${PIUPARTS_OBJECTS%%=*} in
+		console-setup-freebsd)
+			skip_always "non-Linux package"
+			;;
+	esac
+
+fi
+
+
 case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 	acgvision-agent@squeeze)
 		# removed after squeeze
@@ -255,11 +331,6 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 	chef-server-api@wheezy)
 		skip_distupgrade "#681573: chef-solr: fails to install if policy-rc.d forbids to start services"
 		;;
-	console-setup-freebsd@*)
-		if [ "$(uname)" = "Linux" ]; then
-			skip_always "non-Linux package"
-		fi
-		;;
 	cpushare@wheezy)
 		# removed after wheezy
 		skip_distupgrade "#689885: Ships a folder in /var/run or /var/lock"
@@ -349,14 +420,4 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 	usrmerge@*)
 		skip_always "performs piuparts-unfriendy filesystem layout changes"
 		;;
-	xul-ext-compactheader@wheezy)
-		if [ "$(apt-cache show xul-ext-compactheader | sed -n '/^Version:/s/^Version: //p')" = "2.0.5-1" ]; then
-			skip_distupgrade "#837387: uninstallable in wheezy with icedove (>= 1:45)"
-		fi
-		;;
-	xul-ext-compactheader@jessie)
-		if [ "$(apt-cache show xul-ext-compactheader | sed -n '/^Version:/s/^Version: //p')" = "2.0.8-1" ]; then
-			skip_distupgrade "#837387: uninstallable in jessie with icedove (>= 1:45)"
-		fi
-		;;
 esac

debian/changelog

@@ -7,8 +7,15 @@ piuparts (1.0.1) UNRELEASED; urgency=medium
     - Run pre_remove scripts before computing packages to be removed.
     - Fix spurious failure to remove packages with names ending with '+'.
   * piuparts.conf.anbe: Add some more example sections.
+  * piupartslib/packagesdb.py:
+    - Remove stale .kpr files after receiving new logs.
   * piuparts-slave.py:
     - Generate separate tarball names for --merged-usr chroots.
+    - Re-exec on SIGUSR1, picking up updated code and new config sections.
+  * master-bin/detect_piuparts_issues:
+    - Clean up stale temporary and empty files.
+  * master-bin/rotate_master_logs: Delete master logs older than 90 days.
+  * slave-bin/slave_reexec: New, send SIGUSR1 to all running slaves.
 
   [ Holger Levsen ]
   * generate_daily_report: improve statistics.
@@ -160,7 +167,7 @@ piuparts (0.96) unstable; urgency=medium
 
   [ Andreas Beckmann ]
   * bug-templates: Raise severity of "fails to purge" errors to serious.
-   (see #918379)
+    (see #918379)
   * distros.conf: Add jessie-lts as a full distro.
   * piuparts.conf:
     - Use jessie-lts as upgrade target and rename section

master-bin/detect_network_issues.in

@@ -62,7 +62,7 @@ add_pattern "E: Version '.*' for '.*' was not found"
 add_pattern 'E: Method file has died unexpectedly!'
 add_pattern "E: Sub-process rred received a segmentation fault."
 add_pattern "ERROR: Command failed \(status=-7\):.*'apt-get', 'update'"
-add_pattern 'Package .* .* not found in .*, .* is available'
+add_pattern 'Package .* .* not found in .*(, .* is available)?'
 add_pattern "AppStream system cache was updated, but problems were found: Metadata files have errors"
 
 FILE=$(mktemp)

master-bin/detect_piuparts_issues.in

 #!/bin/sh
 
 # Copyright © 2009-2018 Holger Levsen (holger@layer-acht.org)
-# Copyright © 2011-2017 Andreas Beckmann (anbe@debian.org)
+# Copyright © 2011-2019 Andreas Beckmann (anbe@debian.org)
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -96,6 +96,10 @@ for SECTION in $SECTIONS ; do
 	for subdir in fail bugged affected ; do
 		test -d $MASTER/$SECTION/$subdir || continue
 		grep -r -l -E --include '*.log' "$PATTERN" $MASTER/$SECTION/$subdir >> $FILE 2>/dev/null
+	done
+	for subdir in fail bugged affected untestable pass ; do
+		test -d $MASTER/$SECTION/$subdir || continue
+		find $MASTER/$SECTION/$subdir -name 'tmp??????' -mtime +1 -delete 2>/dev/null
 		find $MASTER/$SECTION/$subdir -name '*.log' -size 0 >> $FILE
 	done
 done

master-bin/rotate_master_logs.in

 #!/bin/sh
 #
-# Copyright © 2017 Andreas Beckmann (anbe@debian.org)
+# Copyright © 2017-2019 Andreas Beckmann (anbe@debian.org)
 # Copyright © 2018 Holger Levsen (holger@layer-acht.org)
 #
 # This program is free software; you can redistribute it and/or modify it
@@ -48,6 +48,12 @@ for SECTION in $SECTIONS ; do
 			xz -v9 $log >>$OUTPUT 2>&1
 		fi
 	done
+
+	# delete master logs older than 90 days, but keep at least five old logs
+	oldlogs=$(ls -t1 $SECTION/master.*.log.* 2>/dev/null | tail -n +5)
+	if [ -n "$oldlogs" ]; then
+		find $oldlogs -mtime +90 | xargs -r rm -fv >>$OUTPUT 2>&1
+	fi
 done
 
 LOGDATEDIR="$HTDOCS/logs/$(date -u '+%Y/%m/%d' -d '7 days ago')"

piuparts-slave.py

@@ -29,7 +29,7 @@ import sys
 import stat
 import time
 import logging
-from signal import alarm, signal, SIGALRM, SIGINT, SIGKILL, SIGHUP
+from signal import alarm, signal, SIGALRM, SIGINT, SIGKILL, SIGHUP, SIGUSR1
 import subprocess
 import fcntl
 import random
@@ -50,6 +50,7 @@ MAX_WAIT_TEST_RUN = 90 * 60
 interrupted = False
 old_sigint_handler = None
 got_sighup = False
+got_sigusr1 = False
 
 
 def setup_logging(log_level, log_file_name):
@@ -129,6 +130,14 @@ def sighup_handler(signum, frame):
     print('SIGHUP: Will flush finished logs.')
 
 
+def sigusr1_handler(signum, frame):
+    global got_sigusr1
+    global got_sighup
+    got_sigusr1 = True
+    got_sighup = True
+    print('SIGUSR1: Will restart.')
+
+
 class MasterIsBusy(Exception):
 
     def __init__(self):
@@ -975,6 +984,7 @@ def create_file(filename, contents):
 def main():
     setup_logging(logging.INFO, None)
     signal(SIGHUP, sighup_handler)
+    signal(SIGUSR1, sigusr1_handler)
 
     # For supporting multiple architectures and suites, we take command-line
     # argument(s) referring to section(s) in the configuration file.
@@ -1015,6 +1025,10 @@ def main():
         for section in sorted(sections, key=lambda section: (section.precedence(), section.sleep_until())):
             test_count += section.run(do_processing=(test_count == 0))
 
+        if got_sigusr1:
+            logging.info("Restarting...")
+            os.execv(__file__, sys.argv)
+
         if test_count == 0 and got_sighup:
             # clear SIGHUP state after flushing all sections
             got_sighup = False

piupartslib/packagesdb.py

@@ -236,6 +236,9 @@ class LogDB:
     def _log_name(self, package, version):
         return "%s_%s.log" % (package, version)
 
+    def _kpr_name(self, package, version):
+        return "%s_%s.kpr" % (package, version)
+
     def log_exists2(self, package, version, subdirs):
         log_name = self._log_name(package, version)
         for subdir in subdirs:
@@ -274,6 +277,11 @@ class LogDB:
             self.remove_file(full_name)
         self._evict(full_name)
 
+    def remove_kpr(self, subdir, package, version):
+        full_name = os.path.join(subdir, self._kpr_name(package, version))
+        if os.path.exists(full_name):
+            self.remove_file(full_name)
+
     def stat(self, subdir, package, version):
         full_name = os.path.join(subdir, self._log_name(package, version))
         return os.stat(full_name)
@@ -896,32 +904,24 @@ class PackagesDB:
                     self._logdb.create(self._recycle, package, version, "")
         self._logdb.remove(self._reserved, package, version)
 
-    def pass_package(self, package, version, log):
+    def _process_log(self, package, version, log, subdir, result):
         self._check_for_acceptability_as_filename(package)
         self._check_for_acceptability_as_filename(version)
         self._remove_logs_if_reserved(package, version)
-        if self._logdb.create(self._ok, package, version, log):
-            self._record_submission("pass", package, version)
+        if self._logdb.create(subdir, package, version, log):
+            self._record_submission(result, package, version)
+            self._logdb.remove_kpr(subdir, package, version)
         else:
-            raise LogfileExists(self._ok, package, version)
+            raise LogfileExists(subdir, package, version)
+
+    def pass_package(self, package, version, log):
+        self._process_log(package, version, log, self._ok, "pass")
 
     def fail_package(self, package, version, log):
-        self._check_for_acceptability_as_filename(package)
-        self._check_for_acceptability_as_filename(version)
-        self._remove_logs_if_reserved(package, version)
-        if self._logdb.create(self._fail, package, version, log):
-            self._record_submission("fail", package, version)
-        else:
-            raise LogfileExists(self._fail, package, version)
+        self._process_log(package, version, log, self._fail, "fail")
 
     def make_package_untestable(self, package, version, log):
-        self._check_for_acceptability_as_filename(package)
-        self._check_for_acceptability_as_filename(version)
-        self._remove_logs_if_reserved(package, version)
-        if self._logdb.create(self._evil, package, version, log):
-            self._record_submission("untestable", package, version)
-        else:
-            raise LogfileExists(self._evil, package, version)
+        self._process_log(package, version, log, self._evil, "untestable")
 
     def _get_rdep_dict(self):
         """Return dict of one-level reverse dependencies by package"""

slave-bin/slave_reexec.in

+#!/bin/sh
+set -e
+
+# Copyright © 2019 Andreas Beckmann (anbe@debian.org)
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>
+
+#
+# Send all running piuparts-slaves SIGUSR1 to reexec themselves,
+# picking up new code and sections.
+#
+
+for pid in $(pgrep --full '/usr/bin/python @sharedir@/piuparts/piuparts-slave')
+do
+	echo "Sending SIGUSR1 to ${pid}..."
+	kill -USR1 $pid
+	sleep 5
+done
+echo "Done."