custom-scripts/scripts/post_chroot_unpack_allow_unauthenticated

@@ -9,7 +9,10 @@ Acquire::Check-Valid-Until "false";
 EOF
 fi
 
-if [ "$PIUPARTS_DISTRIBUTION" = "squeeze" ]; then
+if [ "$PIUPARTS_DISTRIBUTION" = "squeeze" ] || \
+   [ "$PIUPARTS_DISTRIBUTION" = "squeeze-lts" ] || \
+   [ "$PIUPARTS_DISTRIBUTION" = "squeeze-backports" ] || \
+   [ "$PIUPARTS_DISTRIBUTION" = "squeeze-backports-sloppy" ]; then
 	echo "Creating /etc/apt/apt.conf.d/unauthenticated-squeeze ..."
 	tee /etc/apt/apt.conf.d/unauthenticated-squeeze <<EOF
 # The squeeze signing key has expired.

custom-scripts/scripts/post_distupgrade_exceptions

@@ -84,6 +84,12 @@ fi
 
 if [ "$PIUPARTS_DISTRIBUTION" = "buster" ]; then
 
+	# libc-bin only upgrades pristine /etc/nsswitch.conf
+	if grep -q '^passwd:.*compat' /etc/nsswitch.conf ; then
+		echo "Switching from compat to files in /etc/nsswitch.conf"
+		sed -r -i '/^(passwd|group|shadow):/ s/compat/files/' /etc/nsswitch.conf
+	fi
+
 	# dpkg does not properly clean up directories getting empty and no longer shipped
 	for dir in /etc/dbus-1/system.d /etc/dbus-1
 	do

custom-scripts/scripts/pre_distupgrade_allow_unauthenticated

@@ -23,6 +23,8 @@ Acquire::Check-Valid-Until "false";
 EOF
 elif [ "$PIUPARTS_DISTRIBUTION_NEXT" = "squeeze-backports" ]; then
 	:
+elif [ "$PIUPARTS_DISTRIBUTION_NEXT" = "squeeze-lts" ]; then
+	:
 elif [ -e /etc/apt/apt.conf.d/unauthenticated-squeeze ]; then
 	echo "FAIL: /etc/apt/apt.conf.d/unauthenticated-squeeze exists"
 	exit 1

custom-scripts/scripts/pre_install_exceptions

@@ -13,7 +13,7 @@ case ${PIUPARTS_OBJECTS%%=*} in
 				log_debug
 				echo 'Yes, do as I say!' | apt-get -y --force-yes install file-rc
 				;;
-			squeeze*|stretch*|sid)
+			squeeze*|stretch*)
 				# force installation and removal of essential package sysv-rc
 				log_debug
 				yes 'Yes, do as I say!' | apt-get -y --force-yes install file-rc

debian/changelog

+piuparts (0.94) unstable; urgency=medium
+
+  [ Holger Levsen ]
+  * master-bin/detect_well_known_errors.py: suppress output if nothing new is
+    found in a section, or section is busy or has been recently processed.
+  * master-bin/generate_daily_report: include number of failures in daily mail.
+  * slave-bin/detect_slave_problems: increase amount of time (from 30 to 60m)
+    a slave has to be inactive before complaining.
+
+  [ Andreas Beckmann ]
+  * piuparts.py:
+    - Logrotate files can only be placed directly in /etc/logrotate.d/.
+  * scripts/post_distupgrade_exceptions:
+    - Handle /etc/nsswitch.conf stretch -> buster upgrade if modifications
+      from libnss-myhostname and friends are present.
+
+  [ Mathieu Parent ]
+  * piuparts.py: Use mknod instead of touch to create missing /dev/ptmx
+    mountpoint, working around findutils bug #912180 (Closes: #911334)
+
+  [ Ivo De Decker ]
+  * piuparts-reports.py: Don't add results from outdated binaries to
+    summary.json.
+
+ -- Holger Levsen <holger@debian.org>  Thu, 08 Nov 2018 12:09:57 +0100
+
 piuparts (0.93) unstable; urgency=medium
 
   [ Andreas Beckmann ]

master-bin/detect_well_known_errors.py

@@ -113,7 +113,7 @@ def detect_well_known_errors(sections, config, problem_list, recheck, recheck_fa
         (section, next_try) = todo.popleft()
         now = time.time()
         if (now < next_try):
-            logging.info("Sleeping while sections are busy")
+            # sleeping, section has been tried recently
             time.sleep(max(30, next_try - now) + 30)
         try:
             (del_cnt, add_cnt) = \
@@ -123,11 +123,11 @@ def detect_well_known_errors(sections, config, problem_list, recheck, recheck_fa
             total_add += add_cnt
             current_time=time.strftime("%a %b %2d %H:%M:%S %Z %Y", time.localtime())
             if del_cnt == 0 and add_cnt == 0:
-                logging.info("%s - %s: nothing new" % (current_time, section))
+                pass # nothing new
             else:
                 logging.info("%s - %s: parsed logfiles: %d removed, %d added" % (current_time, section, del_cnt, add_cnt))
         except Busy:
-            logging.info("Section is busy")
+            # section is busy
             todo.append((section, time.time() + 300))
         except MissingSection:
             pass

master-bin/generate_daily_report.in

@@ -67,7 +67,7 @@ FAILURESTAMP=$HTDOCS/last-failure-report.stamp
 test -f $FAILURESTAMP || touch -d @0 $FAILURESTAMP  # start at the epoch
 touch $FAILURESTAMP.new  # for the next report
 
-echo "New failures:" >> $DAILYREPORT
+DAILYTMP=$(mktemp)
 for SECTION in $SECTIONS ; do
 	test -d $MASTER/$SECTION || continue
 	for DIRECTORY in fail bugged affected untestable ; do
@@ -75,7 +75,11 @@ for SECTION in $SECTIONS ; do
 	done
 	find $MASTER/$SECTION/fail $MASTER/$SECTION/bugged $MASTER/$SECTION/affected $MASTER/$SECTION/untestable \
 		-type f -name '*.log' -newer $FAILURESTAMP -exec ls -1 {} + 2>/dev/null
-done | sed s#^$MASTER#$URLBASE# >> $DAILYREPORT
+done | sed s#^$MASTER#$URLBASE# >> $DAILYTMP
+
+echo "$(cat $DAILYTMP | wc -l) new failures:" >> $DAILYREPORT
+cat $DAILYTMP >> $DAILYREPORT
+rm $DAILYTMP
 
 echo "" >> $DAILYREPORT
 date >> $DAILYREPORT

piuparts-report.py

@@ -1611,6 +1611,8 @@ class Section:
             summary = pkgsummary.new_summary()
 
             for reporting_section in reporting_sections:
+                source_done = {}
+                source_todo = {}
                 for binpkg in self._binary_db.get_all_packages():
                     pkgname = binpkg["Package"]
                     state = self._binary_db.get_package_state(pkgname)
@@ -1619,13 +1621,30 @@ class Section:
                     if flag == 'F':
                         block_cnt = self._binary_db.block_count(pkgname)
                     srcpkg = self._binary_db.get_package(pkgname).source()
+                    srcver = self._binary_db.get_package(pkgname).source_version()
+                    latest_srcver = self._source_db.get_test_versions(srcpkg)
                     url = source_summary_url(
                               web_host, self._doc_root,
                               self._config.section, srcpkg)
 
+                    if (srcver != latest_srcver):
+                        # if the binary package is cruft, we don't add it to
+                        # the report, but we keep track to make sure the
+                        # source package is added
+                        source_todo[srcpkg] = url
+                    else:
                         pkgsummary.add_summary(
                                        summary, reporting_section, srcpkg,
                                        flag, block_cnt, url)
+                        source_done[srcpkg] = 1
+                for srcpkg in source_todo:
+                    # source packages that only have outdated binaries are
+                    # waiting for the build, so we add them as 'W'
+                    # (waiting-to-be-tested)
+                    if srcpkg not in source_done:
+                        pkgsummary.add_summary(
+                                       summary, reporting_section, srcpkg,
+                                       'W', 0, source_todo[srcpkg])
 
             pkgsummary.write_summary(summary, summary_path)
 

piuparts.py

@@ -1565,17 +1565,15 @@ class Chroot:
         self.run_scripts("post_remove")
 
         if not settings.skip_cronfiles_test:
-            cronfiles, cronfiles_list = self.check_if_cronfiles(packages)
-
-        if not settings.skip_cronfiles_test and cronfiles:
-            self.check_output_cronfiles(cronfiles_list)
+            cronfiles = self.check_if_cronfiles(packages)
+            if cronfiles:
+                self.check_output_cronfiles(cronfiles)
 
         if not settings.skip_logrotatefiles_test:
-            logrotatefiles, logrotatefiles_list = self.check_if_logrotatefiles(packages)
-
-        if not settings.skip_logrotatefiles_test and logrotatefiles:
+            logrotatefiles = self.check_if_logrotatefiles(packages)
+            if logrotatefiles:
                 installed = self.install_logrotate()
-            self.check_output_logrotatefiles(logrotatefiles_list)
+                self.check_output_logrotatefiles(logrotatefiles)
                 self.purge_packages(installed)
 
         # Then purge all packages being depended on.
@@ -1757,8 +1755,7 @@ class Chroot:
         dev_ptmx_rel_path = self.relative("dev/ptmx")
         if not os.path.islink(dev_ptmx_rel_path):
             if not os.path.exists(dev_ptmx_rel_path):
-                with open(dev_ptmx_rel_path, 'w'):
-                    pass
+                os.mknod(dev_ptmx_rel_path, 0666 | stat.S_IFCHR, os.makedev(5, 2))
             self.mount(self.relative("dev/pts/ptmx"), "/dev/ptmx", opts="bind", no_mkdir=True)
         p = subprocess.Popen(["tty"], stdout=subprocess.PIPE)
         stdout, _ = p.communicate()
@@ -1829,9 +1826,9 @@ class Chroot:
         """Check if the packages have cron files under /etc/cron.d and in case positive,
         it returns the list of files. """
 
+        # FIXME! Does not work for M-A: same packages
         vdir = self.relative("var/lib/dpkg/info")
         vlist = []
-        has_cronfiles = False
         for p in packages:
             basename = p + ".list"
 
@@ -1846,12 +1843,10 @@ class Chroot:
                         mode = st[stat.ST_MODE]
                         # XXX /etc/cron.d/ files are NOT executables
                         if (mode & stat.S_IEXEC):
-                            if not has_cronfiles:
-                                has_cronfiles = True
                             vlist.append(pathname)
                             logging.info("Package " + p + " contains cron file: " + pathname)
 
-        return has_cronfiles, vlist
+        return vlist
 
     def check_output_cronfiles(self, list):
         """Check if a given list of cronfiles has any output. Executes
@@ -1874,9 +1869,9 @@ class Chroot:
         """Check if the packages have logrotate files under /etc/logrotate.d and in case positive,
         it returns the list of files. """
 
+        # FIXME! Does not work for M-A: same packages
         vdir = self.relative("var/lib/dpkg/info")
         vlist = []
-        has_logrotatefiles = False
         for p in packages:
             basename = p + ".list"
 
@@ -1885,14 +1880,12 @@ class Chroot:
 
             for line in readlines_file(os.path.join(vdir, basename)):
                 pathname = line.strip()
-                if pathname.startswith("/etc/logrotate.d/"):
+                if os.path.dirname(pathname) == "/etc/logrotate.d":
                     if os.path.isfile(self.relative(pathname.strip("/"))):
-                        if not has_logrotatefiles:
-                            has_logrotatefiles = True
                         vlist.append(pathname)
                         logging.info("Package " + p + " contains logrotate file: " + pathname)
 
-        return has_logrotatefiles, vlist
+        return vlist
 
     def install_logrotate(self):
         """Install logrotate for check_output_logrotatefiles, and return the

slave-bin/detect_slave_problems.in

@@ -49,18 +49,15 @@ STATEFILE=$SLAVEROOT/slave-problems
 # clear the statefile daily and whine again
 test $(file_age $STATEFILE) -lt 86000 || rm -f $STATEFILE
 
-# Only complain if screenlog is older than $IDLE_SPEEP + 1 minute (the slave
-# likes to sleep that long) and the problem is new or was not reported within
-# the previous 24 hours.
-if [ $(file_age $SCREENLOG) -le $(($IDLE_SLEEP + 60)) ]; then
+# Only complain if screenlog is older than $IDLE_SLEEP + 30 minutes
+# and the problem is new or was not reported within the previous 24 hours.
+if [ $(file_age $SCREENLOG) -le $(($IDLE_SLEEP + 1800)) ]; then
 	rm -f $STATEFILE
 elif [ ! -f $STATEFILE ]; then
 	{
-		echo "Either a test is running for a very long time (but no test"
-		echo "should run longer than an hour), piuparts-slave hangs or is"
-		echo "not running at all or wasn't started with"
-		echo "~piupartss/bin/slave_run - please investigate and take"
-		echo "appropriate measures!"
+		echo "Either a test is running for a very long time (more than $(( ($IDLE_SLEEP + 1800)/60 )) minutes),"
+		echo "or piuparts-slave hangs or is not running at all or wasn't started with"
+		echo "~piupartss/bin/slave_run - please investigate and take appropriate measures!"
 		echo
 		tail $SCREENLOG
 	} | mail -s "problem with piuparts-slave detected" piupartss