Makefile

@@ -28,8 +28,8 @@ DOCS_GENERATED		 = piuparts.1 piuparts.1.html piuparts_slave_run.8 piuparts_slav
 define placeholder_substitution
 	sed -r \
 	-e 's/__PIUPARTS_VERSION__/$(version)/g' \
-	-e 's%@sharedir@%$(sharedir)%g' \
 	-e 's%@libdir@%$(libdir)%g' \
+	-e 's%@sharedir@%$(sharedir)%g' \
 	$< > $@
 endef
 

TODO

@@ -124,8 +124,6 @@ More ideas for even later:
 - make it possible to call aptitude (or similar) instead of apt-get and allow to
   override the commandline arguments.
 
-- rewrite piuparts-analyze to run over all sections and cache BTS responses
-
 - "decorate" (strike-through) bug links generated by piuparts-analyze to
   indicate resolved state (take package version into account!)
 

custom-scripts/scripts-leftovers/pre_remove_preseed_cleanup

@@ -33,7 +33,8 @@ case ${PIUPARTS_OBJECTS%%=*} in
 		log_debug
 		echo "ifetch-tools ifetch-tools/purge boolean true" | debconf-set-selections
 		;;
-	mlmmj)  #668752
+	mlmmj)
+		#668752
 		log_debug
 		echo "mlmmj mlmmj/remove-on-purge boolean true" | debconf-set-selections
 		;;
@@ -41,6 +42,12 @@ case ${PIUPARTS_OBJECTS%%=*} in
 		log_debug
 		echo "pluxml pluxml/system/purgedata boolean true" | debconf-set-selections
 		;;
+	sympa)
+		#673972
+		log_debug
+		echo "sympa sympa/remove_spool boolean true" | debconf-set-selections
+		;;
+	####################################################################
 	ironic-common)
 		dbconfig_enable_purge ironic-common
 		;;

custom-scripts/scripts/is_testable_uninstallable

@@ -108,6 +108,7 @@ case ${PIUPARTS_OBJECTS%%=*}@${PIUPARTS_DISTRIBUTION} in
 	fossology-scheduler@squeeze|\
 	fossology-web@squeeze|\
 	fossology-web-single@squeeze)
+		# removed after squeeze
 		skip_distupgrade "#595593: fails to install"
 		;;
 	gclcvs@squeeze)

custom-scripts/scripts/post_chroot_unpack_allow_unauthenticated

+#!/bin/sh
+
+if [ "$PIUPARTS_DISTRIBUTION" = "lenny" ]; then
+	echo "Creating /etc/apt/apt.conf.d/unauthenticated-lenny ..."
+	# apt/lenny does not like comments ...
+	tee /etc/apt/apt.conf.d/unauthenticated-lenny <<EOF
+APT::Get::AllowUnauthenticated "true";
+Acquire::Check-Valid-Until "false";
+EOF
+fi
+
+if [ "$PIUPARTS_DISTRIBUTION" = "squeeze" ]; then
+	echo "Creating /etc/apt/apt.conf.d/unauthenticated-squeeze ..."
+	tee /etc/apt/apt.conf.d/unauthenticated-squeeze <<EOF
+# The squeeze signing key has expired.
+APT::Get::AllowUnauthenticated "true";
+# The Release file is not getting updated.
+Acquire::Check-Valid-Until "false";
+EOF
+fi

custom-scripts/scripts/pre_distupgrade_allow_unauthenticated

+#!/bin/sh
+set -e
+
+if [ "$PIUPARTS_DISTRIBUTION_NEXT" = "squeeze" ]; then
+	rm -fv /etc/apt/apt.conf.d/unauthenticated-lenny
+fi
+
+if [ -e /etc/apt/apt.conf.d/unauthenticated-lenny ]; then
+	echo "FAIL: /etc/apt/apt.conf.d/unauthenticated-lenny exists"
+	exit 1
+fi
+
+if [ "$PIUPARTS_DISTRIBUTION_NEXT" = "wheezy" ]; then
+	rm -fv /etc/apt/apt.conf.d/unauthenticated-squeeze
+fi
+
+if [ -e /etc/apt/apt.conf.d/unauthenticated-squeeze ]; then
+	echo "FAIL: /etc/apt/apt.conf.d/unauthenticated-squeeze exists"
+	exit 1
+fi
+
+if [ "$PIUPARTS_DISTRIBUTION_NEXT" = "squeeze" ]; then
+	echo "Creating /etc/apt/apt.conf.d/unauthenticated-squeeze ..."
+	# apt/lenny does not like comments ...
+	tee /etc/apt/apt.conf.d/unauthenticated-squeeze <<EOF
+APT::Get::AllowUnauthenticated "true";
+Acquire::Check-Valid-Until "false";
+EOF
+fi

custom-scripts/scripts/pre_install_exceptions

@@ -8,7 +8,12 @@ log_debug() {
 case ${PIUPARTS_OBJECTS%%=*} in
 	file-rc)
 		case "$PIUPARTS_DISTRIBUTION" in
-			lenny|squeeze*|stretch*|sid)
+			lenny)
+				# force installation and removal of essential package sysv-rc
+				log_debug
+				echo 'Yes, do as I say!' | apt-get -y --force-yes install file-rc
+				;;
+			squeeze*|stretch*|sid)
 				# force installation and removal of essential package sysv-rc
 				log_debug
 				yes 'Yes, do as I say!' | apt-get -y --force-yes install file-rc
@@ -36,10 +41,6 @@ case ${PIUPARTS_OBJECTS%%=*} in
 		# switch init to upstart before installation
 		apt-get install upstart
 		;;
-	sympa)
-		# force removal of spool directories
-		echo "sympa sympa/remove_spool boolean true" | debconf-set-selections
-		;;
 	systemd-sysv)
 		case "$PIUPARTS_DISTRIBUTION" in
 			wheezy*)
@@ -75,12 +76,6 @@ case ${PIUPARTS_OBJECTS%%=*} in
 			log_debug
 			echo "auctex auctex/doauto select Foreground" | debconf-set-selections
 			;;
-	samhain)
-			# work around #749602
-			log_debug
-			mkdir -p /var/state/samhain/
-			touch /var/state/samhain/samhain_file
-			;;
 	bugzilla3)
 			# checksetup.pl goes into infinite loop asking for them
 			log_debug
@@ -88,6 +83,12 @@ case ${PIUPARTS_OBJECTS%%=*} in
 			echo "bugzilla3 bugzilla3/bugzilla_admin_real_name string Bz" | debconf-set-selections
 			echo "bugzilla3 bugzilla3/bugzilla_admin_pwd password bzbzbzbz" | debconf-set-selections
 			;;
+	samhain)
+			# work around #749602
+			log_debug
+			mkdir -p /var/state/samhain/
+			touch /var/state/samhain/samhain_file
+			;;
 esac
 
 if [ "$PIUPARTS_DISTRIBUTION" = "wheezy" ]; then

custom-scripts/scripts/pre_remove_40_find_missing_md5sums

@@ -31,7 +31,7 @@ do
 
 	f2=/var/run/f2.$$
 	>$f2
-	dpkg -L "$pkg" | sed 's,\\,\\\\,' | sort | \
+	dpkg -L "$pkg" | sed 's,\\,\\\\,g' | sort | \
 	while read f ; do
 		if [ "$f" != "${f#package diverts others to: }" ]; then
 			: # skip diversion information

custom-scripts/scripts/pre_remove_exceptions

@@ -6,11 +6,21 @@ log_debug() {
 }
 
 case ${PIUPARTS_OBJECTS%%=*} in
+	apt-listbugs)
+		# cleanup from post_install_exceptions
+		log_debug
+		rm -f /usr/sbin/apt-listbugs
+		dpkg-divert --remove --rename /usr/sbin/apt-listbugs
+		;;
 	file-rc)
+		# removal won't work if sysv-rc isn't reinstalled
 		case "$PIUPARTS_DISTRIBUTION" in
-			lenny|squeeze*)
+			lenny)
+				log_debug
+				echo 'Yes, do as I say!' | apt-get -y --force-yes install sysv-rc
+				;;
+			squeeze*)
 				log_debug
-				# removal won't work if sysv-rc isn't reinstalled
 				yes 'Yes, do as I say!' | apt-get -y --force-yes install sysv-rc
 				;;
 		esac
@@ -22,22 +32,20 @@ case ${PIUPARTS_OBJECTS%%=*} in
 			rm -fv /etc/isdn/*
 		fi
 		;;
-	apt-listbugs)
-		# cleanup from post_install_exceptions
-		log_debug
-		rm -f /usr/sbin/apt-listbugs
-		dpkg-divert --remove --rename /usr/sbin/apt-listbugs
-		;;
 esac
 
 case "$PIUPARTS_DISTRIBUTION" in
 
-	stretch*|buster*|sid)
+	lenny*|squeeze*|wheezy*|jessie*)
+		# --allow-remove-essential was added in apt 1.1 (stretch)
+		;;
+	*)
 		case ${PIUPARTS_OBJECTS%%=*} in
 			init|\
 			education-thin-client|\
 			ltsp-client|\
-			ltsp-client-core)
+			ltsp-client-core|\
+			ltsp-client-core-dbgsym)
 				log_debug
 				# requires removal of essential packages
 				if [ ! -f /etc/apt/apt.conf.d/piuparts-allow-remove-essential ]

debian/changelog

 piuparts (0.93) UNRELEASED; urgency=medium
 
-  * continue development...
+  [ Andreas Beckmann ]
+  * piuparts.conf.anbe: Add some more example sections.
+  * piupartslib/packagesdb.py:
+    - Add methods Package.source(), Package.source_version().
+    - Remove method PackagesDB.get_source().
+  * piuparts-report.py:
+    - Postpone deletion of old logs after the html pages possibly referencing
+      them have been updated, avoiding error 404 while processing a section.
+    - Generate missing/outdated .kpr files.
+    - Simplify known problem report generation.
+    - Mark logs of packages built from an outdated source version as [cruft].
+  * scripts/{post_chroot_unpack,pre_distupgrade}_allow_unauthenticated: New
+    scripts to handle installation from archived releases with expired signing
+    keys (lenny, squeeze). Drop the --no-check-valid-until from *.conf since
+    that is active for all steps in a distupgrade test while the new scripts
+    only enable it for the distros that actually need it.
+  * Overhaul some exceptions.
 
  -- Holger Levsen <holger@debian.org>  Sat, 22 Sep 2018 15:32:55 +0200
 

debian/copyright

@@ -4,7 +4,7 @@ Upstream-Contact: Holger Levsen <holger@debian.org>
 Source: https://salsa.debian.org/debian/piuparts.git
 Copyright: 2005-2008 Lars Wirzenius
            2008-2018 Holger Levsen
-           © 2011-2017 Andreas Beckmann
+           © 2011-2018 Andreas Beckmann
 Comment: Original author is also Lars Wirzenius <liw@iki.fi>
  Upstream is bunch of lunatics who don't make release tarballs publicly
  available. The Debian .orig.tar.gz is what would be released, though.
@@ -12,7 +12,7 @@ Comment: Original author is also Lars Wirzenius <liw@iki.fi>
 Files: *
 Copyright: 2005-2008 Lars Wirzenius <liw@iki.fi>
            2008-2018 Holger Levsen <holger@debian.org>
-           © 2011-2017 Andreas Beckmann <anbe@debian.org>
+           © 2011-2018 Andreas Beckmann <anbe@debian.org>
 License: GPL-2+
  This program is free software; you can redistribute it
  and/or modify it under the terms of the GNU General Public

instances/piuparts.conf-template.pejacevic

@@ -84,8 +84,6 @@ flags-start-squeeze =
 	-i /etc/shells
 # see #604807:
 	--skip-logrotatefiles-test
-# squeeze has been archived
-	--no-check-valid-until
 
 # common flags for tests ending in squeeze
 flags-end-squeeze =
@@ -106,8 +104,6 @@ flags-start-lenny =
 	-i /etc/shells
 # see #604807:
 	--skip-logrotatefiles-test
-# lenny has been archived
-	--no-check-valid-until
 
 
 [global]

instances/piuparts.conf.anbe

@@ -45,7 +45,6 @@ flags-end-wheezy =
 	--scriptsdir /etc/piuparts/scripts-wheezy
 
 flags-start-squeeze =
-	--no-check-valid-until
 	-i /etc/shells
 flags-end-squeeze =
 	--warn-on-debsums-errors
@@ -54,7 +53,6 @@ flags-end-squeeze =
 
 flags-start-lenny =
 	--dpkg-noforce-unsafe-io
-	--no-check-valid-until
 	-i /etc/shells
 flags-end-lenny =
 	--skip-cronfiles-test
@@ -149,8 +147,13 @@ sections =
 	stable2testing2sid/main
 	stretch2proposed/main
 	stretch2buster/main
+	stretch3buster/main
+	stretch2buster_i386/main
+	stretch3buster_i386/main
+	stretch2buster-rcmd/main
 	stretch3buster-rcmd/main
 	stretch2buster-rcmd_i386/main
+	stretch3buster-rcmd_i386/main
 	stretch2buster-iri/main
 	stretch2bpo2buster/main
 	jessie2proposed/main
@@ -224,8 +227,13 @@ sections =
 	stable2testing2sid/contrib
 	stretch2proposed/contrib
 	stretch2buster/contrib
+	stretch3buster/contrib
+	stretch2buster_i386/contrib
+	stretch3buster_i386/contrib
+	stretch2buster-rcmd/contrib
 	stretch3buster-rcmd/contrib
 	stretch2buster-rcmd_i386/contrib
+	stretch3buster-rcmd_i386/contrib
 	stretch2buster-iri/contrib
 	stretch2bpo2buster/contrib
 	jessie2proposed/contrib
@@ -299,8 +307,13 @@ sections =
 	stable2testing2sid/non-free
 	stretch2proposed/non-free
 	stretch2buster/non-free
+	stretch3buster/non-free
+	stretch2buster_i386/non-free
+	stretch3buster_i386/non-free
+	stretch2buster-rcmd/non-free
 	stretch3buster-rcmd/non-free
 	stretch2buster-rcmd_i386/non-free
+	stretch3buster-rcmd_i386/non-free
 	stretch2buster-iri/non-free
 	stretch2bpo2buster/non-free
 	jessie2proposed/non-free
@@ -3805,6 +3818,187 @@ distro = stretch
 upgrade-test-distros = stretch buster
 
 
+[stretch3buster/main]
+precedence = 80
+description = + Testing two-stage upgrades.
+piuparts-flags =
+	--upgrade-before-dist-upgrade
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+arch = amd64
+area = main
+components = main
+distro = stretch
+upgrade-test-distros = stretch buster
+exclude-known-problems = packages_have_been_kept_back_error.conf packages_have_been_kept_back_issue.conf
+
+[stretch3buster/contrib]
+precedence = 95
+description = + Testing two-stage upgrades.
+piuparts-flags =
+	--upgrade-before-dist-upgrade
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+depends-sections = stretch3buster/main stretch3buster/non-free
+arch = amd64
+area = contrib
+distro = stretch
+upgrade-test-distros = stretch buster
+exclude-known-problems = packages_have_been_kept_back_error.conf packages_have_been_kept_back_issue.conf
+
+[stretch3buster/non-free]
+precedence = 95
+description = + Testing two-stage upgrades.
+piuparts-flags =
+	--upgrade-before-dist-upgrade
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+depends-sections = stretch3buster/main stretch3buster/contrib
+arch = amd64
+area = non-free
+distro = stretch
+upgrade-test-distros = stretch buster
+exclude-known-problems = packages_have_been_kept_back_error.conf packages_have_been_kept_back_issue.conf
+
+
+[stretch2buster_i386/main]
+precedence = 80
+piuparts-flags =
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+arch = i386
+area = main
+components = main
+distro = stretch
+upgrade-test-distros = stretch buster
+
+[stretch2buster_i386/contrib]
+precedence = 95
+piuparts-flags =
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+depends-sections = stretch2buster_i386/main stretch2buster_i386/non-free
+arch = i386
+area = contrib
+distro = stretch
+upgrade-test-distros = stretch buster
+
+[stretch2buster_i386/non-free]
+precedence = 95
+piuparts-flags =
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+depends-sections = stretch2buster_i386/main stretch2buster_i386/contrib
+arch = i386
+area = non-free
+distro = stretch
+upgrade-test-distros = stretch buster
+
+
+[stretch3buster_i386/main]
+precedence = 80
+description = + Testing two-stage upgrades.
+piuparts-flags =
+	--upgrade-before-dist-upgrade
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+arch = i386
+area = main
+components = main
+distro = stretch
+upgrade-test-distros = stretch buster
+exclude-known-problems = packages_have_been_kept_back_error.conf packages_have_been_kept_back_issue.conf
+
+[stretch3buster_i386/contrib]
+precedence = 95
+description = + Testing two-stage upgrades.
+piuparts-flags =
+	--upgrade-before-dist-upgrade
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+depends-sections = stretch3buster_i386/main stretch3buster_i386/non-free
+arch = i386
+area = contrib
+distro = stretch
+upgrade-test-distros = stretch buster
+exclude-known-problems = packages_have_been_kept_back_error.conf packages_have_been_kept_back_issue.conf
+
+[stretch3buster_i386/non-free]
+precedence = 95
+description = + Testing two-stage upgrades.
+piuparts-flags =
+	--upgrade-before-dist-upgrade
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+depends-sections = stretch3buster_i386/main stretch3buster_i386/contrib
+arch = i386
+area = non-free
+distro = stretch
+upgrade-test-distros = stretch buster
+exclude-known-problems = packages_have_been_kept_back_error.conf packages_have_been_kept_back_issue.conf
+
+
+[stretch2buster-rcmd/main]
+precedence = 80
+piuparts-flags =
+	--install-recommends
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+arch = amd64
+area = main
+components = main
+distro = stretch
+upgrade-test-distros = stretch buster
+
+[stretch2buster-rcmd/contrib]
+precedence = 95
+piuparts-flags =
+	--install-recommends
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+depends-sections = stretch2buster-rcmd/main stretch2buster-rcmd/non-free
+arch = amd64
+area = contrib
+distro = stretch
+upgrade-test-distros = stretch buster
+
+[stretch2buster-rcmd/non-free]
+precedence = 95
+piuparts-flags =
+	--install-recommends
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+depends-sections = stretch2buster-rcmd/main stretch2buster-rcmd/contrib
+arch = amd64
+area = non-free
+distro = stretch
+upgrade-test-distros = stretch buster
+
+
 [stretch3buster-rcmd/main]
 precedence = 80
 description = + Testing two-stage upgrades with --install-recommends.
@@ -3903,6 +4097,61 @@ distro = stretch
 upgrade-test-distros = stretch buster
 
 
+[stretch3buster-rcmd_i386/main]
+precedence = 80
+description = + Testing two-stage upgrades with --install-recommends.
+piuparts-flags =
+	--install-recommends
+	--fake-essential-packages ca-certificates
+	--upgrade-before-dist-upgrade
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+arch = i386
+area = main
+components = main
+distro = stretch
+upgrade-test-distros = stretch buster
+exclude-known-problems = packages_have_been_kept_back_error.conf packages_have_been_kept_back_issue.conf
+
+[stretch3buster-rcmd_i386/contrib]
+precedence = 95
+description = + Testing two-stage upgrades with --install-recommends.
+piuparts-flags =
+	--install-recommends
+	--fake-essential-packages ca-certificates
+	--upgrade-before-dist-upgrade
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+depends-sections = stretch3buster-rcmd_i386/main stretch3buster-rcmd_i386/non-free
+arch = i386
+area = contrib
+distro = stretch
+upgrade-test-distros = stretch buster
+exclude-known-problems = packages_have_been_kept_back_error.conf packages_have_been_kept_back_issue.conf
+
+[stretch3buster-rcmd_i386/non-free]
+precedence = 95
+description = + Testing two-stage upgrades with --install-recommends.
+piuparts-flags =
+	--install-recommends
+	--fake-essential-packages ca-certificates
+	--upgrade-before-dist-upgrade
+	%(flags-default)s
+	%(flags-start-stretch)s
+	%(flags-end-buster)s
+	--scriptsdir /etc/piuparts/scripts-debug-problemresolver
+depends-sections = stretch3buster-rcmd_i386/main stretch3buster-rcmd_i386/contrib
+arch = i386
+area = non-free
+distro = stretch
+upgrade-test-distros = stretch buster
+exclude-known-problems = packages_have_been_kept_back_error.conf packages_have_been_kept_back_issue.conf
+
+
 [stretch2buster-iri/main]
 precedence = 90
 description = Debian %(area)s: package installation and removal in stretch, then upgrade to buster, installation, removal, and purge test.

piuparts-report.py

@@ -3,7 +3,7 @@
 #
 # Copyright 2005 Lars Wirzenius (liw@iki.fi)
 # Copyright 2009-2018 Holger Levsen (holger@layer-acht.org)
-# Copyright © 2011-2017 Andreas Beckmann (anbe@debian.org)
+# Copyright © 2011-2018 Andreas Beckmann (anbe@debian.org)
 # Copyright 2013 David Steele (dsteele@gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify it
@@ -516,6 +516,7 @@ class Config(piupartslib.conf.Config):
                 "max-reserved": 1,
                 "doc-root": "/",
                 "known-problem-directory": "@sharedir@/piuparts/known_problems",
+                "exclude-known-problems": None,
                 "json-sections": "default",
                 "precedence": 1,
                 "web-host": "piuparts.debian.org",
@@ -978,7 +979,7 @@ class Section:
         return link[:-2]
 
     def link_to_source_summary(self, package_name):
-        source_name = self._binary_db.get_source(package_name)
+        source_name = self._binary_db.get_package(package_name).source()
         link = "<a href=\"%s/%s/source/%s\">%s</a>" % (
             self._doc_root,
                 self._config.section,
@@ -1057,7 +1058,7 @@ class Section:
         return total
 
     def create_maintainer_summaries(self, maintainers, source_data):
-        logging.debug("Writing %d maintainer summaries in %s" % (len(maintainers), self._config.section))
+        logging.debug("Writing %d maintainer summaries" % len(maintainers))
         maintainer_dir = os.path.join(self._output_directory, "maintainer")
         if not os.path.exists(maintainer_dir):
             os.mkdir(maintainer_dir)
@@ -1155,6 +1156,8 @@ class Section:
                 state_style = "labelcell"
 
             binary_version = self._binary_db.get_test_versions(binary)
+            if self._binary_db.get_package(binary).source_version() != source_version:
+                binary_version += " [cruft]"
             binary_arch = self._binary_db.get_control_header(binary, "Architecture")
             archsuffix = ""
             if binary_arch == "all":
@@ -1247,7 +1250,7 @@ class Section:
 
     def create_package_summaries(self, logs_by_dir):
         src_names = sorted(self._source_db.get_all_package_names())
-        logging.debug("Writing %d source summaries in %s" % (len(src_names), self._config.section))
+        logging.debug("Writing %d source summaries" % len(src_names))
 
         maintainers = {}
         source_binary_rows = {}
@@ -1320,6 +1323,16 @@ class Section:
 
         return stats_html
 
+    def prepare_known_problem_reports(self, failures):
+        self._problem_reports = {}
+        for problem in self._problem_list:
+            tpl_text = dwke_update_tpl(self._config.section, problem,
+                                       failures.filtered(problem.name),
+                                       PKG_ERROR_TPL, PROB_TPL,
+                                       self._binary_db, self._source_db)
+            if len(tpl_text):
+                self._problem_reports[problem.name[:-5] + TPL_EXT] = tpl_text
+
     def create_and_link_to_analysises(self, state):
         link = "<ul>\n"
         for template, linktarget in linktarget_by_template:
@@ -1328,11 +1341,9 @@ class Section:
                     or (state == "successfully-tested" and template[-9:] == "issue.tpl"):
                 substats = ""
 
-                tpl = os.path.join(self._output_directory, template)
-                if (os.path.exists(tpl)):
+                if template in self._problem_reports:
                     logging.debug("Writing analysis page for %s" % template)
-                    rows = read_file(tpl)
-                    os.unlink(tpl)
+                    rows = self._problem_reports[template]
 
                     self._write_template_html(
                             os.path.join(self._output_directory, template[:-len(".tpl")] + ".html"),
@@ -1456,6 +1467,8 @@ class Section:
                                          self.link_to_source_summary(package["Package"]))
                 if package["Architecture"] == "all":
                     vlist += ":all"
+                if package.source_version() != self._source_db.get_version(package.source()):
+                    vlist += " [cruft]"
                 if with_counts:
                     vlist += " (%d, %d)" % (self._binary_db.rrdep_count(package["Package"]),
                                             self._binary_db.block_count(package["Package"]))
@@ -1546,9 +1559,6 @@ class Section:
         logging.debug("Copying log files")
         copy_logs(logs_by_dir, self._output_directory)
 
-        logging.debug("Removing old log files")
-        remove_old_logs(logs_by_dir, self._output_directory)
-
         logging.debug("Writing per-dir HTML pages")
         self.print_by_dir(self._output_directory, logs_by_dir)
 
@@ -1556,13 +1566,14 @@ class Section:
 
         self.create_package_summaries(logs_by_dir)
 
-        logging.debug("generate known problem *.tpl")
-        dwke_process_section(self._config.section, '.', self._output_directory, self._problem_list, self._binary_db)
+        logging.debug("Preparing known problem reports")
+        failures = dwke_get_failures(self._binary_db, self._problem_list)
+        self.prepare_known_problem_reports(failures)
 
         logging.debug("Writing section index page")
         self.write_section_index_page(dirs, total_packages)
 
-        logging.debug("Writing stats pages for %s" % self._config.section)
+        logging.debug("Writing state pages")
         self.write_state_pages()
 
         logging.debug("Wrote %d out of %d html files, refreshed %d out of %d unmodified files" % ( \
@@ -1571,6 +1582,9 @@ class Section:
         with open(md5cachefile, "w") as f:
             pickle.dump(self._md5cache['new'], f)
 
+        logging.debug("Removing old log files")
+        remove_old_logs(logs_by_dir, self._output_directory)
+
 
     def generate_summary(self, web_host):
         summary_path = os.path.join(self._output_directory, "summary.json")
@@ -1597,7 +1611,7 @@ class Section:
                     block_cnt = 0
                     if flag == 'F':
                         block_cnt = self._binary_db.block_count(pkgname)
-                    srcpkg = self._binary_db.get_source(pkgname)
+                    srcpkg = self._binary_db.get_package(pkgname).source()
                     url = source_summary_url(
                               web_host, self._doc_root,
                               self._config.section, srcpkg)
@@ -1620,6 +1634,9 @@ class Section:
         if not os.path.exists(self._output_directory):
             os.makedirs(self._output_directory)
         self._problem_list = problem_list
+        if self._config['exclude-known-problems']:
+            excluded = self._config['exclude-known-problems'].split()
+            self._problem_list = [p for p in problem_list if p.name not in excluded]
 
         oldcwd = os.getcwd()
         os.chdir(self._section_directory)
@@ -1676,17 +1693,19 @@ def populate_tpl(tmpl, vals):
     return tmpl
 
 
-def update_tpl(basedir, section, problem, failures, logdict, ftpl, ptpl, pkgsdb):
+def dwke_update_tpl(section, problem, failures, ftpl, ptpl, pkgsdb, srcdb):
 
     pkg_text = ""
     bugged_section = False
     for failure in failures:
         bin_pkg = get_pkg(failure.pkgspec)
-        src_pkg = pkgsdb.get_source(bin_pkg)
+        src_pkg = pkgsdb.get_package(bin_pkg).source()
         rdep_cnt = pkgsdb.rrdep_count(bin_pkg)
         bin_arch = ""
         if pkgsdb.get_control_header(bin_pkg, "Architecture") == "all":
             bin_arch = " [all]"
+        if pkgsdb.get_package(bin_pkg).source_version() != srcdb.get_version(src_pkg):
+            bin_arch += " [cruft]"
 
         if bugged_section is False and failure.where != 'fail':
             bugged_section = True
@@ -1715,31 +1734,23 @@ def update_tpl(basedir, section, problem, failures, logdict, ftpl, ptpl, pkgsdb)
     return ""
 
 
-def update_html(section, html_dir, logdict, problem_list, failures, pkgsdb):
-    for problem in problem_list:
-        tpl_text = update_tpl(html_dir, section, problem,
-                   failures.filtered(problem.name),
-                   logdict,
-                   PKG_ERROR_TPL, PROB_TPL, pkgsdb)
-        if len(tpl_text):
-            with open(os.path.join(html_dir, problem.name[:-5] + TPL_EXT), 'w') as pf:
-                pf.write(tpl_text)
-
-
-def dwke_process_section(section, sectiondir, htmldir, problem_list, pkgsdb):
-    workdirs = [os.path.join(sectiondir, x) for x in KPR_DIRS]
-
-    logdict = get_file_dict(workdirs, LOG_EXT)
+def dwke_get_failures(pkgsdb, problem_list):
+    logdict = get_file_dict(KPR_DIRS, LOG_EXT)
+    kprdict = get_file_dict(KPR_DIRS, KPR_EXT)
+    del_cnt = clean_cache_files(logdict, kprdict)
+    kprdict = get_file_dict(KPR_DIRS, KPR_EXT)
+    add_cnt = make_kprs(logdict, kprdict, problem_list)
 
     failures = FailureManager(logdict)
     failures.sort_by_bugged_and_rdeps(pkgsdb)
 
+    logging.info("parsed logfiles: %d removed, %d added" % (del_cnt, add_cnt))
     for prob in problem_list:
         pcount = len(failures.filtered(prob.name))
         if pcount:
             logging.info("%7d %s" % (pcount, prob.name))
 
-    update_html(section, htmldir, logdict, problem_list, failures, pkgsdb)
+    return failures
 
 # END detect_well_known_errors
 

piupartslib/packagesdb.py

 # -*- coding: utf-8 -*-
 
 # Copyright 2005 Lars Wirzenius (liw@iki.fi)
-# Copyright © 2011-2017 Andreas Beckmann (anbe@debian.org)
+# Copyright © 2011-2018 Andreas Beckmann (anbe@debian.org)
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -75,6 +75,27 @@ class Package(UserDict.UserDict):
     def version(self):
         return self["Version"]
 
+    def source(self):
+        # Binary packages built from the source package with the same name
+        # and version don't have a Source header.
+        if "Source" in self:
+            # If source and binary version differ (e.g. for binNMUs), the
+            # source version is given as the second element in the "Source"
+            # entry. Strip off the optional source version.
+            return self["Source"].split(" ")[0]
+        return self["Package"]
+
+    def source_version(self):
+        # Binary packages built from the source package with the same name
+        # and version don't have a Source header.
+        if "Source" in self:
+            # If source and binary version differ (e.g. for binNMUs), the
+            # source version is given as the second element in the "Source"
+            # entry. Strip off the parentheses around the source version.
+            if " " in self["Source"]:
+                return self["Source"].split(" ")[1][1:-1]
+        return self["Version"]
+
     def set_test_versions(self, tv):
         self["TestVersions"] = tv
 
@@ -685,12 +706,6 @@ class PackagesDB:
             return self._packages[name].test_versions()
         return None
 
-    def get_source(self, name):
-        self._find_all_packages()
-        if name in self._packages:
-            return self.get_control_header(name, "Source")
-        return None
-
     def get_providers(self, name, recurse=True):
         self._find_all_packages()
         providers = []
@@ -711,21 +726,7 @@ class PackagesDB:
 
     def get_control_header(self, package_name, header):
         self._find_all_packages()
-        if header == "Source":
-            # binary packages build from the source package with the same name
-            # don't have a Source header, so let's try:
-            try:
-                _source = self._packages[package_name][header]
-                # for binNMU the Source header in Packages files holds the version
-                # too, so we need to chop it of:
-                if " " in _source:
-                    source, version = _source.split(" ")
-                else:
-                    source = _source
-            except:
-                source = self._packages[package_name]["Package"]
-            return source
-        elif header == "Uploaders":
+        if header == "Uploaders":
             # not all (source) packages have an Uploaders header
             uploaders = ""
             try: