config/debian/lintian.tags

@@ -117,7 +117,6 @@ lintian:
     - package-not-lowercase
     - package-uses-local-diversion
     - section-is-dh_make-template
-    - source-field-does-not-match-pkg-name
     - symlink-has-too-many-up-segments
     - too-many-architectures
     - uploader-address-is-on-localhost

dak/admin.py

@@ -745,6 +745,10 @@ ALLOWED_SUITE_CONFIGS = {
     'mail_whitelist': str,
     'notautomatic': utils.parse_boolean_from_user,
     'origin': str,
+    'overridecodename': str,
+    'overrideorigin': str,
+    'overrideprocess': utils.parse_boolean_from_user,
+    'overridesuite': str,
     'priority': int,
     'signingkeys': SUITE_CONFIG_READ_ONLY,
     'suite_name': SUITE_CONFIG_READ_ONLY,

daklib/cruft.py

@@ -26,7 +26,7 @@ from __future__ import print_function
 from daklib.dbconn import *
 
 from sqlalchemy import func
-from sqlalchemy.orm import object_session
+from sqlalchemy.orm import object_session, aliased
 
 
 def newer_version(lowersuite_name, highersuite_name, session, include_equal=False):
@@ -40,19 +40,94 @@ def newer_version(lowersuite_name, highersuite_name, session, include_equal=Fals
     lowersuite = get_suite(lowersuite_name, session)
     highersuite = get_suite(highersuite_name, session)
 
-    query = session.query(DBSource.source, func.max(DBSource.version)). \
-        with_parent(highersuite).group_by(DBSource.source)
+    def get_suite_sources(suite):
+        q1 = session.query(DBSource.source,
+                           func.max(DBSource.version).label('version')). \
+            with_parent(suite). \
+            group_by(DBSource.source). \
+            subquery()
+
+        return aliased(q1)
+
+    def get_suite_binaries(suite):
+        q1 = session.query(DBBinary.package,
+                           DBSource.source,
+                           func.max(DBSource.version).label('version'),
+                           Architecture.arch_string,
+                           func.max(DBBinary.version).label('binversion')). \
+            join(DBSource). \
+            with_parent(suite). \
+            join(Architecture). \
+            group_by(
+                DBBinary.package,
+                DBSource.source,
+                Architecture.arch_string,
+            ). \
+            subquery()
+
+        return aliased(q1)
+
+    highq = get_suite_sources(highersuite)
+    lowq = get_suite_sources(lowersuite)
+
+    query = session.query(
+            highq.c.source,
+            highq.c.version.label('higherversion'),
+            lowq.c.version.label('lowerversion')
+            ). \
+        join(lowq, highq.c.source == lowq.c.source)
+
+    if include_equal:
+        query = query.filter(highq.c.version <= lowq.c.version)
+    else:
+        query = query.filter(highq.c.version < lowq.c.version)
 
     list = []
-    for (source, higherversion) in query:
-        q = session.query(func.max(DBSource.version)). \
-            filter_by(source=source)
+    # get all sources that have a higher version in lowersuite than in
+    # highersuite
+    for (source, higherversion, lowerversion) in query:
+        q1 = session.query(DBBinary.package,
+                           DBSource.source,
+                           DBSource.version,
+                           Architecture.arch_string
+            ). \
+            join(DBSource). \
+            with_parent(highersuite). \
+            join(Architecture). \
+            filter(DBSource.source == source). \
+            subquery()
+        q2 = session.query(q1.c.arch_string).group_by(q1.c.arch_string)
+        # all architectures for which source has binaries in highersuite
+        archs_high = set(x[0] for x in q2.all())
+
+        highq = get_suite_binaries(highersuite)
+        lowq = get_suite_binaries(lowersuite)
+
+        query = session.query(highq.c.arch_string). \
+            join(lowq, highq.c.source == lowq.c.source). \
+            filter(highq.c.arch_string == lowq.c.arch_string). \
+            filter(highq.c.package == lowq.c.package). \
+            filter(highq.c.source == source)
+
         if include_equal:
-            q = q.filter(DBSource.version >= higherversion)
+            query = query. \
+                filter(highq.c.binversion <= lowq.c.binversion). \
+                filter(highq.c.version <= lowq.c.version)
         else:
-            q = q.filter(DBSource.version > higherversion)
-        lowerversion = q.with_parent(lowersuite).group_by(DBSource.source).scalar()
-        if lowerversion is not None:
+            query = query. \
+                filter(highq.c.binversion < lowq.c.binversion). \
+                filter(highq.c.version < lowq.c.version)
+
+        query = query.group_by(highq.c.arch_string)
+
+        # all architectures for which source has a newer binary in lowersuite
+        archs_newer = set(x[0] for x in query.all())
+
+        # if has at least one binary in lowersuite which is newer than the one
+        # in highersuite on each architecture for which source has binaries in
+        # highersuite, we know that the builds for all relevant architecture
+        # are done, so we can remove the old source with it's binaries
+        if (archs_newer >= archs_high):
             list.append((source, higherversion, lowerversion))
 
     list.sort()

integration-tests/common

@@ -38,3 +38,11 @@ import-fixture-signing-key() {
     ln -sf ${packages:?}/gpg/pubring.gpg ${DAKBASE:?}/keyrings/upload-keyring.gpg
     dak import-keyring -U "%s" ${DAKBASE}/keyrings/upload-keyring.gpg
 }
+
+list_all_suites() {
+    dak admin s list | sort | while read suite
+    do
+        printf "\n\n$suite\n"
+        dak control-suite -l $suite |sort
+    done
+}

integration-tests/dinstall

@@ -19,6 +19,7 @@
 declare public_archives=(ftp-master debian-debug)
 declare -r scriptdir=${DAKBASE}/scripts
 declare -r overridedir=${scriptdir}/override # debian
+echo "Dir::Override \"${overridedir}\";" >> "${DAK_CONFIG}"
 declare -r pdiff_tempdir="${DAKBASE}/_dinstall-pdiff-temp"
 declare -r upload_dir="${DAKBASE}/uploads"
 
@@ -42,7 +43,8 @@ setup_debian_like_archive() {
     dak admin suite-component add unstable main contrib non-free
     dak admin suite add-build-queue unstable buildd-unstable buildd-sid build-queues
     dak admin suite-config set unstable allowcsset=no byhash=yes \
-        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog
+        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
+        overrideprocess=True
 
     dak admin suite add-all-arches unstable-debug "" codename=sid-debug archive=debian-debug
     dak admin suite-component add unstable-debug main contrib non-free
@@ -55,7 +57,8 @@ setup_debian_like_archive() {
     dak admin suite-component add testing main contrib non-free
     dak admin suite-config set testing allowcsset=yes byhash=yes \
         changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
-        changelog=dists/testing/ChangeLog
+        changelog=dists/testing/ChangeLog \
+        overrideprocess=True overrideorigin=unstable
 
     dak admin suite add-all-arches testing-debug "" codename=sid-debug archive=debian-debug
     dak admin suite-component add testing-debug main contrib non-free
@@ -68,7 +71,8 @@ setup_debian_like_archive() {
         codename=buster-proposed-updates archive=ftp-master
     dak admin suite-component add testing-proposed-updates main contrib non-free
     dak admin suite-config set testing-proposed-updates allowcsset=yes byhash=yes \
-        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog
+        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
+        overridesuite=testing
 
     
     # experimental
@@ -77,7 +81,8 @@ setup_debian_like_archive() {
     dak admin suite add-build-queue experimental buildd-experimental buildd-rc-buggy build-queues
     dak admin suite-config set experimental allowcsset=no byhash=yes \
         changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
-        notautomatic=yes butautomaticupgrades=no
+        notautomatic=yes butautomaticupgrades=no \
+        overridesuite=unstable overridecodename=sid
 
 
     # Version constraints

integration-tests/tests/2000-dinstall

@@ -23,6 +23,8 @@ set -u
 . ${DAK_ROOT:?}/integration-tests/setup
 . ${DAK_ROOT:?}/integration-tests/dinstall
 
+echo "Dinstall::AllowSourceOnlyUploads true;" >> ${DAKBASE}/etc/dak.conf
+
 setup_debian_like_archive
 import-fixture-signing-key
 
@@ -36,6 +38,8 @@ dinstall
   upload_changes ${packages:?}/nonfree-package_0.1-1_amd64.changes
   upload_changes ${packages:?}/package_0.1-1_amd64.changes
   upload_changes ${packages:?}/main-contrib-with-debug_0.1-1_amd64.changes
+  upload_changes ${packages:?}/binnmupkg_0.1-1_amd64.changes
+  upload_changes ${packages:?}/pkgnew_0.1-1_amd64.changes
 
   process_uploads
 )
@@ -61,6 +65,8 @@ package admin
 EOF
 
 echo a | dak process-new main-contrib-with-debug_0.1-1_amd64.changes
+echo a | dak process-new binnmupkg_0.1-1_amd64.changes
+echo a | dak process-new pkgnew_0.1-1_amd64.changes
   
   dak process-new --automatic
   dak process-policy new
@@ -78,6 +84,8 @@ main-contrib-with-debug 0.2-1 source
 main-package 0.2-1 all
 package 0.1-1 all
 package 0.1-1 source
+binnmupkg 0.1-1 amd64
+binnmupkg 0.1-1 source
 EOF
 )"
 
@@ -90,11 +98,52 @@ dinstall
 ls -l ${DAKBASE}/ftp-master/pool/main/p/package/package_*.dsc
 echo "Published successfully"
 
-echo "contents of testing:"
-dak control-suite -l testing
-echo ""
-echo "contents of unstable:"
-dak control-suite -l unstable
-echo ""
+(
+  upload_changes ${packages:?}/pkgnew_0.1-2~exp_amd64.changes
+  process_uploads
+
+  echo a | dak process-new pkgnew_0.1-2~exp_amd64.changes
+
+  dak process-new --automatic
+  dak process-policy new
+
+  # pretend the upload to experimental happened (more than) 14 days ago to
+  # trigger NVIU issue below
+  echo "update override set created = created - interval '14 days' where package like 'pkg%';" | psql
+
+  upload_changes ${packages:?}/pkgnew_0.1-2_source.changes
+  process_uploads
+
+  upload_changes ${packages:?}/pkgnew_0.1-2_all.changes
+  process_uploads
+
+  upload_changes ${packages:?}/binnmupkg_0.1-2_source.changes
+  process_uploads
+
+  upload_changes ${packages:?}/binnmupkg_0.1-2_amd64.changes
+  process_uploads
+
+  upload_changes ${packages:?}/binnmupkg_0.1-1+b1_amd64.changes
+  process_uploads
+
+  list_all_suites
+
+  # trigger obsolete override issue: needs 2 dinstalls:
+  # NVIU is done after obsolete overrides, so we needs a second dinstall to
+  # remove the overrides for the packages removed by NVIU
+  dinstall
+  dinstall
+
+  list_all_suites
+
+  upload_changes ${packages:?}/pkgnew_0.1-2_amd64.changes
+  process_uploads
+
+)
+
+list_all_suites
 
 dak ls main-contrib-with-debug
+
+# check that this package has actually been published
+ls -l ${DAKBASE}/ftp-master/pool/main/p/pkgnew/pkg-any3_0.1-2_amd64.deb

tests/fixtures/packages/Makefile

 export GNUPGHOME = $(CURDIR)/gpg
 SHELL = /bin/bash
 
-TAR = linux_42.0 nonfree-package_0.1 package_0.1 package_0.2 package-built-using_0.1 main-contrib-with-debug_0.1 main-contrib-with-debug_0.2
-PACKAGES = $(TAR)
+# List of packages to build
+# for each of these, the source must be available in a directory
+# $PACKAGE_$VERSION
+#
+# if the directory doesn't exist, but there is a directory
+# overlays/$PACKAGE and a directory overlays/$PACKAGE_$VERSION,
+# the contents of both directories will be copied sequentially to
+# $PACKAGE_$VERSION, allowing identical data for similar packages (eg.
+# multiple versions of the same packages) to be stored only once
+#
+# the suffix (_F, _A, _B or _S) specifies the type of build
 
-all: packages
+PACKAGES = \
+linux_42.0_F \
+nonfree-package_0.1_F \
+package_0.1_F \
+package_0.2_F \
+package-built-using_0.1_F \
+main-contrib-with-debug_0.1_F \
+main-contrib-with-debug_0.2_F \
+binnmu_0.1_F \
+binnmu_0.1+b1_B \
+binnmu_0.2_S \
+binnmu_0.2_B \
+pkgnew_0.1_F \
+pkgnew_0.2~exp_F \
+pkgnew_0.2_S \
+pkgnew_0.2_A \
+pkgnew_0.2_B \
 
-tarballs: stamp-tarballs
-stamp-tarballs:
-	set -e; for t in $(TAR); do \
+ALL = $(PACKAGES)
+
+stamp-all: $(ALL)
+	touch $@
+
+%.orig.tar.gz:
+	t=$*; \
+	dir=$${t/_/-}; \
+	base=$${t%_*}; \
+	if [ ! -d $$dir ] ; then\
+		cp -rv overlays/$$base/ ./$$dir/; \
+		cp -Trv overlays/$$dir/ ./$$dir; \
+	fi; \
 	if [ ! -f $$t.orig.tar.gz ]; then \
 		tar -czf $$t.orig.tar.gz --exclude=debian $${t/_/-}; \
-	  fi; \
-	done
-	touch $@
+	fi
 
-packages: stamp-packages
-stamp-packages: stamp-tarballs
-	set -e; for p in $(PACKAGES); do \
-	  (cd $${p/_/-}; dpkg-buildpackage); \
-	done
-	touch $@
+%_F: %.orig.tar.gz
+	p=$*; (cd $${p/_/-}; dpkg-buildpackage -F)
+
+%_A: %.orig.tar.gz
+	p=$*; (cd $${p/_/-}; dpkg-buildpackage -A)
+
+%_B: %.orig.tar.gz
+	p=$*; (cd $${p/_/-}; dpkg-buildpackage -B)
+
+%_S: %.orig.tar.gz
+	p=$*; (cd $${p/_/-}; dpkg-buildpackage -S)
 
 clean:
-	set -e; for p in $(PACKAGES); do \
-	  make -C $${p/_/-} -f debian/rules clean; \
+	set -e; for j in $(ALL); do \
+	  p=$${j%_*}; \
+	  dir=$${p/_/-}; \
+	  if [ -d overlays/$$dir ]; then rm -rf ./$$dir; \
+	    else make -C $$dir -f debian/rules clean; fi; \
 	done
 	rm -f *.tar.gz *.tar.xz *.buildinfo *.dsc *.changes *.diff.gz *.deb
 	rm -f gpg/*~
 	rm -f stamp-*
+
+.PRECIOUS: %.orig.tar.gz

tests/fixtures/packages/main-contrib-with-debug-0.2/Makefile

-
-default: test
-
-clean:
-	rm -f test

tests/fixtures/packages/main-contrib-with-debug-0.2/debian/contrib-with-debug.install

-test usr/bin

tests/fixtures/packages/main-contrib-with-debug-0.2/debian/main-package.install

-some-file usr/share/main-package

tests/fixtures/packages/main-contrib-with-debug-0.2/some-file

-some text

tests/fixtures/packages/main-contrib-with-debug-0.2/test.c

-
-
-int main(void) {
-	return(0);
-}
-

tests/fixtures/packages/overlays/binnmu-0.1+b1/debian/changelog

+binnmupkg (0.1-1+b1) testing-proposed-updates; urgency=medium, binary-only=yes
+
+  * Binary-only non-maintainer upload for amd64; no source changes.
+
+ -- A Maintainer <maint@example.com>  Sat, 13 Jul 2019 16:29:19 +0000
+
+binnmupkg (0.1-1) unstable; urgency=low
+
+  * Initial release.
+
+ -- A Maintainer <maint@example.com>  Fri, 08 Jun 2012 18:10:01 +0200

tests/fixtures/packages/overlays/binnmu-0.1/debian/changelog

+binnmupkg (0.1-1) unstable; urgency=low
+
+  * Initial release.
+
+ -- A Maintainer <maint@example.com>  Fri, 08 Jun 2012 18:10:01 +0200

tests/fixtures/packages/overlays/binnmu-0.2/debian/changelog

+binnmupkg (0.1-2) unstable; urgency=medium
+
+  * Some changes
+
+ -- A Maintainer <maint@example.com>  Sat, 13 Jul 2019 10:29:19 +0000
+
+binnmupkg (0.1-1) unstable; urgency=low
+
+  * Initial release.
+
+ -- A Maintainer <maint@example.com>  Fri, 08 Jun 2012 18:10:01 +0200

tests/fixtures/packages/package-0.2/debian/control → tests/fixtures/packages/overlays/binnmu/debian/control

-Source: package
+Source: binnmupkg
 Section: misc
 Priority: extra
 Maintainer: A Maintainer <maint@example.com>
@@ -6,8 +6,8 @@ Uploaders: An Uploader <uploader@example.com>
 Build-Depends: debhelper (>= 9)
 Rules-Requires-Root: no
 
-Package: package
-Architecture: all
+Package: binnmupkg
+Architecture: any
 Depends: ${misc:Depends}
 Description: a package
  a package