config/debian/dinstall.functions

@@ -212,7 +212,11 @@ function copyoverrides() {
     (
         shopt -s nullglob
         rm -f ${indices}/override.*.gz
-        for ofile in ${overridedir}/override.{jessie,stretch,buster,bullseye,sid}.{,extra.}{main,contrib,non-free}*; do
+        TESTING=$(dak admin suite-config get-value testing codename)
+        STABLE=$(dak admin suite-config get-value stable codename)
+        OLDSTABLE=$(dak admin suite-config get-value oldstable codename)
+        OLDOLDSTABLE=$(dak admin suite-config get-value oldoldstable codename)
+        for ofile in ${overridedir}/override.{$OLDOLDSTABLE,$OLDSTABLE,$STABLE,$TESTING,sid}.{,extra.}{main,contrib,non-free}*; do
             bname=${ofile##*/}
             gzip -9cv --rsyncable  --no-name ${ofile} > ${indices}/${bname}.gz
             chmod g+w ${indices}/${bname}.gz
@@ -347,13 +351,18 @@ function mkfilesindices() {
             sort -u | poolfirst > ../arch-$a.files
     done
 
+    TESTING=$(dak admin suite-config get-value testing codename)
+    STABLE=$(dak admin suite-config get-value stable codename)
+    OLDSTABLE=$(dak admin suite-config get-value oldstable codename)
+    OLDOLDSTABLE=$(dak admin suite-config get-value oldoldstable codename)
+
     (cd $base/ftp/
-     for dist in sid jessie stretch buster bullseye; do
+     for dist in sid $OLDOLDSTABLE $OLDSTABLE $STABLE $TESTING; do
 	 find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip -9 > $base/ftp/indices/files/components/translation-$dist.list.gz
      done
     )
 
-    (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-jessie.list.gz ; zcat translation-stretch.list.gz ; zcat translation-buster.list.gz ; zcat translation-bullseye.list.gz) |
+    (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-$OLDOLDSTABLE.list.gz ; zcat translation-$OLDSTABLE.list.gz ; zcat translation-$STABLE.list.gz ; zcat translation-$TESTING.list.gz) |
         sort -u | poolfirst > ../typical.files
 
     rm -f $ARCHLIST
@@ -533,8 +542,7 @@ function i18n2() {
     mkdir -p ${scriptdir}/i18n/${STAMP}
     cd ${scriptdir}/i18n/${STAMP}
     for suite in stable testing unstable; do
-        codename=$(dak admin s show ${suite}|grep '^Codename')
-        codename=${codename##* }
+        codename=$(dak admin suite-config get-value ${suite} codename)
         echo "Codename is ${codename}"
         dak control-suite -l ${suite} >${codename}
     done

config/debian/dinstall.variables

@@ -23,8 +23,7 @@ DINSTALLSTATE="${webdir}/dinstall.status"
 extimportdists=""
 if [ "${functionname}" = ftp-master.debian.org ]; then
   for suite in testing unstable; do
-    codename=$(dak admin s show ${suite}|grep '^Codename')
-    codename=${codename##* }
+    codename=$(dak admin suite-config get-value ${suite} codename)
     extimportdists="${extimportdists} ${codename}"
   done
 fi

config/debian/lintian.tags

@@ -117,7 +117,6 @@ lintian:
     - package-not-lowercase
     - package-uses-local-diversion
     - section-is-dh_make-template
-    - source-field-does-not-match-pkg-name
     - symlink-has-too-many-up-segments
     - too-many-architectures
     - uploader-address-is-on-localhost

dak/admin.py

@@ -745,6 +745,10 @@ ALLOWED_SUITE_CONFIGS = {
     'mail_whitelist': str,
     'notautomatic': utils.parse_boolean_from_user,
     'origin': str,
+    'overridecodename': str,
+    'overrideorigin': str,
+    'overrideprocess': utils.parse_boolean_from_user,
+    'overridesuite': str,
     'priority': int,
     'signingkeys': SUITE_CONFIG_READ_ONLY,
     'suite_name': SUITE_CONFIG_READ_ONLY,

dak/process_policy.py

@@ -262,6 +262,9 @@ def comment_accept(upload, srcqueue, comments, transaction):
             if os.path.exists(src) and not os.path.exists(dst):
                 fs.copy(src, dst, mode=mode)
 
+    utils.process_buildinfos(upload.policy_queue.path, chg.buildinfo_files,
+                             fs, Logger)
+
     if upload.source is not None and not Options['No-Action']:
         urgency = upload.changes.urgency
         # As per policy 5.6.17, the urgency can be followed by a space and a

dak/process_upload.py

@@ -267,7 +267,8 @@ def accept(directory, upload):
     print("ACCEPT")
 
     upload.install()
-    process_buildinfos(upload)
+    utils.process_buildinfos(upload.directory, upload.changes.buildinfo_files,
+                             upload.transaction.fs, Logger)
 
     accepted_to_real_suite = any(suite.policy_queue is None for suite in upload.final_suites)
     sourceful_upload = 'source' in upload.changes.architectures
@@ -509,25 +510,6 @@ def process_changes(changes_filenames):
     for directory, c in changes:
         process_it(directory, c, keyring_files)
 
-
-def process_buildinfos(upload):
-    cnf = Config()
-
-    if 'Dir::BuildinfoArchive' not in cnf:
-        return
-
-    target_dir = os.path.join(
-        cnf['Dir::BuildinfoArchive'],
-        datetime.datetime.now().strftime('%Y/%m/%d'),
-    )
-
-    for f in upload.changes.buildinfo_files:
-        src = os.path.join(upload.directory, f.filename)
-        dst = utils.find_next_free(os.path.join(target_dir, f.filename))
-
-        Logger.log(["Archiving", f.filename])
-        upload.transaction.fs.copy(src, dst, mode=0o644)
-
 ###############################################################################
 
 

daklib/cruft.py

@@ -26,7 +26,7 @@ from __future__ import print_function
 from daklib.dbconn import *
 
 from sqlalchemy import func
-from sqlalchemy.orm import object_session
+from sqlalchemy.orm import object_session, aliased
 
 
 def newer_version(lowersuite_name, highersuite_name, session, include_equal=False):
@@ -40,19 +40,94 @@ def newer_version(lowersuite_name, highersuite_name, session, include_equal=Fals
     lowersuite = get_suite(lowersuite_name, session)
     highersuite = get_suite(highersuite_name, session)
 
-    query = session.query(DBSource.source, func.max(DBSource.version)). \
-        with_parent(highersuite).group_by(DBSource.source)
+    def get_suite_sources(suite):
+        q1 = session.query(DBSource.source,
+                           func.max(DBSource.version).label('version')). \
+            with_parent(suite). \
+            group_by(DBSource.source). \
+            subquery()
+
+        return aliased(q1)
+
+    def get_suite_binaries(suite):
+        q1 = session.query(DBBinary.package,
+                           DBSource.source,
+                           func.max(DBSource.version).label('version'),
+                           Architecture.arch_string,
+                           func.max(DBBinary.version).label('binversion')). \
+            join(DBSource). \
+            with_parent(suite). \
+            join(Architecture). \
+            group_by(
+                DBBinary.package,
+                DBSource.source,
+                Architecture.arch_string,
+            ). \
+            subquery()
+
+        return aliased(q1)
+
+    highq = get_suite_sources(highersuite)
+    lowq = get_suite_sources(lowersuite)
+
+    query = session.query(
+            highq.c.source,
+            highq.c.version.label('higherversion'),
+            lowq.c.version.label('lowerversion')
+            ). \
+        join(lowq, highq.c.source == lowq.c.source)
+
+    if include_equal:
+        query = query.filter(highq.c.version <= lowq.c.version)
+    else:
+        query = query.filter(highq.c.version < lowq.c.version)
 
     list = []
-    for (source, higherversion) in query:
-        q = session.query(func.max(DBSource.version)). \
-            filter_by(source=source)
+    # get all sources that have a higher version in lowersuite than in
+    # highersuite
+    for (source, higherversion, lowerversion) in query:
+        q1 = session.query(DBBinary.package,
+                           DBSource.source,
+                           DBSource.version,
+                           Architecture.arch_string
+            ). \
+            join(DBSource). \
+            with_parent(highersuite). \
+            join(Architecture). \
+            filter(DBSource.source == source). \
+            subquery()
+        q2 = session.query(q1.c.arch_string).group_by(q1.c.arch_string)
+        # all architectures for which source has binaries in highersuite
+        archs_high = set(x[0] for x in q2.all())
+
+        highq = get_suite_binaries(highersuite)
+        lowq = get_suite_binaries(lowersuite)
+
+        query = session.query(highq.c.arch_string). \
+            join(lowq, highq.c.source == lowq.c.source). \
+            filter(highq.c.arch_string == lowq.c.arch_string). \
+            filter(highq.c.package == lowq.c.package). \
+            filter(highq.c.source == source)
+
         if include_equal:
-            q = q.filter(DBSource.version >= higherversion)
+            query = query. \
+                filter(highq.c.binversion <= lowq.c.binversion). \
+                filter(highq.c.version <= lowq.c.version)
         else:
-            q = q.filter(DBSource.version > higherversion)
-        lowerversion = q.with_parent(lowersuite).group_by(DBSource.source).scalar()
-        if lowerversion is not None:
+            query = query. \
+                filter(highq.c.binversion < lowq.c.binversion). \
+                filter(highq.c.version < lowq.c.version)
+
+        query = query.group_by(highq.c.arch_string)
+
+        # all architectures for which source has a newer binary in lowersuite
+        archs_newer = set(x[0] for x in query.all())
+
+        # if has at least one binary in lowersuite which is newer than the one
+        # in highersuite on each architecture for which source has binaries in
+        # highersuite, we know that the builds for all relevant architecture
+        # are done, so we can remove the old source with it's binaries
+        if (archs_newer >= archs_high):
             list.append((source, higherversion, lowerversion))
 
     list.sort()

daklib/rm.py

@@ -164,7 +164,7 @@ class ReverseDependencyChecker(object):
                FROM source s
                JOIN source_metadata sm ON s.id = sm.src_id
                WHERE s.id in
-                   (SELECT source FROM src_associations
+                   (SELECT src FROM newest_src_association
                        WHERE suite = :suite_id)
                    AND sm.key_id in :metakey_ids
                GROUP BY s.id, s.source''')

daklib/utils.py

@@ -1366,3 +1366,37 @@ def suite_suffix(suite_name):
     elif suite_name in Cnf.value_list('Dinstall::SuiteSuffixSuites'):
         return suffix
     return ''
+
+################################################################################
+
+
+def process_buildinfos(directory, buildinfo_files, fs_transaction, logger):
+    """Copy buildinfo files into Dir::BuildinfoArchive
+
+    @type directory: string
+    @param directory: directory where .changes is stored
+
+    @type buildinfo_files: list of str
+    @param buildinfo_files: names of buildinfo files
+
+    @type fs_transaction: L{daklib.fstransactions.FilesystemTransaction}
+    @param fs_transaction: FilesystemTransaction instance
+
+    @type logger: L{daklib.daklog.Logger}
+    @param logger: logger instance
+    """
+
+    if 'Dir::BuildinfoArchive' not in Cnf:
+        return
+
+    target_dir = os.path.join(
+        Cnf['Dir::BuildinfoArchive'],
+        datetime.datetime.now().strftime('%Y/%m/%d'),
+    )
+
+    for f in buildinfo_files:
+        src = os.path.join(directory, f.filename)
+        dst = find_next_free(os.path.join(target_dir, f.filename))
+
+        logger.log(["Archiving", f.filename])
+        fs_transaction.copy(src, dst, mode=0o644)

integration-tests/common

@@ -38,3 +38,11 @@ import-fixture-signing-key() {
     ln -sf ${packages:?}/gpg/pubring.gpg ${DAKBASE:?}/keyrings/upload-keyring.gpg
     dak import-keyring -U "%s" ${DAKBASE}/keyrings/upload-keyring.gpg
 }
+
+list_all_suites() {
+    dak admin s list | sort | while read suite
+    do
+        printf "\n\n$suite\n"
+        dak control-suite -l $suite |sort
+    done
+}

integration-tests/dinstall

@@ -19,6 +19,7 @@
 declare public_archives=(ftp-master debian-debug)
 declare -r scriptdir=${DAKBASE}/scripts
 declare -r overridedir=${scriptdir}/override # debian
+echo "Dir::Override \"${overridedir}\";" >> "${DAK_CONFIG}"
 declare -r pdiff_tempdir="${DAKBASE}/_dinstall-pdiff-temp"
 declare -r upload_dir="${DAKBASE}/uploads"
 
@@ -42,7 +43,8 @@ setup_debian_like_archive() {
     dak admin suite-component add unstable main contrib non-free
     dak admin suite add-build-queue unstable buildd-unstable buildd-sid build-queues
     dak admin suite-config set unstable allowcsset=no byhash=yes \
-        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog
+        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
+        overrideprocess=True
 
     dak admin suite add-all-arches unstable-debug "" codename=sid-debug archive=debian-debug
     dak admin suite-component add unstable-debug main contrib non-free
@@ -55,7 +57,8 @@ setup_debian_like_archive() {
     dak admin suite-component add testing main contrib non-free
     dak admin suite-config set testing allowcsset=yes byhash=yes \
         changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
-        changelog=dists/testing/ChangeLog
+        changelog=dists/testing/ChangeLog \
+        overrideprocess=True overrideorigin=unstable
 
     dak admin suite add-all-arches testing-debug "" codename=sid-debug archive=debian-debug
     dak admin suite-component add testing-debug main contrib non-free
@@ -68,7 +71,8 @@ setup_debian_like_archive() {
         codename=buster-proposed-updates archive=ftp-master
     dak admin suite-component add testing-proposed-updates main contrib non-free
     dak admin suite-config set testing-proposed-updates allowcsset=yes byhash=yes \
-        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog
+        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
+        overridesuite=testing
 
     
     # experimental
@@ -77,7 +81,8 @@ setup_debian_like_archive() {
     dak admin suite add-build-queue experimental buildd-experimental buildd-rc-buggy build-queues
     dak admin suite-config set experimental allowcsset=no byhash=yes \
         changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
-        notautomatic=yes butautomaticupgrades=no
+        notautomatic=yes butautomaticupgrades=no \
+        overridesuite=unstable overridecodename=sid
 
 
     # Version constraints

integration-tests/tests/2000-dinstall

@@ -23,6 +23,8 @@ set -u
 . ${DAK_ROOT:?}/integration-tests/setup
 . ${DAK_ROOT:?}/integration-tests/dinstall
 
+echo "Dinstall::AllowSourceOnlyUploads true;" >> ${DAKBASE}/etc/dak.conf
+
 setup_debian_like_archive
 import-fixture-signing-key
 
@@ -36,6 +38,8 @@ dinstall
   upload_changes ${packages:?}/nonfree-package_0.1-1_amd64.changes
   upload_changes ${packages:?}/package_0.1-1_amd64.changes
   upload_changes ${packages:?}/main-contrib-with-debug_0.1-1_amd64.changes
+  upload_changes ${packages:?}/binnmupkg_0.1-1_amd64.changes
+  upload_changes ${packages:?}/pkgnew_0.1-1_amd64.changes
 
   process_uploads
 )
@@ -61,6 +65,8 @@ package admin
 EOF
 
 echo a | dak process-new main-contrib-with-debug_0.1-1_amd64.changes
+echo a | dak process-new binnmupkg_0.1-1_amd64.changes
+echo a | dak process-new pkgnew_0.1-1_amd64.changes
   
   dak process-new --automatic
   dak process-policy new
@@ -78,6 +84,8 @@ main-contrib-with-debug 0.2-1 source
 main-package 0.2-1 all
 package 0.1-1 all
 package 0.1-1 source
+binnmupkg 0.1-1 amd64
+binnmupkg 0.1-1 source
 EOF
 )"
 
@@ -90,11 +98,52 @@ dinstall
 ls -l ${DAKBASE}/ftp-master/pool/main/p/package/package_*.dsc
 echo "Published successfully"
 
-echo "contents of testing:"
-dak control-suite -l testing
-echo ""
-echo "contents of unstable:"
-dak control-suite -l unstable
-echo ""
+(
+  upload_changes ${packages:?}/pkgnew_0.1-2~exp_amd64.changes
+  process_uploads
+
+  echo a | dak process-new pkgnew_0.1-2~exp_amd64.changes
+
+  dak process-new --automatic
+  dak process-policy new
+
+  # pretend the upload to experimental happened (more than) 14 days ago to
+  # trigger NVIU issue below
+  echo "update override set created = created - interval '14 days' where package like 'pkg%';" | psql
+
+  upload_changes ${packages:?}/pkgnew_0.1-2_source.changes
+  process_uploads
+
+  upload_changes ${packages:?}/pkgnew_0.1-2_all.changes
+  process_uploads
+
+  upload_changes ${packages:?}/binnmupkg_0.1-2_source.changes
+  process_uploads
+
+  upload_changes ${packages:?}/binnmupkg_0.1-2_amd64.changes
+  process_uploads
+
+  upload_changes ${packages:?}/binnmupkg_0.1-1+b1_amd64.changes
+  process_uploads
+
+  list_all_suites
+
+  # trigger obsolete override issue: needs 2 dinstalls:
+  # NVIU is done after obsolete overrides, so we needs a second dinstall to
+  # remove the overrides for the packages removed by NVIU
+  dinstall
+  dinstall
+
+  list_all_suites
+
+  upload_changes ${packages:?}/pkgnew_0.1-2_amd64.changes
+  process_uploads
+
+)
+
+list_all_suites
 
 dak ls main-contrib-with-debug
+
+# check that this package has actually been published
+ls -l ${DAKBASE}/ftp-master/pool/main/p/pkgnew/pkg-any3_0.1-2_amd64.deb

tests/fixtures/packages/Makefile

 export GNUPGHOME = $(CURDIR)/gpg
 SHELL = /bin/bash
 
-TAR = linux_42.0 nonfree-package_0.1 package_0.1 package_0.2 package-built-using_0.1 main-contrib-with-debug_0.1 main-contrib-with-debug_0.2
-PACKAGES = $(TAR)
+# List of packages to build
+# for each of these, the source must be available in a directory
+# $PACKAGE_$VERSION
+#
+# if the directory doesn't exist, but there is a directory
+# overlays/$PACKAGE and a directory overlays/$PACKAGE_$VERSION,
+# the contents of both directories will be copied sequentially to
+# $PACKAGE_$VERSION, allowing identical data for similar packages (eg.
+# multiple versions of the same packages) to be stored only once
+#
+# the suffix (_F, _A, _B or _S) specifies the type of build
 
-all: packages
+PACKAGES = \
+linux_42.0_F \
+nonfree-package_0.1_F \
+package_0.1_F \
+package_0.2_F \
+package-built-using_0.1_F \
+main-contrib-with-debug_0.1_F \
+main-contrib-with-debug_0.2_F \
+binnmu_0.1_F \
+binnmu_0.1+b1_B \
+binnmu_0.2_S \
+binnmu_0.2_B \
+pkgnew_0.1_F \
+pkgnew_0.2~exp_F \
+pkgnew_0.2_S \
+pkgnew_0.2_A \
+pkgnew_0.2_B \
 
-tarballs: stamp-tarballs
-stamp-tarballs:
-	set -e; for t in $(TAR); do \
+ALL = $(PACKAGES)
+
+stamp-all: $(ALL)
+	touch $@
+
+%.orig.tar.gz:
+	t=$*; \
+	dir=$${t/_/-}; \
+	base=$${t%_*}; \
+	if [ ! -d $$dir ] ; then\
+		cp -rv overlays/$$base/ ./$$dir/; \
+		cp -Trv overlays/$$dir/ ./$$dir; \
+	fi; \
 	if [ ! -f $$t.orig.tar.gz ]; then \
 		tar -czf $$t.orig.tar.gz --exclude=debian $${t/_/-}; \
-	  fi; \
-	done
-	touch $@
+	fi
 
-packages: stamp-packages
-stamp-packages: stamp-tarballs
-	set -e; for p in $(PACKAGES); do \
-	  (cd $${p/_/-}; dpkg-buildpackage); \
-	done
-	touch $@
+%_F: %.orig.tar.gz
+	p=$*; (cd $${p/_/-}; dpkg-buildpackage -F)
+
+%_A: %.orig.tar.gz
+	p=$*; (cd $${p/_/-}; dpkg-buildpackage -A)
+
+%_B: %.orig.tar.gz
+	p=$*; (cd $${p/_/-}; dpkg-buildpackage -B)
+
+%_S: %.orig.tar.gz
+	p=$*; (cd $${p/_/-}; dpkg-buildpackage -S)
 
 clean:
-	set -e; for p in $(PACKAGES); do \
-	  make -C $${p/_/-} -f debian/rules clean; \
+	set -e; for j in $(ALL); do \
+	  p=$${j%_*}; \
+	  dir=$${p/_/-}; \
+	  if [ -d overlays/$$dir ]; then rm -rf ./$$dir; \
+	    else make -C $$dir -f debian/rules clean; fi; \
 	done
 	rm -f *.tar.gz *.tar.xz *.buildinfo *.dsc *.changes *.diff.gz *.deb
 	rm -f gpg/*~
 	rm -f stamp-*
+
+.PRECIOUS: %.orig.tar.gz

tests/fixtures/packages/main-contrib-with-debug-0.2/Makefile

-
-default: test
-
-clean:
-	rm -f test

tests/fixtures/packages/main-contrib-with-debug-0.2/debian/contrib-with-debug.install

-test usr/bin

tests/fixtures/packages/main-contrib-with-debug-0.2/debian/main-package.install

-some-file usr/share/main-package

tests/fixtures/packages/main-contrib-with-debug-0.2/some-file

-some text

tests/fixtures/packages/main-contrib-with-debug-0.2/test.c

-
-
-int main(void) {
-	return(0);
-}
-

tests/fixtures/packages/overlays/binnmu-0.1+b1/debian/changelog

+binnmupkg (0.1-1+b1) testing-proposed-updates; urgency=medium, binary-only=yes
+
+  * Binary-only non-maintainer upload for amd64; no source changes.
+
+ -- A Maintainer <maint@example.com>  Sat, 13 Jul 2019 16:29:19 +0000
+
+binnmupkg (0.1-1) unstable; urgency=low
+
+  * Initial release.
+
+ -- A Maintainer <maint@example.com>  Fri, 08 Jun 2012 18:10:01 +0200

tests/fixtures/packages/overlays/binnmu-0.1/debian/changelog

+binnmupkg (0.1-1) unstable; urgency=low
+
+  * Initial release.
+
+ -- A Maintainer <maint@example.com>  Fri, 08 Jun 2012 18:10:01 +0200