Skip to content
Commits on Source (16)
Show whitespace changes
Inline Side-by-side
config/homedir/ssh/ftpmaster-authorized_keys
View file @ 989f3757
......@@ -2,20 +2,20 @@
# path: config/homedir/ssh/ftpmaster-authorized_keys
# whenever we have to read a new dataset for testing/squeeze-updates this runs
command="/srv/ftp-master.debian.org/dak/scripts/debian/import_dataset.sh --from-ssh-command",restrict,from="138.16.160.12,franck.debian.org,5.153.231.29,2001:41c8:1000:21::21:29,respighi.debian.org" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC19ORZTjAB3fXAnUUWHVPNKVNKF+SVLGqq28M03pNkZgcAfH79q2Q/1lDOajDzxarKTxrP4wyoniiquy4PsZXrUpSyPJgnkxvlaTwDE6OupXJsaCawKcFe6eitFY9tWR2Bu71WXzjHHFI/6G9bLoiVRFAuxP/pOSJQ1doB7NEdQu8i+/hlN++0Itoa9WiHfK7PcBXjFa3eM6NqYKskxp0+cEWKV/GY51/DNp34mUO+e/ad49wz6URK8bCkdIu/ExLIAfQqLOePcSMH8XxGj6hs/Uk/CH+QzyXRZSzDU5G2QLCMN2XaoCosLzZdP7CaiWhvNth09djjPmKA8xeIZpQX release@respighi
command="/srv/ftp-master.debian.org/dak/scripts/debian/import_dataset.sh --from-ssh-command",restrict,from="5.153.231.29,2001:41c8:1000:21::21:29,respighi.debian.org" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC19ORZTjAB3fXAnUUWHVPNKVNKF+SVLGqq28M03pNkZgcAfH79q2Q/1lDOajDzxarKTxrP4wyoniiquy4PsZXrUpSyPJgnkxvlaTwDE6OupXJsaCawKcFe6eitFY9tWR2Bu71WXzjHHFI/6G9bLoiVRFAuxP/pOSJQ1doB7NEdQu8i+/hlN++0Itoa9WiHfK7PcBXjFa3eM6NqYKskxp0+cEWKV/GY51/DNp34mUO+e/ad49wz6URK8bCkdIu/ExLIAfQqLOePcSMH8XxGj6hs/Uk/CH+QzyXRZSzDU5G2QLCMN2XaoCosLzZdP7CaiWhvNth09djjPmKA8xeIZpQX release@respighi
# release team tpu removals
command="/srv/ftp-master.debian.org/dak/scripts/debian/release_team_removals.sh",restrict,from="138.16.160.12,franck.debian.org,5.153.231.29,2001:41c8:1000:21::21:29,respighi.debian.org" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCw6DLpbCsiadOqxenRfW5In7UFG5HoIDt0xV/dRDbqNUUihNcDi6SqlREuSBCA75lOqbhL1w2tWsdsTIMnJeq3Fdr3LdFjIKlG6QQZVThaD3SI76EkGtjt0XQDoN2d4hi0Xn2LOPKz8hxaY4jKYzSUN0TVue3C1EHTJD0S8Grkd5tPaDgXt4pJzHmNwT4r2dH5OT3Y3vJL2UGhbY6Y+rNFfmnKzDcBtNdUTLTtrAfCCMkPITTYrMvZevA9u/SzNenN9qwEQicc06FrycSCi6+XSA+t4k1YNf1NTHhTQEncEX4/FRf+jgbkt1lkchiu+eShx3bUZCsKPuoNEsuWUU5v release@franck
command="/srv/ftp-master.debian.org/dak/scripts/debian/release_team_removals.sh",restrict,from="5.153.231.29,2001:41c8:1000:21::21:29,respighi.debian.org" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCw6DLpbCsiadOqxenRfW5In7UFG5HoIDt0xV/dRDbqNUUihNcDi6SqlREuSBCA75lOqbhL1w2tWsdsTIMnJeq3Fdr3LdFjIKlG6QQZVThaD3SI76EkGtjt0XQDoN2d4hi0Xn2LOPKz8hxaY4jKYzSUN0TVue3C1EHTJD0S8Grkd5tPaDgXt4pJzHmNwT4r2dH5OT3Y3vJL2UGhbY6Y+rNFfmnKzDcBtNdUTLTtrAfCCMkPITTYrMvZevA9u/SzNenN9qwEQicc06FrycSCi6+XSA+t4k1YNf1NTHhTQEncEX4/FRf+jgbkt1lkchiu+eShx3bUZCsKPuoNEsuWUU5v release@franck
# release team p-u comment files
command="/srv/ftp-master.debian.org/dak/scripts/debian/p-u-comment.sh",restrict,from="138.16.160.12,franck.debian.org,5.153.231.29,2001:41c8:1000:21::21:29,respighi.debian.org,209.87.16.72,2607:f8f0:614:1::1274:72,coccia.deban.org" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvamon5WR/xOOHk9T+wzwVIe5cGH90ivLg94kpFcF0Wkmp7vbh7TSdBtjy5w9YbPG4r6sAMna74dLcgPSDiEpWf56ubSDmGKgKf40EgfC7qOO31aWqRNmyuYMmxWZmbRKQ16LdajFfDEGV1Tdy1a5KdU4HgFlHoucuG7p4VHGO7ZJo0t7CQv5C/hF4gSzK+gjcI1uR1N+jUHxW+3yv75XiQpaLyWgaLYF0x4XYwcDJVcqILaySs8MllJcfyPMNvOyvrhGdHIhei8bnACOkfJ9Eeyv7tOj8Rp8ECMgzYE8AUjv5sR4zkKs7KP5MwoAuCdXvE8OMzyu4ZBa37x+yADWd release@franck
command="/srv/ftp-master.debian.org/dak/scripts/debian/p-u-comment.sh",restrict,from="5.153.231.29,2001:41c8:1000:21::21:29,respighi.debian.org,209.87.16.72,2607:f8f0:614:1::1274:72,coccia.deban.org" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvamon5WR/xOOHk9T+wzwVIe5cGH90ivLg94kpFcF0Wkmp7vbh7TSdBtjy5w9YbPG4r6sAMna74dLcgPSDiEpWf56ubSDmGKgKf40EgfC7qOO31aWqRNmyuYMmxWZmbRKQ16LdajFfDEGV1Tdy1a5KdU4HgFlHoucuG7p4VHGO7ZJo0t7CQv5C/hF4gSzK+gjcI1uR1N+jUHxW+3yv75XiQpaLyWgaLYF0x4XYwcDJVcqILaySs8MllJcfyPMNvOyvrhGdHIhei8bnACOkfJ9Eeyv7tOj8Rp8ECMgzYE8AUjv5sR4zkKs7KP5MwoAuCdXvE8OMzyu4ZBa37x+yADWd release@franck
# sync deferred queue overview
## html file
command="rsync --server -logDtpre.iLsfx . /srv/ftp.debian.org/web/deferred.html",restrict,from="128.31.0.69,usper.debian.org" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGvxeT80rDP/lOqfVAiLRGb32o+2XgiheujPbJS6nm74 sync deferred html file
command="rsync --server -logDtpre.iLsfx . /srv/ftp.debian.org/web/deferred.html",restrict,from="128.31.0.69,2603:400a:ffff:bb8::801f:45,usper.debian.org" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGvxeT80rDP/lOqfVAiLRGb32o+2XgiheujPbJS6nm74 sync deferred html file
## deferred dir
command="rsync --server --delete -vvlogDtpre.iLsfx . /srv/ftp.debian.org/web/deferred/",restrict,from="128.31.0.69,usper.debian.org" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPWdzoLnQJ0Zlp5DDKcPOPk5kIYxb9P2XIwcWZk9Nx9d sync deferred html dir
command="rsync --server --delete -vvlogDtpre.iLsfx . /srv/ftp.debian.org/web/deferred/",restrict,from="128.31.0.69,2603:400a:ffff:bb8::801f:45,usper.debian.org" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPWdzoLnQJ0Zlp5DDKcPOPk5kIYxb9P2XIwcWZk9Nx9d sync deferred html dir
## rrd file
command="rsync --server -vlogDtpre.iLsfx . /srv/ftp.debian.org/web/stat/deferred.rrd",restrict,from="128.31.0.69,usper.debian.org" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHyX7qEexkQ7YW6nskVxwF2zGAX8oJB56Tb7DlxM46Pu sync deferred rrd file
command="rsync --server -vlogDtpre.iLsfx . /srv/ftp.debian.org/web/stat/deferred.rrd",restrict,from="128.31.0.69,2603:400a:ffff:bb8::801f:45,usper.debian.org" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHyX7qEexkQ7YW6nskVxwF2zGAX8oJB56Tb7DlxM46Pu sync deferred rrd file
dak/control_suite.py
View file @ 989f3757
......@@ -178,10 +178,11 @@ def version_checks(package, architecture, target_suite, new_version, session, fo
for suite, version in suite_version_list:
cmp = apt_pkg.version_compare(new_version, version)
if suite in must_be_newer_than and cmp < 1:
# for control-suite we allow equal version (for uploads, we don't)
if suite in must_be_newer_than and cmp < 0:
utils.warn("%s (%s): version check violated: %s targeted at %s is *not* newer than %s in %s" % (package, architecture, new_version, target_suite, version, suite))
violations = True
if suite in must_be_older_than and cmp > 1:
if suite in must_be_older_than and cmp > 0:
utils.warn("%s (%s): version check violated: %s targeted at %s is *not* older than %s in %s" % (package, architecture, new_version, target_suite, version, suite))
violations = True
......@@ -216,6 +217,7 @@ def set_suite(file, suite, transaction, britney=False, force=False):
session = transaction.session
suite_id = suite.suite_id
lines = file.readlines()
suites = [suite] + [q.suite for q in suite.copy_queues]
# Our session is already in a transaction
......@@ -257,9 +259,11 @@ def set_suite(file, suite, transaction, britney=False, force=False):
component = pkg.poolfile.component
if architecture == "source":
transaction.copy_source(pkg, suite, component)
for s in suites:
transaction.copy_source(pkg, s, component)
else:
transaction.copy_binary(pkg, suite, component)
for s in suites:
transaction.copy_binary(pkg, s, component)
Logger.log(["added", suite.suite_name, " ".join(key)])
......@@ -289,6 +293,7 @@ def process_file(file, suite, action, transaction, britney=False, force=False):
return
suite_id = suite.suite_id
suites = [suite] + [q.suite for q in suite.copy_queues]
request = []
......@@ -334,7 +339,8 @@ def process_file(file, suite, action, transaction, britney=False, force=False):
utils.warn("'%s_%s_%s' already exists in suite %s." % (package, version, architecture, suite.suite_name))
continue
else:
transaction.copy_source(pkg, suite, component)
for s in suites:
transaction.copy_source(pkg, s, component)
Logger.log(["added", package, version, architecture, suite.suite_name, pkid])
elif action == "remove":
......@@ -361,7 +367,8 @@ def process_file(file, suite, action, transaction, britney=False, force=False):
utils.warn("'%s_%s_%s' already exists in suite %s." % (package, version, architecture, suite))
continue
else:
transaction.copy_binary(pkg, suite, component)
for s in suites:
transaction.copy_binary(pkg, s, component)
Logger.log(["added", package, version, architecture, suite.suite_name, pkid])
elif action == "remove":
if association_id is None:
......
dak/cruft_report.py
View file @ 989f3757
......@@ -35,7 +35,7 @@ Check for obsolete binary packages
################################################################################
from __future__ import print_function
#from __future__ import print_function
import functools
import os
......@@ -303,6 +303,30 @@ def reportNBS(suite_name, suite_id, rdeps=False):
session.close()
def reportNBSMetadata(suite_name, suite_id, session, rdeps=False):
rows = queryNBS_metadata(suite_id, session)
title = 'NBS packages (from metadata) in suite %s' % suite_name
if rows.rowcount > 0:
print('%s\n%s\n' % (title, '-' * len(title)))
for row in rows:
(packages, architecture, source, version) = row
print("* source package %s version %s no longer builds" %
(source, version))
print(" binary package(s): %s" % packages)
print(" on %s" % architecture)
print(" - suggested command:")
message = '"[auto-cruft] NBS (no longer built by %s)"' % source
print(" dak rm -m %s -s %s -a %s -p -R -b %s" %
(message, suite_name, architecture, packages))
if rdeps:
if utils.check_reverse_depends(packages.split(), suite_name, [architecture], session, True):
print()
else:
print(" - No dependency problem found\n")
else:
print()
def reportAllNBS(suite_name, suite_id, session, rdeps=False):
reportWithoutSource(suite_name, suite_id, session, rdeps)
reportNewerAll(suite_name, session)
......@@ -547,9 +571,9 @@ def main():
# Set up checks based on mode
if Options["Mode"] == "daily":
checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu"]
checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "nbs metadata"]
elif Options["Mode"] == "full":
checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "dubious nbs", "bnb", "bms", "anais"]
checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "nbs metadata", "dubious nbs", "bnb", "bms", "anais"]
elif Options["Mode"] == "bdo":
checks = ["nbs", "obsolete source"]
else:
......@@ -582,6 +606,9 @@ def main():
if "nbs" in checks:
reportAllNBS(suite_name, suite_id, session, rdeps)
if "nbs metadata" in checks:
reportNBSMetadata(suite_name, suite_id, session, rdeps)
if "outdated non-free" in checks:
report_outdated_nonfree(suite_name, session, rdeps)
......
dak/dak.py
View file @ 989f3757
......@@ -33,7 +33,7 @@ G{importgraph}
################################################################################
from __future__ import print_function
#from __future__ import print_function
import importlib
import os
......
daklib/cruft.py
View file @ 989f3757
......@@ -307,3 +307,41 @@ with
group by arch_list, source, version)
select * from outdated_packages order by source"""
return session.execute(query, {'suite_id': suite_id})
def queryNBS_metadata(suite_id, session):
"""searches for NBS packages based on metadata extraction of the
newest source for a given suite"""
query = """
select string_agg(bin.package, ' ' order by bin.package), (
select arch_string
from architecture
where id = bin.architecture) as architecture, src.source, newsrc.version
from bin_associations_binaries bin
join src_associations_src src
on src.src = bin.source
and src.suite = bin.suite
join newest_src_association newsrc
on newsrc.source = src.source
and newsrc.version != src.version
and newsrc.suite = bin.suite
where bin.suite = :suite_id
and bin.package not in (
select trim(unnest(string_to_array(meta.value, ',')))
from source_metadata meta
where meta.src_id = (
select newsrc.src
from newest_src_association newsrc
where newsrc.source = (
select s.source
from source s
where s.id = bin.source)
and newsrc.suite = bin.suite)
and key_id = (
select key_id
from metadata_keys
where key = 'Binary'))
group by src.source, newsrc.version, architecture
order by src.source, newsrc.version, bin.architecture"""
return session.execute(query, {'suite_id': suite_id})
integration-tests/common
View file @ 989f3757
......@@ -16,6 +16,8 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
declare -r testname=`basename $0`
_escape_newline() {
echo "$1" | perl -pe 's/\n/\\n/g'
}
......@@ -39,10 +41,31 @@ import-fixture-signing-key() {
dak import-keyring -U "%s" ${DAKBASE}/keyrings/upload-keyring.gpg
}
list_all_suites() {
check_all_suites() {
# list all suites and compare the control-suite output to the reference
# output from the repo
checkname=$1
if [ -z "$checkname" ]
then
echo "E: $testname: check_all_suites called without checkname"
exit 1
fi
csdir="${DAKBASE}/testoutput/$testname/$checkname"
refdir="${DAK_ROOT}/integration-tests/test-output/$testname/$checkname"
if [ ! -d "$refdir" ]
then
echo "E: $testname: dir $refdir missing"
exit 1
fi
mkdir -p "$csdir"
dak admin s list | sort | while read suite
do
printf "\n\n$suite\n"
dak control-suite -l $suite |sort
dak control-suite -l $suite | LANG=C sort | tee "$csdir/$suite"
done
if ! diff -Nur "$refdir" "$csdir"
then
echo "E: $testname $checkname: output different"
exit 1
fi
}
integration-tests/dinstall
View file @ 989f3757
......@@ -70,6 +70,7 @@ setup_debian_like_archive() {
dak admin suite add-all-arches testing-proposed-updates "" \
codename=buster-proposed-updates archive=ftp-master
dak admin suite-component add testing-proposed-updates main contrib non-free
dak admin suite add-build-queue testing-proposed-updates buildd-testing-proposed-updates buildd-buster-proposed-updates build-queues
dak admin suite-config set testing-proposed-updates allowcsset=yes byhash=yes \
changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
overridesuite=testing
......
integration-tests/test-output/0001-basic/after_accept/new 0 → 100644
View file @ 989f3757
nonfree-package 0.1-1 all
nonfree-package 0.1-1 source
integration-tests/test-output/2000-dinstall/after_dinstall_nviu/buildd-experimental 0 → 100644
View file @ 989f3757
pkg-all1 0.1-2~exp all
pkg-all2 0.1-2~exp all
pkg-all3 0.1-2~exp all
pkg-all4 0.1-2~exp all
pkg-any1 0.1-2~exp amd64
pkg-any2 0.1-2~exp amd64
pkg-any3 0.1-2~exp amd64
pkg-any4 0.1-2~exp amd64
pkgnew 0.1-2~exp source
integration-tests/test-output/2000-dinstall/after_dinstall_nviu/buildd-unstable 0 → 100644
View file @ 989f3757
binnmupkg 0.1-2 amd64
binnmupkg 0.1-2 source
contrib-with-debug 0.2-1 amd64
contrib-with-debug-dbgsym 0.2-1 amd64
main-contrib-with-debug 0.2-1 source
main-package 0.2-1 all
package 0.1-1 all
package 0.1-1 source
pkg-all1 0.1-1 all
pkg-all1 0.1-2 all
pkg-all2 0.1-1 all
pkg-all2 0.1-2 all
pkg-all3 0.1-2 all
pkg-all4 0.1-2 all
pkg-any1 0.1-1 amd64
pkg-any2 0.1-1 amd64
pkgnew 0.1-1 source
pkgnew 0.1-2 source
integration-tests/test-output/2000-dinstall/after_dinstall_nviu/experimental 0 → 100644
View file @ 989f3757
pkg-all1 0.1-2~exp all
pkg-all2 0.1-2~exp all
pkg-all3 0.1-2~exp all
pkg-all4 0.1-2~exp all
pkg-any1 0.1-2~exp amd64
pkg-any2 0.1-2~exp amd64
pkg-any3 0.1-2~exp amd64
pkg-any4 0.1-2~exp amd64
pkgnew 0.1-2~exp source
integration-tests/test-output/2000-dinstall/after_dinstall_nviu/new 0 → 100644
View file @ 989f3757
nonfree-package 0.1-1 all
nonfree-package 0.1-1 source
integration-tests/test-output/2000-dinstall/after_dinstall_nviu/testing 0 → 100644
View file @ 989f3757
binnmupkg 0.1-1 amd64
binnmupkg 0.1-1 source
contrib-with-debug 0.2-1 amd64
main-contrib-with-debug 0.2-1 source
main-package 0.2-1 all
package 0.1-1 all
package 0.1-1 source
integration-tests/test-output/2000-dinstall/after_dinstall_nviu/unstable 0 → 100644
View file @ 989f3757
binnmupkg 0.1-2 amd64
binnmupkg 0.1-2 source
contrib-with-debug 0.2-1 amd64
main-contrib-with-debug 0.2-1 source
main-package 0.2-1 all
package 0.1-1 all
package 0.1-1 source
pkg-all1 0.1-1 all
pkg-all1 0.1-2 all
pkg-all2 0.1-1 all
pkg-all2 0.1-2 all
pkg-all3 0.1-2 all
pkg-all4 0.1-2 all
pkg-any1 0.1-1 amd64
pkg-any2 0.1-1 amd64
pkgnew 0.1-1 source
pkgnew 0.1-2 source