reproducible Debian, debrebuild: update code copy from updated...

reproducible Debian, debrebuild: update code copy from updated debian/devscripts!212

 - thanks josch

Signed-off-by: Holger Levsen <holger@layer-acht.org>

bin/rb-debrebuild

@@ -33,8 +33,10 @@ use File::Basename;
 eval {
     require LWP::Simple;
     require LWP::UserAgent;
+    require URI::Escape;    # libwww-perl depends on liburi-perl
     no warnings;
-    $LWP::Simple::ua = LWP::UserAgent->new(agent => 'LWP::UserAgent/srebuild');
+    $LWP::Simple::ua
+      = LWP::UserAgent->new(agent => 'LWP::UserAgent/debrebuild');
     $LWP::Simple::ua->env_proxy();
 };
 if ($@) {
@@ -45,11 +47,11 @@ if ($@) {
     }
 }
 
-my $respect_build_path  = 1;
-my $use_tor             = 0;
-my $outdir              = './';
-my $builder             = 'none';
-my @required_timestamps = ();
+my $respect_build_path = 1;
+my $use_tor            = 0;
+my $outdir             = './';
+my $builder            = 'none';
+my $timestamp          = '';
 
 my %OPTIONS = (
     'help|h'              => sub { usage(0); },
@@ -57,7 +59,7 @@ my %OPTIONS = (
     'respect-build-path!' => \$respect_build_path,
     'output|O=s'          => \$outdir,
     'builder=s'           => \$builder,
-    'timestamp|t=s'       => \@required_timestamps,
+    'timestamp|t=s'       => \$timestamp,
 );
 
 sub usage {
@@ -80,7 +82,9 @@ Options:
                             provided .buildinfo file.
  --output, -O               Directory for the build artifacts (default: ./)
  --builder=BUILDER          Which building software should be used. See section BUILDER
- --timestamp, -t            The required timestamps from snapshot.d.o if you already know them, separated by commas
+ --timestamp, -t            The required unstable main timestamps from snapshot.d.o if you
+                            already know them, separated by commas, or one of the values
+                            "first_seen" or "metasnap". See section TIMESTAMPS.
 
 Note: $me can parse buildinfo files with and without a GPG signature.  However,
 the signature (if present) is discarded as debrebuild does not support verifying
@@ -120,6 +124,27 @@ bug #898446 for details. To enable user namespaces, run:
 
     \$ sudo sysctl -w kernel.unprivileged_userns_clone=1
 
+TIMESTAMPS
+
+The --timestamp option allows one to skip the step of figuring out the correct
+set of required timestamps by listing them separated by commas in the same
+format used in the snapshot.d.o URL. The default is to use the "first_seen"
+attribute from the snapshot.d.o API and download multiple Packages files until
+all required timestamps are found. To explicitly select this mode, use
+--timestamp=first_seen. Lastly, the metasnap.d.n service can be used to figure
+out the right set of timestamps. This mode can be selected by using
+--timestamp=metasnap. In contrast to the "first_seen" mode, the metasnap.d.n
+service will always return a minimal set of timestamps if the package versions
+were at some point part of Debian unstable main.
+
+LIMITATIONS
+
+Currently, the code assumes that all packages were at some point part of Debian
+unstable main. This fails for packages from Debian ports, packages from
+experimental as well as for locally built packages or packages from third
+party repositories. Enabling support for Debian ports and experimental is
+conceptually possible and only needs somebody implementing it.
+
 EOF
 
     exit($exit_code);
@@ -127,9 +152,6 @@ EOF
 
 GetOptions(%OPTIONS);
 
-# support timestamps being separated by a comma
-@required_timestamps = split(/,/, join(',', @required_timestamps));
-
 my $buildinfo = shift @ARGV;
 if (not defined($buildinfo)) {
     print STDERR "ERROR: Missing mandatory buildinfo filename\n";
@@ -434,6 +456,50 @@ if (!defined($src_date)) {
     die "cannot find .dsc\n";
 }
 
+# support timestamps being separated by a comma
+my @required_timestamps = ();
+if ($timestamp eq "first_seen") {
+    # nothing to do, timestamps will be figured out later
+} elsif ($timestamp eq "metasnap") {
+    # acquire the required timestamps using metasnap.d.n
+    print "retrieving required timestamps from metasnap.d.n\n";
+    my $ua = LWP::UserAgent->new(timeout => 10);
+    $ua->env_proxy;
+    my @pkgs = ();
+    foreach my $pkg (@inst_build_deps) {
+        my $pkg_name = $pkg->{name};
+        my $pkg_ver  = $pkg->{version};
+        my $pkg_arch = $pkg->{architecture};
+        if (defined $pkg_arch) {
+            push @pkgs,
+              URI::Escape::uri_escape("$pkg_name:$pkg_arch=$pkg_ver");
+        } else {
+            push @pkgs, URI::Escape::uri_escape("$pkg_name=$pkg_ver");
+        }
+    }
+    my $response
+      = $ua->get('https://metasnap.debian.net/cgi-bin/api'
+          . '?archive=debian'
+          . "&pkgs="
+          . (join "%2C", @pkgs)
+          . "&arch=$build_arch"
+          . '&suite=unstable'
+          . '&comp=main');
+    if (!$response->is_success) {
+        die "request to metasnap.d.n failed: $response->status_line";
+    }
+    foreach my $line (split /\n/, $response->decoded_content) {
+        my ($arch, $t) = split / /, $line, 2;
+        if ($arch ne $build_arch) {
+            die
+"debrebuild is currently unable to handle multiple architectures";
+        }
+        push @required_timestamps, $t;
+    }
+} else {
+    @required_timestamps = split(/,/, $timestamp);
+}
+
 # setup a temporary apt directory
 
 my $tempdir = tempdir(CLEANUP => 1);
@@ -505,7 +571,8 @@ open(FH, '>', $aptconf);
 # commit 475f75506db48a7fa90711fce4ed129f6a14cc9a.
 #
 # Acquire::Check-Valid-Until has to be set to false because the snapshot
-# timestamps might be too far in the past to still be valid.
+# timestamps might be too far in the past to still be valid. This could be
+# fixed by a solution to https://bugs.debian.org/763419
 #
 # Acquire::Languages has to be set to prevent downloading of translations from
 # the mirrors.
@@ -514,6 +581,14 @@ open(FH, '>', $aptconf);
 # so that apt-get update fails if repositories cannot be authenticated. The
 # default value of this option will change to true with apt from Debian
 # Buster.
+#
+# We need APT::Get::allow-downgrades set to true, because even if we choose a
+# base distribution that was released before the state that "unstable"
+# currently is in, the package versions in that stable release might be newer
+# than what is in unstable due to security fixes. Choosing a stable release
+# from an older snapshot timestamp would fix this problem but would defeat the
+# purpose of a base distribution for builders like sbuild which can take
+# advantage of existing chroot environments.
 
 print FH <<EOF;
 Apt {
@@ -523,13 +598,19 @@ Apt {
 
 Dir "$tempdir";
 Dir::State::status "$tempdir/var/lib/dpkg/status";
-Acquire::Check-Valid-Until "false";
 Acquire::Languages "none";
-Acquire::http::Dl-Limit "1000";
-Acquire::https::Dl-Limit "1000";
-Acquire::Retries "5";
 Binary::apt-get::Acquire::AllowInsecureRepositories "false";
 EOF
+my @common_aptopts = (
+    'Acquire::Check-Valid-Until "false";',
+    'Acquire::http::Dl-Limit "1000";',
+    'Acquire::https::Dl-Limit "1000";',
+    'Acquire::Retries "5";',
+    'APT::Get::allow-downgrades "true";',
+);
+foreach my $line (@common_aptopts) {
+    print FH "$line\n";
+}
 close FH;
 
 # add the removed keys because they are not returned by Dpkg::Vendor
@@ -815,13 +896,6 @@ if ($builder ne "none") {
     }
 }
 
-my @aptopts = (
-    'Acquire::Check-Valid-Until "false";',
-    'Acquire::http::Dl-Limit "1000";',
-    'Acquire::https::Dl-Limit "1000";',
-    'Acquire::Retries "5";'
-);
-
 if ($builder eq "none") {
     print "\n";
     print "Manual installation and build\n";
@@ -898,7 +972,7 @@ if ($builder eq "none") {
         die "$config already exists -- refusing to overwrite\n";
     }
     open(FH, '>', $config) or die "cannot open $config: $!\n";
-    foreach my $line (@aptopts) {
+    foreach my $line (@common_aptopts) {
         print FH "$line\n";
     }
     close FH;
@@ -963,7 +1037,8 @@ if ($builder eq "none") {
       'dpkg-buildpackage', '-uc', "--host-arch=$host_arch", "--build=$build"
       or die "dpkg-buildpackage failed\n";
     # we are not interested in the unpacked source directory
-    0 == system 'rm', '-r', $custom_build_path;
+    0 == system 'rm', '-r', $custom_build_path
+      or die "failed to remove $custom_build_path: $?";
     # but instead we want the produced artifacts
     0 == system 'dcmd', 'mv',
       (dirname $custom_build_path)
@@ -993,7 +1068,7 @@ if ($builder eq "none") {
     # the user.
     push @cmd,
         '--chroot-setup-commands=echo '
-      . (shellescape(join '\n', @aptopts))
+      . (shellescape(join '\n', @common_aptopts))
       . ' | tee /etc/apt/apt.conf.d/23-debrebuild.conf';
 
     my @add_depends = ();
@@ -1042,7 +1117,7 @@ if ($builder eq "none") {
         push @cmd, "--build-path=$custom_build_path";
     }
     push @cmd, "${srcpkgname}_$srcpkgver";
-    print(join " ", @cmd) . "\n";
+    print((join " ", @cmd) . "\n");
     0 == system @cmd or die "sbuild failed\n";
 } elsif ($builder eq "mmdebstrap") {
 
@@ -1091,10 +1166,7 @@ if ($builder eq "none") {
         'mmdebstrap',
         "--arch=$build_arch",
         "--variant=apt",
-        '--aptopt=Acquire::Check-Valid-Until "false"',
-        '--aptopt=Acquire::http::Dl-Limit "1000";',
-        '--aptopt=Acquire::https::Dl-Limit "1000";',
-        '--aptopt=Acquire::Retries "5";',
+        (map { "--aptopt=$_" } @common_aptopts),
         '--include=' . (join ' ', @install),
         '--essential-hook=chroot "$1" sh -c "'
           . (
@@ -1127,7 +1199,7 @@ if ($builder eq "none") {
         '/dev/null',
         "deb $base_mirror/$build_date/ $base_dist main"
     );
-    print(join ' ', @cmd) . "\n";
+    print((join ' ', @cmd) . "\n");
 
     0 == system @cmd or die "mmdebstrap failed\n";
 } else {