...
 
Commits (15)
# Use an official Debian stable as a parent image
FROM debian:stable
RUN apt-get update && \
apt-get -y install sudo
RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo
# Copy the current directory contents into the container at /app
COPY virtualization/provision.sh .
# Install any needed packages specified in requirements.txt
RUN ./provision.sh docker
# Make HTTP port 80 available to the world outside this container
EXPOSE 80
# Make PostgreSQL port 5432 available to the world outside this container
EXPOSE 5432
# Define environment variable
ENV NAME Debian
COPY docker/startup.sh .
# Start apache2 & postgres
CMD ["./startup.sh"]
......@@ -2,16 +2,26 @@
# vi: set ft=ruby :
Vagrant.configure(2) do |config|
config.vm.box = "debian/stretch"
config.vm.box = "debian/contrib-stretch64"
config.vm.network "forwarded_port", guest: 80, host: 8080
config.vm.network :forwarded_port, guest: 80, host: 8080
config.vm.provider "virtualbox" do |vb|
vb.memory = "4096"
end
config.vm.provision "shell", path: "vagrant/provision.sh"
config.vm.provision :shell, path: "viztualization/provision.sh", args: "'vagrant'"
# Forward agent when doing 'vagrant ssh' (needed for tunnels to ullmann/alioth)
config.ssh.forward_agent = true
config.vm.post_up_message = <<~HEREDOC
UDD set up at http://localhost:8080/
The database is, by default, empty. Either:
- use a tunnel to the real UDD: vagrant ssh -c /vagrant/vagrant/setup-tunnel.sh
this requires shell access to udd.debian.org and to enable SSH agent forwarding
- import (parts of) the real UDD:
vagrant ssh -c '/vagrant/vagrant/populate-db.sh all'
HEREDOC
end
#!/bin/sh
set -x
set -e
sudo sed -i s/httpredir.debian.org/deb.debian.org/ /etc/apt/sources.list
sudo apt-get update
sudo apt-get install -y apache2 postgresql postgresql-plperl-9.6 postgresql-9.6-debversion ruby-debian ruby-oj rsync python-yaml python-psycopg2 ruby-pg ruby-sequel-pg
# trust local connections
sudo sed -ri 's/(local\s+all\s+all\s+)peer/\1trust/' /etc/postgresql/9.6/main/pg_hba.conf
sudo sed -ri 's/(host\s+all\s+all\s+127.0.0.1\/32\s+)md5/\1trust/' /etc/postgresql/9.6/main/pg_hba.conf
sudo sed -ri 's/(host\s+all\s+all\s+::1\/128\s+)md5/\1trust/' /etc/postgresql/9.6/main/pg_hba.conf
# use port 5452 (same as the real UDD instance on ullmann)
sudo sed -ri 's/^port = 5432/port = 5452/' /etc/postgresql/9.6/main/postgresql.conf
sudo sed -ri 's/^#checkpoint_segments = .*/checkpoint_segments = 256/' /etc/postgresql/9.6/main/postgresql.conf
sudo sed -ri 's/^#synchronous_commit = .*/synchronous_commit = off/' /etc/postgresql/9.6/main/postgresql.conf
sudo sed -ri 's/^#fsync = .*/fsync = off/' /etc/postgresql/9.6/main/postgresql.conf
sudo sed -ri 's/^#shared_buffers = .*/shared_buffers = 1GB/' /etc/postgresql/9.6/main/postgresql.conf
# restart postgresql
sudo service postgresql restart
# create and configure UDD database
sudo -u postgres dropuser udd || true
sudo -u postgres createuser -DRS udd
sudo -u postgres dropuser docker || true
sudo -u postgres createuser -DRS docker
sudo -u postgres dropdb udd || true
sudo -u postgres createdb -T template0 -E SQL_ASCII udd
# create the database, named 'udd', forcing the encoding to SQL_ASCII, since that's the format of the export.
# We base it off 'template0' because 'template1' (the default) might be set to UTF8 which prevents creation
# of new SQL_ASCII databases.
sudo -u postgres psql udd -c 'CREATE EXTENSION debversion'
# Also create a guest user (used by CGIs)
sudo -u postgres dropuser guest || true
sudo -u postgres createuser -lDRS guest
sudo -u postgres psql udd -c 'GRANT usage ON schema public TO PUBLIC;'
sudo -u postgres psql udd -c 'GRANT select ON all tables in schema public TO PUBLIC;'
sudo mkdir -p /srv/udd.debian.org/
sudo ln -sfn /docker /srv/udd.debian.org/udd
# stop postgresql
sudo service postgresql stop
# apache setup
#
# This one gives a 'file not found' error:
# sudo ln -sf /docker/docker/apache.conf /etc/apache2/sites-enabled/000-default.conf
sudo ln -sf /etc/apache2/mods-available/cgi.load /etc/apache2/mods-enabled/
sudo ln -sf /etc/apache2/mods-available/rewrite.load /etc/apache2/mods-enabled/
sudo rm -f /etc/apache2/conf-enabled/serve-cgi-bin.conf
#
# Run apache2 as the docker user. Yes, eek. But this avoids all permission problems.
sudo sed -i 's/APACHE_RUN_USER=www-data/APACHE_RUN_USER=docker/' /etc/apache2/envvars
sudo sed -i 's/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=docker/' /etc/apache2/envvars
sudo chown -R docker:docker /var/log/apache2
# This one gives a 'file not found' error:
# sudo chown -R docker:docker /var/lock/apache2
sudo service apache2 restart
#
sudo chown -R docker:docker /etc/apache2
# stop apache (just in case :)
sudo service apache2 stop
#!/bin/sh
# Start services
sudo service postgresql restart
sudo service apache2 restart
# Keep the container up :-)
tail -f /dev/null
#!/usr/bin/ruby
require 'yaml'
require 'time'
EXCUSES_URL = 'https://release.debian.org/britney/excuses.yaml'
def update_migration_excuses
db = PG.connect(UDD_USER_PG)
db.exec("BEGIN")
db.exec("SET CONSTRAINTS ALL DEFERRED")
db.exec("DELETE FROM migration_excuses")
# all fields: ["missing-builds", "old-binaries"]
db.prepare('me_insert', "INSERT INTO migration_excuses
(item_name, source, migration_policy_verdict, old_version, new_version, is_candidate, excuses, reason, hints, policy_info, dependencies, invalidated_by_other_package, missing_builds, old_binaries)
VALUES
($1, $2, $3, $4, $5, $6, $7::text[], $8::text[], $9::text[], $10, $11, $12, $13, $14)")
excuses = YAML::load(open(EXCUSES_URL).read)
if excuses['generated-date'] < Time::now - 86400
raise "Excuses file is too old"
end
excuses['sources'].each do |e|
hints = e['hints'] ? e['hints'].map { |h| h.values.join(':') } : nil
db.exec_prepared( 'me_insert', [
e['item-name'],
e['source'],
e['migration-policy-verdict'],
e['old-version'],
e['new-version'],
e['is-candidate'],
PG::TextEncoder::Array.new.encode(e['excuses']),
PG::TextEncoder::Array.new.encode(e['reason']),
PG::TextEncoder::Array.new.encode(hints),
JSON::dump(e['policy_info']),
JSON::dump(e['dependencies']),
e['invalidated-by-other-package'],
JSON::dump(e['missing-builds']),
JSON::dump(e['old-binaries'])
])
end
db.exec("COMMIT")
end
......@@ -22,6 +22,7 @@ require 'security-tracker'
require 'upstream'
require 'duck'
require 'wanna-build'
require 'migration_excuses'
require 'optparse'
OptionParser::new do |opts|
......@@ -49,6 +50,7 @@ OptionParser::new do |opts|
when 'upstream' ; Upstream::update_upstream
when 'duck' ; update_duck
when 'wanna-build' ; update_wanna_build
when 'migration-excuses' ; update_migration_excuses
else
puts "Invalid task: #{task}"
exit(1)
......
......@@ -2,6 +2,7 @@
"importers": {
"security-tracker": { "cron":"0 * * * *", "pool":"long-tasks" },
"ci": { "cron":"1 * * * *" },
"migration-excuses": { "cron":"50 * * * *" },
"duck": { "cron":"3 * * * *" },
"upstream": { "cron":"4 * * * *" },
"wanna-build": { "cron":"9 * * * *" },
......
......@@ -26,6 +26,7 @@ my $TESTING='';
my $testing_archs = {};
my $debug = 0;
$| = 1 if ($debug);
my $now = time;
sub debug {
......@@ -89,28 +90,35 @@ sub is_testing_arch {
sub get_depends {
my $handle = shift;
my $query = shift;
my $type = shift;
my $pkgfield = shift;
my $depfields = shift;
my $pkgs = shift;
my $sthc = do_query($handle,$query);
while (my ($pkg,$deps) = $sthc->fetchrow_array()) {
next unless $deps;
foreach my $dep (split(/\s*[|,]\s*/,$deps)) {
my $d = $dep;
$dep =~ s/\([^\)]*\)//og;
$dep =~ s/<[^\>]*>//og;
$dep =~ s/^\s*+//o;
$dep =~ s/\s*+$//o;
if ($dep =~ s/\s*\[([^\]]*)\]//og) {
unless (is_testing_arch($1)) {
#print "skip $pkg $type $dep $d not in testing\n";
next;
while (my $row = $sthc->fetchrow_hashref()) {
my $pkg = $row->{$pkgfield};
for my $field (@$depfields) {
my $deps = $row->{$field};
next unless $deps;
foreach my $dep (split(/\s*[|,]\s*/,$deps)) {
my $d = $dep;
$dep =~ s/\([^\)]*\)//og;
$dep =~ s/<[^\>]*>//og;
$dep =~ s/^\s*+//o;
$dep =~ s/\s*+$//o;
if ($dep =~ s/\s*\[([^\]]*)\]//og) {
unless (is_testing_arch($1)) {
#print "skip $pkg $type $dep $d not in testing\n";
next;
}
}
}
my $info = "$pkg $type $dep";
unless (defined $pkgs->{$dep}) {
$pkgs->{$dep} = $info;
my $type = $field;
$type =~ s/_/-/g;
my $info = "$pkg $type $dep";
unless (defined $pkgs->{$dep}) {
$pkgs->{$dep} = $info;
}
}
}
}
......@@ -184,29 +192,24 @@ while (1) {
my $pkgs = {};
debug "# Getting build-depends for sources\n";
get_depends($dbh,"select source,build_depends from sources
where release='$TESTING' and source in ('".join("','",keys %$newsrcs)."')","build-depends",$pkgs);
get_depends($dbh,"select source,build_depends,build_depends_indep,build_depends_arch from sources
where release='$TESTING' and source in ('".join("','",keys %$newsrcs)."')","source",["build_depends","build_depends_indep","build_depends_arch"],$pkgs);
debug "# Getting sources for build-depends\n";
my $newsrcs_a = add_pkg_sources($dbh,$srcs,$pkgs);
debug "# Getting build-depends-indep for sources\n";
get_depends($dbh,"select source,build_depends_indep from sources
where release='$TESTING' and source in ('".join("','",keys %$newsrcs)."')","build-depends-indep",$pkgs);
my $newsrcs_b = add_pkg_sources($dbh,$srcs,$pkgs);
my $newsrcs_c = {};
if ($do_deps) {
$pkgs = {};
debug "# Getting depends for sources\n";
get_depends($dbh,"select package,depends from packages
where release='$TESTING' and source in ('".join("','",keys %$newsrcs)."')","depends",$pkgs);
where release='$TESTING' and source in ('".join("','",keys %$newsrcs)."')","package",["depends"],$pkgs);
debug "# Getting sources for depends\n";
$newsrcs_c = add_pkg_sources($dbh,$srcs,$pkgs);
}
$newsrcs = {%$newsrcs_a,%$newsrcs_b,%$newsrcs_c};
$newsrcs = {%$newsrcs_a,%$newsrcs_c};
debug "# Adding ".(scalar keys %$newsrcs)." source packages: ".join(" ",sort keys %$newsrcs)."\n";
last unless scalar keys %$newsrcs;
......
......@@ -16,6 +16,7 @@ my $testing = "";
my $removaldelay = 15*24*3600;
my $removaldelay_rdeps = 30*24*3600;
my $debug = 0;
$| = 1 if $debug;
# set to 0 to disable autoremoval of rdeps
my $POPCON_PERCENT = 5; # x% of submissions must have the package installed
......@@ -563,7 +564,7 @@ sub get_autoremoval_info {
sub read_source_depends {
my ($dbh,$needs) = @_;
my $query = "select source, version, build_depends, build_depends_indep from sources ".
my $query = "select source, version, build_depends, build_depends_indep, build_depends_arch from sources ".
" where release = '$testing' and extra_source_only is null;";
my $sthc = do_query($dbh,$query);
while (my $pg = $sthc->fetchrow_hashref()) {
......@@ -573,7 +574,7 @@ sub read_source_depends {
my $depstr = '';
die "Sources has paragraph without Source and Version field\n"
unless defined $src && defined $version;
foreach my $f (qw(build_depends build_depends_indep)) {
foreach my $f (qw(build_depends build_depends_indep build_depends_arch)) {
my $val = $pg->{$f};
next unless $val;
$depstr .= ', ' if $depstr;
......
#!/bin/sh
if [ ${#@} == 0 ]; then
echo "Usage: $0 <apache-user>"
echo "* apache-user: <user that will run Apache (it should be the provider, 'vagrant' or 'docker')>"
fi
set -x
set -e
sudo sed -i s/httpredir.debian.org/deb.debian.org/ /etc/apt/sources.list
......@@ -20,8 +26,8 @@ sudo service postgresql restart
# create and configure UDD database
sudo -u postgres dropuser udd || true
sudo -u postgres createuser -DRS udd
sudo -u postgres dropuser vagrant || true
sudo -u postgres createuser -DRS vagrant
sudo -u postgres dropuser $1 || true
sudo -u postgres createuser -DRS $1
sudo -u postgres dropdb udd || true
sudo -u postgres createdb -T template0 -E SQL_ASCII udd
# create the database, named 'udd', forcing the encoding to SQL_ASCII, since that's the format of the export.
......@@ -37,24 +43,14 @@ sudo mkdir -p /srv/udd.debian.org/
sudo ln -sfn /vagrant /srv/udd.debian.org/udd
# Apache setup
sudo ln -sf /vagrant/vagrant/apache.conf /etc/apache2/sites-enabled/000-default.conf
sudo ln -sf /vagrant/virtualization/apache.conf /etc/apache2/sites-enabled/000-default.conf
sudo ln -sf /etc/apache2/mods-available/cgi.load /etc/apache2/mods-enabled/
sudo ln -sf /etc/apache2/mods-available/rewrite.load /etc/apache2/mods-enabled/
sudo rm -f /etc/apache2/conf-enabled/serve-cgi-bin.conf
#
# Run apache2 as the vagrant user. Yes, eek. But this avoids all permission problems.
sudo sed -i 's/APACHE_RUN_USER=www-data/APACHE_RUN_USER=vagrant/' /etc/apache2/envvars
sudo sed -i 's/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=vagrant/' /etc/apache2/envvars
sudo chown -R vagrant:vagrant /var/log/apache2
sudo chown -R vagrant:vagrant /var/lock/apache2
# Run apache2 as the provider user. Yes, eek. But this avoids all permission problems.
sudo sed -i 's/APACHE_RUN_USER=www-data/APACHE_RUN_USER=$1/' /etc/apache2/envvars
sudo sed -i 's/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=$1/' /etc/apache2/envvars
sudo chown -R $1:$1 /var/log/apache2
sudo chown -R $1:$1 /var/lock/apache2
sudo service apache2 restart
echo "
UDD set up at http://localhost:8080/
The database is empty. Either:
- use a tunnel to the real UDD: vagrant ssh -c /vagrant/vagrant/setup-tunnel.sh
- import (parts of) the real UDD:
vagrant ssh -c '/vagrant/vagrant/populate-db.sh schema'
vagrant ssh -c '/vagrant/vagrant/populate-db.sh udd_logs'
"
......@@ -150,6 +150,7 @@ COLUMNS = [
['crttags', 'release&nbsp;team&nbsp;tags'],
['cautormtime', 'autoremoval&nbsp;time'],
['cwhykey', 'reason for being key package'],
['cbritney', 'britney status'],
]
# copied from /org/bugs.debian.org/etc/config
......@@ -295,6 +296,9 @@ if cgi.params != {}
if cols['cwhykey'] || cols['ckeypackage']
q += ", coalesce(key_packages.reason, '') as whykey"
end
if cols['cbritney']
q += ", coalesce(migration_excuses.migration_policy_verdict, '') as britney"
end
if cols['cmissingbuilds']
q += ", coalesce(mb, '-') as missingbuilds"
end
......@@ -314,6 +318,9 @@ if cgi.params != {}
if cols['cwhykey'] || cols['ckeypackage']
q += " left join key_packages on (bugs.source = key_packages.source) "
end
if cols['cbritney']
q += " left join migration_excuses on (bugs.source = migration_excuses.item_name) "
end
if cols['cmissingbuilds']
q += " left join (
select string_agg(distinct(wannabuild.architecture),', ' order by wannabuild.architecture) mb, source
......
......@@ -4,16 +4,22 @@ $:.unshift('../../rlibs')
require 'udd-db'
require 'pp'
require 'json'
require 'fileutils'
puts "Content-type: text/plain\n\n"
def get_pas
td = `mktemp -d`.chomp
system("cd #{td} && git clone --no-checkout https://buildd.debian.org/git/packages-arch-specific.git > /dev/null 2>&1") or raise
pas = `cd #{td}/packages-arch-specific && git show HEAD:Packages-arch-specific`
raise "error" if $? != 0
FileUtils::rm_rf(td)
return pas
end
# calculate $archs
$archs = {}
pas = `wget --ca-directory /etc/ssl/ca-debian -q -O - "https://anonscm.debian.org/cgit/mirror/packages-arch-specific.git/plain/Packages-arch-specific"`
if $? != 0
puts "unable to fetch Packages-arch-specific from anonscm"
exit 1
end
pas = get_pas
arr = pas.split(/\n/).grep(/^%?[a-z0-9]/).map { |l| l.gsub(/\s*#.*$/,'') }
allarchs=['amd64','i386']
arr.each do |l|
......
......@@ -121,6 +121,7 @@ cautormtime=1&amp;
cmissingbuilds=1&amp;
caffected=1&amp;
cpopcon=1&amp;
cbritney=1&amp;
">Fixed in sid, no activity in the last 5 days, but still not fixed in <%= r_testing %>. Help the fix migrate to <%= r_testing %>!</a></li>
</ul>
</div>
......@@ -230,6 +231,7 @@ cpopcon=1&amp;
<% if cols['cpopcon'] %><th>popcon</th><% end %>
<% if cols['cseverity'] %><th>severity</th><% end %>
<% if cols['chints'] %><th>hints</th><% end %>
<% if cols['cbritney'] %><th>britney status</th><% end %>
<% if cols['cmissingbuilds'] %><th>missing builds in unstable</th><% end %>
<% if cols['cwhykey'] %><th>why key pkg</th><% end %>
<% if cols['cautormtime'] %><th>autoremoval</th><% end %>
......@@ -275,6 +277,7 @@ cpopcon=1&amp;
<%= r['severity'] %></td>
<% end %>
<% if cols['chints'] %><td><%= gen_hints_unblocks(r['rt-hints'] || [], r['unblock-requests'] || []) %> &nbsp;</td><% end %>
<% if cols['cbritney'] %><td><a href="https://release.debian.org/britney/update_excuses.html#<%= r['source']%>"><%= r['britney'] %></a></td><% end %>
<% if cols['cmissingbuilds'] %><td><a href="https://buildd.debian.org/status/package.php?p=<%= r['source']%>&suite=sid"><%= r['missingbuilds'] %></a></td><% end %>
<% if cols['cwhykey'] %><td><%= r['whykey'] %></td><% end %>
<% if cols['cautormtime'] %><td><%= r['autormdate'] %></td><% end %>
......