...
 
Commits (8)
# Use an official Debian stable as a parent image
FROM debian:stable
RUN apt-get update && \
apt-get -y install sudo
RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo
# Copy the current directory contents into the container at /app
COPY docker/provision.sh .
# Install any needed packages specified in requirements.txt
RUN ./provision.sh
# Make HTTP port 80 available to the world outside this container
EXPOSE 80
# Make PostgreSQL port 5432 available to the world outside this container
EXPOSE 5432
# Define environment variable
ENV NAME Debian
COPY docker/startup.sh .
# Start apache2 & postgres
CMD ["./startup.sh"]
......@@ -2,16 +2,27 @@
# vi: set ft=ruby :
Vagrant.configure(2) do |config|
config.vm.box = "debian/stretch"
config.vm.box = "debian/stretch64"
config.vm.network "forwarded_port", guest: 80, host: 8080
config.vm.network :forwarded_port, guest: 80, host: 8080
config.vm.provider "virtualbox" do |vb|
vb.memory = "4096"
vb.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/v-root", "1"]
end
config.vm.provision "shell", path: "vagrant/provision.sh"
config.vm.provision :shell, path: "vagrant/provision.sh", args: "'vagrant'"
# Forward agent when doing 'vagrant ssh' (needed for tunnels to ullmann/alioth)
config.ssh.forward_agent = true
config.vm.post_up_message = <<~HEREDOC
UDD set up at http://localhost:8080/
The database is, by default, empty. Either:
- use a tunnel to the real UDD: vagrant ssh -c /vagrant/vagrant/setup-tunnel.sh
this requires shell access to udd.debian.org and to enable SSH agent forwarding
- import (parts of) the real UDD:
vagrant ssh -c '/vagrant/vagrant/populate-db.sh all'
HEREDOC
end
#!/bin/sh
set -x
set -e
sudo sed -i s/httpredir.debian.org/deb.debian.org/ /etc/apt/sources.list
sudo apt-get update
sudo apt-get install -y apache2 postgresql postgresql-plperl-9.6 postgresql-9.6-debversion ruby-debian ruby-oj rsync python-yaml python-psycopg2 ruby-pg ruby-sequel-pg
# trust local connections
sudo sed -ri 's/(local\s+all\s+all\s+)peer/\1trust/' /etc/postgresql/9.6/main/pg_hba.conf
sudo sed -ri 's/(host\s+all\s+all\s+127.0.0.1\/32\s+)md5/\1trust/' /etc/postgresql/9.6/main/pg_hba.conf
sudo sed -ri 's/(host\s+all\s+all\s+::1\/128\s+)md5/\1trust/' /etc/postgresql/9.6/main/pg_hba.conf
# use port 5452 (same as the real UDD instance on ullmann)
sudo sed -ri 's/^port = 5432/port = 5452/' /etc/postgresql/9.6/main/postgresql.conf
sudo sed -ri 's/^#checkpoint_segments = .*/checkpoint_segments = 256/' /etc/postgresql/9.6/main/postgresql.conf
sudo sed -ri 's/^#synchronous_commit = .*/synchronous_commit = off/' /etc/postgresql/9.6/main/postgresql.conf
sudo sed -ri 's/^#fsync = .*/fsync = off/' /etc/postgresql/9.6/main/postgresql.conf
sudo sed -ri 's/^#shared_buffers = .*/shared_buffers = 1GB/' /etc/postgresql/9.6/main/postgresql.conf
# restart postgresql
sudo service postgresql restart
# create and configure UDD database
sudo -u postgres dropuser udd || true
sudo -u postgres createuser -DRS udd
sudo -u postgres dropuser docker || true
sudo -u postgres createuser -DRS docker
sudo -u postgres dropdb udd || true
sudo -u postgres createdb -T template0 -E SQL_ASCII udd
# create the database, named 'udd', forcing the encoding to SQL_ASCII, since that's the format of the export.
# We base it off 'template0' because 'template1' (the default) might be set to UTF8 which prevents creation
# of new SQL_ASCII databases.
sudo -u postgres psql udd -c 'CREATE EXTENSION debversion'
# Also create a guest user (used by CGIs)
sudo -u postgres dropuser guest || true
sudo -u postgres createuser -lDRS guest
sudo -u postgres psql udd -c 'GRANT usage ON schema public TO PUBLIC;'
sudo -u postgres psql udd -c 'GRANT select ON all tables in schema public TO PUBLIC;'
sudo mkdir -p /srv/udd.debian.org/
sudo ln -sfn /docker /srv/udd.debian.org/udd
# stop postgresql
sudo service postgresql stop
# apache setup
#
# This one gives a 'file not found' error:
# sudo ln -sf /docker/docker/apache.conf /etc/apache2/sites-enabled/000-default.conf
sudo ln -sf /etc/apache2/mods-available/cgi.load /etc/apache2/mods-enabled/
sudo ln -sf /etc/apache2/mods-available/rewrite.load /etc/apache2/mods-enabled/
sudo rm -f /etc/apache2/conf-enabled/serve-cgi-bin.conf
#
# Run apache2 as the docker user. Yes, eek. But this avoids all permission problems.
sudo sed -i 's/APACHE_RUN_USER=www-data/APACHE_RUN_USER=docker/' /etc/apache2/envvars
sudo sed -i 's/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=docker/' /etc/apache2/envvars
sudo chown -R docker:docker /var/log/apache2
# This one gives a 'file not found' error:
# sudo chown -R docker:docker /var/lock/apache2
sudo service apache2 restart
#
sudo chown -R docker:docker /etc/apache2
# stop apache (just in case :)
sudo service apache2 stop
#!/bin/sh
# Start services
sudo service postgresql restart
sudo service apache2 restart
# Keep the container up :-)
tail -f /dev/null
#!/usr/bin/env bash
set -xeuo pipefail
sudo mkfs.ext4 -F /dev/sda2
sudo mkdir /var/lib/postgresql
sudo tee -a /etc/fstab <<EOF
/dev/sda2 /var/lib/postgresql auto nodev,nosuid 0 0
EOF
sudo mount /var/lib/postgresql
#!/usr/bin/env bash
set -xuo pipefail
sudo apt install -y kpartx
sudo swapoff /dev/sda5
sudo sed -i '/swap/d' /etc/fstab
sudo update-initramfs -u -k all
sudo fdisk /dev/sda <<EOF
p
d
5
d
2
n
p
2
p
w
EOF
exit 0
#!/bin/sh
#!/usr/bin/env bash
set -e
set -x
DUMP_URI="https://udd.debian.org/dumps"
sshtarget="lucas@udd.debian.org"
SCHEMA_URI="${DUMP_URI}/udd-schema.sql"
POPCON_URI="${DUMP_URI}/udd-popcon.sql.xz"
BUGS_URI="${DUMP_URI}/udd-bugs.sql.xz"
UDD_URI="${DUMP_URI}/udd.dump"
if [ "$1" = "" ]; then
echo "Specify target as parameter (schema, packages)"
export PGUSER=udd
export PGDATABASE=udd
load() {
while [ "$#" -gt 0 ]; do
case "$1" in
*.xz)
curl "$1" | unxz | psql
;;
*.gz)
curl "$1" | gunzip | psql
;;
*.sql)
curl "$1" | psql
;;
*.dump)
curl "$1" | pg_restore -v -d udd -x --disable-triggers
;;
*)
echo "Unable to process '$1'" >&2
exit 1
;;
esac
shift
done
}
# If running interactively, stop there
if [ -z "$BASH" ] || [[ "$-" == *i* ]]; then
return
fi
# Otherwise, parse arguments and load the corresponding files
set -eo pipefail
if [ "$#" -eq 0 ]; then
echo "Specify at least one target as parameter (schema, packages, all)"
exit 1
fi
while [ "$1" != "" ]; do
if [ "$1" = "all" ]; then
# everything (except DD-restricted)
dumptarget="-c --if-exists --exclude-table=ldap --exclude-table=pts"
elif [ "$1" = "schema" ]; then
# everything, without data (except DD-restricted)
dumptarget="--schema-only -c --if-exists --exclude-table=ldap --exclude-table=pts"
elif [ "$1" = "packages" ]; then
# only tables related to sources/packages
dumptarget="--data-only -n sources -n packages -n packages_summary"
elif [ "$1" = "table" ]; then
# only specified table
dumptarget="--data-only -t $2"
shift
else
echo "Unknown target: $1"
exit 1
fi
shift
fname="udd-dump-$(date +%s).$$.dump"
ssh -t $sshtarget pg_dump --no-owner -p 5452 -Fc -v -f /tmp/$fname $dumptarget service=udd
rsync -avP $sshtarget:/tmp/$fname /run/shm/$fname
ssh $sshtarget rm -f /tmp/$fname
pg_restore -U udd -j 8 -v -d udd /run/shm/$fname
bad_all() {
echo "Target 'all' is incompatible with other targets" >&2
exit 1
}
if [ "$1" = "all" ]; then
[ "$#" -eq 1 ] || bad_all
targets=( "${UDD_URI}" )
else
targets=()
while [ "$1" != "" ]; do
case "$1" in
"all")
bad_all
;;
"schema")
targets+=("${SCHEMA_URI}")
;;
# "packages")
# targets+=("${UDD_URI}")
# ;;
# "table")
*)
echo "Unknown target: '$1'" >&2
exit 1
esac
shift
done
fi
for target in "${targets[@]}"; do
load "${target}"
done
#!/bin/sh
if [ ${#@} == 0 ]; then
echo "Usage: $0 <apache-user>"
echo "* apache-user: <user that will run Apache (it should be the provider, 'vagrant' or 'docker')>"
fi
set -x
set -e
sudo sed -i s/httpredir.debian.org/deb.debian.org/ /etc/apt/sources.list
sudo apt-get update
sudo apt-get install -y apache2 postgresql postgresql-plperl-9.6 postgresql-9.6-debversion ruby-debian ruby-oj rsync python-yaml python-psycopg2 ruby-pg ruby-sequel-pg
sudo apt-get install -y apache2 postgresql postgresql-plperl-9.6 postgresql-9.6-debversion ruby-debian ruby-oj rsync python-yaml python-psycopg2 ruby-pg ruby-sequel-pg curl
# trust local connections
sudo sed -ri 's/(local\s+all\s+all\s+)peer/\1trust/' /etc/postgresql/9.6/main/pg_hba.conf
sudo sed -ri 's/(host\s+all\s+all\s+127.0.0.1\/32\s+)md5/\1trust/' /etc/postgresql/9.6/main/pg_hba.conf
......@@ -21,7 +27,7 @@ sudo service postgresql restart
sudo -u postgres dropuser udd || true
sudo -u postgres createuser -DRS udd
sudo -u postgres dropuser vagrant || true
sudo -u postgres createuser -DRS vagrant
sudo -u postgres createuser -DRS $1
sudo -u postgres dropdb udd || true
sudo -u postgres createdb -T template0 -E SQL_ASCII udd
# create the database, named 'udd', forcing the encoding to SQL_ASCII, since that's the format of the export.
......@@ -42,19 +48,9 @@ sudo ln -sf /etc/apache2/mods-available/cgi.load /etc/apache2/mods-enabled/
sudo ln -sf /etc/apache2/mods-available/rewrite.load /etc/apache2/mods-enabled/
sudo rm -f /etc/apache2/conf-enabled/serve-cgi-bin.conf
#
# Run apache2 as the vagrant user. Yes, eek. But this avoids all permission problems.
sudo sed -i 's/APACHE_RUN_USER=www-data/APACHE_RUN_USER=vagrant/' /etc/apache2/envvars
sudo sed -i 's/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=vagrant/' /etc/apache2/envvars
sudo chown -R vagrant:vagrant /var/log/apache2
sudo chown -R vagrant:vagrant /var/lock/apache2
# Run apache2 as the provider user. Yes, eek. But this avoids all permission problems.
sudo sed -i 's/APACHE_RUN_USER=www-data/APACHE_RUN_USER=$1/' /etc/apache2/envvars
sudo sed -i 's/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=$1/' /etc/apache2/envvars
sudo chown -R $1:$1 /var/log/apache2
sudo chown -R $1:$1 /var/lock/apache2
sudo service apache2 restart
echo "
UDD set up at http://localhost:8080/
The database is empty. Either:
- use a tunnel to the real UDD: vagrant ssh -c /vagrant/vagrant/setup-tunnel.sh
- import (parts of) the real UDD:
vagrant ssh -c '/vagrant/vagrant/populate-db.sh schema'
vagrant ssh -c '/vagrant/vagrant/populate-db.sh udd_logs'
"