bin/cgi-bin/schedule

+#!/usr/bin/python3
+
+# Copyright © 2018 Mattia Rizzolo <mattia@debian.org>
+# Licensed under GPL-2
+
+import os
+import re
+import cgi
+import cgitb
+import subprocess
+cgitb.enable()
+
+
+class ValidationError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+        print('Status: 400 Bad Request')
+        print('Content-Type: text/html; charset="utf-8"')
+        print()
+        print(message)
+
+
+def sanify_field(field_name, field_text):
+    sane_re = re.compile(r'^[a-zA-Z0-9_.+-]+$')
+    if not sane_re.match(field_text):
+        err = '"{}" is not sane (does not match {})'.format(field_name, sane_re)
+        raise ValidationError(err)
+
+def validate(form):
+    if 'pkg' not in form:
+        raise ValidationError('no packages specified')
+    for pkg in form.getlist('pkg'):
+        sanify_field('pkg', pkg)
+
+    known_opts = (
+        'dry-run',
+        'keep-artifacts',
+        'notify',
+        'notify-on-start',
+    )
+    known_opts2 = (
+        'message',
+        'status',
+        'issue',
+        'after',
+        'before',
+    )
+    args = []
+    for opt in known_opts:
+        value = form.getvalue(opt)
+        if value:
+            if value in ('yes', 'true'):
+                args.append('--{}'.format(opt))
+    for opt in known_opts2:
+        value = form.getvalue(opt)
+        if value:
+            sanify_field(opt, value)
+            args.append('--{} {}'.format(opt, value))
+    for f in ('suite', 'architecture'):
+        for i in form.getlist(f):
+            sanify_field(f, i)
+    return args
+
+
+def main(args):
+    processes = []
+    failure = False
+    for s in form.getlist('suite'):
+        for a in form.getlist('architecture'):
+            cmd = (
+                '/srv/jenkins/bin/reproducible_remote_scheduler.py',
+                *args,
+                '--dry-run',
+                '--architecture', a,
+                '--suite', s,
+                *form.getlist('pkg'),
+            )
+            print('Executing: ', cmd)
+            try:
+                p = subprocess.run(
+                    cmd,
+                    check=True,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    env={'LC_USER': user},
+                )
+                processes.append(p)
+            except subprocess.CalledProcessError as e:
+                processes.append(e)
+                failure = True
+
+    if failure:
+        print('Status: 520 Unknown Error')
+        print('Content-Type: text/plain; charset="utf-8"')
+        print()
+        print('Failed to schedule packages.  Please try again later or contact us for support.')
+        print()
+        print('Error log:')
+    else:
+        print('Status: 200 OK')
+        print('Content-Type: text/plain; charset="utf-8"')
+        print()
+        print('Successfully scheduled the requested packages.')
+        print()
+        print('Scheduling log:')
+    for p in processes:
+        print()
+        print('Command ran: ', p.args)
+        print(p.stdout.decode('utf-8', errors='ignore').strip())
+        print('Return code: ', p.returncode)
+
+
+# Check whether the user has successfully authenticated
+try:
+    user = os.environ['SSL_CLIENT_S_DN_CN']
+except KeyError:
+    user = None
+    print('Status: 496 SSL Certificate Required')
+    print('Content-Type: text/plain; charset="utf-8"')
+    print()
+    print('You need to authenticate with a Debian SSO certificate to use this service')
+else:
+    try:
+        form = cgi.FieldStorage()
+        main(validate(form))
+    except Exception:
+        print('Status: 500 Internal Server Error')
+        print('Content-Type: text/html; charset="utf-8"')
+        print()
+        cgitb.handler()
+
+print()
+
+print('WARNING: This endpoint is still a WORK IN PROGRESS. This means that the option details can change, and it will not commit any change.')
+print('User: {}'.format(user))
+print(form)

hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf

@@ -60,6 +60,17 @@ Use https-redirect tests.reproducible-builds.org
 Use https-redirect reproducible-builds.org
 Use https-redirect www.reproducible-builds.org
 
+<VirtualHost *:80>
+	Use common-directives diffoscope.org
+	DocumentRoot /srv/diffoscope.org/wwww
+	AddDefaultCharset utf-8
+
+	<Directory /srv/reproducible-builds.org/www>
+		AllowOverride None
+		Require all granted
+	</Directory>
+</VirtualHost>
+
 <VirtualHost *:443>
 	Use common-directives jenkins.debian.net
 	Use common-directives-ssl jenkins.debian.net
@@ -139,6 +150,18 @@ Use https-redirect www.reproducible-builds.org
 	    Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
 	    Require all granted
 	</Directory>
+	# Use the sso.debian.org CA to validate client certificates
+	# Keep these files up to date with update-debsso-ca
+	SSLCACertificateFile /etc/apache2/ssl/debsso/debsso.crt
+	SSLCARevocationCheck chain
+	SSLCARevocationFile /etc/apache2/ssl/debsso/debsso.crl
+	<Location /cgi-bin/schedule>
+		# Export data about the certificate to the environment
+		SSLOptions +StdEnvVars
+		# Allow access if one does not have a valid certificate,
+		# so we can show a decent error message
+		SSLVerifyClient optional
+	</Location>
 
 	<Proxy *>
 		Require all granted

hosts/jenkins/etc/cron.d/dsa

@@ -9,3 +9,4 @@ MAILTO=root
 0 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-running-kernel
 2 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-packages
 30 * * * * root ( /usr/sbin/service ntp stop ; sleep 2 ; /usr/sbin/ntpdate de.pool.ntp.org ; /usr/sbin/service ntp start ) > /dev/null
+0 0 * * * root mkdir -p /etc/apache2/ssl/debsso && /usr/local/bin/update-debsso-ca --destdir /etc/apache2/ssl/debsso

hosts/jenkins/usr/local/bin/update-debsso-ca

+#!/usr/bin/python3
+
+# Originally downloaded from https://anonscm.debian.org/cgit/debian-sso/debian-sso.git/tree/update-debsso-ca
+
+# Download new versions of the CA certificate and Certificate Revocation List
+# from sso.debian.org and write them out atomically.
+
+import requests
+import tempfile
+import argparse
+import os
+import subprocess
+import ssl
+
+class atomic_writer(object):
+    """
+    Atomically write to a file
+    """
+    def __init__(self, fname, mode, osmode=0o644, sync=True, **kw):
+        self.fname = fname
+        self.osmode = osmode
+        self.sync = sync
+        dirname = os.path.dirname(self.fname)
+        self.fd, self.abspath = tempfile.mkstemp(dir=dirname, text="b" not in mode)
+        self.outfd = open(self.fd, mode, closefd=True, **kw)
+
+    def __enter__(self):
+        return self.outfd
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is None:
+            self.outfd.flush()
+            if self.sync: os.fdatasync(self.fd)
+            os.fchmod(self.fd, self.osmode)
+            os.rename(self.abspath, self.fname)
+        else:
+            os.unlink(self.abspath)
+        self.outfd.close()
+        return False
+
+
+def get_url(url):
+    """
+    Fetch a URL and return the raw result as bytes
+    """
+    bundle='/etc/ssl/ca-debian/ca-certificates.crt'
+    if os.path.exists(bundle):
+        res = requests.get(url, verify=bundle)
+    else:
+        res = requests.get(url)
+    res.raise_for_status()
+    return res.content
+
+
+def update_file(pathname, content, validate=None):
+    """
+    Write content on pathname atomically, and do nothing if pathname exists and
+    has the same content as `content`.
+
+    Returns True if the file has been updated, else False.
+    """
+    try:
+        with open(pathname, "rb") as fd:
+            existing = fd.read()
+    except OSError:
+        existing = None
+
+    if existing == content: return False
+
+    # Validate the contents
+    if validate:
+        validate(content)
+
+    with atomic_writer(pathname, "wb", osmode=0o644) as out:
+        out.write(content)
+    return True
+
+def validate_crt(data):
+    ssl.PEM_cert_to_DER_cert(data.decode("utf-8"))
+
+def validate_crl(data):
+    if not data.startswith(b"-----BEGIN X509 CRL-----"):
+        raise RuntimeError("Data does not begin with a CRL signature")
+    if not data.endswith(b"-----END X509 CRL-----\n"):
+        raise RuntimeError("Data does not end with a CRL footer")
+
+def update(destdir):
+    # Fetch the certificate and the CRL
+    cert = get_url("https://sso.debian.org/ca/ca.pem")
+    crl = get_url("https://sso.debian.org/ca/ca.crl")
+
+    # Write them out atomically
+
+    updated = False
+    updated = update_file(os.path.join(destdir, "debsso.crt"), cert, validate=validate_crt) or updated
+    updated = update_file(os.path.join(destdir, "debsso.crl"), crl, validate=validate_crl) or updated
+    return updated
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--destdir", default=".", help="destination directory. Default: .")
+    parser.add_argument("--onupdate", help="command to run if the file has been updated. Default: do not run anything.")
+    args = parser.parse_args()
+
+    if update(args.destdir):
+        if args.onupdate:
+            subprocess.check_call(["sh", "-c", args.onupdate])
+
+
+
+if __name__ == "__main__":
+    main()

job-cfg/reproducible.yaml

@@ -739,6 +739,13 @@
                     my_gitrepo: 'https://salsa.debian.org/reproducible-builds/reproducible-website.git'
                     my_gitbranches: 'origin/master'
                     my_shell: 'jekyll build -s . -d /srv/reproducible-builds.org/www'
+                - 'builds_diffoscope_website':
+                    my_description: 'Build https://diffoscope.org/ on every git commit to https://salsa.debian.org/reproducible-builds/diffoscope-website.git'
+                    my_timed: ''
+                    my_hname: ''
+                    my_gitrepo: 'https://salsa.debian.org/reproducible-builds/diffoscope-website.git'
+                    my_gitbranches: 'origin/master'
+                    my_shell: 'mkdir -pv /srv/diffoscope.org/www && rsync -rptHCv --dry-run ./ /srv/diffoscope.org/www/'
             my_gitbranches: 'master'
             my_recipients: 'jenkins+debian-reproducible qa-jenkins-scm@lists.alioth.debian.org'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'

update_jdn.sh

@@ -447,6 +447,7 @@ if [ -f /etc/debian_version ] ; then
 				procmail 
 				python3-debian 
 				python3-pystache
+				python3-requests
 				python3-sqlalchemy
 				python3-xdg
 				python3-yaml