reproducible openwrt: add snapshot build script

    packages = []

    for line in package_list_fp:

        if line == '\n':

        if line == '\n' or line == '':

            parser = email.parser.Parser()

            package = parser.parsestr(linebuffer)

            packages.append(package)

#!/usr/bin/env python3

import os

import re

import pystache

import subprocess

import hashlib

from urllib.request import urlopen

from tempfile import mkdtemp, NamedTemporaryFile

from multiprocessing import cpu_count

from multiprocessing import Pool

from time import strftime, gmtime

import shutil

import importlib

# target to be build

target = os.environ.get("TARGET", "ath79/generic")

# version to be build

version = os.environ.get("VERSION", "SNAPSHOT")

# where to store rendered html and diffoscope output

output_dir = os.environ.get("OUTPUT_DIR", "/srv/reproducible-results")

# where to (re)build openwrt

temporary_dir = os.environ.get("TMP_DIR", mkdtemp(dir="/srv/workspace/chroots/"))

# where to find mustache templates

template_dir = os.environ.get(

    "TEMPLATE_DIR", "/srv/jenkins/mustache-templates/reproducible"

)

# where to find the origin builds

openwrt_url = (

    os.environ.get("ORIGIN_URL", "https://downloads.openwrt.org/snapshots/targets/")

    + target

)

# dir of the version + target

target_dir = os.path.join(output_dir, version, target)

# dir where openwrt actually stores binary files

rebuild_dir = temporary_dir + "/bin/targets/" + target

# where to get the openwrt source git

openwrt_git = os.environ.get("OPENWRT_GIT", "https://github.com/openwrt/openwrt.git")

# run a command in shell

def run_command(cmd, cwd=".", ignore_errors=False):

    print("Running {} in {}".format(cmd, cwd))

    proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE)

    response = ""

    # print and store the output at the same time

    while True:

        line = proc.stdout.readline().decode("utf-8")

        if line == "" and proc.poll() != None:

            break

        response += line

        print(line, end="", flush=True)

    if proc.returncode and not ignore_errors:

        print("Error running {}".format(cmd))

        quit()

    return response

# files not to check via diffoscope

meta_files = re.compile(

    "|".join(

        [

            ".+\.buildinfo",

            ".+\.manifest",

            "openwrt-imagebuilder",

            "openwrt-sdk",

            "sha256sums",

            "kernel-debug.tar.bz2",

        ]

    )

)

# the context to fill the mustache tempaltes

context = {

    "targets": [

        {"version": "SNAPSHOT", "name": "ath79/generic"},

        {"version": "SNAPSHOT", "name": "x86/64"},

        {"version": "SNAPSHOT", "name": "ramips/mt7621"},

    ],

    "version": version,

    "commit_string": "",

    "images_repro": 0,

    "images_repro_percent": 0,

    "images_total": 0,

    "packages_repro": 0,

    "packages_repro_percent": 0,

    "packages_total": 0,

    "today": strftime("%Y-%m-%d", gmtime()),

    "diffoscope_version": run_command(["diffoscope", "--version"]).split()[1],

    "target": target,

    "images": [],

    "packages": [],

    "git_log_oneline": "",

    "missing": [],

}

# download file from openwrt server and compare it, store output in target_dir

def diffoscope(origin_name):

    file_origin = NamedTemporaryFile()

    if get_file(openwrt_url + "/" + origin_name, file_origin.name):

        print("Error downloading {}".format(origin_name))

        return

    run_command(

        [

            "diffoscope",

            file_origin.name,

            rebuild_dir + "/" + origin_name,

            "--html",

            target_dir + "/" + origin_name + ".html",

        ],

        ignore_errors=True,

    )

    file_origin.close()

# return sha256sum of given path

def sha256sum(path):

    with open(path, "rb") as hash_file:

        return hashlib.sha256(hash_file.read()).hexdigest()

# return content of online file or stores it locally if path is given

def get_file(url, path=None):

    print("downloading {}".format(url))

    try:

        content = urlopen(url).read()

    except:

        return 1

    if path:

        print("storing to {}".format(path))

        with open(path, "wb") as file_b:

            file_b.write(content)

        return 0

    else:

        return content.decode("utf-8")

# parse the origin sha256sums file from openwrt

def parse_origin_sha256sums():

    sha256sums = get_file(openwrt_url + "/sha256sums")

    return re.findall(r"(.+?) \*(.+?)\n", sha256sums)

# not required for now

# def exchange_signature(origin_path, rebuild_path):

#    file_sig = NamedTemporaryFile()

#    # extract original signatur in temporary file

#    run_command(

#        "./staging_dir/host/bin/fwtool -s {} {}".format(file_sig.name, origin_path),

#        temporary_dir,

#    )

#    # remove random signatur of rebuild

#    run_command(

#        "./staging_dir/host/bin/fwtool -t -s /dev/null {}".format(rebuild_path),

#        temporary_dir,

#    )

#    # add original signature to rebuild file

#    run_command(

#        "./staging_dir/host/bin/fwtool -S {} {}".format(file_sig.name, rebuild_path),

#        temporary_dir,

#    )

#    file_sig.close()

# initial clone of openwrt.git

run_command(["git", "clone", openwrt_git, temporary_dir])

# download buildinfo files

get_file(openwrt_url + "/config.buildinfo", temporary_dir + "/.config")

with open(temporary_dir + "/.config", "a") as config_file:

    # extra options used by the buildbot

    config_file.writelines(

        [

            "CONFIG_CLEAN_IPKG=y\n",

            "CONFIG_TARGET_ROOTFS_TARGZ=y\n",

            "CONFIG_CLEAN_IPKG=y\n",

            'CONFIG_KERNEL_BUILD_USER="builder"\n',

            'CONFIG_KERNEL_BUILD_DOMAIN="buildhost"\n',

        ]

    )

# insecure private key to build the images

with open(temporary_dir + "/key-build", "w") as key_build_file:

    key_build_file.write(

        "Local build key\nRWRCSwAAAAB12EzgExgKPrR4LMduadFAw1Z8teYQAbg/EgKaN9SUNrgteVb81/bjFcvfnKF7jS1WU8cDdT2VjWE4Cp4cxoxJNrZoBnlXI+ISUeHMbUaFmOzzBR7B9u/LhX3KAmLsrPc="

    )

# spoof the official openwrt public key to prevent adding another key in the binary

with open(temporary_dir + "/key-build.pub", "w") as key_build_pub_file:

    key_build_pub_file.write(

        "OpenWrt snapshot release signature\nRWS1BD5w+adc3j2Hqg9+b66CvLR7NlHbsj7wjNVj0XGt/othDgIAOJS+"

    )

# this specific key is odly chmodded to 600

os.chmod(temporary_dir + "/key-build.pub", 0o600)

# download origin buildinfo file containing the feeds

get_file(openwrt_url + "/feeds.buildinfo", temporary_dir + "/feeds.conf")

# get current commit_string to show in website banner

context["commit_string"] = get_file(openwrt_url + "/version.buildinfo")[:-1]

# ... and parse the actual commit to checkout

commit = context["commit_string"].split("-")[1]

# checkout the desired commit

run_command(["git", "checkout", "-f", commit, temporary_dir])

# show the last 20 commit to have an idea what was changed lately

context["git_log_oneline"] = run_command(

    ["git", "log", "--oneline", "-n", "20"], temporary_dir

)

# do as the buildbots do

run_command(["./scripts/feeds", "update"], temporary_dir)

run_command(["./scripts/feeds", "install", "-a"], temporary_dir)

run_command(["make", "defconfig"], temporary_dir)

# actually build everything

run_command(

    ["make", "IGNORE_ERRORS='n m y'", "BUILD_LOG=1", "-j", str(cpu_count() + 1)],

    temporary_dir,

)

# flush the current website dir of target

shutil.rmtree(target_dir, ignore_errors=True)

# and recreate it here

os.makedirs(target_dir + "/packages", exist_ok=True)

# iterate over all sums in origin sha256sums and check rebuild files

for origin in parse_origin_sha256sums():

    origin_sum, origin_name = origin

    # except the meta files defined above

    if meta_files.match(origin_name):

        print("Skipping meta file {}".format(origin_name))

        continue

    rebuild_path = temporary_dir + "/bin/targets/" + target + "/" + origin_name

    # report missing files

    if not os.path.exists(rebuild_path):

        context["missing"].append({"name": origin_name})

    else:

        rebuild_info = {

            "name": origin_name,

            "size": os.path.getsize(rebuild_path),

            "sha256sum": sha256sum(rebuild_path),

            "repro": False,

        }

        # files ending with ipk are considered packages

        if origin_name.endswith(".ipk"):

            if rebuild_info["sha256sum"] == origin_sum:

                rebuild_info["repro"] = True

                context["packages_repro"] += 1

            context["packages"].append(rebuild_info)

        else:

            #everything else should be images

            if rebuild_info["sha256sum"] == origin_sum:

                rebuild_info["repro"] = True

                context["images_repro"] += 1

            context["images"].append(rebuild_info)

# calculate how many images are reproducible

context["images_total"] = len(context["images"])

if context["images_total"]:

    context["images_repro_percent"] = round(

        context["images_repro"] / context["images_total"] * 100.0, 2

    )

# calculate how many packages are reproducible

context["packages_total"] = len(context["packages"])

if context["packages_total"]:

    context["packages_repro_percent"] = round(

        context["packages_repro"] / context["packages_total"] * 100.0, 2

    )

# now render the website

renderer = pystache.Renderer()

mustache_header = renderer.load_template(template_dir + "/header")

mustache_footer = renderer.load_template(template_dir + "/footer")

mustache_target = renderer.load_template(template_dir + "/target")

mustache_index = renderer.load_template(template_dir + "/index")

index_html = renderer.render(mustache_header, context)

index_html += renderer.render(mustache_index, context)

index_html += renderer.render(mustache_footer, context)

target_html = renderer.render(mustache_header, context)

target_html += renderer.render(mustache_target, context)

target_html += renderer.render(mustache_footer, context)

# and store the files

with open(output_dir + "/index.html", "w") as index_file:

    index_file.write(index_html)

with open(target_dir + "/index.html", "w") as target_file:

    target_file.write(target_html)

# get the origin manifest

origin_manifest = get_file(openwrt_url + "/packages/Packages.manifest")

# and store it in the databse

ropp = importlib.import_module("reproducible_openwrt_package_parser")

with open(rebuild_dir + "/packages/Packages.manifest") as rebuild_manifest:

    result = ropp.show_list_difference(origin_manifest, rebuild_manifest.readlines())

    ropp.insert_into_db(result, "{}-rebuild".format(version))

# run diffoscope over non reproducible files in all available threads

pool = Pool(cpu_count() + 1)

pool.map(

    diffoscope,

    map(

        lambda x: x["name"],

        filter(lambda x: not x["repro"], context["images"] + context["packages"]),

    ),

)

# debug option to keep build dir

if not os.environ.get("KEEP_BUILD_DIR"):

    print("removing build dir")

    shutil.rmtree(temporary_dir)

print("all done")

  </div>

  <hr id="footer_separator" />

  <p style="font-size:0.9em;">

    <div id="page_footer">

      This page was built by the jenkins job <a href="https://jenkins.debian.net/job/reproducible_openwrt-target-ar71xx/"> reproducible_openwrt-target-ar71xx</a>

      which is configured via this <a href="https://salsa.debian.org/qa/jenkins.debian.net">git repo</a>.

      There is more information

      <a href="/userContent/about.html">about jenkins.debian.net</a>

      and about <a href="https://wiki.debian.org/ReproducibleBuilds"> reproducible builds of Debian</a>

      available elsewhere.

      <br /> Please send technical feedback about jenkins to <a href="mailto:qa-jenkins-dev@lists.alioth.debian.org">the Debian jenkins development list</a>,

      or as a <a href="https://www.debian.org/Bugs/Reporting">bug report against the <tt>jenkins.debian.org</tt> package</a>.

      Feedback about specific job results should go to their respective lists and/or the BTS.

      <br />

      The code of <a href="https://salsa.debian.org/qa/jenkins.debian.net">jenkins.debian.net</a>

      is mostly GPL-2 licensed. The weather icons are public domain and were taken from the

      <a href="http://tango.freedesktop.org/Tango_Icon_Library" target=_blank> Tango Icon Library</a>.

      Copyright 2014-2019 <a href="mailto:holger@layer-acht.org">Holger Levsen</a>

      and <a href="https://jenkins.debian.net/userContent/thanks.html">many others</a>.

    </div>

  </p>

  </div>

</body>

</html>

<!DOCTYPE html>

<html lang="en-US">

<head>

	<meta charset="UTF-8">

	<meta name="viewport" content="width=device-width">

	<title>Reproducible OpenWrt ?</title>

	<link rel='stylesheet' id='kamikaze-style-css' href='cascade.css?ver=4.0' type='text/css' media='all'>

</head>

<style type="text/css">

	html,

	body {

		margin: 0;

		padding: 0;

		height: 100%;

	}

	body {

		color: #333;

		padding-top: 2em;

		font-family: Helvetica, Arial, sans-serif;

		width: 90%;

		min-width: 700px;

		max-width: 1100px;

		margin: auto;

		font-size: 120%;

		background-color: #ddd;

	}

	h1 {

		font-size: 120%;

		line-height: 1em;

	}

	h2 {

		font-size: 100%;

		line-height: 1em;

	}

	table {

		width: 100%;

		box-shadow: 0 0 0.5em #999;

		margin: 0;

		border: none !important;

		margin-bottom: 2em;

		border-collapse: collapse;

		border-spacing: 0;

	}

	th {

		background: #000;

		background: -webkit-linear-gradient(top, #444, #000);

		background: -moz-linear-gradient(top, #444, #000);

		background: -ms-linear-gradient(top, #444, #000);

		background: -o-linear-gradient(top, #444, #000);

		background: linear-gradient(top, #444, #000);

		font-size: 14px;

		line-height: 24px;

		border: none;

		text-align: left;

		color: #fff;

	}

	tr {

		background: rgba(255, 255, 255, 0.8);

	}

	tr:hover {

		background: rgba(255, 255, 255, 0.6);

	}

	p,

	th,

	td {

		font-size: 14px;

	}

	th,

	td {

		height: 20px;

		vertical-align: middle;

		white-space: nowrap;

		padding: 0.2em 0.5em;

		border: 1px solid #ccc;

	}

	a:link,

	a:visited {

		color: #337ab7;

		font-weight: bold;

		text-decoration: none;

	}

	a:hover,

	a:active,

	a:focus {

		color: #23527c;

		text-decoration: underline;

	}

	.s {

		text-align: right;

		width: 15%;

	}

	.d {

		text-align: center;

		width: 15%;

	}

	.sh {

		font-family: monospace;

	}

</style>

<body>

	<div id="content">

		<pre>

  _______                     ________        __

 |       |.-----.-----.-----.|  |  |  |.----.|  |_

 |   -   ||  _  |  -__|     ||  |  |  ||   _||   _|

 |_______||   __|_____|__|__||________||__|  |____|

          |__| W I R E L E S S   F R E E D O M

 -----------------------------------------------------

 OpenWrt {{ version }}, {{ commit_string }}

 -----------------------------------------------------

    </pre>

	</div>

	<div id="main-content">

		<h1>OpenWrt - <em>reproducible</em> wireless freedom!</h1>

    <p>

      <em>Reproducible builds</em> enable anyone to reproduce bit by bit

      identical binary packages from a given source, so that anyone can verify

      that a given binary derived from the source it was said to be derived.

      There is more information about

      <a href="https://wiki.debian.org/ReproducibleBuilds">reproducible builds on the Debian wiki</a>

      and on

      <a href="https://reproducible-builds.org">https://reproducible-builds.org</a>.

      These pages explain in more depth why this is

      useful, what common issues exist and which

      workarounds and solutions are known.

    </p>

    <p>

      <em>Reproducible OpenWrt</em> is an effort to apply this to OpenWrt. Thus each OpenWrt target is build twice, with a few variations added and then the resulting images and packages from the two builds are compared using <a href="https://tracker.debian.org/diffoscope">diffoscope</a>. OpenWrt generates many different types of raw <code>.bin</code> files, and diffoscope does not know how to parse these. Thus the resulting diffoscope output is not nearly as clear as it could be - hopefully this limitation will be overcome eventually, but in the meanwhile the input components (uImage kernel file, rootfs.tar.gz, and/or rootfs squashfs) can be inspected. Also please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>

    <p>

      There is a weekly run <a href="https://jenkins.debian.net/view/reproducible/job/reproducible_openwrt/">jenkins

        job</a> to test the <code>master</code> branch of <a href="https://github.com/openwrt/openwrt.git">OpenWrt.git</a>. The

      jenkins job is running <a href="https://salsa.debian.org/qa/jenkins.debian.net/tree/master/bin/reproducible_openwrt.sh">reproducible_openwrt.sh</a>

      in a Debian environment and this script is solely responsible for creating

      this page. Feel invited to join <code>#reproducible-builds</code> (on

      irc.oftc.net) to request job runs whenever sensible. Patches and other <a href="mailto:reproducible-builds@lists.alioth.debian.org">feedback</a> are

      very much appreciated - if you want to help, please start by looking at the

      <a href="https://jenkins.debian.net/userContent/todo.html#_reproducible_openwrt">ToDo

        list for OpenWrt</a>, you might find something easy to contribute. <br />Thanks to <a href="https://www.profitbricks.co.uk">Profitbricks</a> for

      donating the virtual machines this is running on!</p>

    <p>

      <ul>

        {{ #targets }}

        <li><a href="{{ version }}/{{ name}}">{{ version }} - {{ name }}</a></li>

        {{ /targets }}

      </ul>

    </p>

    <table>

      <tr>

        <th>git commit log</th>

      </tr>

      <tr>

        <td>

          <pre>{{ git_log_oneline }}</pre>

        </td>

      </tr>

    </table>