Commit 52af44ce authored by Jochen Sprickerhof's avatar Jochen Sprickerhof Committed by Holger Levsen
Browse files

rdn: switch to argparse and separate output() 



(cherry picked from commit 9dc03146a0141e3f74552df2855816a687737cb8)
Signed-off-by: default avatarHolger Levsen <holger@layer-acht.org>
parent ec7825b9

bin/rebuilderd_stats.py

+59 −32
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ 
#

# Licensed under GPL-2



from argparse import ArgumentParser, FileType

from datetime import datetime

from hashlib import file_digest

from os import path
@@ -23,22 +24,39 @@ from zstandard import ZstdDecompressor 




def main() -> None:

    arch = argv[1]

    db = argv[2]

    cx = connect(db)

    parser = ArgumentParser(description="Analyze rebuilderd DB failed packages")

    parser.add_argument(

        "arch",

        help="Architecture",

    )

    parser.add_argument(

        "db",

        help="rebuilderd DB file",

    )

    parser.add_argument(

        "--cache",

        type=Path,

        default=Path(),

        help="Cache file (will be ignored when this script changes).",

    )

    parser.add_argument(

        "datefile",

        type=Path,

        help="Output HTML sorted by build date",

    )

    args = parser.parse_args()

    cx = connect(args.db)

    cx.create_function("regexp", 2, lambda x, y: 1 if search(x, y) else 0)

    cu = cx.cursor()



    cache_filename = Path(argv[3]) if len(argv) > 3 else Path()



    hash_key = None

    with open(argv[0], "rb") as fp:

        hash_key = file_digest(fp, "blake2b").hexdigest()



    cache = {}

    cache_new = {}

    if cache_filename.is_file():

        with open(cache_filename, "rb") as fp:

    if args.cache.is_file():

        with args.cache.open("rb") as fp:

            hash_key_old, cache_old = load(fp)

        if hash_key == hash_key_old:

            cache = cache_old
@@ -105,7 +123,7 @@ def main() -> None: 
        "dpkg-buildpackage failed: dh-r (#1089197)": log_has_re(r"(Source|Binary): r-(cran|bioc|other)-.*dpkg-buildpackage died"),

        "dpkg-buildpackage failed": log_has("E: Build failure (dpkg-buildpackage died"),

        "failed to reproduce: diffoscope timeout (not fatal)": diff_has("TRUNCATED DUE TO TIMEOUT: 600 seconds"),

        "failed to reproduce: dh-buildinfo (#1068809)": diff_has_any([f"buildinfo_{arch}.gz", "buildinfo_all.gz"]),

        "failed to reproduce: dh-buildinfo (#1068809)": diff_has_any([f"buildinfo_{args.arch}.gz", "buildinfo_all.gz"]),

        one_second_string: diff_has_re(r" -([drwx-]{10} [0-9a-z() ]{,60} [0-9]+ [0-9-]{10} [012][0-9]:).* \+\1"),

        "failed to reproduce: dh-r (#1089197)": log_has_re(r"(Source|Binary): r-(cran|bioc|other)-"),

        "failed to reproduce": lambda log, diff: diff,
@@ -113,7 +131,7 @@ def main() -> None: 
        "other errors": lambda *_: True,

    }



    if arch == "all":

    if args.arch == "all":

        del error_messages[one_second_string]



    unzstd = ZstdDecompressor()
@@ -144,10 +162,16 @@ def main() -> None: 


    messages_packages = {k: v for k, v in messages_packages.items() if v}  # Remove empty categories



    if cache_filename.name:

        with open(cache_filename, "wb") as fb:

    if args.cache.name:

        with open(args.cache, "wb") as fb:

            dump((hash_key, cache_new), fb)



    db_size = path.getsize(args.db)

    with args.datefile.open("w") as outfile:

        output(outfile, args.arch, db_size, total, messages_packages)





def output(outfile, arch, db_size, total, messages_packages):

    with urlopen("https://salsa.debian.org/reproducible-builds/reproducible-notes/-/raw/master/packages.yml?ref_type=heads") as rn:

        rn_bugs = safe_load(rn)
@@ -157,12 +181,13 @@ def main() -> None: 
        f"<title>https://reproduce.debian.net/{arch}/stats</title>"

        '<meta name="viewport" content="width=device-width, initial-scale=1">'

        "</head><body>"

        f'<header><h1>https://<a href="/">reproduce</a>.debian.net/<a href="/{arch}">{arch}</a>/stats</h1></header> <main>'

        f'<header><h1>https://<a href="/">reproduce</a>.debian.net/<a href="/{arch}">{arch}</a>/stats</h1></header> <main>',

        file=outfile

    )

    print(f"Database size: {bytes2human(path.getsize(db))}<br/>")

    print(f"Last changed: {datetime.now().replace(microsecond=0)} - updated every 3h.")

    print(f"Database size: {bytes2human(db_size)}<br/>", file=outfile)

    print(f"Last changed: {datetime.now().replace(microsecond=0)} - updated every 3h.", file=outfile)



    print("<table> <tr> <th>error</th> <th colspan=2>number of affected<br/>bad binary packages</th><th>number of affected<br/>bad source packages</th> </tr>")

    print("<table> <tr> <th>error</th> <th colspan=2>number of affected<br/>bad binary packages</th><th>number of affected<br/>bad source packages</th> </tr>", file=outfile)

    bad_packages=0

    bad_sources=0

    for message, packages in messages_packages.items():
@@ -170,18 +195,20 @@ def main() -> None: 
        print(

            f'<tr><td><a href="#{anchor}">{message}</a></td>'

            f'<td style="text-align:right">{len(packages)}</td><td style="text-align:right">({len(packages)/total*100:.2f}%)</td>'

            f'<td style="text-align:right">{len(set([pkg['src'] for pkg in packages]))}</td></tr>'

            f'<td style="text-align:right">{len(set([pkg['src'] for pkg in packages]))}</td></tr>',

            file=outfile

        )

        bad_packages += len(packages)

        bad_sources += len(set([pkg['src'] for pkg in packages]))

    print(

        f'<tr><td><b>total amount of unreproduced packages</b></td>'

        f'<td style="text-align:right"><b>{bad_packages}</b></td>'

        f'<td style="text-align:right" colspan="2"><b>{bad_sources}</b></td></tr>'

        f'<td style="text-align:right" colspan="2"><b>{bad_sources}</b></td></tr>',

        file=outfile

    )

    print("</table>")

    print("</table>", file=outfile)



    print('<p>Packages are grouped by having a bug in <a href="https://salsa.debian.org/reproducible-builds/reproducible-notes">reproducible-notes</a> and sorted by build time (newer logs later) inside the group.</p>')

    print('<p>Packages are grouped by having a bug in <a href="https://salsa.debian.org/reproducible-builds/reproducible-notes">reproducible-notes</a> and sorted by build time (newer logs later) inside the group.</p>', file=outfile)



    if arch not in ['amd64', 'arm64', 'armhf', 'i386']:

        ci_arch = 'amd64'
@@ -191,31 +218,31 @@ def main() -> None: 
        anchor = message.replace(" ", "-")

        message = sub(r'(#)([0-9]*)', r'https://bugs.debian.org/\2', message)

        message = sub(r'(https://[a-z.A-Z_0-9/]*)', r'<a href="\1">\1</a>', message)

        print(f'\n<h2 id="{anchor}">{message}</h2>')

        print(f'\n<h2 id="{anchor}">{message}</h2>', file=outfile)

        span = 0

        bug_break = False

        print("<p><span>")

        print("<p><span>", file=outfile)

        for pkg in sorted(packages, key=lambda pkg: pkg["src"] in rn_bugs and "bugs" in rn_bugs[pkg["src"]], reverse=True):

            if not bug_break and not (pkg["src"] in rn_bugs and "bugs" in rn_bugs[pkg["src"]]):

                bug_break = True

                print("</span></p><p><span>")

            print(f'<a href="https://reproduce.debian.net/{arch}/api/v0/builds/{pkg["id"]}/log">{pkg["name"]}</a> ', end='')

                print("</span></p><p><span>", file=outfile)

            print(f'<a href="https://reproduce.debian.net/{arch}/api/v0/builds/{pkg["id"]}/log">{pkg["name"]}</a> ', end='', file=outfile)

            if pkg["diff"]:

                print(f'<a href="https://reproduce.debian.net/{arch}/api/v0/builds/{pkg["id"]}/diffoscope">💠</a>', end='')

            print(f'<a href="https://tracker.debian.org/pkg/{pkg["src"]}">🍥</a>', end='')

            print(f'<a href="https://tests.reproducible-builds.org/debian/rb-pkg/trixie/{ci_arch}/{pkg["src"]}.html">🔬</a>', end='')

                print(f'<a href="https://reproduce.debian.net/{arch}/api/v0/builds/{pkg["id"]}/diffoscope">💠</a>', end='', file=outfile)

            print(f'<a href="https://tracker.debian.org/pkg/{pkg["src"]}">🍥</a>', end='', file=outfile)

            print(f'<a href="https://tests.reproducible-builds.org/debian/rb-pkg/trixie/{ci_arch}/{pkg["src"]}.html">🔬</a>', end='', file=outfile)

            if pkg["src"] in rn_bugs and "bugs" in rn_bugs[pkg["src"]]:

                for bug in rn_bugs[pkg["src"]]["bugs"]:

                    print(f'<a style="color: #FF0000;" href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug={bug}">#</a>', end='')

            print(" ")

                    print(f'<a style="color: #FF0000;" href="https://bugs.debian.org/cgi-bin/bugreport.cgi?bug={bug}">#</a>', end='', file=outfile)

            print(" ", file=outfile)

            span += 1

            if span == 256:

                span = 0

                print("</span><span>")

        print("</span> ")

        print("</p>")

                print("</span><span>", file=outfile)

        print("</span> ", file=outfile)

        print("</p>", file=outfile)



    print('<br/><br/><hr/><a href="https://salsa.debian.org/qa/jenkins.debian.net/-/blob/master/bin/rebuilderd_stats.py">rebuilderd_stats.py<a/> - patches welcome.</main></body></html>')

    print('<br/><br/><hr/><a href="https://salsa.debian.org/qa/jenkins.debian.net/-/blob/master/bin/rebuilderd_stats.py">rebuilderd_stats.py<a/> - patches welcome.</main></body></html>', file=outfile)





if __name__ == "__main__":

hosts/osuosl5-amd64/etc/cron.d/rebuilderd

+1 −1
Original line number Diff line number Diff line 
MAILTO=root

42 */3 * * * rebuilderd for arch in all amd64 arm64 armel armhf i386 ppc64el riscv64 ; do mkdir -p /srv/rebuilderd/$arch/stats/ && /srv/jenkins/bin/rebuilderd_stats.py $arch /srv/rebuilderd/$arch/rebuilderd.db /srv/rebuilderd/$arch/stats/cache.pickle > /srv/rebuilderd/$arch/stats/index.html.tmp && mv /srv/rebuilderd/$arch/stats/index.html.tmp /srv/rebuilderd/$arch/stats/index.html ; done

42 */3 * * * rebuilderd for arch in all amd64 arm64 armel armhf i386 ppc64el riscv64 ; do mkdir -p /srv/rebuilderd/$arch/stats/ && /srv/jenkins/bin/rebuilderd_stats.py $arch /srv/rebuilderd/$arch/rebuilderd.db --cache /srv/rebuilderd/$arch/stats/cache.pickle /srv/rebuilderd/$arch/stats/index.html.tmp && mv /srv/rebuilderd/$arch/stats/index.html.tmp /srv/rebuilderd/$arch/stats/index.html ; done

23 0,6,12,18 * * * rebuilderd for arch in all amd64 arm64 armel armhf i386 ppc64el riscv64 ; do /srv/jenkins/bin/rebuilderd_graph.sh $arch ; done