bin/rebuilderd_stats: Process packages only once

from collections import defaultdict

from datetime import datetime

from re import search

from re import search, DOTALL

from sqlite3 import connect

from sys import argv

    log = "CAST(b.build_log AS TEXT)"

    r_packages = "p.name like 'r-cran-%' or p.name like 'r-bioc-%' or p.name like 'r-other-%'"

    def log_has(s):

        return lambda log, _: s in log

    def log_has_re(re):

        return lambda log, _: search(re, log, DOTALL)

    def log_has_any(strs):

        def f(log, _):

            for s in strs:

                if s in log:

                    return True

            return False

        return f

    def diff_has(s):

        return lambda _, diff: s in (diff or '')

    def diff_has_re(re):

        return lambda _, diff: search(re, diff or '', DOTALL)

    def diff_has_any(strs):

        def f(_, diff):

            if not diff:

                return False

            for s in strs:

                if s in (diff or ''):

                    return True

            return False

        return f

    error_messages = {

        f"{log} like '%rebuilderd: unexpected error while rebuilding package:"

        " Failed to download build input from%'": "buildinfo file 404 (maybe temporary)",

        f"{log} like '%rebuilderd: unexpected error while rebuilding package:"

        " Failed to download original package from%'": "package file 404 (temporary)",

        f"{log} like '%debsnap: fatal error at line 27%'": "debsnap failed (temporary)",

        f"{log} like '%debsnap: No source files found for%'": "debsnap failed (temporary)",

        f"{log} like '%cannot find:%debootsnap failed"

        " at /usr/bin/debrebuild line 48%'": "packages missing on metasnap (maybe temporary)",

        f"{log} not like '%cannot find:%' and {log} like '%debootsnap failed"

        " at /usr/bin/debrebuild line 48%'": "debootsnap failed (maybe temporary)",

        f"{log} like '%Validation FAILED!!%'": "dscverify failed (temporary)",

        f"{log} like '%400 URL must be absolute"

        "_E: Could not download%sbuild failed%'": "download failed (temporary)",

        f"{log} like '%E: Error creating chroot session: skipping%'": "sbuild chroot failed (temporary)",

        f"{log} like '%E: Disk space is probably not sufficient for building.%'": "sbuild failed due to insufficient disk space",

        f"{log} like '%TRUNCATED DUE TO TIMEOUT: %'"

        f" and {log} like '%inputs/freedict_20%'": "timeout: freedict (#998683)",

        f"{log} like '%TRUNCATED DUE TO TIMEOUT: %'"

        f" and {log} not like '%inputs/freedict_20%'": "timeout (maybe temporary)",

        f"{log} like '%TRUNCATED DUE TO SIZE LIMIT: %'": "size limit (not fatal)",

        f"{log} like '%fakeroot not found, either install the fakeroot%'"

        fr" and {log} regexp 'dpkg is already the newest version \(1\.1[0-8]'": "old dpkg (<1.19.0, temporary)",

        f"{log} like '%fakeroot not found, either install the fakeroot%'"

        fr" and not {log} regexp 'dpkg is already the newest version \(1\.1[0-8]'": "fakeroot not found (https://deb.li/3P46G)",

        f"{log} not like '%fakeroot not found, either install the fakeroot%'"

        f" and not ({r_packages})"

        f" and {log} like '%E: Build failure (dpkg-buildpackage died)%'": "dpkg-buildpackage failed",

        "b.diffoscope like '%TRUNCATED DUE TO TIMEOUT: 600 seconds%'": "diffoscope timeout (not fatal)",

        f"not ({r_packages}) and (b.diffoscope is null or"

        f" (b.diffoscope not like '%buildinfo_{arch}.gz%' and"

        " b.diffoscope not like '%buildinfo_all.gz%' and"

        " b.diffoscope not like '%TRUNCATED DUE TO TIMEOUT: 600 seconds%'))"

        fr" and ({log} regexp 'checking [^ ]*: $'"

        fr" or {log} regexp 'checking [^ ]*: size differs for [^ ]*$'"

        fr" or {log} regexp 'checking [^ ]*: size... $'"

        fr" or {log} regexp 'checking [^ ]*: size... value of [^ ]* differs for [^ ]*$')": "failed to reproduce",

        f"{log} like '%rebuilderd: unexpected error while rebuilding package:"

        " Failed to run diffoscope: No such file or directory (os error 2)%'": "diffoscope not found (fixed)",

        r_packages: "failed to reproduce: R package (#1089197)",

        f"b.diffoscope like '%buildinfo_{arch}.gz%' or b.diffoscope like '%buildinfo_all.gz%'": "failed to reproduce: dh_buildinfo (#1068809)",

        f"({log} like '%.deb: size... md5... sha256... sha1... all OK_' or {log} like '%.deb: size... md5... sha1... sha256... all OK_')": "rebuilderd error"

        "buildinfo file 404 (maybe temporary)": log_has("rebuilderd: unexpected error while rebuilding package: Failed to download build input from"),

        "package file 404 (temporary)": log_has("rebuilderd: unexpected error while rebuilding package: Failed to download original package from"),

        "debsnap failed (temporary)": log_has("debsnap: fatal error at line 27"),

        "debsnap failed to find source files (temporary)": log_has("debsnap: No source files found for"),

        "packages missing on metasnap (maybe temporary)": log_has_re(r"cannot find:.*debootsnap failed at /usr/bin/debrebuild line 48"),

        "debootsnap failed (maybe temporary)": log_has("debootsnap failed at /usr/bin/debrebuild line 48"),

        "dscverify failed (temporary)": log_has("Validation FAILED!!"),

        "download failed (temporary)": log_has_re("400 URL must be absolute.E: Could not download.*sbuild failed"),

        "sbuild chroot failed (temporary)": log_has("E: Error creating chroot session: skipping"),

        "sbuild failed due to insufficient disk space": log_has("E: Disk space is probably not sufficient for building."),

        "timeout: freedict (#998683)": log_has_any(["TRUNCATED DUE TO TIMEOUT: ", "inputs/freedict_20"]),

        "timeout (maybe temporary)": log_has("TRUNCATED DUE TO TIMEOUT: "),

        "size limit (not fatal)": log_has("TRUNCATED DUE TO SIZE LIMIT: "),

        "old dpkg (<1.19.0, temporary)": log_has_re(r"dpkg is already the newest version \(1\.1[0-8].*fakeroot not found, either install the fakeroot"),

        "fakeroot not found (https://deb.li/3P46G)": log_has("fakeroot not found, either install the fakeroot"),

        "dpkg-buildpackage failed": log_has("E: Build failure (dpkg-buildpackage died)"),

        "diffoscope timeout (not fatal)": diff_has("TRUNCATED DUE TO TIMEOUT: 600 seconds"),

        "failed to reproduce: R package (#1089197)": log_has_re(r"(Source|Binary): r-(cran|bioc|other)-"),

        "failed to reproduce: dh_buildinfo (#1068809)": diff_has_any([f"buildinfo_{arch}.gz", "buildinfo_all.gz"]),

        "failed to reproduce": log_has_re(r"checking [^ ]*: (size(...|(value of [^ ]* differs for [^ ]*)))? ?$"),

        "diffoscope not found (fixed)": log_has("rebuilderd: unexpected error while rebuilding package: Failed to run diffoscope: No such file or directory (os error 2)"),

        "rebuilderd error": log_has_any([".deb: size... md5... sha256... sha1... all OK", "deb: size... md5... sha1... sha256... all OK"]),

    }

    messages_packages = defaultdict(list)

    for error, message in error_messages.items():

    messages_packages = {k: [] for k in error_messages.keys()} # Preserve keys order

    for row in cu.execute(

            "SELECT p.name FROM packages p LEFT JOIN builds b ON b.id = p.build_id"

            f" WHERE p.status = 'BAD' and {error}"

        "SELECT p.name, CAST(b.build_log AS TEXT), b.diffoscope FROM packages p LEFT JOIN builds b"

        " ON b.id = p.build_id WHERE p.status = 'BAD'"

    ):

        for message, error_match_fn in error_messages.items():

            if error_match_fn(row[1], row[2]):

                messages_packages[message].append(row[0])

                break

    messages_packages = {k: v for k, v in messages_packages.items() if v} # Remove empty categories

    package_logs = dict()

    bad_packages = set()