Commit c8b1bac2 authored by Jochen Sprickerhof's avatar Jochen Sprickerhof Committed by Holger Levsen
Browse files

rdn stats: add a page sorted by diffoscope size 



(cherry picked from commit 5589afc2064a68c756c1c8d466b91dfb0dc0556a)
Signed-off-by: default avatarHolger Levsen <holger@layer-acht.org>
parent 52af44ce

bin/rebuilderd_stats.py

+28 −7
Original line number Diff line number Diff line
@@ -44,6 +44,11 @@ def main() -> None: 
        type=Path,

        help="Output HTML sorted by build date",

    )

    parser.add_argument(

        "sizefile",

        type=Path,

        help="Output HTML sorted by diffoscope size",

    )

    args = parser.parse_args()

    cx = connect(args.db)

    cx.create_function("regexp", 2, lambda x, y: 1 if search(x, y) else 0)
@@ -147,17 +152,21 @@ def main() -> None: 


        if build_id in cache:

            cache_new[build_id] = cache[build_id]

            messages_packages[cache[build_id]].append({"name": name, "src": src_name, "id": build_id, "diff": diffoscope is not None})

            message, diffsize = cache[build_id]

            messages_packages[message].append({"name": name, "src": src_name, "id": build_id, "diff": diff_size})

            continue



        build_log = unzstd.stream_reader(build_log).read().decode("utf-8")

        if diffoscope:

            diffoscope = unzstd.stream_reader(diffoscope).read().decode("utf-8")

        else:

            diffoscope = ""



        for message, error_match_fn in error_messages.items():

            if error_match_fn(build_log, diffoscope):

                cache_new[build_id] = message

                messages_packages[message].append({"name": name, "src": src_name, "id": build_id, "diff": diffoscope is not None})

                difflen = len(diffoscope)

                cache_new[build_id] = (message, difflen)

                messages_packages[message].append({"name": name, "src": src_name, "id": build_id, "diff": difflen})

                break



    messages_packages = {k: v for k, v in messages_packages.items() if v}  # Remove empty categories
@@ -168,10 +177,12 @@ def main() -> None: 


    db_size = path.getsize(args.db)

    with args.datefile.open("w") as outfile:

        output(outfile, args.arch, db_size, total, messages_packages)

        output(outfile, args.arch, db_size, total, messages_packages, False)

    with args.sizefile.open("w") as outfile:

        output(outfile, args.arch, db_size, total, messages_packages, True)





def output(outfile, arch, db_size, total, messages_packages):

def output(outfile, arch, db_size, total, messages_packages, size_sort):

    with urlopen("https://salsa.debian.org/reproducible-builds/reproducible-notes/-/raw/master/packages.yml?ref_type=heads") as rn:

        rn_bugs = safe_load(rn)
@@ -208,8 +219,18 @@ def output(outfile, arch, db_size, total, messages_packages): 
    )

    print("</table>", file=outfile)



    if size_sort:

        print('<p>Packages are grouped by having a bug in <a href="https://salsa.debian.org/reproducible-builds/reproducible-notes">reproducible-notes</a> and sorted by diffoscope size (bigger logs later) inside the group.</p>', file=outfile)

    else:

        print('<p>Packages are grouped by having a bug in <a href="https://salsa.debian.org/reproducible-builds/reproducible-notes">reproducible-notes</a> and sorted by build time (newer logs later) inside the group.</p>', file=outfile)



    def sorter(pkg):

        if pkg["src"] in rn_bugs and "bugs" in rn_bugs[pkg["src"]]:

            return -1

        if size_sort:

            return pkg["diff"]

        return 1



    if arch not in ['amd64', 'arm64', 'armhf', 'i386']:

        ci_arch = 'amd64'

    else:
@@ -222,7 +243,7 @@ def output(outfile, arch, db_size, total, messages_packages): 
        span = 0

        bug_break = False

        print("<p><span>", file=outfile)

        for pkg in sorted(packages, key=lambda pkg: pkg["src"] in rn_bugs and "bugs" in rn_bugs[pkg["src"]], reverse=True):

        for pkg in sorted(packages, key=sorter):

            if not bug_break and not (pkg["src"] in rn_bugs and "bugs" in rn_bugs[pkg["src"]]):

                bug_break = True

                print("</span></p><p><span>", file=outfile)

hosts/osuosl5-amd64/etc/cron.d/rebuilderd

+1 −1
Original line number Diff line number Diff line 
MAILTO=root

42 */3 * * * rebuilderd for arch in all amd64 arm64 armel armhf i386 ppc64el riscv64 ; do mkdir -p /srv/rebuilderd/$arch/stats/ && /srv/jenkins/bin/rebuilderd_stats.py $arch /srv/rebuilderd/$arch/rebuilderd.db --cache /srv/rebuilderd/$arch/stats/cache.pickle /srv/rebuilderd/$arch/stats/index.html.tmp && mv /srv/rebuilderd/$arch/stats/index.html.tmp /srv/rebuilderd/$arch/stats/index.html ; done

42 */3 * * * rebuilderd for arch in all amd64 arm64 armel armhf i386 ppc64el riscv64 ; do mkdir -p /srv/rebuilderd/$arch/stats/ && /srv/jenkins/bin/rebuilderd_stats.py $arch /srv/rebuilderd/$arch/rebuilderd.db --cache /srv/rebuilderd/$arch/stats/cache.pickle /srv/rebuilderd/$arch/stats/index.html.tmp /srv/rebuilderd/$arch/stats/index_date.html.tmp && mv /srv/rebuilderd/$arch/stats/index.html.tmp /srv/rebuilderd/$arch/stats/index.html && mv /srv/rebuilderd/$arch/stats/index_date.html.tmp /srv/rebuilderd/$arch/stats/index_date.html ; done

23 0,6,12,18 * * * rebuilderd for arch in all amd64 arm64 armel armhf i386 ppc64el riscv64 ; do /srv/jenkins/bin/rebuilderd_graph.sh $arch ; done