reproducible_html_live_status.py 8.87 KB
Newer Older
1 2 3
#!/usr/bin/python3
# -*- coding: utf-8 -*-
#
4
# Copyright © 2015-2017 Holger Levsen <holger@layer-acht.org>
5
#           © 2018      Mattia Rizzolo <mattia@mapreri.org>
6 7 8 9 10
# based on ~jenkins.d.n:~mattia/status.sh by Mattia Rizzolo <mattia@mapreri.org>
# Licensed under GPL-2
#
# Depends: python3

11
from string import Template
12
from sqlalchemy import select, func, cast, Integer, and_, bindparam
13

14
from rblib import query_db, db_table
15
from rblib.confparse import log
16
from rblib.models import Package, Status
17
from rblib.utils import convert_into_hms_string
18 19 20 21 22 23 24
from rblib.html import tab, create_main_navigation, write_html_page
from reproducible_html_indexes import build_leading_text_section
from rblib.const import (
    DISTRO_BASE, DISTRO_URL, DISTRO_URI,
    ARCHS, SUITES,
    defaultsuite,
)
25

26 27 28 29 30 31
# sqlalchemy table definitions needed for queries
results = db_table('results')
sources = db_table('sources')
schedule = db_table('schedule')
stats_build = db_table('stats_build')

32 33
def convert_into_status_html(statusname):
    if statusname == 'None':
34
        return ''
35 36 37
    status = Status.get(statusname)
    return '{n} <img src="/static/{icon}" alt="{n}" title="{n}" />'.format(
            n=status.value.name, icon=status.value.icon)
38 39


40 41 42 43
def generate_schedule(arch):
    """ the schedule pages are very different than others index pages """
    log.info('Building the schedule index page for ' + arch + '...')
    title = 'Packages currently scheduled on ' + arch + ' for testing for build reproducibility'
44 45 46 47 48 49 50 51

    # 'AND h.name=s.name AND h.suite=s.suite AND h.architecture=s.architecture'
    # in this query and the query below is needed due to not using package_id
    # in the stats_build table, which should be fixed...
    averagesql = select([
        func.coalesce(func.avg(cast(stats_build.c.build_duration, Integer)), 0)
    ]).where(
        and_(
52
            stats_build.c.status.in_(('reproducible', 'FTBR')),
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
            stats_build.c.name == sources.c.name,
            stats_build.c.suite == sources.c.suite,
            stats_build.c.architecture == sources.c.architecture,
        )
    ).as_scalar()

    query = select([
        schedule.c.date_scheduled,
        sources.c.suite,
        sources.c.architecture,
        sources.c.name,
        results.c.status,
        results.c.build_duration,
        averagesql
    ]).select_from(
        sources.join(schedule).join(results, isouter=True)
    ).where(
        and_(
            schedule.c.date_build_started == None,
            sources.c.architecture == bindparam('arch'),
        )
    ).order_by(
        schedule.c.date_scheduled
    )

78 79
    text = Template('$tot packages are currently scheduled for testing on $arch:')
    html = ''
80
    rows = query_db(query.params({'arch': arch}))
81
    html += build_leading_text_section({'text': text}, rows, defaultsuite, arch)
82
    html += generate_live_status_table(arch)
83
    html += '<p><table class="scheduled">\n' + tab
84
    html += '<tr><th class="center">#</th><th class="center">scheduled at</th><th class="center">suite</th>'
85
    html += '<th class="center">arch</th><th class="center">source package</th><th class="center">previous build status</th><th class="center">previous build duration</th><th class="center">average build duration</th></tr>\n'
86
    for row in rows:
87
        # 0: date_scheduled, 1: suite, 2: arch, 3: pkg name 4: previous status 5: previous build duration 6. avg build duration
88
        pkg = row[3]
89
        duration = convert_into_hms_string(row[5])
90
        avg_duration = convert_into_hms_string(row[6])
91 92
        html += tab + '<tr><td>&nbsp;</td><td>' + row[0] + '</td>'
        html += '<td>' + row[1] + '</td><td>' + row[2] + '</td><td><code>'
93
        html += Package(pkg).html_link(row[1], row[2])
94
        html += '</code></td><td>'+convert_into_status_html(str(row[4]))+'</td><td>'+duration+'</td><td>' + avg_duration + '</td></tr>\n'
95
    html += '</table></p>\n'
96 97 98
    destfile = DISTRO_BASE + '/index_' + arch + '_scheduled.html'
    desturl = DISTRO_URL + '/index_' + arch + '_scheduled.html'
    suite_arch_nav_template = DISTRO_URI + '/index_{{arch}}_scheduled.html'
99 100
    left_nav_html = create_main_navigation(arch=arch, no_suite=True,
        displayed_page='scheduled', suite_arch_nav_template=suite_arch_nav_template)
101 102
    write_html_page(title=title, body=html, destfile=destfile, style_note=True,
                    refresh_every=60, left_nav_html=left_nav_html)
103 104
    log.info("Page generated at " + desturl)

105

106
def generate_live_status_table(arch):
107 108 109 110
    averagesql = select([
        func.coalesce(func.avg(cast(stats_build.c.build_duration, Integer)), 0)
    ]).where(
        and_(
111
            stats_build.c.status.in_(('reproducible', 'FTBR')),
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
            stats_build.c.name == sources.c.name,
            stats_build.c.suite == sources.c.suite,
            stats_build.c.architecture == sources.c.architecture,
        )
    ).as_scalar()

    query = select([
        sources.c.id,
        sources.c.suite,
        sources.c.architecture,
        sources.c.name,
        sources.c.version,
        schedule.c.date_build_started,
        results.c.status,
        results.c.build_duration,
        averagesql,
        schedule.c.job,
    ]).select_from(
        sources.join(schedule).join(results, isouter=True)
    ).where(
        and_(
            schedule.c.date_build_started != None,
            sources.c.architecture == bindparam('arch'),
        )
    ).order_by(
        schedule.c.date_scheduled
    )
139
    html = ''
140
    rows = query_db(query.params({'arch': arch}))
141
    html += '<p><table class="scheduled">\n' + tab
142 143 144 145
    html += '<tr><th class="center">#</th><th class="center">src pkg id</th><th class="center">suite</th><th class="center">arch</th>'
    html += '<th class=\"center\">source package</th><th class=\"center\">version</th></th>'
    html += '<th class=\"center\">build started</th><th class=\"center\">previous build status</th>'
    html += '<th class=\"center\">previous build duration</th><th class=\"center\">average build duration</th><th class=\"center\">builder job</th>'
146
    html += '</tr>\n'
147
    counter = 0
148
    for row in rows:
149
        counter += 1
150 151 152
        suite = row[1]
        arch = row[2]
        pkg = row[3]
153
        duration = convert_into_hms_string(row[7])
154
        avg_duration = convert_into_hms_string(row[8])
155
        html += tab + '<tr><td>&nbsp;</td><td>' + str(row[0]) + '</td>'
156
        html += '<td>' + suite + '</td><td>' + arch + '</td>'
157
        html += '<td><code>' + Package(pkg).html_link(suite, arch, bugs=False) + '</code></td>'
158
        html += '<td>' + str(row[4]) + '</td><td>' + str(row[5]) + '</td>'
159
        html += '<td>' + convert_into_status_html(str(row[6])) + '</td><td>' + duration + '</td><td>' + avg_duration + '</td>'
160
        html += '<td><a href="https://tests.reproducible-builds.org/cgi-bin/nph-logwatch?' + str(row[9]) + '">' + str(row[9]) + '</a></td>'
161 162
        html += '</tr>\n'
    html += '</table></p>\n'
163
    return html
164

165 166 167 168 169
def generate_oldies(arch):
    log.info('Building the oldies page for ' + arch + '...')
    title = 'Oldest results on ' + arch
    html = ''
    for suite in SUITES:
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
        query = select([
            sources.c.suite,
            sources.c.architecture,
            sources.c.name,
            results.c.status,
            results.c.build_date
        ]).select_from(
            results.join(sources)
        ).where(
            and_(
                sources.c.suite == bindparam('suite'),
                sources.c.architecture == bindparam('arch'),
                results.c.status != 'blacklisted'
            )
        ).order_by(
            results.c.build_date
        ).limit(15)
187
        text = Template('Oldest results on $suite/$arch:')
188
        rows = query_db(query.params({'arch': arch, 'suite': suite}))
189 190 191 192 193 194 195 196 197
        html += build_leading_text_section({'text': text}, rows, suite, arch)
        html += '<p><table class="scheduled">\n' + tab
        html += '<tr><th class="center">#</th><th class="center">suite</th><th class="center">arch</th>'
        html += '<th class="center">source package</th><th class="center">status</th><th class="center">build date</th></tr>\n'
        for row in rows:
            # 0: suite, 1: arch, 2: pkg name 3: status 4: build date
            pkg = row[2]
            html += tab + '<tr><td>&nbsp;</td><td>' + row[0] + '</td>'
            html += '<td>' + row[1] + '</td><td><code>'
198
            html += Package(pkg).html_link(row[0], row[1])
199 200
            html += '</code></td><td>'+convert_into_status_html(str(row[3]))+'</td><td>' + row[4] + '</td></tr>\n'
        html += '</table></p>\n'
201 202
    destfile = DISTRO_BASE + '/index_' + arch + '_oldies.html'
    desturl = DISTRO_URL + '/index_' + arch + '_oldies.html'
203 204 205
    left_nav_html = create_main_navigation(arch=arch)
    write_html_page(title=title, body=html, destfile=destfile, style_note=True,
                    refresh_every=60, left_nav_html=left_nav_html)
206 207
    log.info("Page generated at " + desturl)

208
if __name__ == '__main__':
209 210
    for arch in ARCHS:
        generate_schedule(arch)
211
        generate_oldies(arch)