reproducible_common.py 26 KB
Newer Older
1
2
#!/usr/bin/python3
# -*- coding: utf-8 -*-
3
#
4
# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
5
6
# Based on the reproducible_common.sh by © 2014 Holger Levsen <holger@layer-acht.org>
# Licensed under GPL-2
7
#
8
# Depends: python3 python3-psycopg2
9
#
10
# This is included by all reproducible_*.py scripts, it contains common functions
11
12
13
14

import os
import re
import sys
15
import json
16
import errno
17
import atexit
18
19
20
import sqlite3
import logging
import argparse
21
import psycopg2
22
import html as HTML
23
from string import Template
24
from subprocess import call
25
from traceback import print_exception
26
from datetime import datetime, timedelta
27
28
29
30

DEBUG = False
QUIET = False

31
# tested suites
32
SUITES = ['testing', 'unstable', 'experimental']
33
# tested architectures
Holger Levsen's avatar
Holger Levsen committed
34
ARCHS = ['amd64', 'armhf']
35
36
37
# defaults
defaultsuite = 'unstable'
defaultarch = 'amd64'
38

39
BIN_PATH = '/srv/jenkins/bin'
40
BASE = '/var/lib/jenkins/userContent/reproducible'
41

42
REPRODUCIBLE_JSON = BASE + '/reproducible.json'
43
REPRODUCIBLE_TRACKER_JSON = BASE + '/reproducible-tracker.json'
44
REPRODUCIBLE_DB = '/var/lib/jenkins/reproducible.db'
45

46
DBD_URI = '/dbd'
47
DBDTXT_URI = '/dbdtxt'
48
LOGS_URI = '/logs'
49
DIFFS_URI = '/logdiffs'
50
NOTES_URI = '/notes'
51
ISSUES_URI = '/issues'
52
RB_PKG_URI = '/rb-pkg'
53
54
55
RBUILD_URI = '/rbuild'
BUILDINFO_URI = '/buildinfo'
DBD_PATH = BASE + DBD_URI
56
DBDTXT_PATH = BASE + DBDTXT_URI
57
LOGS_PATH = BASE + LOGS_URI
58
DIFFS_PATH = BASE + DIFFS_URI
59
60
61
NOTES_PATH = BASE + NOTES_URI
ISSUES_PATH = BASE + ISSUES_URI
RB_PKG_PATH = BASE + RB_PKG_URI
62
63
RBUILD_PATH = BASE + RBUILD_URI
BUILDINFO_PATH = BASE + BUILDINFO_URI
64
65
66
67
68
69
70
71

REPRODUCIBLE_URL = 'https://reproducible.debian.net'
JENKINS_URL = 'https://jenkins.debian.net'

parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()
group.add_argument("-d", "--debug", action="store_true")
group.add_argument("-q", "--quiet", action="store_true")
72
73
parser.add_argument("--ignore-missing-files", action="store_true",
                    help="useful for local testing, where you don't have all the build logs, etc..")
74
args, unknown_args = parser.parse_known_args()
75
76
log_level = logging.INFO
if args.debug or DEBUG:
77
    DEBUG = True
78
79
80
    log_level = logging.DEBUG
if args.quiet or QUIET:
    log_level = logging.ERROR
81
82
83
84
85
86
log = logging.getLogger(__name__)
log.setLevel(log_level)
sh = logging.StreamHandler()
sh.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
log.addHandler(sh)

87
88
started_at = datetime.now()
log.info('Starting at %s', started_at)
89
90

log.debug("BIN_PATH:\t" + BIN_PATH)
91
92
93
log.debug("BASE:\t\t" + BASE)
log.debug("DBD_URI:\t\t" + DBD_URI)
log.debug("DBD_PATH:\t" + DBD_PATH)
94
95
log.debug("DBDTXT_URI:\t" + DBDTXT_URI)
log.debug("DBDTXT_PATH:\t" + DBDTXT_PATH)
96
97
log.debug("LOGS_URI:\t" + LOGS_URI)
log.debug("LOGS_PATH:\t" + LOGS_PATH)
98
99
log.debug("DIFFS_URI:\t" + DIFFS_URI)
log.debug("DIFFS_PATH:\t" + DIFFS_PATH)
100
101
102
103
104
105
log.debug("NOTES_URI:\t" + NOTES_URI)
log.debug("ISSUES_URI:\t" + ISSUES_URI)
log.debug("NOTES_PATH:\t" + NOTES_PATH)
log.debug("ISSUES_PATH:\t" + ISSUES_PATH)
log.debug("RB_PKG_URI:\t" + RB_PKG_URI)
log.debug("RB_PKG_PATH:\t" + RB_PKG_PATH)
106
107
108
109
log.debug("RBUILD_URI:\t" + RBUILD_URI)
log.debug("RBUILD_PATH:\t" + RBUILD_PATH)
log.debug("BUILDINFO_URI:\t" + BUILDINFO_URI)
log.debug("BUILDINFO_PATH:\t" + BUILDINFO_PATH)
110
111
112
113
log.debug("REPRODUCIBLE_DB:\t" + REPRODUCIBLE_DB)
log.debug("REPRODUCIBLE_JSON:\t" + REPRODUCIBLE_JSON)
log.debug("JENKINS_URL:\t\t" + JENKINS_URL)
log.debug("REPRODUCIBLE_URL:\t" + REPRODUCIBLE_URL)
114

115
116
if args.ignore_missing_files:
    log.warning("Missing files will be ignored!")
117
118
119

tab = '  '

120
html_header = Template("""<!DOCTYPE html>
121
122
<html>
  <head>
123
124
      <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
      <meta name="viewport" content="width=device-width" />
125
      <link href="/static/style.css" type="text/css" rel="stylesheet" />
126
127
      <title>$page_title</title>
  </head>
128
  <body $padding>""")
129
130
131
132
133
134
135
html_footer = Template("""
    <hr />
    <p style="font-size:0.9em;">
      There is more information <a href="%s/userContent/about.html">about
      jenkins.debian.net</a> and about
      <a href="https://wiki.debian.org/ReproducibleBuilds"> reproducible builds
      of Debian</a> available elsewhere. Last update: $date.
136
      Copyright 2014-2015 <a href="mailto:holger@layer-acht.org">Holger Levsen</a> and others,
137
138
139
140
141
      GPL-2 licensed. The weather icons are public domain and have been taken
      from the <a href=http://tango.freedesktop.org/Tango_Icon_Library target=_blank>
      Tango Icon Library</a>.
     </p>
  </body>
142
</html>""" % (JENKINS_URL))
143
144
145
146

html_head_page = Template((tab*2).join("""
<header>
  <h2>$page_title</h2>
147
  <nav><ul>
148
149
    <li>Have a look at:</li>
    <li>
150
      <a href="/$suite/$arch/index_reproducible.html" target="_parent">
151
        <img src="/static/weather-clear.png" alt="reproducible icon" />
152
      </a>
153
154
    </li>
    <li>
155
      <a href="/$suite/$arch/index_FTBR.html" target="_parent">
156
        <img src="/static/weather-showers-scattered.png" alt="FTBR icon" />
157
      </a>
158
159
    </li>
    <li>
160
      <a href="/$suite/$arch/index_FTBFS.html" target="_parent">
161
        <img src="/static/weather-storm.png" alt="FTBFS icon" />
162
      </a>
163
164
    </li>
    <li>
165
166
      <a href="/$suite/$arch/index_depwait.html" target="_parent">
        <img src="/static/weather-snow.png" alt="depwait icon" />
167
      </a>
168
169
    </li>
    <li>
170
      <a href="/$suite/$arch/index_not_for_us.html" target="_parent">
171
        <img src="/static/weather-few-clouds-night.png" alt="not_for_us icon" />
172
      </a>
173
174
    </li>
    <li>
175
176
      <a href="/$suite/$arch/index_404.html" target="_parent">
        <img src="/static/weather-severe-alert.png" alt="404 icon" />
177
      </a>
178
    </li>
179
    <li>
180
181
      <a href="/$suite/$arch/index_blacklisted.html" target="_parent">
        <img src="/static/error.png" alt="blacklisted icon" />
182
183
      </a>
    </li>
184
    <li><a href="/index_issues.html">issues</a></li>
185
186
    <li><a href="/$suite/$arch/index_notes.html">packages with notes</a></li>
    <li><a href="/$suite/$arch/index_no_notes.html">packages without notes</a></li>
187
    <li><a href="/index_scheduled.html">currently scheduled</a></li>
188
$links
189
    <li><a href="/index_repositories.html">repositories overview</a></li>
Holger Levsen's avatar
Holger Levsen committed
190
    <li><a href="/reproducible.html">reproducible stats</a></li>
191
    <li><a href="https://wiki.debian.org/ReproducibleBuilds" target="_blank">wiki</a></li>
192
  </ul></nav>
193
</header>""".splitlines(True)))
194

195

196
html_foot_page_style_note = Template((tab*2).join("""
197
198
199
<p style="font-size:0.9em;">
  A package name displayed with a bold font is an indication that this
  package has a note. Visited packages are linked in green, those which
200
  have not been visited are linked in blue.<br />
201
202
203
  A <code><span class="bug">&#35;</span></code> sign after the name of a
  package indicates that a bug is filed against it. Likewise, a
  <code><span class="bug-patch">&#43;</span></code> sign indicates there is
204
205
  a patch available, a <code><span class="bug-pending">P</span></code> means a
  pending bug while <code><span class="bug-done">&#35;</span></code>
206
  indicates a closed bug. In cases of several bugs, the symbol is repeated.
207
</p>""".splitlines(True)))
208
209
210
211


url2html = re.compile(r'((mailto\:|((ht|f)tps?)\://|file\:///){1}\S+)')

212
# filter used on the index_FTBFS pages and for the reproducible.json
213
filtered_issues = (
214
    'bad_handling_of_extra_warnings',
215
216
    'ftbfs_wdatetime',
    'ftbfs_wdatetime_due_to_swig',
217
218
219
220
221
    'ftbfs_pbuilder_malformed_dsc',
    'ftbfs_in_jenkins_setup',
    'ftbfs_build_depends_not_available_on_amd64',
    'ftbfs_due_to_root_username',
    'ftbfs_due_to_virtual_dependencies')
222
223
224
225
226
227
228
229
230
filter_query = ''
for issue in filtered_issues:
    if filter_query == '':
        filter_query = 'n.issues LIKE "%' + issue + '%"'
        filter_html = '<a href="' + REPRODUCIBLE_URL + ISSUES_URI + '/$suite/' + issue + '_issue.html">' + issue + '</a>'
    else:
        filter_query += ' OR n.issues LIKE "%' + issue + '%"'
        filter_html += ' or <a href="' + REPRODUCIBLE_URL + ISSUES_URI + '/$suite/' + issue + '_issue.html">' + issue + '</a>'

231

232
233
234
235
236
237
@atexit.register
def print_time():
    log.info('Finished at %s, took: %s', datetime.now(),
             datetime.now()-started_at)


238
239
def print_critical_message(msg):
    print('\n\n\n')
240
241
242
243
244
    try:
        for line in msg.splitlines():
            log.critical(line)
    except AttributeError:
        log.critical(msg)
245
246
    print('\n\n\n')

247

248
249
250
251
252
253
254
255
256
257
class bcolors:
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'
    RED = '\033[91m'
    GOOD = '\033[92m'
    WARN = '\033[93m' + UNDERLINE
    FAIL = RED + BOLD + UNDERLINE
    ENDC = '\033[0m'


258
def _gen_links(suite, arch):
259
260
261
262
    links = [
        ('last_24h', '<li><a href="/{suite}/{arch}/index_last_24h.html">packages tested in the last 24h</a></li>'),
        ('last_48h', '<li><a href="/{suite}/{arch}/index_last_48h.html">packages tested in the last 48h</a></li>'),
        ('all_abc', '<li><a href="/{suite}/{arch}/index_all_abc.html">all tested packages (sorted alphabetically)</a></li>'),
263
        ('notify', '<li><a href="/index_notify.html" title="notify icon">⚑</a></li>'),
264
        ('dd-list', '<li><a href="/{suite}/index_dd-list.html">maintainers of unreproducible packages</a></li>'),
265
266
267
268
        ('pkg_sets', '<li><a href="/{suite}/{arch}/index_pkg_sets.html">package sets stats</a></li>')
    ]
    html = ''
    for link in links:
269
270
        if link[0] == 'pkg_sets' and suite == 'experimental':
            html += link[1].format(suite=defaultsuite, arch=arch) + '\n'
271
            continue
272
        html += link[1].format(suite=suite, arch=arch) + '\n'
273
    for i in SUITES:  # suite links
274
275
        if arch == 'armhf' and i != 'unstable':
            continue
Holger Levsen's avatar
Holger Levsen committed
276
        html += '<li><a href="/' + i + '/index_suite_' + arch + '_stats.html">suite: ' + i + '</a></li>'
277
    if arch == 'amd64':
278
        html += '<li><a href="/unstable/index_suite_armhf_stats.html\">arch: armhf</a></li>'
Mattia Rizzolo's avatar
Mattia Rizzolo committed
279
    else:
280
        html += '<li><a href="/unstable/index_suite_amd64_stats.html\">arch: amd64</a></li>'
281
    return html
282
283


284
def write_html_page(title, body, destfile, suite=defaultsuite, arch=defaultarch, noheader=False, style_note=False, noendpage=False, packages=False):
285
    now = datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC')
286
    html = ''
287
288
289
290
291
    # this removes the padding if we are writing a package page
    padding = 'class="wrapper"' if packages else ''
    html += html_header.substitute(
            page_title=title,
            padding=padding)
292
    if not noheader:
293
        links = _gen_links(suite, arch)
294
295
        html += html_head_page.substitute(
            page_title=title,
296
297
            suite=suite,
            arch=arch,
298
            links=links)
299
    html += body
300
301
    if style_note:
        html += html_foot_page_style_note.substitute()
302
303
304
305
    if not noendpage:
        html += html_footer.substitute(date=now)
    else:
        html += '</body>\n</html>'
306
307
308
309
310
    try:
        os.makedirs(destfile.rsplit('/', 1)[0], exist_ok=True)
    except OSError as e:
        if e.errno != errno.EEXIST:  # that's 'File exists' error (errno 17)
            raise
311
    with open(destfile, 'w', encoding='UTF-8') as fd:
312
313
        fd.write(html)

314
def start_db_connection():
315
    return sqlite3.connect(REPRODUCIBLE_DB, timeout=60)
316
317

def query_db(query):
318
    cursor = conn_db.cursor()
319
320
321
322
323
    try:
        cursor.execute(query)
    except:
        print_critical_message('Error execting this query:\n' + query)
        raise
324
    conn_db.commit()
325
326
    return cursor.fetchall()

327
328
329
330
331
332
333
def start_udd_connection():
    username = "public-udd-mirror"
    password = "public-udd-mirror"
    host = "public-udd-mirror.xvm.mit.edu"
    port = 5432
    db = "udd"
    try:
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
        try:
            log.debug("Starting connection to the UDD database")
            conn = psycopg2.connect(
                database=db,
                user=username,
                host=host,
                password=password,
                connect_timeout=5,
            )
        except psycopg2.OperationalError as err:
            if str(err) == 'timeout expired\n':
                log.error('Connection to the UDD database replice timed out. '
                          'Maybe the machine is offline or just unavailable.')
                log.error('Failing nicely anyway, all queries will return an '
                          'empty response.')
                return None
            else:
                raise
352
    except:
353
354
355
356
357
358
359
        log.error('Erorr connecting to the UDD database replica.' +
                  'The full error is:')
        exc_type, exc_value, exc_traceback = sys.exc_info()
        print_exception(exc_type, exc_value, exc_traceback)
        log.error('Failing nicely anyway, all queries will return an empty ' +
                  'response.')
        return None
360
361
362
363
    conn.set_client_encoding('utf8')
    return conn

def query_udd(query):
364
365
366
367
368
    if not conn_udd:
        log.error('There has been an error connecting to the UDD database. ' +
                  'Please look for a previous error for more information.')
        log.error('Failing nicely anyway, returning an empty response.')
        return []
369
    cursor = conn_udd.cursor()
370
371
372
373
374
375
376
377
378
    try:
        cursor.execute(query)
    except:
        log.error('The UDD server encountered a issue while executing the ' +
                  'query. The full error is:')
        exc_type, exc_value, exc_traceback = sys.exc_info()
        print_exception(exc_type, exc_value, exc_traceback)
        log.error('Failing nicely anyway, returning an empty response.')
        return []
379
380
    return cursor.fetchall()

381

382
383
384
385
386
387
388
389
def package_has_notes(package):
    # not a really serious check, it'd be better to check the yaml file
    path = NOTES_PATH + '/' + package + '_note.html'
    if os.access(path, os.R_OK):
        return True
    else:
        return False

390

391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
def link_package(package, suite, arch, bugs={}):
    url = RB_PKG_URI + '/' + suite + '/' + arch + '/' + package + '.html'
    query = 'SELECT n.issues, n.bugs, n.comments ' + \
            'FROM notes AS n JOIN sources AS s ON s.id=n.package_id ' + \
            'WHERE s.name="{pkg}" AND s.suite="{suite}" ' + \
            'AND s.architecture="{arch}"'
    try:
        notes = query_db(query.format(pkg=package, suite=suite, arch=arch))[0]
    except IndexError:  # no notes for this package
        html = '<a href="' + url + '" class="package">' + package  + '</a>'
    else:
        title = ''
        for issue in json.loads(notes[0]):
            title += issue + '\n'
        for bug in json.loads(notes[1]):
            title += '#' + str(bug) + '\n'
        if notes[2]:
            title += notes[2]
        title = HTML.escape(title.strip())
        html = '<a href="' + url + '" class="noted" title="' + title + \
               '">' + package + '</a>'
    finally:
413
        html += get_trailing_icon(package, bugs) + '\n'
414
415
416
    return html


417
418
419
420
421
422
423
424
def link_packages(packages, suite, arch):
    bugs = get_bugs()
    html = ''
    for pkg in packages:
        html += link_package(pkg, suite, arch, bugs)
    return html


425
426
427
def join_status_icon(status, package=None, version=None):
    table = {'reproducible' : 'weather-clear.png',
             'FTBFS': 'weather-storm.png',
428
             'FTBR' : 'weather-showers-scattered.png',
429
             '404': 'weather-severe-alert.png',
430
             'depwait': 'weather-snow.png',
431
432
             'not for us': 'weather-few-clouds-night.png',
             'not_for_us': 'weather-few-clouds-night.png',
433
             'untested': 'weather-clear-night.png',
434
435
436
             'blacklisted': 'error.png'}
    if status == 'unreproducible':
            status = 'FTBR'
437
438
    elif status == 'not for us':
            status = 'not_for_us'
439
440
441
442
443
444
445
446
447
    log.debug('Linking status ⇔ icon. package: ' + str(package) + ' @ ' +
              str(version) + ' status: ' + status)
    try:
        return (status, table[status])
    except KeyError:
        log.error('Status of package ' + package + ' (' + status +
                  ') not recognized')
        return (status, '')

448
449
450
451
452
453
def strip_epoch(version):
    """
    Stip the epoch out of the version string. Some file (e.g. buildlogs, debs)
    do not have epoch in their filenames.
    """
    try:
454
        return version.split(':', 1)[1]
455
456
457
    except IndexError:
        return version

458
def pkg_has_buildinfo(package, version=False, suite=defaultsuite, arch=defaultarch):
459
460
461
462
463
    """
    if there is no version specified it will use the version listed in
    reproducible.db
    """
    if not version:
464
        query = 'SELECT r.version ' + \
465
                'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' + \
466
467
                'WHERE s.name="{}" AND s.suite="{}" AND s.architecture="{}"'
        query = query.format(package, suite, arch)
468
        version = str(query_db(query)[0][0])
469
    buildinfo = BUILDINFO_PATH + '/' + suite + '/' + arch + '/' + package + \
470
                '_' + strip_epoch(version) + '_amd64.buildinfo'
471
472
473
474
    if os.access(buildinfo, os.R_OK):
        return True
    else:
        return False
475

476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493

def pkg_has_rbuild(package, version=False, suite=defaultsuite, arch=defaultarch):
    if not version:
        query = 'SELECT r.version ' + \
                'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' + \
                'WHERE s.name="{}" AND s.suite="{}" AND s.architecture="{}"'
        query = query.format(package, suite, arch)
        version = str(query_db(query)[0][0])
    rbuild = RBUILD_PATH + '/' + suite + '/' + arch + '/' + package + '_' + \
             strip_epoch(version) + '.rbuild.log'
    if os.access(rbuild, os.R_OK):
        return (rbuild, os.stat(rbuild).st_size)
    elif os.access(rbuild+'.gz', os.R_OK):
        return (rbuild+'.gz', os.stat(rbuild+'.gz').st_size)
    else:
        return ()


494
495
496
497
498
499
500
501
502
503
def get_bugs():
    """
    This function returns a dict:
    { "package_name": {
        bug1: {patch: True, done: False},
        bug2: {patch: False, done: False},
       }
    }
    """
    query = """
504
        SELECT bugs.id, bugs.source, bugs.done, ARRAY_AGG(tags.tag)
505
506
507
508
        FROM bugs JOIN bugs_tags ON bugs.id = bugs_tags.id
                  JOIN bugs_usertags ON bugs_tags.id = bugs_usertags.id
                  JOIN sources ON bugs.source=sources.source
                  LEFT JOIN (
509
510
                    SELECT id, tag FROM bugs_tags
                    WHERE tag='patch' OR tag='pending'
511
                  ) AS tags ON bugs.id = tags.id
512
513
514
515
516
517
518
519
520
        WHERE bugs_usertags.email = 'reproducible-builds@lists.alioth.debian.org'
        AND bugs.id NOT IN (
            SELECT id
            FROM bugs_usertags
            WHERE email = 'reproducible-builds@lists.alioth.debian.org'
            AND (
                bugs_usertags.tag = 'toolchain'
                OR bugs_usertags.tag = 'infrastructure')
            )
521
        GROUP BY bugs.id, bugs.source, bugs.done
522
523
    """
    # returns a list of tuples [(id, source, done)]
524
525
526
    global conn_udd
    if not conn_udd:
        conn_udd = start_udd_connection()
527
528
529
    global bugs
    if bugs:
        return bugs
530
531
532
533
534
535
536
    rows = query_udd(query)
    log.info("finding out which usertagged bugs have been closed or at least have patches")
    packages = {}

    for bug in rows:
        if bug[1] not in packages:
            packages[bug[1]] = {}
537
        # bug[0] = bug_id, bug[1] = source_name, bug[2] = who_when_done,
538
539
540
541
        # bug[3] = tag (patch or pending)
        packages[bug[1]][bug[0]] = {
            'done': False, 'patch': False, 'pending': False
        }
542
        if bug[2]:  # if the bug is done
543
            packages[bug[1]][bug[0]]['done'] = True
544
        if 'patch' in bug[3]:  # the bug is patched
545
            packages[bug[1]][bug[0]]['patch'] = True
546
547
        if 'pending' in bug[3]:  # the bug is pending
            packages[bug[1]][bug[0]]['pending'] = True
548
549
    return packages

550

551
552
553
554
def get_trailing_icon(package, bugs):
    html = ''
    if package in bugs:
        for bug in bugs[package]:
555
            html += '<a href="https://bugs.debian.org/{bug}">'.format(bug=bug)
556
557
558
            html += '<span class="'
            if bugs[package][bug]['done']:
                html += 'bug-done" title="#' + str(bug) + ', done">#</span>'
559
560
            elif bugs[package][bug]['pending']:
                html += 'bug-pending" title="#' + str(bug) + ', pending">P</span>'
561
562
563
            elif bugs[package][bug]['patch']:
                html += 'bug-patch" title="#' + str(bug) + ', with patch">+</span>'
            else:
564
                html += 'bug" title="#' + str(bug) + '">#</span>'
565
            html += '</a>'
566
567
568
    return html


569
570
571
572
573
574
575
576
577
578
579
580
def get_trailing_bug_icon(bug, bugs, package=None):
    html = ''
    if not package:
        for pkg in bugs.keys():
            if get_trailing_bug_icon(bug, bugs, pkg):
                return get_trailing_bug_icon(bug, bugs, pkg)
    else:
        try:
            if bug in bugs[package].keys():
                html += '<span class="'
                if bugs[package][bug]['done']:
                    html += 'bug-done" title="#' + str(bug) + ', done">#'
581
582
                elif bugs[package][bug]['pending']:
                    html += 'bug-pending" title="#' + str(bug) + ', pending">P'
583
584
                elif bugs[package][bug]['patch']:
                    html += 'bug-patch" title="#' + str(bug) + ', with patch">+'
585
586
                else:
                    html += 'bug">'
587
588
589
590
591
                html += '</span>'
        except KeyError:
            pass
    return html

592

593
594
595
596
597
598
def irc_msg(msg):
    kgb = ['kgb-client', '--conf', '/srv/jenkins/kgb/debian-reproducible.conf',
           '--relay-msg']
    kgb.extend(str(msg).strip().split())
    call(kgb)

599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675

class Bug:
    def __init__(self, bug):
        self.bug = bug

    def __str__(self):
        return str(self.bug)


class Issue:
    def __init__(self, name):
        self.name = name
        query = 'SELECT url, description  FROM issues WHERE name="{}"'
        result = query_db(query.format(self.name))
        try:
            self.url = result[0][0]
        except IndexError:
            self.url = ''
        try:
            self.desc = result[0][0]
        except IndexError:
            self.desc = ''


class Note:
    def __init__(self, pkg, results):
        log.debug(str(results))
        self.issues = [Issue(x) for x in json.loads(results[0])]
        self.bugs = [Bug(x) for x in json.loads(results[1])]
        self.comment = results[2]


class NotedPkg:
    def __init__(self, package, suite, arch):
        self.package = package
        self.suite = suite
        self.arch = arch
        query = 'SELECT n.issues, n.bugs, n.comments ' + \
                'FROM sources AS s JOIN notes AS n ON s.id=n.package_id ' + \
                'WHERE s.name="{}" AND s.suite="{}" AND s.architecture="{}"'
        result = query_db(query.format(self.package, self.suite, self.arch))
        try:
            result = result[0]
        except IndexError:
            self.note = None
        else:
            self.note = Note(self, result)

class Build:
    def __init__(self, package, suite, arch):
        self.package = package
        self.suite = suite
        self.arch = arch
        self.status = False
        self.version = False
        self.build_date = False
        self._get_package_status()

    def _get_package_status(self):
        try:
            query = 'SELECT r.status, r.version, r.build_date ' + \
                    'FROM results AS r JOIN sources AS s ' + \
                    'ON r.package_id=s.id WHERE s.name="{}" ' + \
                    'AND s.architecture="{}" AND s.suite="{}"'
            query = query.format(self.package, self.arch, self.suite)
            result = query_db(query)[0]
        except IndexError:  # not tested, look whether it actually exists
            query = 'SELECT version FROM sources WHERE name="{}" ' + \
                    'AND suite="{}" AND architecture="{}"'
            query = query.format(self.package, self.suite, self.arch)
            try:
                result = query_db(query)[0][0]
                if result:
                    result = ('untested', str(result), False)
            except IndexError:  # there is no package with this name in this
                return          # suite/arch, or none at all
        self.status = str(result[0])
676
677
678
679
680
681
682
683
684
685
        if self.status != 'blacklisted':
            self.version = str(result[1])
        else
            query = 'SELECT version FROM sources WHERE name="{}" ' + \
                    'AND suite="{}" AND architecture="{}"'
            query = query.format(self.package, self.suite, self.arch)
            try:
                self.version = query_db(query)[0][0]
            except IndexError:      # there is no package with this name in this
                self.version = ''   # suite/arch, or none at all
Holger Levsen's avatar
Holger Levsen committed
686
687
        if result[2]:
            self.build_date = str(result[2]) + ' UTC'
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734


class Package:
    def __init__(self, name, no_notes=False):
        self.name = name
        self._status = {}
        for suite in SUITES:
            self._status[suite] = {}
            for arch in ARCHS:
                self._status[suite][arch] = Build(self.name, suite, arch)
                if not no_notes:
                    self.note = NotedPkg(self.name, suite, arch).note
                else:
                    self.note = False
        try:
            self.status = self._status[defaultsuite][defaultarch].status
        except KeyError:
            self.status = False
        query = 'SELECT notify_maintainer FROM sources WHERE name="{}"'
        try:
            result = int(query_db(query.format(self.name))[0][0])
        except IndexError:
            result = 0
        self.notify_maint = '⚑' if result == 1 else ''

    def get_status(self, suite, arch):
        """ This returns False if the package does not exists in this suite """
        try:
            return self._status[suite][arch].status
        except KeyError:
            return False

    def get_build_date(self, suite, arch):
        """ This returns False if the package does not exists in this suite """
        try:
            return self._status[suite][arch].build_date
        except KeyError:
            return False

    def get_tested_version(self, suite, arch):
        """ This returns False if the package does not exists in this suite """
        try:
            return self._status[suite][arch].version
        except KeyError:
            return False


735
# init the databases connections
736
737
738
conn_db = start_db_connection()  # the local sqlite3 reproducible db
# get_bugs() is the only user of this, let it initialize the connection itself,
# during it's first call to speed up things when unneeded
739
# also "share" the bugs, to avoid collecting them multiple times per run
740
conn_udd = None
741
bugs = None