Skip to content
GitLab
Commits (4)
Hide whitespace changes
Inline Side-by-side
clean-notes
View file @ 071ded16
......@@ -11,6 +11,7 @@
from apt_pkg import version_compare
import apt
import sys
import yaml
import argparse
......@@ -37,39 +38,56 @@ This script is also able to show you other information:
* not-usertagged: list bugs listed in notes but without usertags
* archived-bugs: list bugs that are archived, probably they need to be removed
"""
parser = argparse.ArgumentParser(description=desc,
formatter_class=argparse.RawTextHelpFormatter
)
parser.add_argument('-e', '--enable', action='append', default=[],
help="enable a particular view (choose from above)")
parser.add_argument('-d', '--disable', action='append', default=[],
help="disable a particular view (choose from above)")
parser.add_argument("--ignore-duplicates", action="store_true",
help="ATTENTION! This option remove any duplicated " +
"field present in the file!")
parser.add_argument('-s', '--sort-only', action='store_true',
help='Only sort the notes, do no clean up')
parser = argparse.ArgumentParser(
description=desc, formatter_class=argparse.RawTextHelpFormatter
)
parser.add_argument(
"-e",
"--enable",
action="append",
default=[],
help="enable a particular view (choose from above)",
)
parser.add_argument(
"-d",
"--disable",
action="append",
default=[],
help="disable a particular view (choose from above)",
)
parser.add_argument(
"--ignore-duplicates",
action="store_true",
help="ATTENTION! This option remove any duplicated " + "field present in the file!",
)
parser.add_argument(
"-s", "--sort-only", action="store_true", help="Only sort the notes, do no clean up"
)
parser.add_argument("-v", "--verbose", action="store_true")
parser.add_argument("-n", "--dry-run", action="store_true")
args = type('DefaultArgs', (object,), {
'verbose': False,
'ignore_duplicates': False,
})
args = type(
"DefaultArgs",
(object,),
{
"verbose": False,
"ignore_duplicates": False,
},
)
if __name__ == '__main__':
if __name__ == "__main__":
args = parser.parse_args()
disabled_features = [
'fixed-magically',
'new-tested-version',
'missing-usertagged',
'not-usertagged',
'now-fixed',
# 'missing-version',
'archived-bugs',
]
"fixed-magically",
"new-tested-version",
"missing-usertagged",
"not-usertagged",
"now-fixed",
# 'missing-version',
"archived-bugs",
]
args.disable.extend(disabled_features)
for feature in args.enable:
if feature in args.disable:
......@@ -100,25 +118,27 @@ log.debug(args)
def scalar_node_eq(self, other):
return self.id == other.id and \
self.tag == other.tag and \
self.value == other.value
return self.id == other.id and self.tag == other.tag and self.value == other.value
yaml.nodes.ScalarNode.__eq__ = scalar_node_eq
def scalar_node_hash(self):
return hash((self.id, self.tag, self.value))
yaml.nodes.ScalarNode.__hash__ = scalar_node_hash
def compose_mapping_node(self, anchor):
start_event = self.get_event()
tag = start_event.tag
if tag is None or tag == u'!':
if tag is None or tag == u"!":
tag = self.resolve(yaml.nodes.MappingNode, None, start_event.implicit)
node = yaml.nodes.MappingNode(tag, [],
start_event.start_mark, None,
flow_style=start_event.flow_style)
node = yaml.nodes.MappingNode(
tag, [], start_event.start_mark, None, flow_style=start_event.flow_style
)
if anchor is not None:
self.anchors[anchor] = node
seen = set()
......@@ -126,10 +146,12 @@ def compose_mapping_node(self, anchor):
key_event = self.peek_event()
item_key = self.compose_node(node, None)
if item_key in seen:
raise yaml.composer.ComposerError("while composing a mapping",
start_event.start_mark,
"found duplicate key",
key_event.start_mark)
raise yaml.composer.ComposerError(
"while composing a mapping",
start_event.start_mark,
"found duplicate key",
key_event.start_mark,
)
seen.add(item_key)
item_value = self.compose_node(node, item_key)
node.value.append((item_key, item_value))
......@@ -137,26 +159,33 @@ def compose_mapping_node(self, anchor):
node.end_mark = end_event.end_mark
return node
if __name__ == '__main__':
if __name__ == "__main__":
if not args.ignore_duplicates:
yaml.composer.Composer.compose_mapping_node = compose_mapping_node
def check_notes_validity(notes, testedpkgs):
errors = False
for pkg in sorted(notes):
note = notes[pkg]
if pkg not in testedpkgs:
if 'issues' in note and (
'blacklisted_on_jenkins' in note['issues'] or
'ftbfs_in_jenkins_setup' in note['issues'] or
'ftbfs_build_depends_not_available_on_amd64' in note['issues']
):
pass
if "issues" in note and "ftbfs_in_jenkins_setup" in note["issues"]:
pass
else:
log.critical(
"the package %s was never tested. Maybe it's misspelled?",
pkg,
)
errors = True
continue
if testedpkgs[pkg]["status"] == "blacklisted":
if "ftbfs_in_jenkins_setup" not in note.get("issues", []):
log.critical(
"the package %s is blacklisted, but no note was found", pkg
)
errors = True
return errors
def check_bugs(notes):
......@@ -165,75 +194,104 @@ def check_bugs(notes):
"""
bugs = []
for pkg in sorted(notes, key=str):
if 'bugs' in notes[pkg]:
for bug in notes[pkg]['bugs']:
if "bugs" in notes[pkg]:
for bug in notes[pkg]["bugs"]:
bugs.append((str(pkg), int(bug)))
log.debug("looking throught bugs listed in the notes and check whether " +
"they are usertagged")
log.debug(
"looking throught bugs listed in the notes and check whether "
+ "they are usertagged"
)
if not bugs:
return
bugs_list = sorted([x[1] for x in bugs])
bugs_package = {x[1]: x[0] for x in bugs}
ids = 'id='
ids = "id="
for bug in bugs_list[:-1]:
ids += '%s OR id=' % bug
ids += "%s OR id=" % bug
ids += str(bugs_list[-1])
rows = remote.query_udd(conn_udd, """SELECT id FROM bugs_usertags WHERE
rows = remote.query_udd(
conn_udd,
"""SELECT id FROM bugs_usertags WHERE
email='reproducible-builds@lists.alioth.debian.org' AND (
%s )""" % ids)
%s )"""
% ids,
)
import webbrowser
for bug in bugs_list:
# the results from SELECT are a list of one-element tuples, so we have
# have to look up 1-tuples with the bug number in the list
if (bug,) not in rows:
log.info("https://bugs.debian.org/%s in package %s "
"is not usertagged", bug, bugs_package[bug])
log.info(
"https://bugs.debian.org/%s in package %s " "is not usertagged",
bug,
bugs_package[bug],
)
webbrowser.open("https://bugs.debian.org/{}".format(bug))
def find_old_notes(testedpkgs, notes):
log.debug("parsing the reproducible.json and the notes to find weirdness")
toremove = []
for pkg in sorted(notes, key=str):
if 'version' not in notes[pkg] and \
'missing-version' not in args.disable:
if "version" not in notes[pkg] and "missing-version" not in args.disable:
log.info("There is no version set for the package " + pkg)
continue
try:
item = testedpkgs[pkg]
except KeyError:
# this is due to
# https://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/commit/?id=275309
# https://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/commit/?id=275309
# and later commits to that file, otherwise this would be
# quite a issue
log.debug(pkg + ' was not tested. Skipping cruft check.')
log.debug(pkg + " was not tested. Skipping cruft check.")
continue
if item['version'] == notes[pkg].get('version') and \
item['status'] == 'reproducible' and \
(notes[pkg].get('issues') and
'timestamps_in_png' not in notes[pkg]['issues']) and \
'fixed-magically' not in args.disable:
log.info("%s/%s has a note for the version %s but that version is "
"reproducible", remote.RB_SITE, pkg, item['version'])
if item['status'] == 'reproducible' and \
notes[pkg].get('version') and \
'now-fixed' not in args.disable and \
(notes[pkg].get('issues') and
'timestamps_in_png' not in notes[pkg]['issues']) and \
version_compare(str(item['version']),
str(notes[pkg]['version'])) > 0:
log.info("The package " + pkg +
" is now reproducible but still listed in the notes")
if (
item["version"] == notes[pkg].get("version")
and item["status"] == "reproducible"
and (
notes[pkg].get("issues")
and "timestamps_in_png" not in notes[pkg]["issues"]
)
and "fixed-magically" not in args.disable
):
log.info(
"%s/%s has a note for the version %s but that version is "
"reproducible",
remote.RB_SITE,
pkg,
item["version"],
)
if (
item["status"] == "reproducible"
and notes[pkg].get("version")
and "now-fixed" not in args.disable
and (
notes[pkg].get("issues")
and "timestamps_in_png" not in notes[pkg]["issues"]
)
and version_compare(str(item["version"]), str(notes[pkg]["version"])) > 0
):
log.info(
"The package "
+ pkg
+ " is now reproducible but still listed in the notes"
)
toremove.append(pkg)
if notes[pkg].get('version') and \
version_compare(str(item['version']),
str(notes[pkg]['version'])) > 0:
if 'new-tested-version' not in args.disable:
if (
notes[pkg].get("version")
and version_compare(str(item["version"]), str(notes[pkg]["version"])) > 0
):
if "new-tested-version" not in args.disable:
log.info("The package " + pkg + " has a new tested version")
return toremove
def get_bugs():
query = 'SELECT * FROM bugs_usertags ' + \
"WHERE email='reproducible-builds@lists.alioth.debian.org'"
query = (
"SELECT * FROM bugs_usertags "
+ "WHERE email='reproducible-builds@lists.alioth.debian.org'"
)
rows = remote.query_udd(conn_udd, query)
# returns a list of tuples (email, tag, id)
bugs = {}
......@@ -246,18 +304,23 @@ def get_bugs():
def detect_archived_bugs(notes):
query = 'SELECT u.id ' \
'FROM bugs_usertags AS u JOIN archived_bugs AS a ON u.id=a.id ' \
"WHERE u.email='reproducible-builds@lists.alioth.debian.org'"
query = (
"SELECT u.id "
"FROM bugs_usertags AS u JOIN archived_bugs AS a ON u.id=a.id "
"WHERE u.email='reproducible-builds@lists.alioth.debian.org'"
)
rows = [x[0] for x in remote.query_udd(conn_udd, query)]
for pkg in sorted(notes.keys()):
try:
for bug in notes[pkg]['bugs']:
for bug in notes[pkg]["bugs"]:
if bug in rows:
log.warning(
'https://bugs.debian.org/%s in %s/%s is archived',
bug, remote.RB_SITE, pkg)
notes[pkg]['bugs'].remove(bug)
"https://bugs.debian.org/%s in %s/%s is archived",
bug,
remote.RB_SITE,
pkg,
)
notes[pkg]["bugs"].remove(bug)
except KeyError:
pass
......@@ -275,39 +338,39 @@ def parse_bugs(bugs):
"""
log.debug("find out if filed bugs are also noted in the notes")
packages = {}
ids = ''
ids = ""
bugs_list = sorted(bugs.keys())
ignored_tags = ['toolchain', 'infrastructure']
OR = ''
ignored_tags = ["toolchain", "infrastructure"]
OR = ""
for bug in bugs_list[:-1]:
if not [i for i in bugs[bug] if i in ignored_tags]:
ids += '%sb.id=%s' % (OR, bug)
OR = ' OR '
ids += "%sb.id=%s" % (OR, bug)
OR = " OR "
if not [i for i in bugs[bugs_list[-1]] if i in ignored_tags]:
ids += OR + 'b.id=%s' % bugs_list[-1]
ids += OR + "b.id=%s" % bugs_list[-1]
# the join avoid picking virtual packages
query = "SELECT DISTINCT b.id, b.source, b.done "
query += "FROM bugs AS b JOIN sources AS s ON b.source=s.source "
query += "WHERE %s""" % ids
query += "WHERE %s" "" % ids
log.debug(query)
rows = remote.query_udd(conn_udd, query)
log.info('%d bugs found', len(rows))
log.info("%d bugs found", len(rows))
for item in rows:
if item[2]: # do not consider closed bugs
continue
try:
packages[item[1]]['bugs'].append(item[0])
packages[item[1]]["bugs"].append(item[0])
except KeyError:
try:
packages[item[1]]['bugs'] = [item[0]]
packages[item[1]]["bugs"] = [item[0]]
except KeyError:
packages[item[1]] = {}
packages[item[1]]['bugs'] = [item[0]]
packages[item[1]]["bugs"] = [item[0]]
for tag in bugs[item[0]]:
try:
packages[item[1]]['usertags'].append(tag)
packages[item[1]]["usertags"].append(tag)
except KeyError:
packages[item[1]]['usertags'] = [tag]
packages[item[1]]["usertags"] = [tag]
return packages
......@@ -316,27 +379,36 @@ def join_notes_bugs(notes, bugs):
if package not in testedpkgs:
# maybe the bug is for a removed package?
continue
for bug in sorted(bugs[package]['bugs']):
for bug in sorted(bugs[package]["bugs"]):
try:
if 'bugs' in notes[package]:
if bug not in notes[package]['bugs']:
log.info("https://bugs.debian.org/%s"
" in package %s"
" is not listed in notes.git.",
bug, package)
notes[package]['bugs'].append(bug)
if "bugs" in notes[package]:
if bug not in notes[package]["bugs"]:
log.info(
"https://bugs.debian.org/%s"
" in package %s"
" is not listed in notes.git.",
bug,
package,
)
notes[package]["bugs"].append(bug)
else:
log.info("https://bugs.debian.org/%s in %s is not listed "
"in notes.git.", bug, package)
notes[package]['bugs'] = [bug]
log.info(
"https://bugs.debian.org/%s in %s is not listed "
"in notes.git.",
bug,
package,
)
notes[package]["bugs"] = [bug]
except KeyError:
log.info("https://bugs.debian.org/%s in "
"%s is not listed in notes.git.",
bug, package)
log.info(
"https://bugs.debian.org/%s in " "%s is not listed in notes.git.",
bug,
package,
)
notes[package] = {}
notes[package]['bugs'] = [bug]
notes[package]["bugs"] = [bug]
# just try guessing the version (prefers unstable)
notes[package]['version'] = testedpkgs[package]['version']
notes[package]["version"] = testedpkgs[package]["version"]
return notes
......@@ -346,29 +418,33 @@ def cleanup_notes(notes, toremove):
return notes
if __name__ == '__main__':
if 'not-usertagged' not in args.disable or \
'missing-usertagged' not in args.disable or \
'archived-bugs' not in args.disable:
if __name__ == "__main__":
if (
"not-usertagged" not in args.disable
or "missing-usertagged" not in args.disable
or "archived-bugs" not in args.disable
):
conn_udd = remote.start_udd_connection()
notes = yamlfiles.load_notes()
if not args.sort_only:
testedpkgs = remote.load_reproducible_status()
check_notes_validity(notes, testedpkgs)
errors = check_notes_validity(notes, testedpkgs)
toremove = find_old_notes(testedpkgs, notes)
else:
toremove = []
notes = cleanup_notes(notes, toremove)
if 'not-usertagged' not in args.disable:
if "not-usertagged" not in args.disable:
check_bugs(notes)
if 'missing-usertagged' not in args.disable:
if "missing-usertagged" not in args.disable:
bugs = get_bugs()
bugs = parse_bugs(bugs)
notes = join_notes_bugs(notes, bugs)
if 'archived-bugs' not in args.disable:
if "archived-bugs" not in args.disable:
detect_archived_bugs(notes)
if not args.dry_run:
yamlfiles.write_out(notes)
else:
log.info("Don't write out the %s file, as requested (dry-run).",
yamlfiles.NOTES_YAML)
log.info(
"Don't write out the %s file, as requested (dry-run).", yamlfiles.NOTES_YAML
)
sys.exit(errors)