Skip to content
GitLab
Commits (3)
Hide whitespace changes
Inline Side-by-side
bin/generate-draft
View file @ 9fdd1cbd
......@@ -13,32 +13,29 @@ import time
import yaml
PROJECTS = (
'diffoscope',
'diffoscope-website',
'strip-nondeterminism',
'disorderfs',
'reprotest',
'trydiffoscope',
'try.diffoscope.org',
'reproducible-website',
'rb-mailx-ansible',
'jenkins.debian.net',
"diffoscope",
"diffoscope-website",
"strip-nondeterminism",
"disorderfs",
"reprotest",
"trydiffoscope",
"try.diffoscope.org",
"reproducible-website",
"rb-mailx-ansible",
"jenkins.debian.net",
)
def main(*args):
for x in PROJECTS + ('reproducible-notes',):
for x in PROJECTS + ("reproducible-notes",):
ensure_dir(sibling_repo_gitdir(x))
previous_month = datetime.date.today().replace(day=1) - \
datetime.timedelta(days=1)
previous_month = datetime.date.today().replace(day=1) - datetime.timedelta(days=1)
data = get_data(previous_month.year, previous_month.month)
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(os.path.dirname(__file__))
)
print(env.get_template('generate-draft.template').render(**data))
env = jinja2.Environment(loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))
print(env.get_template("generate-draft.template").render(**data))
return 0
......@@ -48,13 +45,13 @@ def log(msg, *args, **kwargs):
def sibling_repo_gitdir(path):
toplevel = os.path.dirname(subprocess.check_output((
'git',
'rev-parse',
'--show-toplevel',
)).decode('utf-8'))
toplevel = os.path.dirname(
subprocess.check_output(("git", "rev-parse", "--show-toplevel",)).decode(
"utf-8"
)
)
return os.path.join(toplevel, path, '.git')
return os.path.join(toplevel, path, ".git")
def ensure_dir(path):
......@@ -63,7 +60,7 @@ def ensure_dir(path):
def get_data(year, month, max_age=3600):
filename = '/tmp/generate-draft-{:04d}{:02d}.pickle'.format(year, month)
filename = "/tmp/generate-draft-{:04d}{:02d}.pickle".format(year, month)
try:
mtime = os.path.getmtime(filename)
......@@ -72,7 +69,7 @@ def get_data(year, month, max_age=3600):
if mtime > mtime_self and mtime >= time.time() - max_age:
log("Using cache from {}", filename)
with open(filename, 'rb') as f:
with open(filename, "rb") as f:
return pickle.load(f)
except (EOFError, OSError):
pass
......@@ -81,57 +78,68 @@ def get_data(year, month, max_age=3600):
month_start = datetime.datetime(year=year, month=month, day=1)
month_end = month_start.replace(
day=calendar.monthlen(year, month),
hour=23,
minute=59
day=calendar.monthlen(year, month), hour=23, minute=59
)
month_start_ts = int(month_start.timestamp())
month_end_ts = int(month_end.timestamp())
data = {x: y(month_start_ts, month_end_ts) for x, y in (
('author', get_author),
('commits', get_commits),
('uploads', get_uploads),
('patches', get_patches),
('ftbfs_bugs', get_ftbfs_bugs),
('issues_yml', get_issues_yml),
('packages_yml', get_packages_yml),
('packages_stats', get_packages_stats),
)}
data.update({
'projects': PROJECTS,
'month_year': month_start.strftime('%B %Y'),
'title_year': '{:04d}'.format(year),
'title_month': '{:02d}'.format(month),
})
data = {
x: y(month_start_ts, month_end_ts)
for x, y in (
("authors", get_authors),
("commits", get_commits),
("uploads", get_uploads),
("patches", get_patches),
("ftbfs_bugs", get_ftbfs_bugs),
("issues_yml", get_issues_yml),
("packages_yml", get_packages_yml),
("packages_stats", get_packages_stats),
)
}
data.update(
{
"projects": PROJECTS,
"month_year": month_start.strftime("%B %Y"),
"title_year": "{:04d}".format(year),
"title_month": "{:02d}".format(month),
}
)
log("Saving cache to {}", filename)
with open(filename, 'wb') as f:
with open(filename, "wb") as f:
pickle.dump(data, f)
return data
def get_author(month_start, month_end):
return os.environ.get('DEBFULLNAME', 'FIXME')
def get_authors(month_start, month_end):
lines = subprocess.check_output(
(
"git",
"log",
'--pretty=format:"%an%x09"',
"_reports/{}-{:02d}.md".format(month_start.year, month_start.month),
)
).decode("utf-8")
authors = list(sorted(set(x.replace('"', "").strip() for x in lines.splitlines())))
if len(authors) < 2:
return " and ".join(authors)
return "{} and {}".format(", ".join(authors[:-1]), authors[-1],)
def get_ftbfs_bugs(month_start, month_end):
return bugs(
month_start,
month_end,
"bugs_usertags.tag = '{}'".format('ftbfs'),
)
return bugs(month_start, month_end, "bugs_usertags.tag = '{}'".format("ftbfs"),)
def get_patches(month_start, month_end):
return bugs(
month_start,
month_end,
"id IN (SELECT id FROM bugs_tags WHERE tag = 'patch')",
month_start, month_end, "id IN (SELECT id FROM bugs_tags WHERE tag = 'patch')",
)
......@@ -139,12 +147,12 @@ def bugs(month_start, month_end, extra="true"):
log("Querying UDD for usertagged bugs with filter: {}", extra)
fields = (
'id',
'source',
'submitter',
'submitter_name',
'title',
'arrival',
"id",
"source",
"submitter",
"submitter_name",
"title",
"arrival",
)
sql = """
......@@ -161,38 +169,39 @@ def bugs(month_start, month_end, extra="true"):
AND
CAST(arrival AS DATE) BETWEEN
to_timestamp(@{month_start}) AND to_timestamp(@{month_end})
""".format(**{
'fields': ', '.join(fields),
'extra': extra,
'month_start': month_start,
'month_end': month_end,
})
""".format(
**{
"fields": ", ".join(fields),
"extra": extra,
"month_start": month_start,
"month_end": month_end,
}
)
seen = set()
result = {}
for x in udd(sql, fields):
if x['id'] in seen:
if x["id"] in seen:
continue
seen.add(x['id'])
seen.add(x["id"])
result.setdefault(x['submitter_name'], []).append(x)
result.setdefault(x["submitter_name"], []).append(x)
return {
x: list(sorted(y, key=lambda x: x['id'])) for x, y in result.items()
}
return {x: list(sorted(y, key=lambda x: x["id"])) for x, y in result.items()}
def get_uploads(month_start, month_end):
log("Querying UDD for uploads")
fields = (
'source',
'version',
'distribution',
'signed_by_name',
"source",
"version",
"distribution",
"signed_by_name",
)
data = udd("""
data = udd(
"""
SELECT
{fields}
FROM
......@@ -204,29 +213,38 @@ def get_uploads(month_start, month_end):
to_timestamp(@{month_start}) AND to_timestamp(@{month_end})
ORDER BY
date
""".format(**{
'fields': ', '.join(fields),
'sources': ', '.join("'{}'".format(x) for x in PROJECTS),
'month_start': month_start,
'month_end': month_end,
}), fields)
""".format(
**{
"fields": ", ".join(fields),
"sources": ", ".join("'{}'".format(x) for x in PROJECTS),
"month_start": month_start,
"month_end": month_end,
}
),
fields,
)
result = {}
for x in data:
result.setdefault(x['source'], []).append(x)
result.setdefault(x["source"], []).append(x)
return result
def udd(query, fields):
lines = subprocess.check_output("""
lines = subprocess.check_output(
"""
echo "{}" | ssh quantz.debian.org psql --no-psqlrc service=udd
""".format(query), shell=True)
""".format(
query
),
shell=True,
)
data = []
for line in lines.splitlines()[2:]:
split = line.decode('utf-8').split('|')
split = line.decode("utf-8").split("|")
if len(split) != len(fields):
continue
......@@ -243,11 +261,11 @@ def get_commits(month_start, month_end):
def get_issues_yml(month_start, month_end):
return commits(month_start, month_end, 'reproducible-notes', 'issues.yml')
return commits(month_start, month_end, "reproducible-notes", "issues.yml")
def get_packages_yml(month_start, month_end):
return commits(month_start, month_end, 'reproducible-notes', 'packages.yml')
return commits(month_start, month_end, "reproducible-notes", "packages.yml")
def open_packages_yml(date):
......@@ -255,8 +273,9 @@ def open_packages_yml(date):
"git show $(git rev-list -n1 --until @{0} "
"origin/master):packages.yml".format(date),
shell=True,
cwd=sibling_repo_gitdir('reproducible-notes'),
stdout=subprocess.PIPE).stdout
cwd=sibling_repo_gitdir("reproducible-notes"),
stdout=subprocess.PIPE,
).stdout
def get_packages_stats(month_start, month_end):
......@@ -276,52 +295,54 @@ def get_packages_stats(month_start, month_end):
}
def commits(month_start, month_end, project, path='.'):
def commits(month_start, month_end, project, path="."):
# Assume its in the parent dir
git_dir = sibling_repo_gitdir(project)
subprocess.check_call(('git', 'fetch', 'origin'), cwd=git_dir)
output = subprocess.check_output((
'git',
'log',
'origin/master',
'--since', '@{}'.format(month_start),
'--until', '@{}'.format(month_end),
'--pretty=format:%an\t%h\t%s',
'--no-merges',
'--all',
'--',
path,
':(exclude,glob)_blog/posts/*.md',
), cwd=git_dir).decode('utf-8')
subprocess.check_call(("git", "fetch", "origin"), cwd=git_dir)
output = subprocess.check_output(
(
"git",
"log",
"origin/master",
"--since",
"@{}".format(month_start),
"--until",
"@{}".format(month_end),
"--pretty=format:%an\t%h\t%s",
"--no-merges",
"--all",
"--",
path,
":(exclude,glob)_blog/posts/*.md",
),
cwd=git_dir,
).decode("utf-8")
result = collections.defaultdict(list)
for x in output.splitlines():
author, sha, title = x.split('\t', 2)
author, sha, title = x.split("\t", 2)
for pattern in (
r'^dependabot$',
):
for pattern in (r"^dependabot$",):
if re.search(pattern, author) is not None:
continue
for pattern in (
r'^--fix-deterministic$',
r'^Add missing usertagged bugs$',
r'^Remove archived bugs$',
r'^Release .* to Debian .*$',
r"^--fix-deterministic$",
r"^Add missing usertagged bugs$",
r"^Remove archived bugs$",
r"^Release .* to Debian .*$",
):
if re.search(pattern, title) is not None:
continue
result[author].append({
'sha': sha,
'title': title.replace('_', '\_'),
})
result[author].append(
{"sha": sha, "title": title.replace("_", "\_"),}
)
return result
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main(*sys.argv[1:]))
bin/generate-draft.template
View file @ 9fdd1cbd
......@@ -105,4 +105,4 @@ If you are interested in contributing the Reproducible Builds project, please vi
---
This month's report was written by {{ author }}. It was subsequently reviewed by a bunch of Reproducible Builds folks on IRC and the mailing list.
This month's report was written by {{ authors }}. It was subsequently reviewed by a bunch of Reproducible Builds folks on IRC and the mailing list.
citests.md
View file @ 9fdd1cbd
......@@ -4,7 +4,8 @@ title: Continuous tests
permalink: /citests/
order: 10
---
## Continuous tests
# Continuous tests
<div markdown="1">
The following projects are tested within the reproducible test infrastructure.
......