Commit eb0ae674 authored by Chris Lamb's avatar Chris Lamb 💬

Move to monthly report structure.

parent e4498a1d
......@@ -20,6 +20,8 @@ collections:
output: true
events:
output: true
reports:
output: true
exclude:
- README
......
---
layout: new/default
---
<h1><a href="{{ page.url }}">{{ page.title }}</a></h1>
<p>&larr; <a href="{{ "/reports/" | prepend: site.baseurl }}">View all monthly reports</a></p>
{% unless page.published %}
<div class="col-12 alert alert-warning" role="alert">
<strong>This is an unpublished draft report.</strong>
<a href="https://salsa.debian.org/reproducible-builds/reproducible-website.git/tree/_reports/{{ page.year }}{{ page.month }}.md">(source)</a>
</div>
{% endunless %}
<hr>
<div class="blog-post-content">
{{ content }}
</div>
<hr>
<p>&larr; <a href="{{ "/reports/" | prepend: site.baseurl }}">View all monthly reports</a></p>
---
layout: new/blog
week: 207
layout: new/report
year: "2019"
month: "04"
title: "Reproducible Builds in April 2019"
---
* fontconfig
......
......@@ -11,8 +11,6 @@ import sys
import time
import yaml
WEEK_1_END = 1430611200 # May 3 2015, 00:00 UTC, Sunday
PROJECTS = (
'diffoscope',
'diffoscope-website',
......@@ -29,9 +27,9 @@ def main(*args):
for x in PROJECTS + ('reproducible-notes',):
ensure_dir(sibling_repo_gitdir(x))
week = int(args[0]) if len(args) > 0 else prev_week()
today = datetime.datetime.utcnow()
data = get_data(week)
data = get_data(today.year, today.month)
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(os.path.dirname(__file__))
......@@ -45,11 +43,6 @@ def log(msg, *args, **kwargs):
print("I: " + msg.format(*args, **kwargs), file=sys.stderr)
def prev_week():
now = int(time.time())
return ((now - WEEK_1_END) // (7 * 24 * 3600) + 1)
def sibling_repo_gitdir(path):
toplevel = os.path.dirname(subprocess.check_output((
'git',
......@@ -65,8 +58,8 @@ def ensure_dir(path):
raise ValueError("not a directory: {}".format(path))
def get_data(week, max_age=3600):
filename = '/tmp/generate-draft-{}.pickle'.format(week)
def get_data(year, month, max_age=3600):
filename = '/tmp/generate-draft-{:04d}{:02d}.pickle'.format(year, month)
try:
mtime = os.path.getmtime(filename)
......@@ -82,10 +75,13 @@ def get_data(week, max_age=3600):
log("Getting new data")
week_end = WEEK_1_END + (week - 1) * 7 * 24 * 3600 # exclusive
week_start = week_end - 7 * 24 * 3600 # inclusive
month_start = datetime.date(year=year, month=month, day=1)
month_end = month_start.replace(day=calendar.monthlen(year, month))
month_start_ts = int(month_start.timestamp())
month_end_ts = int(month_end.timestamp())
data = {x: y(week_start, week_end) for x, y in (
data = {x: y(month_start_ts, month_end_ts) for x, y in (
('author', get_author),
('commits', get_commits),
('uploads', get_uploads),
......@@ -97,10 +93,10 @@ def get_data(week, max_age=3600):
)}
data.update({
'week': week,
'week_start': datetime.datetime.utcfromtimestamp(week_start),
'week_end': datetime.datetime.utcfromtimestamp(week_end - 1),
'projects': PROJECTS,
'title': month_start.strftime('Reproducible Builds in %B %Y'),
'title_year': '{:04d}'.format(year),
'title_month': '{:02d}'.format(month),
})
log("Saving cache to {}", filename)
......@@ -111,27 +107,27 @@ def get_data(week, max_age=3600):
return data
def get_author(week_start, week_end):
def get_author(month_start, month_end):
return os.environ.get('DEBFULLNAME', 'FIXME')
def get_ftbfs_bugs(week_start, week_end):
def get_ftbfs_bugs(month_start, month_end):
return bugs(
week_start,
week_end,
month_start,
month_end,
"bugs_usertags.tag = '{}'".format('ftbfs'),
)
def get_patches(week_start, week_end):
def get_patches(month_start, month_end):
return bugs(
week_start,
week_end,
month_start,
month_end,
"id IN (SELECT id FROM bugs_tags WHERE tag = 'patch')",
)
def bugs(week_start, week_end, extra="true"):
def bugs(month_start, month_end, extra="true"):
log("Querying UDD for usertagged bugs with filter: {}", extra)
fields = (
......@@ -156,12 +152,12 @@ def bugs(week_start, week_end, extra="true"):
{extra}
AND
CAST(arrival AS DATE) BETWEEN
to_timestamp(@{week_start}) AND to_timestamp(@{week_end})
to_timestamp(@{month_start}) AND to_timestamp(@{month_end})
""".format(**{
'fields': ', '.join(fields),
'extra': extra,
'week_start': week_start,
'week_end': week_end,
'month_start': month_start,
'month_end': month_end,
})
seen = set()
......@@ -178,7 +174,7 @@ def bugs(week_start, week_end, extra="true"):
}
def get_uploads(week_start, week_end):
def get_uploads(month_start, month_end):
log("Querying UDD for uploads")
fields = (
......@@ -197,14 +193,14 @@ def get_uploads(week_start, week_end):
source IN ({sources})
AND
CAST(date AS date) BETWEEN
to_timestamp(@{week_start}) AND to_timestamp(@{week_end})
to_timestamp(@{month_start}) AND to_timestamp(@{month_end})
ORDER BY
date
""".format(**{
'fields': ', '.join(fields),
'sources': ', '.join("'{}'".format(x) for x in PROJECTS),
'week_start': week_start,
'week_end': week_end,
'month_start': month_start,
'month_end': month_end,
}), fields)
result = {}
......@@ -234,16 +230,16 @@ def udd(query, fields):
return data
def get_commits(week_start, week_end):
return {x: commits(week_start, week_end, x) for x in PROJECTS}
def get_commits(month_start, month_end):
return {x: commits(month_start, month_end, x) for x in PROJECTS}
def get_issues_yml(week_start, week_end):
return commits(week_start, week_end, 'reproducible-notes', 'issues.yml')
def get_issues_yml(month_start, month_end):
return commits(month_start, month_end, 'reproducible-notes', 'issues.yml')
def get_packages_yml(week_start, week_end):
return commits(week_start, week_end, 'reproducible-notes', 'packages.yml')
def get_packages_yml(month_start, month_end):
return commits(month_start, month_end, 'reproducible-notes', 'packages.yml')
def open_packages_yml(date):
......@@ -255,9 +251,9 @@ def open_packages_yml(date):
stdout=subprocess.PIPE).stdout
def get_packages_stats(week_start, week_end):
old = yaml.safe_load(open_packages_yml(week_start))
new = yaml.safe_load(open_packages_yml(week_end))
def get_packages_stats(month_start, month_end):
old = yaml.safe_load(open_packages_yml(month_start))
new = yaml.safe_load(open_packages_yml(month_end))
removed = set(old.keys()) - set(new.keys())
added = set(new.keys()) - set(old.keys())
......@@ -272,7 +268,7 @@ def get_packages_stats(week_start, week_end):
}
def commits(week_start, week_end, project, path='.'):
def commits(month_start, month_end, project, path='.'):
# Assume its in the parent dir
git_dir = sibling_repo_gitdir(project)
......@@ -282,8 +278,8 @@ def commits(week_start, week_end, project, path='.'):
'git',
'log',
'origin/master',
'--since', '@{}'.format(week_start),
'--until', '@{}'.format(week_end),
'--since', '@{}'.format(month_start),
'--until', '@{}'.format(month_end),
'--pretty=format:%an\t%h\t%s',
'--no-merges',
'--all',
......
---
layout: new/blog
week: {{ week }}
layout: new/report
year: "{{ title_year }}"
month: "{{ title_month }}"
title: "{{ title }}"
---
Here's what happened in the [Reproducible Builds](https://reproducible-builds.org) effort between {{ week_start.strftime('%A %B') }} {{ week_start.day }} and {{ week_end.strftime('%A %B') }} {{ week_end.day }} {{ week_end.year }}:
* On [our mailing list](https://lists.reproducible-builds.org/pipermail/rb-general/) this month: FIXME
* On [our mailing list](https://lists.reproducible-builds.org/pipermail/rb-general/) this week: FIXME
* {{ packages_stats['added'] }} reviews of Debian packages were added, {{ packages_stats['updated'] }} were updated and {{ packages_stats['removed'] }} were removed in this week, adding to [our knowledge about identified issues](https://tests.reproducible-builds.org/debian/index_issues.html). FIXME issue types have been updated: {% for _, xs in issues_yml.items()|sort %}{% for x in xs %}[{{ x['title'] }}](https://salsa.debian.org/reproducible-builds/reproducible-notes/commit/{{ x['sha'] }}), {% endfor %}{% endfor %}
* {{ packages_stats['added'] }} reviews of Debian packages were added, {{ packages_stats['updated'] }} were updated and {{ packages_stats['removed'] }} were removed this month, adding to [our knowledge about identified issues](https://tests.reproducible-builds.org/debian/index_issues.html). FIXME issue types have been updated: {% for _, xs in issues_yml.items()|sort %}{% for x in xs %}[{{ x['title'] }}](https://salsa.debian.org/reproducible-builds/reproducible-notes/commit/{{ x['sha'] }}), {% endfor %}{% endfor %}
## Packages reviewed and fixed, and bugs filed
......@@ -22,7 +22,7 @@ In addition, build failure bugs were reported by:
{% for project in projects %}
## {{ project }} development
{% for x in uploads[project] %}
{{ project }} version `{{ x['version'] }}` was [uploaded to Debian {{ x['distribution'] }}](https://tracker.debian.org/pkg/{{ project }}?FIXME) by {{ x['signed_by_name'] }}. It [included contributions already covered in previous weeks](https://salsa.debian.org/reproducible-builds/{{ project }}/commits/{% if project != 'diffoscope' %}debian/{% endif %}{{ x['version'] }}) as well as new ones from:
{{ project }} version `{{ x['version'] }}` was [uploaded to Debian {{ x['distribution'] }}](https://tracker.debian.org/pkg/{{ project }}?FIXME) by {{ x['signed_by_name'] }}. It [included contributions already covered in previous months](https://salsa.debian.org/reproducible-builds/{{ project }}/commits/{% if project != 'diffoscope' %}debian/{% endif %}{{ x['version'] }}) as well as new ones from:
{% endfor %}
{% for x, ys in commits[project].items()|sort %}* {{ x }}:{% for y in ys %}
......@@ -32,4 +32,4 @@ In addition, build failure bugs were reported by:
---
This week's edition was written by {{ author }} & reviewed by a bunch of Reproducible Builds folks on IRC & the mailing lists.
This months's report was written by {{ author }} & reviewed by a bunch of Reproducible Builds folks on IRC & the mailing lists.
......@@ -2,18 +2,20 @@
set -eu
WEEK="${1}"
YEAR="${1}"
MONTH="$(printf "%02d" "${2}")"
TAG="${YEAR}${MONTH}"
URL="https://reproducible-builds.org/blog/posts/${WEEK}/"
URL="https://reproducible-builds.org/blog/reports/${YEAR}${MONTH}/"
DATE="$(date --utc +'%Y-%m-%d %H:%M:%S')"
if ! shift 1
if ! shift 2
then
echo "${0}: usage: ${0} <week>" >&2
echo "${0}: usage: ${0} <year> <month>" >&2
exit 2
fi
FILENAME="_blog/posts/${WEEK}.md"
FILENAME="_reports/${TAG}.md"
if grep -qs FIXME "${FILENAME}"
then
......@@ -23,7 +25,7 @@ fi
if ! grep -qs 'published: ' "${FILENAME}"
then
sed -i -e "s@^\(week: ${WEEK}\)@\1\npublished: ${DATE}@g" "${FILENAME}"
sed -i -e "s@^\(title: .*\)@\1\npublished: ${DATE}@g" "${FILENAME}"
fi
git add "${FILENAME}"
......@@ -31,12 +33,12 @@ git add "${FILENAME}"
if git commit -m "published as ${URL}"
then
git log -1
git tag -s "${WEEK}" -m "Publish week ${WEEK}"
git tag -s "${TAG}" -m "Publish report for ${TAG}"
echo
echo "Now verify the results and run:"
echo
echo " $ git push origin master && git push origin ${WEEK}"
echo " $ git push origin master && git push origin ${TAG}"
echo
while true
......@@ -46,7 +48,7 @@ then
case "${X}" in
Y|"")
git push origin master && git push origin ${WEEK}
git push origin master && git push origin ${TAG}
break
;;
N|n)
......@@ -61,7 +63,7 @@ echo
echo
echo "After ensuring this has been published, Tweet this via:"
echo
echo "%tweet What happened in the @ReproBuilds effort between $(tr '\n' ' ' < "${FILENAME}" | sed -n -e 's@.* effort between \([^:]*\):.*@\1@p'): https://reproducible-builds.org/blog/posts/${WEEK}/"
echo "%tweet Here's happened in the @ReproBuilds effort in the past month ${URL}"
echo
printf "Waiting for page to be published "
while :
......@@ -69,7 +71,7 @@ do
sleep 1
printf "."
if ! curl -qs "${URL}" 2>/dev/null | grep -qs "This is an unpublished draft post."
if ! curl -qs "${URL}" 2>/dev/null | grep -qs "This is an unpublished draft report."
then
break
fi
......
---
layout: new/default
title: Blog
permalink: /blog/
order: 6
---
# Blog
Every week we publish a weekly report on what we have been up to. ([RSS/Atom feed]({{ "/blog/index.rss" | prepend: site.baseurl }}))
{% assign by_week = site.blog | sort: 'week' | reverse %}
{% for page in by_week %}
{% if page.published %}
* [Week {{ page.week }}]({{ "/blog/posts/" | append: page.week | append: "/" | prepend: site.baseurl }})
{% endif %}
{% endfor %}
<meta http-equiv="refresh" content="0; url={{ "/reports/" | prepend: site.baseurl }}">
......@@ -12,18 +12,27 @@ layout: null
<lastBuildDate>{{ site.time | date_to_rfc822 }}</lastBuildDate>
<generator>Jekyll v{{ jekyll.version }}</generator>
{% assign sorted = site.blog | sort: 'published' | reverse %}
{% for page in sorted limit:10 %}
{% if page.published %}
{% assign reports = site.reports | sort: 'year,month' | where_exp: 'item', 'item.published' | reverse %}
{% for x in reports limit:10 %}
<item>
<title>Reproducible Builds: Weekly report #{{ page.week }}</title>
<pubDate>{{ page.published | date_to_rfc822 }}</pubDate>
<link>{{ page.url | prepend: site.url }}</link>
<guid isPermaLink="true">{{ page.url | prepend: site.url }}</guid>
<description>{{ page.content | xml_escape }}</description>
<title>{{ x.title }}</title>
<pubDate>{{ x.published | date_to_rfc822 }}</pubDate>
<link>{{ x.url | prepend: site.url }}</link>
<guid isPermaLink="true">{{ x.url | prepend: site.url }}</guid>
<description>{{ x.content | xml_escape }}</description>
</item>
{% endif %}
{% endfor %}
{% assign posts = site.blog | sort: 'published' | where_exp: 'item', 'item.published' | reverse %}
{% for x in posts limit:10 %}
<item>
<title>Reproducible Builds: Weekly report #{{ x.week }}</title>
<pubDate>{{ x.published | date_to_rfc822 }}</pubDate>
<link>{{ x.url | prepend: site.url }}</link>
<guid isPermaLink="true">{{ x.url | prepend: site.url }}</guid>
<description>{{ x.content | xml_escape }}</description>
</item>
{% endfor %}
</channel>
</rss>
---
layout: new/default
title: Reports
permalink: /reports/
order: 6
---
{% assign reports = site.reports | sort: 'year,month' | where_exp: 'item', 'item.published' | reverse %}
{% if reports.size > 0 %}
# Reports
Every month we publish a report on what we have been up to. ([RSS/Atom feed]({{ "/blog/index.rss" | prepend: site.baseurl }}))
{% for x in reports %}
* [{{ x.title }}]({{ x.url | prepend: site.baseurl }})
{% endfor %}
## Weekly reports
We previously published weekly reports on what we have been up to:
{% else %}
# Blog
Every week we publish a weekly report on what we have been up to. ([RSS/Atom feed]({{ "/blog/index.rss" | prepend: site.baseurl }}))
{% endif %}
{% assign posts = site.blog | sort: 'week' | where_exp: 'item', 'item.published' | reverse %}
{% for x in posts %}
* [Week {{ x.week }}]({{ x.url | prepend: site.baseurl }})
{% endfor %}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment