From a2d9bcbfb121bf4d86a43325e4a885401cbb3360 Mon Sep 17 00:00:00 2001 From: Ivan Udovichenko Date: Thu, 14 Jan 2016 22:51:41 +0200 Subject: [PATCH 001/257] Add debian folder. --- debian/changelog | 5 +++ debian/compat | 1 + debian/control | 75 +++++++++++++++++++++++++++++++++ debian/copyright | 28 ++++++++++++ debian/gbp.conf | 9 ++++ debian/python-reno-doc.doc-base | 9 ++++ debian/rules | 51 ++++++++++++++++++++++ debian/source/format | 1 + debian/source/options | 1 + debian/watch | 3 ++ 10 files changed, 183 insertions(+) create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/gbp.conf create mode 100644 debian/python-reno-doc.doc-base create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100644 debian/source/options create mode 100644 debian/watch diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..41d0e37 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +python-reno (1.3.0-1) unstable; urgency=medium + + * Initial release. (Closes: #XXXXXX) + + -- Ivan Udovichenko Thu, 14 Jan 2016 17:45:02 +0200 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..a46b000 --- /dev/null +++ b/debian/control @@ -0,0 +1,75 @@ +Source: python-reno +Section: python +Priority: optional +Maintainer: PKG OpenStack +Uploaders: Ivan Udovichenko +Build-Depends: debhelper (>= 9), + dh-python, + python-setuptools, + python-all (>= 2.7), + python3-setuptools, + python3-all (>= 3.3), + python-sphinx, +Build-Depends-Indep: python-babel, + python-coverage (>= 3.6), + python-hacking (>= 0.10.0), + python-mock (>= 1.2), + python-oslosphinx (>= 2.5.0), + python-oslotest (>= 1.10.0), + python-pbr (>= 1.4), + python-subunit (>= 0.0.18), + python-testrepository (>= 0.0.18), + python-testscenarios (>= 0.4), + python-testtools (>= 1.4.0), + python-yaml (>= 3.1.0), + python3-babel, + python3-coverage (>= 3.6), + python3-hacking (>= 0.10.0), + python3-mock (>= 1.2), + python3-oslotest (>= 1.10.0), + python3-pbr (>= 1.4), + python3-subunit (>= 0.0.18), + python3-testrepository (>= 0.0.18), + python3-testscenarios (>= 0.4), + python3-testtools (>= 1.4.0), +Standards-Version: 3.9.6 +Vcs-Browser: http://anonscm.debian.org/gitweb/?p=openstack/python-reno.git +Vcs-Git: git://anonscm.debian.org/openstack/python-reno.git +Homepage: http://www.openstack.org/ + +Package: python-reno +Architecture: all +Depends: python-pbr (>= 1.4), + python-yaml (>= 3.1.0), + ${python:Depends}, + ${misc:Depends} +Suggests: python-reno-doc +Description: RElease NOtes manager - Python 2.x + Reno is a release notes manager for storing release notes in a git + repository and then building documentation from them. + . + This package contains the Python 2.x module. + +Package: python3-reno +Architecture: all +Depends: python3-pbr (>= 1.4), + python3-yaml (>= 3.1.0), + ${python3:Depends}, + ${misc:Depends} +Suggests: python-reno-doc +Description: RElease NOtes manager - Python 3.x + Reno is a release notes manager for storing release notes in a git + repository and then building documentation from them. + . + This package contains the Python 3.x module. + +Package: python-reno-doc +Section: doc +Architecture: all +Depends: ${misc:Depends}, ${sphinxdoc:Depends} +Description: RElease NOtes manager - doc + Reno is a release notes manager for storing release notes in a git + repository and then building documentation from them. + . + This package contains the documentation. + diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..0d2cbef --- /dev/null +++ b/debian/copyright @@ -0,0 +1,28 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: reno +Source: http://www.openstack.org/ + +Files: debian/* +Copyright: 2015, Ivan Udovichenko +License: Apache-2 + +Files: * +Copyright: (c) 2013, OpenStack +License: Apache-2 + +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian-based systems the full text of the Apache version 2.0 license + can be found in /usr/share/common-licenses/Apache-2.0. + diff --git a/debian/gbp.conf b/debian/gbp.conf new file mode 100644 index 0000000..fd8ec27 --- /dev/null +++ b/debian/gbp.conf @@ -0,0 +1,9 @@ +[DEFAULT] +upstream-branch = master +debian-branch = debian/unstable +upstream-tag = %(version)s +compression = xz + +[git-buildpackage] +export-dir = ../build-area/ + diff --git a/debian/python-reno-doc.doc-base b/debian/python-reno-doc.doc-base new file mode 100644 index 0000000..b8dd916 --- /dev/null +++ b/debian/python-reno-doc.doc-base @@ -0,0 +1,9 @@ +Document: python-reno-doc +Title: reno Documentation +Author: N/A +Abstract: Sphinx documentation for reno +Section: Programming/Python + +Format: HTML +Index: /usr/share/doc/python-reno-doc/html/index.html +Files: /usr/share/doc/python-reno-doc/html/* diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..c7abe1c --- /dev/null +++ b/debian/rules @@ -0,0 +1,51 @@ +#!/usr/bin/make -f + +PYTHONS:=$(shell pyversions -vr) +PYTHON3S:=$(shell py3versions -vr) + +UPSTREAM_GIT = git://github.com/openstack/reno.git +-include /usr/share/openstack-pkg-tools/pkgos.make + +%: + dh $@ --buildsystem=python_distutils --with python2,python3,sphinxdoc + +override_dh_install: + set -e ; for pyvers in $(PYTHONS); do \ + python$$pyvers setup.py install --install-layout=deb \ + --root $(CURDIR)/debian/python-reno; \ + done + set -e ; for pyvers in $(PYTHON3S); do \ + python$$pyvers setup.py install --install-layout=deb \ + --root $(CURDIR)/debian/python3-reno; \ + done + rm -rf $(CURDIR)/debian/python*-reno/usr/lib/python*/dist-packages/*.pth + +override_dh_auto_test: +ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) + set -e ; for pyvers in $(PYTHONS) $(PYTHON3S); do \ + python$$pyvers setup.py test ; \ + done +endif + +override_dh_sphinxdoc: + sphinx-build -b html doc/source debian/python-reno-doc/usr/share/doc/python-reno-doc/html + dh_sphinxdoc -O--buildsystem=python_distutils + + +override_dh_clean: + dh_clean -O--buildsystem=python_distutils + rm -rf build + + +# Commands not to run +override_dh_installcatalogs: +override_dh_installemacsen override_dh_installifupdown: +override_dh_installinfo override_dh_installmenu: +override_dh_installmime override_dh_installmodules: +override_dh_installlogcheck override_dh_installpam: +override_dh_installppp override_dh_installudev: +override_dh_installwm override_dh_installxfonts: +override_dh_gconf override_dh_icons override_dh_perl: +override_dh_usrlocal override_dh_installcron: +override_dh_installdebconf override_dh_installlogrotate: +override_dh_installgsettings: diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/source/options b/debian/source/options new file mode 100644 index 0000000..cb61fa5 --- /dev/null +++ b/debian/source/options @@ -0,0 +1 @@ +extend-diff-ignore = "^[^/]*[.]egg-info/" diff --git a/debian/watch b/debian/watch new file mode 100644 index 0000000..078fafc --- /dev/null +++ b/debian/watch @@ -0,0 +1,3 @@ +version=3 +http://pypi.python.org/packages/source/r/reno reno-(.*).tar.gz + -- GitLab From 052206e189f87fdc89c6c6562bea6c8033f1179a Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 15 Jan 2016 16:39:05 +0000 Subject: [PATCH 002/257] manage stderr output from external commands Only show error output from external commands in debug mode. This suppresses "fatal" messages that are fatal to git, but not reno, for example. Change-Id: Ic9a9fcf30fd2f9ff2c0a837de45dd062b7a900e8 Related-Bug: 1534613 --- reno/utils.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/reno/utils.py b/reno/utils.py index fce1772..137d3e1 100644 --- a/reno/utils.py +++ b/reno/utils.py @@ -11,12 +11,16 @@ # under the License. import binascii +import logging +import os import os.path import random import subprocess from reno import defaults +LOG = logging.getLogger(__name__) + def get_notes_dir(args): """Return the path to the release notes directory.""" @@ -41,5 +45,15 @@ def get_random_string(nbytes=8): def check_output(*args, **kwds): """Unicode-aware wrapper for subprocess.check_output""" - raw = subprocess.check_output(*args, **kwds) - return raw.decode('utf-8') + process = subprocess.Popen(stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + *args, **kwds) + output, errors = process.communicate() + retcode = process.poll() + if errors: + LOG.debug('error output from (%s): %s', + ' '.join(*args), + errors.rstrip()) + if retcode: + raise subprocess.CalledProcessError(retcode, args, output=output) + return output.decode('utf-8') -- GitLab From 8ca98818d3038dccca89ea772e757c27dcb247e7 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sat, 16 Jan 2016 06:19:32 +0000 Subject: [PATCH 003/257] Added myself as uploader. --- debian/control | 21 +++++++++++---------- debian/copyright | 1 - 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/debian/control b/debian/control index a46b000..f67490f 100644 --- a/debian/control +++ b/debian/control @@ -2,14 +2,15 @@ Source: python-reno Section: python Priority: optional Maintainer: PKG OpenStack -Uploaders: Ivan Udovichenko +Uploaders: Ivan Udovichenko , + Thomas Goirand , Build-Depends: debhelper (>= 9), dh-python, - python-setuptools, python-all (>= 2.7), - python3-setuptools, - python3-all (>= 3.3), + python-setuptools, python-sphinx, + python3-all (>= 3.3), + python3-setuptools, Build-Depends-Indep: python-babel, python-coverage (>= 3.6), python-hacking (>= 0.10.0), @@ -41,9 +42,9 @@ Package: python-reno Architecture: all Depends: python-pbr (>= 1.4), python-yaml (>= 3.1.0), + ${misc:Depends}, ${python:Depends}, - ${misc:Depends} -Suggests: python-reno-doc +Suggests: python-reno-doc, Description: RElease NOtes manager - Python 2.x Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. @@ -54,9 +55,9 @@ Package: python3-reno Architecture: all Depends: python3-pbr (>= 1.4), python3-yaml (>= 3.1.0), + ${misc:Depends}, ${python3:Depends}, - ${misc:Depends} -Suggests: python-reno-doc +Suggests: python-reno-doc, Description: RElease NOtes manager - Python 3.x Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. @@ -66,10 +67,10 @@ Description: RElease NOtes manager - Python 3.x Package: python-reno-doc Section: doc Architecture: all -Depends: ${misc:Depends}, ${sphinxdoc:Depends} +Depends: ${misc:Depends}, + ${sphinxdoc:Depends}, Description: RElease NOtes manager - doc Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. . This package contains the documentation. - diff --git a/debian/copyright b/debian/copyright index 0d2cbef..9bd26f8 100644 --- a/debian/copyright +++ b/debian/copyright @@ -25,4 +25,3 @@ License: Apache-2 . On Debian-based systems the full text of the Apache version 2.0 license can be found in /usr/share/common-licenses/Apache-2.0. - -- GitLab From 38a5a40807692c9b70da16c0af98445b6804fbe8 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sat, 16 Jan 2016 06:22:42 +0000 Subject: [PATCH 004/257] Fixed (build-)depends. --- debian/control | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/debian/control b/debian/control index f67490f..0741812 100644 --- a/debian/control +++ b/debian/control @@ -6,33 +6,31 @@ Uploaders: Ivan Udovichenko , Thomas Goirand , Build-Depends: debhelper (>= 9), dh-python, - python-all (>= 2.7), + python-all, + python-pbr (>= 1.4), python-setuptools, python-sphinx, - python3-all (>= 3.3), + python3-all, + python3-pbr (>= 1.4), python3-setuptools, Build-Depends-Indep: python-babel, - python-coverage (>= 3.6), - python-hacking (>= 0.10.0), - python-mock (>= 1.2), + python-coverage, + python-hacking, + python-mock (>= 1.3), python-oslosphinx (>= 2.5.0), python-oslotest (>= 1.10.0), - python-pbr (>= 1.4), - python-subunit (>= 0.0.18), - python-testrepository (>= 0.0.18), - python-testscenarios (>= 0.4), + python-testscenarios, python-testtools (>= 1.4.0), - python-yaml (>= 3.1.0), + python-yaml, python3-babel, python3-coverage (>= 3.6), - python3-hacking (>= 0.10.0), - python3-mock (>= 1.2), + python3-mock (>= 1.3), python3-oslotest (>= 1.10.0), - python3-pbr (>= 1.4), - python3-subunit (>= 0.0.18), - python3-testrepository (>= 0.0.18), - python3-testscenarios (>= 0.4), + python3-subunit, + python3-testscenarios, python3-testtools (>= 1.4.0), + subunit, + testrepository, Standards-Version: 3.9.6 Vcs-Browser: http://anonscm.debian.org/gitweb/?p=openstack/python-reno.git Vcs-Git: git://anonscm.debian.org/openstack/python-reno.git @@ -41,7 +39,7 @@ Homepage: http://www.openstack.org/ Package: python-reno Architecture: all Depends: python-pbr (>= 1.4), - python-yaml (>= 3.1.0), + python-yaml, ${misc:Depends}, ${python:Depends}, Suggests: python-reno-doc, @@ -54,7 +52,7 @@ Description: RElease NOtes manager - Python 2.x Package: python3-reno Architecture: all Depends: python3-pbr (>= 1.4), - python3-yaml (>= 3.1.0), + python3-yaml, ${misc:Depends}, ${python3:Depends}, Suggests: python-reno-doc, -- GitLab From c8d7a76dacd652627f94c94aeb9abe34e14302d4 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sat, 16 Jan 2016 06:26:41 +0000 Subject: [PATCH 005/257] Fixed debian/copyright. --- debian/copyright | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/debian/copyright b/debian/copyright index 9bd26f8..04d565e 100644 --- a/debian/copyright +++ b/debian/copyright @@ -2,12 +2,13 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: reno Source: http://www.openstack.org/ -Files: debian/* -Copyright: 2015, Ivan Udovichenko +Files: * +Copyright: (c) 2015-2016, OpenStack Foundation License: Apache-2 -Files: * -Copyright: (c) 2013, OpenStack +Files: debian/* +Copyright: (c) 2015, Ivan Udovichenko + (c) 2016, Thomas Goirand License: Apache-2 License: Apache-2 -- GitLab From cda96faad888742ebb9760969939c72456ad8f57 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 17 Jan 2016 06:14:22 +0000 Subject: [PATCH 006/257] Fixed gbp.conf --- debian/gbp.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/gbp.conf b/debian/gbp.conf index fd8ec27..10f9500 100644 --- a/debian/gbp.conf +++ b/debian/gbp.conf @@ -4,6 +4,6 @@ debian-branch = debian/unstable upstream-tag = %(version)s compression = xz -[git-buildpackage] +[buildpackage] export-dir = ../build-area/ -- GitLab From cbe66e6246d8f2830151b23699da2fe77bf12587 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 17 Jan 2016 06:14:34 +0000 Subject: [PATCH 007/257] Added OSLO_PACKAGE_VERSION and disabled unit tests. --- debian/rules | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/debian/rules b/debian/rules index c7abe1c..268c98f 100755 --- a/debian/rules +++ b/debian/rules @@ -6,6 +6,8 @@ PYTHON3S:=$(shell py3versions -vr) UPSTREAM_GIT = git://github.com/openstack/reno.git -include /usr/share/openstack-pkg-tools/pkgos.make +export OSLO_PACKAGE_VERSION=$(shell dpkg-parsechangelog | grep Version: | cut -d' ' -f2 | sed -e 's/^[[:digit:]]*://' -e 's/[-].*//' -e 's/~/.0/' | head -n 1) + %: dh $@ --buildsystem=python_distutils --with python2,python3,sphinxdoc @@ -22,9 +24,10 @@ override_dh_install: override_dh_auto_test: ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) - set -e ; for pyvers in $(PYTHONS) $(PYTHON3S); do \ - python$$pyvers setup.py test ; \ - done + echo "Unit tests are timing out generating GPG keys..." +# set -e ; for pyvers in $(PYTHONS) $(PYTHON3S); do \ +# python$$pyvers setup.py test ; \ +# done endif override_dh_sphinxdoc: -- GitLab From cd8cabfa225c415a5c61c1afa2fe6cce1332a0ab Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 17 Jan 2016 06:16:13 +0000 Subject: [PATCH 008/257] Copy .rst files instead of generating html doc --- debian/rules | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/debian/rules b/debian/rules index 268c98f..ce463a7 100755 --- a/debian/rules +++ b/debian/rules @@ -31,8 +31,11 @@ ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) endif override_dh_sphinxdoc: - sphinx-build -b html doc/source debian/python-reno-doc/usr/share/doc/python-reno-doc/html - dh_sphinxdoc -O--buildsystem=python_distutils + # Disabling sphinx doc generation as it fails if no git files are present + mkdir -p debian/python-reno-doc/usr/share/doc/python-reno-doc/html + cp -auxf doc/source/*.rst debian/python-reno-doc/usr/share/doc/python-reno-doc/html + #sphinx-build -b html doc/source debian/python-reno-doc/usr/share/doc/python-reno-doc/html + #dh_sphinxdoc -O--buildsystem=python_distutils override_dh_clean: -- GitLab From 05728acc306cff150dc86c6e73bc64dcf55bc8ad Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 17 Jan 2016 06:17:29 +0000 Subject: [PATCH 009/257] Take over the changelog. --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 41d0e37..749764e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,4 +2,4 @@ python-reno (1.3.0-1) unstable; urgency=medium * Initial release. (Closes: #XXXXXX) - -- Ivan Udovichenko Thu, 14 Jan 2016 17:45:02 +0200 + -- Thomas Goirand Sun, 17 Jan 2016 06:17:15 +0000 -- GitLab From 34338d66d988b480a1fe11129feac5f64ee8019d Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 17 Jan 2016 06:19:11 +0000 Subject: [PATCH 010/257] changelog closes ITP --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 749764e..5d031bf 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ python-reno (1.3.0-1) unstable; urgency=medium - * Initial release. (Closes: #XXXXXX) + * Initial release. (Closes: #811150) -- Thomas Goirand Sun, 17 Jan 2016 06:17:15 +0000 -- GitLab From f881bbb7741839dbbddcb3c583e21eb5ce112a1c Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 17 Jan 2016 06:19:35 +0000 Subject: [PATCH 011/257] Removed doc-base file. --- debian/python-reno-doc.doc-base | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 debian/python-reno-doc.doc-base diff --git a/debian/python-reno-doc.doc-base b/debian/python-reno-doc.doc-base deleted file mode 100644 index b8dd916..0000000 --- a/debian/python-reno-doc.doc-base +++ /dev/null @@ -1,9 +0,0 @@ -Document: python-reno-doc -Title: reno Documentation -Author: N/A -Abstract: Sphinx documentation for reno -Section: Programming/Python - -Format: HTML -Index: /usr/share/doc/python-reno-doc/html/index.html -Files: /usr/share/doc/python-reno-doc/html/* -- GitLab From 21bff7417d53a80953d601356684003ee59bab16 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 17 Jan 2016 06:20:33 +0000 Subject: [PATCH 012/257] Fixed watch file to use github tag. --- debian/watch | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/watch b/debian/watch index 078fafc..59efb5e 100644 --- a/debian/watch +++ b/debian/watch @@ -1,3 +1,4 @@ version=3 -http://pypi.python.org/packages/source/r/reno reno-(.*).tar.gz +opts="uversionmangle=s/\.(b|rc)/~$1/" \ +https://github.com/openstack/reno/tags .*/(\d[\d\.]+)\.tar\.gz -- GitLab From bc17fbe572cb27c8362940264b24e66a66fa0a81 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 17 Jan 2016 06:21:30 +0000 Subject: [PATCH 013/257] Using alternative for /usr/bin/reno --- debian/python-reno.postinst | 11 +++++++++++ debian/python-reno.postrm | 11 +++++++++++ debian/python-reno.prerm | 11 +++++++++++ debian/python3-reno.postinst | 11 +++++++++++ debian/python3-reno.postrm | 11 +++++++++++ debian/python3-reno.prerm | 11 +++++++++++ debian/rules | 2 ++ 7 files changed, 68 insertions(+) create mode 100644 debian/python-reno.postinst create mode 100644 debian/python-reno.postrm create mode 100644 debian/python-reno.prerm create mode 100644 debian/python3-reno.postinst create mode 100644 debian/python3-reno.postrm create mode 100644 debian/python3-reno.prerm diff --git a/debian/python-reno.postinst b/debian/python-reno.postinst new file mode 100644 index 0000000..341c60e --- /dev/null +++ b/debian/python-reno.postinst @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +if [ "$1" = "configure" ] ; then + update-alternatives --install /usr/bin/reno reno /usr/bin/python2-reno 300 +fi + +#DEBHELPER# + +exit 0 diff --git a/debian/python-reno.postrm b/debian/python-reno.postrm new file mode 100644 index 0000000..3bb38a0 --- /dev/null +++ b/debian/python-reno.postrm @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +if [ "$1" = "remove" ] || [ "$1" = "disappear" ] ; then + update-alternatives --remove reno /usr/bin/python2-reno +fi + +#DEBHELPER# + +exit 0 diff --git a/debian/python-reno.prerm b/debian/python-reno.prerm new file mode 100644 index 0000000..1e512a6 --- /dev/null +++ b/debian/python-reno.prerm @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +if [ "$1" = "remove" ] ; then + update-alternatives --remove reno /usr/bin/python2-reno +fi + +#DEBHELPER# + +exit 0 diff --git a/debian/python3-reno.postinst b/debian/python3-reno.postinst new file mode 100644 index 0000000..d2689da --- /dev/null +++ b/debian/python3-reno.postinst @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +if [ "$1" = "configure" ] ; then + update-alternatives --install /usr/bin/reno reno /usr/bin/python3-reno 200 +fi + +#DEBHELPER# + +exit 0 diff --git a/debian/python3-reno.postrm b/debian/python3-reno.postrm new file mode 100644 index 0000000..7a3f7d6 --- /dev/null +++ b/debian/python3-reno.postrm @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +if [ "$1" = "remove" ] || [ "$1" = "disappear" ] ; then + update-alternatives --remove reno /usr/bin/python3-reno +fi + +#DEBHELPER# + +exit 0 diff --git a/debian/python3-reno.prerm b/debian/python3-reno.prerm new file mode 100644 index 0000000..b05f778 --- /dev/null +++ b/debian/python3-reno.prerm @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +if [ "$1" = "remove" ] ; then + update-alternatives --remove reno /usr/bin/python3-reno +fi + +#DEBHELPER# + +exit 0 diff --git a/debian/rules b/debian/rules index ce463a7..d0d7bc0 100755 --- a/debian/rules +++ b/debian/rules @@ -21,6 +21,8 @@ override_dh_install: --root $(CURDIR)/debian/python3-reno; \ done rm -rf $(CURDIR)/debian/python*-reno/usr/lib/python*/dist-packages/*.pth + mv $(CURDIR)/debian/python-reno/usr/bin/reno $(CURDIR)/debian/python-reno/usr/bin/python2-reno + mv $(CURDIR)/debian/python3-reno/usr/bin/reno $(CURDIR)/debian/python3-reno/usr/bin/python3-reno override_dh_auto_test: ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) -- GitLab From a3863fdcd74389c359f707bb3f95088771347eb1 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Mon, 18 Jan 2016 05:01:00 +0000 Subject: [PATCH 014/257] Fixed copyright according to FTP masters recommendations. --- debian/copyright | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/copyright b/debian/copyright index 04d565e..7adc8f2 100644 --- a/debian/copyright +++ b/debian/copyright @@ -3,7 +3,8 @@ Upstream-Name: reno Source: http://www.openstack.org/ Files: * -Copyright: (c) 2015-2016, OpenStack Foundation +Copyright: (c) 2010-2016, OpenStack Foundation + (c) 2013, Hewlett-Packard Development Company, L.P. License: Apache-2 Files: debian/* -- GitLab From 207776787f3d4ac7fdbe2e4a7e50e1a8a0304bbd Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 26 Jan 2016 12:46:47 -0500 Subject: [PATCH 015/257] fix detection of pre-release tags in git log We had two different regexes being used for detecting tags. Unify them into a single expression that works in both situations. Convert that expression to use "verbose" mode to include some inline documentation for the parts of the pattern. Add more details to the existing debug output when new tags are found. Add more debug output at the end to show how many files were actually detected. Change-Id: I7104f459c948011f198fed04303ea5cafb59f223 Closes-Bug: #1537451 Signed-off-by: Doug Hellmann --- .../notes/bug-1537451-f44591da125ba09d.yaml | 6 ++ reno/scanner.py | 15 +++-- reno/tests/test_scanner.py | 60 +++++++++++++++++++ 3 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/bug-1537451-f44591da125ba09d.yaml diff --git a/releasenotes/notes/bug-1537451-f44591da125ba09d.yaml b/releasenotes/notes/bug-1537451-f44591da125ba09d.yaml new file mode 100644 index 0000000..f033466 --- /dev/null +++ b/releasenotes/notes/bug-1537451-f44591da125ba09d.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - Resolves `a bug `__ + with properly detecting pre-release versions in the existing + history of a repository that resulted in some release notes not + appearing in the report output. diff --git a/reno/scanner.py b/reno/scanner.py index e8593da..261d8dd 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -22,7 +22,6 @@ import sys from reno import utils -_TAG_PAT = re.compile('tag: ([\d\.]+)') LOG = logging.getLogger(__name__) @@ -111,7 +110,11 @@ def _get_unique_id(filename): # lines that have no tags or are completely blank, and we might have # "tag:" or not. This pattern is used to find the tag entries on each # line, ignoring tags that don't look like version numbers. -TAG_RE = re.compile('(?:[(]|tag: )([\d.ab]+)[,)]') +TAG_RE = re.compile(''' + (?:[(]|tag:\s) # look for tag: prefix and drop + ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and pre-releases + [,)] # possible trailing comma or closing paren +''', flags=re.VERBOSE | re.UNICODE) def _get_version_tags_on_branch(reporoot, branch): @@ -204,7 +207,7 @@ def get_notes_by_version(reporoot, notesdir, branch=None): # The first line of the block will include the SHA and may # include tags, the other lines are filenames. sha = hlines[0].split(' ')[0] - tags = _TAG_PAT.findall(hlines[0]) + tags = TAG_RE.findall(hlines[0]) # Filter the files based on the notes directory we were # given. We cannot do this in the git log command directly # because it means we end up skipping some of the tags if the @@ -224,8 +227,8 @@ def get_notes_by_version(reporoot, notesdir, branch=None): tags = [current_version] else: current_version = tags[0] - LOG.debug('%s has tags, updating current version to %s' % - (sha, current_version)) + LOG.debug('%s has tags %s (%r), updating current version to %s' % + (sha, tags, hlines[0], current_version)) # Remember each version we have seen. if current_version not in versions: @@ -291,4 +294,6 @@ def get_notes_by_version(reporoot, notesdir, branch=None): # so just sort based on the unique id. trimmed[ov] = sorted(files_and_tags[ov]) + LOG.debug('[reno] found %d versions and %d files', + len(trimmed.keys()), sum(len(ov) for ov in trimmed.values())) return trimmed diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 1a334a2..fb29e97 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -460,6 +460,66 @@ class BasicTest(Base): ) +class PreReleaseTest(Base): + + def test_alpha(self): + self._make_python_package() + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a1') + f1 = self._add_notes_file('slug1') + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a2') + raw_results = scanner.get_notes_by_version( + self.reporoot, + 'releasenotes/notes', + ) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0.0a2': [f1], + }, + results, + ) + + def test_beta(self): + self._make_python_package() + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b1') + f1 = self._add_notes_file('slug1') + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b2') + raw_results = scanner.get_notes_by_version( + self.reporoot, + 'releasenotes/notes', + ) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0.0b2': [f1], + }, + results, + ) + + def test_release_candidate(self): + self._make_python_package() + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc1') + f1 = self._add_notes_file('slug1') + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc2') + raw_results = scanner.get_notes_by_version( + self.reporoot, + 'releasenotes/notes', + ) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0.0rc2': [f1], + }, + results, + ) + + class MergeCommitTest(Base): def test_1(self): -- GitLab From 53891ddb1a55e0d6a801c75c3df7d3073a4749f7 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 27 Jan 2016 17:20:17 -0500 Subject: [PATCH 016/257] add flag to collapse pre-releases into final releases Add a new flag to combine pre-release notes into their final release version after that version is tagged. The flag defaults to False to preserve the current behavior. Change-Id: I6b9c058289f0baa3e39048b3fa78f6af81bdd83b Signed-off-by: Doug Hellmann --- doc/source/sphinxext.rst | 6 ++ ...ollapse-pre-releases-0b24e0bab46d7cf1.yaml | 4 + reno/lister.py | 6 +- reno/main.py | 12 +++ reno/report.py | 6 +- reno/scanner.py | 46 ++++++++++- reno/sphinxext.py | 7 +- reno/tests/test_scanner.py | 77 +++++++++++++++++++ 8 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/collapse-pre-releases-0b24e0bab46d7cf1.yaml diff --git a/doc/source/sphinxext.rst b/doc/source/sphinxext.rst index b43407c..e1ad86a 100644 --- a/doc/source/sphinxext.rst +++ b/doc/source/sphinxext.rst @@ -46,6 +46,12 @@ Enable the extension by adding ``'reno.sphinxext'`` to the A comma separated list of versions to include in the notes. The default is to include all versions found on ``branch``. + *collapse-pre-releases* + + A flag indicating that notes attached to pre-release versions + should be incorporated into the notes for the final release, + after the final release is tagged. + Examples ======== diff --git a/releasenotes/notes/collapse-pre-releases-0b24e0bab46d7cf1.yaml b/releasenotes/notes/collapse-pre-releases-0b24e0bab46d7cf1.yaml new file mode 100644 index 0000000..9cd1675 --- /dev/null +++ b/releasenotes/notes/collapse-pre-releases-0b24e0bab46d7cf1.yaml @@ -0,0 +1,4 @@ +--- +features: + - Add a flag to collapse pre-release notes into their final release, + if the final release tag is present. \ No newline at end of file diff --git a/reno/lister.py b/reno/lister.py index b60267a..c71f0af 100644 --- a/reno/lister.py +++ b/reno/lister.py @@ -25,7 +25,11 @@ def list_cmd(args): LOG.debug('starting list') reporoot = args.reporoot.rstrip('/') + '/' notesdir = utils.get_notes_dir(args) - notes = scanner.get_notes_by_version(reporoot, notesdir, args.branch) + collapse = args.collapse_pre_releases + notes = scanner.get_notes_by_version( + reporoot, notesdir, args.branch, + collapse_pre_releases=collapse, + ) if args.version: versions = args.version else: diff --git a/reno/main.py b/reno/main.py index 7f4cfb5..867c035 100644 --- a/reno/main.py +++ b/reno/main.py @@ -76,6 +76,12 @@ def main(argv=sys.argv[1:]): default=None, help='the branch to scan, defaults to the current', ) + do_list.add_argument( + '--collapse-pre-releases', + action='store_true', + default=False, + help='combine pre-releases with their final release', + ) do_list.set_defaults(func=lister.list_cmd) do_report = subparsers.add_parser( @@ -102,6 +108,12 @@ def main(argv=sys.argv[1:]): action='append', help='the version(s) to include, defaults to all', ) + do_report.add_argument( + '--collapse-pre-releases', + action='store_true', + default=False, + help='combine pre-releases with their final release', + ) do_report.set_defaults(func=report.report_cmd) args = parser.parse_args() diff --git a/reno/report.py b/reno/report.py index 8f37c09..d7d3c65 100644 --- a/reno/report.py +++ b/reno/report.py @@ -21,7 +21,11 @@ def report_cmd(args): "Generates a release notes report" reporoot = args.reporoot.rstrip('/') + '/' notesdir = utils.get_notes_dir(args) - notes = scanner.get_notes_by_version(reporoot, notesdir, args.branch) + collapse = args.collapse_pre_releases + notes = scanner.get_notes_by_version( + reporoot, notesdir, args.branch, + collapse_pre_releases=collapse, + ) if args.version: versions = args.version else: diff --git a/reno/scanner.py b/reno/scanner.py index 261d8dd..5867e97 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -115,6 +115,9 @@ TAG_RE = re.compile(''' ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and pre-releases [,)] # possible trailing comma or closing paren ''', flags=re.VERBOSE | re.UNICODE) +PRE_RELEASE_RE = re.compile(''' + \.(\d+(?:[ab]|rc)+\d*)$ +''', flags=re.VERBOSE | re.UNICODE) def _get_version_tags_on_branch(reporoot, branch): @@ -146,13 +149,24 @@ def _get_version_tags_on_branch(reporoot, branch): return tags -def get_notes_by_version(reporoot, notesdir, branch=None): +def get_notes_by_version(reporoot, notesdir, branch=None, + collapse_pre_releases=False): """Return an OrderedDict mapping versions to lists of notes files. The versions are presented in reverse chronological order. Notes files are associated with the earliest version for which they were available, regardless of whether they changed later. + + :param reporoot: Path to the root of the git repository. + :type reporoot: str + :param notesdir: The directory under *reporoot* with the release notes. + :type notesdir: str + :param branch: The name of the branch to scan. Defaults to current. + :type branch: str + :param collapse_pre_releases: When true, merge pre-release versions + into the final release, if it is present. + :type collapse_pre_releases: bool """ LOG.debug('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch)) @@ -277,6 +291,36 @@ def get_notes_by_version(reporoot, notesdir, branch=None): LOG.debug(msg) print(msg, file=sys.stderr) + # Combine pre-releases into the final release, if we are told to + # and the final release exists. + if collapse_pre_releases: + collapsing = files_and_tags + files_and_tags = collections.OrderedDict() + for ov in versions_by_date: + if ov not in collapsing: + # We don't need to collapse this one because there are + # no notes attached to it. + continue + pre_release_match = PRE_RELEASE_RE.search(ov) + LOG.debug('checking %r', ov) + if pre_release_match: + # Remove the trailing pre-release part of the version + # from the string. + pre_rel_str = pre_release_match.groups()[0] + canonical_ver = ov[:-len(pre_rel_str)].rstrip('.') + if canonical_ver not in versions_by_date: + # This canonical version was never tagged, so we + # do not want to collapse the pre-releases. Reset + # to the original version. + canonical_ver = ov + else: + LOG.debug('combining into %r', canonical_ver) + else: + canonical_ver = ov + if canonical_ver not in files_and_tags: + files_and_tags[canonical_ver] = [] + files_and_tags[canonical_ver].extend(collapsing[ov]) + # Only return the parts of files_and_tags that actually have # filenames associated with the versions. trimmed = collections.OrderedDict() diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 07359e5..f1d62e5 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -33,6 +33,7 @@ class ReleaseNotesDirective(rst.Directive): 'relnotessubdir': directives.unchanged, 'notesdir': directives.unchanged, 'version': directives.unchanged, + 'collapse-pre-releases': directives.flag, } def run(self): @@ -50,12 +51,16 @@ class ReleaseNotesDirective(rst.Directive): defaults.RELEASE_NOTES_SUBDIR) notessubdir = self.options.get('notesdir', defaults.NOTES_SUBDIR) version_opt = self.options.get('version') + collapse = self.options.get('collapse-pre-releases') notesdir = os.path.join(relnotessubdir, notessubdir) info('scanning %s for %s release notes' % (os.path.join(reporoot, notesdir), branch or 'current branch')) - notes = scanner.get_notes_by_version(reporoot, notesdir, branch) + notes = scanner.get_notes_by_version( + reporoot, notesdir, branch, + collapse_pre_releases=collapse, + ) if version_opt is not None: versions = [ v.strip() diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index fb29e97..33ddb12 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -519,6 +519,83 @@ class PreReleaseTest(Base): results, ) + def test_collapse(self): + files = [] + self._make_python_package() + files.append(self._add_notes_file('slug1')) + self._run_git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') + files.append(self._add_notes_file('slug2')) + self._run_git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') + files.append(self._add_notes_file('slug3')) + self._run_git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') + files.append(self._add_notes_file('slug4')) + self._run_git('tag', '-s', '-m', 'full release tag', '1.0.0') + raw_results = scanner.get_notes_by_version( + self.reporoot, + 'releasenotes/notes', + collapse_pre_releases=True, + ) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0': files, + }, + results, + ) + + def test_collapse_without_full_release(self): + self._make_python_package() + f1 = self._add_notes_file('slug1') + self._run_git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') + f2 = self._add_notes_file('slug2') + self._run_git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') + f3 = self._add_notes_file('slug3') + self._run_git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') + raw_results = scanner.get_notes_by_version( + self.reporoot, + 'releasenotes/notes', + collapse_pre_releases=True, + ) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0.0a1': [f1], + '1.0.0.0b1': [f2], + '1.0.0.0rc1': [f3], + }, + results, + ) + + def test_collapse_without_notes(self): + self._make_python_package() + self._run_git('tag', '-s', '-m', 'earlier tag', '0.1.0') + f1 = self._add_notes_file('slug1') + self._run_git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') + f2 = self._add_notes_file('slug2') + self._run_git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') + f3 = self._add_notes_file('slug3') + self._run_git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') + raw_results = scanner.get_notes_by_version( + self.reporoot, + 'releasenotes/notes', + collapse_pre_releases=True, + ) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0.0a1': [f1], + '1.0.0.0b1': [f2], + '1.0.0.0rc1': [f3], + }, + results, + ) + class MergeCommitTest(Base): -- GitLab From 75e06c544188321ae3784544d0d26451406b054b Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 3 Feb 2016 19:21:44 -0500 Subject: [PATCH 017/257] add earliest_version option to scanner Add an option to the scanner to tell it not to include all of the history, but to stop at a specific version (inclusive). This will allow us to configure release series history pages to only show versions that are part of that series (otherwise, scanning stable/newton will include mitaka releases from the master branch, for example). Change-Id: I53b4b95e13c99d0a19f53e2f3e836ffe67428211 Signed-off-by: Doug Hellmann --- doc/source/sphinxext.rst | 7 +++++++ reno/lister.py | 1 + reno/main.py | 10 ++++++++++ reno/report.py | 1 + reno/scanner.py | 7 ++++++- reno/sphinxext.py | 3 +++ reno/tests/test_scanner.py | 24 ++++++++++++++++++++++++ 7 files changed, 52 insertions(+), 1 deletion(-) diff --git a/doc/source/sphinxext.rst b/doc/source/sphinxext.rst index e1ad86a..1ccf4d7 100644 --- a/doc/source/sphinxext.rst +++ b/doc/source/sphinxext.rst @@ -52,6 +52,13 @@ Enable the extension by adding ``'reno.sphinxext'`` to the should be incorporated into the notes for the final release, after the final release is tagged. + *earliest-version* + + A string containing the version number of the earliest version to + be included. For example, when scanning a branch, this is + typically set to the version used to create the branch to limit + the output to only versions on that branch. + Examples ======== diff --git a/reno/lister.py b/reno/lister.py index c71f0af..eece975 100644 --- a/reno/lister.py +++ b/reno/lister.py @@ -29,6 +29,7 @@ def list_cmd(args): notes = scanner.get_notes_by_version( reporoot, notesdir, args.branch, collapse_pre_releases=collapse, + earliest_version=args.earliest_version, ) if args.version: versions = args.version diff --git a/reno/main.py b/reno/main.py index 867c035..246cb83 100644 --- a/reno/main.py +++ b/reno/main.py @@ -82,6 +82,11 @@ def main(argv=sys.argv[1:]): default=False, help='combine pre-releases with their final release', ) + do_list.add_argument( + '--earliest-version', + default=None, + help='stop when this version is reached in the history', + ) do_list.set_defaults(func=lister.list_cmd) do_report = subparsers.add_parser( @@ -114,6 +119,11 @@ def main(argv=sys.argv[1:]): default=False, help='combine pre-releases with their final release', ) + do_report.add_argument( + '--earliest-version', + default=None, + help='stop when this version is reached in the history', + ) do_report.set_defaults(func=report.report_cmd) args = parser.parse_args() diff --git a/reno/report.py b/reno/report.py index d7d3c65..439ee83 100644 --- a/reno/report.py +++ b/reno/report.py @@ -25,6 +25,7 @@ def report_cmd(args): notes = scanner.get_notes_by_version( reporoot, notesdir, args.branch, collapse_pre_releases=collapse, + earliest_version=args.earliest_version, ) if args.version: versions = args.version diff --git a/reno/scanner.py b/reno/scanner.py index 5867e97..26a5dc6 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -150,7 +150,8 @@ def _get_version_tags_on_branch(reporoot, branch): def get_notes_by_version(reporoot, notesdir, branch=None, - collapse_pre_releases=False): + collapse_pre_releases=False, + earliest_version=None): """Return an OrderedDict mapping versions to lists of notes files. The versions are presented in reverse chronological order. @@ -337,6 +338,10 @@ def get_notes_by_version(reporoot, notesdir, branch=None, # same order, but it doesn't really matter what order that is, # so just sort based on the unique id. trimmed[ov] = sorted(files_and_tags[ov]) + # If we have been told to stop at a version, we can do that + # now. + if earliest_version and ov == earliest_version: + break LOG.debug('[reno] found %d versions and %d files', len(trimmed.keys()), sum(len(ov) for ov in trimmed.values())) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index f1d62e5..81723a8 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -34,6 +34,7 @@ class ReleaseNotesDirective(rst.Directive): 'notesdir': directives.unchanged, 'version': directives.unchanged, 'collapse-pre-releases': directives.flag, + 'earliest-version': directives.unchanged, } def run(self): @@ -52,6 +53,7 @@ class ReleaseNotesDirective(rst.Directive): notessubdir = self.options.get('notesdir', defaults.NOTES_SUBDIR) version_opt = self.options.get('version') collapse = self.options.get('collapse-pre-releases') + earliest_version = self.options.get('earliest-version') notesdir = os.path.join(relnotessubdir, notessubdir) info('scanning %s for %s release notes' % @@ -60,6 +62,7 @@ class ReleaseNotesDirective(rst.Directive): notes = scanner.get_notes_by_version( reporoot, notesdir, branch, collapse_pre_releases=collapse, + earliest_version=earliest_version, ) if version_opt is not None: versions = [ diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 33ddb12..7e5e3ef 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -459,6 +459,30 @@ class BasicTest(Base): results, ) + def test_limit_by_earliest_version(self): + self._make_python_package() + self._add_notes_file() + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + f2 = self._add_notes_file() + self._run_git('tag', '-s', '-m', 'middle tag', '2.0.0') + f3 = self._add_notes_file() + self._run_git('tag', '-s', '-m', 'last tag', '3.0.0') + raw_results = scanner.get_notes_by_version( + self.reporoot, + 'releasenotes/notes', + earliest_version='2.0.0', + ) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'2.0.0': [f2], + '3.0.0': [f3], + }, + results, + ) + class PreReleaseTest(Base): -- GitLab From f4e2d66942bf344cf0974d87ff16e27e5803f05a Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 8 Feb 2016 16:43:21 -0500 Subject: [PATCH 018/257] add release note for earliest-version feature Change-Id: I595f2b2d3eead9b3c6c5aa8a11d6745c5fead0d4 Signed-off-by: Doug Hellmann --- .../notes/add-earliest-version-6f3d634770e855d0.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 releasenotes/notes/add-earliest-version-6f3d634770e855d0.yaml diff --git a/releasenotes/notes/add-earliest-version-6f3d634770e855d0.yaml b/releasenotes/notes/add-earliest-version-6f3d634770e855d0.yaml new file mode 100644 index 0000000..b5454d3 --- /dev/null +++ b/releasenotes/notes/add-earliest-version-6f3d634770e855d0.yaml @@ -0,0 +1,6 @@ +--- +features: + - Add the ability to limit queries by stopping at an "earliest + version". This is intended to be used when scanning a branch, for + example, to stop at a point when the branch was created and not + include all of the history from the parent branch. \ No newline at end of file -- GitLab From 11c887ef62ab761de834c3fe0466092bd5b6161c Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 10 Feb 2016 17:04:36 -0500 Subject: [PATCH 019/257] collapse pre-release notes into regular releases by default Change the default for the collapse flag to True and provide options for setting it to false. Change-Id: Ia17f39dc3bb576f26f159eee3903445a2f541c14 Signed-off-by: Doug Hellmann --- reno/main.py | 16 ++++++++++++++-- reno/scanner.py | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/reno/main.py b/reno/main.py index 246cb83..b117ee2 100644 --- a/reno/main.py +++ b/reno/main.py @@ -79,9 +79,15 @@ def main(argv=sys.argv[1:]): do_list.add_argument( '--collapse-pre-releases', action='store_true', - default=False, + default=True, help='combine pre-releases with their final release', ) + do_list.add_argument( + '--no-collapse-pre-releases', + action='store_false', + dest='collapse_pre_releases', + help='show pre-releases separately', + ) do_list.add_argument( '--earliest-version', default=None, @@ -116,9 +122,15 @@ def main(argv=sys.argv[1:]): do_report.add_argument( '--collapse-pre-releases', action='store_true', - default=False, + default=True, help='combine pre-releases with their final release', ) + do_report.add_argument( + '--no-collapse-pre-releases', + action='store_false', + dest='collapse_pre_releases', + help='show pre-releases separately', + ) do_report.add_argument( '--earliest-version', default=None, diff --git a/reno/scanner.py b/reno/scanner.py index 26a5dc6..15b645d 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -150,7 +150,7 @@ def _get_version_tags_on_branch(reporoot, branch): def get_notes_by_version(reporoot, notesdir, branch=None, - collapse_pre_releases=False, + collapse_pre_releases=True, earliest_version=None): """Return an OrderedDict mapping versions to lists of notes files. -- GitLab From 5cca59c865113bc48f8d3d2edd8ed00e14ba71eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nov=C3=BD?= Date: Sun, 28 Feb 2016 15:48:25 +0100 Subject: [PATCH 020/257] Fixed VCS URLs (https). --- debian/changelog | 6 ++++++ debian/control | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 5d031bf..c6662c6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +python-reno (1.3.0-2) UNRELEASED; urgency=medium + + * Fixed VCS URLs (https). + + -- Ondřej Nový Sun, 28 Feb 2016 15:48:25 +0100 + python-reno (1.3.0-1) unstable; urgency=medium * Initial release. (Closes: #811150) diff --git a/debian/control b/debian/control index 0741812..2b745e5 100644 --- a/debian/control +++ b/debian/control @@ -32,8 +32,8 @@ Build-Depends-Indep: python-babel, subunit, testrepository, Standards-Version: 3.9.6 -Vcs-Browser: http://anonscm.debian.org/gitweb/?p=openstack/python-reno.git -Vcs-Git: git://anonscm.debian.org/openstack/python-reno.git +Vcs-Browser: https://anonscm.debian.org/cgit/openstack/python-reno.git/ +Vcs-Git: https://anonscm.debian.org/git/openstack/python-reno.git Homepage: http://www.openstack.org/ Package: python-reno -- GitLab From a9362189cab9bab0559f8937f0ece6eae7b4af41 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sun, 28 Feb 2016 10:39:21 -0500 Subject: [PATCH 021/257] use less entropy in unit tests Fix the logic for dealing with entropy in the unit tests so we consume less. Change-Id: I1faebfd5de0b9ae150bc2298df5d797fbbf83c07 Signed-off-by: Doug Hellmann --- reno/tests/test_scanner.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 7e5e3ef..4e6d0bb 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -65,7 +65,7 @@ class GPGKeyFixture(fixtures.Fixture): gnupg_version_re = re.compile('^gpg\s.*\s([\d+])\.([\d+])\.([\d+])') gnupg_version = utils.check_output(['gpg', '--version'], cwd=tempdir.path) - for line in gnupg_version[0].split('\n'): + for line in gnupg_version.split('\n'): gnupg_version = gnupg_version_re.match(line) if gnupg_version: gnupg_version = (int(gnupg_version.group(1)), @@ -98,17 +98,17 @@ class GPGKeyFixture(fixtures.Fixture): # Note that --quick-random (--debug-quick-random in GnuPG 2.x) # does not have a corresponding preferences file setting and # must be passed explicitly on the command line instead - # if gnupg_version[0] == 1: - # gnupg_random = '--quick-random' - # elif gnupg_version[0] >= 2: - # gnupg_random = '--debug-quick-random' - # else: - # gnupg_random = '' - subprocess.check_call( - ['gpg', '--gen-key', '--batch', - # gnupg_random, - config_file], - cwd=tempdir.path) + if gnupg_version[0] == 1: + gnupg_random = '--quick-random' + elif gnupg_version[0] >= 2: + gnupg_random = '--debug-quick-random' + else: + gnupg_random = '' + cmd = ['gpg', '--gen-key', '--batch'] + if gnupg_random: + cmd.append(gnupg_random) + cmd.append(config_file) + subprocess.check_call(cmd, cwd=tempdir.path) class Base(base.TestCase): -- GitLab From 668013575a9da988fd818ecd8920c87af2f28c8f Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Mon, 29 Feb 2016 17:35:37 +0000 Subject: [PATCH 022/257] * Use patch to use less entropy in unit tests. * Reactivated unit tests. --- debian/changelog | 9 ++- debian/patches/series | 1 + .../use_less_entropy_in_unit_tests.patch | 59 +++++++++++++++++++ debian/rules | 16 +++-- 4 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 debian/patches/series create mode 100644 debian/patches/use_less_entropy_in_unit_tests.patch diff --git a/debian/changelog b/debian/changelog index c6662c6..3077637 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,13 @@ -python-reno (1.3.0-2) UNRELEASED; urgency=medium +python-reno (1.3.0-2) unstable; urgency=medium + [ Thomas Goirand ] + * Use patch to use less entropy in unit tests. + * Reactivated unit tests. + + [ Ondřej Nový ] * Fixed VCS URLs (https). - -- Ondřej Nový Sun, 28 Feb 2016 15:48:25 +0100 + -- Thomas Goirand Mon, 29 Feb 2016 17:34:49 +0000 python-reno (1.3.0-1) unstable; urgency=medium diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 0000000..cc26a2e --- /dev/null +++ b/debian/patches/series @@ -0,0 +1 @@ +use_less_entropy_in_unit_tests.patch diff --git a/debian/patches/use_less_entropy_in_unit_tests.patch b/debian/patches/use_less_entropy_in_unit_tests.patch new file mode 100644 index 0000000..da9f7e3 --- /dev/null +++ b/debian/patches/use_less_entropy_in_unit_tests.patch @@ -0,0 +1,59 @@ +From a9362189cab9bab0559f8937f0ece6eae7b4af41 Mon Sep 17 00:00:00 2001 +From: Doug Hellmann +Date: Sun, 28 Feb 2016 10:39:21 -0500 +Subject: [PATCH] use less entropy in unit tests + +Fix the logic for dealing with entropy in the unit tests so we consume +less. + +Change-Id: I1faebfd5de0b9ae150bc2298df5d797fbbf83c07 +Signed-off-by: Doug Hellmann +--- + reno/tests/test_scanner.py | 24 ++++++++++++------------ + 1 file changed, 12 insertions(+), 12 deletions(-) + +diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py +index 7e5e3ef..4e6d0bb 100644 +--- a/reno/tests/test_scanner.py ++++ b/reno/tests/test_scanner.py +@@ -65,7 +65,7 @@ class GPGKeyFixture(fixtures.Fixture): + gnupg_version_re = re.compile('^gpg\s.*\s([\d+])\.([\d+])\.([\d+])') + gnupg_version = utils.check_output(['gpg', '--version'], + cwd=tempdir.path) +- for line in gnupg_version[0].split('\n'): ++ for line in gnupg_version.split('\n'): + gnupg_version = gnupg_version_re.match(line) + if gnupg_version: + gnupg_version = (int(gnupg_version.group(1)), +@@ -98,17 +98,17 @@ class GPGKeyFixture(fixtures.Fixture): + # Note that --quick-random (--debug-quick-random in GnuPG 2.x) + # does not have a corresponding preferences file setting and + # must be passed explicitly on the command line instead +- # if gnupg_version[0] == 1: +- # gnupg_random = '--quick-random' +- # elif gnupg_version[0] >= 2: +- # gnupg_random = '--debug-quick-random' +- # else: +- # gnupg_random = '' +- subprocess.check_call( +- ['gpg', '--gen-key', '--batch', +- # gnupg_random, +- config_file], +- cwd=tempdir.path) ++ if gnupg_version[0] == 1: ++ gnupg_random = '--quick-random' ++ elif gnupg_version[0] >= 2: ++ gnupg_random = '--debug-quick-random' ++ else: ++ gnupg_random = '' ++ cmd = ['gpg', '--gen-key', '--batch'] ++ if gnupg_random: ++ cmd.append(gnupg_random) ++ cmd.append(config_file) ++ subprocess.check_call(cmd, cwd=tempdir.path) + + + class Base(base.TestCase): +-- +1.9.1 + diff --git a/debian/rules b/debian/rules index d0d7bc0..c895884 100755 --- a/debian/rules +++ b/debian/rules @@ -26,10 +26,18 @@ override_dh_install: override_dh_auto_test: ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) - echo "Unit tests are timing out generating GPG keys..." -# set -e ; for pyvers in $(PYTHONS) $(PYTHON3S); do \ -# python$$pyvers setup.py test ; \ -# done + @echo "===> Running tests" + set -e ; set -x ; for i in 2.7 $(PYTHON3S) ; do \ + PYMAJOR=`echo $$i | cut -d'.' -f1` ; \ + echo "===> Testing with python$$i (python$$PYMAJOR)" ; \ + rm -rf .testrepository ; \ + testr-python$$PYMAJOR init ; \ + TEMP_REZ=`mktemp -t` ; \ + PYTHONPATH=$(CURDIR) PYTHON=python$$i testr-python$$PYMAJOR run --subunit | tee $$TEMP_REZ | subunit2pyunit ; \ + cat $$TEMP_REZ | subunit-filter -s --no-passthrough | subunit-stats ; \ + rm -f $$TEMP_REZ ; \ + testr-python$$PYMAJOR slowest ; \ + done endif override_dh_sphinxdoc: -- GitLab From 87b8e8cca21fa565d2c7db1ff80dd77d43711a6b Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Mon, 29 Feb 2016 17:46:39 +0000 Subject: [PATCH 023/257] Fix patch header --- .../patches/use_less_entropy_in_unit_tests.patch | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/debian/patches/use_less_entropy_in_unit_tests.patch b/debian/patches/use_less_entropy_in_unit_tests.patch index da9f7e3..3cb8ef5 100644 --- a/debian/patches/use_less_entropy_in_unit_tests.patch +++ b/debian/patches/use_less_entropy_in_unit_tests.patch @@ -1,16 +1,12 @@ -From a9362189cab9bab0559f8937f0ece6eae7b4af41 Mon Sep 17 00:00:00 2001 -From: Doug Hellmann +Description: use less entropy in unit tests + Fix the logic for dealing with entropy in the unit tests so we consume + less. +Author: Doug Hellmann Date: Sun, 28 Feb 2016 10:39:21 -0500 -Subject: [PATCH] use less entropy in unit tests - -Fix the logic for dealing with entropy in the unit tests so we consume -less. - Change-Id: I1faebfd5de0b9ae150bc2298df5d797fbbf83c07 Signed-off-by: Doug Hellmann ---- - reno/tests/test_scanner.py | 24 ++++++++++++------------ - 1 file changed, 12 insertions(+), 12 deletions(-) +Origin: upstream, https://review.openstack.org/#/c/285812 +Last-Update: 2016-03-01 diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 7e5e3ef..4e6d0bb 100644 -- GitLab From 13b932f0f2e5f180f75b22583a7b9d9a4a83521a Mon Sep 17 00:00:00 2001 From: Mike Perez Date: Tue, 1 Mar 2016 11:01:32 -0800 Subject: [PATCH 024/257] Add deprecations section to usage documentation This includes information on the section deprecations and a consideration when using it. Change-Id: I581a8afa3491eef74af1472c57d0c35273f45f89 --- doc/source/usage.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/usage.rst b/doc/source/usage.rst index e9e1269..74a5390 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -64,6 +64,13 @@ upgrade A list of upgrade notes in the release. For example, if a database schema alteration is needed. +deprecations + + A list of features, APIs, configuration options to be deprecated in the + release. Deprecations should not be used for something that is removed in the + release, use upgrade section instead. Deprecation should allow time for users + to make necessary changes for the removal to happen in a future release. + critical A list of *fixed* critical bugs. @@ -96,6 +103,8 @@ entirely. - List known issues here, or remove this section. upgrade: - List upgrade notes here, or remove this section. + deprecations: + - List deprecation notes here, or remove this section critical: - Add critical notes here, or remove this section. security: -- GitLab From 627a1da7ce8e2db6554a5f220111cb00d1670a46 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 7 Mar 2016 13:24:51 -0500 Subject: [PATCH 025/257] always show coverage report for test runs Change-Id: I487137bd8a57c7cabee46ad9e7b9180702561c17 Signed-off-by: Doug Hellmann --- .gitignore | 1 + tox.ini | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e625780..ad619f7 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ ChangeLog *~ .*.swp .*sw? +/cover/ diff --git a/tox.ini b/tox.ini index 884c772..cdb05ea 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,9 @@ setenv = deps = -r{toxinidir}/test-requirements.txt .[sphinx] -commands = python setup.py test --slowest --testr-args='{posargs}' +commands = + python setup.py test --slowest --coverage --coverage-package-name=reno --testr-args='{posargs}' + coverage report --show-missing [testenv:pep8] commands = flake8 -- GitLab From abad748259539914bba35f63bcd59d23ead8aa42 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 7 Mar 2016 14:01:10 -0500 Subject: [PATCH 026/257] improve test coverage Change-Id: I76a22e918bca369c3666b4265b238829f6d0ca0b Signed-off-by: Doug Hellmann --- reno/tests/test_create.py | 22 ++++++++ reno/tests/test_formatter.py | 98 ++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 reno/tests/test_formatter.py diff --git a/reno/tests/test_create.py b/reno/tests/test_create.py index 6663dd8..7b84c16 100644 --- a/reno/tests/test_create.py +++ b/reno/tests/test_create.py @@ -15,6 +15,7 @@ from reno import create from reno.tests import base +import fixtures import mock @@ -29,3 +30,24 @@ class TestPickFileName(base.TestCase): 'somepath', 'someslug', ) + + @mock.patch('os.path.exists') + def test_random_enough(self, exists): + exists.return_value = False + result = create._pick_note_file_name('somepath', 'someslug') + self.assertIn('somepath', result) + self.assertIn('someslug', result) + + +class TestCreate(base.TestCase): + + def setUp(self): + super(TestCreate, self).setUp() + self.tmpdir = self.useFixture(fixtures.TempDir()).path + + def test_create_from_template(self): + filename = create._pick_note_file_name(self.tmpdir, 'theslug') + create._make_note_file(filename) + with open(filename, 'r') as f: + body = f.read() + self.assertEqual(create._TEMPLATE, body) diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py new file mode 100644 index 0000000..04aebb9 --- /dev/null +++ b/reno/tests/test_formatter.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import textwrap + +from reno import formatter +from reno.tests import base + +from oslotest import mockpatch + + +class TestFormatter(base.TestCase): + + scanner_output = { + '0.0.0': [('note1', 'shaA')], + '1.0.0': [('note2', 'shaB'), ('note3', 'shaC')], + } + + versions = ['0.0.0', '1.0.0'] + + note_bodies = { + 'note1': textwrap.dedent(""" + prelude: > + This is the prelude. + """), + 'note2': textwrap.dedent(""" + issues: + - This is the first issue. + - This is the second issue. + """), + 'note3': textwrap.dedent(""" + features: + - We added a feature! + """) + } + + def _get_note_body(self, reporoot, filename, sha): + return self.note_bodies.get(filename, '') + + def setUp(self): + super(TestFormatter, self).setUp() + self.useFixture( + mockpatch.Patch('reno.scanner.get_file_at_commit', + new=self._get_note_body) + ) + + def test_with_title(self): + result = formatter.format_report( + reporoot=None, + scanner_output=self.scanner_output, + versions_to_include=self.versions, + title='This is the title', + ) + self.assertIn('This is the title', result) + + def test_versions(self): + result = formatter.format_report( + reporoot=None, + scanner_output=self.scanner_output, + versions_to_include=self.versions, + title='This is the title', + ) + self.assertIn('0.0.0\n=====', result) + self.assertIn('1.0.0\n=====', result) + + def test_without_title(self): + result = formatter.format_report( + reporoot=None, + scanner_output=self.scanner_output, + versions_to_include=self.versions, + title=None, + ) + self.assertNotIn('This is the title', result) + + def test_section_order(self): + result = formatter.format_report( + reporoot=None, + scanner_output=self.scanner_output, + versions_to_include=self.versions, + title=None, + ) + prelude_pos = result.index('This is the prelude.') + issues_pos = result.index('This is the first issue.') + features_pos = result.index('We added a feature!') + expected = [prelude_pos, features_pos, issues_pos] + actual = list(sorted([prelude_pos, features_pos, issues_pos])) + self.assertEqual(expected, actual) -- GitLab From 2a3b26abfb8fec584210aee2468f02ba0feceab3 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 7 Mar 2016 15:22:50 -0500 Subject: [PATCH 027/257] refactor argument buildup to make it more reusable We have the same query arguments to 2 commands and the next patch will add a third. Refactor things to make that more consistent. Change-Id: I757fe90c2afaa316a8067956fa375871536e77cd Signed-off-by: Doug Hellmann --- reno/main.py | 85 ++++++++++++++++++---------------------------------- 1 file changed, 29 insertions(+), 56 deletions(-) diff --git a/reno/main.py b/reno/main.py index b117ee2..0aef1b8 100644 --- a/reno/main.py +++ b/reno/main.py @@ -19,6 +19,33 @@ from reno import defaults from reno import lister from reno import report +_query_args = [ + (('--version',), + dict(default=[], + action='append', + help='the version(s) to include, defaults to all')), + (('--branch',), + dict(default=None, + help='the branch to scan, defaults to the current')), + (('--collapse-pre-releases',), + dict(action='store_true', + default=True, + help='combine pre-releases with their final release')), + (('--no-collapse-pre-releases',), + dict(action='store_false', + dest='collapse_pre_releases', + help='show pre-releases separately')), + (('--earliest-version',), + dict(default=None, + help='stop when this version is reached in the history')), +] + + +def _build_query_arg_group(parser): + group = parser.add_argument_group('query') + for args, kwds in _query_args: + group.add_argument(*args, **kwds) + def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser() @@ -61,38 +88,11 @@ def main(argv=sys.argv[1:]): 'list', help='list notes files based on query arguments', ) + _build_query_arg_group(do_list) do_list.add_argument( 'reporoot', help='root of the git repository', ) - do_list.add_argument( - '--version', - default=[], - action='append', - help='the version(s) to include, defaults to all', - ) - do_list.add_argument( - '--branch', - default=None, - help='the branch to scan, defaults to the current', - ) - do_list.add_argument( - '--collapse-pre-releases', - action='store_true', - default=True, - help='combine pre-releases with their final release', - ) - do_list.add_argument( - '--no-collapse-pre-releases', - action='store_false', - dest='collapse_pre_releases', - help='show pre-releases separately', - ) - do_list.add_argument( - '--earliest-version', - default=None, - help='stop when this version is reached in the history', - ) do_list.set_defaults(func=lister.list_cmd) do_report = subparsers.add_parser( @@ -108,34 +108,7 @@ def main(argv=sys.argv[1:]): default=None, help='output filename, defaults to stdout', ) - do_report.add_argument( - '--branch', - default=None, - help='the branch to scan, defaults to the current', - ) - do_report.add_argument( - '--version', - default=[], - action='append', - help='the version(s) to include, defaults to all', - ) - do_report.add_argument( - '--collapse-pre-releases', - action='store_true', - default=True, - help='combine pre-releases with their final release', - ) - do_report.add_argument( - '--no-collapse-pre-releases', - action='store_false', - dest='collapse_pre_releases', - help='show pre-releases separately', - ) - do_report.add_argument( - '--earliest-version', - default=None, - help='stop when this version is reached in the history', - ) + _build_query_arg_group(do_report) do_report.set_defaults(func=report.report_cmd) args = parser.parse_args() -- GitLab From 0b459b8337c1db4b2ae217386864327e9b9ef782 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 7 Mar 2016 15:57:47 -0500 Subject: [PATCH 028/257] add 'cache' command to write a cache file Generate YAML data based on the query arguments. A future patch will read the cache, if present, instead of scanning the git history. Change-Id: I711577b40d6030e670a0620a38809e61b9ebf512 Signed-off-by: Doug Hellmann --- reno/cache.py | 88 +++++++++++++++++++++++++++++++++++++++ reno/main.py | 17 ++++++++ reno/tests/test_cache.py | 89 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+) create mode 100644 reno/cache.py create mode 100644 reno/tests/test_cache.py diff --git a/reno/cache.py b/reno/cache.py new file mode 100644 index 0000000..5c8d923 --- /dev/null +++ b/reno/cache.py @@ -0,0 +1,88 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from reno import scanner +from reno import utils + +import yaml + +import sys + + +def build_cache_db(reporoot, notesdir, branch, collapse_pre_releases, + versions_to_include, earliest_version): + notes = scanner.get_notes_by_version( + reporoot, notesdir, branch, + collapse_pre_releases=collapse_pre_releases, + earliest_version=earliest_version, + ) + + # Default to including all versions returned by the scanner. + if not versions_to_include: + versions_to_include = list(notes.keys()) + + # Build a cache data structure including the file contents as well + # as the basic data returned by the scanner. + file_contents = {} + for version in versions_to_include: + for filename, sha in notes[version]: + body = scanner.get_file_at_commit( + reporoot, + filename, + sha, + ) + # We want to save the contents of the file, which is YAML, + # inside another YAML file. That looks terribly ugly with + # all of the escapes needed to format it properly as + # embedded YAML, so parse the input and convert it to a + # data structure that can be serialized cleanly. + y = yaml.safe_load(body) + file_contents[filename] = y + + cache = { + 'notes': [ + {'version': k, 'files': v} + for k, v in notes.items() + ], + 'file-contents': file_contents, + } + return cache + + +def cache_cmd(args): + "Generates a release notes cache" + reporoot = args.reporoot.rstrip('/') + '/' + notesdir = utils.get_notes_dir(args) + if args.output: + stream = open(args.output, 'w') + else: + stream = sys.stdout + try: + cache = build_cache_db( + reporoot=reporoot, + notesdir=notesdir, + branch=args.branch, + collapse_pre_releases=args.collapse_pre_releases, + versions_to_include=args.version, + earliest_version=args.earliest_version, + ) + yaml.safe_dump( + cache, + stream, + allow_unicode=True, + explicit_start=True, + encoding='utf-8', + ) + finally: + if args.output: + stream.close() + return diff --git a/reno/main.py b/reno/main.py index 0aef1b8..dfc0143 100644 --- a/reno/main.py +++ b/reno/main.py @@ -14,6 +14,7 @@ import argparse import logging import sys +from reno import cache from reno import create from reno import defaults from reno import lister @@ -111,6 +112,22 @@ def main(argv=sys.argv[1:]): _build_query_arg_group(do_report) do_report.set_defaults(func=report.report_cmd) + do_cache = subparsers.add_parser( + 'cache', + help='generate release notes cache', + ) + do_cache.add_argument( + 'reporoot', + help='root of the git repository', + ) + do_cache.add_argument( + '--output', '-o', + default=None, + help='output filename, defaults to stdout', + ) + _build_query_arg_group(do_cache) + do_cache.set_defaults(func=cache.cache_cmd) + args = parser.parse_args() logging.basicConfig( diff --git a/reno/tests/test_cache.py b/reno/tests/test_cache.py new file mode 100644 index 0000000..c6da923 --- /dev/null +++ b/reno/tests/test_cache.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import textwrap + +from reno import cache +from reno.tests import base + +from oslotest import mockpatch + +import mock + + +class TestCache(base.TestCase): + + scanner_output = { + '0.0.0': [('note1', 'shaA')], + '1.0.0': [('note2', 'shaB'), ('note3', 'shaC')], + } + + note_bodies = { + 'note1': textwrap.dedent(""" + prelude: > + This is the prelude. + """), + 'note2': textwrap.dedent(""" + issues: + - This is the first issue. + - This is the second issue. + """), + 'note3': textwrap.dedent(""" + features: + - We added a feature! + """) + } + + def _get_note_body(self, reporoot, filename, sha): + return self.note_bodies.get(filename, '') + + def setUp(self): + super(TestCache, self).setUp() + self.useFixture( + mockpatch.Patch('reno.scanner.get_file_at_commit', + new=self._get_note_body) + ) + + def test_build_cache_db(self): + with mock.patch('reno.scanner.get_notes_by_version') as gnbv: + gnbv.return_value = self.scanner_output + db = cache.build_cache_db( + reporoot=None, + notesdir=None, + branch=None, + collapse_pre_releases=True, + versions_to_include=[], + earliest_version=None, + ) + expected = { + 'notes': [ + {'version': k, 'files': v} + for k, v in self.scanner_output.items() + ], + 'file-contents': { + 'note1': { + 'prelude': 'This is the prelude.\n', + }, + 'note2': { + 'issues': [ + 'This is the first issue.', + 'This is the second issue.', + ], + }, + 'note3': { + 'features': ['We added a feature!'], + }, + }, + } + self.assertEqual(expected, db) -- GitLab From 9cb8c4bf1b9140e979f96843fba2c639b15c9e3b Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 7 Mar 2016 17:44:25 -0500 Subject: [PATCH 029/257] use the cache file instead of scanner when possible Define a loader API on top of the cache and scanner APIs and update the list, report, and sphinxext modules to use that instead of calling the scanner directly. Change-Id: I2899c2ae9bb46919a375ffe4f195b239cff389ef Signed-off-by: Doug Hellmann --- .gitignore | 1 + reno/formatter.py | 19 ++---- reno/lister.py | 12 ++-- reno/loader.py | 109 +++++++++++++++++++++++++++++++++++ reno/main.py | 4 ++ reno/report.py | 13 +++-- reno/sphinxext.py | 13 +++-- reno/tests/test_formatter.py | 64 +++++++++++--------- 8 files changed, 176 insertions(+), 59 deletions(-) create mode 100644 reno/loader.py diff --git a/.gitignore b/.gitignore index ad619f7..ee3e871 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,4 @@ ChangeLog .*.swp .*sw? /cover/ +/releasenotes/notes/reno.cache diff --git a/reno/formatter.py b/reno/formatter.py index 10112f0..517da87 100644 --- a/reno/formatter.py +++ b/reno/formatter.py @@ -12,10 +12,6 @@ from __future__ import print_function -from reno import scanner - -import yaml - _SECTION_ORDER = [ ('features', 'New Features'), @@ -41,7 +37,7 @@ def _indent_for_list(text, prefix=' '): ]) + '\n' -def format_report(reporoot, scanner_output, versions_to_include, title=None): +def format_report(loader, versions_to_include, title=None): report = [] if title: report.append('=' * len(title)) @@ -52,14 +48,9 @@ def format_report(reporoot, scanner_output, versions_to_include, title=None): # Read all of the notes files. file_contents = {} for version in versions_to_include: - for filename, sha in scanner_output[version]: - body = scanner.get_file_at_commit( - reporoot, - filename, - sha, - ) - y = yaml.safe_load(body) - file_contents[filename] = y + for filename, sha in loader[version]: + body = loader.parse_note_file(filename, sha) + file_contents[filename] = body for version in versions_to_include: report.append(version) @@ -67,7 +58,7 @@ def format_report(reporoot, scanner_output, versions_to_include, title=None): report.append('') # Add the preludes. - notefiles = scanner_output[version] + notefiles = loader[version] for n, sha in notefiles: if 'prelude' in file_contents[n]: report.append(file_contents[n]['prelude']) diff --git a/reno/lister.py b/reno/lister.py index eece975..c204ecd 100644 --- a/reno/lister.py +++ b/reno/lister.py @@ -14,7 +14,7 @@ from __future__ import print_function import logging -from reno import scanner +from reno import loader from reno import utils LOG = logging.getLogger(__name__) @@ -26,17 +26,19 @@ def list_cmd(args): reporoot = args.reporoot.rstrip('/') + '/' notesdir = utils.get_notes_dir(args) collapse = args.collapse_pre_releases - notes = scanner.get_notes_by_version( - reporoot, notesdir, args.branch, + ldr = loader.Loader( + reporoot=reporoot, + notesdir=notesdir, + branch=args.branch, collapse_pre_releases=collapse, earliest_version=args.earliest_version, ) if args.version: versions = args.version else: - versions = notes.keys() + versions = ldr.versions for version in versions: - notefiles = notes[version] + notefiles = ldr[version] print(version) for n, sha in notefiles: if n.startswith(reporoot): diff --git a/reno/loader.py b/reno/loader.py new file mode 100644 index 0000000..e3b3ee7 --- /dev/null +++ b/reno/loader.py @@ -0,0 +1,109 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os.path + +from reno import scanner + +import yaml + +LOG = logging.getLogger(__name__) + + +def get_cache_filename(reporoot, notesdir): + return os.path.join(reporoot, notesdir, 'reno.cache') + + +class Loader(object): + "Load the release notes for a given repository." + + def __init__(self, reporoot, notesdir, branch=None, + collapse_pre_releases=True, + earliest_version=None, + ignore_cache=False): + """Initialize a Loader. + + The versions are presented in reverse chronological order. + + Notes files are associated with the earliest version for which + they were available, regardless of whether they changed later. + + :param reporoot: Path to the root of the git repository. + :type reporoot: str + :param notesdir: The directory under *reporoot* with the release notes. + :type notesdir: str + :param branch: The name of the branch to scan. Defaults to current. + :type branch: str + :param collapse_pre_releases: When true, merge pre-release versions + into the final release, if it is present. + :type collapse_pre_releases: bool + :param earliest_version: The oldest version to include. + :type earliest_version: str + :param ignore_cache: Do not load a cache file if it is present. + :type ignore_cache: bool + + """ + self._reporoot = reporoot + self._notesdir = notesdir + self._branch = branch + self._collapse_pre_releases = collapse_pre_releases + self._earliest_version = earliest_version + self._ignore_cache = ignore_cache + + self._cache = None + self._scanner_output = None + self._cache_filename = get_cache_filename(reporoot, notesdir) + + self._load_data() + + def _load_data(self): + cache_file_exists = os.path.exists(self._cache_filename) + + if self._ignore_cache and cache_file_exists: + LOG.debug('ignoring cache file %s', self._cache_filename) + + if (not self._ignore_cache) and cache_file_exists: + with open(self._cache_filename, 'r') as f: + self._cache = yaml.safe_load(f.read()) + # Save the cached scanner output to the same attribute + # it would be in if we had loaded it "live". This + # simplifies some of the logic in the other methods. + self._scanner_output = { + n['version']: n['files'] + for n in self._cache['notes'] + } + else: + self._scanner_output = scanner.get_notes_by_version( + reporoot=self._reporoot, + notesdir=self._notesdir, + branch=self._branch, + collapse_pre_releases=self._collapse_pre_releases, + earliest_version=self._earliest_version, + ) + + @property + def versions(self): + "A list of all of the versions found." + return list(self._scanner_output.keys()) + + def __getitem__(self, version): + "Return data about the files that should go into a given version." + return self._scanner_output[version] + + def parse_note_file(self, filename, sha): + "Return the data structure encoded in the note file." + if self._cache: + return self._cache['file-contents'][filename] + else: + body = scanner.get_file_at_commit(self._reporoot, filename, sha) + return yaml.safe_load(body) diff --git a/reno/main.py b/reno/main.py index dfc0143..248a25b 100644 --- a/reno/main.py +++ b/reno/main.py @@ -39,6 +39,10 @@ _query_args = [ (('--earliest-version',), dict(default=None, help='stop when this version is reached in the history')), + (('--ignore-cache',), + dict(default=False, + action='store_true', + help='if there is a cache file present, do not use it')), ] diff --git a/reno/report.py b/reno/report.py index 439ee83..bf59f48 100644 --- a/reno/report.py +++ b/reno/report.py @@ -13,7 +13,7 @@ from __future__ import print_function from reno import formatter -from reno import scanner +from reno import loader from reno import utils @@ -22,18 +22,19 @@ def report_cmd(args): reporoot = args.reporoot.rstrip('/') + '/' notesdir = utils.get_notes_dir(args) collapse = args.collapse_pre_releases - notes = scanner.get_notes_by_version( - reporoot, notesdir, args.branch, + ldr = loader.Loader( + reporoot=reporoot, + notesdir=notesdir, + branch=args.branch, collapse_pre_releases=collapse, earliest_version=args.earliest_version, ) if args.version: versions = args.version else: - versions = notes.keys() + versions = ldr.versions text = formatter.format_report( - reporoot, - notes, + ldr, versions, title='Release Notes', ) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 81723a8..131f0c6 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -14,7 +14,7 @@ import os.path from reno import defaults from reno import formatter -from reno import scanner +from reno import loader from docutils import nodes from docutils.parsers import rst @@ -59,8 +59,10 @@ class ReleaseNotesDirective(rst.Directive): info('scanning %s for %s release notes' % (os.path.join(reporoot, notesdir), branch or 'current branch')) - notes = scanner.get_notes_by_version( - reporoot, notesdir, branch, + ldr = loader.Loader( + reporoot=reporoot, + notesdir=notesdir, + branch=branch, collapse_pre_releases=collapse, earliest_version=earliest_version, ) @@ -70,10 +72,9 @@ class ReleaseNotesDirective(rst.Directive): for v in version_opt.split(',') ] else: - versions = notes.keys() + versions = ldr.versions text = formatter.format_report( - reporoot, - notes, + ldr, versions, title=title, ) diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py index 04aebb9..e39be24 100644 --- a/reno/tests/test_formatter.py +++ b/reno/tests/test_formatter.py @@ -12,12 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. -import textwrap - from reno import formatter +from reno import loader from reno.tests import base -from oslotest import mockpatch +import mock class TestFormatter(base.TestCase): @@ -30,19 +29,20 @@ class TestFormatter(base.TestCase): versions = ['0.0.0', '1.0.0'] note_bodies = { - 'note1': textwrap.dedent(""" - prelude: > - This is the prelude. - """), - 'note2': textwrap.dedent(""" - issues: - - This is the first issue. - - This is the second issue. - """), - 'note3': textwrap.dedent(""" - features: - - We added a feature! - """) + 'note1': { + 'prelude': 'This is the prelude.', + }, + 'note2': { + 'issues': [ + 'This is the first issue.', + 'This is the second issue.', + ], + }, + 'note3': { + 'features': [ + 'We added a feature!', + ], + }, } def _get_note_body(self, reporoot, filename, sha): @@ -50,15 +50,26 @@ class TestFormatter(base.TestCase): def setUp(self): super(TestFormatter, self).setUp() - self.useFixture( - mockpatch.Patch('reno.scanner.get_file_at_commit', - new=self._get_note_body) - ) + + def _load(ldr): + ldr._scanner_output = self.scanner_output + ldr._cache = { + 'file-contents': self.note_bodies + } + + with mock.patch('reno.loader.Loader._load_data', _load): + self.ldr = loader.Loader( + reporoot='reporoot', + notesdir='notesdir', + branch=None, + collapse_pre_releases=None, + earliest_version=None, + ignore_cache=False, + ) def test_with_title(self): result = formatter.format_report( - reporoot=None, - scanner_output=self.scanner_output, + loader=self.ldr, versions_to_include=self.versions, title='This is the title', ) @@ -66,8 +77,7 @@ class TestFormatter(base.TestCase): def test_versions(self): result = formatter.format_report( - reporoot=None, - scanner_output=self.scanner_output, + loader=self.ldr, versions_to_include=self.versions, title='This is the title', ) @@ -76,8 +86,7 @@ class TestFormatter(base.TestCase): def test_without_title(self): result = formatter.format_report( - reporoot=None, - scanner_output=self.scanner_output, + loader=self.ldr, versions_to_include=self.versions, title=None, ) @@ -85,8 +94,7 @@ class TestFormatter(base.TestCase): def test_section_order(self): result = formatter.format_report( - reporoot=None, - scanner_output=self.scanner_output, + loader=self.ldr, versions_to_include=self.versions, title=None, ) -- GitLab From c805665f6b661dcc8409cd27c7fc741e6fef31d6 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 7 Mar 2016 17:44:45 -0500 Subject: [PATCH 030/257] make the cache command write to a file by default Compute the name of the file the loader is going to look for and write to that if no other output is given. Use '-' to write to stdout. Change-Id: I3f2cebb1c7a8e77cf8ed8d78c8ab85ccdd4ee325 Signed-off-by: Doug Hellmann --- reno/cache.py | 12 +++++++++--- reno/main.py | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/reno/cache.py b/reno/cache.py index 5c8d923..f870c1b 100644 --- a/reno/cache.py +++ b/reno/cache.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +from reno import loader from reno import scanner from reno import utils @@ -62,10 +63,15 @@ def cache_cmd(args): "Generates a release notes cache" reporoot = args.reporoot.rstrip('/') + '/' notesdir = utils.get_notes_dir(args) - if args.output: + if args.output == '-': + stream = sys.stdout + close_stream = False + elif args.output: stream = open(args.output, 'w') + close_stream = True else: - stream = sys.stdout + stream = open(loader.get_cache_filename(reporoot, notesdir), 'w') + close_stream = True try: cache = build_cache_db( reporoot=reporoot, @@ -83,6 +89,6 @@ def cache_cmd(args): encoding='utf-8', ) finally: - if args.output: + if close_stream: stream.close() return diff --git a/reno/main.py b/reno/main.py index 248a25b..cba08ad 100644 --- a/reno/main.py +++ b/reno/main.py @@ -127,7 +127,9 @@ def main(argv=sys.argv[1:]): do_cache.add_argument( '--output', '-o', default=None, - help='output filename, defaults to stdout', + help=('output filename, ' + 'defaults to the cache file within the notesdir, ' + 'use "-" for stdout'), ) _build_query_arg_group(do_cache) do_cache.set_defaults(func=cache.cache_cmd) -- GitLab From 0994c39c82cafb93c321e7b169f214bf803b75d3 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Thu, 10 Mar 2016 15:35:32 +0100 Subject: [PATCH 031/257] Added git as runtime and build depends. --- debian/changelog | 6 ++++++ debian/control | 9 ++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 3077637..0705e6c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +python-reno (1.3.0-3) UNRELEASED; urgency=medium + + * Added git as runtime and build depends. + + -- Thomas Goirand Thu, 10 Mar 2016 15:35:14 +0100 + python-reno (1.3.0-2) unstable; urgency=medium [ Thomas Goirand ] diff --git a/debian/control b/debian/control index 2b745e5..3b7aaed 100644 --- a/debian/control +++ b/debian/control @@ -13,7 +13,8 @@ Build-Depends: debhelper (>= 9), python3-all, python3-pbr (>= 1.4), python3-setuptools, -Build-Depends-Indep: python-babel, +Build-Depends-Indep: git, + python-babel, python-coverage, python-hacking, python-mock (>= 1.3), @@ -38,7 +39,8 @@ Homepage: http://www.openstack.org/ Package: python-reno Architecture: all -Depends: python-pbr (>= 1.4), +Depends: git, + python-pbr (>= 1.4), python-yaml, ${misc:Depends}, ${python:Depends}, @@ -51,7 +53,8 @@ Description: RElease NOtes manager - Python 2.x Package: python3-reno Architecture: all -Depends: python3-pbr (>= 1.4), +Depends: git, + python3-pbr (>= 1.4), python3-yaml, ${misc:Depends}, ${python3:Depends}, -- GitLab From 8df79bf2b8908d344f86011e64299686e118fb7e Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 15 Mar 2016 13:56:21 -0400 Subject: [PATCH 032/257] handle deleted notes properly We had handling for renamed notes but not for deleted files. Add handling with tests. Change-Id: I20e40c6e60eee7840018abec07d23bc0be7cf696 Signed-off-by: Doug Hellmann --- reno/scanner.py | 47 ++++++++++++++++++++++++++++++-------- reno/tests/test_scanner.py | 47 ++++++++++++++++++++++++++++++++++++++ reno/utils.py | 8 ++++--- 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 15b645d..99c27bb 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -197,6 +197,9 @@ def get_notes_by_version(reporoot, notesdir, branch=None, # renames. last_name_by_id = {} + # Remember uniqueids that have had files deleted. + uniqueids_deleted = collections.defaultdict(set) + # FIXME(dhellmann): This might need to be more line-oriented for # longer histories. log_cmd = [ @@ -256,22 +259,45 @@ def get_notes_by_version(reporoot, notesdir, branch=None, for f in filenames: # Updated as older tags are found, handling edits to release # notes. - LOG.debug('setting earliest reference to %s to %s' % - (f, tags[0])) uniqueid = _get_unique_id(f) + LOG.debug('%s: found file %s', + uniqueid, f) + LOG.debug('%s: setting earliest reference to %s' % + (uniqueid, tags[0])) earliest_seen[uniqueid] = tags[0] if uniqueid in last_name_by_id: # We already have a filename for this id from a # new commit, so use that one in case the name has # changed. - LOG.debug('%s was seen before' % f) + LOG.debug('%s: was seen before in %s', + uniqueid, last_name_by_id[uniqueid]) continue - if _file_exists_at_commit(reporoot, f, sha): - # Remember this filename as the most recent version of - # the unique id we have seen, in case the name - # changed from an older commit. - last_name_by_id[uniqueid] = (f, sha) - LOG.debug('remembering %s as filename for %s' % (f, uniqueid)) + elif _file_exists_at_commit(reporoot, f, sha): + LOG.debug('%s: looking for %s in deleted files %s', + uniqueid, f, uniqueids_deleted[uniqueid]) + if f in uniqueids_deleted[uniqueid]: + # The file exists in the commit, but was deleted + # later in the history. + LOG.debug('%s: skipping deleted file %s', + uniqueid, f) + else: + # Remember this filename as the most recent version of + # the unique id we have seen, in case the name + # changed from an older commit. + last_name_by_id[uniqueid] = (f, sha) + LOG.debug('%s: remembering %s as filename', + uniqueid, f) + else: + # Track files that have been deleted. The rename logic + # above checks for repeated references to files that + # are deleted later, and the inversion logic below + # checks for any remaining values and skips those + # entries. + LOG.debug('%s: saw a file that no longer exists', + uniqueid) + uniqueids_deleted[uniqueid].add(f) + LOG.debug('%s: deleted files %s', + uniqueid, uniqueids_deleted[uniqueid]) # Invert earliest_seen to make a list of notes files for each # version. @@ -283,6 +309,9 @@ def get_notes_by_version(reporoot, notesdir, branch=None, for uniqueid, version in earliest_seen.items(): try: base, sha = last_name_by_id[uniqueid] + if base in uniqueids_deleted.get(uniqueid, set()): + LOG.debug('skipping deleted note %s' % uniqueid) + continue files_and_tags[version].append((base, sha)) except KeyError: # Unable to find the file again, skip it to avoid breaking diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 4e6d0bb..0b13b4c 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -483,6 +483,53 @@ class BasicTest(Base): results, ) + def test_delete_file(self): + self._make_python_package() + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file('slug1') + f2 = self._add_notes_file('slug2') + self._run_git('rm', f1) + self._git_commit('remove note file') + self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + raw_results = scanner.get_notes_by_version( + self.reporoot, + 'releasenotes/notes', + ) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'2.0.0': [f2], + }, + results, + ) + + def test_rename_then_delete_file(self): + self._make_python_package() + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file('slug1') + f2 = f1.replace('slug1', 'slug2') + self._run_git('mv', f1, f2) + self._git_commit('rename note file') + self._run_git('rm', f2) + self._git_commit('remove note file') + f3 = self._add_notes_file('slug3') + self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + raw_results = scanner.get_notes_by_version( + self.reporoot, + 'releasenotes/notes', + ) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'2.0.0': [f3], + }, + results, + ) + class PreReleaseTest(Base): diff --git a/reno/utils.py b/reno/utils.py index 137d3e1..7ab4afa 100644 --- a/reno/utils.py +++ b/reno/utils.py @@ -51,9 +51,11 @@ def check_output(*args, **kwds): output, errors = process.communicate() retcode = process.poll() if errors: - LOG.debug('error output from (%s): %s', - ' '.join(*args), - errors.rstrip()) + LOG.debug('ran: %s', ' '.join(*args)) + LOG.debug('returned: %s', retcode) + LOG.debug('error output: %s', errors.rstrip()) + LOG.debug('regular output: %s', output.rstrip()) if retcode: + LOG.debug('raising error') raise subprocess.CalledProcessError(retcode, args, output=output) return output.decode('utf-8') -- GitLab From 11a2b7a88d34d69b1068e8ce100b0cf265bff181 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 7 Apr 2016 04:02:07 -0400 Subject: [PATCH 033/257] default to collapsing pre-releases in sphinxext Change-Id: I2e7b3af24cff19c60e933cd7766c438aa76ed1b8 Signed-off-by: Doug Hellmann --- reno/sphinxext.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 81723a8..e643dbf 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -52,7 +52,9 @@ class ReleaseNotesDirective(rst.Directive): defaults.RELEASE_NOTES_SUBDIR) notessubdir = self.options.get('notesdir', defaults.NOTES_SUBDIR) version_opt = self.options.get('version') - collapse = self.options.get('collapse-pre-releases') + # FIXME(dhellmann): Force this flag True for now and figure + # out how Sphinx passes a "false" flag later. + collapse = True # 'collapse-pre-releases' in self.options earliest_version = self.options.get('earliest-version') notesdir = os.path.join(relnotessubdir, notessubdir) -- GitLab From 1d7c3d88d7dcaa07da709126ca265e6a740f751a Mon Sep 17 00:00:00 2001 From: ZhiQiang Fan Date: Fri, 29 Apr 2016 20:27:26 +0800 Subject: [PATCH 034/257] [Trivial] Remove executable privilege of doc/source/conf.py It is a configuration file, rather than a script. Change-Id: I645bf620596a61275253e15d9f74f639e3a4b3dc --- doc/source/conf.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 doc/source/conf.py diff --git a/doc/source/conf.py b/doc/source/conf.py old mode 100755 new mode 100644 -- GitLab From 0f8d6b56400a48afd8b5bd6566529a8341980efe Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Wed, 18 May 2016 10:24:00 +0200 Subject: [PATCH 035/257] Closes #824593 --- debian/changelog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 0705e6c..5d67118 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,8 @@ -python-reno (1.3.0-3) UNRELEASED; urgency=medium +python-reno (1.3.0-3) unstable; urgency=medium - * Added git as runtime and build depends. + * Added git as runtime and build depends (Closes: #824593). - -- Thomas Goirand Thu, 10 Mar 2016 15:35:14 +0100 + -- Thomas Goirand Wed, 18 May 2016 10:23:32 +0200 python-reno (1.3.0-2) unstable; urgency=medium -- GitLab From 1f98300ee075fe45c2b9ead4aab1cb87c58ec192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nov=C3=BD?= Date: Thu, 19 May 2016 21:31:19 +0200 Subject: [PATCH 036/257] d/rules: Changed UPSTREAM_GIT protocol to https --- debian/changelog | 6 ++++++ debian/rules | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 5d67118..e2aab22 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +python-reno (1.3.0-4) UNRELEASED; urgency=medium + + * d/rules: Changed UPSTREAM_GIT protocol to https + + -- Ondřej Nový Thu, 19 May 2016 21:31:19 +0200 + python-reno (1.3.0-3) unstable; urgency=medium * Added git as runtime and build depends (Closes: #824593). diff --git a/debian/rules b/debian/rules index c895884..f318bf2 100755 --- a/debian/rules +++ b/debian/rules @@ -3,7 +3,7 @@ PYTHONS:=$(shell pyversions -vr) PYTHON3S:=$(shell py3versions -vr) -UPSTREAM_GIT = git://github.com/openstack/reno.git +UPSTREAM_GIT := https://github.com/openstack/reno.git -include /usr/share/openstack-pkg-tools/pkgos.make export OSLO_PACKAGE_VERSION=$(shell dpkg-parsechangelog | grep Version: | cut -d' ' -f2 | sed -e 's/^[[:digit:]]*://' -e 's/[-].*//' -e 's/~/.0/' | head -n 1) -- GitLab From 38b015817e20feef2a8a050ae82494b24ddc8002 Mon Sep 17 00:00:00 2001 From: "ChangBo Guo(gcb)" Date: Mon, 30 May 2016 13:40:13 +0800 Subject: [PATCH 037/257] Clean up oslo-incubator stuff Change-Id: If8ef4537fec846a81e5a4be79b28f37b28c16550 --- openstack-common.conf | 6 ------ tox.ini | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 openstack-common.conf diff --git a/openstack-common.conf b/openstack-common.conf deleted file mode 100644 index ebac829..0000000 --- a/openstack-common.conf +++ /dev/null @@ -1,6 +0,0 @@ -[DEFAULT] - -# The list of modules to copy from oslo-incubator.git - -# The base module to hold the copy of openstack.common -base=reno diff --git a/tox.ini b/tox.ini index cdb05ea..41bedef 100644 --- a/tox.ini +++ b/tox.ini @@ -36,4 +36,4 @@ commands = oslo_debug_helper {posargs} show-source = True ignore = E123,E125 builtins = _ -exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build -- GitLab From 45878e06b2b781fb298dd654f4a49fcd84334d5a Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Fri, 3 Jun 2016 15:14:26 -0700 Subject: [PATCH 038/257] Ignore empty sections in notes Users may do this, and it doesn't make sense to explode when a valid section is stated but empty. Change-Id: I1b30be6122893563cdc4c41172e28ed5a69a0e3c Closes-Bug: #1588992 --- reno/formatter.py | 1 + reno/tests/test_formatter.py | 1 + 2 files changed, 2 insertions(+) diff --git a/reno/formatter.py b/reno/formatter.py index 517da87..749dfbe 100644 --- a/reno/formatter.py +++ b/reno/formatter.py @@ -68,6 +68,7 @@ def format_report(loader, versions_to_include, title=None): notes = [ n for fn, sha in notefiles + if file_contents[fn].get(section_name) for n in file_contents[fn].get(section_name, []) ] if notes: diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py index e39be24..129fe56 100644 --- a/reno/tests/test_formatter.py +++ b/reno/tests/test_formatter.py @@ -42,6 +42,7 @@ class TestFormatter(base.TestCase): 'features': [ 'We added a feature!', ], + 'upgrade': None, }, } -- GitLab From 0c103c36e6b105e3c1863434b04e3045714493a5 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 16 Jun 2016 18:34:09 -0400 Subject: [PATCH 039/257] report extra files with warnings If the user forgets to add the .yaml extension, we ignore the file. Instead of doing that silently, report that we see something and that we're ignoring it. Change-Id: I18130951a09a216339d3b63dc7decc67e766ee2d Signed-off-by: Doug Hellmann --- reno/scanner.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 99c27bb..8f41065 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -233,11 +233,12 @@ def get_notes_by_version(reporoot, notesdir, branch=None, # files. Even if this list ends up empty, we continue doing # the other processing so that we record all of the known # versions. - filenames = [ - f - for f in hlines[2:] - if fnmatch.fnmatch(f, notesdir + '/*.yaml') - ] + filenames = [] + for f in hlines[2:]: + if fnmatch.fnmatch(f, notesdir + '/*.yaml'): + filenames.append(f) + elif fnmatch.fnmatch(f, notesdir + '/*'): + LOG.warn('found and ignored extra file %s', f) # If there are no tags in this block, assume the most recently # seen version. -- GitLab From 92ddd8f692c84db666f28226efe7a03bbfbfc0dc Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 20 Jun 2016 14:43:18 -0400 Subject: [PATCH 040/257] add API for writing the cache file Provide a stable API for writing to the cache file for projects that don't want to invoke the shell command. Change-Id: Iba80e313b6dba8182aebcba3f7bf99460226d75d Signed-off-by: Doug Hellmann --- reno/cache.py | 58 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/reno/cache.py b/reno/cache.py index f870c1b..025a0c6 100644 --- a/reno/cache.py +++ b/reno/cache.py @@ -10,14 +10,15 @@ # License for the specific language governing permissions and limitations # under the License. +import os +import sys + from reno import loader from reno import scanner from reno import utils import yaml -import sys - def build_cache_db(reporoot, notesdir, branch, collapse_pre_releases, versions_to_include, earliest_version): @@ -59,27 +60,41 @@ def build_cache_db(reporoot, notesdir, branch, collapse_pre_releases, return cache -def cache_cmd(args): - "Generates a release notes cache" - reporoot = args.reporoot.rstrip('/') + '/' - notesdir = utils.get_notes_dir(args) - if args.output == '-': +def write_cache_db(reporoot, notesdir, branch, collapse_pre_releases, + versions_to_include, earliest_version, + outfilename=None): + """Create a cache database file for the release notes data. + + Build the cache database from scanning the project history and + write it to a file within the project. + + By default, the data is written to the same file the scanner will + try to read when it cannot look at the git history. If outfilename + is given and is '-' the data is written to stdout + instead. Otherwise, if outfilename is given, the data overwrites + the named file. + + """ + if outfilename == '-': stream = sys.stdout close_stream = False - elif args.output: - stream = open(args.output, 'w') + elif outfilename: + stream = open(outfilename, 'w') close_stream = True else: - stream = open(loader.get_cache_filename(reporoot, notesdir), 'w') + outfilename = loader.get_cache_filename(reporoot, notesdir) + if not os.path.exists(os.path.dirname(outfilename)): + os.makedirs(os.path.dirname(outfilename)) + stream = open(outfilename, 'w') close_stream = True try: cache = build_cache_db( reporoot=reporoot, notesdir=notesdir, - branch=args.branch, - collapse_pre_releases=args.collapse_pre_releases, - versions_to_include=args.version, - earliest_version=args.earliest_version, + branch=branch, + collapse_pre_releases=collapse_pre_releases, + versions_to_include=versions_to_include, + earliest_version=earliest_version, ) yaml.safe_dump( cache, @@ -91,4 +106,19 @@ def cache_cmd(args): finally: if close_stream: stream.close() + + +def cache_cmd(args): + "Generates a release notes cache" + reporoot = args.reporoot.rstrip('/') + '/' + notesdir = utils.get_notes_dir(args) + write_cache_db( + reporoot=reporoot, + notesdir=notesdir, + branch=args.branch, + collapse_pre_releases=args.collapse_pre_releases, + versions_to_include=args.version, + earliest_version=args.earliest_version, + outfilename=args.output, + ) return -- GitLab From 2a86047efeea9ba5ea8aee2bea984326a93f69df Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 22 Jun 2016 11:37:38 -0400 Subject: [PATCH 041/257] add warnings for malformated input Log warnings if the input doesn't match the type of data expected. Eventually we may want to enforce these rules by raising exceptions, but for now it should be enough to help someone debug their problem to just print a warning. Change-Id: I9016041bf13e9047d1894d2284b57b0a94554977 Signed-off-by: Doug Hellmann --- reno/loader.py | 42 ++++++++++++++++-- reno/tests/base.py | 14 +++++- reno/tests/test_loader.py | 92 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 reno/tests/test_loader.py diff --git a/reno/loader.py b/reno/loader.py index e3b3ee7..5faa530 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -15,6 +15,7 @@ import os.path from reno import scanner +import six import yaml LOG = logging.getLogger(__name__) @@ -101,9 +102,44 @@ class Loader(object): return self._scanner_output[version] def parse_note_file(self, filename, sha): - "Return the data structure encoded in the note file." + """Return the data structure encoded in the note file. + + Emit warnings for content that does not look valid in some + way, but return it anway for backwards-compatibility. + + """ if self._cache: - return self._cache['file-contents'][filename] + content = self._cache['file-contents'][filename] else: body = scanner.get_file_at_commit(self._reporoot, filename, sha) - return yaml.safe_load(body) + content = yaml.safe_load(body) + + for section_name, section_content in content.items(): + if section_name == 'prelude': + if not isinstance(section_content, six.string_types): + LOG.warning( + ('The prelude section of %s ' + 'does not parse as a single string. ' + 'Is the YAML input escaped properly?') % + filename, + ) + else: + if not isinstance(section_content, list): + LOG.warning( + ('The %s section of %s ' + 'does not parse as a list of strings. ' + 'Is the YAML input escaped properly?') % ( + section_name, filename), + ) + else: + for item in section_content: + if not isinstance(item, six.string_types): + LOG.warning( + ('The item %r in the %s section of %s ' + 'parses as a %s instead of a string. ' + 'Is the YAML input escaped properly?' + ) % (item, section_name, + filename, type(item)), + ) + + return content diff --git a/reno/tests/base.py b/reno/tests/base.py index 1c30cdb..c1f9829 100644 --- a/reno/tests/base.py +++ b/reno/tests/base.py @@ -15,9 +15,19 @@ # License for the specific language governing permissions and limitations # under the License. -from oslotest import base +import fixtures +import testtools -class TestCase(base.BaseTestCase): +class TestCase(testtools.TestCase): """Test case base class for all unit tests.""" + + def setUp(self): + super(TestCase, self).setUp() + self._stdout_fixture = fixtures.StringStream('stdout') + self.stdout = self.useFixture(self._stdout_fixture).stream + self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.stdout)) + self._stderr_fixture = fixtures.StringStream('stderr') + self.stderr = self.useFixture(self._stderr_fixture).stream + self.useFixture(fixtures.MonkeyPatch('sys.stderr', self.stderr)) diff --git a/reno/tests/test_loader.py b/reno/tests/test_loader.py new file mode 100644 index 0000000..50b6520 --- /dev/null +++ b/reno/tests/test_loader.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import textwrap + +from reno import loader +from reno.tests import base + +import fixtures +import mock +import six +import yaml + + +class TestValidate(base.TestCase): + + scanner_output = { + '0.0.0': [('note', 'shaA')], + } + + versions = ['0.0.0'] + + def setUp(self): + super(TestValidate, self).setUp() + self.logger = self.useFixture( + fixtures.FakeLogger( + format='%(message)s', + level=logging.WARNING, + ) + ) + + def _make_loader(self, note_bodies): + def _load(ldr): + ldr._scanner_output = self.scanner_output + ldr._cache = { + 'file-contents': {'note1': note_bodies}, + } + + with mock.patch('reno.loader.Loader._load_data', _load): + return loader.Loader( + reporoot='reporoot', + notesdir='notesdir', + branch=None, + collapse_pre_releases=None, + earliest_version=None, + ignore_cache=False, + ) + + def test_prelude_list(self): + note_bodies = yaml.safe_load(textwrap.dedent(''' + prelude: + - This is the first comment. + - This is a second. + ''')) + self.assertIsInstance(note_bodies['prelude'], list) + ldr = self._make_loader(note_bodies) + ldr.parse_note_file('note1', None) + self.assertIn('prelude', self.logger.output) + + def test_non_prelude_single_string(self): + note_bodies = yaml.safe_load(textwrap.dedent(''' + issues: | + This is a single string. + ''')) + print(type(note_bodies['issues'])) + self.assertIsInstance(note_bodies['issues'], six.string_types) + ldr = self._make_loader(note_bodies) + ldr.parse_note_file('note1', None) + self.assertIn('list of strings', self.logger.output) + + def test_note_with_colon_as_dict(self): + note_bodies = yaml.safe_load(textwrap.dedent(''' + issues: + - This is the first issue. + - dict: This is parsed as a dictionary. + ''')) + self.assertIsInstance(note_bodies['issues'][-1], dict) + ldr = self._make_loader(note_bodies) + ldr.parse_note_file('note1', None) + self.assertIn('dict', self.logger.output) -- GitLab From 96b0641269358e04d1c6cd10f327879b55025a94 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 22 Jun 2016 11:44:24 -0400 Subject: [PATCH 042/257] ignore all coverage output files Change-Id: I30176af6ff0f5117f16ed79ce423799371caf26d Signed-off-by: Doug Hellmann --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ee3e871..7c5611f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ lib64 pip-log.txt # Unit test / coverage reports -.coverage +.coverage* .tox nosetests.xml .testrepository -- GitLab From 34105eaf91a378c5f00275c1f9e836f10d49f977 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Mon, 27 Jun 2016 16:17:30 -0700 Subject: [PATCH 043/257] Properly declare dependency on six Reno uses six, but doesn't declare the dependency. This declares it. Line taken from global-requirements.txt Change-Id: I83e65a9dbda2f421bb6c4d2cc1d081b049fd259d Closes-bug: #1596731 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2dc6330..ba446a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ pbr<2.0,>=1.4 Babel>=1.3 PyYAML>=3.1.0 +six>=1.9.0 -- GitLab From bcaa1929d080543935ae737adc5c1973250e90ff Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Tue, 28 Jun 2016 10:57:38 -0700 Subject: [PATCH 044/257] Have import order follow standard and fix misspelling Have the import order follow the OpenStack standard: http://docs.openstack.org/developer/hacking/#import-order-template Basically the reno imports should be the final section of imports. Fix a misspelled word. Change-Id: I3401838398c051a9bdafad5f39d43a6af3de1058 --- reno/cache.py | 4 ++-- reno/loader.py | 6 +++--- reno/sphinxext.py | 8 ++++---- reno/tests/test_cache.py | 7 +++---- reno/tests/test_create.py | 6 +++--- reno/tests/test_formatter.py | 4 ++-- reno/tests/test_loader.py | 6 +++--- reno/tests/test_scanner.py | 8 ++++---- reno/tests/test_utils.py | 6 +++--- 9 files changed, 27 insertions(+), 28 deletions(-) diff --git a/reno/cache.py b/reno/cache.py index 025a0c6..e46fc98 100644 --- a/reno/cache.py +++ b/reno/cache.py @@ -13,12 +13,12 @@ import os import sys +import yaml + from reno import loader from reno import scanner from reno import utils -import yaml - def build_cache_db(reporoot, notesdir, branch, collapse_pre_releases, versions_to_include, earliest_version): diff --git a/reno/loader.py b/reno/loader.py index 5faa530..93aa1aa 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -13,11 +13,11 @@ import logging import os.path -from reno import scanner - import six import yaml +from reno import scanner + LOG = logging.getLogger(__name__) @@ -105,7 +105,7 @@ class Loader(object): """Return the data structure encoded in the note file. Emit warnings for content that does not look valid in some - way, but return it anway for backwards-compatibility. + way, but return it anyway for backwards-compatibility. """ if self._cache: diff --git a/reno/sphinxext.py b/reno/sphinxext.py index de1cda7..e5de33f 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -12,16 +12,16 @@ import os.path -from reno import defaults -from reno import formatter -from reno import loader - from docutils import nodes from docutils.parsers import rst from docutils.parsers.rst import directives from docutils.statemachine import ViewList from sphinx.util.nodes import nested_parse_with_titles +from reno import defaults +from reno import formatter +from reno import loader + class ReleaseNotesDirective(rst.Directive): diff --git a/reno/tests/test_cache.py b/reno/tests/test_cache.py index c6da923..794d85e 100644 --- a/reno/tests/test_cache.py +++ b/reno/tests/test_cache.py @@ -14,12 +14,11 @@ import textwrap -from reno import cache -from reno.tests import base - +import mock from oslotest import mockpatch -import mock +from reno import cache +from reno.tests import base class TestCache(base.TestCase): diff --git a/reno/tests/test_create.py b/reno/tests/test_create.py index 7b84c16..04675fb 100644 --- a/reno/tests/test_create.py +++ b/reno/tests/test_create.py @@ -12,12 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -from reno import create -from reno.tests import base - import fixtures import mock +from reno import create +from reno.tests import base + class TestPickFileName(base.TestCase): diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py index 129fe56..30048fe 100644 --- a/reno/tests/test_formatter.py +++ b/reno/tests/test_formatter.py @@ -12,12 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. +import mock + from reno import formatter from reno import loader from reno.tests import base -import mock - class TestFormatter(base.TestCase): diff --git a/reno/tests/test_loader.py b/reno/tests/test_loader.py index 50b6520..fe68a0c 100644 --- a/reno/tests/test_loader.py +++ b/reno/tests/test_loader.py @@ -15,14 +15,14 @@ import logging import textwrap -from reno import loader -from reno.tests import base - import fixtures import mock import six import yaml +from reno import loader +from reno.tests import base + class TestValidate(base.TestCase): diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 0b13b4c..abafc4d 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -19,15 +19,15 @@ import re import subprocess import textwrap +import fixtures +import mock +from testtools.content import text_content + from reno import create from reno import scanner from reno.tests import base from reno import utils -import fixtures -import mock -from testtools.content import text_content - _SETUP_TEMPLATE = """ import setuptools diff --git a/reno/tests/test_utils.py b/reno/tests/test_utils.py index 8e31100..cb76cc7 100644 --- a/reno/tests/test_utils.py +++ b/reno/tests/test_utils.py @@ -12,12 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -from reno.tests import base -from reno import utils - import mock import six +from reno.tests import base +from reno import utils + class TestGetRandomString(base.TestCase): -- GitLab From 2f89f7a0b0b71b40b37ddb605ddae6cef1398e6a Mon Sep 17 00:00:00 2001 From: EdLeafe Date: Thu, 30 Jun 2016 16:02:00 +0000 Subject: [PATCH 045/257] Make note template follow correct formatting The template used to create a new release note does not follow the ReST standards[0]. This can be confusing to a contributor who may be unfamiliar with the formatting requirements, as they will tend to follow the template format, and end up with incorrect ReST. This patch adds the required vertical bar and newline to each section. [0] http://docs.openstack.org/developer/reno/usage.html#formatting Change-Id: Ie8cbe524376a3c3e1e11446abdc6e5fa00233d51 --- reno/create.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/reno/create.py b/reno/create.py index b37ee9f..c9dee7c 100644 --- a/reno/create.py +++ b/reno/create.py @@ -32,56 +32,64 @@ prelude: > major features or adding release theme details should have a prelude. features: - - List new features here, or remove this section. + - | + List new features here, or remove this section. All of the list items in this section are combined when the release notes are rendered, so the text needs to be worded so that it does not depend on any information only available in another section, such as the prelude. This may mean repeating some details. issues: - - List known issues here, or remove this section. + - | + List known issues here, or remove this section. All of the list items in this section are combined when the release notes are rendered, so the text needs to be worded so that it does not depend on any information only available in another section, such as the prelude. This may mean repeating some details. upgrade: - - List upgrade notes here, or remove this section. + - | + List upgrade notes here, or remove this section. All of the list items in this section are combined when the release notes are rendered, so the text needs to be worded so that it does not depend on any information only available in another section, such as the prelude. This may mean repeating some details. deprecations: - - List deprecations notes here, or remove this section. + - | + List deprecations notes here, or remove this section. All of the list items in this section are combined when the release notes are rendered, so the text needs to be worded so that it does not depend on any information only available in another section, such as the prelude. This may mean repeating some details. critical: - - Add critical notes here, or remove this section. + - | + Add critical notes here, or remove this section. All of the list items in this section are combined when the release notes are rendered, so the text needs to be worded so that it does not depend on any information only available in another section, such as the prelude. This may mean repeating some details. security: - - Add security notes here, or remove this section. + - | + Add security notes here, or remove this section. All of the list items in this section are combined when the release notes are rendered, so the text needs to be worded so that it does not depend on any information only available in another section, such as the prelude. This may mean repeating some details. fixes: - - Add normal bug fixes here, or remove this section. + - | + Add normal bug fixes here, or remove this section. All of the list items in this section are combined when the release notes are rendered, so the text needs to be worded so that it does not depend on any information only available in another section, such as the prelude. This may mean repeating some details. other: - - Add other notes here, or remove this section. + - | + Add other notes here, or remove this section. All of the list items in this section are combined when the release notes are rendered, so the text needs to be worded so that it does not depend on any -- GitLab From e5b172b666452f62546dc50dcb92ed00799bbad2 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 12 Jul 2016 13:42:35 -0400 Subject: [PATCH 046/257] add python 3.5 classifier and default tox env Change-Id: Ibabe35a68fed1973145ace6f712a9f6a30a532ce Signed-off-by: Doug Hellmann --- setup.cfg | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1d8a008..6bf03b6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,6 +18,7 @@ classifier = Programming Language :: Python :: 3 Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 [files] packages = diff --git a/tox.ini b/tox.ini index 41bedef..4229cef 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = py34,py27,pep8 +envlist = py35,py34,py27,pep8 skipsdist = True [testenv] -- GitLab From 0139533b51195e090cdc8c9e14534cfa278ff30c Mon Sep 17 00:00:00 2001 From: "Swapnil Kulkarni (coolsvap)" Date: Thu, 21 Jul 2016 17:02:14 +0000 Subject: [PATCH 047/257] Remove discover from test-requirements It's only needed for python < 2.7 which is not supported Change-Id: I973bfa4514f46299e9d0a3de2c26e111c66825cb --- test-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c352436..ce26e18 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,7 +7,6 @@ hacking<0.11,>=0.10.0 mock>=1.2 coverage>=3.6 -discover python-subunit>=0.0.18 oslosphinx>=2.5.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 -- GitLab From 750bdc021b8b3daf7b4faf09bd8bb1fe527d9d91 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 11 Jul 2016 16:33:06 -0500 Subject: [PATCH 048/257] Add YAML configuration parsing While discussing I7539fdeada14a73ae4e18a125bb0e3947f08e8d1 Doug and Harry realized that it would be much better if some of these values could be specified in a config file, whether INI or YAML. This adds a simple function to allow options to be specified in the config file named "config.yml" in the --rel-notes-dir. Change-Id: Ie25e1eb3da66cc627d93af585b0893469d6a7b2e --- doc/source/usage.rst | 25 +++++ .../add-config-file-e77084792c1dc695.yaml | 10 ++ reno/config.py | 73 ++++++++++++++ reno/defaults.py | 1 + reno/lister.py | 1 + reno/loader.py | 26 ++++- reno/main.py | 9 +- reno/tests/test_config.py | 94 +++++++++++++++++++ 8 files changed, 234 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/add-config-file-e77084792c1dc695.yaml create mode 100644 reno/config.py create mode 100644 reno/tests/test_config.py diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 74a5390..cbd13c4 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -147,3 +147,28 @@ repeated). Notes are output in the order they are found by ``git log`` looking over the history of the branch. This is deterministic, but not necessarily predictable or mutable. + +Configuring Reno +================ + +Reno looks for an optional ``config.yml`` file in your release notes +directory. This file may contain optional flags that you might use with a +command. If the values do not apply to the command, they are ignored in the +configuration file. For example, a couple reno commands allow you to specify + +- ``--branch`` +- ``--earliest-version`` +- ``--collapse-pre-releases``/``--no-collapse-pre-releases`` +- ``--ignore-cache`` + +So you might write a config file (if you use these often) like: + +.. code-block:: yaml + + --- + branch: master + earliest_version: 12.0.0 + collapse_pre_releases: false + +These will be parsed first and then the CLI options will be applied after +the config files. diff --git a/releasenotes/notes/add-config-file-e77084792c1dc695.yaml b/releasenotes/notes/add-config-file-e77084792c1dc695.yaml new file mode 100644 index 0000000..b6787df --- /dev/null +++ b/releasenotes/notes/add-config-file-e77084792c1dc695.yaml @@ -0,0 +1,10 @@ +--- +prelude: > + Reno now supports having a configuration file! +features: + - | + Reno now supports having a ``config.yaml`` file in your release notes + directory. It will search for file in the directory specified by + ``--rel-notes-dir`` and parse it. It will apply whatever options are + valid for that particular command. If an option is not relevant to a + particular sub-command, it will not attempt to apply them. diff --git a/reno/config.py b/reno/config.py new file mode 100644 index 0000000..2980ee5 --- /dev/null +++ b/reno/config.py @@ -0,0 +1,73 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import logging +import os.path + +import yaml + +from reno import defaults + +LOG = logging.getLogger(__name__) + + +def get_config_path(relnotesdir): + """Generate the path to the config file. + + :param str relnotesdir: + The directory containing release notes. + :returns: + The path to the config file in the release notes directory. + :rtype: + str + """ + return os.path.join(relnotesdir, defaults.RELEASE_NOTES_CONFIG_FILENAME) + + +def read_config(config_file): + """Read and parse the config file. + + :param str config_file: + The path to the config file to parse. + :returns: + The YAML parsed into a dictionary, otherwise, or an empty dictionary + if the path does not exist. + :rtype: + dict + """ + if not os.path.exists(config_file): + return {} + + with open(config_file, 'r') as fd: + return yaml.safe_load(fd) + + +def parse_config_into(parsed_arguments): + """Parse the user config onto the namespace arguments. + + :param parsed_arguments: + The result of calling :meth:`argparse.ArgumentParser.parse_args`. + :type parsed_arguments: + argparse.Namespace + """ + config_path = get_config_path(parsed_arguments.relnotesdir) + config_values = read_config(config_path) + + for key in config_values.keys(): + try: + getattr(parsed_arguments, key) + except AttributeError: + LOG.info('Option "%s" does not apply to this particular command.' + '. Ignoring...', key) + continue + setattr(parsed_arguments, key, config_values[key]) + + parsed_arguments._config = config_values diff --git a/reno/defaults.py b/reno/defaults.py index b30a18d..124fb59 100644 --- a/reno/defaults.py +++ b/reno/defaults.py @@ -12,3 +12,4 @@ RELEASE_NOTES_SUBDIR = 'releasenotes' NOTES_SUBDIR = 'notes' +RELEASE_NOTES_CONFIG_FILENAME = 'config.yml' diff --git a/reno/lister.py b/reno/lister.py index c204ecd..9d5e080 100644 --- a/reno/lister.py +++ b/reno/lister.py @@ -32,6 +32,7 @@ def list_cmd(args): branch=args.branch, collapse_pre_releases=collapse, earliest_version=args.earliest_version, + config=args._config ) if args.version: versions = args.version diff --git a/reno/loader.py b/reno/loader.py index 93aa1aa..60fe8e5 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -16,6 +16,7 @@ import os.path import six import yaml +from reno import config from reno import scanner LOG = logging.getLogger(__name__) @@ -31,7 +32,8 @@ class Loader(object): def __init__(self, reporoot, notesdir, branch=None, collapse_pre_releases=True, earliest_version=None, - ignore_cache=False): + ignore_cache=False, + parsedconfig=None): """Initialize a Loader. The versions are presented in reverse chronological order. @@ -52,13 +54,22 @@ class Loader(object): :type earliest_version: str :param ignore_cache: Do not load a cache file if it is present. :type ignore_cache: bool - + :param parsedconfig: Parsed configuration from file + :type parsedconfig: dict """ self._reporoot = reporoot self._notesdir = notesdir self._branch = branch - self._collapse_pre_releases = collapse_pre_releases - self._earliest_version = earliest_version + self._config = parsedconfig + if parsedconfig is None: + # NOTE(sigmavirus24): This should only happen when it is being + # used by the reStructuredText directive in our sphinx extension. + notesconfig = config.get_config_path(notesdir) + self._config = config.read_config(notesconfig) + self._collapse_pre_releases = self._value_from_config( + 'collapse_pre_releases', collapse_pre_releases, default=True) + self._earliest_version = self._value_from_config('earliest_version', + earliest_version) self._ignore_cache = ignore_cache self._cache = None @@ -67,6 +78,13 @@ class Loader(object): self._load_data() + def _value_from_config(self, name, value, default=None): + # NOTE(sigmavirus24): If it's the default value for the parameter + # definition then we might want to look at the config. + if value is default: + value = self._config.get(name, default) + return value + def _load_data(self): cache_file_exists = os.path.exists(self._cache_filename) diff --git a/reno/main.py b/reno/main.py index cba08ad..5e0a0ff 100644 --- a/reno/main.py +++ b/reno/main.py @@ -15,6 +15,7 @@ import logging import sys from reno import cache +from reno import config from reno import create from reno import defaults from reno import lister @@ -134,7 +135,13 @@ def main(argv=sys.argv[1:]): _build_query_arg_group(do_cache) do_cache.set_defaults(func=cache.cache_cmd) - args = parser.parse_args() + original_args = parser.parse_args(argv) + config.parse_config_into(original_args) + # NOTE(sigmavirus24): We parse twice to avoid having to guess if a parsed + # option is the default value or not. This allows us to apply the config + # to the proper command and then make sure that the command-line values + # take precedence + args = parser.parse_args(argv, original_args) logging.basicConfig( level=args.verbosity, diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py new file mode 100644 index 0000000..bb16d87 --- /dev/null +++ b/reno/tests/test_config.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import argparse +import os + +import fixtures + +from reno import config +from reno import defaults +from reno.tests import base + + +class TestConfig(base.TestCase): + EXAMPLE_CONFIG = """ +branch: master +collapse_pre_releases: false +earliest_version: true +""" + + def setUp(self): + super(TestConfig, self).setUp() + # Temporary directory to store our config + self.tempdir = self.useFixture(fixtures.TempDir()) + + # Argument parser and parsed arguments for our config function + parser = argparse.ArgumentParser() + parser.add_argument('--branch') + parser.add_argument('--collapse-pre-releases') + parser.add_argument('--earliest-version') + self.args = parser.parse_args([]) + self.args.relnotesdir = self.tempdir.path + + config_path = self.tempdir.join(defaults.RELEASE_NOTES_CONFIG_FILENAME) + + with open(config_path, 'w') as fd: + fd.write(self.EXAMPLE_CONFIG) + + self.addCleanup(os.unlink, config_path) + self.config_path = config_path + + def test_applies_relevant_config_values(self): + """Verify that our config function overrides default values.""" + config.parse_config_into(self.args) + del self.args._config + expected_value = { + 'relnotesdir': self.tempdir.path, + 'branch': 'master', + 'collapse_pre_releases': False, + 'earliest_version': True, + } + self.assertDictEqual(expected_value, vars(self.args)) + + def test_does_not_add_extra_options(self): + """Show that earliest_version is not set when missing.""" + del self.args.earliest_version + self.assertEqual(0, getattr(self.args, 'earliest_version', 0)) + + config.parse_config_into(self.args) + del self.args._config + expected_value = { + 'relnotesdir': self.tempdir.path, + 'branch': 'master', + 'collapse_pre_releases': False, + } + + self.assertDictEqual(expected_value, vars(self.args)) + + def test_get_congfig_path(self): + """Show that we generate the path appropriately.""" + self.assertEqual('releasenotes/config.yml', + config.get_config_path('releasenotes')) + + def test_read_config_shortcircuits(self): + """Verify we don't try to open a non-existent file.""" + self.assertDictEqual({}, + config.read_config('fake/path/to/config.yml')) + + def test_read_config(self): + """Verify we read and parse the config file specified if it exists.""" + self.assertDictEqual({'branch': 'master', + 'collapse_pre_releases': False, + 'earliest_version': True}, + config.read_config(self.config_path)) -- GitLab From 82c8025e2ef8c250ac0098352ff11c7168f333ab Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 22 Jul 2016 16:43:37 -0400 Subject: [PATCH 049/257] define Config class Define a class to hold all of the configuration settings, with methods to read a configuration file and to change the option values based on command line arguments. Change-Id: I70b3ebdb5f5c3cc6a9386ae947aaeeb912bae39c Signed-off-by: Doug Hellmann --- reno/cache.py | 12 +-- reno/config.py | 145 +++++++++++++++++++++++---------- reno/create.py | 4 +- reno/defaults.py | 2 - reno/lister.py | 14 ++-- reno/loader.py | 27 ++----- reno/main.py | 16 ++-- reno/report.py | 13 +-- reno/sphinxext.py | 28 +++++-- reno/tests/test_config.py | 151 ++++++++++++++++++++++------------- reno/tests/test_formatter.py | 2 + reno/tests/test_loader.py | 2 + reno/utils.py | 6 +- 13 files changed, 257 insertions(+), 165 deletions(-) diff --git a/reno/cache.py b/reno/cache.py index e46fc98..8b19d24 100644 --- a/reno/cache.py +++ b/reno/cache.py @@ -108,17 +108,17 @@ def write_cache_db(reporoot, notesdir, branch, collapse_pre_releases, stream.close() -def cache_cmd(args): +def cache_cmd(args, conf): "Generates a release notes cache" - reporoot = args.reporoot.rstrip('/') + '/' - notesdir = utils.get_notes_dir(args) + reporoot = conf.reporoot.rstrip('/') + '/' + notesdir = utils.get_notes_dir(conf) write_cache_db( reporoot=reporoot, notesdir=notesdir, - branch=args.branch, - collapse_pre_releases=args.collapse_pre_releases, + branch=conf.branch, + collapse_pre_releases=conf.collapse_pre_releases, versions_to_include=args.version, - earliest_version=args.earliest_version, + earliest_version=conf.earliest_version, outfilename=args.output, ) return diff --git a/reno/config.py b/reno/config.py index 2980ee5..cd75d2a 100644 --- a/reno/config.py +++ b/reno/config.py @@ -14,60 +14,117 @@ import os.path import yaml -from reno import defaults - LOG = logging.getLogger(__name__) -def get_config_path(relnotesdir): - """Generate the path to the config file. +class Config(object): + + _FILENAME = 'config.yaml' + + _OPTS = { + # The root directory of the git repository to scan. + 'reporoot': '.', - :param str relnotesdir: - The directory containing release notes. - :returns: - The path to the config file in the release notes directory. - :rtype: - str - """ - return os.path.join(relnotesdir, defaults.RELEASE_NOTES_CONFIG_FILENAME) + # The notes subdirectory within the relnotesdir where the + # notes live. + 'notesdir': 'notes', + # Should pre-release versions be merged into the final release + # of the same number (1.0.0.0a1 notes appear under 1.0.0). + 'collapse_pre_releases': True, -def read_config(config_file): - """Read and parse the config file. + # The git branch to scan. Defaults to the "current" branch + # checked out. + 'branch': None, - :param str config_file: - The path to the config file to parse. - :returns: - The YAML parsed into a dictionary, otherwise, or an empty dictionary - if the path does not exist. - :rtype: - dict - """ - if not os.path.exists(config_file): - return {} + # The earliest version to be included. This is usually the + # lowest version number, and is meant to be the oldest + # version. + 'earliest_version': None, + } + + @classmethod + def get_default(cls, opt): + "Return the default for an option." + try: + return cls._OPTS[opt] + except KeyError: + raise ValueError('unknown option name %r' % (opt,)) - with open(config_file, 'r') as fd: - return yaml.safe_load(fd) + def __init__(self, relnotesdir): + """Instantiate a Config object + :param str relnotesdir: + The directory containing release notes. -def parse_config_into(parsed_arguments): - """Parse the user config onto the namespace arguments. + """ + self.relnotesdir = relnotesdir + # Initialize attributes from the defaults. + self.override(**self._OPTS) - :param parsed_arguments: - The result of calling :meth:`argparse.ArgumentParser.parse_args`. - :type parsed_arguments: - argparse.Namespace - """ - config_path = get_config_path(parsed_arguments.relnotesdir) - config_values = read_config(config_path) + self._filename = os.path.join(relnotesdir, self._FILENAME) + self._contents = {} + self._load_file() - for key in config_values.keys(): + def _load_file(self): try: - getattr(parsed_arguments, key) - except AttributeError: - LOG.info('Option "%s" does not apply to this particular command.' - '. Ignoring...', key) - continue - setattr(parsed_arguments, key, config_values[key]) - - parsed_arguments._config = config_values + with open(self._filename, 'r') as fd: + self._contents = yaml.safe_load(fd) + except IOError as err: + LOG.info('did not load config file %s: %s', + self._filename, err) + else: + self.override(**self._contents) + + def override(self, **kwds): + """Set the values of the named configuration options. + + Take the values of the keyword arguments as the current value + of the same option, regardless of whether a value is already + present. + + """ + for n, v in kwds.items(): + if n not in self._OPTS: + LOG.warning('ignoring unknown configuration value %r = %r', + n, v) + else: + setattr(self, n, v) + + def override_from_parsed_args(self, parsed_args): + """Set the values of the configuration options from parsed CLI args. + + This method assumes that the DEST values for the command line + arguments are named the same as the configuration options. + + """ + arg_values = { + o: getattr(parsed_args, o) + for o in self._OPTS.keys() + if hasattr(parsed_args, o) + } + self.override(**arg_values) + + +# def parse_config_into(parsed_arguments): + +# """Parse the user config onto the namespace arguments. + +# :param parsed_arguments: +# The result of calling :meth:`argparse.ArgumentParser.parse_args`. +# :type parsed_arguments: +# argparse.Namespace +# """ +# config_path = get_config_path(parsed_arguments.relnotesdir) +# config_values = read_config(config_path) + +# for key in config_values.keys(): +# try: +# getattr(parsed_arguments, key) +# except AttributeError: +# LOG.info('Option "%s" does not apply to this particular command.' +# '. Ignoring...', key) +# continue +# setattr(parsed_arguments, key, config_values[key]) + +# parsed_arguments._config = config_values diff --git a/reno/create.py b/reno/create.py index c9dee7c..d695a07 100644 --- a/reno/create.py +++ b/reno/create.py @@ -120,9 +120,9 @@ def _make_note_file(filename): f.write(_TEMPLATE) -def create_cmd(args): +def create_cmd(args, conf): "Create a new release note file from the template." - notesdir = utils.get_notes_dir(args) + notesdir = utils.get_notes_dir(conf) # NOTE(dhellmann): There is a short race window where we might try # to pick a name that does not exist, then overwrite the file if # it is created before we try to write it. This isn't a problem diff --git a/reno/defaults.py b/reno/defaults.py index 124fb59..1b2cf00 100644 --- a/reno/defaults.py +++ b/reno/defaults.py @@ -11,5 +11,3 @@ # under the License. RELEASE_NOTES_SUBDIR = 'releasenotes' -NOTES_SUBDIR = 'notes' -RELEASE_NOTES_CONFIG_FILENAME = 'config.yml' diff --git a/reno/lister.py b/reno/lister.py index 9d5e080..70e15a6 100644 --- a/reno/lister.py +++ b/reno/lister.py @@ -20,19 +20,19 @@ from reno import utils LOG = logging.getLogger(__name__) -def list_cmd(args): +def list_cmd(args, conf): "List notes files based on query arguments" LOG.debug('starting list') - reporoot = args.reporoot.rstrip('/') + '/' - notesdir = utils.get_notes_dir(args) - collapse = args.collapse_pre_releases + reporoot = conf.reporoot.rstrip('/') + '/' + notesdir = utils.get_notes_dir(conf) + collapse = conf.collapse_pre_releases ldr = loader.Loader( reporoot=reporoot, notesdir=notesdir, - branch=args.branch, + branch=conf.branch, collapse_pre_releases=collapse, - earliest_version=args.earliest_version, - config=args._config + earliest_version=conf.earliest_version, + conf=conf, ) if args.version: versions = args.version diff --git a/reno/loader.py b/reno/loader.py index 60fe8e5..e5594f8 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -16,7 +16,6 @@ import os.path import six import yaml -from reno import config from reno import scanner LOG = logging.getLogger(__name__) @@ -33,7 +32,7 @@ class Loader(object): collapse_pre_releases=True, earliest_version=None, ignore_cache=False, - parsedconfig=None): + conf=None): """Initialize a Loader. The versions are presented in reverse chronological order. @@ -54,22 +53,15 @@ class Loader(object): :type earliest_version: str :param ignore_cache: Do not load a cache file if it is present. :type ignore_cache: bool - :param parsedconfig: Parsed configuration from file - :type parsedconfig: dict + :param conf: Parsed configuration from file + :type conf: reno.config.Config """ self._reporoot = reporoot self._notesdir = notesdir self._branch = branch - self._config = parsedconfig - if parsedconfig is None: - # NOTE(sigmavirus24): This should only happen when it is being - # used by the reStructuredText directive in our sphinx extension. - notesconfig = config.get_config_path(notesdir) - self._config = config.read_config(notesconfig) - self._collapse_pre_releases = self._value_from_config( - 'collapse_pre_releases', collapse_pre_releases, default=True) - self._earliest_version = self._value_from_config('earliest_version', - earliest_version) + self._config = conf + self._collapse_pre_releases = self._config.collapse_pre_releases + self._earliest_version = self._config.earliest_version self._ignore_cache = ignore_cache self._cache = None @@ -78,13 +70,6 @@ class Loader(object): self._load_data() - def _value_from_config(self, name, value, default=None): - # NOTE(sigmavirus24): If it's the default value for the parameter - # definition then we might want to look at the config. - if value is default: - value = self._config.get(name, default) - return value - def _load_data(self): cache_file_exists = os.path.exists(self._cache_filename) diff --git a/reno/main.py b/reno/main.py index 5e0a0ff..01eac0f 100644 --- a/reno/main.py +++ b/reno/main.py @@ -27,11 +27,11 @@ _query_args = [ action='append', help='the version(s) to include, defaults to all')), (('--branch',), - dict(default=None, + dict(default=config.Config.get_default('branch'), help='the branch to scan, defaults to the current')), (('--collapse-pre-releases',), dict(action='store_true', - default=True, + default=config.Config.get_default('collapse_pre_releases'), help='combine pre-releases with their final release')), (('--no-collapse-pre-releases',), dict(action='store_false', @@ -135,17 +135,13 @@ def main(argv=sys.argv[1:]): _build_query_arg_group(do_cache) do_cache.set_defaults(func=cache.cache_cmd) - original_args = parser.parse_args(argv) - config.parse_config_into(original_args) - # NOTE(sigmavirus24): We parse twice to avoid having to guess if a parsed - # option is the default value or not. This allows us to apply the config - # to the proper command and then make sure that the command-line values - # take precedence - args = parser.parse_args(argv, original_args) + args = parser.parse_args(argv) + conf = config.Config(args.relnotesdir) + conf.override_from_parsed_args(args) logging.basicConfig( level=args.verbosity, format='%(message)s', ) - return args.func(args) + return args.func(args, conf) diff --git a/reno/report.py b/reno/report.py index bf59f48..ed79641 100644 --- a/reno/report.py +++ b/reno/report.py @@ -17,17 +17,18 @@ from reno import loader from reno import utils -def report_cmd(args): +def report_cmd(args, conf): "Generates a release notes report" - reporoot = args.reporoot.rstrip('/') + '/' - notesdir = utils.get_notes_dir(args) - collapse = args.collapse_pre_releases + reporoot = conf.reporoot.rstrip('/') + '/' + notesdir = utils.get_notes_dir(conf) + collapse = conf.collapse_pre_releases ldr = loader.Loader( reporoot=reporoot, notesdir=notesdir, - branch=args.branch, + branch=conf.branch, collapse_pre_releases=collapse, - earliest_version=args.earliest_version, + earliest_version=conf.earliest_version, + conf=conf, ) if args.version: versions = args.version diff --git a/reno/sphinxext.py b/reno/sphinxext.py index e5de33f..9dc7ddb 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -18,6 +18,7 @@ from docutils.parsers.rst import directives from docutils.statemachine import ViewList from sphinx.util.nodes import nested_parse_with_titles +from reno import config from reno import defaults from reno import formatter from reno import loader @@ -50,23 +51,34 @@ class ReleaseNotesDirective(rst.Directive): reporoot = os.path.abspath(reporoot_opt) relnotessubdir = self.options.get('relnotessubdir', defaults.RELEASE_NOTES_SUBDIR) - notessubdir = self.options.get('notesdir', defaults.NOTES_SUBDIR) + conf = config.Config(relnotessubdir) + opt_overrides = { + 'reporoot': reporoot, + } + if 'notesdir' in self.options: + opt_overrides['notesdir'] = self.options.get('notesdir') version_opt = self.options.get('version') # FIXME(dhellmann): Force this flag True for now and figure # out how Sphinx passes a "false" flag later. - collapse = True # 'collapse-pre-releases' in self.options - earliest_version = self.options.get('earliest-version') + # 'collapse-pre-releases' in self.options + opt_overrides['collapse_pre_releases'] = True + if 'earliest-version' in self.options: + opt_overrides['earliest_version'] = self.options.get( + 'earliest-version') + conf.override(**opt_overrides) - notesdir = os.path.join(relnotessubdir, notessubdir) + notesdir = os.path.join(relnotessubdir, conf.notesdir) info('scanning %s for %s release notes' % - (os.path.join(reporoot, notesdir), branch or 'current branch')) + (os.path.join(conf.reporoot, notesdir), + branch or 'current branch')) ldr = loader.Loader( - reporoot=reporoot, + reporoot=conf.reporoot, notesdir=notesdir, branch=branch, - collapse_pre_releases=collapse, - earliest_version=earliest_version, + collapse_pre_releases=conf.collapse_pre_releases, + earliest_version=conf.earliest_version, + conf=conf, ) if version_opt is not None: versions = [ diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py index bb16d87..a7bf8ef 100644 --- a/reno/tests/test_config.py +++ b/reno/tests/test_config.py @@ -17,15 +17,15 @@ import os import fixtures from reno import config -from reno import defaults +from reno import main from reno.tests import base +import mock + class TestConfig(base.TestCase): EXAMPLE_CONFIG = """ -branch: master collapse_pre_releases: false -earliest_version: true """ def setUp(self): @@ -33,62 +33,103 @@ earliest_version: true # Temporary directory to store our config self.tempdir = self.useFixture(fixtures.TempDir()) - # Argument parser and parsed arguments for our config function - parser = argparse.ArgumentParser() - parser.add_argument('--branch') - parser.add_argument('--collapse-pre-releases') - parser.add_argument('--earliest-version') - self.args = parser.parse_args([]) - self.args.relnotesdir = self.tempdir.path - - config_path = self.tempdir.join(defaults.RELEASE_NOTES_CONFIG_FILENAME) - + def test_defaults(self): + c = config.Config(self.tempdir.path) + actual = { + o: getattr(c, o) + for o in config.Config._OPTS.keys() + } + self.assertEqual(config.Config._OPTS, actual) + + def test_override(self): + c = config.Config(self.tempdir.path) + c.override( + collapse_pre_releases=False, + ) + actual = { + o: getattr(c, o) + for o in config.Config._OPTS.keys() + } + expected = {} + expected.update(config.Config._OPTS) + expected['collapse_pre_releases'] = False + self.assertEqual(expected, actual) + + def test_override_multiple(self): + c = config.Config(self.tempdir.path) + c.override( + notesdir='value1', + ) + c.override( + notesdir='value2', + ) + actual = { + o: getattr(c, o) + for o in config.Config._OPTS.keys() + } + expected = {} + expected.update(config.Config._OPTS) + expected['notesdir'] = 'value2' + self.assertEqual(expected, actual) + + def test_load_file_not_present(self): + with mock.patch.object(config.LOG, 'info') as logger: + config.Config(self.tempdir.path) + self.assertEqual(1, logger.call_count) + + def test_load_file(self): + config_path = self.tempdir.join(config.Config._FILENAME) with open(config_path, 'w') as fd: fd.write(self.EXAMPLE_CONFIG) - self.addCleanup(os.unlink, config_path) - self.config_path = config_path - - def test_applies_relevant_config_values(self): - """Verify that our config function overrides default values.""" - config.parse_config_into(self.args) - del self.args._config - expected_value = { - 'relnotesdir': self.tempdir.path, - 'branch': 'master', - 'collapse_pre_releases': False, - 'earliest_version': True, - } - self.assertDictEqual(expected_value, vars(self.args)) - - def test_does_not_add_extra_options(self): - """Show that earliest_version is not set when missing.""" - del self.args.earliest_version - self.assertEqual(0, getattr(self.args, 'earliest_version', 0)) - - config.parse_config_into(self.args) - del self.args._config - expected_value = { - 'relnotesdir': self.tempdir.path, - 'branch': 'master', - 'collapse_pre_releases': False, - } + c = config.Config(self.tempdir.path) + self.assertEqual(False, c.collapse_pre_releases) - self.assertDictEqual(expected_value, vars(self.args)) + def test_get_default(self): + d = config.Config.get_default('notesdir') + self.assertEqual('notes', d) - def test_get_congfig_path(self): - """Show that we generate the path appropriately.""" - self.assertEqual('releasenotes/config.yml', - config.get_config_path('releasenotes')) + def test_get_default_unknown(self): + self.assertRaises( + ValueError, + config.Config.get_default, + 'unknownopt', + ) - def test_read_config_shortcircuits(self): - """Verify we don't try to open a non-existent file.""" - self.assertDictEqual({}, - config.read_config('fake/path/to/config.yml')) + def _run_override_from_parsed_args(self, argv): + parser = argparse.ArgumentParser() + main._build_query_arg_group(parser) + args = parser.parse_args(argv) + c = config.Config(self.tempdir.path) + c.override_from_parsed_args(args) + return c + + def test_override_from_parsed_args_empty(self): + c = self._run_override_from_parsed_args([]) + actual = { + o: getattr(c, o) + for o in config.Config._OPTS.keys() + } + self.assertEqual(config.Config._OPTS, actual) + + def test_override_from_parsed_args(self): + c = self._run_override_from_parsed_args([ + '--no-collapse-pre-releases', + ]) + actual = { + o: getattr(c, o) + for o in config.Config._OPTS.keys() + } + expected = {} + expected.update(config.Config._OPTS) + expected['collapse_pre_releases'] = False + self.assertEqual(expected, actual) - def test_read_config(self): - """Verify we read and parse the config file specified if it exists.""" - self.assertDictEqual({'branch': 'master', - 'collapse_pre_releases': False, - 'earliest_version': True}, - config.read_config(self.config_path)) + def test_override_from_parsed_args_ignore_non_options(self): + parser = argparse.ArgumentParser() + main._build_query_arg_group(parser) + parser.add_argument('not_a_config_option') + args = parser.parse_args(['value']) + c = config.Config(self.tempdir.path) + c.override_from_parsed_args(args) + self.assertFalse(hasattr(c, 'not_a_config_option')) diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py index 30048fe..6327906 100644 --- a/reno/tests/test_formatter.py +++ b/reno/tests/test_formatter.py @@ -14,6 +14,7 @@ import mock +from reno import config from reno import formatter from reno import loader from reno.tests import base @@ -66,6 +67,7 @@ class TestFormatter(base.TestCase): collapse_pre_releases=None, earliest_version=None, ignore_cache=False, + conf=config.Config('reporoot/releasenotes'), ) def test_with_title(self): diff --git a/reno/tests/test_loader.py b/reno/tests/test_loader.py index fe68a0c..0ed50bd 100644 --- a/reno/tests/test_loader.py +++ b/reno/tests/test_loader.py @@ -20,6 +20,7 @@ import mock import six import yaml +from reno import config from reno import loader from reno.tests import base @@ -56,6 +57,7 @@ class TestValidate(base.TestCase): collapse_pre_releases=None, earliest_version=None, ignore_cache=False, + conf=config.Config('reporoot/releasenotes'), ) def test_prelude_list(self): diff --git a/reno/utils.py b/reno/utils.py index 7ab4afa..a298278 100644 --- a/reno/utils.py +++ b/reno/utils.py @@ -17,14 +17,12 @@ import os.path import random import subprocess -from reno import defaults - LOG = logging.getLogger(__name__) -def get_notes_dir(args): +def get_notes_dir(conf): """Return the path to the release notes directory.""" - return os.path.join(args.relnotesdir, defaults.NOTES_SUBDIR) + return os.path.join(conf.relnotesdir, conf.notesdir) def get_random_string(nbytes=8): -- GitLab From d32fda8b34248a6c30dd510735fecee7495e69b1 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 22 Jul 2016 16:57:53 -0400 Subject: [PATCH 050/257] let the Config class clean up the reporoot we're given Instead of repeating the cleanup logic in all of the commands, let the Config class do it in one place. Change-Id: I3d3d4484b2656b54650e196e058c26c4c3dcdeb6 Signed-off-by: Doug Hellmann --- reno/cache.py | 2 +- reno/config.py | 11 ++++++++++- reno/lister.py | 2 +- reno/report.py | 2 +- reno/tests/test_config.py | 15 +++++++++++++++ 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/reno/cache.py b/reno/cache.py index 8b19d24..18fc2de 100644 --- a/reno/cache.py +++ b/reno/cache.py @@ -110,7 +110,7 @@ def write_cache_db(reporoot, notesdir, branch, collapse_pre_releases, def cache_cmd(args, conf): "Generates a release notes cache" - reporoot = conf.reporoot.rstrip('/') + '/' + reporoot = conf.reporoot notesdir = utils.get_notes_dir(conf) write_cache_db( reporoot=reporoot, diff --git a/reno/config.py b/reno/config.py index cd75d2a..09c58e3 100644 --- a/reno/config.py +++ b/reno/config.py @@ -23,7 +23,7 @@ class Config(object): _OPTS = { # The root directory of the git repository to scan. - 'reporoot': '.', + 'reporoot': './', # The notes subdirectory within the relnotesdir where the # notes live. @@ -105,6 +105,15 @@ class Config(object): } self.override(**arg_values) + @property + def reporoot(self): + return self._reporoot + + # Ensure that the 'reporoot' value always only ends in one '/'. + @reporoot.setter + def reporoot(self, value): + self._reporoot = value.rstrip('/') + '/' + # def parse_config_into(parsed_arguments): diff --git a/reno/lister.py b/reno/lister.py index 70e15a6..d38b752 100644 --- a/reno/lister.py +++ b/reno/lister.py @@ -23,7 +23,7 @@ LOG = logging.getLogger(__name__) def list_cmd(args, conf): "List notes files based on query arguments" LOG.debug('starting list') - reporoot = conf.reporoot.rstrip('/') + '/' + reporoot = conf.reporoot notesdir = utils.get_notes_dir(conf) collapse = conf.collapse_pre_releases ldr = loader.Loader( diff --git a/reno/report.py b/reno/report.py index ed79641..a09e5ed 100644 --- a/reno/report.py +++ b/reno/report.py @@ -19,7 +19,7 @@ from reno import utils def report_cmd(args, conf): "Generates a release notes report" - reporoot = conf.reporoot.rstrip('/') + '/' + reporoot = conf.reporoot notesdir = utils.get_notes_dir(conf) collapse = conf.collapse_pre_releases ldr = loader.Loader( diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py index a7bf8ef..ad3b77d 100644 --- a/reno/tests/test_config.py +++ b/reno/tests/test_config.py @@ -133,3 +133,18 @@ collapse_pre_releases: false c = config.Config(self.tempdir.path) c.override_from_parsed_args(args) self.assertFalse(hasattr(c, 'not_a_config_option')) + + +class TestConfigProperties(base.TestCase): + + def setUp(self): + super(TestConfigProperties, self).setUp() + # Temporary directory to store our config + self.tempdir = self.useFixture(fixtures.TempDir()) + self.c = config.Config(self.tempdir.path) + + def test_reporoot(self): + self.c.reporoot = 'blah//' + self.assertEqual('blah/', self.c.reporoot) + self.c.reporoot = 'blah' + self.assertEqual('blah/', self.c.reporoot) -- GitLab From 0355be32b424360a3fc250fa80fdcf5929a35f64 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 22 Jul 2016 17:12:27 -0400 Subject: [PATCH 051/257] let the Config object give us the path to the notes Add a property to combine the release notes subdirectory and the notes directory within it to build the path to look for notes files. Remove the old utils function for doing the same work. Change-Id: I7f6aa8c7e59f6e4559a583cad4b9bc5b574c77e0 Signed-off-by: Doug Hellmann --- reno/config.py | 4 ++++ reno/create.py | 3 +-- reno/lister.py | 3 --- reno/loader.py | 10 ++++------ reno/report.py | 3 --- reno/sphinxext.py | 1 - reno/tests/test_config.py | 7 ++++++- reno/tests/test_formatter.py | 1 - reno/tests/test_loader.py | 1 - reno/utils.py | 5 ----- 10 files changed, 15 insertions(+), 23 deletions(-) diff --git a/reno/config.py b/reno/config.py index 09c58e3..4bd6fa7 100644 --- a/reno/config.py +++ b/reno/config.py @@ -114,6 +114,10 @@ class Config(object): def reporoot(self, value): self._reporoot = value.rstrip('/') + '/' + @property + def notespath(self): + "The path in the repo where notes are kept." + return os.path.join(self.relnotesdir, self.notesdir) # def parse_config_into(parsed_arguments): diff --git a/reno/create.py b/reno/create.py index d695a07..2e56d85 100644 --- a/reno/create.py +++ b/reno/create.py @@ -122,7 +122,6 @@ def _make_note_file(filename): def create_cmd(args, conf): "Create a new release note file from the template." - notesdir = utils.get_notes_dir(conf) # NOTE(dhellmann): There is a short race window where we might try # to pick a name that does not exist, then overwrite the file if # it is created before we try to write it. This isn't a problem @@ -130,7 +129,7 @@ def create_cmd(args, conf): # their local git tree, and so there should not be any concurrency # concern. slug = args.slug.replace(' ', '-') - filename = _pick_note_file_name(notesdir, slug) + filename = _pick_note_file_name(conf.notespath, slug) _make_note_file(filename) print('Created new notes file in %s' % filename) return diff --git a/reno/lister.py b/reno/lister.py index d38b752..45618b6 100644 --- a/reno/lister.py +++ b/reno/lister.py @@ -15,7 +15,6 @@ from __future__ import print_function import logging from reno import loader -from reno import utils LOG = logging.getLogger(__name__) @@ -24,11 +23,9 @@ def list_cmd(args, conf): "List notes files based on query arguments" LOG.debug('starting list') reporoot = conf.reporoot - notesdir = utils.get_notes_dir(conf) collapse = conf.collapse_pre_releases ldr = loader.Loader( reporoot=reporoot, - notesdir=notesdir, branch=conf.branch, collapse_pre_releases=collapse, earliest_version=conf.earliest_version, diff --git a/reno/loader.py b/reno/loader.py index e5594f8..fa8f28f 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -28,7 +28,7 @@ def get_cache_filename(reporoot, notesdir): class Loader(object): "Load the release notes for a given repository." - def __init__(self, reporoot, notesdir, branch=None, + def __init__(self, reporoot, branch=None, collapse_pre_releases=True, earliest_version=None, ignore_cache=False, @@ -42,8 +42,6 @@ class Loader(object): :param reporoot: Path to the root of the git repository. :type reporoot: str - :param notesdir: The directory under *reporoot* with the release notes. - :type notesdir: str :param branch: The name of the branch to scan. Defaults to current. :type branch: str :param collapse_pre_releases: When true, merge pre-release versions @@ -57,7 +55,7 @@ class Loader(object): :type conf: reno.config.Config """ self._reporoot = reporoot - self._notesdir = notesdir + self._notespath = conf.notespath self._branch = branch self._config = conf self._collapse_pre_releases = self._config.collapse_pre_releases @@ -66,7 +64,7 @@ class Loader(object): self._cache = None self._scanner_output = None - self._cache_filename = get_cache_filename(reporoot, notesdir) + self._cache_filename = get_cache_filename(reporoot, self._notespath) self._load_data() @@ -89,7 +87,7 @@ class Loader(object): else: self._scanner_output = scanner.get_notes_by_version( reporoot=self._reporoot, - notesdir=self._notesdir, + notesdir=self._notespath, branch=self._branch, collapse_pre_releases=self._collapse_pre_releases, earliest_version=self._earliest_version, diff --git a/reno/report.py b/reno/report.py index a09e5ed..eb55eb6 100644 --- a/reno/report.py +++ b/reno/report.py @@ -14,17 +14,14 @@ from __future__ import print_function from reno import formatter from reno import loader -from reno import utils def report_cmd(args, conf): "Generates a release notes report" reporoot = conf.reporoot - notesdir = utils.get_notes_dir(conf) collapse = conf.collapse_pre_releases ldr = loader.Loader( reporoot=reporoot, - notesdir=notesdir, branch=conf.branch, collapse_pre_releases=collapse, earliest_version=conf.earliest_version, diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 9dc7ddb..8dc5ef0 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -74,7 +74,6 @@ class ReleaseNotesDirective(rst.Directive): ldr = loader.Loader( reporoot=conf.reporoot, - notesdir=notesdir, branch=branch, collapse_pre_releases=conf.collapse_pre_releases, earliest_version=conf.earliest_version, diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py index ad3b77d..2208d76 100644 --- a/reno/tests/test_config.py +++ b/reno/tests/test_config.py @@ -141,10 +141,15 @@ class TestConfigProperties(base.TestCase): super(TestConfigProperties, self).setUp() # Temporary directory to store our config self.tempdir = self.useFixture(fixtures.TempDir()) - self.c = config.Config(self.tempdir.path) + self.c = config.Config('releasenotes') def test_reporoot(self): self.c.reporoot = 'blah//' self.assertEqual('blah/', self.c.reporoot) self.c.reporoot = 'blah' self.assertEqual('blah/', self.c.reporoot) + + def test_notespath(self): + self.assertEqual('releasenotes/notes', self.c.notespath) + self.c.override(notesdir='thenotes') + self.assertEqual('releasenotes/thenotes', self.c.notespath) diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py index 6327906..22c52a9 100644 --- a/reno/tests/test_formatter.py +++ b/reno/tests/test_formatter.py @@ -62,7 +62,6 @@ class TestFormatter(base.TestCase): with mock.patch('reno.loader.Loader._load_data', _load): self.ldr = loader.Loader( reporoot='reporoot', - notesdir='notesdir', branch=None, collapse_pre_releases=None, earliest_version=None, diff --git a/reno/tests/test_loader.py b/reno/tests/test_loader.py index 0ed50bd..3ae6d3b 100644 --- a/reno/tests/test_loader.py +++ b/reno/tests/test_loader.py @@ -52,7 +52,6 @@ class TestValidate(base.TestCase): with mock.patch('reno.loader.Loader._load_data', _load): return loader.Loader( reporoot='reporoot', - notesdir='notesdir', branch=None, collapse_pre_releases=None, earliest_version=None, diff --git a/reno/utils.py b/reno/utils.py index a298278..88cd098 100644 --- a/reno/utils.py +++ b/reno/utils.py @@ -20,11 +20,6 @@ import subprocess LOG = logging.getLogger(__name__) -def get_notes_dir(conf): - """Return the path to the release notes directory.""" - return os.path.join(conf.relnotesdir, conf.notesdir) - - def get_random_string(nbytes=8): """Return a fixed-length random string -- GitLab From c022901e2e839ba309a38c5b83275d4cda282a77 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 22 Jul 2016 17:37:30 -0400 Subject: [PATCH 052/257] use Config instead of individual args with parameters Instead of passing an increasing number of arguments around through the system, set up the Config instance correctly and use it. Also make the reporoot a required argument for Config, and let the release note subdirectory be an optional argument with a default. Change-Id: I5fc6358bb496f44e3fd68c89ad71b06067dd7425 Signed-off-by: Doug Hellmann --- reno/cache.py | 31 ++----- reno/config.py | 16 ++-- reno/lister.py | 9 +-- reno/loader.py | 42 +++------- reno/main.py | 2 +- reno/report.py | 10 +-- reno/scanner.py | 19 ++--- reno/sphinxext.py | 14 +--- reno/tests/test_cache.py | 8 +- reno/tests/test_config.py | 5 +- reno/tests/test_formatter.py | 8 +- reno/tests/test_loader.py | 7 +- reno/tests/test_scanner.py | 153 +++++++++-------------------------- 13 files changed, 92 insertions(+), 232 deletions(-) diff --git a/reno/cache.py b/reno/cache.py index 18fc2de..c31ca88 100644 --- a/reno/cache.py +++ b/reno/cache.py @@ -17,16 +17,10 @@ import yaml from reno import loader from reno import scanner -from reno import utils -def build_cache_db(reporoot, notesdir, branch, collapse_pre_releases, - versions_to_include, earliest_version): - notes = scanner.get_notes_by_version( - reporoot, notesdir, branch, - collapse_pre_releases=collapse_pre_releases, - earliest_version=earliest_version, - ) +def build_cache_db(conf, versions_to_include): + notes = scanner.get_notes_by_version(conf) # Default to including all versions returned by the scanner. if not versions_to_include: @@ -38,7 +32,7 @@ def build_cache_db(reporoot, notesdir, branch, collapse_pre_releases, for version in versions_to_include: for filename, sha in notes[version]: body = scanner.get_file_at_commit( - reporoot, + conf.reporoot, filename, sha, ) @@ -60,8 +54,7 @@ def build_cache_db(reporoot, notesdir, branch, collapse_pre_releases, return cache -def write_cache_db(reporoot, notesdir, branch, collapse_pre_releases, - versions_to_include, earliest_version, +def write_cache_db(conf, versions_to_include, outfilename=None): """Create a cache database file for the release notes data. @@ -82,19 +75,15 @@ def write_cache_db(reporoot, notesdir, branch, collapse_pre_releases, stream = open(outfilename, 'w') close_stream = True else: - outfilename = loader.get_cache_filename(reporoot, notesdir) + outfilename = loader.get_cache_filename(conf.reporoot, conf.notespath) if not os.path.exists(os.path.dirname(outfilename)): os.makedirs(os.path.dirname(outfilename)) stream = open(outfilename, 'w') close_stream = True try: cache = build_cache_db( - reporoot=reporoot, - notesdir=notesdir, - branch=branch, - collapse_pre_releases=collapse_pre_releases, + conf, versions_to_include=versions_to_include, - earliest_version=earliest_version, ) yaml.safe_dump( cache, @@ -110,15 +99,9 @@ def write_cache_db(reporoot, notesdir, branch, collapse_pre_releases, def cache_cmd(args, conf): "Generates a release notes cache" - reporoot = conf.reporoot - notesdir = utils.get_notes_dir(conf) write_cache_db( - reporoot=reporoot, - notesdir=notesdir, - branch=conf.branch, - collapse_pre_releases=conf.collapse_pre_releases, + conf=conf, versions_to_include=args.version, - earliest_version=conf.earliest_version, outfilename=args.output, ) return diff --git a/reno/config.py b/reno/config.py index 4bd6fa7..d1af5f2 100644 --- a/reno/config.py +++ b/reno/config.py @@ -14,6 +14,8 @@ import os.path import yaml +from reno import defaults + LOG = logging.getLogger(__name__) @@ -22,9 +24,6 @@ class Config(object): _FILENAME = 'config.yaml' _OPTS = { - # The root directory of the git repository to scan. - 'reporoot': './', - # The notes subdirectory within the relnotesdir where the # notes live. 'notesdir': 'notes', @@ -51,18 +50,23 @@ class Config(object): except KeyError: raise ValueError('unknown option name %r' % (opt,)) - def __init__(self, relnotesdir): + def __init__(self, reporoot, relnotesdir=defaults.RELEASE_NOTES_SUBDIR): """Instantiate a Config object + :param str reporoot: + The root directory of the repository. :param str relnotesdir: - The directory containing release notes. + The directory containing release notes. Defaults to + 'releasenotes'. """ + self.reporoot = reporoot self.relnotesdir = relnotesdir # Initialize attributes from the defaults. self.override(**self._OPTS) - self._filename = os.path.join(relnotesdir, self._FILENAME) + self._filename = os.path.join(self.reporoot, relnotesdir, + self._FILENAME) self._contents = {} self._load_file() diff --git a/reno/lister.py b/reno/lister.py index 45618b6..aa1ec4d 100644 --- a/reno/lister.py +++ b/reno/lister.py @@ -23,14 +23,7 @@ def list_cmd(args, conf): "List notes files based on query arguments" LOG.debug('starting list') reporoot = conf.reporoot - collapse = conf.collapse_pre_releases - ldr = loader.Loader( - reporoot=reporoot, - branch=conf.branch, - collapse_pre_releases=collapse, - earliest_version=conf.earliest_version, - conf=conf, - ) + ldr = loader.Loader(conf) if args.version: versions = args.version else: diff --git a/reno/loader.py b/reno/loader.py index fa8f28f..036e73c 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -28,11 +28,8 @@ def get_cache_filename(reporoot, notesdir): class Loader(object): "Load the release notes for a given repository." - def __init__(self, reporoot, branch=None, - collapse_pre_releases=True, - earliest_version=None, - ignore_cache=False, - conf=None): + def __init__(self, conf, + ignore_cache=False): """Initialize a Loader. The versions are presented in reverse chronological order. @@ -40,31 +37,24 @@ class Loader(object): Notes files are associated with the earliest version for which they were available, regardless of whether they changed later. - :param reporoot: Path to the root of the git repository. - :type reporoot: str - :param branch: The name of the branch to scan. Defaults to current. - :type branch: str - :param collapse_pre_releases: When true, merge pre-release versions - into the final release, if it is present. - :type collapse_pre_releases: bool - :param earliest_version: The oldest version to include. - :type earliest_version: str - :param ignore_cache: Do not load a cache file if it is present. - :type ignore_cache: bool :param conf: Parsed configuration from file :type conf: reno.config.Config + :param ignore_cache: Do not load a cache file if it is present. + :type ignore_cache: bool """ - self._reporoot = reporoot - self._notespath = conf.notespath - self._branch = branch self._config = conf - self._collapse_pre_releases = self._config.collapse_pre_releases - self._earliest_version = self._config.earliest_version self._ignore_cache = ignore_cache + self._reporoot = conf.reporoot + self._notespath = conf.notespath + self._branch = conf.branch + self._collapse_pre_releases = conf.collapse_pre_releases + self._earliest_version = conf.earliest_version + self._cache = None self._scanner_output = None - self._cache_filename = get_cache_filename(reporoot, self._notespath) + self._cache_filename = get_cache_filename(self._reporoot, + self._notespath) self._load_data() @@ -85,13 +75,7 @@ class Loader(object): for n in self._cache['notes'] } else: - self._scanner_output = scanner.get_notes_by_version( - reporoot=self._reporoot, - notesdir=self._notespath, - branch=self._branch, - collapse_pre_releases=self._collapse_pre_releases, - earliest_version=self._earliest_version, - ) + self._scanner_output = scanner.get_notes_by_version(self._config) @property def versions(self): diff --git a/reno/main.py b/reno/main.py index 01eac0f..3888692 100644 --- a/reno/main.py +++ b/reno/main.py @@ -136,7 +136,7 @@ def main(argv=sys.argv[1:]): do_cache.set_defaults(func=cache.cache_cmd) args = parser.parse_args(argv) - conf = config.Config(args.relnotesdir) + conf = config.Config(args.reporoot, args.relnotesdir) conf.override_from_parsed_args(args) logging.basicConfig( diff --git a/reno/report.py b/reno/report.py index eb55eb6..e59520f 100644 --- a/reno/report.py +++ b/reno/report.py @@ -18,15 +18,7 @@ from reno import loader def report_cmd(args, conf): "Generates a release notes report" - reporoot = conf.reporoot - collapse = conf.collapse_pre_releases - ldr = loader.Loader( - reporoot=reporoot, - branch=conf.branch, - collapse_pre_releases=collapse, - earliest_version=conf.earliest_version, - conf=conf, - ) + ldr = loader.Loader(conf) if args.version: versions = args.version else: diff --git a/reno/scanner.py b/reno/scanner.py index 8f41065..4f8525f 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -149,9 +149,7 @@ def _get_version_tags_on_branch(reporoot, branch): return tags -def get_notes_by_version(reporoot, notesdir, branch=None, - collapse_pre_releases=True, - earliest_version=None): +def get_notes_by_version(conf): """Return an OrderedDict mapping versions to lists of notes files. The versions are presented in reverse chronological order. @@ -161,15 +159,12 @@ def get_notes_by_version(reporoot, notesdir, branch=None, :param reporoot: Path to the root of the git repository. :type reporoot: str - :param notesdir: The directory under *reporoot* with the release notes. - :type notesdir: str - :param branch: The name of the branch to scan. Defaults to current. - :type branch: str - :param collapse_pre_releases: When true, merge pre-release versions - into the final release, if it is present. - :type collapse_pre_releases: bool """ + reporoot = conf.reporoot + notesdir = conf.notespath + branch = conf.branch + LOG.debug('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch)) # Determine all of the tags known on the branch, in their date @@ -324,7 +319,7 @@ def get_notes_by_version(reporoot, notesdir, branch=None, # Combine pre-releases into the final release, if we are told to # and the final release exists. - if collapse_pre_releases: + if conf.collapse_pre_releases: collapsing = files_and_tags files_and_tags = collections.OrderedDict() for ov in versions_by_date: @@ -370,7 +365,7 @@ def get_notes_by_version(reporoot, notesdir, branch=None, trimmed[ov] = sorted(files_and_tags[ov]) # If we have been told to stop at a version, we can do that # now. - if earliest_version and ov == earliest_version: + if conf.earliest_version and ov == conf.earliest_version: break LOG.debug('[reno] found %d versions and %d files', diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 8dc5ef0..8bff0d6 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -51,10 +51,8 @@ class ReleaseNotesDirective(rst.Directive): reporoot = os.path.abspath(reporoot_opt) relnotessubdir = self.options.get('relnotessubdir', defaults.RELEASE_NOTES_SUBDIR) - conf = config.Config(relnotessubdir) - opt_overrides = { - 'reporoot': reporoot, - } + conf = config.Config(reporoot, relnotessubdir) + opt_overrides = {} if 'notesdir' in self.options: opt_overrides['notesdir'] = self.options.get('notesdir') version_opt = self.options.get('version') @@ -72,13 +70,7 @@ class ReleaseNotesDirective(rst.Directive): (os.path.join(conf.reporoot, notesdir), branch or 'current branch')) - ldr = loader.Loader( - reporoot=conf.reporoot, - branch=branch, - collapse_pre_releases=conf.collapse_pre_releases, - earliest_version=conf.earliest_version, - conf=conf, - ) + ldr = loader.Loader(conf) if version_opt is not None: versions = [ v.strip() diff --git a/reno/tests/test_cache.py b/reno/tests/test_cache.py index 794d85e..e62d85a 100644 --- a/reno/tests/test_cache.py +++ b/reno/tests/test_cache.py @@ -18,6 +18,7 @@ import mock from oslotest import mockpatch from reno import cache +from reno import config from reno.tests import base @@ -53,17 +54,14 @@ class TestCache(base.TestCase): mockpatch.Patch('reno.scanner.get_file_at_commit', new=self._get_note_body) ) + self.c = config.Config('.') def test_build_cache_db(self): with mock.patch('reno.scanner.get_notes_by_version') as gnbv: gnbv.return_value = self.scanner_output db = cache.build_cache_db( - reporoot=None, - notesdir=None, - branch=None, - collapse_pre_releases=True, + self.c, versions_to_include=[], - earliest_version=None, ) expected = { 'notes': [ diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py index 2208d76..313f537 100644 --- a/reno/tests/test_config.py +++ b/reno/tests/test_config.py @@ -78,7 +78,10 @@ collapse_pre_releases: false self.assertEqual(1, logger.call_count) def test_load_file(self): - config_path = self.tempdir.join(config.Config._FILENAME) + rn_path = self.tempdir.join('releasenotes') + os.mkdir(rn_path) + config_path = self.tempdir.join('releasenotes/' + + config.Config._FILENAME) with open(config_path, 'w') as fd: fd.write(self.EXAMPLE_CONFIG) self.addCleanup(os.unlink, config_path) diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py index 22c52a9..42dc52c 100644 --- a/reno/tests/test_formatter.py +++ b/reno/tests/test_formatter.py @@ -59,14 +59,12 @@ class TestFormatter(base.TestCase): 'file-contents': self.note_bodies } + self.c = config.Config('reporoot') + with mock.patch('reno.loader.Loader._load_data', _load): self.ldr = loader.Loader( - reporoot='reporoot', - branch=None, - collapse_pre_releases=None, - earliest_version=None, + self.c, ignore_cache=False, - conf=config.Config('reporoot/releasenotes'), ) def test_with_title(self): diff --git a/reno/tests/test_loader.py b/reno/tests/test_loader.py index 3ae6d3b..8ca9cf8 100644 --- a/reno/tests/test_loader.py +++ b/reno/tests/test_loader.py @@ -41,6 +41,7 @@ class TestValidate(base.TestCase): level=logging.WARNING, ) ) + self.c = config.Config('reporoot') def _make_loader(self, note_bodies): def _load(ldr): @@ -51,12 +52,8 @@ class TestValidate(base.TestCase): with mock.patch('reno.loader.Loader._load_data', _load): return loader.Loader( - reporoot='reporoot', - branch=None, - collapse_pre_releases=None, - earliest_version=None, + self.c, ignore_cache=False, - conf=config.Config('reporoot/releasenotes'), ) def test_prelude_list(self): diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index abafc4d..a0982ad 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -23,6 +23,7 @@ import fixtures import mock from testtools.content import text_content +from reno import config from reno import create from reno import scanner from reno.tests import base @@ -179,10 +180,7 @@ class Base(base.TestCase): self.useFixture(fixtures.NestedTempfile()) self.temp_dir = self.useFixture(fixtures.TempDir()).path self.reporoot = os.path.join(self.temp_dir, 'reporoot') - self.notesdir = os.path.join(self.reporoot, - 'releasenotes', - 'notes', - ) + self.c = config.Config(self.reporoot) self._git_setup() self._counter = itertools.count(1) self.get_note_num = lambda: next(self._counter) @@ -192,10 +190,7 @@ class BasicTest(Base): def test_non_python_no_tags(self): filename = self._add_notes_file() - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -208,10 +203,7 @@ class BasicTest(Base): def test_python_no_tags(self): self._make_python_package() filename = self._add_notes_file() - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -225,10 +217,7 @@ class BasicTest(Base): filename = self._add_notes_file() self._add_other_file('not-a-release-note.txt') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -241,10 +230,7 @@ class BasicTest(Base): def test_note_commit_tagged(self): filename = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -258,10 +244,7 @@ class BasicTest(Base): self._make_python_package() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') filename = self._add_notes_file() - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -276,10 +259,7 @@ class BasicTest(Base): self._add_other_file('ignore-1.txt') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') self._add_other_file('ignore-2.txt') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -294,10 +274,7 @@ class BasicTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file() f2 = self._add_notes_file() - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -312,10 +289,7 @@ class BasicTest(Base): f1 = self._add_notes_file(commit=False) f2 = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -331,10 +305,7 @@ class BasicTest(Base): f1 = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = self._add_notes_file() - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -354,10 +325,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug2') self._run_git('mv', f1, f2) self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -376,10 +344,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug0') self._run_git('mv', f1, f2) self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -398,10 +363,7 @@ class BasicTest(Base): with open(os.path.join(self.reporoot, f1), 'w') as f: f.write('---\npreamble: new contents for file') self._git_commit('edit note file') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -420,10 +382,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug2') self._run_git('mv', f1, f2) self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -445,10 +404,7 @@ class BasicTest(Base): 'slug1-0000000000000001') self._run_git('mv', f1, f2) self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -467,11 +423,10 @@ class BasicTest(Base): self._run_git('tag', '-s', '-m', 'middle tag', '2.0.0') f3 = self._add_notes_file() self._run_git('tag', '-s', '-m', 'last tag', '3.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', + self.c.override( earliest_version='2.0.0', ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -491,10 +446,7 @@ class BasicTest(Base): self._run_git('rm', f1) self._git_commit('remove note file') self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -516,10 +468,7 @@ class BasicTest(Base): self._git_commit('remove note file') f3 = self._add_notes_file('slug3') self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -538,10 +487,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a2') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -557,10 +503,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b2') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -576,10 +519,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc2') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -601,11 +541,10 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') files.append(self._add_notes_file('slug4')) self._run_git('tag', '-s', '-m', 'full release tag', '1.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', + self.c.override( collapse_pre_releases=True, ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -624,11 +563,10 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') f3 = self._add_notes_file('slug3') self._run_git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', + self.c.override( collapse_pre_releases=True, ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -650,11 +588,10 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') f3 = self._add_notes_file('slug3') self._run_git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', + self.c.override( collapse_pre_releases=True, ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -682,10 +619,7 @@ class MergeCommitTest(Base): self._run_git('merge', '--no-ff', 'test_merge_commit') self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -713,10 +647,7 @@ class MergeCommitTest(Base): self._run_git('merge', '--no-ff', 'test_merge_commit') self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -748,10 +679,7 @@ class MergeCommitTest(Base): self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') self._add_other_file('ignore-3.txt') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -787,10 +715,7 @@ class MergeCommitTest(Base): self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') self._add_other_file('ignore-3.txt') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -840,10 +765,7 @@ class BranchTest(Base): f21 = self._add_notes_file('slug21') log_text = self._run_git('log') self.addDetail('git log', text_content(log_text)) - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -865,11 +787,10 @@ class BranchTest(Base): log_text = self._run_git('log', '--pretty=%x00%H %d', '--name-only', 'stable/2') self.addDetail('git log', text_content(log_text)) - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - 'stable/2', + self.c.override( + branch='stable/2', ) + raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() -- GitLab From c745d30c8b83db868783fa724d3f832206f9d8b3 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 22 Jul 2016 17:01:21 -0400 Subject: [PATCH 053/257] set the default for reporoot for the command line Use the default reporoot of '.' for all commands that take the value as an argument, and make the argument optional. Change-Id: I02ce81df91a9252273dad7cb84a1f330187bc3e7 Signed-off-by: Doug Hellmann --- .../notes/default-repository-root-cli-85d23034bef81619.yaml | 5 +++++ reno/main.py | 6 ++++++ 2 files changed, 11 insertions(+) create mode 100644 releasenotes/notes/default-repository-root-cli-85d23034bef81619.yaml diff --git a/releasenotes/notes/default-repository-root-cli-85d23034bef81619.yaml b/releasenotes/notes/default-repository-root-cli-85d23034bef81619.yaml new file mode 100644 index 0000000..ebd74fc --- /dev/null +++ b/releasenotes/notes/default-repository-root-cli-85d23034bef81619.yaml @@ -0,0 +1,5 @@ +--- +features: + - Set the default value of the reporoot argument + for all command line programs to "." and make + it an optional parameter. \ No newline at end of file diff --git a/reno/main.py b/reno/main.py index 3888692..6d4807c 100644 --- a/reno/main.py +++ b/reno/main.py @@ -97,6 +97,8 @@ def main(argv=sys.argv[1:]): _build_query_arg_group(do_list) do_list.add_argument( 'reporoot', + default='.', + nargs='?', help='root of the git repository', ) do_list.set_defaults(func=lister.list_cmd) @@ -107,6 +109,8 @@ def main(argv=sys.argv[1:]): ) do_report.add_argument( 'reporoot', + default='.', + nargs='?', help='root of the git repository', ) do_report.add_argument( @@ -123,6 +127,8 @@ def main(argv=sys.argv[1:]): ) do_cache.add_argument( 'reporoot', + default='.', + nargs='?', help='root of the git repository', ) do_cache.add_argument( -- GitLab From a590a3064faa841f5f93b9a58428d59d796157fc Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 3 Aug 2016 18:03:33 -0400 Subject: [PATCH 054/257] return the name of the cache file created Have the function that creates the cache file return the name of the file created so the caller doesn't have to know how that name is constructed from default values. Change-Id: I576045e1591badc84722e28072c7aed36a35be38 Signed-off-by: Doug Hellmann --- reno/cache.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/reno/cache.py b/reno/cache.py index c31ca88..44922e1 100644 --- a/reno/cache.py +++ b/reno/cache.py @@ -67,6 +67,8 @@ def write_cache_db(conf, versions_to_include, instead. Otherwise, if outfilename is given, the data overwrites the named file. + Return the name of the file created, if any. + """ if outfilename == '-': stream = sys.stdout @@ -95,6 +97,7 @@ def write_cache_db(conf, versions_to_include, finally: if close_stream: stream.close() + return outfilename def cache_cmd(args, conf): -- GitLab From 7f7d4a32671964424c27f3021f1d0dd5fee593ef Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 3 Aug 2016 18:04:08 -0400 Subject: [PATCH 055/257] set relnotesdir to default inside config Rather than using a default value in the function signature, look for None and set the default. This allows callers to not have to look for an option to be defined to decide when to pass relnotesdir or not. Change-Id: I8535db2f35fa69faa009b4d368b21e25735016ee Signed-off-by: Doug Hellmann --- reno/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reno/config.py b/reno/config.py index d1af5f2..5e47eb8 100644 --- a/reno/config.py +++ b/reno/config.py @@ -50,7 +50,7 @@ class Config(object): except KeyError: raise ValueError('unknown option name %r' % (opt,)) - def __init__(self, reporoot, relnotesdir=defaults.RELEASE_NOTES_SUBDIR): + def __init__(self, reporoot, relnotesdir=None): """Instantiate a Config object :param str reporoot: @@ -61,6 +61,8 @@ class Config(object): """ self.reporoot = reporoot + if relnotesdir is None: + relnotesdir = defaults.RELEASE_NOTES_SUBDIR self.relnotesdir = relnotesdir # Initialize attributes from the defaults. self.override(**self._OPTS) -- GitLab From fe0925fa2134115fdcabb26cdfe0b623f86b6c36 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 4 Aug 2016 09:10:04 -0400 Subject: [PATCH 056/257] add missing reporoot arg to 'new' command In some of the earlier reorganization, the reporoot argument became a required parameter. Add it as an argument to the new command with the same default as elsewhere. It is the last argument, so it can be left off and existing scripts using the new command will continue to work. Change-Id: I8033f5d896201dba394346fd19e77f7c11408d6c Signed-off-by: Doug Hellmann --- reno/main.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/reno/main.py b/reno/main.py index 6d4807c..50af629 100644 --- a/reno/main.py +++ b/reno/main.py @@ -88,6 +88,12 @@ def main(argv=sys.argv[1:]): 'slug', help='descriptive title of note (keep it short)', ) + do_new.add_argument( + 'reporoot', + default='.', + nargs='?', + help='root of the git repository', + ) do_new.set_defaults(func=create.create_cmd) do_list = subparsers.add_parser( -- GitLab From 9cff0d7d906a51704db675f2fbb9d07f524f8c55 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Tue, 9 Aug 2016 13:24:03 -0400 Subject: [PATCH 057/257] Add debugging section to docs Adds a 'debugging' section to the usage documentation, with a quick note to help users uncover errors masked by sphinx includes. Change-Id: I9c24931c3acb0ab7588221f278287549eee0b3cd --- doc/source/usage.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/source/usage.rst b/doc/source/usage.rst index cbd13c4..07ba2d3 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -172,3 +172,14 @@ So you might write a config file (if you use these often) like: These will be parsed first and then the CLI options will be applied after the config files. + +Debugging +========= + +The way release notes are included into sphinx documents may mask where +formatting errors occur. To generate the release notes manually, so that +they can be put into a sphinx document directly for debugging, run: + +.. code-block:: + + reno report . -- GitLab From 9d0a4ba2ce47027da7860567818939b80253a2db Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Fri, 2 Sep 2016 10:54:18 +0200 Subject: [PATCH 058/257] Fix RST code-block on its own will surpress the argument, add console so that the next line gets displayed. Also add "$" to signify the command line interaction. Change-Id: I2107356a1063f7f7e1c5c68ef821aebe2897969d --- doc/source/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 07ba2d3..20d45ec 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -180,6 +180,6 @@ The way release notes are included into sphinx documents may mask where formatting errors occur. To generate the release notes manually, so that they can be put into a sphinx document directly for debugging, run: -.. code-block:: +.. code-block:: console - reno report . + $ reno report . -- GitLab From 4495c4fd240294f98602f0f1c40594602612b360 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Fri, 9 Sep 2016 15:14:56 +0200 Subject: [PATCH 059/257] Add gnupg as build-depends-indep (Closes: #834685). --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 5d67118..31d9d95 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +python-reno (1.3.0-4) unstable; urgency=medium + + * Add gnupg as build-depends-indep (Closes: #834685). + + -- Thomas Goirand Fri, 09 Sep 2016 14:11:39 +0200 + python-reno (1.3.0-3) unstable; urgency=medium * Added git as runtime and build depends (Closes: #824593). -- GitLab From 7ea533623433502e4d107ff653706dbfb038ec65 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Fri, 9 Sep 2016 13:19:53 +0000 Subject: [PATCH 060/257] Standards-Version: 3.9.8 --- debian/changelog | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index c7eff8d..f53417b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,7 @@ python-reno (1.3.0-4) unstable; urgency=medium [ Thomas Goirand ] * Add gnupg as build-depends-indep (Closes: #834685). + * Standards-Version: 3.9.8 (no change). -- Thomas Goirand Fri, 09 Sep 2016 14:11:39 +0200 diff --git a/debian/control b/debian/control index 3b7aaed..51b6f13 100644 --- a/debian/control +++ b/debian/control @@ -32,7 +32,7 @@ Build-Depends-Indep: git, python3-testtools (>= 1.4.0), subunit, testrepository, -Standards-Version: 3.9.6 +Standards-Version: 3.9.8 Vcs-Browser: https://anonscm.debian.org/cgit/openstack/python-reno.git/ Vcs-Git: https://anonscm.debian.org/git/openstack/python-reno.git Homepage: http://www.openstack.org/ -- GitLab From 149747d0e6012159fbf206bef4a2295bd01dee48 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Fri, 23 Sep 2016 14:43:17 +0100 Subject: [PATCH 061/257] Wrap template at ~79 characters The default template gives some people the impression that reno notes need to be wrapped at < 50 characters or so. Since this isn't the case, wrap the template at something a little more standard and head off such questions. Change-Id: I77b43a3e749f404170a2a193953439f247fa61b0 --- reno/create.py | 106 ++++++++++++++++++++++--------------------------- 1 file changed, 47 insertions(+), 59 deletions(-) diff --git a/reno/create.py b/reno/create.py index 2e56d85..d69937f 100644 --- a/reno/create.py +++ b/reno/create.py @@ -20,81 +20,69 @@ from reno import utils _TEMPLATE = """\ --- prelude: > - Replace this text with content to appear at the - top of the section for this release. All of the - prelude content is merged together and then rendered - separately from the items listed in other parts of - the file, so the text needs to be worded so that - both the prelude and the other items make sense - when read independently. This may mean repeating - some details. Not every release note - requires a prelude. Usually only notes describing - major features or adding release theme details should - have a prelude. + Replace this text with content to appear at the top of the section for this + release. All of the prelude content is merged together and then rendered + separately from the items listed in other parts of the file, so the text + needs to be worded so that both the prelude and the other items make sense + when read independently. This may mean repeating some details. Not every + release note requires a prelude. Usually only notes describing major + features or adding release theme details should have a prelude. features: - | - List new features here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. + List new features here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. issues: - | - List known issues here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. + List known issues here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. upgrade: - | - List upgrade notes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. + List upgrade notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. deprecations: - | - List deprecations notes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. + List deprecations notes here, or remove this section. All of the list + items in this section are combined when the release notes are rendered, so + the text needs to be worded so that it does not depend on any information + only available in another section, such as the prelude. This may mean + repeating some details. critical: - | - Add critical notes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. + Add critical notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. security: - | - Add security notes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. + Add security notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. fixes: - | - Add normal bug fixes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. + Add normal bug fixes here, or remove this section. All of the list items + in this section are combined when the release notes are rendered, so the + text needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. other: - | - Add other notes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. + Add other notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. """ -- GitLab From fbd074bc4bfd73c8e7dbce6d8aba8373b09e16b6 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 29 Sep 2016 14:22:21 -0400 Subject: [PATCH 062/257] add reference to project team guide for openstack projects We do not want to put instructions that are specific to OpenStack in a general-purpose tool, but it's OK to link to those instructions elsewhere since a lot of people will start by looking at the reno documentation. Change-Id: I73f9fd71cfc24d67420adb70509a2775ad022627 Signed-off-by: Doug Hellmann --- doc/source/usage.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 20d45ec..47cd317 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -183,3 +183,12 @@ they can be put into a sphinx document directly for debugging, run: .. code-block:: console $ reno report . + +Within OpenStack +================ + +The OpenStack project maintains separate instructions for configuring +the CI jobs and other project-specific settings used for reno. Refer +to the `Managing Release Notes +`__ +section of the Project Team Guide for details. -- GitLab From f92ef1b72517eddb858471c42b17d90294a3b8dd Mon Sep 17 00:00:00 2001 From: gecong1973 Date: Tue, 4 Oct 2016 14:09:54 +0800 Subject: [PATCH 063/257] Replace LOG.wirn with LOG.warning logging.warn is deprecated in Python 3.[1] [1] https://docs.python.org/3/library/logging.html#logging.warning Change-Id: I5cdb63752a14b5a3d06e54efd0f9ad3ee0494ebb --- reno/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reno/scanner.py b/reno/scanner.py index 4f8525f..ecb7788 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -233,7 +233,7 @@ def get_notes_by_version(conf): if fnmatch.fnmatch(f, notesdir + '/*.yaml'): filenames.append(f) elif fnmatch.fnmatch(f, notesdir + '/*'): - LOG.warn('found and ignored extra file %s', f) + LOG.warning('found and ignored extra file %s', f) # If there are no tags in this block, assume the most recently # seen version. -- GitLab From 4a02ed5600462ed1efdd308f0346d4f24e8b0935 Mon Sep 17 00:00:00 2001 From: Sharat Sharma Date: Tue, 4 Oct 2016 12:13:27 +0530 Subject: [PATCH 064/257] Changed the link to home-page Change-Id: Ibbdfce8b7e08eda9a9f033af8a6f2278e5a51b58 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 6bf03b6..98333b1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ description-file = README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org -home-page = http://www.openstack.org/ +home-page = http://docs.openstack.org/developer/reno/ classifier = Environment :: OpenStack Intended Audience :: Information Technology -- GitLab From c11859d6a08e36d94df80d43e582ca5cdc6bec10 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 10 Oct 2016 14:04:28 -0400 Subject: [PATCH 065/257] quiet gpg commands in tests Change-Id: Ie25c50ec7ce3c2f07820012d39589e73499ebc5f Signed-off-by: Doug Hellmann --- reno/tests/test_scanner.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index a0982ad..0300a7f 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -109,7 +109,13 @@ class GPGKeyFixture(fixtures.Fixture): if gnupg_random: cmd.append(gnupg_random) cmd.append(config_file) - subprocess.check_call(cmd, cwd=tempdir.path) + subprocess.check_call( + cmd, + cwd=tempdir.path, + # Direct stderr to its own pipe, from which we don't read, + # to quiet the commands. + stderr=subprocess.PIPE, + ) class Base(base.TestCase): -- GitLab From 5ab603955007359735b867085362c4d319f2702a Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 10 Oct 2016 14:06:57 -0400 Subject: [PATCH 066/257] use unicode literals in scanner tests Change-Id: I34f2209e179efac9f1367b987966f24981756897 Signed-off-by: Doug Hellmann --- reno/tests/test_scanner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 0300a7f..b5db0b1 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import unicode_literals + import itertools import logging import os.path -- GitLab From d29fd6d3f4ac4fffd88847cb94b60e580d727e43 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 10 Oct 2016 14:25:08 -0400 Subject: [PATCH 067/257] log git commands run in scanner tests Log the commands we run to make debugging easier Change-Id: Ib12efa207642bb2ca76f45faaa76981db9496e9a Signed-off-by: Doug Hellmann --- reno/tests/test_scanner.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index b5db0b1..0c6e16d 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -123,10 +123,13 @@ class GPGKeyFixture(fixtures.Fixture): class Base(base.TestCase): def _run_git(self, *args): - return utils.check_output( + logging.debug('$ git %s', ' '.join(args)) + output = utils.check_output( ['git'] + list(args), cwd=self.reporoot, ) + logging.debug(output) + return output def _git_setup(self): os.makedirs(self.reporoot) -- GitLab From 9013febab8077c08b3d1983f16998333e764c794 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 10 Oct 2016 15:12:45 -0400 Subject: [PATCH 068/257] log scanner tests in a way that makes them easier to debug Change-Id: I2452bb8916a78666d0795770a4f753d9b82f630f Signed-off-by: Doug Hellmann --- reno/tests/test_scanner.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 0c6e16d..c59fdb2 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -122,13 +122,15 @@ class GPGKeyFixture(fixtures.Fixture): class Base(base.TestCase): + logger = logging.getLogger('test') + def _run_git(self, *args): - logging.debug('$ git %s', ' '.join(args)) + self.logger.debug('$ git %s', ' '.join(args)) output = utils.check_output( ['git'] + list(args), cwd=self.reporoot, ) - logging.debug(output) + self.logger.debug(output) return output def _git_setup(self): @@ -176,7 +178,7 @@ class Base(base.TestCase): def setUp(self): super(Base, self).setUp() - self.logger = self.useFixture( + self.fake_logger = self.useFixture( fixtures.FakeLogger( format='%(levelname)8s %(name)s %(message)s', level=logging.DEBUG, @@ -474,11 +476,16 @@ class BasicTest(Base): f1 = self._add_notes_file('slug1') f2 = f1.replace('slug1', 'slug2') self._run_git('mv', f1, f2) + self._run_git('status') self._git_commit('rename note file') self._run_git('rm', f2) self._git_commit('remove note file') f3 = self._add_notes_file('slug3') self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + log_results = self._run_git('log', '--topo-order', + '--pretty=%H %d', + '--name-only') + self.addDetail('git log', text_content(log_results)) raw_results = scanner.get_notes_by_version(self.c) results = { k: [f for (f, n) in v] -- GitLab From e2fe07d13f2d49309efcb046098181811217762c Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 10 Oct 2016 15:48:06 -0400 Subject: [PATCH 069/257] fix branch handling in sphinx extension Somewhere in the conversion from explicit arguments to configuration options the branch value was not being passed to the scanner from the sphinx extension. Change-Id: If430c76c8447b5e5d9d1e7d3a5caab104138c33c Signed-off-by: Doug Hellmann --- reno/sphinxext.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 8bff0d6..e561e6c 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -63,6 +63,8 @@ class ReleaseNotesDirective(rst.Directive): if 'earliest-version' in self.options: opt_overrides['earliest_version'] = self.options.get( 'earliest-version') + if branch: + opt_overrides['branch'] = branch conf.override(**opt_overrides) notesdir = os.path.join(relnotessubdir, conf.notesdir) @@ -78,6 +80,7 @@ class ReleaseNotesDirective(rst.Directive): ] else: versions = ldr.versions + info('got versions %s' % (versions,)) text = formatter.format_report( ldr, versions, -- GitLab From 6f6e7addfb7b1bda65efecb362fb206731bcab2e Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 10 Oct 2016 14:04:04 -0400 Subject: [PATCH 070/257] stop scanning at the base of a branch By default, stop scanning when we hit the version that is at the base of a branch. Change-Id: I4eaa03d15bafa8dc3bdbdae3707295040482ec04 Signed-off-by: Doug Hellmann --- ...stop-scanning-branch-e5a8937c248acc99.yaml | 7 + reno/scanner.py | 79 +++++++- reno/tests/test_scanner.py | 168 +++++++++++++++++- 3 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/stop-scanning-branch-e5a8937c248acc99.yaml diff --git a/releasenotes/notes/stop-scanning-branch-e5a8937c248acc99.yaml b/releasenotes/notes/stop-scanning-branch-e5a8937c248acc99.yaml new file mode 100644 index 0000000..3caccc5 --- /dev/null +++ b/releasenotes/notes/stop-scanning-branch-e5a8937c248acc99.yaml @@ -0,0 +1,7 @@ +--- +features: + - Automatically stop scanning branches at the point where they + diverge from master. This avoids having release notes from older + versions, that appear on master before the branch, from showing up + in the versions from the branch. This logic is only applied to + branches created from master. diff --git a/reno/scanner.py b/reno/scanner.py index 4f8525f..559aa52 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -78,6 +78,64 @@ def _get_unique_id(filename): return uniqueid +def _get_branch_base(reporoot, branch): + "Return the tag at base of the branch." + # Based on + # http://stackoverflow.com/questions/1527234/finding-a-branch-point-with-git + # git rev-list $(git rev-list --first-parent \ + # ^origin/stable/newton master | tail -n1)^^! + # + # Determine the list of commits accessible from the branch we are + # supposed to be scanning, but not on master. + cmd = [ + 'git', + 'rev-list', + '--first-parent', + branch, # on the branch + '^master', # not on master + ] + try: + LOG.debug(' '.join(cmd)) + parents = utils.check_output(cmd, cwd=reporoot).strip() + if not parents: + # There are no commits on the branch, yet, so we can use + # our current-version logic. + return _get_current_version(reporoot, branch) + except subprocess.CalledProcessError as e: + LOG.warning('failed to retrieve branch base: %s [%s]', + e, e.output.strip()) + return None + parent = parents.splitlines()[-1] + LOG.debug('parent = %r', parent) + # Now get the previous commit, which should be the one we tagged + # to create the branch. + cmd = [ + 'git', + 'rev-list', + '{}^^!'.format(parent), + ] + try: + sha = utils.check_output(cmd, cwd=reporoot).strip() + LOG.debug('sha = %r', sha) + except subprocess.CalledProcessError as e: + LOG.warning('failed to retrieve branch base: %s [%s]', + e, e.output.strip()) + return None + # Now get the tag for that commit. + cmd = [ + 'git', + 'describe', + '--abbrev=0', + sha, + ] + try: + return utils.check_output(cmd, cwd=reporoot).strip() + except subprocess.CalledProcessError as e: + LOG.warning('failed to retrieve branch base: %s [%s]', + e, e.output.strip()) + return None + + # The git log output from _get_tags_on_branch() looks like this sample # from the openstack/nova repository for git 1.9.1: # @@ -164,9 +222,26 @@ def get_notes_by_version(conf): reporoot = conf.reporoot notesdir = conf.notespath branch = conf.branch + earliest_version = conf.earliest_version + collapse_pre_releases = conf.collapse_pre_releases LOG.debug('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch)) + # If the user has not told us where to stop, try to work it out + # for ourselves. If branch is set and is not "master", then we + # want to stop at the base of the branch. + if (not earliest_version) and branch and (branch != 'master'): + LOG.debug('determining earliest_version from branch') + earliest_version = _get_branch_base(reporoot, branch) + if earliest_version and collapse_pre_releases: + if PRE_RELEASE_RE.search(earliest_version): + # The earliest version won't actually be the pre-release + # that might have been tagged when the branch was created, + # but the final version. Strip the pre-release portion of + # the version number. + earliest_version = '.'.join(earliest_version.split('.')[:-1]) + LOG.debug('using earliest_version = %r', earliest_version) + # Determine all of the tags known on the branch, in their date # order. We scan the commit history in topological order to ensure # we have the commits in the right version, so we might encounter @@ -319,7 +394,7 @@ def get_notes_by_version(conf): # Combine pre-releases into the final release, if we are told to # and the final release exists. - if conf.collapse_pre_releases: + if collapse_pre_releases: collapsing = files_and_tags files_and_tags = collections.OrderedDict() for ov in versions_by_date: @@ -365,7 +440,7 @@ def get_notes_by_version(conf): trimmed[ov] = sorted(files_and_tags[ov]) # If we have been told to stop at a version, we can do that # now. - if conf.earliest_version and ov == conf.earliest_version: + if earliest_version and ov == earliest_version: break LOG.debug('[reno] found %d versions and %d files', diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index c59fdb2..810fca4 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -815,13 +815,179 @@ class BranchTest(Base): } self.assertEqual( { - '1.0.0': [self.f1], '2.0.0': [self.f2], '2.0.0-1': [f21], }, results, ) + def test_pre_release_branch_no_collapse(self): + f4 = self._add_notes_file('slug4') + self._run_git('tag', '-s', '-m', 'pre-release', '4.0.0.0rc1') + # Add a commit on master after the tag + self._add_notes_file('slug5') + # Move back to the tag and create the branch + self._run_git('checkout', '4.0.0.0rc1') + self._run_git('checkout', '-b', 'stable/4') + # Create a commit on the branch + f41 = self._add_notes_file('slug41') + log_text = self._run_git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self._run_git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + collapse_pre_releases=False, + ) + raw_results = scanner.get_notes_by_version(self.c) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0.0rc1': [f4], + '4.0.0.0rc1-1': [f41], + }, + results, + ) + + def test_pre_release_branch_collapse(self): + f4 = self._add_notes_file('slug4') + self._run_git('tag', '-s', '-m', 'pre-release', '4.0.0.0rc1') + # Add a commit on master after the tag + self._add_notes_file('slug5') + # Move back to the tag and create the branch + self._run_git('checkout', '4.0.0.0rc1') + self._run_git('checkout', '-b', 'stable/4') + # Create a commit on the branch + f41 = self._add_notes_file('slug41') + self._run_git('tag', '-s', '-m', 'release', '4.0.0') + log_text = self._run_git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self._run_git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + collapse_pre_releases=True, + ) + raw_results = scanner.get_notes_by_version(self.c) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0': [f4, f41], + }, + results, + ) + + def test_full_release_branch(self): + f4 = self._add_notes_file('slug4') + self._run_git('tag', '-s', '-m', 'release', '4.0.0') + # Add a commit on master after the tag + self._add_notes_file('slug5') + # Move back to the tag and create the branch + self._run_git('checkout', '4.0.0') + self._run_git('checkout', '-b', 'stable/4') + # Create a commit on the branch + f41 = self._add_notes_file('slug41') + log_text = self._run_git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self._run_git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + ) + raw_results = scanner.get_notes_by_version(self.c) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0': [f4], + '4.0.0-1': [f41], + }, + results, + ) + + def test_branch_tip_of_master(self): + # We have branched from master, but not added any commits to + # master. + f4 = self._add_notes_file('slug4') + self._run_git('tag', '-s', '-m', 'release', '4.0.0') + self._run_git('checkout', '-b', 'stable/4') + # Create a commit on the branch + f41 = self._add_notes_file('slug41') + f42 = self._add_notes_file('slug42') + log_text = self._run_git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self._run_git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + ) + raw_results = scanner.get_notes_by_version(self.c) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0': [f4], + '4.0.0-2': [f41, f42], + }, + results, + ) + + def test_branch_no_more_commits(self): + # We have branched from master, but not added any commits to + # our branch or to master. + f4 = self._add_notes_file('slug4') + self._run_git('tag', '-s', '-m', 'release', '4.0.0') + self._run_git('checkout', '-b', 'stable/4') + # Create a commit on the branch + log_text = self._run_git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self._run_git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + ) + raw_results = scanner.get_notes_by_version(self.c) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0': [f4], + }, + results, + ) + class GetTagsParseTest(base.TestCase): -- GitLab From 7ee2a78a8a865980ed9a2f07be3f55211e5a90b3 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 10 Oct 2016 15:34:51 -0400 Subject: [PATCH 071/257] add configuration option to not stop at branch base The previous commit changes the default behavior to always stop scanning at the base of a branch. This change adds a configuration option to allow that behavior to be disabled, so that revisions along the history of the branch prior to the point where it diverged from master can be included. The new default behavior established in the previous commit is not changed. Change-Id: I2c4968e1291c1b7d268896cfbb79e320d4085bce Signed-off-by: Doug Hellmann --- doc/source/usage.rst | 2 ++ ...anning-branch-option-6a0156b183814d7f.yaml | 9 ++++++ reno/config.py | 4 +++ reno/main.py | 9 ++++++ reno/scanner.py | 4 ++- reno/sphinxext.py | 4 ++- reno/tests/test_scanner.py | 28 +++++++++++++++++++ 7 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/stop-scanning-branch-option-6a0156b183814d7f.yaml diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 47cd317..b005d28 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -160,6 +160,7 @@ configuration file. For example, a couple reno commands allow you to specify - ``--earliest-version`` - ``--collapse-pre-releases``/``--no-collapse-pre-releases`` - ``--ignore-cache`` +- ``--stop-at-branch-base``/``--no-stop-at-branch-base`` So you might write a config file (if you use these often) like: @@ -169,6 +170,7 @@ So you might write a config file (if you use these often) like: branch: master earliest_version: 12.0.0 collapse_pre_releases: false + stop_at_branch_base: true These will be parsed first and then the CLI options will be applied after the config files. diff --git a/releasenotes/notes/stop-scanning-branch-option-6a0156b183814d7f.yaml b/releasenotes/notes/stop-scanning-branch-option-6a0156b183814d7f.yaml new file mode 100644 index 0000000..fbba24b --- /dev/null +++ b/releasenotes/notes/stop-scanning-branch-option-6a0156b183814d7f.yaml @@ -0,0 +1,9 @@ +--- +features: + - Add a new configuration option, stop_at_branch_base, to control + whether or not the scanner stops looking for changes at the point + where a branch diverges from master. The default is True, meaning + that the scanner does stop. A false value means that versions that + appear on master from a point earlier than when the branch was + created will be included when scanning the branch for release + notes. diff --git a/reno/config.py b/reno/config.py index 5e47eb8..224d57f 100644 --- a/reno/config.py +++ b/reno/config.py @@ -32,6 +32,10 @@ class Config(object): # of the same number (1.0.0.0a1 notes appear under 1.0.0). 'collapse_pre_releases': True, + # Should the scanner stop at the base of a branch (True) or go + # ahead and scan the entire history (False)? + 'stop_at_branch_base': True, + # The git branch to scan. Defaults to the "current" branch # checked out. 'branch': None, diff --git a/reno/main.py b/reno/main.py index 50af629..3ca29c3 100644 --- a/reno/main.py +++ b/reno/main.py @@ -44,6 +44,15 @@ _query_args = [ dict(default=False, action='store_true', help='if there is a cache file present, do not use it')), + (('--stop-at-branch-base',), + dict(action='store_true', + default=True, + dest='stop_at_branch_base', + help='stop scanning when the branch meets master')), + (('--no-stop-at-branch-base',), + dict(action='store_false', + dest='stop_at_branch_base', + help='do not stop scanning when the branch meets master')), ] diff --git a/reno/scanner.py b/reno/scanner.py index 559aa52..8247dfd 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -224,13 +224,15 @@ def get_notes_by_version(conf): branch = conf.branch earliest_version = conf.earliest_version collapse_pre_releases = conf.collapse_pre_releases + stop_at_branch_base = conf.stop_at_branch_base LOG.debug('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch)) # If the user has not told us where to stop, try to work it out # for ourselves. If branch is set and is not "master", then we # want to stop at the base of the branch. - if (not earliest_version) and branch and (branch != 'master'): + if (stop_at_branch_base and + (not earliest_version) and branch and (branch != 'master')): LOG.debug('determining earliest_version from branch') earliest_version = _get_branch_base(reporoot, branch) if earliest_version and collapse_pre_releases: diff --git a/reno/sphinxext.py b/reno/sphinxext.py index e561e6c..018ce71 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -36,6 +36,7 @@ class ReleaseNotesDirective(rst.Directive): 'version': directives.unchanged, 'collapse-pre-releases': directives.flag, 'earliest-version': directives.unchanged, + 'stop-at-branch-base': directives.flag, } def run(self): @@ -56,10 +57,11 @@ class ReleaseNotesDirective(rst.Directive): if 'notesdir' in self.options: opt_overrides['notesdir'] = self.options.get('notesdir') version_opt = self.options.get('version') - # FIXME(dhellmann): Force this flag True for now and figure + # FIXME(dhellmann): Force these flags True for now and figure # out how Sphinx passes a "false" flag later. # 'collapse-pre-releases' in self.options opt_overrides['collapse_pre_releases'] = True + opt_overrides['stop_at_branch_base'] = True if 'earliest-version' in self.options: opt_overrides['earliest_version'] = self.options.get( 'earliest-version') diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 810fca4..94b99f2 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -821,6 +821,34 @@ class BranchTest(Base): results, ) + def test_files_stable_from_master_no_stop_base(self): + self._run_git('checkout', '2.0.0') + self._run_git('checkout', '-b', 'stable/2') + f21 = self._add_notes_file('slug21') + self._run_git('checkout', 'master') + log_text = self._run_git('log', '--pretty=%x00%H %d', '--name-only', + 'stable/2') + self.addDetail('git log', text_content(log_text)) + self.c.override( + branch='stable/2', + ) + self.c.override( + stop_at_branch_base=False, + ) + raw_results = scanner.get_notes_by_version(self.c) + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '1.0.0': [self.f1], + '2.0.0': [self.f2], + '2.0.0-1': [f21], + }, + results, + ) + def test_pre_release_branch_no_collapse(self): f4 = self._add_notes_file('slug4') self._run_git('tag', '-s', '-m', 'pre-release', '4.0.0.0rc1') -- GitLab From 4964ab786d0199da8d16d51c4ec4fbfee8ac7516 Mon Sep 17 00:00:00 2001 From: Cedric Brandily Date: Thu, 10 Nov 2016 16:09:44 +0100 Subject: [PATCH 072/257] Support to set a custom template used to create new notes Currently the template used to create new notes is hardcoded in reno source code. It's fine when reno is used inside OpenStack community but external projects would perhaps like to use a custom template in order to better match their needs. This change supports to set through "template" attribute in config.yaml a custom template which will be used by reno new to create new notes. Change-Id: I83a870eb9906c26e5ee946d53b9c154867180971 --- doc/source/usage.rst | 6 +- ...port-custom-template-0534a2199cfec44c.yaml | 8 ++ reno/config.py | 71 ++++++++++++++++++ reno/create.py | 75 +------------------ reno/tests/test_config.py | 5 ++ reno/tests/test_create.py | 4 +- reno/tests/test_scanner.py | 2 +- 7 files changed, 95 insertions(+), 76 deletions(-) create mode 100644 releasenotes/notes/support-custom-template-0534a2199cfec44c.yaml diff --git a/doc/source/usage.rst b/doc/source/usage.rst index b005d28..e8985de 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -35,7 +35,8 @@ being installed globally. For example By default the new note is created under ``./releasenotes/notes``. Use the ``--rel-notes-dir`` to change the parent directory (the ``notes`` -subdirectory is always appended). +subdirectory is always appended). It's also possible to set a custom +template to create notes (see `Configuring Reno`_ ). Editing a Release Note ====================== @@ -171,6 +172,9 @@ So you might write a config file (if you use these often) like: earliest_version: 12.0.0 collapse_pre_releases: false stop_at_branch_base: true + template: | + + ... These will be parsed first and then the CLI options will be applied after the config files. diff --git a/releasenotes/notes/support-custom-template-0534a2199cfec44c.yaml b/releasenotes/notes/support-custom-template-0534a2199cfec44c.yaml new file mode 100644 index 0000000..b47e859 --- /dev/null +++ b/releasenotes/notes/support-custom-template-0534a2199cfec44c.yaml @@ -0,0 +1,8 @@ +--- +prelude: > + Support to set a custom template used to create new notes +features: + - | + Reno now supports to set through ``template`` attribute in + ``config.yaml`` a custom template which will be used by reno new + to create notes. diff --git a/reno/config.py b/reno/config.py index 224d57f..b9b4637 100644 --- a/reno/config.py +++ b/reno/config.py @@ -18,6 +18,74 @@ from reno import defaults LOG = logging.getLogger(__name__) +_TEMPLATE = """\ +--- +prelude: > + Replace this text with content to appear at the top of the section for this + release. All of the prelude content is merged together and then rendered + separately from the items listed in other parts of the file, so the text + needs to be worded so that both the prelude and the other items make sense + when read independently. This may mean repeating some details. Not every + release note requires a prelude. Usually only notes describing major + features or adding release theme details should have a prelude. +features: + - | + List new features here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +issues: + - | + List known issues here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +upgrade: + - | + List upgrade notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +deprecations: + - | + List deprecations notes here, or remove this section. All of the list + items in this section are combined when the release notes are rendered, so + the text needs to be worded so that it does not depend on any information + only available in another section, such as the prelude. This may mean + repeating some details. +critical: + - | + Add critical notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +security: + - | + Add security notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +fixes: + - | + Add normal bug fixes here, or remove this section. All of the list items + in this section are combined when the release notes are rendered, so the + text needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +other: + - | + Add other notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +""" + class Config(object): @@ -44,6 +112,9 @@ class Config(object): # lowest version number, and is meant to be the oldest # version. 'earliest_version': None, + + # The template used by reno new to create a note. + 'template': _TEMPLATE } @classmethod diff --git a/reno/create.py b/reno/create.py index d69937f..187829f 100644 --- a/reno/create.py +++ b/reno/create.py @@ -17,75 +17,6 @@ import os from reno import utils -_TEMPLATE = """\ ---- -prelude: > - Replace this text with content to appear at the top of the section for this - release. All of the prelude content is merged together and then rendered - separately from the items listed in other parts of the file, so the text - needs to be worded so that both the prelude and the other items make sense - when read independently. This may mean repeating some details. Not every - release note requires a prelude. Usually only notes describing major - features or adding release theme details should have a prelude. -features: - - | - List new features here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -issues: - - | - List known issues here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -upgrade: - - | - List upgrade notes here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -deprecations: - - | - List deprecations notes here, or remove this section. All of the list - items in this section are combined when the release notes are rendered, so - the text needs to be worded so that it does not depend on any information - only available in another section, such as the prelude. This may mean - repeating some details. -critical: - - | - Add critical notes here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -security: - - | - Add security notes here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -fixes: - - | - Add normal bug fixes here, or remove this section. All of the list items - in this section are combined when the release notes are rendered, so the - text needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -other: - - | - Add other notes here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -""" - - def _pick_note_file_name(notesdir, slug): "Pick a unique name in notesdir." for i in range(50): @@ -100,12 +31,12 @@ def _pick_note_file_name(notesdir, slug): ) -def _make_note_file(filename): +def _make_note_file(filename, template): notesdir = os.path.dirname(filename) if not os.path.exists(notesdir): os.makedirs(notesdir) with open(filename, 'w') as f: - f.write(_TEMPLATE) + f.write(template) def create_cmd(args, conf): @@ -118,6 +49,6 @@ def create_cmd(args, conf): # concern. slug = args.slug.replace(' ', '-') filename = _pick_note_file_name(conf.notespath, slug) - _make_note_file(filename) + _make_note_file(filename, conf.template) print('Created new notes file in %s' % filename) return diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py index 313f537..c2e0237 100644 --- a/reno/tests/test_config.py +++ b/reno/tests/test_config.py @@ -156,3 +156,8 @@ class TestConfigProperties(base.TestCase): self.assertEqual('releasenotes/notes', self.c.notespath) self.c.override(notesdir='thenotes') self.assertEqual('releasenotes/thenotes', self.c.notespath) + + def test_template(self): + self.assertEqual(config._TEMPLATE, self.c.template) + self.c.override(template='i-am-a-template') + self.assertEqual('i-am-a-template', self.c.template) diff --git a/reno/tests/test_create.py b/reno/tests/test_create.py index 04675fb..1fc305a 100644 --- a/reno/tests/test_create.py +++ b/reno/tests/test_create.py @@ -47,7 +47,7 @@ class TestCreate(base.TestCase): def test_create_from_template(self): filename = create._pick_note_file_name(self.tmpdir, 'theslug') - create._make_note_file(filename) + create._make_note_file(filename, 'i-am-a-template') with open(filename, 'r') as f: body = f.read() - self.assertEqual(create._TEMPLATE, body) + self.assertEqual('i-am-a-template', body) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 94b99f2..dbad7c4 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -158,7 +158,7 @@ class Base(base.TestCase): basename = '%s-%016x.yaml' % (slug, n) filename = os.path.join(self.reporoot, 'releasenotes', 'notes', basename) - create._make_note_file(filename) + create._make_note_file(filename, 'i-am-also-a-template') self._git_commit('add %s' % basename) return os.path.join('releasenotes', 'notes', basename) -- GitLab From 92a9bcdb142f00e2e46d4711f84c6dbf60a13407 Mon Sep 17 00:00:00 2001 From: Cedric Brandily Date: Thu, 10 Nov 2016 20:09:42 +0100 Subject: [PATCH 073/257] Enable to create and edit a note with reno new Commonly, we edit a note just after its creation with reno new command. This change enables to do both with reno new --edit, it creates a new note and edits it with your editor (defined with EDITOR env variable). Change-Id: I7866277c8ceaae41b9b9d0225ee33dcc37f3be6b --- doc/source/usage.rst | 8 ++++++++ .../notes/support-edit-ec5c01ad6144815a.yaml | 7 +++++++ reno/create.py | 11 +++++++++++ reno/main.py | 5 +++++ reno/tests/test_create.py | 12 ++++++++++++ 5 files changed, 43 insertions(+) create mode 100644 releasenotes/notes/support-edit-ec5c01ad6144815a.yaml diff --git a/doc/source/usage.rst b/doc/source/usage.rst index e8985de..e89c215 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -32,6 +32,14 @@ being installed globally. For example releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml +The ``--edit`` option enables to edit the note just after its creation. + +:: + + $ reno new slug-goes-here --edit + ... Open your editor (defined with EDITOR environment variable) ... + Created new notes file in releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml + By default the new note is created under ``./releasenotes/notes``. Use the ``--rel-notes-dir`` to change the parent directory (the ``notes`` diff --git a/releasenotes/notes/support-edit-ec5c01ad6144815a.yaml b/releasenotes/notes/support-edit-ec5c01ad6144815a.yaml new file mode 100644 index 0000000..48753c8 --- /dev/null +++ b/releasenotes/notes/support-edit-ec5c01ad6144815a.yaml @@ -0,0 +1,7 @@ +--- +prelude: > + Enable to create and edit a note with reno new --edit. +features: + - | + Reno now enables with reno new --edit to create a note and edit it with + your editor (defined with EDITOR environment variable). diff --git a/reno/create.py b/reno/create.py index 187829f..0b5e0ea 100644 --- a/reno/create.py +++ b/reno/create.py @@ -13,6 +13,7 @@ from __future__ import print_function import os +import subprocess from reno import utils @@ -39,6 +40,13 @@ def _make_note_file(filename, template): f.write(template) +def _edit_file(filename): + if 'EDITOR' not in os.environ: + return False + subprocess.call([os.environ['EDITOR'], filename]) + return True + + def create_cmd(args, conf): "Create a new release note file from the template." # NOTE(dhellmann): There is a short race window where we might try @@ -50,5 +58,8 @@ def create_cmd(args, conf): slug = args.slug.replace(' ', '-') filename = _pick_note_file_name(conf.notespath, slug) _make_note_file(filename, conf.template) + if args.edit and not _edit_file(filename): + print('Was unable to edit the new note. EDITOR environment variable ' + 'is missing!') print('Created new notes file in %s' % filename) return diff --git a/reno/main.py b/reno/main.py index 3ca29c3..79f8589 100644 --- a/reno/main.py +++ b/reno/main.py @@ -93,6 +93,11 @@ def main(argv=sys.argv[1:]): 'new', help='create a new note', ) + do_new.add_argument( + '--edit', + action='store_true', + help='Edit note after its creation (require EDITOR env variable)', + ) do_new.add_argument( 'slug', help='descriptive title of note (keep it short)', diff --git a/reno/tests/test_create.py b/reno/tests/test_create.py index 1fc305a..2ad374a 100644 --- a/reno/tests/test_create.py +++ b/reno/tests/test_create.py @@ -51,3 +51,15 @@ class TestCreate(base.TestCase): with open(filename, 'r') as f: body = f.read() self.assertEqual('i-am-a-template', body) + + def test_edit(self): + self.useFixture(fixtures.EnvironmentVariable('EDITOR', 'myeditor')) + with mock.patch('subprocess.call') as call_mock: + self.assertTrue(create._edit_file('somepath')) + call_mock.assert_called_once_with(['myeditor', 'somepath']) + + def test_edit_without_editor_env_var(self): + self.useFixture(fixtures.EnvironmentVariable('EDITOR')) + with mock.patch('subprocess.call') as call_mock: + self.assertFalse(create._edit_file('somepath')) + call_mock.assert_not_called() -- GitLab From 37cd75904d7193ff66e08669285ac048b49cdf78 Mon Sep 17 00:00:00 2001 From: Cao Xuan Hoang Date: Thu, 17 Nov 2016 10:18:17 +0700 Subject: [PATCH 074/257] Clean imports in code This patch set modifies lines which are importing objects instead of modules. As per openstack import guide lines, user should import modules in a file not objects. http://docs.openstack.org/developer/hacking/#imports Change-Id: I0307642df8157d7a70b7fd8bb77892b19a539575 --- reno/sphinxext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 018ce71..7903d90 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -15,7 +15,7 @@ import os.path from docutils import nodes from docutils.parsers import rst from docutils.parsers.rst import directives -from docutils.statemachine import ViewList +from docutils import statemachine from sphinx.util.nodes import nested_parse_with_titles from reno import config @@ -89,7 +89,7 @@ class ReleaseNotesDirective(rst.Directive): title=title, ) source_name = '<' + __name__ + '>' - result = ViewList() + result = statemachine.ViewList() for line in text.splitlines(): result.append(line, source_name) -- GitLab From d90bb77ab2ea9888696141a020727288a7c5ac46 Mon Sep 17 00:00:00 2001 From: Zane Bitter Date: Thu, 1 Dec 2016 11:24:59 -0500 Subject: [PATCH 075/257] Link to reStructuredText primer from usage docs Change-Id: I1282801ffead6393a557cc80164f138603b91019 --- doc/source/usage.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/usage.rst b/doc/source/usage.rst index e89c215..268976e 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -50,7 +50,7 @@ Editing a Release Note ====================== The note file is a YAML file with several sections. All of the text is -interpreted as having reStructuredText formatting. +interpreted as having `reStructuredText`_ formatting. prelude @@ -126,7 +126,7 @@ entirely. Formatting ---------- -Release notes may include embedded reStructuredText, including simple +Release notes may include embedded `reStructuredText`_, including simple inline markup like emphasis and pre-formatted text as well as complex body structures such as nested lists and tables. To use these formatting features, the note must be escaped from the YAML parser. @@ -143,6 +143,8 @@ the value with ``|``. For example: See :doc:`examples` for the rendered version of the note. +.. _reStructuredText: http://www.sphinx-doc.org/en/stable/rest.html + Generating a Report =================== -- GitLab From 17a248dc8447d3bbb37fa03b4ba2a75d040e6acf Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 6 Dec 2016 16:37:57 -0500 Subject: [PATCH 076/257] skip the test that fails on git 2.9.2 Add a skip rule to avoid running the test that fails on git version 2.9.2. When the logic involved is rewritten using dulwich in a later patch, the skip will be removed. Change-Id: Id8b6f74fe4feb9bec8f47c2141abfc7d463b0e57 Signed-off-by: Doug Hellmann --- reno/tests/test_scanner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index dbad7c4..7a11351 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -20,6 +20,7 @@ import os.path import re import subprocess import textwrap +import unittest import fixtures import mock @@ -54,6 +55,8 @@ packages = testpkg """ +GIT_VERSION = utils.check_output(['git', '--version']).strip() + class GPGKeyFixture(fixtures.Fixture): """Creates a GPG key for testing. @@ -470,7 +473,10 @@ class BasicTest(Base): results, ) + @unittest.skipIf(GIT_VERSION == 'git version 2.9.2', + 'Skipping for git version 2.9.2') def test_rename_then_delete_file(self): + print(GIT_VERSION) self._make_python_package() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1') -- GitLab From db4e3647e210299afabe56a526b85344fee6fc2d Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 5 Dec 2016 16:17:30 -0500 Subject: [PATCH 077/257] refactor existing implementation into a class As part of the re-implementation work using dulwich I want to create a Scanner class, to replace some of the module-level functions in the scanner module now. This patch moves existing code around into something like what I would expect that API to look like, without changing the implementation details. The class API is still subject to change in the future, but this should make it a little easier to review those implementation changes. Change-Id: If92f6ff5bce955bbdcb748cee52ad71210ae313b Signed-off-by: Doug Hellmann --- reno/cache.py | 9 +- reno/loader.py | 5 +- reno/scanner.py | 669 +++++++++++++++++++------------------ reno/tests/test_cache.py | 6 +- reno/tests/test_scanner.py | 101 ++++-- 5 files changed, 412 insertions(+), 378 deletions(-) diff --git a/reno/cache.py b/reno/cache.py index 44922e1..ad852db 100644 --- a/reno/cache.py +++ b/reno/cache.py @@ -20,7 +20,8 @@ from reno import scanner def build_cache_db(conf, versions_to_include): - notes = scanner.get_notes_by_version(conf) + s = scanner.Scanner(conf) + notes = s.get_notes_by_version() # Default to including all versions returned by the scanner. if not versions_to_include: @@ -31,11 +32,7 @@ def build_cache_db(conf, versions_to_include): file_contents = {} for version in versions_to_include: for filename, sha in notes[version]: - body = scanner.get_file_at_commit( - conf.reporoot, - filename, - sha, - ) + body = s.get_file_at_commit(filename, sha) # We want to save the contents of the file, which is YAML, # inside another YAML file. That looks terribly ugly with # all of the escapes needed to format it properly as diff --git a/reno/loader.py b/reno/loader.py index 036e73c..5d70cdb 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -52,6 +52,7 @@ class Loader(object): self._earliest_version = conf.earliest_version self._cache = None + self._scanner = scanner.Scanner(self._config) self._scanner_output = None self._cache_filename = get_cache_filename(self._reporoot, self._notespath) @@ -75,7 +76,7 @@ class Loader(object): for n in self._cache['notes'] } else: - self._scanner_output = scanner.get_notes_by_version(self._config) + self._scanner_output = self._scanner.get_notes_by_version() @property def versions(self): @@ -96,7 +97,7 @@ class Loader(object): if self._cache: content = self._cache['file-contents'][filename] else: - body = scanner.get_file_at_commit(self._reporoot, filename, sha) + body = self._scanner.get_file_at_commit(filename, sha) content = yaml.safe_load(body) for section_name, section_content in content.items(): diff --git a/reno/scanner.py b/reno/scanner.py index 611f366..cd35291 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -25,48 +25,6 @@ from reno import utils LOG = logging.getLogger(__name__) -def _get_current_version(reporoot, branch=None): - """Return the current version of the repository. - - If the repo appears to contain a python project, use setup.py to - get the version so pbr (if used) can do its thing. Otherwise, use - git describe. - - """ - cmd = ['git', 'describe', '--tags'] - if branch is not None: - cmd.append(branch) - try: - result = utils.check_output(cmd, cwd=reporoot).strip() - if '-' in result: - # Descriptions that come after a commit look like - # 2.0.0-1-abcde, and we want to remove the SHA value from - # the end since we only care about the version number - # itself, but we need to recognize that the change is - # unreleased so keep the -1 part. - result, dash, ignore = result.rpartition('-') - except subprocess.CalledProcessError: - # This probably means there are no tags. - result = '0.0.0' - return result - - -def get_file_at_commit(reporoot, filename, sha): - "Return the contents of the file if it exists at the commit, or None." - try: - return utils.check_output( - ['git', 'show', '%s:%s' % (sha, filename)], - cwd=reporoot, - ) - except subprocess.CalledProcessError: - return None - - -def _file_exists_at_commit(reporoot, filename, sha): - "Return true if the file exists at the given commit." - return bool(get_file_at_commit(reporoot, filename, sha)) - - def _get_unique_id(filename): base = os.path.basename(filename) root, ext = os.path.splitext(base) @@ -78,64 +36,6 @@ def _get_unique_id(filename): return uniqueid -def _get_branch_base(reporoot, branch): - "Return the tag at base of the branch." - # Based on - # http://stackoverflow.com/questions/1527234/finding-a-branch-point-with-git - # git rev-list $(git rev-list --first-parent \ - # ^origin/stable/newton master | tail -n1)^^! - # - # Determine the list of commits accessible from the branch we are - # supposed to be scanning, but not on master. - cmd = [ - 'git', - 'rev-list', - '--first-parent', - branch, # on the branch - '^master', # not on master - ] - try: - LOG.debug(' '.join(cmd)) - parents = utils.check_output(cmd, cwd=reporoot).strip() - if not parents: - # There are no commits on the branch, yet, so we can use - # our current-version logic. - return _get_current_version(reporoot, branch) - except subprocess.CalledProcessError as e: - LOG.warning('failed to retrieve branch base: %s [%s]', - e, e.output.strip()) - return None - parent = parents.splitlines()[-1] - LOG.debug('parent = %r', parent) - # Now get the previous commit, which should be the one we tagged - # to create the branch. - cmd = [ - 'git', - 'rev-list', - '{}^^!'.format(parent), - ] - try: - sha = utils.check_output(cmd, cwd=reporoot).strip() - LOG.debug('sha = %r', sha) - except subprocess.CalledProcessError as e: - LOG.warning('failed to retrieve branch base: %s [%s]', - e, e.output.strip()) - return None - # Now get the tag for that commit. - cmd = [ - 'git', - 'describe', - '--abbrev=0', - sha, - ] - try: - return utils.check_output(cmd, cwd=reporoot).strip() - except subprocess.CalledProcessError as e: - LOG.warning('failed to retrieve branch base: %s [%s]', - e, e.output.strip()) - return None - - # The git log output from _get_tags_on_branch() looks like this sample # from the openstack/nova repository for git 1.9.1: # @@ -207,244 +107,349 @@ def _get_version_tags_on_branch(reporoot, branch): return tags -def get_notes_by_version(conf): - """Return an OrderedDict mapping versions to lists of notes files. +class Scanner(object): - The versions are presented in reverse chronological order. + def __init__(self, conf): + self.conf = conf + self.reporoot = self.conf.reporoot - Notes files are associated with the earliest version for which - they were available, regardless of whether they changed later. + def _get_current_version(self, branch=None): + """Return the current version of the repository. - :param reporoot: Path to the root of the git repository. - :type reporoot: str - """ + If the repo appears to contain a python project, use setup.py to + get the version so pbr (if used) can do its thing. Otherwise, use + git describe. - reporoot = conf.reporoot - notesdir = conf.notespath - branch = conf.branch - earliest_version = conf.earliest_version - collapse_pre_releases = conf.collapse_pre_releases - stop_at_branch_base = conf.stop_at_branch_base - - LOG.debug('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch)) - - # If the user has not told us where to stop, try to work it out - # for ourselves. If branch is set and is not "master", then we - # want to stop at the base of the branch. - if (stop_at_branch_base and - (not earliest_version) and branch and (branch != 'master')): - LOG.debug('determining earliest_version from branch') - earliest_version = _get_branch_base(reporoot, branch) - if earliest_version and collapse_pre_releases: - if PRE_RELEASE_RE.search(earliest_version): - # The earliest version won't actually be the pre-release - # that might have been tagged when the branch was created, - # but the final version. Strip the pre-release portion of - # the version number. - earliest_version = '.'.join(earliest_version.split('.')[:-1]) - LOG.debug('using earliest_version = %r', earliest_version) - - # Determine all of the tags known on the branch, in their date - # order. We scan the commit history in topological order to ensure - # we have the commits in the right version, so we might encounter - # the tags in a different order during that phase. - versions_by_date = _get_version_tags_on_branch(reporoot, branch) - LOG.debug('versions by date %r' % (versions_by_date,)) - versions = [] - earliest_seen = collections.OrderedDict() - - # Determine the current version, which might be an unreleased or - # dev version if there are unreleased commits at the head of the - # branch in question. Since the version may not already be known, - # make sure it is in the list of versions by date. And since it is - # the most recent version, go ahead and insert it at the front of - # the list. - current_version = _get_current_version(reporoot, branch) - LOG.debug('current repository version: %s' % current_version) - if current_version not in versions_by_date: - LOG.debug('adding %s to versions by date' % current_version) - versions_by_date.insert(0, current_version) - - # Remember the most current filename for each id, to allow for - # renames. - last_name_by_id = {} - - # Remember uniqueids that have had files deleted. - uniqueids_deleted = collections.defaultdict(set) - - # FIXME(dhellmann): This might need to be more line-oriented for - # longer histories. - log_cmd = [ - 'git', 'log', - '--topo-order', # force traversal order rather than date order - '--pretty=%x00%H %d', # output contents in parsable format - '--name-only' # only include the names of the files in the patch - ] - if branch is not None: - log_cmd.append(branch) - LOG.debug('running %s' % ' '.join(log_cmd)) - history_results = utils.check_output(log_cmd, cwd=reporoot) - history = history_results.split('\x00') - current_version = current_version - for h in history: - h = h.strip() - if not h: - continue - # print(h) - - hlines = h.splitlines() - - # The first line of the block will include the SHA and may - # include tags, the other lines are filenames. - sha = hlines[0].split(' ')[0] - tags = TAG_RE.findall(hlines[0]) - # Filter the files based on the notes directory we were - # given. We cannot do this in the git log command directly - # because it means we end up skipping some of the tags if the - # commits being tagged don't include any release note - # files. Even if this list ends up empty, we continue doing - # the other processing so that we record all of the known - # versions. - filenames = [] - for f in hlines[2:]: - if fnmatch.fnmatch(f, notesdir + '/*.yaml'): - filenames.append(f) - elif fnmatch.fnmatch(f, notesdir + '/*'): - LOG.warning('found and ignored extra file %s', f) - - # If there are no tags in this block, assume the most recently - # seen version. - if not tags: - tags = [current_version] - else: - current_version = tags[0] - LOG.debug('%s has tags %s (%r), updating current version to %s' % - (sha, tags, hlines[0], current_version)) - - # Remember each version we have seen. - if current_version not in versions: - LOG.debug('%s is a new version' % current_version) - versions.append(current_version) - - LOG.debug('%s contains files %s' % (sha, filenames)) - - # Remember the files seen, using their UUID suffix as a unique id. - for f in filenames: - # Updated as older tags are found, handling edits to release - # notes. - uniqueid = _get_unique_id(f) - LOG.debug('%s: found file %s', - uniqueid, f) - LOG.debug('%s: setting earliest reference to %s' % - (uniqueid, tags[0])) - earliest_seen[uniqueid] = tags[0] - if uniqueid in last_name_by_id: - # We already have a filename for this id from a - # new commit, so use that one in case the name has - # changed. - LOG.debug('%s: was seen before in %s', - uniqueid, last_name_by_id[uniqueid]) - continue - elif _file_exists_at_commit(reporoot, f, sha): - LOG.debug('%s: looking for %s in deleted files %s', - uniqueid, f, uniqueids_deleted[uniqueid]) - if f in uniqueids_deleted[uniqueid]: - # The file exists in the commit, but was deleted - # later in the history. - LOG.debug('%s: skipping deleted file %s', - uniqueid, f) - else: - # Remember this filename as the most recent version of - # the unique id we have seen, in case the name - # changed from an older commit. - last_name_by_id[uniqueid] = (f, sha) - LOG.debug('%s: remembering %s as filename', - uniqueid, f) - else: - # Track files that have been deleted. The rename logic - # above checks for repeated references to files that - # are deleted later, and the inversion logic below - # checks for any remaining values and skips those - # entries. - LOG.debug('%s: saw a file that no longer exists', - uniqueid) - uniqueids_deleted[uniqueid].add(f) - LOG.debug('%s: deleted files %s', - uniqueid, uniqueids_deleted[uniqueid]) - - # Invert earliest_seen to make a list of notes files for each - # version. - files_and_tags = collections.OrderedDict() - for v in versions: - files_and_tags[v] = [] - # Produce a list of the actual files present in the repository. If - # a note is removed, this step should let us ignore it. - for uniqueid, version in earliest_seen.items(): + """ + cmd = ['git', 'describe', '--tags'] + if branch is not None: + cmd.append(branch) + try: + result = utils.check_output(cmd, cwd=self.reporoot).strip() + if '-' in result: + # Descriptions that come after a commit look like + # 2.0.0-1-abcde, and we want to remove the SHA value from + # the end since we only care about the version number + # itself, but we need to recognize that the change is + # unreleased so keep the -1 part. + result, dash, ignore = result.rpartition('-') + except subprocess.CalledProcessError: + # This probably means there are no tags. + result = '0.0.0' + return result + + def _get_branch_base(self, branch): + "Return the tag at base of the branch." + # Based on + # http://stackoverflow.com/questions/1527234/finding-a-branch-point-with-git + # git rev-list $(git rev-list --first-parent \ + # ^origin/stable/newton master | tail -n1)^^! + # + # Determine the list of commits accessible from the branch we are + # supposed to be scanning, but not on master. + cmd = [ + 'git', + 'rev-list', + '--first-parent', + branch, # on the branch + '^master', # not on master + ] + try: + LOG.debug(' '.join(cmd)) + parents = utils.check_output(cmd, cwd=self.reporoot).strip() + if not parents: + # There are no commits on the branch, yet, so we can use + # our current-version logic. + return self._get_current_version(branch) + except subprocess.CalledProcessError as e: + LOG.warning('failed to retrieve branch base: %s [%s]', + e, e.output.strip()) + return None + parent = parents.splitlines()[-1] + LOG.debug('parent = %r', parent) + # Now get the previous commit, which should be the one we tagged + # to create the branch. + cmd = [ + 'git', + 'rev-list', + '{}^^!'.format(parent), + ] try: - base, sha = last_name_by_id[uniqueid] - if base in uniqueids_deleted.get(uniqueid, set()): - LOG.debug('skipping deleted note %s' % uniqueid) + sha = utils.check_output(cmd, cwd=self.reporoot).strip() + LOG.debug('sha = %r', sha) + except subprocess.CalledProcessError as e: + LOG.warning('failed to retrieve branch base: %s [%s]', + e, e.output.strip()) + return None + # Now get the tag for that commit. + cmd = [ + 'git', + 'describe', + '--abbrev=0', + sha, + ] + try: + return utils.check_output(cmd, cwd=self.reporoot).strip() + except subprocess.CalledProcessError as e: + LOG.warning('failed to retrieve branch base: %s [%s]', + e, e.output.strip()) + return None + + def get_file_at_commit(self, filename, sha): + "Return the contents of the file if it exists at the commit, or None." + try: + return utils.check_output( + ['git', 'show', '%s:%s' % (sha, filename)], + cwd=self.reporoot, + ) + except subprocess.CalledProcessError: + return None + + def _file_exists_at_commit(self, filename, sha): + "Return true if the file exists at the given commit." + return bool(self.get_file_at_commit(filename, sha)) + + def get_notes_by_version(self): + """Return an OrderedDict mapping versions to lists of notes files. + + The versions are presented in reverse chronological order. + + Notes files are associated with the earliest version for which + they were available, regardless of whether they changed later. + + :param reporoot: Path to the root of the git repository. + :type reporoot: str + """ + + reporoot = self.reporoot + notesdir = self.conf.notespath + branch = self.conf.branch + earliest_version = self.conf.earliest_version + collapse_pre_releases = self.conf.collapse_pre_releases + stop_at_branch_base = self.conf.stop_at_branch_base + + LOG.debug('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch)) + + # If the user has not told us where to stop, try to work it out + # for ourselves. If branch is set and is not "master", then we + # want to stop at the base of the branch. + if (stop_at_branch_base and + (not earliest_version) and branch and (branch != 'master')): + LOG.debug('determining earliest_version from branch') + earliest_version = self._get_branch_base(branch) + if earliest_version and collapse_pre_releases: + if PRE_RELEASE_RE.search(earliest_version): + # The earliest version won't actually be the pre-release + # that might have been tagged when the branch was created, + # but the final version. Strip the pre-release portion of + # the version number. + earliest_version = '.'.join( + earliest_version.split('.')[:-1] + ) + LOG.debug('using earliest_version = %r', earliest_version) + + # Determine all of the tags known on the branch, in their date + # order. We scan the commit history in topological order to ensure + # we have the commits in the right version, so we might encounter + # the tags in a different order during that phase. + versions_by_date = _get_version_tags_on_branch(reporoot, branch) + LOG.debug('versions by date %r' % (versions_by_date,)) + versions = [] + earliest_seen = collections.OrderedDict() + + # Determine the current version, which might be an unreleased or + # dev version if there are unreleased commits at the head of the + # branch in question. Since the version may not already be known, + # make sure it is in the list of versions by date. And since it is + # the most recent version, go ahead and insert it at the front of + # the list. + current_version = self._get_current_version(branch) + LOG.debug('current repository version: %s' % current_version) + if current_version not in versions_by_date: + LOG.debug('adding %s to versions by date' % current_version) + versions_by_date.insert(0, current_version) + + # Remember the most current filename for each id, to allow for + # renames. + last_name_by_id = {} + + # Remember uniqueids that have had files deleted. + uniqueids_deleted = collections.defaultdict(set) + + # FIXME(dhellmann): This might need to be more line-oriented for + # longer histories. + log_cmd = [ + 'git', 'log', + '--topo-order', # force traversal order rather than date order + '--pretty=%x00%H %d', # output contents in parsable format + '--name-only' # only include the names of the files in the patch + ] + if branch is not None: + log_cmd.append(branch) + LOG.debug('running %s' % ' '.join(log_cmd)) + history_results = utils.check_output(log_cmd, cwd=reporoot) + history = history_results.split('\x00') + current_version = current_version + for h in history: + h = h.strip() + if not h: continue - files_and_tags[version].append((base, sha)) - except KeyError: - # Unable to find the file again, skip it to avoid breaking - # the build. - msg = ('[reno] unable to find file associated ' - 'with unique id %r, skipping') % uniqueid - LOG.debug(msg) - print(msg, file=sys.stderr) - - # Combine pre-releases into the final release, if we are told to - # and the final release exists. - if collapse_pre_releases: - collapsing = files_and_tags + # print(h) + + hlines = h.splitlines() + + # The first line of the block will include the SHA and may + # include tags, the other lines are filenames. + sha = hlines[0].split(' ')[0] + tags = TAG_RE.findall(hlines[0]) + # Filter the files based on the notes directory we were + # given. We cannot do this in the git log command directly + # because it means we end up skipping some of the tags if the + # commits being tagged don't include any release note + # files. Even if this list ends up empty, we continue doing + # the other processing so that we record all of the known + # versions. + filenames = [] + for f in hlines[2:]: + if fnmatch.fnmatch(f, notesdir + '/*.yaml'): + filenames.append(f) + elif fnmatch.fnmatch(f, notesdir + '/*'): + LOG.warning('found and ignored extra file %s', f) + + # If there are no tags in this block, assume the most recently + # seen version. + if not tags: + tags = [current_version] + else: + current_version = tags[0] + LOG.debug('%s has tags %s (%r), ' + 'updating current version to %s' % + (sha, tags, hlines[0], current_version)) + + # Remember each version we have seen. + if current_version not in versions: + LOG.debug('%s is a new version' % current_version) + versions.append(current_version) + + LOG.debug('%s contains files %s' % (sha, filenames)) + + # Remember the files seen, using their UUID suffix as a unique id. + for f in filenames: + # Updated as older tags are found, handling edits to release + # notes. + uniqueid = _get_unique_id(f) + LOG.debug('%s: found file %s', + uniqueid, f) + LOG.debug('%s: setting earliest reference to %s' % + (uniqueid, tags[0])) + earliest_seen[uniqueid] = tags[0] + if uniqueid in last_name_by_id: + # We already have a filename for this id from a + # new commit, so use that one in case the name has + # changed. + LOG.debug('%s: was seen before in %s', + uniqueid, last_name_by_id[uniqueid]) + continue + elif self._file_exists_at_commit(f, sha): + LOG.debug('%s: looking for %s in deleted files %s', + uniqueid, f, uniqueids_deleted[uniqueid]) + if f in uniqueids_deleted[uniqueid]: + # The file exists in the commit, but was deleted + # later in the history. + LOG.debug('%s: skipping deleted file %s', + uniqueid, f) + else: + # Remember this filename as the most recent version of + # the unique id we have seen, in case the name + # changed from an older commit. + last_name_by_id[uniqueid] = (f, sha) + LOG.debug('%s: remembering %s as filename', + uniqueid, f) + else: + # Track files that have been deleted. The rename logic + # above checks for repeated references to files that + # are deleted later, and the inversion logic below + # checks for any remaining values and skips those + # entries. + LOG.debug('%s: saw a file that no longer exists', + uniqueid) + uniqueids_deleted[uniqueid].add(f) + LOG.debug('%s: deleted files %s', + uniqueid, uniqueids_deleted[uniqueid]) + + # Invert earliest_seen to make a list of notes files for each + # version. files_and_tags = collections.OrderedDict() + for v in versions: + files_and_tags[v] = [] + # Produce a list of the actual files present in the repository. If + # a note is removed, this step should let us ignore it. + for uniqueid, version in earliest_seen.items(): + try: + base, sha = last_name_by_id[uniqueid] + if base in uniqueids_deleted.get(uniqueid, set()): + LOG.debug('skipping deleted note %s' % uniqueid) + continue + files_and_tags[version].append((base, sha)) + except KeyError: + # Unable to find the file again, skip it to avoid breaking + # the build. + msg = ('[reno] unable to find file associated ' + 'with unique id %r, skipping') % uniqueid + LOG.debug(msg) + print(msg, file=sys.stderr) + + # Combine pre-releases into the final release, if we are told to + # and the final release exists. + if collapse_pre_releases: + collapsing = files_and_tags + files_and_tags = collections.OrderedDict() + for ov in versions_by_date: + if ov not in collapsing: + # We don't need to collapse this one because there are + # no notes attached to it. + continue + pre_release_match = PRE_RELEASE_RE.search(ov) + LOG.debug('checking %r', ov) + if pre_release_match: + # Remove the trailing pre-release part of the version + # from the string. + pre_rel_str = pre_release_match.groups()[0] + canonical_ver = ov[:-len(pre_rel_str)].rstrip('.') + if canonical_ver not in versions_by_date: + # This canonical version was never tagged, so we + # do not want to collapse the pre-releases. Reset + # to the original version. + canonical_ver = ov + else: + LOG.debug('combining into %r', canonical_ver) + else: + canonical_ver = ov + if canonical_ver not in files_and_tags: + files_and_tags[canonical_ver] = [] + files_and_tags[canonical_ver].extend(collapsing[ov]) + + # Only return the parts of files_and_tags that actually have + # filenames associated with the versions. + trimmed = collections.OrderedDict() for ov in versions_by_date: - if ov not in collapsing: - # We don't need to collapse this one because there are - # no notes attached to it. + if not files_and_tags.get(ov): continue - pre_release_match = PRE_RELEASE_RE.search(ov) - LOG.debug('checking %r', ov) - if pre_release_match: - # Remove the trailing pre-release part of the version - # from the string. - pre_rel_str = pre_release_match.groups()[0] - canonical_ver = ov[:-len(pre_rel_str)].rstrip('.') - if canonical_ver not in versions_by_date: - # This canonical version was never tagged, so we - # do not want to collapse the pre-releases. Reset - # to the original version. - canonical_ver = ov - else: - LOG.debug('combining into %r', canonical_ver) - else: - canonical_ver = ov - if canonical_ver not in files_and_tags: - files_and_tags[canonical_ver] = [] - files_and_tags[canonical_ver].extend(collapsing[ov]) - - # Only return the parts of files_and_tags that actually have - # filenames associated with the versions. - trimmed = collections.OrderedDict() - for ov in versions_by_date: - if not files_and_tags.get(ov): - continue - # Sort the notes associated with the version so they are in a - # deterministic order, to avoid having the same data result in - # different output depending on random factors. Earlier - # versions of the scanner assumed the notes were recorded in - # chronological order based on the commit date, but with the - # change to use topological sorting that is no longer - # necessarily true. We want the notes to always show up in the - # same order, but it doesn't really matter what order that is, - # so just sort based on the unique id. - trimmed[ov] = sorted(files_and_tags[ov]) - # If we have been told to stop at a version, we can do that - # now. - if earliest_version and ov == earliest_version: - break - - LOG.debug('[reno] found %d versions and %d files', - len(trimmed.keys()), sum(len(ov) for ov in trimmed.values())) - return trimmed + # Sort the notes associated with the version so they are in a + # deterministic order, to avoid having the same data result in + # different output depending on random factors. Earlier + # versions of the scanner assumed the notes were recorded in + # chronological order based on the commit date, but with the + # change to use topological sorting that is no longer + # necessarily true. We want the notes to always show up in the + # same order, but it doesn't really matter what order that is, + # so just sort based on the unique id. + trimmed[ov] = sorted(files_and_tags[ov]) + # If we have been told to stop at a version, we can do that + # now. + if earliest_version and ov == earliest_version: + break + + LOG.debug('[reno] found %d versions and %d files', + len(trimmed.keys()), sum(len(ov) for ov in trimmed.values())) + return trimmed diff --git a/reno/tests/test_cache.py b/reno/tests/test_cache.py index e62d85a..ef7d120 100644 --- a/reno/tests/test_cache.py +++ b/reno/tests/test_cache.py @@ -45,19 +45,19 @@ class TestCache(base.TestCase): """) } - def _get_note_body(self, reporoot, filename, sha): + def _get_note_body(self, filename, sha): return self.note_bodies.get(filename, '') def setUp(self): super(TestCache, self).setUp() self.useFixture( - mockpatch.Patch('reno.scanner.get_file_at_commit', + mockpatch.Patch('reno.scanner.Scanner.get_file_at_commit', new=self._get_note_body) ) self.c = config.Config('.') def test_build_cache_db(self): - with mock.patch('reno.scanner.get_notes_by_version') as gnbv: + with mock.patch('reno.scanner.Scanner.get_notes_by_version') as gnbv: gnbv.return_value = self.scanner_output db = cache.build_cache_db( self.c, diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 7a11351..5869a20 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -197,6 +197,7 @@ class Base(base.TestCase): self.temp_dir = self.useFixture(fixtures.TempDir()).path self.reporoot = os.path.join(self.temp_dir, 'reporoot') self.c = config.Config(self.reporoot) + self.scanner = scanner.Scanner(self.c) self._git_setup() self._counter = itertools.count(1) self.get_note_num = lambda: next(self._counter) @@ -206,7 +207,7 @@ class BasicTest(Base): def test_non_python_no_tags(self): filename = self._add_notes_file() - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -219,7 +220,7 @@ class BasicTest(Base): def test_python_no_tags(self): self._make_python_package() filename = self._add_notes_file() - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -233,7 +234,7 @@ class BasicTest(Base): filename = self._add_notes_file() self._add_other_file('not-a-release-note.txt') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -246,7 +247,7 @@ class BasicTest(Base): def test_note_commit_tagged(self): filename = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -260,7 +261,7 @@ class BasicTest(Base): self._make_python_package() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') filename = self._add_notes_file() - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -275,7 +276,7 @@ class BasicTest(Base): self._add_other_file('ignore-1.txt') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') self._add_other_file('ignore-2.txt') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -290,7 +291,7 @@ class BasicTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file() f2 = self._add_notes_file() - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -305,7 +306,7 @@ class BasicTest(Base): f1 = self._add_notes_file(commit=False) f2 = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -321,7 +322,7 @@ class BasicTest(Base): f1 = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = self._add_notes_file() - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -341,7 +342,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug2') self._run_git('mv', f1, f2) self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -360,7 +361,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug0') self._run_git('mv', f1, f2) self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -379,7 +380,7 @@ class BasicTest(Base): with open(os.path.join(self.reporoot, f1), 'w') as f: f.write('---\npreamble: new contents for file') self._git_commit('edit note file') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -398,7 +399,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug2') self._run_git('mv', f1, f2) self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -420,7 +421,7 @@ class BasicTest(Base): 'slug1-0000000000000001') self._run_git('mv', f1, f2) self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -442,7 +443,7 @@ class BasicTest(Base): self.c.override( earliest_version='2.0.0', ) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -462,7 +463,7 @@ class BasicTest(Base): self._run_git('rm', f1) self._git_commit('remove note file') self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -492,7 +493,7 @@ class BasicTest(Base): '--pretty=%H %d', '--name-only') self.addDetail('git log', text_content(log_results)) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -511,7 +512,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a2') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -527,7 +528,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b2') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -543,7 +544,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc2') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -568,7 +569,7 @@ class PreReleaseTest(Base): self.c.override( collapse_pre_releases=True, ) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -590,7 +591,7 @@ class PreReleaseTest(Base): self.c.override( collapse_pre_releases=True, ) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -615,7 +616,7 @@ class PreReleaseTest(Base): self.c.override( collapse_pre_releases=True, ) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -643,7 +644,7 @@ class MergeCommitTest(Base): self._run_git('merge', '--no-ff', 'test_merge_commit') self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -671,7 +672,7 @@ class MergeCommitTest(Base): self._run_git('merge', '--no-ff', 'test_merge_commit') self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -703,7 +704,7 @@ class MergeCommitTest(Base): self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') self._add_other_file('ignore-3.txt') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -739,7 +740,7 @@ class MergeCommitTest(Base): self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') self._add_other_file('ignore-3.txt') - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -789,7 +790,7 @@ class BranchTest(Base): f21 = self._add_notes_file('slug21') log_text = self._run_git('log') self.addDetail('git log', text_content(log_text)) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -814,7 +815,7 @@ class BranchTest(Base): self.c.override( branch='stable/2', ) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -841,7 +842,7 @@ class BranchTest(Base): self.c.override( stop_at_branch_base=False, ) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -877,7 +878,7 @@ class BranchTest(Base): branch='stable/4', collapse_pre_releases=False, ) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -913,7 +914,7 @@ class BranchTest(Base): branch='stable/4', collapse_pre_releases=True, ) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -946,7 +947,7 @@ class BranchTest(Base): self.c.override( branch='stable/4', ) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -979,7 +980,7 @@ class BranchTest(Base): self.c.override( branch='stable/4', ) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -1010,7 +1011,7 @@ class BranchTest(Base): self.c.override( branch='stable/4', ) - raw_results = scanner.get_notes_by_version(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -1183,3 +1184,33 @@ class GetTagsParseTest(base.TestCase): actual = scanner._get_version_tags_on_branch('reporoot', branch=None) self.assertEqual(self.EXPECTED, actual) + + +class VersionTest(Base): + + def setUp(self): + super(VersionTest, self).setUp() + self._make_python_package() + self.f1 = self._add_notes_file('slug1') + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.f2 = self._add_notes_file('slug2') + self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self._add_notes_file('slug3') + self._run_git('tag', '-s', '-m', 'first tag', '3.0.0') + + def test_tagged_head(self): + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_current_version(None) + self.assertEqual( + '3.0.0', + results, + ) + + def test_head_after_tag(self): + self._add_notes_file('slug4') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_current_version(None) + self.assertEqual( + '3.0.0-1', + results, + ) -- GitLab From 50900951691b9f2d869b0a4cd907b9a86cd5caac Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 5 Dec 2016 18:26:14 -0500 Subject: [PATCH 078/257] use dulwich to determine the tags on a branch Replace the git commands for determine the tags on a branch with calls to dulwich. Because the Scanner class scans for the tags when it is created and caches the data, we need to update the tests to instantiate the Scanner after the git repo is created and the test commits are applied. The main API entry point into reno is the Loader, and that is not changed because the Loader does not instantiate a Scanner until it knows it needs it. Change-Id: Icf7c7a47a768175a7cb31ab374aca4fa4b44a5cb Signed-off-by: Doug Hellmann --- reno/loader.py | 3 +- reno/scanner.py | 71 +++++++----- reno/tests/test_scanner.py | 219 ++++++++++--------------------------- requirements.txt | 1 + 4 files changed, 99 insertions(+), 195 deletions(-) diff --git a/reno/loader.py b/reno/loader.py index 5d70cdb..f11196e 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -52,7 +52,7 @@ class Loader(object): self._earliest_version = conf.earliest_version self._cache = None - self._scanner = scanner.Scanner(self._config) + self._scanner = None self._scanner_output = None self._cache_filename = get_cache_filename(self._reporoot, self._notespath) @@ -76,6 +76,7 @@ class Loader(object): for n in self._cache['notes'] } else: + self._scanner = scanner.Scanner(self._config) self._scanner_output = self._scanner.get_notes_by_version() @property diff --git a/reno/scanner.py b/reno/scanner.py index cd35291..86f3791 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -20,6 +20,9 @@ import re import subprocess import sys +from dulwich import refs +from dulwich import repo + from reno import utils LOG = logging.getLogger(__name__) @@ -78,40 +81,48 @@ PRE_RELEASE_RE = re.compile(''' ''', flags=re.VERBOSE | re.UNICODE) -def _get_version_tags_on_branch(reporoot, branch): - """Return tags from the branch, in date order. - - Need to get the list of tags in the right order, because the topo - search breaks the date ordering. Use git to ask for the tags in - order, rather than trying to sort them, because many repositories - have "non-standard" tags or have renumbered projects (from - date-based to SemVer), for which sorting would require complex - logic. - - """ - tags = [] - tag_cmd = [ - 'git', 'log', - '--simplify-by-decoration', - '--pretty="%d"', - ] - if branch: - tag_cmd.append(branch) - LOG.debug('running %s' % ' '.join(tag_cmd)) - tag_results = utils.check_output(tag_cmd, cwd=reporoot) - LOG.debug(tag_results) - for line in tag_results.splitlines(): - LOG.debug('line %r' % line) - for match in TAG_RE.findall(line): - tags.append(match) - return tags - - class Scanner(object): def __init__(self, conf): self.conf = conf self.reporoot = self.conf.reporoot + self._repo = repo.Repo(self.reporoot) + self._load_tags() + + def _load_tags(self): + self._all_tags = { + k.partition(b'/tags/')[-1].decode('utf-8'): v + for k, v in self._repo.get_refs().items() + if k.startswith(b'refs/tags/') + } + self._shas_to_tags = {} + for tag, tag_sha in self._all_tags.items(): + # The tag has its own SHA, but the tag refers to the commit and + # that's the SHA we'll see when we scan commits on a branch. + tag_obj = self._repo[tag_sha] + tagged_sha = tag_obj.object[1] + self._shas_to_tags.setdefault(tagged_sha, []).append(tag) + + def _get_tags_on_branch(self, branch): + "Return a list of tag names on the given branch." + results = [] + if branch: + branch_ref = b'refs/heads/' + branch.encode('utf-8') + if not refs.check_ref_format(branch_ref): + raise ValueError( + '{!r} does not look like a valid branch reference'.format( + branch_ref)) + branch_head = self._repo.refs[branch_ref] + else: + branch_head = self._repo.refs[b'HEAD'] + w = self._repo.get_walker(branch_head) + for c in w: + # shas_to_tags has encoded versions of the shas + # but the commit object gives us a decoded version + sha = c.commit.sha().hexdigest().encode('ascii') + if sha in self._shas_to_tags: + results.extend(self._shas_to_tags[sha]) + return results def _get_current_version(self, branch=None): """Return the current version of the repository. @@ -252,7 +263,7 @@ class Scanner(object): # order. We scan the commit history in topological order to ensure # we have the commits in the right version, so we might encounter # the tags in a different order during that phase. - versions_by_date = _get_version_tags_on_branch(reporoot, branch) + versions_by_date = self._get_tags_on_branch(branch) LOG.debug('versions by date %r' % (versions_by_date,)) versions = [] earliest_seen = collections.OrderedDict() diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 5869a20..3696364 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -19,11 +19,9 @@ import logging import os.path import re import subprocess -import textwrap import unittest import fixtures -import mock from testtools.content import text_content from reno import config @@ -197,7 +195,6 @@ class Base(base.TestCase): self.temp_dir = self.useFixture(fixtures.TempDir()).path self.reporoot = os.path.join(self.temp_dir, 'reporoot') self.c = config.Config(self.reporoot) - self.scanner = scanner.Scanner(self.c) self._git_setup() self._counter = itertools.count(1) self.get_note_num = lambda: next(self._counter) @@ -207,6 +204,7 @@ class BasicTest(Base): def test_non_python_no_tags(self): filename = self._add_notes_file() + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -220,6 +218,7 @@ class BasicTest(Base): def test_python_no_tags(self): self._make_python_package() filename = self._add_notes_file() + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -234,6 +233,7 @@ class BasicTest(Base): filename = self._add_notes_file() self._add_other_file('not-a-release-note.txt') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -247,6 +247,7 @@ class BasicTest(Base): def test_note_commit_tagged(self): filename = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -261,6 +262,7 @@ class BasicTest(Base): self._make_python_package() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') filename = self._add_notes_file() + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -276,6 +278,7 @@ class BasicTest(Base): self._add_other_file('ignore-1.txt') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') self._add_other_file('ignore-2.txt') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -291,6 +294,7 @@ class BasicTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file() f2 = self._add_notes_file() + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -306,6 +310,7 @@ class BasicTest(Base): f1 = self._add_notes_file(commit=False) f2 = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -322,6 +327,7 @@ class BasicTest(Base): f1 = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = self._add_notes_file() + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -342,6 +348,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug2') self._run_git('mv', f1, f2) self._git_commit('rename note file') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -361,6 +368,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug0') self._run_git('mv', f1, f2) self._git_commit('rename note file') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -380,6 +388,7 @@ class BasicTest(Base): with open(os.path.join(self.reporoot, f1), 'w') as f: f.write('---\npreamble: new contents for file') self._git_commit('edit note file') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -399,6 +408,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug2') self._run_git('mv', f1, f2) self._git_commit('rename note file') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -421,6 +431,7 @@ class BasicTest(Base): 'slug1-0000000000000001') self._run_git('mv', f1, f2) self._git_commit('rename note file') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -443,6 +454,7 @@ class BasicTest(Base): self.c.override( earliest_version='2.0.0', ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -463,6 +475,7 @@ class BasicTest(Base): self._run_git('rm', f1) self._git_commit('remove note file') self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -493,6 +506,7 @@ class BasicTest(Base): '--pretty=%H %d', '--name-only') self.addDetail('git log', text_content(log_results)) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -512,6 +526,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a2') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -528,6 +543,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b2') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -544,6 +560,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc2') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -569,6 +586,7 @@ class PreReleaseTest(Base): self.c.override( collapse_pre_releases=True, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -591,6 +609,7 @@ class PreReleaseTest(Base): self.c.override( collapse_pre_releases=True, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -616,6 +635,7 @@ class PreReleaseTest(Base): self.c.override( collapse_pre_releases=True, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -644,6 +664,7 @@ class MergeCommitTest(Base): self._run_git('merge', '--no-ff', 'test_merge_commit') self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -672,6 +693,7 @@ class MergeCommitTest(Base): self._run_git('merge', '--no-ff', 'test_merge_commit') self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -704,6 +726,7 @@ class MergeCommitTest(Base): self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') self._add_other_file('ignore-3.txt') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -740,6 +763,7 @@ class MergeCommitTest(Base): self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') self._add_other_file('ignore-3.txt') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -788,8 +812,9 @@ class BranchTest(Base): self._run_git('checkout', '2.0.0') self._run_git('checkout', '-b', 'stable/2') f21 = self._add_notes_file('slug21') - log_text = self._run_git('log') + log_text = self._run_git('log', '--decorate') self.addDetail('git log', text_content(log_text)) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -815,6 +840,7 @@ class BranchTest(Base): self.c.override( branch='stable/2', ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -842,6 +868,7 @@ class BranchTest(Base): self.c.override( stop_at_branch_base=False, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -878,6 +905,7 @@ class BranchTest(Base): branch='stable/4', collapse_pre_releases=False, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -914,6 +942,7 @@ class BranchTest(Base): branch='stable/4', collapse_pre_releases=True, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -947,6 +976,7 @@ class BranchTest(Base): self.c.override( branch='stable/4', ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -980,6 +1010,7 @@ class BranchTest(Base): self.c.override( branch='stable/4', ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -1011,6 +1042,7 @@ class BranchTest(Base): self.c.override( branch='stable/4', ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -1024,166 +1056,25 @@ class BranchTest(Base): ) -class GetTagsParseTest(base.TestCase): - - EXPECTED = [ - '2.0.0', - '1.8.1', - '1.8.0', - '1.7.1', - '1.7.0', - '1.6.0', - '1.5.0', - '1.4.0', - '1.3.0', - '1.2.0', - '1.1.0', - '1.0.0', - '0.11.2', - '0.11.1', - '0.11.0', - '0.10.1', - '0.10.0', - '0.9.0', - '0.8.0', - '0.7.1', - '0.7.0', - '0.6.0', - '0.5.1', - '0.5.0', - '0.4.2', - '0.4.1', - '0.4.0', - '0.3.2', - '0.3.1', - '0.3.0', - '0.2.5', - '0.2.4', - '0.2.3', - '0.2.2', - '0.2.1', - '0.2.0', - '0.1.3', - '0.1.2', - '0.1.1', - '0.1.0', - ] - - def test_keystoneclient_ubuntu_1_9_1(self): - # git 1.9.1 as it produces output on ubuntu for python-keystoneclient - # git log --simplify-by-decoration --pretty="%d" - tag_list_output = textwrap.dedent(""" - (HEAD, origin/master, origin/HEAD, gerrit/master, master) - (apu/master) - (tag: 2.0.0) - (tag: 1.8.1) - (tag: 1.8.0) - (tag: 1.7.1) - (tag: 1.7.0) - (tag: 1.6.0) - (tag: 1.5.0) - (tag: 1.4.0) - (uncap-requirements) - (tag: 1.3.0) - (tag: 1.2.0) - (tag: 1.1.0) - (tag: 1.0.0) - (tag: 0.11.2) - (tag: 0.11.1) - (tag: 0.11.0) - (tag: 0.10.1) - (tag: 0.10.0) - (tag: 0.9.0) - (tag: 0.8.0) - (tag: 0.7.1) - (tag: 0.7.0) - (tag: 0.6.0) - (tag: 0.5.1) - (tag: 0.5.0) - (tag: 0.4.2) - (tag: 0.4.1) - (tag: 0.4.0) - (tag: 0.3.2) - (tag: 0.3.1) - (tag: 0.3.0) - (tag: 0.2.5) - (tag: 0.2.4) - (tag: 0.2.3) - (tag: 0.2.2) - (tag: 0.2.1) - (tag: 0.2.0) - - (origin/feature/keystone-v3, gerrit/feature/keystone-v3) - (tag: 0.1.3) - (tag: 0.1.2) - (tag: 0.1.1) - (tag: 0.1.0) - (tag: folsom-1) - (tag: essex-rc1) - (tag: essex-4) - (tag: essex-3) - """) - with mock.patch('reno.utils.check_output') as co: - co.return_value = tag_list_output - actual = scanner._get_version_tags_on_branch('reporoot', - branch=None) - self.assertEqual(self.EXPECTED, actual) - - def test_keystoneclient_rhel_1_7_1(self): - # git 1.7.1 as it produces output on RHEL 6 for python-keystoneclient - # git log --simplify-by-decoration --pretty="%d" - tag_list_output = textwrap.dedent(""" - (HEAD, origin/master, origin/HEAD, master) - (tag: 2.0.0) - (tag: 1.8.1) - (tag: 1.8.0) - (tag: 1.7.1) - (tag: 1.7.0) - (tag: 1.6.0) - (tag: 1.5.0) - (tag: 1.4.0) - (tag: 1.3.0) - (tag: 1.2.0) - (tag: 1.1.0) - (tag: 1.0.0) - (tag: 0.11.2) - (tag: 0.11.1) - (tag: 0.11.0) - (tag: 0.10.1) - (tag: 0.10.0) - (tag: 0.9.0) - (tag: 0.8.0) - (tag: 0.7.1) - (tag: 0.7.0) - (tag: 0.6.0) - (tag: 0.5.1) - (tag: 0.5.0) - (tag: 0.4.2) - (tag: 0.4.1) - (tag: 0.4.0) - (tag: 0.3.2) - (tag: 0.3.1) - (tag: 0.3.0) - (tag: 0.2.5) - (tag: 0.2.4) - (tag: 0.2.3) - (tag: 0.2.2) - (tag: 0.2.1) - (tag: 0.2.0) - (tag: 0.1.3) - (0.1.2) - (tag: 0.1.1) - (0.1.0) - (tag: folsom-1) - (tag: essex-rc1) - (essex-4) - (essex-3) - """) - with mock.patch('reno.utils.check_output') as co: - co.return_value = tag_list_output - actual = scanner._get_version_tags_on_branch('reporoot', - branch=None) - self.assertEqual(self.EXPECTED, actual) +class TagsTest(Base): + + def setUp(self): + super(TagsTest, self).setUp() + self._make_python_package() + self.f1 = self._add_notes_file('slug1') + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.f2 = self._add_notes_file('slug2') + self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self._add_notes_file('slug3') + self._run_git('tag', '-s', '-m', 'first tag', '3.0.0') + + def test_tags_without_count(self): + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch(None) + self.assertEqual( + ['3.0.0', '2.0.0', '1.0.0'], + results, + ) class VersionTest(Base): diff --git a/requirements.txt b/requirements.txt index ba446a2..7e2b9e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ pbr<2.0,>=1.4 Babel>=1.3 PyYAML>=3.1.0 six>=1.9.0 +dulwich>=0.15.0 # Apache-2.0 -- GitLab From 6d0041cf6d8f3e907544dd6122e9fd9c46fcff9c Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 6 Dec 2016 16:19:37 -0500 Subject: [PATCH 079/257] use dulwich to find the current version on a branch Update _get_tags_on_branch to optionally count the patches past the first tag to produce a version like x.y.z-n to match what "git describe" produces. Change-Id: I41f095932a78d4a9d0462b9231d2f0b35ef7c4f2 Signed-off-by: Doug Hellmann --- reno/scanner.py | 50 ++++++++++++++++---------------------- reno/tests/test_scanner.py | 17 +++++++++++++ 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 86f3791..c72540a 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -103,9 +103,7 @@ class Scanner(object): tagged_sha = tag_obj.object[1] self._shas_to_tags.setdefault(tagged_sha, []).append(tag) - def _get_tags_on_branch(self, branch): - "Return a list of tag names on the given branch." - results = [] + def _get_walker_for_branch(self, branch): if branch: branch_ref = b'refs/heads/' + branch.encode('utf-8') if not refs.check_ref_format(branch_ref): @@ -115,39 +113,33 @@ class Scanner(object): branch_head = self._repo.refs[branch_ref] else: branch_head = self._repo.refs[b'HEAD'] - w = self._repo.get_walker(branch_head) - for c in w: + return self._repo.get_walker(branch_head) + + def _get_tags_on_branch(self, branch, with_count=False): + "Return a list of tag names on the given branch." + results = [] + count = 0 + for c in self._get_walker_for_branch(branch): # shas_to_tags has encoded versions of the shas # but the commit object gives us a decoded version sha = c.commit.sha().hexdigest().encode('ascii') if sha in self._shas_to_tags: - results.extend(self._shas_to_tags[sha]) + if with_count and count and not results: + val = '{}-{}'.format(self._shas_to_tags[sha][0], count) + results.append(val) + else: + results.extend(self._shas_to_tags[sha]) + else: + count += 1 return results def _get_current_version(self, branch=None): - """Return the current version of the repository. - - If the repo appears to contain a python project, use setup.py to - get the version so pbr (if used) can do its thing. Otherwise, use - git describe. - - """ - cmd = ['git', 'describe', '--tags'] - if branch is not None: - cmd.append(branch) - try: - result = utils.check_output(cmd, cwd=self.reporoot).strip() - if '-' in result: - # Descriptions that come after a commit look like - # 2.0.0-1-abcde, and we want to remove the SHA value from - # the end since we only care about the version number - # itself, but we need to recognize that the change is - # unreleased so keep the -1 part. - result, dash, ignore = result.rpartition('-') - except subprocess.CalledProcessError: - # This probably means there are no tags. - result = '0.0.0' - return result + "Return the current version of the repository, like git describe." + tags = self._get_tags_on_branch(branch, with_count=True) + if not tags: + # Never tagged. + return '0.0.0' + return tags[0] def _get_branch_base(self, branch): "Return the tag at base of the branch." diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 3696364..bc650ec 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1076,6 +1076,23 @@ class TagsTest(Base): results, ) + def test_tags_with_count_tagged_head(self): + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch(None, with_count=True) + self.assertEqual( + ['3.0.0', '2.0.0', '1.0.0'], + results, + ) + + def test_tags_with_count_head_after_tag(self): + self._add_notes_file('slug4') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch(None, with_count=True) + self.assertEqual( + ['3.0.0-1', '2.0.0', '1.0.0'], + results, + ) + class VersionTest(Base): -- GitLab From 871853f4b5326f2d0f72cbdbe478f245e9c03743 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 6 Dec 2016 17:41:51 -0500 Subject: [PATCH 080/257] use dulwich to get the contents of a file Implement "git show sha:filename" by traversing the tree associated with a commit to retrieve the contents of a given file at that point in the git history. Use a subclass of the dulwich Repo, because this is something we can propose upstream when we have time later. Change-Id: I6ddeb2ca3586146b4a839aa5a09d7dce225451f2 Signed-off-by: Doug Hellmann --- reno/scanner.py | 47 +++++++++++++++++++----- reno/tests/test_scanner.py | 73 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index c72540a..871c8b6 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -81,12 +81,49 @@ PRE_RELEASE_RE = re.compile(''' ''', flags=re.VERBOSE | re.UNICODE) +class RenoRepo(repo.Repo): + + def _get_file_from_tree(self, filename, tree): + "Given a tree object, traverse it to find the file." + try: + if os.sep in filename: + # The tree entry will only have a single level of the + # directory name, so if we have a / in our filename we + # know we're going to have to keep traversing the + # tree. + prefix, _, trailing = filename.partition(os.sep) + mode, subtree_sha = tree[prefix.encode('utf-8')] + subtree = self[subtree_sha] + return self._get_file_from_tree(trailing, subtree) + else: + # The tree entry will point to the blob with the + # contents of the file. + mode, file_blob_sha = tree[filename.encode('utf-8')] + file_blob = self[file_blob_sha] + return file_blob.data + except KeyError: + # Some part of the filename wasn't found, so the file is + # not present. Return the sentinel value. + return None + + def get_file_at_commit(self, filename, sha): + "Return the contents of the file if it exists at the commit, or None." + # Get the tree associated with the commit identified by the + # input SHA, then look through the items in the tree to find + # the one with the path matching the filename. Take the + # associated SHA from the tree and get the file contents from + # the repository. + commit = self[sha.encode('ascii')] + tree = self[commit.tree] + return self._get_file_from_tree(filename, tree) + + class Scanner(object): def __init__(self, conf): self.conf = conf self.reporoot = self.conf.reporoot - self._repo = repo.Repo(self.reporoot) + self._repo = RenoRepo(self.reporoot) self._load_tags() def _load_tags(self): @@ -200,13 +237,7 @@ class Scanner(object): def get_file_at_commit(self, filename, sha): "Return the contents of the file if it exists at the commit, or None." - try: - return utils.check_output( - ['git', 'show', '%s:%s' % (sha, filename)], - cwd=self.reporoot, - ) - except subprocess.CalledProcessError: - return None + return self._repo.get_file_at_commit(filename, sha) def _file_exists_at_commit(self, filename, sha): "Return true if the file exists at the given commit." diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index bc650ec..bfb6651 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -151,7 +151,8 @@ class Base(base.TestCase): f.write('adding %s\n' % name) self._git_commit('add %s' % name) - def _add_notes_file(self, slug='slug', commit=True, legacy=False): + def _add_notes_file(self, slug='slug', commit=True, legacy=False, + contents='i-am-also-a-template'): n = self.get_note_num() if legacy: basename = '%016x-%s.yaml' % (n, slug) @@ -159,7 +160,7 @@ class Base(base.TestCase): basename = '%s-%016x.yaml' % (slug, n) filename = os.path.join(self.reporoot, 'releasenotes', 'notes', basename) - create._make_note_file(filename, 'i-am-also-a-template') + create._make_note_file(filename, contents) self._git_commit('add %s' % basename) return os.path.join('releasenotes', 'notes', basename) @@ -519,6 +520,74 @@ class BasicTest(Base): ) +class FileContentsTest(Base): + + def test_basic_file(self): + # Prove that we can get a file we have committed. + f1 = self._add_notes_file(contents='well-known-contents') + r = scanner.RenoRepo(self.reporoot) + contents = r.get_file_at_commit(f1, 'HEAD') + self.assertEqual( + b'well-known-contents', + contents, + ) + + def test_no_such_file(self): + # Returns None when the file does not exist at all. + # (we have to commit something, otherwise there is no HEAD) + self._add_notes_file(contents='well-known-contents') + r = scanner.RenoRepo(self.reporoot) + contents = r.get_file_at_commit('no-such-dir/no-such-file', 'HEAD') + self.assertEqual( + None, + contents, + ) + + def test_edit_file_and_commit(self): + # Prove that we can edit a file and see the changes. + f1 = self._add_notes_file(contents='initial-contents') + with open(os.path.join(self.reporoot, f1), 'w') as f: + f.write('new contents for file') + self._git_commit('edit note file') + r = scanner.RenoRepo(self.reporoot) + contents = r.get_file_at_commit(f1, 'HEAD') + self.assertEqual( + b'new contents for file', + contents, + ) + + def test_earlier_version_of_edited_file(self): + # Prove that we are not always just returning the most current + # version of a file. + f1 = self._add_notes_file(contents='initial-contents') + with open(os.path.join(self.reporoot, f1), 'w') as f: + f.write('new contents for file') + self._git_commit('edit note file') + self.scanner = scanner.Scanner(self.c) + r = scanner.RenoRepo(self.reporoot) + head = r.head() + parent = r.get_parents(head)[0] + parent = parent.decode('ascii') + contents = r.get_file_at_commit(f1, parent) + self.assertEqual( + b'initial-contents', + contents, + ) + + def test_edit_file_without_commit(self): + # Prove we are not picking up the contents from the local + # filesystem outside of the git history. + f1 = self._add_notes_file(contents='initial-contents') + with open(os.path.join(self.reporoot, f1), 'w') as f: + f.write('new contents for file') + r = scanner.RenoRepo(self.reporoot) + contents = r.get_file_at_commit(f1, 'HEAD') + self.assertEqual( + b'initial-contents', + contents, + ) + + class PreReleaseTest(Base): def test_alpha(self): -- GitLab From 7bb2f0b794b336dfd31b7164336a8c07cfea36ce Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 7 Dec 2016 11:43:53 -0500 Subject: [PATCH 081/257] move tag management into repo subclass Rather than having the Scanner manage the set of tags in the repo, add logic to the Repo class for doing it. This can eventually move upstream. Change-Id: I5c5f82380281d8c220901c541c25cdd8d1a7998d Signed-off-by: Doug Hellmann --- reno/scanner.py | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 871c8b6..e131df7 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -83,6 +83,30 @@ PRE_RELEASE_RE = re.compile(''' class RenoRepo(repo.Repo): + # Populated by _load_tags(). + _all_tags = None + _shas_to_tags = None + + def _load_tags(self): + self._all_tags = { + k.partition(b'/tags/')[-1].decode('utf-8'): v + for k, v in self.get_refs().items() + if k.startswith(b'refs/tags/') + } + self._shas_to_tags = {} + for tag, tag_sha in self._all_tags.items(): + # The tag has its own SHA, but the tag refers to the commit and + # that's the SHA we'll see when we scan commits on a branch. + tag_obj = self[tag_sha] + tagged_sha = tag_obj.object[1] + self._shas_to_tags.setdefault(tagged_sha, []).append(tag) + + def get_tags_on_commit(self, sha): + "Return the tag(s) on a commit." + if self._all_tags is None: + self._load_tags() + return self._shas_to_tags.get(sha) + def _get_file_from_tree(self, filename, tree): "Given a tree object, traverse it to find the file." try: @@ -124,21 +148,6 @@ class Scanner(object): self.conf = conf self.reporoot = self.conf.reporoot self._repo = RenoRepo(self.reporoot) - self._load_tags() - - def _load_tags(self): - self._all_tags = { - k.partition(b'/tags/')[-1].decode('utf-8'): v - for k, v in self._repo.get_refs().items() - if k.startswith(b'refs/tags/') - } - self._shas_to_tags = {} - for tag, tag_sha in self._all_tags.items(): - # The tag has its own SHA, but the tag refers to the commit and - # that's the SHA we'll see when we scan commits on a branch. - tag_obj = self._repo[tag_sha] - tagged_sha = tag_obj.object[1] - self._shas_to_tags.setdefault(tagged_sha, []).append(tag) def _get_walker_for_branch(self, branch): if branch: @@ -160,12 +169,13 @@ class Scanner(object): # shas_to_tags has encoded versions of the shas # but the commit object gives us a decoded version sha = c.commit.sha().hexdigest().encode('ascii') - if sha in self._shas_to_tags: + tags = self._repo.get_tags_on_commit(sha) + if tags: if with_count and count and not results: - val = '{}-{}'.format(self._shas_to_tags[sha][0], count) + val = '{}-{}'.format(tags[0], count) results.append(val) else: - results.extend(self._shas_to_tags[sha]) + results.extend(tags) else: count += 1 return results -- GitLab From b1abbc988994ab2e20f7481f55eb37f8bd49a168 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 7 Dec 2016 11:48:01 -0500 Subject: [PATCH 082/257] add tests for determining the branch base Add a set of unit tests for the method that finds the tags on the commit at the base of a branch. Change-Id: Iff3a8677510010d5aaf600cfa865a2b6924db7b7 Signed-off-by: Doug Hellmann --- reno/tests/test_scanner.py | 52 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index bfb6651..4b0fe30 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -865,6 +865,58 @@ class UniqueIdTest(Base): self.assertEqual('0000000000000001', uid) +class BranchBaseTest(Base): + + def setUp(self): + super(BranchBaseTest, self).setUp() + self._make_python_package() + self._add_notes_file('slug1') + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self._add_notes_file('slug2') + self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self._add_notes_file('slug3') + self._run_git('tag', '-s', '-m', 'first tag', '3.0.0') + self._run_git('checkout', '2.0.0') + self._run_git('branch', 'not-master') + self._run_git('checkout', 'master') + self.scanner = scanner.Scanner(self.c) + + def test_current_branch_no_extra_commits(self): + # checkout the branch and then ask for its base + self._run_git('checkout', 'not-master') + self.assertEqual( + '2.0.0', + self.scanner._get_branch_base('not-master'), + ) + + def test_current_branch_extra_commit(self): + # checkout the branch and then ask for its base + self._run_git('checkout', 'not-master') + self._add_notes_file('slug4') + self.assertEqual( + '2.0.0', + self.scanner._get_branch_base('not-master'), + ) + + def test_alternate_branch_no_extra_commits(self): + # checkout master and then ask for the alternate branch base + self._run_git('checkout', 'master') + self.assertEqual( + '2.0.0', + self.scanner._get_branch_base('not-master'), + ) + + def test_alternate_branch_extra_commit(self): + # checkout master and then ask for the alternate branch base + self._run_git('checkout', 'not-master') + self._add_notes_file('slug4') + self._run_git('checkout', 'master') + self.assertEqual( + '2.0.0', + self.scanner._get_branch_base('not-master'), + ) + + class BranchTest(Base): def setUp(self): -- GitLab From e01d992a9f202f31b54420ebba97e43a95b2c437 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 7 Dec 2016 11:48:27 -0500 Subject: [PATCH 083/257] use dulwich to determine the branch base Scan the repository ourselves to find the intersection of the two branches. Change-Id: I3ad7bc8e51518f23b9bcc5d0e745c219a2cb5838 Signed-off-by: Doug Hellmann --- reno/scanner.py | 65 ++++++++++++------------------------------------- 1 file changed, 15 insertions(+), 50 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index e131df7..e2ae2a3 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -17,7 +17,6 @@ import fnmatch import logging import os.path import re -import subprocess import sys from dulwich import refs @@ -195,55 +194,21 @@ class Scanner(object): # git rev-list $(git rev-list --first-parent \ # ^origin/stable/newton master | tail -n1)^^! # - # Determine the list of commits accessible from the branch we are - # supposed to be scanning, but not on master. - cmd = [ - 'git', - 'rev-list', - '--first-parent', - branch, # on the branch - '^master', # not on master - ] - try: - LOG.debug(' '.join(cmd)) - parents = utils.check_output(cmd, cwd=self.reporoot).strip() - if not parents: - # There are no commits on the branch, yet, so we can use - # our current-version logic. - return self._get_current_version(branch) - except subprocess.CalledProcessError as e: - LOG.warning('failed to retrieve branch base: %s [%s]', - e, e.output.strip()) - return None - parent = parents.splitlines()[-1] - LOG.debug('parent = %r', parent) - # Now get the previous commit, which should be the one we tagged - # to create the branch. - cmd = [ - 'git', - 'rev-list', - '{}^^!'.format(parent), - ] - try: - sha = utils.check_output(cmd, cwd=self.reporoot).strip() - LOG.debug('sha = %r', sha) - except subprocess.CalledProcessError as e: - LOG.warning('failed to retrieve branch base: %s [%s]', - e, e.output.strip()) - return None - # Now get the tag for that commit. - cmd = [ - 'git', - 'describe', - '--abbrev=0', - sha, - ] - try: - return utils.check_output(cmd, cwd=self.reporoot).strip() - except subprocess.CalledProcessError as e: - LOG.warning('failed to retrieve branch base: %s [%s]', - e, e.output.strip()) - return None + # Build the set of all commits that appear on the master + # branch, then scan the commits that appear on the specified + # branch until we find something that is on both. + master_commits = set( + c.commit.sha().hexdigest() + for c in self._get_walker_for_branch('master') + ) + for c in self._get_walker_for_branch(branch): + if c.commit.sha().hexdigest() in master_commits: + # We got to this commit via the branch, but it is also + # on master, so this is the base. + tags = self._repo.get_tags_on_commit( + c.commit.sha().hexdigest().encode('ascii')) + return tags[0] + return None def get_file_at_commit(self, filename, sha): "Return the contents of the file if it exists at the commit, or None." -- GitLab From bb215519076838fbc89178b472b6ea18cbe0bb9c Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 7 Dec 2016 11:58:28 -0500 Subject: [PATCH 084/257] optimize check for the current version Rewrite the logic for finding the current version to stop as soon as the first tag is encountered. Change-Id: Iaa776a126daa2bfe7ec69e990b47e626d194bcee Signed-off-by: Doug Hellmann --- reno/scanner.py | 53 ++++++++++++++++++++++++-------------- reno/tests/test_scanner.py | 20 ++++++-------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index e2ae2a3..be0a23f 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -104,7 +104,7 @@ class RenoRepo(repo.Repo): "Return the tag(s) on a commit." if self._all_tags is None: self._load_tags() - return self._shas_to_tags.get(sha) + return self._shas_to_tags.get(sha, []) def _get_file_from_tree(self, filename, tree): "Given a tree object, traverse it to find the file." @@ -148,44 +148,57 @@ class Scanner(object): self.reporoot = self.conf.reporoot self._repo = RenoRepo(self.reporoot) - def _get_walker_for_branch(self, branch): + def _get_branch_head(self, branch): if branch: branch_ref = b'refs/heads/' + branch.encode('utf-8') if not refs.check_ref_format(branch_ref): raise ValueError( '{!r} does not look like a valid branch reference'.format( branch_ref)) - branch_head = self._repo.refs[branch_ref] - else: - branch_head = self._repo.refs[b'HEAD'] + return self._repo.refs[branch_ref] + return self._repo.refs[b'HEAD'] + + def _get_walker_for_branch(self, branch): + branch_head = self._get_branch_head(branch) return self._repo.get_walker(branch_head) - def _get_tags_on_branch(self, branch, with_count=False): + def _get_tags_on_branch(self, branch): "Return a list of tag names on the given branch." results = [] - count = 0 for c in self._get_walker_for_branch(branch): # shas_to_tags has encoded versions of the shas # but the commit object gives us a decoded version sha = c.commit.sha().hexdigest().encode('ascii') tags = self._repo.get_tags_on_commit(sha) - if tags: - if with_count and count and not results: - val = '{}-{}'.format(tags[0], count) - results.append(val) - else: - results.extend(tags) - else: - count += 1 + results.extend(tags) return results def _get_current_version(self, branch=None): "Return the current version of the repository, like git describe." - tags = self._get_tags_on_branch(branch, with_count=True) - if not tags: - # Never tagged. - return '0.0.0' - return tags[0] + # This is similar to _get_tags_on_branch() except that it + # counts up to where the tag appears and it returns when it + # finds the first tagged commit (there is no need to scan the + # rest of the branch). + commit = self._repo[self._get_branch_head(branch)] + count = 0 + while commit: + # shas_to_tags has encoded versions of the shas + # but the commit object gives us a decoded version + sha = commit.sha().hexdigest().encode('ascii') + tags = self._repo.get_tags_on_commit(sha) + if tags: + if count: + val = '{}-{}'.format(tags[-1], count) + else: + val = tags[0] + return val + if commit.parents: + # Only traverse the first parent of each node. + commit = self._repo[commit.parents[0]] + count += 1 + else: + commit = None + return '0.0.0' def _get_branch_base(self, branch): "Return the tag at base of the branch." diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 4b0fe30..6ce17ae 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1189,7 +1189,7 @@ class TagsTest(Base): self._add_notes_file('slug3') self._run_git('tag', '-s', '-m', 'first tag', '3.0.0') - def test_tags_without_count(self): + def test_master(self): self.scanner = scanner.Scanner(self.c) results = self.scanner._get_tags_on_branch(None) self.assertEqual( @@ -1197,20 +1197,16 @@ class TagsTest(Base): results, ) - def test_tags_with_count_tagged_head(self): - self.scanner = scanner.Scanner(self.c) - results = self.scanner._get_tags_on_branch(None, with_count=True) - self.assertEqual( - ['3.0.0', '2.0.0', '1.0.0'], - results, - ) - - def test_tags_with_count_head_after_tag(self): + def test_not_master(self): + self._run_git('checkout', '2.0.0') + self._run_git('checkout', '-b', 'not-master') self._add_notes_file('slug4') + self._run_git('tag', '-s', '-m', 'not on master', '2.0.1') + self._run_git('checkout', 'master') self.scanner = scanner.Scanner(self.c) - results = self.scanner._get_tags_on_branch(None, with_count=True) + results = self.scanner._get_tags_on_branch('not-master') self.assertEqual( - ['3.0.0-1', '2.0.0', '1.0.0'], + ['2.0.1', '2.0.0', '1.0.0'], results, ) -- GitLab From 85a3fe990806c4f0e3cf7a3ada79706e0adc39b3 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 7 Dec 2016 13:57:13 -0500 Subject: [PATCH 085/257] ensure tags are returned in a consistent order We need to figure out the most current tag applied in some cases, so always sort the tags returned from get_tags_on_commit() to appear in date order. Update _get_current_version() to use the most recently applied tag, as git-describe does. Change-Id: Ibfc3e11f2b23167e8a06f3615e5722be3b267c8f Signed-off-by: Doug Hellmann --- reno/scanner.py | 12 ++++++++---- reno/tests/test_scanner.py | 19 +++++++++++++++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index be0a23f..4600b7c 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -101,10 +101,14 @@ class RenoRepo(repo.Repo): self._shas_to_tags.setdefault(tagged_sha, []).append(tag) def get_tags_on_commit(self, sha): - "Return the tag(s) on a commit." + "Return the tag(s) on a commit, in application order." if self._all_tags is None: self._load_tags() - return self._shas_to_tags.get(sha, []) + tags_on_commit = self._shas_to_tags.get(sha, []) + tags_on_commit.sort( + key=lambda x: self[self._all_tags[x]].tag_time + ) + return tags_on_commit def _get_file_from_tree(self, filename, tree): "Given a tree object, traverse it to find the file." @@ -190,7 +194,7 @@ class Scanner(object): if count: val = '{}-{}'.format(tags[-1], count) else: - val = tags[0] + val = tags[-1] return val if commit.parents: # Only traverse the first parent of each node. @@ -220,7 +224,7 @@ class Scanner(object): # on master, so this is the base. tags = self._repo.get_tags_on_commit( c.commit.sha().hexdigest().encode('ascii')) - return tags[0] + return tags[-1] return None def get_file_at_commit(self, filename, sha): diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 6ce17ae..6029cf6 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -19,6 +19,7 @@ import logging import os.path import re import subprocess +import time import unittest import fixtures @@ -1219,9 +1220,9 @@ class VersionTest(Base): self.f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') self.f2 = self._add_notes_file('slug2') - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') self._add_notes_file('slug3') - self._run_git('tag', '-s', '-m', 'first tag', '3.0.0') + self._run_git('tag', '-s', '-m', 'third tag', '3.0.0') def test_tagged_head(self): self.scanner = scanner.Scanner(self.c) @@ -1239,3 +1240,17 @@ class VersionTest(Base): '3.0.0-1', results, ) + + def test_multiple_tags(self): + # The timestamp resolution appears to be 1 second, so sleep to + # ensure distinct timestamps for the 2 tags. In practice it is + # unlikely that anything could apply 2 signed tags within a + # single second (certainly not a person). + time.sleep(1) + self._run_git('tag', '-s', '-m', 'fourth tag', '4.0.0') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_current_version(None) + self.assertEqual( + '4.0.0', + results, + ) -- GitLab From 6f7d62e3a64e181cb0c1892fdc3993374b2dd215 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 12 Dec 2016 11:44:00 -0500 Subject: [PATCH 086/257] add function for reducing change list to operations Change-Id: I669f1f47bcd4e03629c5bfb1986d9f0427c649ee Signed-off-by: Doug Hellmann --- reno/scanner.py | 105 +++++++++++++++++++ reno/tests/test_scanner.py | 209 +++++++++++++++++++++++++++++++++++++ 2 files changed, 314 insertions(+) diff --git a/reno/scanner.py b/reno/scanner.py index 4600b7c..7aeaaa5 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -19,6 +19,7 @@ import os.path import re import sys +from dulwich import diff_tree from dulwich import refs from dulwich import repo @@ -80,6 +81,110 @@ PRE_RELEASE_RE = re.compile(''' ''', flags=re.VERBOSE | re.UNICODE) +def _note_file(notesdir, name): + """Return bool indicating if the filename looks like a note file. + + This is used to filter the files in changes based on the notes + directory we were given. We cannot do this in the walker directly + because it means we end up skipping some of the tags if the + commits being tagged don't include any release note files. + + """ + if not name: + return False + if fnmatch.fnmatch(name, notesdir + '/*.yaml'): + return True + elif fnmatch.fnmatch(name, notesdir + '/*'): + LOG.warning('found and ignored extra file %s', name) + return False + + +def _aggregate_changes(walk_entry, notesdir): + """Collapse a WalkEntry based on our notion of uniqueness for file uids. + + Each WalkEntry contains a list of TreeChange instances which + describe changes between the old and new repository trees. The + change has a type, and new and old paths and shas. + + Simple add, delete, and change operations are handled directly. + + There is a rename type, but detection of renamed files is + incomplete so we handle that ourselves based on the UID value + built into the filenames (under the assumption that if someone + changes that part of the filename they want it treated as a + different file for some reason). If we see both an add and a + delete for a given UID treat that as a rename. + + The SHA values returned are for the commit, rather than the blob + values in the TreeChange objects. + + The path values in the change entries are encoded, so we decode + them to compare them against the notesdir and file pattern in + _note_file() and then return the decoded values to make consuming + them easier. + + """ + by_uid = collections.defaultdict(list) + sha = walk_entry.commit.id + LOG.debug('entry for commit %s', sha) + for ec in walk_entry.changes(): + LOG.debug('change %r', ec) + if not isinstance(ec, list): + ec = [ec] + else: + ec = ec + for c in ec: + if c.type == diff_tree.CHANGE_ADD: + path = c.new.path.decode('utf-8') if c.new.path else None + if _note_file(notesdir, path): + uid = _get_unique_id(path) + by_uid[uid].append((c.type, path, sha)) + else: + LOG.debug('ignoring') + elif c.type == diff_tree.CHANGE_DELETE: + path = c.old.path.decode('utf-8') if c.old.path else None + if _note_file(notesdir, path): + uid = _get_unique_id(path) + by_uid[uid].append((c.type, path)) + else: + LOG.debug('ignoring') + elif c.type == diff_tree.CHANGE_MODIFY: + path = c.new.path.decode('utf-8') if c.new.path else None + if _note_file(notesdir, path): + uid = _get_unique_id(path) + by_uid[uid].append((c.type, path, sha)) + else: + LOG.debug('ignoring') + else: + raise ValueError('unhandled change type: {!r}'.format(c)) + + results = [] + for uid, changes in sorted(by_uid.items()): + if len(changes) == 1: + results.append((uid,) + changes[0]) + else: + types = set(c[0] for c in changes) + if types == set([diff_tree.CHANGE_ADD, diff_tree.CHANGE_DELETE]): + # A rename, combine the data from the add and delete entries. + added = [ + c for c in changes if c[0] == diff_tree.CHANGE_ADD + ][0] + deled = [ + c for c in changes if c[0] == diff_tree.CHANGE_DELETE + ][0] + results.append( + (uid, diff_tree.CHANGE_RENAME, deled[1]) + added[1:] + ) + elif types == set([diff_tree.CHANGE_MODIFY]): + # Merge commit with modifications to the same files in + # different commits. + for c in changes: + results.append((uid, diff_tree.CHANGE_MODIFY, c[1], sha)) + else: + raise ValueError('Unrecognized changes: {!r}'.format(changes)) + return results + + class RenoRepo(repo.Repo): # Populated by _load_tags(). diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 6029cf6..4108169 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -22,7 +22,10 @@ import subprocess import time import unittest +from dulwich import diff_tree +from dulwich import objects import fixtures +import mock from testtools.content import text_content from reno import config @@ -1254,3 +1257,209 @@ class VersionTest(Base): '4.0.0', results, ) + + +class AggregateChangesTest(Base): + + def test_ignore(self): + entry = mock.Mock() + n = self.get_note_num() + name = 'prefix/add-%016x' % n # no .yaml extension + entry.commit.id = 'commit-id' + entry.changes.return_value = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ) + ) + ] + results = scanner._aggregate_changes(entry, 'prefix') + self.assertEqual( + [], + results, + ) + + def test_add(self): + entry = mock.Mock() + n = self.get_note_num() + name = 'prefix/add-%016x.yaml' % n + entry.commit.id = 'commit-id' + entry.changes.return_value = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ) + ) + ] + results = list(scanner._aggregate_changes(entry, 'prefix')) + self.assertEqual( + [('%016x' % n, 'add', name, 'commit-id')], + results, + ) + + def test_delete(self): + entry = mock.Mock() + n = self.get_note_num() + name = 'prefix/delete-%016x.yaml' % n + entry.commit.id = 'commit-id' + entry.changes.return_value = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_DELETE, + old=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ), + new=objects.TreeEntry(path=None, mode=None, sha=None) + ) + ] + results = list(scanner._aggregate_changes(entry, 'prefix')) + self.assertEqual( + [('%016x' % n, 'delete', name)], + results, + ) + + def test_change(self): + entry = mock.Mock() + n = self.get_note_num() + name = 'prefix/change-%016x.yaml' % n + entry.commit.id = 'commit-id' + entry.changes.return_value = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_MODIFY, + old=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='old-sha', + ), + new=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='new-sha', + ), + ) + ] + results = list(scanner._aggregate_changes(entry, 'prefix')) + self.assertEqual( + [('%016x' % n, 'modify', name, 'commit-id')], + results, + ) + + def test_add_then_delete(self): + entry = mock.Mock() + n = self.get_note_num() + new_name = 'prefix/new-%016x.yaml' % n + old_name = 'prefix/old-%016x.yaml' % n + entry.commit.id = 'commit-id' + entry.changes.return_value = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=new_name.encode('utf-8'), + mode='0222', + sha='new-hash', + ) + ), + diff_tree.TreeChange( + type=diff_tree.CHANGE_DELETE, + old=objects.TreeEntry( + path=old_name.encode('utf-8'), + mode='0222', + sha='old-hash', + ), + new=objects.TreeEntry(path=None, mode=None, sha=None) + ) + ] + results = list(scanner._aggregate_changes(entry, 'prefix')) + self.assertEqual( + [('%016x' % n, 'rename', old_name, new_name, 'commit-id')], + results, + ) + + def test_delete_then_add(self): + entry = mock.Mock() + n = self.get_note_num() + new_name = 'prefix/new-%016x.yaml' % n + old_name = 'prefix/old-%016x.yaml' % n + entry.commit.id = 'commit-id' + entry.changes.return_value = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_DELETE, + old=objects.TreeEntry( + path=old_name.encode('utf-8'), + mode='0222', + sha='old-hash', + ), + new=objects.TreeEntry(path=None, mode=None, sha=None) + ), + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=new_name.encode('utf-8'), + mode='0222', + sha='new-hash', + ) + ), + ] + results = list(scanner._aggregate_changes(entry, 'prefix')) + self.assertEqual( + [('%016x' % n, 'rename', old_name, new_name, 'commit-id')], + results, + ) + + def test_tree_changes(self): + # Under some conditions when dulwich sees merge commits, + # changes() returns a list with nested lists. See commit + # cc11da6dcfb1dbaa015e9804b6a23f7872380c1b in this repo for an + # example. + entry = mock.Mock() + n = self.get_note_num() + # The files modified by the commit are actually + # reno/scanner.py, but the fake names are used in this test to + # comply with the rest of the configuration for the scanner. + old_name = 'prefix/old-%016x.yaml' % n + entry.commit.id = 'commit-id' + entry.changes.return_value = [[ + diff_tree.TreeChange( + type='modify', + old=diff_tree.TreeEntry( + path=old_name.encode('utf-8'), + mode=33188, + sha=b'8247dfdd116fd0e3cc4ba32328e4a3eafd227de6', + ), + new=diff_tree.TreeEntry( + path=old_name.encode('utf-8'), + mode=33188, + sha=b'611f3663f54afb1f018a6a8680b6488da50ac340', + ), + ), + diff_tree.TreeChange( + type='modify', + old=diff_tree.TreeEntry( + path=old_name.encode('utf-8'), + mode=33188, + sha=b'ecb7788066eefa9dc8f110b56360efe7b1140b84', + ), + new=diff_tree.TreeEntry( + path=old_name.encode('utf-8'), + mode=33188, + sha=b'611f3663f54afb1f018a6a8680b6488da50ac340', + ), + ), + ]] + results = list(scanner._aggregate_changes(entry, 'prefix')) + self.assertEqual( + [('%016x' % n, 'modify', old_name, 'commit-id'), + ('%016x' % n, 'modify', old_name, 'commit-id')], + results, + ) -- GitLab From 4bb6d478ac49cab464b49756e3db276564778295 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 9 Dec 2016 17:09:50 -0500 Subject: [PATCH 087/257] use dulwich to implement get_notes_by_version Replace the use of git-log with a traversal of the dulwich history. The topo ordering in dulwich does not match the git command line output, so we have our own in _topo_traversal(). I'll try to upstream that separately. Use the _aggregate_changes() method from the previous commit to turn a given commit into the set of changes to make to the note history. The topo sort in dulwich relied too much on the date ordering, which means the entries were sometimes produced in the right order and sometimes the wrong order. The issue is better exposed when the commits have more significant time differences, so add a sleep to the tests after committing and after merging to ensure that repo changes have distinct timestamps. Remove the skip from the test that exposed the issue with the git porcelain output format change to show that the new implementation passes the test. Since we're no longer scanning the git log history, we no longer need the TAG_RE regex. Change-Id: Ibfa1249f9d56a9e27b5a9f435ac8033f4bae24b6 Signed-off-by: Doug Hellmann --- reno/scanner.py | 338 +++++++++++++++++++++++-------------- reno/tests/test_scanner.py | 15 +- 2 files changed, 216 insertions(+), 137 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 7aeaaa5..00cf2d0 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -23,10 +23,13 @@ from dulwich import diff_tree from dulwich import refs from dulwich import repo -from reno import utils - LOG = logging.getLogger(__name__) +# What does a pre-release version number look like? +PRE_RELEASE_RE = re.compile(''' + \.(\d+(?:[ab]|rc)+\d*)$ +''', flags=re.VERBOSE | re.UNICODE) + def _get_unique_id(filename): base = os.path.basename(filename) @@ -39,48 +42,6 @@ def _get_unique_id(filename): return uniqueid -# The git log output from _get_tags_on_branch() looks like this sample -# from the openstack/nova repository for git 1.9.1: -# -# git log --simplify-by-decoration --pretty="%d" -# (HEAD, origin/master, origin/HEAD, gerrit/master, master) -# (apu/master) -# (tag: 13.0.0.0b1) -# (tag: 12.0.0.0rc3, tag: 12.0.0) -# -# And this for git 1.7.1 (RHEL): -# -# $ git log --simplify-by-decoration --pretty="%d" -# (HEAD, origin/master, origin/HEAD, master) -# (tag: 13.0.0.0b1) -# (tag: 12.0.0.0rc3, tag: 12.0.0) -# (tag: 12.0.0.0rc2) -# (tag: 2015.1.0rc3, tag: 2015.1.0) -# ... -# (tag: folsom-2) -# (tag: folsom-1) -# (essex-1) -# (diablo-2) -# (diablo-1) -# (2011.2) -# -# The difference in the tags with "tag:" and without appears to be -# caused by some being annotated and others not. -# -# So we might have multiple tags on a given commit, and we might have -# lines that have no tags or are completely blank, and we might have -# "tag:" or not. This pattern is used to find the tag entries on each -# line, ignoring tags that don't look like version numbers. -TAG_RE = re.compile(''' - (?:[(]|tag:\s) # look for tag: prefix and drop - ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and pre-releases - [,)] # possible trailing comma or closing paren -''', flags=re.VERBOSE | re.UNICODE) -PRE_RELEASE_RE = re.compile(''' - \.(\d+(?:[ab]|rc)+\d*)$ -''', flags=re.VERBOSE | re.UNICODE) - - def _note_file(notesdir, name): """Return bool indicating if the filename looks like a note file. @@ -332,6 +293,99 @@ class Scanner(object): return tags[-1] return None + def _topo_traversal(self, branch): + """Generator that yields the branch entries in topological order. + + The topo ordering in dulwich does not match the git command line + output, so we have our own that follows the branch being merged + into the mainline before following the mainline. This ensures that + tags on the mainline appear in the right place relative to the + merge points, regardless of the commit date on the entry. + + # * d1239b6 (HEAD -> master) Merge branch 'new-branch' + # |\ + # | * 9478612 (new-branch) one commit on branch + # * | 303e21d second commit on master + # * | 0ba5186 first commit on master + # |/ + # * a7f573d original commit on master + + """ + head = self._get_branch_head(branch) + + # Map SHA values to Entry objects, because we will be traversing + # commits not entries. + all = {} + + children = {} + + # Populate all and children structures by traversing the + # entire graph once. It doesn't matter what order we do this + # the first time, since we're just recording the relationships + # of the nodes. + for e in self._repo.get_walker(head): + all[e.commit.id] = e + for p in e.commit.parents: + children.setdefault(p, set()).add(e.commit.id) + + # Track what we have already emitted. + emitted = set() + + # Use a deque as a stack with the nodes left to process. This + # lets us avoid recursion, since we have no idea how deep some + # branches might be. + todo = collections.deque() + todo.appendleft(head) + + while todo: + sha = todo.popleft() + entry = all[sha] + + # If a node has multiple children, it is the start point + # for a branch that was merged back into the rest of the + # tree. We will have already processed the merge commit + # and are traversing either the branch that was merged in + # or the base into which it was merged. We want to stop + # traversing the branch that was merged in at the point + # where the branch was created, because we are trying to + # linearize the history. At that point, we go back to the + # merge node and take the other parent node, which should + # lead us back to the origin of the branch through the + # mainline. + unprocessed_children = [ + c + for c in children.get(sha, set()) + if c not in emitted + ] + + if not unprocessed_children: + # All children have been processed. Remember that we have + # processed this node and then emit the entry. + emitted.add(sha) + yield entry + + # Now put the parents on the stack from left to right + # so they are processed right to left. If the node is + # already on the stack, leave it to be processed in + # the original order where it was added. + # + # NOTE(dhellmann): It's not clear if this is the right + # solution, or if we should re-stack and then ignore + # duplicate emissions at the top of this + # loop. Checking if the item is already on the todo + # stack isn't very expensive, since we don't expect it + # to grow very large, but it's not clear the output + # will be produced in the right order. + for p in entry.commit.parents: + if p not in todo: + todo.appendleft(p) + + else: + # Has unprocessed children. Do not emit, and do not + # restack, since when we get to the other child they will + # stack it. + pass + def get_file_at_commit(self, filename, sha): "Return the contents of the file if it exists at the commit, or None." return self._repo.get_file_at_commit(filename, sha) @@ -399,114 +453,139 @@ class Scanner(object): if current_version not in versions_by_date: LOG.debug('adding %s to versions by date' % current_version) versions_by_date.insert(0, current_version) + LOG.debug('versions by date %r' % (versions_by_date,)) # Remember the most current filename for each id, to allow for # renames. last_name_by_id = {} # Remember uniqueids that have had files deleted. - uniqueids_deleted = collections.defaultdict(set) - - # FIXME(dhellmann): This might need to be more line-oriented for - # longer histories. - log_cmd = [ - 'git', 'log', - '--topo-order', # force traversal order rather than date order - '--pretty=%x00%H %d', # output contents in parsable format - '--name-only' # only include the names of the files in the patch - ] - if branch is not None: - log_cmd.append(branch) - LOG.debug('running %s' % ' '.join(log_cmd)) - history_results = utils.check_output(log_cmd, cwd=reporoot) - history = history_results.split('\x00') - current_version = current_version - for h in history: - h = h.strip() - if not h: - continue - # print(h) - - hlines = h.splitlines() - - # The first line of the block will include the SHA and may - # include tags, the other lines are filenames. - sha = hlines[0].split(' ')[0] - tags = TAG_RE.findall(hlines[0]) - # Filter the files based on the notes directory we were - # given. We cannot do this in the git log command directly - # because it means we end up skipping some of the tags if the - # commits being tagged don't include any release note - # files. Even if this list ends up empty, we continue doing - # the other processing so that we record all of the known - # versions. - filenames = [] - for f in hlines[2:]: - if fnmatch.fnmatch(f, notesdir + '/*.yaml'): - filenames.append(f) - elif fnmatch.fnmatch(f, notesdir + '/*'): - LOG.warning('found and ignored extra file %s', f) + uniqueids_deleted = set() + + for entry in self._topo_traversal(branch): + + sha = entry.commit.id + tags_on_commit = self._repo.get_tags_on_commit(sha) + LOG.debug('%s encountered', sha) # If there are no tags in this block, assume the most recently # seen version. + tags = tags_on_commit if not tags: tags = [current_version] else: - current_version = tags[0] - LOG.debug('%s has tags %s (%r), ' + current_version = tags_on_commit[-1] + LOG.debug('%s has tags %s, ' 'updating current version to %s' % - (sha, tags, hlines[0], current_version)) + (sha, tags_on_commit, current_version)) # Remember each version we have seen. if current_version not in versions: LOG.debug('%s is a new version' % current_version) versions.append(current_version) - LOG.debug('%s contains files %s' % (sha, filenames)) + # Look for changes to notes files in this commit. + for change in _aggregate_changes(entry, notesdir): + uniqueid = change[0] + LOG.debug('%s: found change: %s', uniqueid, change[1:]) - # Remember the files seen, using their UUID suffix as a unique id. - for f in filenames: - # Updated as older tags are found, handling edits to release - # notes. - uniqueid = _get_unique_id(f) - LOG.debug('%s: found file %s', - uniqueid, f) + # Update the "earliest" version where a UID appears + # every time we see it, because we are scanning the + # history in reverse order so "early" items come + # later. LOG.debug('%s: setting earliest reference to %s' % - (uniqueid, tags[0])) - earliest_seen[uniqueid] = tags[0] - if uniqueid in last_name_by_id: - # We already have a filename for this id from a - # new commit, so use that one in case the name has - # changed. - LOG.debug('%s: was seen before in %s', - uniqueid, last_name_by_id[uniqueid]) + (uniqueid, current_version)) + earliest_seen[uniqueid] = current_version + + c_type = change[1] + + # If we have recorded that a UID was deleted, that + # means that was the last change made to the file and + # we can ignore it. + if uniqueid in uniqueids_deleted: + LOG.debug( + '%s: has already been deleted, ignoring this change', + uniqueid, + ) continue - elif self._file_exists_at_commit(f, sha): - LOG.debug('%s: looking for %s in deleted files %s', - uniqueid, f, uniqueids_deleted[uniqueid]) - if f in uniqueids_deleted[uniqueid]: - # The file exists in the commit, but was deleted - # later in the history. - LOG.debug('%s: skipping deleted file %s', - uniqueid, f) + + if c_type == diff_tree.CHANGE_ADD: + # A note is being added in this commit. If we have + # not seen it before, it was added here and never + # changed. + if uniqueid not in last_name_by_id: + path, sha = change[-2:] + last_name_by_id[uniqueid] = (path, sha) + LOG.debug( + '%s: updating last_name_by_id with %r', + uniqueid, (path, sha)) + else: + LOG.debug( + '%s: add for file we have already seen', + uniqueid) + + elif c_type == diff_tree.CHANGE_DELETE: + # This file is being deleted without a rename. If + # we have already seen the UID before, that means + # that after the file was deleted another file + # with the same UID was added back. In that case + # we do not want to treat it as deleted. + # + # Never store deleted files in last_name_by_id so + # we can safely use all of those entries to build + # the history data. + if uniqueid not in last_name_by_id: + uniqueids_deleted.add(uniqueid) + LOG.debug( + '%s: remembering deleted note', + uniqueid, + ) + else: + LOG.debug( + '%s: delete for file re-added after the delete', + uniqueid) + + elif c_type == diff_tree.CHANGE_RENAME: + # The file is being renamed. We may have seen it + # before, if there were subsequent modifications, + # so only store the name information if it is not + # there already. + if uniqueid not in last_name_by_id: + path, sha = change[-2:] + last_name_by_id[uniqueid] = (path, sha) + LOG.debug( + '%s: updating last_name_by_id with %r', + uniqueid, (path, sha)) + else: + LOG.debug( + '%s: renamed file already known with the new name', + uniqueid) + + elif c_type == diff_tree.CHANGE_MODIFY: + # An existing file is being modified. We may have + # seen it before, if there were subsequent + # modifications, so only store the name + # information if it is not there already. + if uniqueid not in last_name_by_id: + path, sha = change[-2:] + last_name_by_id[uniqueid] = (path, sha) + LOG.debug( + '%s: updating last_name_by_id with %r', + uniqueid, (path, sha)) else: - # Remember this filename as the most recent version of - # the unique id we have seen, in case the name - # changed from an older commit. - last_name_by_id[uniqueid] = (f, sha) - LOG.debug('%s: remembering %s as filename', - uniqueid, f) + LOG.debug( + '%s: modified file already known', + uniqueid) + else: - # Track files that have been deleted. The rename logic - # above checks for repeated references to files that - # are deleted later, and the inversion logic below - # checks for any remaining values and skips those - # entries. - LOG.debug('%s: saw a file that no longer exists', - uniqueid) - uniqueids_deleted[uniqueid].add(f) - LOG.debug('%s: deleted files %s', - uniqueid, uniqueids_deleted[uniqueid]) + raise ValueError( + 'unknown change instructions {!r}'.format(change) + ) + + LOG.debug('before inversion') + LOG.debug('versions: %r', versions) + LOG.debug('last_name_by_id: %r', last_name_by_id) + LOG.debug('earliest_seen: %r', earliest_seen) # Invert earliest_seen to make a list of notes files for each # version. @@ -518,9 +597,6 @@ class Scanner(object): for uniqueid, version in earliest_seen.items(): try: base, sha = last_name_by_id[uniqueid] - if base in uniqueids_deleted.get(uniqueid, set()): - LOG.debug('skipping deleted note %s' % uniqueid) - continue files_and_tags[version].append((base, sha)) except KeyError: # Unable to find the file again, skip it to avoid breaking diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 4108169..da511ff 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -20,7 +20,6 @@ import os.path import re import subprocess import time -import unittest from dulwich import diff_tree from dulwich import objects @@ -57,8 +56,6 @@ packages = testpkg """ -GIT_VERSION = utils.check_output(['git', '--version']).strip() - class GPGKeyFixture(fixtures.Fixture): """Creates a GPG key for testing. @@ -149,6 +146,8 @@ class Base(base.TestCase): def _git_commit(self, message='commit message'): self._run_git('add', '.') self._run_git('commit', '-m', message) + self._run_git('show', '--pretty=format:%H') + time.sleep(0.1) # force a delay between commits def _add_other_file(self, name): with open(os.path.join(self.reporoot, name), 'w') as f: @@ -492,10 +491,7 @@ class BasicTest(Base): results, ) - @unittest.skipIf(GIT_VERSION == 'git version 2.9.2', - 'Skipping for git version 2.9.2') def test_rename_then_delete_file(self): - print(GIT_VERSION) self._make_python_package() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1') @@ -734,7 +730,9 @@ class MergeCommitTest(Base): n2 = self._add_notes_file() self._run_git('checkout', 'master') self._add_other_file('ignore-1.txt') + # Merge the branch into master. self._run_git('merge', '--no-ff', 'test_merge_commit') + time.sleep(0.1) # force a delay between commits self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') self.scanner = scanner.Scanner(self.c) @@ -763,7 +761,10 @@ class MergeCommitTest(Base): n2 = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') self._add_other_file('ignore-1.txt') + # Merge the branch into master. self._run_git('merge', '--no-ff', 'test_merge_commit') + time.sleep(0.1) # force a delay between commits + self._run_git('show') self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') self.scanner = scanner.Scanner(self.c) @@ -796,6 +797,7 @@ class MergeCommitTest(Base): self._add_other_file('ignore-1.txt') self._run_git('tag', '-s', '-m', 'second tag', '1.1.0') self._run_git('merge', '--no-ff', 'test_merge_commit') + time.sleep(0.1) # force a delay between commits self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') self._add_other_file('ignore-3.txt') @@ -833,6 +835,7 @@ class MergeCommitTest(Base): n3 = self._add_notes_file() self._run_git('tag', '-s', '-m', 'second tag', '1.1.0') self._run_git('merge', '--no-ff', 'test_merge_commit') + time.sleep(0.1) # force a delay between commits self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') self._add_other_file('ignore-3.txt') -- GitLab From 934ac77cc558113d73bfe7d8f8f24e9dc14790c9 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 12 Dec 2016 14:27:27 -0500 Subject: [PATCH 088/257] deal with unsigned tags dulwich returns different data for unsigned tags Change-Id: Ifcbbbb8c816c2e82999c52a551044b996f7c1b11 Signed-off-by: Doug Hellmann --- reno/scanner.py | 38 ++++++++++++++++++++++++-------------- reno/tests/test_scanner.py | 10 ++++++++++ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 00cf2d0..53c0bb8 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -20,7 +20,7 @@ import re import sys from dulwich import diff_tree -from dulwich import refs +from dulwich import objects from dulwich import repo LOG = logging.getLogger(__name__) @@ -160,21 +160,35 @@ class RenoRepo(repo.Repo): } self._shas_to_tags = {} for tag, tag_sha in self._all_tags.items(): - # The tag has its own SHA, but the tag refers to the commit and - # that's the SHA we'll see when we scan commits on a branch. tag_obj = self[tag_sha] - tagged_sha = tag_obj.object[1] - self._shas_to_tags.setdefault(tagged_sha, []).append(tag) + if isinstance(tag_obj, objects.Tag): + # A signed tag has its own SHA, but the tag refers to + # the commit and that's the SHA we'll see when we scan + # commits on a branch. + tagged_sha = tag_obj.object[1] + date = tag_obj.tag_time + elif isinstance(tag_obj, objects.Commit): + # Unsigned tags refer directly to commits. This seems + # to especially happen when the tag definition moves + # to the packed-refs list instead of being represented + # by its own file. + tagged_sha = tag_obj.id + date = tag_obj.commit_time + else: + raise ValueError( + ('Unrecognized tag object {!r} with ' + 'tag {} and SHA {!r}: {}').format( + tag_obj, tag, tag_sha, type(tag_obj)) + ) + self._shas_to_tags.setdefault(tagged_sha, []).append((tag, date)) def get_tags_on_commit(self, sha): "Return the tag(s) on a commit, in application order." if self._all_tags is None: self._load_tags() - tags_on_commit = self._shas_to_tags.get(sha, []) - tags_on_commit.sort( - key=lambda x: self[self._all_tags[x]].tag_time - ) - return tags_on_commit + tags_and_dates = self._shas_to_tags.get(sha, []) + tags_and_dates.sort(key=lambda x: x[1]) + return [t[0] for t in tags_and_dates] def _get_file_from_tree(self, filename, tree): "Given a tree object, traverse it to find the file." @@ -221,10 +235,6 @@ class Scanner(object): def _get_branch_head(self, branch): if branch: branch_ref = b'refs/heads/' + branch.encode('utf-8') - if not refs.check_ref_format(branch_ref): - raise ValueError( - '{!r} does not look like a valid branch reference'.format( - branch_ref)) return self._repo.refs[branch_ref] return self._repo.refs[b'HEAD'] diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index da511ff..79be037 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1217,6 +1217,16 @@ class TagsTest(Base): results, ) + def test_unsigned(self): + self._add_notes_file('slug4') + self._run_git('tag', '-m', 'first tag', '4.0.0') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch(None) + self.assertEqual( + ['4.0.0', '3.0.0', '2.0.0', '1.0.0'], + results, + ) + class VersionTest(Base): -- GitLab From 38fcc978618e92c3e4a63c4b2d4db48c97f02dcc Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 12 Dec 2016 14:28:10 -0500 Subject: [PATCH 089/257] set up logging in the sphinx extension We have logging throughout the scanner, but Sphinx doesn't set up logging. We can do that when our extension is loaded. This change also modifies the log levels for a couple of messages. Change-Id: Ic766cf5fd0918a8e2afc0cdbe8e3f58a28bdeeee Signed-off-by: Doug Hellmann --- reno/scanner.py | 28 ++++++++++++++-------------- reno/sphinxext.py | 6 ++++++ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 53c0bb8..0532d3f 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -423,7 +423,7 @@ class Scanner(object): collapse_pre_releases = self.conf.collapse_pre_releases stop_at_branch_base = self.conf.stop_at_branch_base - LOG.debug('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch)) + LOG.info('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch)) # If the user has not told us where to stop, try to work it out # for ourselves. If branch is set and is not "master", then we @@ -441,7 +441,7 @@ class Scanner(object): earliest_version = '.'.join( earliest_version.split('.')[:-1] ) - LOG.debug('using earliest_version = %r', earliest_version) + LOG.info('earliest version on the branch is %s', earliest_version) # Determine all of the tags known on the branch, in their date # order. We scan the commit history in topological order to ensure @@ -526,9 +526,9 @@ class Scanner(object): if uniqueid not in last_name_by_id: path, sha = change[-2:] last_name_by_id[uniqueid] = (path, sha) - LOG.debug( - '%s: updating last_name_by_id with %r', - uniqueid, (path, sha)) + LOG.info( + '%s: update to %s in commit %s', + uniqueid, path, sha) else: LOG.debug( '%s: add for file we have already seen', @@ -546,9 +546,9 @@ class Scanner(object): # the history data. if uniqueid not in last_name_by_id: uniqueids_deleted.add(uniqueid) - LOG.debug( - '%s: remembering deleted note', - uniqueid, + LOG.info( + '%s: note deleted in %s', + uniqueid, sha, ) else: LOG.debug( @@ -563,9 +563,9 @@ class Scanner(object): if uniqueid not in last_name_by_id: path, sha = change[-2:] last_name_by_id[uniqueid] = (path, sha) - LOG.debug( - '%s: updating last_name_by_id with %r', - uniqueid, (path, sha)) + LOG.info( + '%s: update to %s in commit %s', + uniqueid, path, sha) else: LOG.debug( '%s: renamed file already known with the new name', @@ -579,9 +579,9 @@ class Scanner(object): if uniqueid not in last_name_by_id: path, sha = change[-2:] last_name_by_id[uniqueid] = (path, sha) - LOG.debug( - '%s: updating last_name_by_id with %r', - uniqueid, (path, sha)) + LOG.info( + '%s: update to %s in commit %s', + uniqueid, path, sha) else: LOG.debug( '%s: modified file already known', diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 7903d90..e7fc22a 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import logging import os.path from docutils import nodes @@ -101,3 +102,8 @@ class ReleaseNotesDirective(rst.Directive): def setup(app): app.add_directive('release-notes', ReleaseNotesDirective) + + logging.basicConfig( + level=logging.INFO, + format='[%(name)s] %(message)s', + ) -- GitLab From 079bcc6d2ad755af1a5f51c06900953d033238c9 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 12 Dec 2016 14:31:01 -0500 Subject: [PATCH 090/257] deal with remote branches When there is no local version of the branch, the reference looks different. Change-Id: I81c7a6e2d664636945f93a5f7e0a364949ea1ccc Signed-off-by: Doug Hellmann --- reno/scanner.py | 11 +++++++++-- reno/tests/test_scanner.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 0532d3f..a8e6375 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -234,8 +234,15 @@ class Scanner(object): def _get_branch_head(self, branch): if branch: - branch_ref = b'refs/heads/' + branch.encode('utf-8') - return self._repo.refs[branch_ref] + candidates = [ + b'refs/heads/' + branch.encode('utf-8'), + b'refs/remotes/' + branch.encode('utf-8'), + ] + for branch_ref in candidates: + if branch_ref in self._repo.refs: + return self._repo.refs[branch_ref] + # If we end up here we didn't find any of the candidates. + raise ValueError('Unknown branch {!r}'.format(branch)) return self._repo.refs[b'HEAD'] def _get_walker_for_branch(self, branch): diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 79be037..a456ea7 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1183,6 +1183,42 @@ class BranchTest(Base): results, ) + def test_remote_branches(self): + self._run_git('checkout', '2.0.0') + self._run_git('checkout', '-b', 'stable/2') + self._run_git('checkout', 'master') + scanner1 = scanner.Scanner(self.c) + head1 = scanner1._get_branch_head('stable/2') + self.assertIsNotNone(head1) + print('head1', head1) + # Create a second repository by cloning the first. + print(utils.check_output( + ['git', 'clone', self.reporoot, 'reporoot2'], + cwd=self.temp_dir, + )) + reporoot2 = os.path.join(self.temp_dir, 'reporoot2') + print(utils.check_output( + ['git', 'remote', 'update'], + cwd=reporoot2, + )) + print(utils.check_output( + ['git', 'remote', '-v'], + cwd=reporoot2, + )) + print(utils.check_output( + ['find', '.git/refs'], + cwd=reporoot2, + )) + print(utils.check_output( + ['git', 'branch', '-a'], + cwd=reporoot2, + )) + c2 = config.Config(reporoot2) + scanner2 = scanner.Scanner(c2) + head2 = scanner2._get_branch_head('origin/stable/2') + self.assertIsNotNone(head2) + self.assertEqual(head1, head2) + class TagsTest(Base): -- GitLab From 66f88814f8f16a22f383f418df4c0f7d20733a48 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 12 Dec 2016 15:25:36 -0500 Subject: [PATCH 091/257] shortcut the branch scan by looking at the version number When we get to the version number at the base of the branch, stop scanning for more changes. Change-Id: I872c2d0f586b23995cc4b20e83e8e5e3bab49dae Signed-off-by: Doug Hellmann --- reno/scanner.py | 46 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index a8e6375..64554cc 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -430,7 +430,23 @@ class Scanner(object): collapse_pre_releases = self.conf.collapse_pre_releases stop_at_branch_base = self.conf.stop_at_branch_base - LOG.info('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch)) + LOG.info('scanning %s/%s (branch=%s)', + reporoot.rstrip('/'), notesdir.lstrip('/'), branch) + + # Determine all of the tags known on the branch, in their date + # order. We scan the commit history in topological order to ensure + # we have the commits in the right version, so we might encounter + # the tags in a different order during that phase. + versions_by_date = self._get_tags_on_branch(branch) + LOG.debug('versions by date %r' % (versions_by_date,)) + if earliest_version and earliest_version not in versions_by_date: + raise ValueError( + 'earliest-version set to unknown revision {!r}'.format( + earliest_version)) + + # If the user has told us where to stop, use that as the + # default. + branch_base_tag = earliest_version # If the user has not told us where to stop, try to work it out # for ourselves. If branch is set and is not "master", then we @@ -439,6 +455,7 @@ class Scanner(object): (not earliest_version) and branch and (branch != 'master')): LOG.debug('determining earliest_version from branch') earliest_version = self._get_branch_base(branch) + branch_base_tag = earliest_version if earliest_version and collapse_pre_releases: if PRE_RELEASE_RE.search(earliest_version): # The earliest version won't actually be the pre-release @@ -448,14 +465,13 @@ class Scanner(object): earliest_version = '.'.join( earliest_version.split('.')[:-1] ) - LOG.info('earliest version on the branch is %s', earliest_version) + if earliest_version: + LOG.info('earliest version to include is %s', earliest_version) + else: + LOG.info('including entire branch history') + if branch_base_tag: + LOG.info('stopping scan at %s', branch_base_tag) - # Determine all of the tags known on the branch, in their date - # order. We scan the commit history in topological order to ensure - # we have the commits in the right version, so we might encounter - # the tags in a different order during that phase. - versions_by_date = self._get_tags_on_branch(branch) - LOG.debug('versions by date %r' % (versions_by_date,)) versions = [] earliest_seen = collections.OrderedDict() @@ -479,11 +495,10 @@ class Scanner(object): # Remember uniqueids that have had files deleted. uniqueids_deleted = set() - for entry in self._topo_traversal(branch): + for counter, entry in enumerate(self._topo_traversal(branch), 1): sha = entry.commit.id tags_on_commit = self._repo.get_tags_on_commit(sha) - LOG.debug('%s encountered', sha) # If there are no tags in this block, assume the most recently # seen version. @@ -492,9 +507,9 @@ class Scanner(object): tags = [current_version] else: current_version = tags_on_commit[-1] - LOG.debug('%s has tags %s, ' - 'updating current version to %s' % - (sha, tags_on_commit, current_version)) + LOG.debug('%s has tags %s', sha, tags_on_commit) + LOG.info('%s %5d updating current version to %s', + sha, counter, current_version) # Remember each version we have seen. if current_version not in versions: @@ -504,7 +519,6 @@ class Scanner(object): # Look for changes to notes files in this commit. for change in _aggregate_changes(entry, notesdir): uniqueid = change[0] - LOG.debug('%s: found change: %s', uniqueid, change[1:]) # Update the "earliest" version where a UID appears # every time we see it, because we are scanning the @@ -599,6 +613,10 @@ class Scanner(object): 'unknown change instructions {!r}'.format(change) ) + if branch_base_tag and branch_base_tag in tags: + LOG.info('reached end of branch after %d commits', counter) + break + LOG.debug('before inversion') LOG.debug('versions: %r', versions) LOG.debug('last_name_by_id: %r', last_name_by_id) -- GitLab From aa8f5799bae3d002fab9106bf7735a48447e9f88 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 13 Dec 2016 10:34:03 -0500 Subject: [PATCH 092/257] traversal performance improvements Only compute the differences in the subdirectory where notes are stored, not the entire tree. The subtree diff is faster to compute, and lets us ignore commits that aren't interesting to us. These changes reduced the scan time for the master branch of openstack/nova from 5:33 to 0:32. Change-Id: Ic85b873ad353b3652717c9807ad5f69c473d8c60 Signed-off-by: Doug Hellmann --- reno/scanner.py | 117 +++++++++++++++++++++++++++++++------ reno/tests/test_scanner.py | 28 ++++----- 2 files changed, 114 insertions(+), 31 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 64554cc..b8f3675 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -42,7 +42,7 @@ def _get_unique_id(filename): return uniqueid -def _note_file(notesdir, name): +def _note_file(name): """Return bool indicating if the filename looks like a note file. This is used to filter the files in changes based on the notes @@ -53,19 +53,72 @@ def _note_file(notesdir, name): """ if not name: return False - if fnmatch.fnmatch(name, notesdir + '/*.yaml'): + if fnmatch.fnmatch(name, '*.yaml'): return True - elif fnmatch.fnmatch(name, notesdir + '/*'): + else: LOG.warning('found and ignored extra file %s', name) return False -def _aggregate_changes(walk_entry, notesdir): - """Collapse a WalkEntry based on our notion of uniqueness for file uids. +def _changes_in_subdir(repo, walk_entry, subdir): + """Iterator producing changes of interest to reno. - Each WalkEntry contains a list of TreeChange instances which - describe changes between the old and new repository trees. The - change has a type, and new and old paths and shas. + The default changes() method of a WalkEntry computes all of the + changes in the entire repo at that point. We only care about + changes in a subdirectory, so this reimplements + WalkeEntry.changes() with that filter in place. + + The alternative, passing paths to the TreeWalker, does not work + because we need all of the commits in sequence so we can tell when + the tag changes. We have to look at every commit to see if it + either has a tag, a note file, or both. + + NOTE(dhellmann): The TreeChange entries returned as a result of + the manipulation done by this function have the subdir prefix + stripped. + + """ + commit = walk_entry.commit + store = repo.object_store + + parents = walk_entry._get_parents(commit) + + if not parents: + changes_func = diff_tree.tree_changes + parent_subtree = None + elif len(parents) == 1: + changes_func = diff_tree.tree_changes + parent_tree = repo[repo[parents[0]].tree] + parent_subtree = repo._get_subtree(parent_tree, subdir) + if parent_subtree: + parent_subtree = parent_subtree.sha().hexdigest().encode('ascii') + else: + changes_func = diff_tree.tree_changes_for_merge + parent_subtree = [ + repo._get_subtree(repo[repo[p].tree], subdir) + for p in parents + ] + parent_subtree = [ + p.sha().hexdigest().encode('ascii') + for p in parent_subtree + if p + ] + subdir_tree = repo._get_subtree(repo[commit.tree], subdir) + if subdir_tree: + commit_subtree = subdir_tree.sha().hexdigest().encode('ascii') + else: + commit_subtree = None + if parent_subtree == commit_subtree: + return [] + return changes_func(store, parent_subtree, commit_subtree) + + +def _aggregate_changes(walk_entry, changes, notesdir): + """Collapse a series of changes based on uniqueness for file uids. + + The list of TreeChange instances describe changes between the old + and new repository trees. The change has a type, and new and old + paths and shas. Simple add, delete, and change operations are handled directly. @@ -85,10 +138,10 @@ def _aggregate_changes(walk_entry, notesdir): them easier. """ - by_uid = collections.defaultdict(list) sha = walk_entry.commit.id LOG.debug('entry for commit %s', sha) - for ec in walk_entry.changes(): + by_uid = collections.defaultdict(list) + for ec in changes: LOG.debug('change %r', ec) if not isinstance(ec, list): ec = [ec] @@ -97,21 +150,21 @@ def _aggregate_changes(walk_entry, notesdir): for c in ec: if c.type == diff_tree.CHANGE_ADD: path = c.new.path.decode('utf-8') if c.new.path else None - if _note_file(notesdir, path): + if _note_file(path): uid = _get_unique_id(path) by_uid[uid].append((c.type, path, sha)) else: LOG.debug('ignoring') elif c.type == diff_tree.CHANGE_DELETE: path = c.old.path.decode('utf-8') if c.old.path else None - if _note_file(notesdir, path): + if _note_file(path): uid = _get_unique_id(path) by_uid[uid].append((c.type, path)) else: LOG.debug('ignoring') elif c.type == diff_tree.CHANGE_MODIFY: path = c.new.path.decode('utf-8') if c.new.path else None - if _note_file(notesdir, path): + if _note_file(path): uid = _get_unique_id(path) by_uid[uid].append((c.type, path, sha)) else: @@ -190,6 +243,29 @@ class RenoRepo(repo.Repo): tags_and_dates.sort(key=lambda x: x[1]) return [t[0] for t in tags_and_dates] + def _get_subtree(self, tree, path): + "Given a tree SHA and a path, return the SHA of the subtree." + try: + if os.sep in path: + # The tree entry will only have a single level of the + # directory name, so if we have a / in our filename we + # know we're going to have to keep traversing the + # tree. + prefix, _, trailing = path.partition(os.sep) + mode, subtree_sha = tree[prefix.encode('utf-8')] + subtree = self[subtree_sha] + return self._get_subtree(subtree, trailing) + else: + # The tree entry will point to the SHA of the contents + # of the subtree. + mode, sha = tree[path.encode('utf-8')] + result = self[sha] + return result + except KeyError: + # Some part of the path wasn't found, so the subtree is + # not present. Return the sentinel value. + return None + def _get_file_from_tree(self, filename, tree): "Given a tree object, traverse it to find the file." try: @@ -517,7 +593,8 @@ class Scanner(object): versions.append(current_version) # Look for changes to notes files in this commit. - for change in _aggregate_changes(entry, notesdir): + changes = _changes_in_subdir(self._repo, entry, notesdir) + for change in _aggregate_changes(entry, changes, notesdir): uniqueid = change[0] # Update the "earliest" version where a UID appears @@ -546,7 +623,9 @@ class Scanner(object): # changed. if uniqueid not in last_name_by_id: path, sha = change[-2:] - last_name_by_id[uniqueid] = (path, sha) + fullpath = os.path.join(notesdir, path) + last_name_by_id[uniqueid] = (fullpath, + sha.decode('ascii')) LOG.info( '%s: update to %s in commit %s', uniqueid, path, sha) @@ -583,7 +662,9 @@ class Scanner(object): # there already. if uniqueid not in last_name_by_id: path, sha = change[-2:] - last_name_by_id[uniqueid] = (path, sha) + fullpath = os.path.join(notesdir, path) + last_name_by_id[uniqueid] = (fullpath, + sha.decode('ascii')) LOG.info( '%s: update to %s in commit %s', uniqueid, path, sha) @@ -599,7 +680,9 @@ class Scanner(object): # information if it is not there already. if uniqueid not in last_name_by_id: path, sha = change[-2:] - last_name_by_id[uniqueid] = (path, sha) + fullpath = os.path.join(notesdir, path) + last_name_by_id[uniqueid] = (fullpath, + sha.decode('ascii')) LOG.info( '%s: update to %s in commit %s', uniqueid, path, sha) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index a456ea7..e06db14 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1315,7 +1315,7 @@ class AggregateChangesTest(Base): n = self.get_note_num() name = 'prefix/add-%016x' % n # no .yaml extension entry.commit.id = 'commit-id' - entry.changes.return_value = [ + changes = [ diff_tree.TreeChange( type=diff_tree.CHANGE_ADD, old=objects.TreeEntry(path=None, mode=None, sha=None), @@ -1326,7 +1326,7 @@ class AggregateChangesTest(Base): ) ) ] - results = scanner._aggregate_changes(entry, 'prefix') + results = scanner._aggregate_changes(entry, changes, 'prefix') self.assertEqual( [], results, @@ -1337,7 +1337,7 @@ class AggregateChangesTest(Base): n = self.get_note_num() name = 'prefix/add-%016x.yaml' % n entry.commit.id = 'commit-id' - entry.changes.return_value = [ + changes = [ diff_tree.TreeChange( type=diff_tree.CHANGE_ADD, old=objects.TreeEntry(path=None, mode=None, sha=None), @@ -1348,7 +1348,7 @@ class AggregateChangesTest(Base): ) ) ] - results = list(scanner._aggregate_changes(entry, 'prefix')) + results = list(scanner._aggregate_changes(entry, changes, 'prefix')) self.assertEqual( [('%016x' % n, 'add', name, 'commit-id')], results, @@ -1359,7 +1359,7 @@ class AggregateChangesTest(Base): n = self.get_note_num() name = 'prefix/delete-%016x.yaml' % n entry.commit.id = 'commit-id' - entry.changes.return_value = [ + changes = [ diff_tree.TreeChange( type=diff_tree.CHANGE_DELETE, old=objects.TreeEntry( @@ -1370,7 +1370,7 @@ class AggregateChangesTest(Base): new=objects.TreeEntry(path=None, mode=None, sha=None) ) ] - results = list(scanner._aggregate_changes(entry, 'prefix')) + results = list(scanner._aggregate_changes(entry, changes, 'prefix')) self.assertEqual( [('%016x' % n, 'delete', name)], results, @@ -1381,7 +1381,7 @@ class AggregateChangesTest(Base): n = self.get_note_num() name = 'prefix/change-%016x.yaml' % n entry.commit.id = 'commit-id' - entry.changes.return_value = [ + changes = [ diff_tree.TreeChange( type=diff_tree.CHANGE_MODIFY, old=objects.TreeEntry( @@ -1396,7 +1396,7 @@ class AggregateChangesTest(Base): ), ) ] - results = list(scanner._aggregate_changes(entry, 'prefix')) + results = list(scanner._aggregate_changes(entry, changes, 'prefix')) self.assertEqual( [('%016x' % n, 'modify', name, 'commit-id')], results, @@ -1408,7 +1408,7 @@ class AggregateChangesTest(Base): new_name = 'prefix/new-%016x.yaml' % n old_name = 'prefix/old-%016x.yaml' % n entry.commit.id = 'commit-id' - entry.changes.return_value = [ + changes = [ diff_tree.TreeChange( type=diff_tree.CHANGE_ADD, old=objects.TreeEntry(path=None, mode=None, sha=None), @@ -1428,7 +1428,7 @@ class AggregateChangesTest(Base): new=objects.TreeEntry(path=None, mode=None, sha=None) ) ] - results = list(scanner._aggregate_changes(entry, 'prefix')) + results = list(scanner._aggregate_changes(entry, changes, 'prefix')) self.assertEqual( [('%016x' % n, 'rename', old_name, new_name, 'commit-id')], results, @@ -1440,7 +1440,7 @@ class AggregateChangesTest(Base): new_name = 'prefix/new-%016x.yaml' % n old_name = 'prefix/old-%016x.yaml' % n entry.commit.id = 'commit-id' - entry.changes.return_value = [ + changes = [ diff_tree.TreeChange( type=diff_tree.CHANGE_DELETE, old=objects.TreeEntry( @@ -1460,7 +1460,7 @@ class AggregateChangesTest(Base): ) ), ] - results = list(scanner._aggregate_changes(entry, 'prefix')) + results = list(scanner._aggregate_changes(entry, changes, 'prefix')) self.assertEqual( [('%016x' % n, 'rename', old_name, new_name, 'commit-id')], results, @@ -1478,7 +1478,7 @@ class AggregateChangesTest(Base): # comply with the rest of the configuration for the scanner. old_name = 'prefix/old-%016x.yaml' % n entry.commit.id = 'commit-id' - entry.changes.return_value = [[ + changes = [[ diff_tree.TreeChange( type='modify', old=diff_tree.TreeEntry( @@ -1506,7 +1506,7 @@ class AggregateChangesTest(Base): ), ), ]] - results = list(scanner._aggregate_changes(entry, 'prefix')) + results = list(scanner._aggregate_changes(entry, changes, 'prefix')) self.assertEqual( [('%016x' % n, 'modify', old_name, 'commit-id'), ('%016x' % n, 'modify', old_name, 'commit-id')], -- GitLab From 24be107c3cbed91671a0a55447ba04c48fc3049b Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 13 Dec 2016 11:13:26 -0500 Subject: [PATCH 093/257] logging improvements Clean up the log output so that it includes more useful information for a user without being a complete info dump of the internals. Also fix the style of some logging calls to remove the string interpolation. Change-Id: I173801b2690e9982abfac1cb95fdf7af450e162d Signed-off-by: Doug Hellmann --- reno/scanner.py | 53 ++++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index b8f3675..271919e 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -139,15 +139,14 @@ def _aggregate_changes(walk_entry, changes, notesdir): """ sha = walk_entry.commit.id - LOG.debug('entry for commit %s', sha) by_uid = collections.defaultdict(list) for ec in changes: - LOG.debug('change %r', ec) if not isinstance(ec, list): ec = [ec] else: ec = ec for c in ec: + LOG.debug('change %r', c) if c.type == diff_tree.CHANGE_ADD: path = c.new.path.decode('utf-8') if c.new.path else None if _note_file(path): @@ -507,7 +506,8 @@ class Scanner(object): stop_at_branch_base = self.conf.stop_at_branch_base LOG.info('scanning %s/%s (branch=%s)', - reporoot.rstrip('/'), notesdir.lstrip('/'), branch) + reporoot.rstrip('/'), notesdir.lstrip('/'), + branch or '*current*') # Determine all of the tags known on the branch, in their date # order. We scan the commit history in topological order to ensure @@ -560,9 +560,7 @@ class Scanner(object): current_version = self._get_current_version(branch) LOG.debug('current repository version: %s' % current_version) if current_version not in versions_by_date: - LOG.debug('adding %s to versions by date' % current_version) versions_by_date.insert(0, current_version) - LOG.debug('versions by date %r' % (versions_by_date,)) # Remember the most current filename for each id, to allow for # renames. @@ -576,6 +574,8 @@ class Scanner(object): sha = entry.commit.id tags_on_commit = self._repo.get_tags_on_commit(sha) + LOG.debug('%06d %s %s', counter, sha, tags_on_commit) + # If there are no tags in this block, assume the most recently # seen version. tags = tags_on_commit @@ -583,9 +583,8 @@ class Scanner(object): tags = [current_version] else: current_version = tags_on_commit[-1] - LOG.debug('%s has tags %s', sha, tags_on_commit) - LOG.info('%s %5d updating current version to %s', - sha, counter, current_version) + LOG.info('%06d %s updating current version to %s', + counter, sha, current_version) # Remember each version we have seen. if current_version not in versions: @@ -601,8 +600,8 @@ class Scanner(object): # every time we see it, because we are scanning the # history in reverse order so "early" items come # later. - LOG.debug('%s: setting earliest reference to %s' % - (uniqueid, current_version)) + LOG.debug('%s: setting earliest reference to %s', + uniqueid, current_version) earliest_seen[uniqueid] = current_version c_type = change[1] @@ -628,11 +627,13 @@ class Scanner(object): sha.decode('ascii')) LOG.info( '%s: update to %s in commit %s', - uniqueid, path, sha) + uniqueid, path, sha, + ) else: LOG.debug( '%s: add for file we have already seen', - uniqueid) + uniqueid, + ) elif c_type == diff_tree.CHANGE_DELETE: # This file is being deleted without a rename. If @@ -653,7 +654,8 @@ class Scanner(object): else: LOG.debug( '%s: delete for file re-added after the delete', - uniqueid) + uniqueid, + ) elif c_type == diff_tree.CHANGE_RENAME: # The file is being renamed. We may have seen it @@ -667,11 +669,13 @@ class Scanner(object): sha.decode('ascii')) LOG.info( '%s: update to %s in commit %s', - uniqueid, path, sha) + uniqueid, path, sha, + ) else: LOG.debug( '%s: renamed file already known with the new name', - uniqueid) + uniqueid, + ) elif c_type == diff_tree.CHANGE_MODIFY: # An existing file is being modified. We may have @@ -685,11 +689,13 @@ class Scanner(object): sha.decode('ascii')) LOG.info( '%s: update to %s in commit %s', - uniqueid, path, sha) + uniqueid, path, sha, + ) else: LOG.debug( '%s: modified file already known', - uniqueid) + uniqueid, + ) else: raise ValueError( @@ -700,11 +706,6 @@ class Scanner(object): LOG.info('reached end of branch after %d commits', counter) break - LOG.debug('before inversion') - LOG.debug('versions: %r', versions) - LOG.debug('last_name_by_id: %r', last_name_by_id) - LOG.debug('earliest_seen: %r', earliest_seen) - # Invert earliest_seen to make a list of notes files for each # version. files_and_tags = collections.OrderedDict() @@ -719,7 +720,7 @@ class Scanner(object): except KeyError: # Unable to find the file again, skip it to avoid breaking # the build. - msg = ('[reno] unable to find file associated ' + msg = ('unable to find release notes file associated ' 'with unique id %r, skipping') % uniqueid LOG.debug(msg) print(msg, file=sys.stderr) @@ -775,6 +776,8 @@ class Scanner(object): if earliest_version and ov == earliest_version: break - LOG.debug('[reno] found %d versions and %d files', - len(trimmed.keys()), sum(len(ov) for ov in trimmed.values())) + LOG.debug( + 'found %d versions and %d files', + len(trimmed.keys()), sum(len(ov) for ov in trimmed.values()), + ) return trimmed -- GitLab From a333993a5f775d31a272e8f99e821b67cc9795bd Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 13 Dec 2016 13:37:59 -0500 Subject: [PATCH 094/257] create GitRepoFixture Change-Id: If18eef4130d3f8f70a254de21203840fb25a804f Signed-off-by: Doug Hellmann --- reno/tests/test_scanner.py | 380 +++++++++++++++++++------------------ 1 file changed, 195 insertions(+), 185 deletions(-) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index e06db14..364d18d 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -122,11 +122,25 @@ class GPGKeyFixture(fixtures.Fixture): ) -class Base(base.TestCase): +class GitRepoFixture(fixtures.Fixture): - logger = logging.getLogger('test') + logger = logging.getLogger('git') + + def __init__(self, reporoot): + self.reporoot = reporoot + super(GitRepoFixture, self).__init__() + + def setUp(self): + super(GitRepoFixture, self).setUp() + self.useFixture(GPGKeyFixture()) + os.makedirs(self.reporoot) + self.git('init', '.') + self.git('config', '--local', 'user.email', 'example@example.com') + self.git('config', '--local', 'user.name', 'reno developer') + self.git('config', '--local', 'user.signingkey', + 'example@example.com') - def _run_git(self, *args): + def git(self, *args): self.logger.debug('$ git %s', ' '.join(args)) output = utils.check_output( ['git'] + list(args), @@ -135,24 +149,21 @@ class Base(base.TestCase): self.logger.debug(output) return output - def _git_setup(self): - os.makedirs(self.reporoot) - self._run_git('init', '.') - self._run_git('config', '--local', 'user.email', 'example@example.com') - self._run_git('config', '--local', 'user.name', 'reno developer') - self._run_git('config', '--local', 'user.signingkey', - 'example@example.com') - - def _git_commit(self, message='commit message'): - self._run_git('add', '.') - self._run_git('commit', '-m', message) - self._run_git('show', '--pretty=format:%H') + def commit(self, message='commit message'): + self.git('add', '.') + self.git('commit', '-m', message) + self.git('show', '--pretty=format:%H') time.sleep(0.1) # force a delay between commits - def _add_other_file(self, name): + def add_file(self, name): with open(os.path.join(self.reporoot, name), 'w') as f: f.write('adding %s\n' % name) - self._git_commit('add %s' % name) + self.commit('add %s' % name) + + +class Base(base.TestCase): + + logger = logging.getLogger('test') def _add_notes_file(self, slug='slug', commit=True, legacy=False, contents='i-am-also-a-template'): @@ -164,7 +175,7 @@ class Base(base.TestCase): filename = os.path.join(self.reporoot, 'releasenotes', 'notes', basename) create._make_note_file(filename, contents) - self._git_commit('add %s' % basename) + self.repo.commit('add %s' % basename) return os.path.join('releasenotes', 'notes', basename) def _make_python_package(self): @@ -179,7 +190,7 @@ class Base(base.TestCase): init = os.path.join(pkgdir, '__init__.py') with open(init, 'w') as f: f.write("Test package") - self._git_commit('add test package') + self.repo.commit('add test package') def setUp(self): super(Base, self).setUp() @@ -194,12 +205,11 @@ class Base(base.TestCase): # directory to permit using git config --global without stepping on # developer configuration. self.useFixture(fixtures.TempHomeDir()) - self.useFixture(GPGKeyFixture()) self.useFixture(fixtures.NestedTempfile()) self.temp_dir = self.useFixture(fixtures.TempDir()).path self.reporoot = os.path.join(self.temp_dir, 'reporoot') + self.repo = self.useFixture(GitRepoFixture(self.reporoot)) self.c = config.Config(self.reporoot) - self._git_setup() self._counter = itertools.count(1) self.get_note_num = lambda: next(self._counter) @@ -235,8 +245,8 @@ class BasicTest(Base): def test_note_before_tag(self): filename = self._add_notes_file() - self._add_other_file('not-a-release-note.txt') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.add_file('not-a-release-note.txt') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -250,7 +260,7 @@ class BasicTest(Base): def test_note_commit_tagged(self): filename = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -264,7 +274,7 @@ class BasicTest(Base): def test_note_commit_after_tag(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') filename = self._add_notes_file() self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() @@ -279,9 +289,9 @@ class BasicTest(Base): def test_other_commit_after_tag(self): filename = self._add_notes_file() - self._add_other_file('ignore-1.txt') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - self._add_other_file('ignore-2.txt') + self.repo.add_file('ignore-1.txt') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.add_file('ignore-2.txt') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -295,7 +305,7 @@ class BasicTest(Base): def test_multiple_notes_after_tag(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file() f2 = self._add_notes_file() self.scanner = scanner.Scanner(self.c) @@ -313,7 +323,7 @@ class BasicTest(Base): self._make_python_package() f1 = self._add_notes_file(commit=False) f2 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -327,9 +337,9 @@ class BasicTest(Base): def test_multiple_tags(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = self._add_notes_file() self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() @@ -346,12 +356,12 @@ class BasicTest(Base): def test_rename_file(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = f1.replace('slug1', 'slug2') - self._run_git('mv', f1, f2) - self._git_commit('rename note file') + self.repo.git('mv', f1, f2) + self.repo.commit('rename note file') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -366,12 +376,12 @@ class BasicTest(Base): def test_rename_file_sort_earlier(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = f1.replace('slug1', 'slug0') - self._run_git('mv', f1, f2) - self._git_commit('rename note file') + self.repo.git('mv', f1, f2) + self.repo.commit('rename note file') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -386,12 +396,12 @@ class BasicTest(Base): def test_edit_file(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') with open(os.path.join(self.reporoot, f1), 'w') as f: f.write('---\npreamble: new contents for file') - self._git_commit('edit note file') + self.repo.commit('edit note file') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -406,12 +416,12 @@ class BasicTest(Base): def test_legacy_file(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1', legacy=True) - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = f1.replace('slug1', 'slug2') - self._run_git('mv', f1, f2) - self._git_commit('rename note file') + self.repo.git('mv', f1, f2) + self.repo.commit('rename note file') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -426,15 +436,15 @@ class BasicTest(Base): def test_rename_legacy_file_to_new(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1', legacy=True) - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') # Rename the file with the new convention of placing the UUID # after the slug instead of before. f2 = f1.replace('0000000000000001-slug1', 'slug1-0000000000000001') - self._run_git('mv', f1, f2) - self._git_commit('rename note file') + self.repo.git('mv', f1, f2) + self.repo.commit('rename note file') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -450,11 +460,11 @@ class BasicTest(Base): def test_limit_by_earliest_version(self): self._make_python_package() self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f2 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'middle tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'middle tag', '2.0.0') f3 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'last tag', '3.0.0') + self.repo.git('tag', '-s', '-m', 'last tag', '3.0.0') self.c.override( earliest_version='2.0.0', ) @@ -473,12 +483,12 @@ class BasicTest(Base): def test_delete_file(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1') f2 = self._add_notes_file('slug2') - self._run_git('rm', f1) - self._git_commit('remove note file') - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('rm', f1) + self.repo.commit('remove note file') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -493,17 +503,17 @@ class BasicTest(Base): def test_rename_then_delete_file(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1') f2 = f1.replace('slug1', 'slug2') - self._run_git('mv', f1, f2) - self._run_git('status') - self._git_commit('rename note file') - self._run_git('rm', f2) - self._git_commit('remove note file') + self.repo.git('mv', f1, f2) + self.repo.git('status') + self.repo.commit('rename note file') + self.repo.git('rm', f2) + self.repo.commit('remove note file') f3 = self._add_notes_file('slug3') - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') - log_results = self._run_git('log', '--topo-order', + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') + log_results = self.repo.git('log', '--topo-order', '--pretty=%H %d', '--name-only') self.addDetail('git log', text_content(log_results)) @@ -548,7 +558,7 @@ class FileContentsTest(Base): f1 = self._add_notes_file(contents='initial-contents') with open(os.path.join(self.reporoot, f1), 'w') as f: f.write('new contents for file') - self._git_commit('edit note file') + self.repo.commit('edit note file') r = scanner.RenoRepo(self.reporoot) contents = r.get_file_at_commit(f1, 'HEAD') self.assertEqual( @@ -562,7 +572,7 @@ class FileContentsTest(Base): f1 = self._add_notes_file(contents='initial-contents') with open(os.path.join(self.reporoot, f1), 'w') as f: f.write('new contents for file') - self._git_commit('edit note file') + self.repo.commit('edit note file') self.scanner = scanner.Scanner(self.c) r = scanner.RenoRepo(self.reporoot) head = r.head() @@ -592,9 +602,9 @@ class PreReleaseTest(Base): def test_alpha(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0a1') f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a2') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0a2') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -609,9 +619,9 @@ class PreReleaseTest(Base): def test_beta(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0b1') f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b2') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0b2') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -626,9 +636,9 @@ class PreReleaseTest(Base): def test_release_candidate(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0rc1') f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc2') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0rc2') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -645,13 +655,13 @@ class PreReleaseTest(Base): files = [] self._make_python_package() files.append(self._add_notes_file('slug1')) - self._run_git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') + self.repo.git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') files.append(self._add_notes_file('slug2')) - self._run_git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') + self.repo.git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') files.append(self._add_notes_file('slug3')) - self._run_git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') + self.repo.git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') files.append(self._add_notes_file('slug4')) - self._run_git('tag', '-s', '-m', 'full release tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'full release tag', '1.0.0') self.c.override( collapse_pre_releases=True, ) @@ -670,11 +680,11 @@ class PreReleaseTest(Base): def test_collapse_without_full_release(self): self._make_python_package() f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') + self.repo.git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') f2 = self._add_notes_file('slug2') - self._run_git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') + self.repo.git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') f3 = self._add_notes_file('slug3') - self._run_git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') + self.repo.git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') self.c.override( collapse_pre_releases=True, ) @@ -694,13 +704,13 @@ class PreReleaseTest(Base): def test_collapse_without_notes(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'earlier tag', '0.1.0') + self.repo.git('tag', '-s', '-m', 'earlier tag', '0.1.0') f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') + self.repo.git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') f2 = self._add_notes_file('slug2') - self._run_git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') + self.repo.git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') f3 = self._add_notes_file('slug3') - self._run_git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') + self.repo.git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') self.c.override( collapse_pre_releases=True, ) @@ -725,16 +735,16 @@ class MergeCommitTest(Base): # Create changes on master and in the branch # in order so the history is "normal" n1 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - self._run_git('checkout', '-b', 'test_merge_commit') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('checkout', '-b', 'test_merge_commit') n2 = self._add_notes_file() - self._run_git('checkout', 'master') - self._add_other_file('ignore-1.txt') + self.repo.git('checkout', 'master') + self.repo.add_file('ignore-1.txt') # Merge the branch into master. - self._run_git('merge', '--no-ff', 'test_merge_commit') + self.repo.git('merge', '--no-ff', 'test_merge_commit') time.sleep(0.1) # force a delay between commits - self._add_other_file('ignore-2.txt') - self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') + self.repo.add_file('ignore-2.txt') + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.0') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -754,19 +764,19 @@ class MergeCommitTest(Base): def test_2(self): # Create changes on the branch before the tag into which it is # actually merged. - self._add_other_file('ignore-0.txt') - self._run_git('checkout', '-b', 'test_merge_commit') + self.repo.add_file('ignore-0.txt') + self.repo.git('checkout', '-b', 'test_merge_commit') n1 = self._add_notes_file() - self._run_git('checkout', 'master') + self.repo.git('checkout', 'master') n2 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - self._add_other_file('ignore-1.txt') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.add_file('ignore-1.txt') # Merge the branch into master. - self._run_git('merge', '--no-ff', 'test_merge_commit') + self.repo.git('merge', '--no-ff', 'test_merge_commit') time.sleep(0.1) # force a delay between commits - self._run_git('show') - self._add_other_file('ignore-2.txt') - self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') + self.repo.git('show') + self.repo.add_file('ignore-2.txt') + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.0') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -788,19 +798,19 @@ class MergeCommitTest(Base): # actually merged, with another tag in between the time of the # commit and the time of the merge. This should reflect the # order of events described in bug #1522153. - self._add_other_file('ignore-0.txt') - self._run_git('checkout', '-b', 'test_merge_commit') + self.repo.add_file('ignore-0.txt') + self.repo.git('checkout', '-b', 'test_merge_commit') n1 = self._add_notes_file() - self._run_git('checkout', 'master') + self.repo.git('checkout', 'master') n2 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - self._add_other_file('ignore-1.txt') - self._run_git('tag', '-s', '-m', 'second tag', '1.1.0') - self._run_git('merge', '--no-ff', 'test_merge_commit') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.add_file('ignore-1.txt') + self.repo.git('tag', '-s', '-m', 'second tag', '1.1.0') + self.repo.git('merge', '--no-ff', 'test_merge_commit') time.sleep(0.1) # force a delay between commits - self._add_other_file('ignore-2.txt') - self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') - self._add_other_file('ignore-3.txt') + self.repo.add_file('ignore-2.txt') + self.repo.git('tag', '-s', '-m', 'third tag', '2.0.0') + self.repo.add_file('ignore-3.txt') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -825,20 +835,20 @@ class MergeCommitTest(Base): # actually merged, with another tag in between the time of the # commit and the time of the merge. This should reflect the # order of events described in bug #1522153. - self._add_other_file('ignore-0.txt') - self._run_git('checkout', '-b', 'test_merge_commit') + self.repo.add_file('ignore-0.txt') + self.repo.git('checkout', '-b', 'test_merge_commit') n1 = self._add_notes_file() - self._run_git('checkout', 'master') + self.repo.git('checkout', 'master') n2 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - self._add_other_file('ignore-1.txt') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.add_file('ignore-1.txt') n3 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'second tag', '1.1.0') - self._run_git('merge', '--no-ff', 'test_merge_commit') + self.repo.git('tag', '-s', '-m', 'second tag', '1.1.0') + self.repo.git('merge', '--no-ff', 'test_merge_commit') time.sleep(0.1) # force a delay between commits - self._add_other_file('ignore-2.txt') - self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') - self._add_other_file('ignore-3.txt') + self.repo.add_file('ignore-2.txt') + self.repo.git('tag', '-s', '-m', 'third tag', '2.0.0') + self.repo.add_file('ignore-3.txt') self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { @@ -878,19 +888,19 @@ class BranchBaseTest(Base): super(BranchBaseTest, self).setUp() self._make_python_package() self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') self._add_notes_file('slug2') - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') self._add_notes_file('slug3') - self._run_git('tag', '-s', '-m', 'first tag', '3.0.0') - self._run_git('checkout', '2.0.0') - self._run_git('branch', 'not-master') - self._run_git('checkout', 'master') + self.repo.git('tag', '-s', '-m', 'first tag', '3.0.0') + self.repo.git('checkout', '2.0.0') + self.repo.git('branch', 'not-master') + self.repo.git('checkout', 'master') self.scanner = scanner.Scanner(self.c) def test_current_branch_no_extra_commits(self): # checkout the branch and then ask for its base - self._run_git('checkout', 'not-master') + self.repo.git('checkout', 'not-master') self.assertEqual( '2.0.0', self.scanner._get_branch_base('not-master'), @@ -898,7 +908,7 @@ class BranchBaseTest(Base): def test_current_branch_extra_commit(self): # checkout the branch and then ask for its base - self._run_git('checkout', 'not-master') + self.repo.git('checkout', 'not-master') self._add_notes_file('slug4') self.assertEqual( '2.0.0', @@ -907,7 +917,7 @@ class BranchBaseTest(Base): def test_alternate_branch_no_extra_commits(self): # checkout master and then ask for the alternate branch base - self._run_git('checkout', 'master') + self.repo.git('checkout', 'master') self.assertEqual( '2.0.0', self.scanner._get_branch_base('not-master'), @@ -915,9 +925,9 @@ class BranchBaseTest(Base): def test_alternate_branch_extra_commit(self): # checkout master and then ask for the alternate branch base - self._run_git('checkout', 'not-master') + self.repo.git('checkout', 'not-master') self._add_notes_file('slug4') - self._run_git('checkout', 'master') + self.repo.git('checkout', 'master') self.assertEqual( '2.0.0', self.scanner._get_branch_base('not-master'), @@ -930,17 +940,17 @@ class BranchTest(Base): super(BranchTest, self).setUp() self._make_python_package() self.f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') self.f2 = self._add_notes_file('slug2') - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') self._add_notes_file('slug3') - self._run_git('tag', '-s', '-m', 'first tag', '3.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '3.0.0') def test_files_current_branch(self): - self._run_git('checkout', '2.0.0') - self._run_git('checkout', '-b', 'stable/2') + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'stable/2') f21 = self._add_notes_file('slug21') - log_text = self._run_git('log', '--decorate') + log_text = self.repo.git('log', '--decorate') self.addDetail('git log', text_content(log_text)) self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() @@ -958,11 +968,11 @@ class BranchTest(Base): ) def test_files_stable_from_master(self): - self._run_git('checkout', '2.0.0') - self._run_git('checkout', '-b', 'stable/2') + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'stable/2') f21 = self._add_notes_file('slug21') - self._run_git('checkout', 'master') - log_text = self._run_git('log', '--pretty=%x00%H %d', '--name-only', + self.repo.git('checkout', 'master') + log_text = self.repo.git('log', '--pretty=%x00%H %d', '--name-only', 'stable/2') self.addDetail('git log', text_content(log_text)) self.c.override( @@ -983,11 +993,11 @@ class BranchTest(Base): ) def test_files_stable_from_master_no_stop_base(self): - self._run_git('checkout', '2.0.0') - self._run_git('checkout', '-b', 'stable/2') + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'stable/2') f21 = self._add_notes_file('slug21') - self._run_git('checkout', 'master') - log_text = self._run_git('log', '--pretty=%x00%H %d', '--name-only', + self.repo.git('checkout', 'master') + log_text = self.repo.git('log', '--pretty=%x00%H %d', '--name-only', 'stable/2') self.addDetail('git log', text_content(log_text)) self.c.override( @@ -1013,20 +1023,20 @@ class BranchTest(Base): def test_pre_release_branch_no_collapse(self): f4 = self._add_notes_file('slug4') - self._run_git('tag', '-s', '-m', 'pre-release', '4.0.0.0rc1') + self.repo.git('tag', '-s', '-m', 'pre-release', '4.0.0.0rc1') # Add a commit on master after the tag self._add_notes_file('slug5') # Move back to the tag and create the branch - self._run_git('checkout', '4.0.0.0rc1') - self._run_git('checkout', '-b', 'stable/4') + self.repo.git('checkout', '4.0.0.0rc1') + self.repo.git('checkout', '-b', 'stable/4') # Create a commit on the branch f41 = self._add_notes_file('slug41') - log_text = self._run_git( + log_text = self.repo.git( 'log', '--pretty=%x00%H %d', '--name-only', '--graph', '--all', '--decorate', ) self.addDetail('git log', text_content(log_text)) - rev_list = self._run_git('rev-list', '--first-parent', + rev_list = self.repo.git('rev-list', '--first-parent', '^stable/4', 'master') self.addDetail('rev-list', text_content(rev_list)) self.c.override( @@ -1049,21 +1059,21 @@ class BranchTest(Base): def test_pre_release_branch_collapse(self): f4 = self._add_notes_file('slug4') - self._run_git('tag', '-s', '-m', 'pre-release', '4.0.0.0rc1') + self.repo.git('tag', '-s', '-m', 'pre-release', '4.0.0.0rc1') # Add a commit on master after the tag self._add_notes_file('slug5') # Move back to the tag and create the branch - self._run_git('checkout', '4.0.0.0rc1') - self._run_git('checkout', '-b', 'stable/4') + self.repo.git('checkout', '4.0.0.0rc1') + self.repo.git('checkout', '-b', 'stable/4') # Create a commit on the branch f41 = self._add_notes_file('slug41') - self._run_git('tag', '-s', '-m', 'release', '4.0.0') - log_text = self._run_git( + self.repo.git('tag', '-s', '-m', 'release', '4.0.0') + log_text = self.repo.git( 'log', '--pretty=%x00%H %d', '--name-only', '--graph', '--all', '--decorate', ) self.addDetail('git log', text_content(log_text)) - rev_list = self._run_git('rev-list', '--first-parent', + rev_list = self.repo.git('rev-list', '--first-parent', '^stable/4', 'master') self.addDetail('rev-list', text_content(rev_list)) self.c.override( @@ -1085,20 +1095,20 @@ class BranchTest(Base): def test_full_release_branch(self): f4 = self._add_notes_file('slug4') - self._run_git('tag', '-s', '-m', 'release', '4.0.0') + self.repo.git('tag', '-s', '-m', 'release', '4.0.0') # Add a commit on master after the tag self._add_notes_file('slug5') # Move back to the tag and create the branch - self._run_git('checkout', '4.0.0') - self._run_git('checkout', '-b', 'stable/4') + self.repo.git('checkout', '4.0.0') + self.repo.git('checkout', '-b', 'stable/4') # Create a commit on the branch f41 = self._add_notes_file('slug41') - log_text = self._run_git( + log_text = self.repo.git( 'log', '--pretty=%x00%H %d', '--name-only', '--graph', '--all', '--decorate', ) self.addDetail('git log', text_content(log_text)) - rev_list = self._run_git('rev-list', '--first-parent', + rev_list = self.repo.git('rev-list', '--first-parent', '^stable/4', 'master') self.addDetail('rev-list', text_content(rev_list)) self.c.override( @@ -1122,17 +1132,17 @@ class BranchTest(Base): # We have branched from master, but not added any commits to # master. f4 = self._add_notes_file('slug4') - self._run_git('tag', '-s', '-m', 'release', '4.0.0') - self._run_git('checkout', '-b', 'stable/4') + self.repo.git('tag', '-s', '-m', 'release', '4.0.0') + self.repo.git('checkout', '-b', 'stable/4') # Create a commit on the branch f41 = self._add_notes_file('slug41') f42 = self._add_notes_file('slug42') - log_text = self._run_git( + log_text = self.repo.git( 'log', '--pretty=%x00%H %d', '--name-only', '--graph', '--all', '--decorate', ) self.addDetail('git log', text_content(log_text)) - rev_list = self._run_git('rev-list', '--first-parent', + rev_list = self.repo.git('rev-list', '--first-parent', '^stable/4', 'master') self.addDetail('rev-list', text_content(rev_list)) self.c.override( @@ -1156,15 +1166,15 @@ class BranchTest(Base): # We have branched from master, but not added any commits to # our branch or to master. f4 = self._add_notes_file('slug4') - self._run_git('tag', '-s', '-m', 'release', '4.0.0') - self._run_git('checkout', '-b', 'stable/4') + self.repo.git('tag', '-s', '-m', 'release', '4.0.0') + self.repo.git('checkout', '-b', 'stable/4') # Create a commit on the branch - log_text = self._run_git( + log_text = self.repo.git( 'log', '--pretty=%x00%H %d', '--name-only', '--graph', '--all', '--decorate', ) self.addDetail('git log', text_content(log_text)) - rev_list = self._run_git('rev-list', '--first-parent', + rev_list = self.repo.git('rev-list', '--first-parent', '^stable/4', 'master') self.addDetail('rev-list', text_content(rev_list)) self.c.override( @@ -1184,9 +1194,9 @@ class BranchTest(Base): ) def test_remote_branches(self): - self._run_git('checkout', '2.0.0') - self._run_git('checkout', '-b', 'stable/2') - self._run_git('checkout', 'master') + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'stable/2') + self.repo.git('checkout', 'master') scanner1 = scanner.Scanner(self.c) head1 = scanner1._get_branch_head('stable/2') self.assertIsNotNone(head1) @@ -1226,11 +1236,11 @@ class TagsTest(Base): super(TagsTest, self).setUp() self._make_python_package() self.f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') self.f2 = self._add_notes_file('slug2') - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') self._add_notes_file('slug3') - self._run_git('tag', '-s', '-m', 'first tag', '3.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '3.0.0') def test_master(self): self.scanner = scanner.Scanner(self.c) @@ -1241,11 +1251,11 @@ class TagsTest(Base): ) def test_not_master(self): - self._run_git('checkout', '2.0.0') - self._run_git('checkout', '-b', 'not-master') + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'not-master') self._add_notes_file('slug4') - self._run_git('tag', '-s', '-m', 'not on master', '2.0.1') - self._run_git('checkout', 'master') + self.repo.git('tag', '-s', '-m', 'not on master', '2.0.1') + self.repo.git('checkout', 'master') self.scanner = scanner.Scanner(self.c) results = self.scanner._get_tags_on_branch('not-master') self.assertEqual( @@ -1255,7 +1265,7 @@ class TagsTest(Base): def test_unsigned(self): self._add_notes_file('slug4') - self._run_git('tag', '-m', 'first tag', '4.0.0') + self.repo.git('tag', '-m', 'first tag', '4.0.0') self.scanner = scanner.Scanner(self.c) results = self.scanner._get_tags_on_branch(None) self.assertEqual( @@ -1270,11 +1280,11 @@ class VersionTest(Base): super(VersionTest, self).setUp() self._make_python_package() self.f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') self.f2 = self._add_notes_file('slug2') - self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.0') self._add_notes_file('slug3') - self._run_git('tag', '-s', '-m', 'third tag', '3.0.0') + self.repo.git('tag', '-s', '-m', 'third tag', '3.0.0') def test_tagged_head(self): self.scanner = scanner.Scanner(self.c) @@ -1299,7 +1309,7 @@ class VersionTest(Base): # unlikely that anything could apply 2 signed tags within a # single second (certainly not a person). time.sleep(1) - self._run_git('tag', '-s', '-m', 'fourth tag', '4.0.0') + self.repo.git('tag', '-s', '-m', 'fourth tag', '4.0.0') self.scanner = scanner.Scanner(self.c) results = self.scanner._get_current_version(None) self.assertEqual( -- GitLab From 1ead95dba8e29c9ace21477bda83e11f4cf6606a Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 14 Dec 2016 09:50:05 -0500 Subject: [PATCH 095/257] support removed stable branches OpenStack's workflow includes a step to remove a stable branch when the version is marked for end-of-life. At that point we add a $name-eol tag to replace the branch. Update the reference searching logic to support this to avoid breaking release notes builds at that point. Change-Id: I49734fbf4c3499e148367c9b6dd6f4f49ba17956 Signed-off-by: Doug Hellmann --- reno/scanner.py | 33 ++++++++++++++++++++++----------- reno/tests/test_scanner.py | 10 ++++++++-- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 271919e..3b755d9 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -307,21 +307,32 @@ class Scanner(object): self.reporoot = self.conf.reporoot self._repo = RenoRepo(self.reporoot) - def _get_branch_head(self, branch): - if branch: + def _get_ref(self, name): + if name: candidates = [ - b'refs/heads/' + branch.encode('utf-8'), - b'refs/remotes/' + branch.encode('utf-8'), + 'refs/heads/' + name, + 'refs/remotes/' + name, + 'refs/tags/' + name, + # If a stable branch was removed, look for its EOL tag. + 'refs/tags/' + (name.rpartition('/')[-1] + '-eol'), ] - for branch_ref in candidates: - if branch_ref in self._repo.refs: - return self._repo.refs[branch_ref] + for ref in candidates: + key = ref.encode('utf-8') + if key in self._repo.refs: + sha = self._repo.refs[key] + o = self._repo[sha] + if isinstance(o, objects.Tag): + # Branches point directly to commits, but + # signed tags point to the signature and we + # need to dereference it to get to the commit. + sha = o.object[1] + return sha # If we end up here we didn't find any of the candidates. - raise ValueError('Unknown branch {!r}'.format(branch)) + raise ValueError('Unknown reference {!r}'.format(name)) return self._repo.refs[b'HEAD'] def _get_walker_for_branch(self, branch): - branch_head = self._get_branch_head(branch) + branch_head = self._get_ref(branch) return self._repo.get_walker(branch_head) def _get_tags_on_branch(self, branch): @@ -341,7 +352,7 @@ class Scanner(object): # counts up to where the tag appears and it returns when it # finds the first tagged commit (there is no need to scan the # rest of the branch). - commit = self._repo[self._get_branch_head(branch)] + commit = self._repo[self._get_ref(branch)] count = 0 while commit: # shas_to_tags has encoded versions of the shas @@ -403,7 +414,7 @@ class Scanner(object): # * a7f573d original commit on master """ - head = self._get_branch_head(branch) + head = self._get_ref(branch) # Map SHA values to Entry objects, because we will be traversing # commits not entries. diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 364d18d..326eebd 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1198,7 +1198,7 @@ class BranchTest(Base): self.repo.git('checkout', '-b', 'stable/2') self.repo.git('checkout', 'master') scanner1 = scanner.Scanner(self.c) - head1 = scanner1._get_branch_head('stable/2') + head1 = scanner1._get_ref('stable/2') self.assertIsNotNone(head1) print('head1', head1) # Create a second repository by cloning the first. @@ -1225,7 +1225,7 @@ class BranchTest(Base): )) c2 = config.Config(reporoot2) scanner2 = scanner.Scanner(c2) - head2 = scanner2._get_branch_head('origin/stable/2') + head2 = scanner2._get_ref('origin/stable/2') self.assertIsNotNone(head2) self.assertEqual(head1, head2) @@ -1250,6 +1250,12 @@ class TagsTest(Base): results, ) + def test_get_ref(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('3.0.0') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + def test_not_master(self): self.repo.git('checkout', '2.0.0') self.repo.git('checkout', '-b', 'not-master') -- GitLab From b633b5ec45c4bfb4680ea7e2e6c03d9b7d05daa0 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 14 Dec 2016 10:25:19 -0500 Subject: [PATCH 096/257] add more tests for _get_ref Change-Id: I428d117609ac6658dfbc568d1c67f4cb1191d8db Signed-off-by: Doug Hellmann --- reno/tests/test_scanner.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 326eebd..19dd88f 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1230,6 +1230,41 @@ class BranchTest(Base): self.assertEqual(head1, head2) +class GetRefTest(Base): + + def setUp(self): + super(GetRefTest, self).setUp() + self._make_python_package() + self.f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('branch', 'stable/foo') + self.repo.git('tag', 'bar-eol') + + def test_signed_tag(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('1.0.0') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_unsigned_tag(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('bar-eol') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_eol_tag_from_branch(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('stable/bar') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_head(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref(None) + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + class TagsTest(Base): def setUp(self): -- GitLab From 389d4672c8bab9197e9c1a6e429d4eb7d1f0849f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 14 Dec 2016 11:17:43 -0500 Subject: [PATCH 097/257] update release notes Add notes for the dulwich work and the eol handling. Clean up some existing notes that added prelude values incorrectly. Change-Id: Id37f2f7ce8e2ce9723e5e8c3ecbebe227b62f4e2 Signed-off-by: Doug Hellmann --- .../notes/add-config-file-e77084792c1dc695.yaml | 2 -- .../notes/branches-eol-bcafc2a007a1eb9f.yaml | 12 ++++++++++++ .../notes/dulwich-rewrite-3a5377162d97402b.yaml | 5 +++++ .../support-custom-template-0534a2199cfec44c.yaml | 2 -- .../notes/support-edit-ec5c01ad6144815a.yaml | 4 +--- 5 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/branches-eol-bcafc2a007a1eb9f.yaml create mode 100644 releasenotes/notes/dulwich-rewrite-3a5377162d97402b.yaml diff --git a/releasenotes/notes/add-config-file-e77084792c1dc695.yaml b/releasenotes/notes/add-config-file-e77084792c1dc695.yaml index b6787df..32ec204 100644 --- a/releasenotes/notes/add-config-file-e77084792c1dc695.yaml +++ b/releasenotes/notes/add-config-file-e77084792c1dc695.yaml @@ -1,6 +1,4 @@ --- -prelude: > - Reno now supports having a configuration file! features: - | Reno now supports having a ``config.yaml`` file in your release notes diff --git a/releasenotes/notes/branches-eol-bcafc2a007a1eb9f.yaml b/releasenotes/notes/branches-eol-bcafc2a007a1eb9f.yaml new file mode 100644 index 0000000..0a852d9 --- /dev/null +++ b/releasenotes/notes/branches-eol-bcafc2a007a1eb9f.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Explicitly allow reno to scan starting from a tag by specifying the + tag where a branch name would otherwise be used. + - | + Add logic to allow reno to detect a branch that has been marked as + end-of-life using the OpenStack community's process of tagging the + HEAD of a stable/foo branch foo-eol before deleting the + branch. This means that references to "stable/foo" are translated + to "foo-eol" when the branch does not exist, and that Sphinx + directives do not need to be manually updated. \ No newline at end of file diff --git a/releasenotes/notes/dulwich-rewrite-3a5377162d97402b.yaml b/releasenotes/notes/dulwich-rewrite-3a5377162d97402b.yaml new file mode 100644 index 0000000..d29bd26 --- /dev/null +++ b/releasenotes/notes/dulwich-rewrite-3a5377162d97402b.yaml @@ -0,0 +1,5 @@ +--- +prelude: > + This release includes a significant rewrite of the internal logic of + reno to access git data through the dulwich library instead of the + git command line porcelain. diff --git a/releasenotes/notes/support-custom-template-0534a2199cfec44c.yaml b/releasenotes/notes/support-custom-template-0534a2199cfec44c.yaml index b47e859..cb3bed8 100644 --- a/releasenotes/notes/support-custom-template-0534a2199cfec44c.yaml +++ b/releasenotes/notes/support-custom-template-0534a2199cfec44c.yaml @@ -1,6 +1,4 @@ --- -prelude: > - Support to set a custom template used to create new notes features: - | Reno now supports to set through ``template`` attribute in diff --git a/releasenotes/notes/support-edit-ec5c01ad6144815a.yaml b/releasenotes/notes/support-edit-ec5c01ad6144815a.yaml index 48753c8..2997e1e 100644 --- a/releasenotes/notes/support-edit-ec5c01ad6144815a.yaml +++ b/releasenotes/notes/support-edit-ec5c01ad6144815a.yaml @@ -1,7 +1,5 @@ --- -prelude: > - Enable to create and edit a note with reno new --edit. features: - | - Reno now enables with reno new --edit to create a note and edit it with + Reno now enables with reno new ``--edit`` to create a note and edit it with your editor (defined with EDITOR environment variable). -- GitLab From 652b116c1dc38f15cf523362f5bae3ad9067df2f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 14 Dec 2016 13:19:59 -0500 Subject: [PATCH 098/257] reconfigure release notes display to include newton Now that we have a stable branch, we need to adjust the way we show release notes to include the information from that branch. Change-Id: I88157b3d401103df76d7b667069fc8b965442bf8 Signed-off-by: Doug Hellmann --- doc/source/history.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/source/history.rst b/doc/source/history.rst index ec77acc..7427aa4 100644 --- a/doc/source/history.rst +++ b/doc/source/history.rst @@ -1 +1,8 @@ -.. release-notes:: Release Notes +=============== + Release Notes +=============== + +.. release-notes:: Mainline + +.. release-notes:: Newton Series + :branch: origin/stable/newton -- GitLab From 8c2e769d7e5abe0e3ecb9e02433caaf54a54758c Mon Sep 17 00:00:00 2001 From: Flavio Percoco Date: Thu, 24 Nov 2016 11:59:03 +0100 Subject: [PATCH 099/257] Show team and repo badges on README This patch adds the team's and repository's badges to the README file. The motivation behind this is to communicate the project status and features at first glance. For more information about this effort, please read this email thread: http://lists.openstack.org/pipermail/openstack-dev/2016-October/105562.html To see an example of how this would look like check: https://gist.github.com/c7736667911fe912a3d8118fb28e8667 Change-Id: I7587e825a0be26097b143ce31786b7b83cfabbd7 --- README.rst | 3 +++ setup.cfg | 1 + 2 files changed, 4 insertions(+) diff --git a/README.rst b/README.rst index ba6bc88..6b6a411 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,9 @@ Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. +.. image:: http://governance.openstack.org/badges/reno.svg + :target: http://governance.openstack.org/reference/tags/index.html + Project Meta-data ================= diff --git a/setup.cfg b/setup.cfg index 98333b1..09187ec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ console_scripts = [extras] sphinx = sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 + docutils>=0.11,!=0.13.1 # OSI-Approved Open Source, Public Domain [build_sphinx] source-dir = doc/source -- GitLab From 74ae4e686cfffc3b9089dac4b5b6a9e36701225b Mon Sep 17 00:00:00 2001 From: git-harry Date: Thu, 15 Dec 2016 16:57:46 +0000 Subject: [PATCH 100/257] Add support for tags tagging other tags _load_tags builds a mapping, _shas_to_tags, of all the tags on a repo. The SHAs in question should correspond to commits. When an annotated tag is used to tag another annotated tag, _shas_to_tags is updated with the SHA of the tagged tag and not the commit at the end of the chain. This change modifies _load_tags so that if a tag references another tag, the chain of tags is followed until it is exhausted. Change-Id: Ifd970b6c7884169ac1272b84f5c1a1b056230f50 --- reno/scanner.py | 56 ++++++++++++++++++++++++-------------- reno/tests/test_scanner.py | 20 ++++++++++++++ 2 files changed, 56 insertions(+), 20 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 3b755d9..30fe23b 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -204,6 +204,41 @@ class RenoRepo(repo.Repo): _all_tags = None _shas_to_tags = None + def _get_commit_from_tag(self, tag, tag_sha): + """Return the commit referenced by the tag and when it was tagged.""" + tag_obj = self[tag_sha] + + if isinstance(tag_obj, objects.Tag): + # A signed tag has its own SHA, but the tag refers to + # the commit and that's the SHA we'll see when we scan + # commits on a branch. + git_obj = tag_obj + while True: + # Tags can point to other tags, in such cases follow the chain + # of tags until there are no more. + child_obj = self[git_obj.object[1]] + if isinstance(child_obj, objects.Tag): + git_obj = child_obj + else: + break + + tagged_sha = git_obj.object[1] + date = tag_obj.tag_time + elif isinstance(tag_obj, objects.Commit): + # Unsigned tags refer directly to commits. This seems + # to especially happen when the tag definition moves + # to the packed-refs list instead of being represented + # by its own file. + tagged_sha = tag_obj.id + date = tag_obj.commit_time + else: + raise ValueError( + ('Unrecognized tag object {!r} with ' + 'tag {} and SHA {!r}: {}').format( + tag_obj, tag, tag_sha, type(tag_obj)) + ) + return tagged_sha, date + def _load_tags(self): self._all_tags = { k.partition(b'/tags/')[-1].decode('utf-8'): v @@ -212,26 +247,7 @@ class RenoRepo(repo.Repo): } self._shas_to_tags = {} for tag, tag_sha in self._all_tags.items(): - tag_obj = self[tag_sha] - if isinstance(tag_obj, objects.Tag): - # A signed tag has its own SHA, but the tag refers to - # the commit and that's the SHA we'll see when we scan - # commits on a branch. - tagged_sha = tag_obj.object[1] - date = tag_obj.tag_time - elif isinstance(tag_obj, objects.Commit): - # Unsigned tags refer directly to commits. This seems - # to especially happen when the tag definition moves - # to the packed-refs list instead of being represented - # by its own file. - tagged_sha = tag_obj.id - date = tag_obj.commit_time - else: - raise ValueError( - ('Unrecognized tag object {!r} with ' - 'tag {} and SHA {!r}: {}').format( - tag_obj, tag, tag_sha, type(tag_obj)) - ) + tagged_sha, date = self._get_commit_from_tag(tag, tag_sha) self._shas_to_tags.setdefault(tagged_sha, []).append((tag, date)) def get_tags_on_commit(self, sha): diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 19dd88f..11b8564 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1314,6 +1314,26 @@ class TagsTest(Base): results, ) + def test_tagged_tag_annotated(self): + time.sleep(1) + self.repo.git('tag', '-s', '-m', 'fourth tag', '4.0.0', '3.0.0') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch(None) + self.assertEqual( + ['3.0.0', '4.0.0', '2.0.0', '1.0.0'], + results, + ) + + def test_tagged_tag_lightweight(self): + time.sleep(1) + self.repo.git('tag', '-m', 'fourth tag', '4.0.0', '3.0.0') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch(None) + self.assertEqual( + ['3.0.0', '4.0.0', '2.0.0', '1.0.0'], + results, + ) + class VersionTest(Base): -- GitLab From fc5ac7f495740af382411bcb9737bf03cb256491 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 15 Dec 2016 12:09:28 -0500 Subject: [PATCH 101/257] use dulwich's tree traversal to look up repo contents dulwich.objects.Tree has a method lookup_path() that returns the permissions and sha associated with a name of a thing under the Tree (either another Tree instance or a Blob containing a file). This change replaces the logic we have in our RenoRepo subclass for traversing Trees with calls to lookup_path(), while still retaining separate methods for handling the return value appropriately based on context. Change-Id: I29a91fa31313918e3dee9e41340b21a55cb8fa90 Signed-off-by: Doug Hellmann --- reno/scanner.py | 54 ++++++++++++++----------------------------------- 1 file changed, 15 insertions(+), 39 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 3b755d9..ce04743 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -245,48 +245,15 @@ class RenoRepo(repo.Repo): def _get_subtree(self, tree, path): "Given a tree SHA and a path, return the SHA of the subtree." try: - if os.sep in path: - # The tree entry will only have a single level of the - # directory name, so if we have a / in our filename we - # know we're going to have to keep traversing the - # tree. - prefix, _, trailing = path.partition(os.sep) - mode, subtree_sha = tree[prefix.encode('utf-8')] - subtree = self[subtree_sha] - return self._get_subtree(subtree, trailing) - else: - # The tree entry will point to the SHA of the contents - # of the subtree. - mode, sha = tree[path.encode('utf-8')] - result = self[sha] - return result + mode, tree_sha = tree.lookup_path(self.get_object, + path.encode('utf-8')) except KeyError: # Some part of the path wasn't found, so the subtree is # not present. Return the sentinel value. return None - - def _get_file_from_tree(self, filename, tree): - "Given a tree object, traverse it to find the file." - try: - if os.sep in filename: - # The tree entry will only have a single level of the - # directory name, so if we have a / in our filename we - # know we're going to have to keep traversing the - # tree. - prefix, _, trailing = filename.partition(os.sep) - mode, subtree_sha = tree[prefix.encode('utf-8')] - subtree = self[subtree_sha] - return self._get_file_from_tree(trailing, subtree) - else: - # The tree entry will point to the blob with the - # contents of the file. - mode, file_blob_sha = tree[filename.encode('utf-8')] - file_blob = self[file_blob_sha] - return file_blob.data - except KeyError: - # Some part of the filename wasn't found, so the file is - # not present. Return the sentinel value. - return None + else: + tree = self[tree_sha] + return tree def get_file_at_commit(self, filename, sha): "Return the contents of the file if it exists at the commit, or None." @@ -297,7 +264,16 @@ class RenoRepo(repo.Repo): # the repository. commit = self[sha.encode('ascii')] tree = self[commit.tree] - return self._get_file_from_tree(filename, tree) + try: + mode, blob_sha = tree.lookup_path(self.get_object, + filename.encode('utf-8')) + except KeyError: + # Some part of the filename wasn't found, so the file is + # not present. Return the sentinel value. + return None + else: + blob = self[blob_sha] + return blob.data class Scanner(object): -- GitLab From 51986ae2b94d38ec53eeab180f9ff56c7ac6ff1e Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 22 Dec 2016 09:08:21 -0500 Subject: [PATCH 102/257] fix a problem scanning for the base of a branch with no tag When there is no tag at the base of a branch, the scanner cannot figure out where to stop scanning. Avoid an exception and allow the scanner to continue to the end of the history. Change-Id: I9bed42724d5ab0e8d11ab0c781e215a71af5e99a Signed-off-by: Doug Hellmann --- reno/scanner.py | 10 +++++++++- reno/tests/test_scanner.py | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/reno/scanner.py b/reno/scanner.py index 949c6ab..75c2fac 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -385,7 +385,15 @@ class Scanner(object): # on master, so this is the base. tags = self._repo.get_tags_on_commit( c.commit.sha().hexdigest().encode('ascii')) - return tags[-1] + if tags: + return tags[-1] + else: + # Naughty, naughty, branching without tagging. + LOG.error( + ('There is no tag on commit %s at the base of %s. ' + 'Branch scan short-cutting is disabled.'), + c.commit.sha().hexdigest(), branch) + return None return None def _topo_traversal(self, branch): diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 11b8564..62c4fa8 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -933,6 +933,15 @@ class BranchBaseTest(Base): self.scanner._get_branch_base('not-master'), ) + def test_no_tag_at_base(self): + # remove the tag at the branch point + self.repo.git('tag', '-d', '2.0.0') + self._add_notes_file('slug4') + self.repo.git('checkout', 'master') + self.assertIsNone( + self.scanner._get_branch_base('not-master') + ) + class BranchTest(Base): -- GitLab From 3b2062d0eb01038b54afea3c80259ea62261c3ea Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 22 Dec 2016 11:55:12 -0500 Subject: [PATCH 103/257] fix the logic for determining where to stop scanning a branch We don't want to stop *at* the tag at the base, we want to stop at the *version* before that tag, which may actually be several tags back in history. Closes-bug: #1652092 Change-Id: I107651fce3fcb06d9f155e50ae4decd905fa932d Signed-off-by: Doug Hellmann --- ...ranch-base-detection-95300805f26a0c15.yaml | 7 ++ reno/scanner.py | 76 ++++++++++-- reno/tests/test_scanner.py | 109 ++++++++++++++++++ 3 files changed, 183 insertions(+), 9 deletions(-) create mode 100644 releasenotes/notes/fix-branch-base-detection-95300805f26a0c15.yaml diff --git a/releasenotes/notes/fix-branch-base-detection-95300805f26a0c15.yaml b/releasenotes/notes/fix-branch-base-detection-95300805f26a0c15.yaml new file mode 100644 index 0000000..10eaf58 --- /dev/null +++ b/releasenotes/notes/fix-branch-base-detection-95300805f26a0c15.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fix a problem with the way reno automatically detects the initial + version in a branch that prevented it from including all of the + notes associated with a release, especially if the branch was + created at a pre-release version number. Bug #1652092 \ No newline at end of file diff --git a/reno/scanner.py b/reno/scanner.py index 75c2fac..7a5b5fb 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -31,6 +31,17 @@ PRE_RELEASE_RE = re.compile(''' ''', flags=re.VERBOSE | re.UNICODE) +def _parse_version(v): + parts = v.split('.') + ['0', '0', '0'] + result = [] + for p in parts[:3]: + try: + result.append(int(p)) + except ValueError: + result.append(p) + return result + + def _get_unique_id(filename): base = os.path.basename(filename) root, ext = os.path.splitext(base) @@ -497,6 +508,43 @@ class Scanner(object): "Return true if the file exists at the given commit." return bool(self.get_file_at_commit(filename, sha)) + @staticmethod + def _find_scan_stop_point(earliest_version, versions_by_date, + collapse_pre_releases): + """Return the version to use to stop the scan. + + Use the list of versions_by_date to get the tag with a + different version created *before* the branch to ensure that + we include notes that go with that version that *is* in the + branch. + + :param earliest_version: Version string of the earliest + version to be included in the output. + :param versions_by_date: List of version strings in reverse + chronological order. + :param collapse_pre_releases: Boolean indicating whether we are + collapsing pre-releases or not. If false, the next tag + is used, regardless of its version. + + """ + earliest_parts = _parse_version(earliest_version) + try: + idx = versions_by_date.index(earliest_version) + 1 + except ValueError: + # The version we were given is not present, use a full + # scan. + return None + if not collapse_pre_releases: + # We just take the next tag. + return versions_by_date[idx] + # We need to look for a different version. + for candidate in versions_by_date[idx:]: + parts = _parse_version(candidate) + if parts != earliest_parts: + # The candidate is a different version, use it. + return candidate + return None + def get_notes_by_version(self): """Return an OrderedDict mapping versions to lists of notes files. @@ -533,16 +581,23 @@ class Scanner(object): # If the user has told us where to stop, use that as the # default. - branch_base_tag = earliest_version + if earliest_version: + scan_stop_tag = self._find_scan_stop_point( + earliest_version, versions_by_date, collapse_pre_releases) + else: + scan_stop_tag = None - # If the user has not told us where to stop, try to work it out - # for ourselves. If branch is set and is not "master", then we - # want to stop at the base of the branch. + # If the user has not told us where to stop, try to work it + # out for ourselves. If branch is set and is not "master", + # then we want to stop at the version before the tag at the + # base of the branch, which involves a bit of searching. if (stop_at_branch_base and (not earliest_version) and branch and (branch != 'master')): LOG.debug('determining earliest_version from branch') earliest_version = self._get_branch_base(branch) - branch_base_tag = earliest_version + scan_stop_tag = self._find_scan_stop_point( + earliest_version, versions_by_date, + collapse_pre_releases) if earliest_version and collapse_pre_releases: if PRE_RELEASE_RE.search(earliest_version): # The earliest version won't actually be the pre-release @@ -556,8 +611,8 @@ class Scanner(object): LOG.info('earliest version to include is %s', earliest_version) else: LOG.info('including entire branch history') - if branch_base_tag: - LOG.info('stopping scan at %s', branch_base_tag) + if scan_stop_tag: + LOG.info('stopping scan at %s', scan_stop_tag) versions = [] earliest_seen = collections.OrderedDict() @@ -713,8 +768,11 @@ class Scanner(object): 'unknown change instructions {!r}'.format(change) ) - if branch_base_tag and branch_base_tag in tags: - LOG.info('reached end of branch after %d commits', counter) + if scan_stop_tag and scan_stop_tag in tags: + LOG.info( + ('reached end of branch after %d commits at %s ' + 'with tags %s'), + counter, sha, tags) break # Invert earliest_seen to make a list of notes files for each diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 62c4fa8..46713fc 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1102,6 +1102,44 @@ class BranchTest(Base): results, ) + def test_pre_release_note_before_branch(self): + f4 = self._add_notes_file('slug4') + self.repo.git('tag', '-s', '-m', 'beta', '4.0.0.0b1') + self.repo.add_file('not-a-release-note.txt') + self.repo.git('tag', '-s', '-m', 'pre-release', '4.0.0.0rc1') + # Add a commit on master after the tag + self._add_notes_file('slug5') + # Move back to the tag and create the branch + self.repo.git('checkout', '4.0.0.0rc1') + self.repo.git('checkout', '-b', 'stable/4') + # Create a commit on the branch + f41 = self._add_notes_file('slug41') + self.repo.git('tag', '-s', '-m', 'release', '4.0.0') + log_text = self.repo.git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self.repo.git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + collapse_pre_releases=True, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0': [f4, f41], + }, + results, + ) + def test_full_release_branch(self): f4 = self._add_notes_file('slug4') self.repo.git('tag', '-s', '-m', 'release', '4.0.0') @@ -1239,6 +1277,77 @@ class BranchTest(Base): self.assertEqual(head1, head2) +class ScanStopPointTest(Base): + + def setUp(self): + super(ScanStopPointTest, self).setUp() + self.scanner = scanner.Scanner(self.c) + + def test_invalid_earliest_version(self): + self.assertIsNone( + self.scanner._find_scan_stop_point( + 'not.a.numeric.version', [], True), + ) + + def test_unknown_version(self): + self.assertIsNone( + self.scanner._find_scan_stop_point( + '1.0.0', [], True), + ) + + def test_only_version(self): + self.assertIsNone( + self.scanner._find_scan_stop_point( + '1.0.0', ['1.0.0'], True), + ) + + def test_beta_collapse(self): + self.assertEqual( + '1.0.0', + self.scanner._find_scan_stop_point( + '2.0.0.0b1', ['2.0.0', '2.0.0.0rc1', '2.0.0.0b1', '1.0.0'], + True), + ) + + def test_rc_collapse(self): + self.assertEqual( + '1.0.0', + self.scanner._find_scan_stop_point( + '2.0.0.0rc1', ['2.0.0', '2.0.0.0rc1', '2.0.0.0b1', '1.0.0'], + True), + ) + + def test_rc_no_collapse(self): + self.assertEqual( + '2.0.0.0b1', + self.scanner._find_scan_stop_point( + '2.0.0.0rc1', ['2.0.0', '2.0.0.0rc1', '2.0.0.0b1', '1.0.0'], + False), + ) + + def test_nova_newton(self): + self.assertEqual( + '13.0.0.0rc3', + self.scanner._find_scan_stop_point( + '14.0.0', + [u'14.0.3', u'14.0.2', u'14.0.1', u'14.0.0.0rc2', + u'14.0.0', u'14.0.0.0rc1', u'14.0.0.0b3', u'14.0.0.0b2', + u'14.0.0.0b1', u'13.0.0.0rc3', u'13.0.0', u'13.0.0.0rc2', + u'13.0.0.0rc1', u'13.0.0.0b3', u'13.0.0.0b2', u'13.0.0.0b1', + u'12.0.0.0rc3', u'12.0.0', u'12.0.0.0rc2', u'12.0.0.0rc1', + u'12.0.0.0b3', u'12.0.0.0b2', u'12.0.0.0b1', u'12.0.0a0', + u'2015.1.0rc3', u'2015.1.0', u'2015.1.0rc2', u'2015.1.0rc1', + u'2015.1.0b3', u'2015.1.0b2', u'2015.1.0b1', u'2014.2.rc2', + u'2014.2', u'2014.2.rc1', u'2014.2.b3', u'2014.2.b2', + u'2014.2.b1', u'2014.1.rc1', u'2014.1.b3', u'2014.1.b2', + u'2014.1.b1', u'2013.2.rc1', u'2013.2.b3', u'2013.1.rc1', + u'folsom-2', u'folsom-1', u'essex-1', u'diablo-2', + u'diablo-1', u'2011.2', u'2011.2rc1', u'2011.2gamma1', + u'2011.1rc1', u'0.9.0'], + True), + ) + + class GetRefTest(Base): def setUp(self): -- GitLab From 8756c1dd2a99e17d69013bb6cb0511f5a981c0fa Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 22 Dec 2016 13:48:02 -0500 Subject: [PATCH 104/257] tone down the warning for missing configuration file The warning logged when we can't load a configuration file looks like an error because it has "Errno" in it. It's not really an error, though, so log a more gentle message saying that the configuration file does not exist where we expect to find it. Change-Id: I30698162615b494f4ed2d23f119309e9fcdf3edd Signed-off-by: Doug Hellmann --- reno/config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/reno/config.py b/reno/config.py index b9b4637..4b06ad5 100644 --- a/reno/config.py +++ b/reno/config.py @@ -9,6 +9,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import errno import logging import os.path @@ -152,8 +153,12 @@ class Config(object): with open(self._filename, 'r') as fd: self._contents = yaml.safe_load(fd) except IOError as err: - LOG.info('did not load config file %s: %s', - self._filename, err) + if err.errno == errno.ENOENT: + LOG.info('no configuration file in %s', + self._filename) + else: + LOG.warning('did not load config file %s: %s', + self._filename, err) else: self.override(**self._contents) -- GitLab From 2f93c4dedea9edc1ffb4a96f6554469e235d75d7 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 22 Dec 2016 19:29:15 -0500 Subject: [PATCH 105/257] centralize handling of branches without base tags Move the logic for handling an empty earliest_version value to the place where that value is used to find the scanning stop point so that all entries into that code path are protected from bad input data. Change-Id: Id54f8d48dd5dff6c6744c128a258d335a4586e6b Closes-Bug: #1652178 Signed-off-by: Doug Hellmann --- reno/scanner.py | 9 ++++----- reno/tests/test_scanner.py | 6 ++++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 7a5b5fb..4e8a4b6 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -527,6 +527,8 @@ class Scanner(object): is used, regardless of its version. """ + if not earliest_version: + return None earliest_parts = _parse_version(earliest_version) try: idx = versions_by_date.index(earliest_version) + 1 @@ -581,11 +583,8 @@ class Scanner(object): # If the user has told us where to stop, use that as the # default. - if earliest_version: - scan_stop_tag = self._find_scan_stop_point( - earliest_version, versions_by_date, collapse_pre_releases) - else: - scan_stop_tag = None + scan_stop_tag = self._find_scan_stop_point( + earliest_version, versions_by_date, collapse_pre_releases) # If the user has not told us where to stop, try to work it # out for ourselves. If branch is set and is not "master", diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 46713fc..af0ab57 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1289,6 +1289,12 @@ class ScanStopPointTest(Base): 'not.a.numeric.version', [], True), ) + def test_none(self): + self.assertIsNone( + self.scanner._find_scan_stop_point( + None, [], True), + ) + def test_unknown_version(self): self.assertIsNone( self.scanner._find_scan_stop_point( -- GitLab From 0043296ec8b1bf15e265be80eb19df50ee7822fd Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 22 Dec 2016 16:46:18 -0500 Subject: [PATCH 106/257] refactor change tracking in scanner When we add support for tracking changes that haven't been committed, we will need to apply the same logic for managing the info we're tracking in several different loops. Isolate it in a class to make it easier to reuse. Change-Id: I98c90bcbc8819ce46d6335ef650c954846ec045a Signed-off-by: Doug Hellmann --- reno/scanner.py | 272 ++++++++++++++++++++++++++++-------------------- 1 file changed, 162 insertions(+), 110 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 4e8a4b6..e02ed17 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -209,6 +209,146 @@ def _aggregate_changes(walk_entry, changes, notesdir): return results +class _ChangeTracker(object): + + def __init__(self): + # Track the versions we have seen and the earliest version for + # which we have seen a given note's unique id. + self.versions = [] + self.earliest_seen = collections.OrderedDict() + # Remember the most current filename for each id, to allow for + # renames. + self.last_name_by_id = {} + # Remember uniqueids that have had files deleted. + self.uniqueids_deleted = set() + + def _common(self, uniqueid, version): + if version not in self.versions: + self.versions.append(version) + # Update the "earliest" version where a UID appears + # every time we see it, because we are scanning the + # history in reverse order so "early" items come + # later. + LOG.debug('%s: setting earliest reference to %s', + uniqueid, version) + self.earliest_seen[uniqueid] = version + + def add(self, filename, sha, version): + uniqueid = _get_unique_id(filename) + self._common(uniqueid, version) + LOG.info('%s: adding %s from %s', + uniqueid, filename, version) + + # If we have recorded that a UID was deleted, that + # means that was the last change made to the file and + # we can ignore it. + if uniqueid in self.uniqueids_deleted: + LOG.debug( + '%s: has already been deleted, ignoring this change', + uniqueid, + ) + return + + # A note is being added in this commit. If we have + # not seen it before, it was added here and never + # changed. + if uniqueid not in self.last_name_by_id: + self.last_name_by_id[uniqueid] = (filename, sha) + LOG.info( + '%s: new %s in commit %s', + uniqueid, filename, sha, + ) + else: + LOG.debug( + '%s: add for file we have already seen', + uniqueid, + ) + + def rename(self, filename, sha, version): + uniqueid = _get_unique_id(filename) + self._common(uniqueid, version) + + # If we have recorded that a UID was deleted, that + # means that was the last change made to the file and + # we can ignore it. + if uniqueid in self.uniqueids_deleted: + LOG.debug( + '%s: has already been deleted, ignoring this change', + uniqueid, + ) + return + + # The file is being renamed. We may have seen it + # before, if there were subsequent modifications, + # so only store the name information if it is not + # there already. + if uniqueid not in self.last_name_by_id: + self.last_name_by_id[uniqueid] = (filename, sha) + LOG.info( + '%s: update to %s in commit %s', + uniqueid, filename, sha, + ) + else: + LOG.debug( + '%s: renamed file already known with the new name', + uniqueid, + ) + + def modify(self, filename, sha, version): + uniqueid = _get_unique_id(filename) + self._common(uniqueid, version) + + # If we have recorded that a UID was deleted, that + # means that was the last change made to the file and + # we can ignore it. + if uniqueid in self.uniqueids_deleted: + LOG.debug( + '%s: has already been deleted, ignoring this change', + uniqueid, + ) + return + + # An existing file is being modified. We may have + # seen it before, if there were subsequent + # modifications, so only store the name + # information if it is not there already. + if uniqueid not in self.last_name_by_id: + self.last_name_by_id[uniqueid] = (filename, sha) + LOG.info( + '%s: update to %s in commit %s', + uniqueid, filename, sha, + ) + else: + LOG.debug( + '%s: modified file already known', + uniqueid, + ) + + def delete(self, filename, sha, version): + uniqueid = _get_unique_id(filename) + self._common(uniqueid, version) + # This file is being deleted without a rename. If + # we have already seen the UID before, that means + # that after the file was deleted another file + # with the same UID was added back. In that case + # we do not want to treat it as deleted. + # + # Never store deleted files in last_name_by_id so + # we can safely use all of those entries to build + # the history data. + if uniqueid not in self.last_name_by_id: + self.uniqueids_deleted.add(uniqueid) + LOG.info( + '%s: note deleted in %s', + uniqueid, sha, + ) + else: + LOG.debug( + '%s: delete for file re-added after the delete', + uniqueid, + ) + + class RenoRepo(repo.Repo): # Populated by _load_tags(). @@ -613,9 +753,6 @@ class Scanner(object): if scan_stop_tag: LOG.info('stopping scan at %s', scan_stop_tag) - versions = [] - earliest_seen = collections.OrderedDict() - # Determine the current version, which might be an unreleased or # dev version if there are unreleased commits at the head of the # branch in question. Since the version may not already be known, @@ -627,13 +764,9 @@ class Scanner(object): if current_version not in versions_by_date: versions_by_date.insert(0, current_version) - # Remember the most current filename for each id, to allow for - # renames. - last_name_by_id = {} - - # Remember uniqueids that have had files deleted. - uniqueids_deleted = set() - + # Track the versions we have seen and the earliest version for + # which we have seen a given note's unique id. + tracker = _ChangeTracker() for counter, entry in enumerate(self._topo_traversal(branch), 1): sha = entry.commit.id @@ -651,116 +784,35 @@ class Scanner(object): LOG.info('%06d %s updating current version to %s', counter, sha, current_version) - # Remember each version we have seen. - if current_version not in versions: - LOG.debug('%s is a new version' % current_version) - versions.append(current_version) - - # Look for changes to notes files in this commit. + # Look for changes to notes files in this commit. The + # change has only the basename of the path file, so we + # need to prefix that with the notesdir before giving it + # to the tracker. changes = _changes_in_subdir(self._repo, entry, notesdir) for change in _aggregate_changes(entry, changes, notesdir): uniqueid = change[0] - # Update the "earliest" version where a UID appears - # every time we see it, because we are scanning the - # history in reverse order so "early" items come - # later. - LOG.debug('%s: setting earliest reference to %s', - uniqueid, current_version) - earliest_seen[uniqueid] = current_version - c_type = change[1] - # If we have recorded that a UID was deleted, that - # means that was the last change made to the file and - # we can ignore it. - if uniqueid in uniqueids_deleted: - LOG.debug( - '%s: has already been deleted, ignoring this change', - uniqueid, - ) - continue - if c_type == diff_tree.CHANGE_ADD: - # A note is being added in this commit. If we have - # not seen it before, it was added here and never - # changed. - if uniqueid not in last_name_by_id: - path, sha = change[-2:] - fullpath = os.path.join(notesdir, path) - last_name_by_id[uniqueid] = (fullpath, - sha.decode('ascii')) - LOG.info( - '%s: update to %s in commit %s', - uniqueid, path, sha, - ) - else: - LOG.debug( - '%s: add for file we have already seen', - uniqueid, - ) + path, blob_sha = change[-2:] + fullpath = os.path.join(notesdir, path) + tracker.add(fullpath, sha, current_version) elif c_type == diff_tree.CHANGE_DELETE: - # This file is being deleted without a rename. If - # we have already seen the UID before, that means - # that after the file was deleted another file - # with the same UID was added back. In that case - # we do not want to treat it as deleted. - # - # Never store deleted files in last_name_by_id so - # we can safely use all of those entries to build - # the history data. - if uniqueid not in last_name_by_id: - uniqueids_deleted.add(uniqueid) - LOG.info( - '%s: note deleted in %s', - uniqueid, sha, - ) - else: - LOG.debug( - '%s: delete for file re-added after the delete', - uniqueid, - ) + path = change[-1] + fullpath = os.path.join(notesdir, path) + tracker.delete(fullpath, sha, current_version) elif c_type == diff_tree.CHANGE_RENAME: - # The file is being renamed. We may have seen it - # before, if there were subsequent modifications, - # so only store the name information if it is not - # there already. - if uniqueid not in last_name_by_id: - path, sha = change[-2:] - fullpath = os.path.join(notesdir, path) - last_name_by_id[uniqueid] = (fullpath, - sha.decode('ascii')) - LOG.info( - '%s: update to %s in commit %s', - uniqueid, path, sha, - ) - else: - LOG.debug( - '%s: renamed file already known with the new name', - uniqueid, - ) + path, blob_sha = change[-2:] + fullpath = os.path.join(notesdir, path) + tracker.rename(fullpath, sha, current_version) elif c_type == diff_tree.CHANGE_MODIFY: - # An existing file is being modified. We may have - # seen it before, if there were subsequent - # modifications, so only store the name - # information if it is not there already. - if uniqueid not in last_name_by_id: - path, sha = change[-2:] - fullpath = os.path.join(notesdir, path) - last_name_by_id[uniqueid] = (fullpath, - sha.decode('ascii')) - LOG.info( - '%s: update to %s in commit %s', - uniqueid, path, sha, - ) - else: - LOG.debug( - '%s: modified file already known', - uniqueid, - ) + path, blob_sha = change[-2:] + fullpath = os.path.join(notesdir, path) + tracker.modify(fullpath, sha, current_version) else: raise ValueError( @@ -777,13 +829,13 @@ class Scanner(object): # Invert earliest_seen to make a list of notes files for each # version. files_and_tags = collections.OrderedDict() - for v in versions: + for v in tracker.versions: files_and_tags[v] = [] # Produce a list of the actual files present in the repository. If # a note is removed, this step should let us ignore it. - for uniqueid, version in earliest_seen.items(): + for uniqueid, version in tracker.earliest_seen.items(): try: - base, sha = last_name_by_id[uniqueid] + base, sha = tracker.last_name_by_id[uniqueid] files_and_tags[version].append((base, sha)) except KeyError: # Unable to find the file again, skip it to avoid breaking -- GitLab From d9db082967cfbf99c420d5141e9bd5957c5b0b98 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Wed, 28 Dec 2016 16:10:21 +0100 Subject: [PATCH 107/257] Remove link to modindex The documentation build does not generate any module index, thus remove the link to the page. The page http://docs.openstack.org/developer/reno/py-modindex.html does not exist. Change-Id: I79bc9033cfa2edb4eeebb8b65e61f41eb78b7a42 --- doc/source/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index f33586e..2032fee 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -23,6 +23,5 @@ Indices and tables ================== * :ref:`genindex` -* :ref:`modindex` * :ref:`search` -- GitLab From c2018e1d007d93a08e5c7535d2a0bac9374b7d87 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 2 Jan 2017 12:26:32 -0500 Subject: [PATCH 108/257] add the irc channel to the readme file Change-Id: Iaa3286c00f6a77202817e17bececed0c05af660f Signed-off-by: Doug Hellmann --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 6b6a411..039264a 100644 --- a/README.rst +++ b/README.rst @@ -15,6 +15,7 @@ Project Meta-data * Documentation: http://docs.openstack.org/developer/reno * Source: http://git.openstack.org/cgit/openstack/reno * Bugs: http://bugs.launchpad.net/reno +* IRC: #openstack-release Features ======== -- GitLab From f8fc8f97ff20026582742e3e7838cdd0ed5cad68 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 22 Dec 2016 16:47:19 -0500 Subject: [PATCH 109/257] teach the scanner to look at uncommitted files Update the Scanner to look at staged and unstaged changes to files in the local copy of the repository. We can't detect unknown files, yet. Closes-Bug: #1553155 Change-Id: I77ed60e6f8b8f819aabb361f34cf779623907f7b Signed-off-by: Doug Hellmann --- ...include-working-copy-d0aed2e77bb095e6.yaml | 7 ++ reno/scanner.py | 58 +++++++++++- reno/tests/test_scanner.py | 90 +++++++++++++++++++ 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/include-working-copy-d0aed2e77bb095e6.yaml diff --git a/releasenotes/notes/include-working-copy-d0aed2e77bb095e6.yaml b/releasenotes/notes/include-working-copy-d0aed2e77bb095e6.yaml new file mode 100644 index 0000000..5a02790 --- /dev/null +++ b/releasenotes/notes/include-working-copy-d0aed2e77bb095e6.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Include the local working copy when scanning the history of the + current branch. Notes files must at least be staged to indicate + that they will eventually be part of the history, but subsequent + changes to the file do not need to also be staged to be seen. diff --git a/reno/scanner.py b/reno/scanner.py index e02ed17..713b00f 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -20,7 +20,9 @@ import re import sys from dulwich import diff_tree +from dulwich import index as d_index from dulwich import objects +from dulwich import porcelain from dulwich import repo LOG = logging.getLogger(__name__) @@ -423,7 +425,23 @@ class RenoRepo(repo.Repo): return tree def get_file_at_commit(self, filename, sha): - "Return the contents of the file if it exists at the commit, or None." + """Return the contents of the file. + + If sha is None, return the working copy of the file. If the + file cannot be read from the working dir, return None. + + If the sha is not None and the file exists at the commit, + return the data from the stored blob. If the file does not + exist at the commit, return None. + + """ + if sha is None: + # Get the copy from the working directory. + try: + with open(os.path.join(self.path, filename), 'r') as f: + return f.read() + except IOError: + return None # Get the tree associated with the commit identified by the # input SHA, then look through the items in the tree to find # the one with the path matching the filename. Take the @@ -763,10 +781,48 @@ class Scanner(object): LOG.debug('current repository version: %s' % current_version) if current_version not in versions_by_date: versions_by_date.insert(0, current_version) + versions_by_date.insert(0, '*working-copy*') # Track the versions we have seen and the earliest version for # which we have seen a given note's unique id. tracker = _ChangeTracker() + + # Process the local index, if we are scanning the current + # branch. + if not branch: + prefix = notesdir.rstrip('/') + '/' + index = self._repo.open_index() + + # Pretend anything known to the repo and changed but not + # staged is part of the fake version '*working-copy*'. + LOG.debug('scanning unstaged changes') + for fname in d_index.get_unstaged_changes(index, self.reporoot): + fname = fname.decode('utf-8') + LOG.debug('found unstaged file %s', fname) + if fname.startswith(prefix) and _note_file(fname): + fullpath = os.path.join(self.reporoot, fname) + if os.path.exists(fullpath): + LOG.debug('found file %s', fullpath) + tracker.add(fname, None, '*working-copy*') + else: + LOG.debug('deleted file %s', fullpath) + tracker.delete(fname, None, '*working-copy*') + + # Pretend anything in the index is part of the fake + # version "*working-copy*". + LOG.debug('scanning staged schanges') + changes = porcelain.get_tree_changes(self._repo) + for fname in changes['add']: + fname = fname.decode('utf-8') + tracker.add(fname, None, '*working-copy*') + for fname in changes['modify']: + fname = fname.decode('utf-8') + tracker.modify(fname, None, '*working-copy*') + for fname in changes['delete']: + fname = fname.decode('utf-8') + tracker.delete(fname, None, '*working-copy*') + + # Process the git commit history. for counter, entry in enumerate(self._topo_traversal(branch), 1): sha = entry.commit.id diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index af0ab57..a419104 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -20,6 +20,7 @@ import os.path import re import subprocess import time +import unittest from dulwich import diff_tree from dulwich import objects @@ -529,6 +530,82 @@ class BasicTest(Base): results, ) + def test_staged_file(self): + # Prove that we can get a file we have staged. + # Start with a standard commit and tag + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + # Now stage a release note + n = self.get_note_num() + basename = 'staged-note-%016x.yaml' % n + filename = os.path.join(self.reporoot, 'releasenotes', 'notes', + basename) + create._make_note_file(filename, 'staged note') + self.repo.git('add', filename) + status_results = self.repo.git('status') + self.addDetail('git status', text_content(status_results)) + # Now run the scanner + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + self.assertEqual( + {'*working-copy*': [ + (os.path.join('releasenotes', 'notes', basename), + None)], + }, + raw_results, + ) + + @unittest.skip('dulwich does not know how to identify new files') + def test_added_tagged_not_staged(self): + # Prove that we can get a file we have created but not staged. + # Start with a standard commit and tag + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + # Now create a note without staging it + n = self.get_note_num() + basename = 'staged-note-%016x.yaml' % n + filename = os.path.join(self.reporoot, 'releasenotes', 'notes', + basename) + create._make_note_file(filename, 'staged note') + status_results = self.repo.git('status') + self.addDetail('git status', text_content(status_results)) + # Now run the scanner + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + # Take the staged version of the file, but associate it with + # tagged version 1.0.0 because the file was added before that + # version. + self.assertEqual( + {'1.0.0': [(os.path.join('releasenotes', 'notes', basename), + None)], + }, + raw_results, + ) + + def test_modified_tagged_not_staged(self): + # Prove that we can get a file we have changed but not staged. + # Start with a standard commit and tag + self._make_python_package() + f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + # Now modify the note + fullpath = os.path.join(self.repo.reporoot, f1) + with open(fullpath, 'w') as f: + f.write('modified first note') + status_results = self.repo.git('status') + self.addDetail('git status', text_content(status_results)) + # Now run the scanner + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + # Take the staged version of the file, but associate it with + # tagged version 1.0.0 because the file was added before that + # version. + self.assertEqual( + {'1.0.0': [(f1, None)], + }, + raw_results, + ) + class FileContentsTest(Base): @@ -597,6 +674,19 @@ class FileContentsTest(Base): contents, ) + def test_staged_file(self): + # Prove we are not picking up the contents from the local + # filesystem outside of the git history. + f1 = self._add_notes_file(contents='initial-contents') + with open(os.path.join(self.reporoot, f1), 'w') as f: + f.write('new contents for file') + r = scanner.RenoRepo(self.reporoot) + contents = r.get_file_at_commit(f1, None) + self.assertEqual( + 'new contents for file', + contents, + ) + class PreReleaseTest(Base): -- GitLab From 5003ea2c91a481d449d07b3d6129961d33a9b862 Mon Sep 17 00:00:00 2001 From: git-harry Date: Thu, 15 Dec 2016 21:52:16 +0000 Subject: [PATCH 110/257] Add support for custom tag version schemes This change adds the ability to specify regular expressions to define a customised versioning scheme for release tags and pre-release tags. By default this change supports the current versioning scheme used by OpenStack. To customise, update the config.yaml file with the appropriate values. For example, for tags with versions like 'v1.0.0' and pre-release versions like 'v1.0.0rc1' the following could be added to config.yaml: release_tag_re: 'v\d\.\d\.\d(rc\d+)?' pre_release_tag_re: '(?Prc\d+$)' Change-Id: I7539fdeada14a73ae4e18a125bb0e3947f08e8d1 --- .../custom-tag-versions-d02028b6d35db967.yaml | 15 +++++ reno/config.py | 21 ++++++- reno/scanner.py | 57 ++++++++++++++----- 3 files changed, 77 insertions(+), 16 deletions(-) create mode 100644 releasenotes/notes/custom-tag-versions-d02028b6d35db967.yaml diff --git a/releasenotes/notes/custom-tag-versions-d02028b6d35db967.yaml b/releasenotes/notes/custom-tag-versions-d02028b6d35db967.yaml new file mode 100644 index 0000000..75d0871 --- /dev/null +++ b/releasenotes/notes/custom-tag-versions-d02028b6d35db967.yaml @@ -0,0 +1,15 @@ +--- +features: + - | + Add the ability to specify regular expressions to a define a + customised versioning scheme for release tags and pre-release tags. + + By default this change supports the current versioning scheme used by + OpenStack. + + To customise, update the config.yaml file with the appropriate values. + For example, for tags with versions like 'v1.0.0' and pre-release + versions like 'v1.0.0rc1' the following could be added to config.yaml: + + release_tag_re: 'v\d\.\d\.\d(rc\d+)?' + pre_release_tag_re: '(?Prc\d+$)' diff --git a/reno/config.py b/reno/config.py index 4b06ad5..95563f1 100644 --- a/reno/config.py +++ b/reno/config.py @@ -115,7 +115,26 @@ class Config(object): 'earliest_version': None, # The template used by reno new to create a note. - 'template': _TEMPLATE + 'template': _TEMPLATE, + + # The RE pattern used to match the repo tags representing a valid + # release version. The pattern is compiled with the verbose and unicode + # flags enabled. + 'release_tag_re': ''' + ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and + # pre-releases + ''', + + # The RE pattern used to check if a valid release version tag is also a + # valid pre-release version. The pattern is compiled with the verbose + # and unicode flags enabled. The pattern must define a group called + # 'pre_release' that matches the pre-release part of the tag and any + # separator, e.g for pre-release version '12.0.0.0rc1' the default RE + # pattern will identify '.0rc1' as the value of the group + # 'pre_release'. + 'pre_release_tag_re': ''' + (?P\.\d+(?:[ab]|rc)+\d*)$ + ''', } @classmethod diff --git a/reno/scanner.py b/reno/scanner.py index 713b00f..7b12571 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -27,11 +27,6 @@ from dulwich import repo LOG = logging.getLogger(__name__) -# What does a pre-release version number look like? -PRE_RELEASE_RE = re.compile(''' - \.(\d+(?:[ab]|rc)+\d*)$ -''', flags=re.VERBOSE | re.UNICODE) - def _parse_version(v): parts = v.split('.') + ['0', '0', '0'] @@ -467,6 +462,14 @@ class Scanner(object): self.conf = conf self.reporoot = self.conf.reporoot self._repo = RenoRepo(self.reporoot) + self.release_tag_re = re.compile( + self.conf.release_tag_re, + flags=re.VERBOSE | re.UNICODE, + ) + self.pre_release_tag_re = re.compile( + self.conf.pre_release_tag_re, + flags=re.VERBOSE | re.UNICODE, + ) def _get_ref(self, name): if name: @@ -496,6 +499,10 @@ class Scanner(object): branch_head = self._get_ref(branch) return self._repo.get_walker(branch_head) + def _get_valid_tags_on_commit(self, sha): + return [tag for tag in self._repo.get_tags_on_commit(sha) + if self.release_tag_re.match(tag)] + def _get_tags_on_branch(self, branch): "Return a list of tag names on the given branch." results = [] @@ -503,7 +510,7 @@ class Scanner(object): # shas_to_tags has encoded versions of the shas # but the commit object gives us a decoded version sha = c.commit.sha().hexdigest().encode('ascii') - tags = self._repo.get_tags_on_commit(sha) + tags = self._get_valid_tags_on_commit(sha) results.extend(tags) return results @@ -519,7 +526,7 @@ class Scanner(object): # shas_to_tags has encoded versions of the shas # but the commit object gives us a decoded version sha = commit.sha().hexdigest().encode('ascii') - tags = self._repo.get_tags_on_commit(sha) + tags = self._get_valid_tags_on_commit(sha) if tags: if count: val = '{}-{}'.format(tags[-1], count) @@ -534,6 +541,27 @@ class Scanner(object): commit = None return '0.0.0' + def _strip_pre_release(self, tag): + """Return tag with pre-release identifier removed if present.""" + pre_release_match = self.pre_release_tag_re.search(tag) + if pre_release_match: + try: + start = pre_release_match.start('pre_release') + end = pre_release_match.end('pre_release') + except IndexError: + raise ValueError( + ("The pre-release tag regular expression, {!r}, is missing" + " a group named 'pre_release'.").format( + self.pre_release_tag_re.pattern + ) + ) + else: + stripped_tag = tag[:start] + tag[end:] + else: + stripped_tag = tag + + return stripped_tag + def _get_branch_base(self, branch): "Return the tag at base of the branch." # Based on @@ -552,7 +580,7 @@ class Scanner(object): if c.commit.sha().hexdigest() in master_commits: # We got to this commit via the branch, but it is also # on master, so this is the base. - tags = self._repo.get_tags_on_commit( + tags = self._get_valid_tags_on_commit( c.commit.sha().hexdigest().encode('ascii')) if tags: return tags[-1] @@ -756,13 +784,13 @@ class Scanner(object): earliest_version, versions_by_date, collapse_pre_releases) if earliest_version and collapse_pre_releases: - if PRE_RELEASE_RE.search(earliest_version): + if self.pre_release_tag_re.search(earliest_version): # The earliest version won't actually be the pre-release # that might have been tagged when the branch was created, # but the final version. Strip the pre-release portion of # the version number. - earliest_version = '.'.join( - earliest_version.split('.')[:-1] + earliest_version = self._strip_pre_release( + earliest_version ) if earliest_version: LOG.info('earliest version to include is %s', earliest_version) @@ -826,7 +854,7 @@ class Scanner(object): for counter, entry in enumerate(self._topo_traversal(branch), 1): sha = entry.commit.id - tags_on_commit = self._repo.get_tags_on_commit(sha) + tags_on_commit = self._get_valid_tags_on_commit(sha) LOG.debug('%06d %s %s', counter, sha, tags_on_commit) @@ -911,13 +939,12 @@ class Scanner(object): # We don't need to collapse this one because there are # no notes attached to it. continue - pre_release_match = PRE_RELEASE_RE.search(ov) + pre_release_match = self.pre_release_tag_re.search(ov) LOG.debug('checking %r', ov) if pre_release_match: # Remove the trailing pre-release part of the version # from the string. - pre_rel_str = pre_release_match.groups()[0] - canonical_ver = ov[:-len(pre_rel_str)].rstrip('.') + canonical_ver = self._strip_pre_release(ov) if canonical_ver not in versions_by_date: # This canonical version was never tagged, so we # do not want to collapse the pre-releases. Reset -- GitLab From 4af98bb280d688b13dd619c8542cffdc6c845a5d Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 10 Jan 2017 17:10:54 -0500 Subject: [PATCH 111/257] do not test python 3.4 by default The gate only tests Python 2.7 and 3.5, so there's no need to include 3.4 in the list that runs locally on a development system. Change-Id: If88fdb9b889b351b46353a27e011fae352122cc3 Signed-off-by: Doug Hellmann --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4229cef..7b45ac9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = py35,py34,py27,pep8 +envlist = py35,py27,pep8 skipsdist = True [testenv] -- GitLab From b11e2fbf29ba7499a0e09a812e9b71d9ed293d1a Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 19 Jan 2017 16:19:08 -0500 Subject: [PATCH 112/257] add a null logging handler Python libraries that log should set a NullHandler in case the application where they are used does not set up logging. Reno's use in the Sphinx extension and in some release tools generates warnings the first time logging calls are made because of the missing handler. Change-Id: Ia8455d72f4b5e861d023b541a8ea9db786d53637 Signed-off-by: Doug Hellmann --- reno/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/reno/__init__.py b/reno/__init__.py index a61b2d0..ce87e72 100644 --- a/reno/__init__.py +++ b/reno/__init__.py @@ -12,8 +12,14 @@ # License for the specific language governing permissions and limitations # under the License. +import logging + import pbr.version __version__ = pbr.version.VersionInfo( 'reno').version_string() + +# Configure a null logger so that if reno is used as a library by an +# application that does not configure logging there are no warnings. +logging.getLogger(__name__).addHandler(logging.NullHandler()) -- GitLab From b8df9a0baff5f5c6d7ed0bf82843f66f54b8cbba Mon Sep 17 00:00:00 2001 From: Cao Xuan Hoang Date: Fri, 20 Jan 2017 14:49:32 +0700 Subject: [PATCH 113/257] Remove support for py33 Python 3.3 is not supported from Mitaka, as per Infra. This patch removes the support for the same. Change-Id: If946e71f50fae4213a46f872584079b48108411a --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 09187ec..b50edb0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,6 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 -- GitLab From 1db83bde73b94d9aa11cd6265f4c69a5af233b87 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 1 Feb 2017 13:10:30 -0500 Subject: [PATCH 114/257] fix logic for deciding when to stop scanning a branch When scanning a given branch, we don't want to stop at the base of the current branch, we want to stop at the base of the *previous* branch on master. That pulls in the full history of the current branch from where the older branch diverged. Given this history: (master) | 1.0.0 | \ (stable/a) | \ | 1.0.1 2.0.0 | 2.0.1 | \ (stable/b) | \ | 2.0.2 2.1.0 The notes for stable/b branch should include versions 2.0.2, 2.0.1, and 2.0.0 (which is on master) but not 1.0.0 which is the previous release. Change-Id: If1feddadc1a8e24b163667cd84f5b9e098951c69 Signed-off-by: Doug Hellmann --- ...ption-branch-name-re-8ecfe93195b8824e.yaml | 15 ++ reno/config.py | 5 + reno/scanner.py | 77 ++++++++- reno/tests/test_scanner.py | 161 +++++++++++++----- 4 files changed, 204 insertions(+), 54 deletions(-) create mode 100644 releasenotes/notes/config-option-branch-name-re-8ecfe93195b8824e.yaml diff --git a/releasenotes/notes/config-option-branch-name-re-8ecfe93195b8824e.yaml b/releasenotes/notes/config-option-branch-name-re-8ecfe93195b8824e.yaml new file mode 100644 index 0000000..c3150ca --- /dev/null +++ b/releasenotes/notes/config-option-branch-name-re-8ecfe93195b8824e.yaml @@ -0,0 +1,15 @@ +--- +features: + - | + Add a configuration option ``branch_name_re`` to hold a regular expression + for choosing "interesting" branches when trying to automatically detect + how far back the scanner should look. The default is 'stable/.+', which + works for the OpenStack practice of creating branches named after the + stable series of releases. +fixes: + - | + Fixes the logic for determining how far back in history to look when + scanning a given branch. Reno now looks for the base of the "previous" + branch, as determined by looking at branches matching ``branch_name_re`` + in lexical order. This may not work if branches are created using + version numbers as their names. diff --git a/reno/config.py b/reno/config.py index 95563f1..47d44b1 100644 --- a/reno/config.py +++ b/reno/config.py @@ -135,6 +135,11 @@ class Config(object): 'pre_release_tag_re': ''' (?P\.\d+(?:[ab]|rc)+\d*)$ ''', + + # The pattern for names for branches that are relevant when + # scanning history to determine where to stop, to find the + # "base" of a branch. Other branches are ignored. + 'branch_name_re': 'stable/.+', } @classmethod diff --git a/reno/scanner.py b/reno/scanner.py index 7b12571..25ffa98 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -470,6 +470,10 @@ class Scanner(object): self.conf.pre_release_tag_re, flags=re.VERBOSE | re.UNICODE, ) + self.branch_name_re = re.compile( + self.conf.branch_name_re, + flags=re.VERBOSE | re.UNICODE, + ) def _get_ref(self, name): if name: @@ -694,9 +698,43 @@ class Scanner(object): "Return true if the file exists at the given commit." return bool(self.get_file_at_commit(filename, sha)) - @staticmethod - def _find_scan_stop_point(earliest_version, versions_by_date, - collapse_pre_releases): + def _get_earlier_branch(self, branch): + "Return the name of the branch created before the given branch." + # FIXME(dhellmann): Assumes branches come in order based on + # name. That may not be true for projects that branch based on + # version numbers instead of names. + if branch.startswith('origin/'): + branch = branch[7:] + LOG.debug('looking for the branch before %s', branch) + refs = self._repo.get_refs() + LOG.debug('refs %s', list(refs.keys())) + branch_names = set() + for r in refs.keys(): + name = None + r = r.decode('utf-8') + if r.startswith('refs/remotes/origin/'): + name = r[20:] + elif r.startswith('refs/heads/'): + name = r[11:] + if name and self.branch_name_re.search(name): + branch_names.add(name) + branch_names = list(sorted(branch_names)) + if branch not in branch_names: + LOG.debug('Could not find branch %r among %s', + branch, branch_names) + return None + LOG.debug('found branches %s', branch_names) + current = branch_names.index(branch) + if current == 0: + # This is the first branch. + LOG.debug('%s appears to be the first branch', branch) + return None + previous = branch_names[current - 1] + LOG.debug('found earlier branch %s', previous) + return previous + + def _find_scan_stop_point(self, earliest_version, versions_by_date, + collapse_pre_releases, branch): """Return the version to use to stop the scan. Use the list of versions_by_date to get the tag with a @@ -711,6 +749,7 @@ class Scanner(object): :param collapse_pre_releases: Boolean indicating whether we are collapsing pre-releases or not. If false, the next tag is used, regardless of its version. + :param branch: The name of the branch we are scanning. """ if not earliest_version: @@ -722,7 +761,17 @@ class Scanner(object): # The version we were given is not present, use a full # scan. return None - if not collapse_pre_releases: + # We need to look for the previous branch's root. + if branch and branch != 'master': + previous_branch = self._get_earlier_branch(branch) + if not previous_branch: + # This was the first branch, so scan the whole + # history. + return None + previous_base = self._get_branch_base(previous_branch) + return previous_base + is_pre_release = bool(self.pre_release_tag_re.search(earliest_version)) + if is_pre_release and not collapse_pre_releases: # We just take the next tag. return versions_by_date[idx] # We need to look for a different version. @@ -770,7 +819,8 @@ class Scanner(object): # If the user has told us where to stop, use that as the # default. scan_stop_tag = self._find_scan_stop_point( - earliest_version, versions_by_date, collapse_pre_releases) + earliest_version, versions_by_date, + collapse_pre_releases, branch) # If the user has not told us where to stop, try to work it # out for ourselves. If branch is set and is not "master", @@ -779,10 +829,15 @@ class Scanner(object): if (stop_at_branch_base and (not earliest_version) and branch and (branch != 'master')): LOG.debug('determining earliest_version from branch') - earliest_version = self._get_branch_base(branch) + branch_base = self._get_branch_base(branch) scan_stop_tag = self._find_scan_stop_point( - earliest_version, versions_by_date, - collapse_pre_releases) + branch_base, versions_by_date, + collapse_pre_releases, branch) + if not scan_stop_tag: + earliest_version = branch_base + else: + idx = versions_by_date.index(scan_stop_tag) + earliest_version = versions_by_date[idx - 1] if earliest_version and collapse_pre_releases: if self.pre_release_tag_re.search(earliest_version): # The earliest version won't actually be the pre-release @@ -932,6 +987,7 @@ class Scanner(object): # Combine pre-releases into the final release, if we are told to # and the final release exists. if collapse_pre_releases: + LOG.debug('collapsing pre-release versions into final releases') collapsing = files_and_tags files_and_tags = collections.OrderedDict() for ov in versions_by_date: @@ -958,12 +1014,16 @@ class Scanner(object): files_and_tags[canonical_ver] = [] files_and_tags[canonical_ver].extend(collapsing[ov]) + LOG.debug('files_and_tags %s', + {k: len(v) for k, v in files_and_tags.items()}) # Only return the parts of files_and_tags that actually have # filenames associated with the versions. + LOG.debug('trimming') trimmed = collections.OrderedDict() for ov in versions_by_date: if not files_and_tags.get(ov): continue + LOG.debug('keeping %s', ov) # Sort the notes associated with the version so they are in a # deterministic order, to avoid having the same data result in # different output depending on random factors. Earlier @@ -977,6 +1037,7 @@ class Scanner(object): # If we have been told to stop at a version, we can do that # now. if earliest_version and ov == earliest_version: + LOG.debug('stopping trimming at %s', earliest_version) break LOG.debug( diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index a419104..5fc8d71 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1367,80 +1367,149 @@ class BranchTest(Base): self.assertEqual(head1, head2) -class ScanStopPointTest(Base): +class ScanStopPointPrereleaseVersionsTest(Base): def setUp(self): - super(ScanStopPointTest, self).setUp() + super(ScanStopPointPrereleaseVersionsTest, self).setUp() self.scanner = scanner.Scanner(self.c) + self._make_python_package() + self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first series', '1.0.0.0rc1') + self.repo.git('checkout', '-b', 'stable/a') + self._add_notes_file('slug2') + self._add_notes_file('slug3') + self.repo.git('tag', '-s', '-m', 'second tag', '1.0.0') + self.repo.git('checkout', 'master') + self._add_notes_file('slug4') + self._add_notes_file('slug5') + self.repo.git('tag', '-s', '-m', 'second series', '2.0.0.0b3') + self._add_notes_file('slug6') + self._add_notes_file('slug7') + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.0.0rc1') + self.repo.git('checkout', '-b', 'stable/b') + self._add_notes_file('slug8') + self._add_notes_file('slug9') + self.repo.git('tag', '-s', '-m', 'third tag', '2.0.0') + self.repo.git('checkout', 'master') - def test_invalid_earliest_version(self): - self.assertIsNone( + def test_beta_collapse(self): + self.assertEqual( + '1.0.0.0rc1', self.scanner._find_scan_stop_point( - 'not.a.numeric.version', [], True), + '2.0.0.0b3', ['2.0.0.0b3', '1.0.0.0rc1'], + True, 'master'), ) - def test_none(self): - self.assertIsNone( + def test_rc_collapse_master(self): + self.assertEqual( + '1.0.0.0rc1', self.scanner._find_scan_stop_point( - None, [], True), + '2.0.0.0rc1', ['2.0.0.0rc1', '2.0.0.0b3', '1.0.0.0rc1'], + True, 'master'), ) - def test_unknown_version(self): - self.assertIsNone( + def test_rc_collapse_branch(self): + self.assertEqual( + '1.0.0.0rc1', self.scanner._find_scan_stop_point( - '1.0.0', [], True), + '2.0.0.0rc1', ['2.0.0.0rc1', '2.0.0.0b3', '1.0.0.0rc1'], + True, 'stable/b'), ) - def test_only_version(self): - self.assertIsNone( + def test_rc_no_collapse(self): + self.assertEqual( + '2.0.0.0b3', self.scanner._find_scan_stop_point( - '1.0.0', ['1.0.0'], True), + '2.0.0.0rc1', ['2.0.0.0rc1', '2.0.0.0b3', '1.0.0.0rc1'], + False, 'master'), ) - def test_beta_collapse(self): + def test_stable_branch_with_collapse(self): self.assertEqual( - '1.0.0', + '1.0.0.0rc1', + self.scanner._find_scan_stop_point( + '2.0.0', ['2.0.0', '2.0.0.0rc1', '2.0.0.0b3', '1.0.0.0rc1'], + True, 'stable/b'), + ) + + # def test_nova_newton(self): + # self.assertEqual( + # '13.0.0.0rc3', + # self.scanner._find_scan_stop_point( + # '14.0.0', + # [u'14.0.3', u'14.0.2', u'14.0.1', u'14.0.0.0rc2', + # u'14.0.0', u'14.0.0.0rc1', u'14.0.0.0b3', u'14.0.0.0b2', + # u'14.0.0.0b1', u'13.0.0.0rc3', u'13.0.0', u'13.0.0.0rc2', + # u'13.0.0.0rc1', u'13.0.0.0b3', u'13.0.0.0b2', u'13.0.0.0b1', + # u'12.0.0.0rc3', u'12.0.0', u'12.0.0.0rc2', u'12.0.0.0rc1', + # u'12.0.0.0b3', u'12.0.0.0b2', u'12.0.0.0b1', u'12.0.0a0', + # u'2015.1.0rc3', u'2015.1.0', u'2015.1.0rc2', u'2015.1.0rc1', + # u'2015.1.0b3', u'2015.1.0b2', u'2015.1.0b1', u'2014.2.rc2', + # u'2014.2', u'2014.2.rc1', u'2014.2.b3', u'2014.2.b2', + # u'2014.2.b1', u'2014.1.rc1', u'2014.1.b3', u'2014.1.b2', + # u'2014.1.b1', u'2013.2.rc1', u'2013.2.b3', u'2013.1.rc1', + # u'folsom-2', u'folsom-1', u'essex-1', u'diablo-2', + # u'diablo-1', u'2011.2', u'2011.2rc1', u'2011.2gamma1', + # u'2011.1rc1', u'0.9.0'], + # True), + # ) + + +class ScanStopPointRegularVersionsTest(Base): + + def setUp(self): + super(ScanStopPointRegularVersionsTest, self).setUp() + self.scanner = scanner.Scanner(self.c) + self._make_python_package() + self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first series', '1.0.0') + self.repo.git('checkout', '-b', 'stable/a') + self._add_notes_file('slug2') + self._add_notes_file('slug3') + self.repo.git('tag', '-s', '-m', 'second tag', '1.0.1') + self.repo.git('checkout', 'master') + self._add_notes_file('slug4') + self._add_notes_file('slug5') + self.repo.git('tag', '-s', '-m', 'second series', '2.0.0') + self._add_notes_file('slug6') + self._add_notes_file('slug7') + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.1') + self.repo.git('checkout', '-b', 'stable/b') + self._add_notes_file('slug8') + self._add_notes_file('slug9') + self.repo.git('tag', '-s', '-m', 'third tag', '2.0.2') + self.repo.git('checkout', 'master') + + def test_invalid_earliest_version(self): + self.assertIsNone( self.scanner._find_scan_stop_point( - '2.0.0.0b1', ['2.0.0', '2.0.0.0rc1', '2.0.0.0b1', '1.0.0'], - True), + 'not.a.numeric.version', [], True, 'stable/b'), ) - def test_rc_collapse(self): - self.assertEqual( - '1.0.0', + def test_none(self): + self.assertIsNone( self.scanner._find_scan_stop_point( - '2.0.0.0rc1', ['2.0.0', '2.0.0.0rc1', '2.0.0.0b1', '1.0.0'], - True), + None, [], True, 'stable/b'), ) - def test_rc_no_collapse(self): - self.assertEqual( - '2.0.0.0b1', + def test_unknown_version(self): + self.assertIsNone( self.scanner._find_scan_stop_point( - '2.0.0.0rc1', ['2.0.0', '2.0.0.0rc1', '2.0.0.0b1', '1.0.0'], - False), + '2.0.2', [], True, 'stable/b'), ) - def test_nova_newton(self): + def test_only_version(self): + self.assertIsNone( + self.scanner._find_scan_stop_point( + '2.0.2', ['1.0.0'], True, 'stable/b'), + ) + + def test_find_prior_branch(self): self.assertEqual( - '13.0.0.0rc3', + '1.0.0', self.scanner._find_scan_stop_point( - '14.0.0', - [u'14.0.3', u'14.0.2', u'14.0.1', u'14.0.0.0rc2', - u'14.0.0', u'14.0.0.0rc1', u'14.0.0.0b3', u'14.0.0.0b2', - u'14.0.0.0b1', u'13.0.0.0rc3', u'13.0.0', u'13.0.0.0rc2', - u'13.0.0.0rc1', u'13.0.0.0b3', u'13.0.0.0b2', u'13.0.0.0b1', - u'12.0.0.0rc3', u'12.0.0', u'12.0.0.0rc2', u'12.0.0.0rc1', - u'12.0.0.0b3', u'12.0.0.0b2', u'12.0.0.0b1', u'12.0.0a0', - u'2015.1.0rc3', u'2015.1.0', u'2015.1.0rc2', u'2015.1.0rc1', - u'2015.1.0b3', u'2015.1.0b2', u'2015.1.0b1', u'2014.2.rc2', - u'2014.2', u'2014.2.rc1', u'2014.2.b3', u'2014.2.b2', - u'2014.2.b1', u'2014.1.rc1', u'2014.1.b3', u'2014.1.b2', - u'2014.1.b1', u'2013.2.rc1', u'2013.2.b3', u'2013.1.rc1', - u'folsom-2', u'folsom-1', u'essex-1', u'diablo-2', - u'diablo-1', u'2011.2', u'2011.2rc1', u'2011.2gamma1', - u'2011.1rc1', u'0.9.0'], - True), + '2.0.2', ['2.0.2', '2.0.1', '2.0.0', '1.0.0'], + True, 'stable/b'), ) -- GitLab From d464c7e1144e050c96efcc269f7b48b2fbf39bcd Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 1 Feb 2017 13:55:06 -0500 Subject: [PATCH 115/257] only show recent releases on the current series pages Change the scanner to stop looking at history when it encounters the base of a branch matching branch_name_re. This causes less history to appear on the "current series" or "unreleased" pages. Change-Id: I7b309b549a90eb30d6c2206caef83996557ecd4b Signed-off-by: Doug Hellmann --- ...show-less-unreleased-802781a1a3bf110e.yaml | 7 ++ reno/scanner.py | 64 ++++++++++++------- reno/tests/test_scanner.py | 54 +++++++++++++++- 3 files changed, 99 insertions(+), 26 deletions(-) create mode 100644 releasenotes/notes/show-less-unreleased-802781a1a3bf110e.yaml diff --git a/releasenotes/notes/show-less-unreleased-802781a1a3bf110e.yaml b/releasenotes/notes/show-less-unreleased-802781a1a3bf110e.yaml new file mode 100644 index 0000000..268c071 --- /dev/null +++ b/releasenotes/notes/show-less-unreleased-802781a1a3bf110e.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + The scanner for the "current" branch (usually master) now stops when it + encounters the base of an earlier branch matching the branch_name_re config + option. This results in less history appearing on the unreleased pages, + while still actually showing the current series and any unreleased notes. diff --git a/reno/scanner.py b/reno/scanner.py index 25ffa98..483d601 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -698,14 +698,8 @@ class Scanner(object): "Return true if the file exists at the given commit." return bool(self.get_file_at_commit(filename, sha)) - def _get_earlier_branch(self, branch): - "Return the name of the branch created before the given branch." - # FIXME(dhellmann): Assumes branches come in order based on - # name. That may not be true for projects that branch based on - # version numbers instead of names. - if branch.startswith('origin/'): - branch = branch[7:] - LOG.debug('looking for the branch before %s', branch) + def _get_series_branches(self): + "Get branches matching the branch_name_re config option." refs = self._repo.get_refs() LOG.debug('refs %s', list(refs.keys())) branch_names = set() @@ -718,7 +712,17 @@ class Scanner(object): name = r[11:] if name and self.branch_name_re.search(name): branch_names.add(name) - branch_names = list(sorted(branch_names)) + return list(sorted(branch_names)) + + def _get_earlier_branch(self, branch): + "Return the name of the branch created before the given branch." + # FIXME(dhellmann): Assumes branches come in order based on + # name. That may not be true for projects that branch based on + # version numbers instead of names. + if branch.startswith('origin/'): + branch = branch[7:] + LOG.debug('looking for the branch before %s', branch) + branch_names = self._get_series_branches() if branch not in branch_names: LOG.debug('Could not find branch %r among %s', branch, branch_names) @@ -801,9 +805,15 @@ class Scanner(object): collapse_pre_releases = self.conf.collapse_pre_releases stop_at_branch_base = self.conf.stop_at_branch_base - LOG.info('scanning %s/%s (branch=%s)', + LOG.info('scanning %s/%s (branch=%s earliest_version=%s)', reporoot.rstrip('/'), notesdir.lstrip('/'), - branch or '*current*') + branch or '*current*', earliest_version) + + # Determine the current version, which might be an unreleased or + # dev version if there are unreleased commits at the head of the + # branch in question. + current_version = self._get_current_version(branch) + LOG.debug('current repository version: %s' % current_version) # Determine all of the tags known on the branch, in their date # order. We scan the commit history in topological order to ensure @@ -823,11 +833,21 @@ class Scanner(object): collapse_pre_releases, branch) # If the user has not told us where to stop, try to work it - # out for ourselves. If branch is set and is not "master", - # then we want to stop at the version before the tag at the - # base of the branch, which involves a bit of searching. - if (stop_at_branch_base and - (not earliest_version) and branch and (branch != 'master')): + # out for ourselves. + if not branch and not earliest_version and stop_at_branch_base: + # On the current branch, stop at the point where the most + # recent branch was created, if we can find one. + LOG.debug('working on current branch without earliest_version') + branches = self._get_series_branches() + if branches: + LOG.debug('looking at base of %s to stop scanning master', + branches[-1]) + scan_stop_tag = self._get_branch_base(branches[-1]) + earliest_version = current_version + elif branch and stop_at_branch_base and not earliest_version: + # If branch is set and is not "master", + # then we want to stop at the version before the tag at the + # base of the branch, which involves a bit of searching. LOG.debug('determining earliest_version from branch') branch_base = self._get_branch_base(branch) scan_stop_tag = self._find_scan_stop_point( @@ -854,14 +874,10 @@ class Scanner(object): if scan_stop_tag: LOG.info('stopping scan at %s', scan_stop_tag) - # Determine the current version, which might be an unreleased or - # dev version if there are unreleased commits at the head of the - # branch in question. Since the version may not already be known, - # make sure it is in the list of versions by date. And since it is - # the most recent version, go ahead and insert it at the front of - # the list. - current_version = self._get_current_version(branch) - LOG.debug('current repository version: %s' % current_version) + # Since the version may not already be known, make sure it is + # in the list of versions by date. And since it is the most + # recent version, go ahead and insert it at the front of the + # list. if current_version not in versions_by_date: versions_by_date.insert(0, current_version) versions_by_date.insert(0, '*working-copy*') diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 5fc8d71..6fe08dd 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -606,6 +606,58 @@ class BasicTest(Base): raw_results, ) + def test_stop_on_master_with_other_branch(self): + self._make_python_package() + self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'middle tag', '2.0.0') + self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'last tag', '3.0.0') + self.repo.git('branch', 'stable/a') + f4 = self._add_notes_file() + self.c.override( + earliest_version=None, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'3.0.0-1': [f4], + }, + results, + ) + + def test_stop_on_master_without_limits_or_branches(self): + self._make_python_package() + f1 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f2 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'middle tag', '2.0.0') + f3 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'last tag', '3.0.0') + f4 = self._add_notes_file() + self.c.override( + earliest_version=None, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'3.0.0-1': [f4], + '3.0.0': [f3], + '2.0.0': [f2], + '1.0.0': [f1], + }, + results, + ) + class FileContentsTest(Base): @@ -1059,8 +1111,6 @@ class BranchTest(Base): } self.assertEqual( { - '1.0.0': [self.f1], - '2.0.0': [self.f2], '2.0.0-1': [f21], }, results, -- GitLab From 9e2be7fd7db270c5588180b0d6c1dc3f8bdb6d1f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Feb 2017 12:32:23 -0500 Subject: [PATCH 116/257] try to discover the repository root in sphinx builds When building under Sphinx on readthedocs.org, the current directory for the build process is $reporoot/docs/source instead of $reporoot. Use dulwich to discover the $reporoot path to support these cases. Closes-Bug: #1661319 Change-Id: I66ca28a69d93a0a40946523102b38bd74c5ce958 Signed-off-by: Doug Hellmann --- reno/sphinxext.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index e7fc22a..85b1a2a 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -19,6 +19,7 @@ from docutils.parsers.rst import directives from docutils import statemachine from sphinx.util.nodes import nested_parse_with_titles +from dulwich import repo from reno import config from reno import defaults from reno import formatter @@ -51,6 +52,9 @@ class ReleaseNotesDirective(rst.Directive): branch = self.options.get('branch') reporoot_opt = self.options.get('reporoot', '.') reporoot = os.path.abspath(reporoot_opt) + # When building on RTD.org the root directory may not be + # the current directory, so look for it. + reporoot = repo.Repo.discover(reporoot).path relnotessubdir = self.options.get('relnotessubdir', defaults.RELEASE_NOTES_SUBDIR) conf = config.Config(reporoot, relnotessubdir) -- GitLab From c8904a6584dda9fcabe896517f8650fae1aa3412 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Feb 2017 16:17:51 -0500 Subject: [PATCH 117/257] remove cruft from readme Change-Id: I01ffa02e50f26b7768c0d42745359e4d31ccf677 Signed-off-by: Doug Hellmann --- README.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.rst b/README.rst index 039264a..3753eec 100644 --- a/README.rst +++ b/README.rst @@ -16,8 +16,3 @@ Project Meta-data * Source: http://git.openstack.org/cgit/openstack/reno * Bugs: http://bugs.launchpad.net/reno * IRC: #openstack-release - -Features -======== - -* TODO -- GitLab From 4dd403d39d847cff3e66dcb9483035cb9ff57dac Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Feb 2017 16:18:00 -0500 Subject: [PATCH 118/257] show full history When the branch-end detection changes in I7b309b549a90eb30d6c2206caef83996557ecd4b merge the history page will no longer show the full release notes. Update the page to show the notes for unreleased changes as well as everything on master. Change-Id: Idf177e7e61da552334b596009ef12b06a523436c Signed-off-by: Doug Hellmann --- doc/source/history.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/history.rst b/doc/source/history.rst index 7427aa4..d5f14e8 100644 --- a/doc/source/history.rst +++ b/doc/source/history.rst @@ -2,7 +2,10 @@ Release Notes =============== +.. release-notes:: Unreleased + .. release-notes:: Mainline + :branch: origin/master .. release-notes:: Newton Series :branch: origin/stable/newton -- GitLab From efceabc12c3c2fa545e3233cd41036ebb3e538a3 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Feb 2017 16:24:10 -0500 Subject: [PATCH 119/257] ignore staged files that are not notes Filter out staged files that do not look like they would contain release notes, as we do with unstaged files and with the commit history. Change-Id: I772a82f444b89b51f29f4e1b5b9b31dee1890ff0 Signed-off-by: Doug Hellmann --- reno/scanner.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 7b12571..eb7197c 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -842,13 +842,16 @@ class Scanner(object): changes = porcelain.get_tree_changes(self._repo) for fname in changes['add']: fname = fname.decode('utf-8') - tracker.add(fname, None, '*working-copy*') + if fname.startswith(prefix) and _note_file(fname): + tracker.add(fname, None, '*working-copy*') for fname in changes['modify']: fname = fname.decode('utf-8') - tracker.modify(fname, None, '*working-copy*') + if fname.startswith(prefix) and _note_file(fname): + tracker.modify(fname, None, '*working-copy*') for fname in changes['delete']: fname = fname.decode('utf-8') - tracker.delete(fname, None, '*working-copy*') + if fname.startswith(prefix) and _note_file(fname): + tracker.delete(fname, None, '*working-copy*') # Process the git commit history. for counter, entry in enumerate(self._topo_traversal(branch), 1): -- GitLab From 962e9086e00e85468c297cce5b16664d9bdbccc2 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Feb 2017 17:31:51 -0500 Subject: [PATCH 120/257] documentation improvements Expand the readme to include more description of the benefits of using reno. Clean up the organization in index.rst. Remove an unused readme.rst from within the documentation tree -- nothing imported it or linked to it. Clean up phrasing and formatting in the usage page. Change-Id: Id3f0b71f199f7ad15d5fc0b3df67e05b7a8f2b19 Signed-off-by: Doug Hellmann --- README.rst | 56 ++++++++++++++++++++++++++++++++------ doc/source/index.rst | 18 +++---------- doc/source/readme.rst | 1 - doc/source/usage.rst | 62 ++++++++++++++++++++++--------------------- 4 files changed, 83 insertions(+), 54 deletions(-) delete mode 100644 doc/source/readme.rst diff --git a/README.rst b/README.rst index 3753eec..bdeaf08 100644 --- a/README.rst +++ b/README.rst @@ -1,18 +1,58 @@ -=============================== - reno -- Release Notes Manager -=============================== +========================================= + reno: A New Way to Manage Release Notes +========================================= -Reno is a release notes manager for storing release notes in a git -repository and then building documentation from them. +Reno is a release notes manager designed with high throughput in mind, +supporting fast distributed development teams without introducing +additional development processes. Our goal is to encourage detailed +and accurate release notes for every release. -.. image:: http://governance.openstack.org/badges/reno.svg - :target: http://governance.openstack.org/reference/tags/index.html +Reno uses git to store its data, along side the code being +described. This means release notes can be written when the code +changes are fresh, so no details are forgotten. It also means that +release notes can go through the same review process used for managing +code and other documentation changes. + +Reno stores each release note in a separate file to enable a large +number of developers to work on multiple patches simultaneously, all +targeting the same branch, without worrying about merge +conflicts. This cuts down on the need to rebase or otherwise manually +resolve conflicts, and keeps a development team moving quickly. + +Reno also supports multiple branches, allowing release notes to be +back-ported from master to maintenance branches together with the +code for bug fixes. + +Reno organizes notes into logical groups based on whether they +describe new features, bug fixes, known issues, or other topics of +interest to the user. Contributors categorize individual notes as they +are added, and reno combines them before publishing. + +Notes can be styled using reStructuredText directives, and reno's +Sphinx integration makes it easy to incorporate release notes into +automated documentation builds. + +Notes are automatically associated with the release version based on +the git tags applied to the repository, so it is not necessary to +track changes manually using a bug tracker or other tool, or to worry +that an important change will be missed when the release notes are +written by hand all at one time, just before a release. + +Modifications to notes are incorporated when the notes are shown in +their original location in the history. This feature makes it possible +to correct typos or otherwise fix a published release note after a +release is made, but have the new note content associated with the +original version number. Notes also can be deleted, eliminating them +from future documentation builds. Project Meta-data ================= +.. image:: http://governance.openstack.org/badges/reno.svg + :target: http://governance.openstack.org/reference/tags/index.html + * Free software: Apache license * Documentation: http://docs.openstack.org/developer/reno * Source: http://git.openstack.org/cgit/openstack/reno * Bugs: http://bugs.launchpad.net/reno -* IRC: #openstack-release +* IRC: #openstack-release on freenode diff --git a/doc/source/index.rst b/doc/source/index.rst index 2032fee..c77378c 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,12 +1,7 @@ -.. reno documentation master file, created by - sphinx-quickstart on Tue Jul 9 22:26:36 2013. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. include:: ../../README.rst -Welcome to reno's documentation! -======================================================== - -Contents: +Contents +======== .. toctree:: :maxdepth: 2 @@ -18,10 +13,3 @@ Contents: contributing history examples - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`search` - diff --git a/doc/source/readme.rst b/doc/source/readme.rst deleted file mode 100644 index a6210d3..0000000 --- a/doc/source/readme.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../README.rst diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 268976e..b0ef557 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -7,8 +7,8 @@ Creating New Release Notes The ``reno`` command line tool is used to create a new release note file in the correct format and with a unique name. The ``new`` -subcommand combines a random suffix with a "slug" value to make the -new file with a unique name that is easy to identify again later. +subcommand combines a random suffix with a "slug" value to create +the file with a unique name that is easy to identify again later. :: @@ -32,19 +32,19 @@ being installed globally. For example releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml -The ``--edit`` option enables to edit the note just after its creation. +The ``--edit`` option opens the new note in a text editor. :: $ reno new slug-goes-here --edit - ... Open your editor (defined with EDITOR environment variable) ... + ... Opens the editor set in the EDITOR environment variable, editing the new file ... Created new notes file in releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml -By default the new note is created under ``./releasenotes/notes``. Use -the ``--rel-notes-dir`` to change the parent directory (the ``notes`` -subdirectory is always appended). It's also possible to set a custom -template to create notes (see `Configuring Reno`_ ). +By default, the new note is created under ``./releasenotes/notes``. +The ``--rel-notes-dir`` command-line flag changes the parent directory +(the ``notes`` subdirectory is always appended). It's also possible to +set a custom template to create notes (see `Configuring Reno`_ ). Editing a Release Note ====================== @@ -123,8 +123,8 @@ entirely. other: - Add other notes here, or remove this section. -Formatting ----------- +Note File Syntax +---------------- Release notes may include embedded `reStructuredText`_, including simple inline markup like emphasis and pre-formatted text as well as complex @@ -155,25 +155,20 @@ that is checked out). To limit the report to a subset of the available versions on the branch, use the ``--version`` option (it can be repeated). -Notes are output in the order they are found by ``git log`` looking -over the history of the branch. This is deterministic, but not -necessarily predictable or mutable. +Notes are output in the order they are found when scanning the git +history of the branch using topological ordering. This is +deterministic, but not necessarily predictable or mutable. Configuring Reno ================ -Reno looks for an optional ``config.yml`` file in your release notes -directory. This file may contain optional flags that you might use with a -command. If the values do not apply to the command, they are ignored in the -configuration file. For example, a couple reno commands allow you to specify - -- ``--branch`` -- ``--earliest-version`` -- ``--collapse-pre-releases``/``--no-collapse-pre-releases`` -- ``--ignore-cache`` -- ``--stop-at-branch-base``/``--no-stop-at-branch-base`` - -So you might write a config file (if you use these often) like: +Reno looks for an optional ``config.yml`` file in the release notes +directory. If the values in the configuration file do not apply to +the command being run, they are ignored. For example, some reno +commands take inputs controlling the branch, earliest revision, and +other common parameters that control which notes are included in the +output. Because they are commonly set options, a configuration file +may be the most convenient way to manage the values consistently. .. code-block:: yaml @@ -186,15 +181,22 @@ So you might write a config file (if you use these often) like: ... -These will be parsed first and then the CLI options will be applied after -the config files. +Many of the settings in the configuration file can be overridden by +using command-line switches. For example: + +- ``--branch`` +- ``--earliest-version`` +- ``--collapse-pre-releases``/``--no-collapse-pre-releases`` +- ``--ignore-cache`` +- ``--stop-at-branch-base``/``--no-stop-at-branch-base`` Debugging ========= -The way release notes are included into sphinx documents may mask where -formatting errors occur. To generate the release notes manually, so that -they can be put into a sphinx document directly for debugging, run: +The true location of formatting errors in release notes may be masked +because of the way release notes are included into sphinx documents. +To generate the release notes manually, so that they can be put into a +sphinx document directly for debugging, use the ``report`` command. .. code-block:: console -- GitLab From 7ecf63312fb487ef8feb3a16a7801c0ecdd35c05 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Mon, 20 Feb 2017 11:59:59 -0500 Subject: [PATCH 121/257] fix reference to config.yaml The configuration file has to be named config.yaml, so fix the documentation not to refer to config.yml. Change-Id: Icb2ef9f99a043a4e9a460f984436dcfdbe0cb41b --- doc/source/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/usage.rst b/doc/source/usage.rst index b0ef557..e21da98 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -162,7 +162,7 @@ deterministic, but not necessarily predictable or mutable. Configuring Reno ================ -Reno looks for an optional ``config.yml`` file in the release notes +Reno looks for an optional ``config.yaml`` file in the release notes directory. If the values in the configuration file do not apply to the command being run, they are ignored. For example, some reno commands take inputs controlling the branch, earliest revision, and -- GitLab From 10ccdda0eb8c1932dc4c8c2a66f46f0e7cf8bb0a Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Tue, 21 Feb 2017 14:04:13 -0500 Subject: [PATCH 122/257] fix some minor formatting issues with release notes - Use monospaced fonts for git refs - Put config.yaml excerpts into a code block - Hyperlink bug references Change-Id: I3ff8f65d1886f25746927ef2ad35d1db87439d2c --- .../config-option-branch-name-re-8ecfe93195b8824e.yaml | 2 +- .../notes/custom-tag-versions-d02028b6d35db967.yaml | 8 ++++---- .../fix-branch-base-detection-95300805f26a0c15.yaml | 3 ++- .../notes/show-less-unreleased-802781a1a3bf110e.yaml | 9 +++++---- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/releasenotes/notes/config-option-branch-name-re-8ecfe93195b8824e.yaml b/releasenotes/notes/config-option-branch-name-re-8ecfe93195b8824e.yaml index c3150ca..d776c0c 100644 --- a/releasenotes/notes/config-option-branch-name-re-8ecfe93195b8824e.yaml +++ b/releasenotes/notes/config-option-branch-name-re-8ecfe93195b8824e.yaml @@ -3,7 +3,7 @@ features: - | Add a configuration option ``branch_name_re`` to hold a regular expression for choosing "interesting" branches when trying to automatically detect - how far back the scanner should look. The default is 'stable/.+', which + how far back the scanner should look. The default is ``stable/.+``, which works for the OpenStack practice of creating branches named after the stable series of releases. fixes: diff --git a/releasenotes/notes/custom-tag-versions-d02028b6d35db967.yaml b/releasenotes/notes/custom-tag-versions-d02028b6d35db967.yaml index 75d0871..c1ce543 100644 --- a/releasenotes/notes/custom-tag-versions-d02028b6d35db967.yaml +++ b/releasenotes/notes/custom-tag-versions-d02028b6d35db967.yaml @@ -8,8 +8,8 @@ features: OpenStack. To customise, update the config.yaml file with the appropriate values. - For example, for tags with versions like 'v1.0.0' and pre-release - versions like 'v1.0.0rc1' the following could be added to config.yaml: + For example, for tags with versions like ``v1.0.0`` and pre-release + versions like ``v1.0.0rc1`` the following could be added to config.yaml:: - release_tag_re: 'v\d\.\d\.\d(rc\d+)?' - pre_release_tag_re: '(?Prc\d+$)' + release_tag_re: 'v\d\.\d\.\d(rc\d+)?' + pre_release_tag_re: '(?Prc\d+$)' diff --git a/releasenotes/notes/fix-branch-base-detection-95300805f26a0c15.yaml b/releasenotes/notes/fix-branch-base-detection-95300805f26a0c15.yaml index 10eaf58..ffbc8a0 100644 --- a/releasenotes/notes/fix-branch-base-detection-95300805f26a0c15.yaml +++ b/releasenotes/notes/fix-branch-base-detection-95300805f26a0c15.yaml @@ -4,4 +4,5 @@ fixes: Fix a problem with the way reno automatically detects the initial version in a branch that prevented it from including all of the notes associated with a release, especially if the branch was - created at a pre-release version number. Bug #1652092 \ No newline at end of file + created at a pre-release version number. + `Bug #1652092 `__ diff --git a/releasenotes/notes/show-less-unreleased-802781a1a3bf110e.yaml b/releasenotes/notes/show-less-unreleased-802781a1a3bf110e.yaml index 268c071..20673e0 100644 --- a/releasenotes/notes/show-less-unreleased-802781a1a3bf110e.yaml +++ b/releasenotes/notes/show-less-unreleased-802781a1a3bf110e.yaml @@ -1,7 +1,8 @@ --- features: - | - The scanner for the "current" branch (usually master) now stops when it - encounters the base of an earlier branch matching the branch_name_re config - option. This results in less history appearing on the unreleased pages, - while still actually showing the current series and any unreleased notes. + The scanner for the "current" branch (usually ``master``) now stops + when it encounters the base of an earlier branch matching the + ``branch_name_re`` config option. This results in less history + appearing on the unreleased pages, while still actually showing + the current series and any unreleased notes. -- GitLab From a92bef64d4b07b650b4020b521839b702a8d0a36 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 22 Feb 2017 22:59:12 -0500 Subject: [PATCH 123/257] add sha info to ChangeTracker debug output Add the sha info to the debug output from the change tracker so it is easier to follow why changes are being recorded. Change-Id: If1fae2002cb06d87b61491ecec95599b793ae357 Signed-off-by: Doug Hellmann --- reno/scanner.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index f73a741..73df536 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -219,20 +219,24 @@ class _ChangeTracker(object): # Remember uniqueids that have had files deleted. self.uniqueids_deleted = set() - def _common(self, uniqueid, version): + def _common(self, uniqueid, sha, version): if version not in self.versions: self.versions.append(version) # Update the "earliest" version where a UID appears # every time we see it, because we are scanning the # history in reverse order so "early" items come # later. - LOG.debug('%s: setting earliest reference to %s', - uniqueid, version) + if uniqueid in self.earliest_seen: + LOG.debug('%s: resetting earliest reference from %s to %s for %s', + uniqueid, self.earliest_seen[uniqueid], version, sha) + else: + LOG.debug('%s: setting earliest reference to %s for %s', + uniqueid, version, sha) self.earliest_seen[uniqueid] = version def add(self, filename, sha, version): uniqueid = _get_unique_id(filename) - self._common(uniqueid, version) + self._common(uniqueid, sha, version) LOG.info('%s: adding %s from %s', uniqueid, filename, version) @@ -263,7 +267,7 @@ class _ChangeTracker(object): def rename(self, filename, sha, version): uniqueid = _get_unique_id(filename) - self._common(uniqueid, version) + self._common(uniqueid, sha, version) # If we have recorded that a UID was deleted, that # means that was the last change made to the file and @@ -293,7 +297,7 @@ class _ChangeTracker(object): def modify(self, filename, sha, version): uniqueid = _get_unique_id(filename) - self._common(uniqueid, version) + self._common(uniqueid, sha, version) # If we have recorded that a UID was deleted, that # means that was the last change made to the file and @@ -323,7 +327,7 @@ class _ChangeTracker(object): def delete(self, filename, sha, version): uniqueid = _get_unique_id(filename) - self._common(uniqueid, version) + self._common(uniqueid, sha, version) # This file is being deleted without a rename. If # we have already seen the UID before, that means # that after the file was deleted another file @@ -994,6 +998,8 @@ class Scanner(object): for uniqueid, version in tracker.earliest_seen.items(): try: base, sha = tracker.last_name_by_id[uniqueid] + LOG.debug('%s: sorting %s into version %s', + uniqueid, base, version) files_and_tags[version].append((base, sha)) except KeyError: # Unable to find the file again, skip it to avoid breaking -- GitLab From b0ba2eeea5b816887ace3e72fe3beb2e3838e705 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 22 Feb 2017 22:59:51 -0500 Subject: [PATCH 124/257] add filename and sha in comments in report output Add debugging details with the filename and sha for the version of the content used so that each note that comes out in the report indicates where the content is from. Change-Id: I569f3dbd3df16f880c8f3fa0a1b5b1c1b77be45d Signed-off-by: Doug Hellmann --- .../show-note-filename-in-report-a1118c917588b58d.yaml | 7 +++++++ reno/formatter.py | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/show-note-filename-in-report-a1118c917588b58d.yaml diff --git a/releasenotes/notes/show-note-filename-in-report-a1118c917588b58d.yaml b/releasenotes/notes/show-note-filename-in-report-a1118c917588b58d.yaml new file mode 100644 index 0000000..286b754 --- /dev/null +++ b/releasenotes/notes/show-note-filename-in-report-a1118c917588b58d.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + The report output now includes debugging details with the filename + and sha for the version of the content used to indicate where the + content is from to assist with debugging formatting or content + issues. diff --git a/reno/formatter.py b/reno/formatter.py index 749dfbe..2aa8e30 100644 --- a/reno/formatter.py +++ b/reno/formatter.py @@ -61,12 +61,13 @@ def format_report(loader, versions_to_include, title=None): notefiles = loader[version] for n, sha in notefiles: if 'prelude' in file_contents[n]: + report.append('.. %s @ %s\n' % (n, sha)) report.append(file_contents[n]['prelude']) report.append('') for section_name, section_title in _SECTION_ORDER: notes = [ - n + (n, fn, sha) for fn, sha in notefiles if file_contents[fn].get(section_name) for n in file_contents[fn].get(section_name, []) @@ -75,7 +76,8 @@ def format_report(loader, versions_to_include, title=None): report.append(section_title) report.append('-' * len(section_title)) report.append('') - for n in notes: + for n, fn, sha in notes: + report.append('.. %s @ %s\n' % (fn, sha)) report.append('- %s' % _indent_for_list(n)) report.append('') -- GitLab From 081a4145e18c82acba877ee22c180b3428c773f6 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Tue, 21 Feb 2017 11:19:08 -0500 Subject: [PATCH 125/257] make sections configurable The list of section identifiers and corresponding display names, and the order in which they are rendered, was hard-coded. Some projects want to customise this, so move it into the Config object so that it can now be specified via config.yaml, e.g. sections: - [features, New Features] - [issues, Known Issues] - [upgrade, Upgrade Notes] - [api, API Changes] - [security, Security Issues] - [fixes, Bug Fixes] Change-Id: I914572c6a07ca81c54965b4b5a6b6aba50b3a787 --- doc/source/usage.rst | 15 ++- ...nfig-option-sections-9c68b070698e984a.yaml | 7 ++ reno/config.py | 14 +++ reno/formatter.py | 16 +--- reno/report.py | 1 + reno/sphinxext.py | 1 + reno/tests/test_formatter.py | 94 ++++++++++++++----- 7 files changed, 111 insertions(+), 37 deletions(-) create mode 100644 releasenotes/notes/config-option-sections-9c68b070698e984a.yaml diff --git a/doc/source/usage.rst b/doc/source/usage.rst index e21da98..53fb7a3 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -50,13 +50,16 @@ Editing a Release Note ====================== The note file is a YAML file with several sections. All of the text is -interpreted as having `reStructuredText`_ formatting. +interpreted as having `reStructuredText`_ formatting. The permitted +sections are configurable (see below) but default to the following +list: prelude General comments about the release. The prelude from all notes in a section are combined, in note order, to produce a single prelude - introducing that release. + introducing that release. This section is always included, regardless + of what sections are configured. features @@ -177,6 +180,14 @@ may be the most convenient way to manage the values consistently. earliest_version: 12.0.0 collapse_pre_releases: false stop_at_branch_base: true + sections: + # The prelude section is implicitly included. + - [features, New Features] + - [issues, Known Issues] + - [upgrade, Upgrade Notes] + - [api, API Changes] + - [security, Security Issues] + - [fixes, Bug Fixes] template: | ... diff --git a/releasenotes/notes/config-option-sections-9c68b070698e984a.yaml b/releasenotes/notes/config-option-sections-9c68b070698e984a.yaml new file mode 100644 index 0000000..73c77bc --- /dev/null +++ b/releasenotes/notes/config-option-sections-9c68b070698e984a.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Add a configuration option ``sections`` to hold the list of + permitted section identifiers and corresponding display names. + This also determines the order in which sections are collated. + diff --git a/reno/config.py b/reno/config.py index 47d44b1..2dd6105 100644 --- a/reno/config.py +++ b/reno/config.py @@ -140,6 +140,20 @@ class Config(object): # scanning history to determine where to stop, to find the # "base" of a branch. Other branches are ignored. 'branch_name_re': 'stable/.+', + + # The identifiers and names of permitted sections in the + # release notes, in the order in which the final report will + # be generated. + 'sections': [ + ['features', 'New Features'], + ['issues', 'Known Issues'], + ['upgrade', 'Upgrade Notes'], + ['deprecations', 'Deprecation Notes'], + ['critical', 'Critical Issues'], + ['security', 'Security Issues'], + ['fixes', 'Bug Fixes'], + ['other', 'Other Notes'], + ], } @classmethod diff --git a/reno/formatter.py b/reno/formatter.py index 749dfbe..aebe6f2 100644 --- a/reno/formatter.py +++ b/reno/formatter.py @@ -13,18 +13,6 @@ from __future__ import print_function -_SECTION_ORDER = [ - ('features', 'New Features'), - ('issues', 'Known Issues'), - ('upgrade', 'Upgrade Notes'), - ('deprecations', 'Deprecation Notes'), - ('critical', 'Critical Issues'), - ('security', 'Security Issues'), - ('fixes', 'Bug Fixes'), - ('other', 'Other Notes'), -] - - def _indent_for_list(text, prefix=' '): """Indent some text to make it work as a list entry. @@ -37,7 +25,7 @@ def _indent_for_list(text, prefix=' '): ]) + '\n' -def format_report(loader, versions_to_include, title=None): +def format_report(loader, config, versions_to_include, title=None): report = [] if title: report.append('=' * len(title)) @@ -64,7 +52,7 @@ def format_report(loader, versions_to_include, title=None): report.append(file_contents[n]['prelude']) report.append('') - for section_name, section_title in _SECTION_ORDER: + for section_name, section_title in config.sections: notes = [ n for fn, sha in notefiles diff --git a/reno/report.py b/reno/report.py index e59520f..2a59410 100644 --- a/reno/report.py +++ b/reno/report.py @@ -25,6 +25,7 @@ def report_cmd(args, conf): versions = ldr.versions text = formatter.format_report( ldr, + conf, versions, title='Release Notes', ) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 85b1a2a..3079f70 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -90,6 +90,7 @@ class ReleaseNotesDirective(rst.Directive): info('got versions %s' % (versions,)) text = formatter.format_report( ldr, + conf, versions, title=title, ) diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py index 42dc52c..23ed3e1 100644 --- a/reno/tests/test_formatter.py +++ b/reno/tests/test_formatter.py @@ -20,7 +20,7 @@ from reno import loader from reno.tests import base -class TestFormatter(base.TestCase): +class TestFormatterBase(base.TestCase): scanner_output = { '0.0.0': [('note1', 'shaA')], @@ -29,29 +29,11 @@ class TestFormatter(base.TestCase): versions = ['0.0.0', '1.0.0'] - note_bodies = { - 'note1': { - 'prelude': 'This is the prelude.', - }, - 'note2': { - 'issues': [ - 'This is the first issue.', - 'This is the second issue.', - ], - }, - 'note3': { - 'features': [ - 'We added a feature!', - ], - 'upgrade': None, - }, - } - def _get_note_body(self, reporoot, filename, sha): return self.note_bodies.get(filename, '') def setUp(self): - super(TestFormatter, self).setUp() + super(TestFormatterBase, self).setUp() def _load(ldr): ldr._scanner_output = self.scanner_output @@ -67,9 +49,31 @@ class TestFormatter(base.TestCase): ignore_cache=False, ) + +class TestFormatter(TestFormatterBase): + + note_bodies = { + 'note1': { + 'prelude': 'This is the prelude.', + }, + 'note2': { + 'issues': [ + 'This is the first issue.', + 'This is the second issue.', + ], + }, + 'note3': { + 'features': [ + 'We added a feature!', + ], + 'upgrade': None, + }, + } + def test_with_title(self): result = formatter.format_report( loader=self.ldr, + config=self.c, versions_to_include=self.versions, title='This is the title', ) @@ -78,6 +82,7 @@ class TestFormatter(base.TestCase): def test_versions(self): result = formatter.format_report( loader=self.ldr, + config=self.c, versions_to_include=self.versions, title='This is the title', ) @@ -87,14 +92,16 @@ class TestFormatter(base.TestCase): def test_without_title(self): result = formatter.format_report( loader=self.ldr, + config=self.c, versions_to_include=self.versions, title=None, ) self.assertNotIn('This is the title', result) - def test_section_order(self): + def test_default_section_order(self): result = formatter.format_report( loader=self.ldr, + config=self.c, versions_to_include=self.versions, title=None, ) @@ -104,3 +111,48 @@ class TestFormatter(base.TestCase): expected = [prelude_pos, features_pos, issues_pos] actual = list(sorted([prelude_pos, features_pos, issues_pos])) self.assertEqual(expected, actual) + + +class TestFormatterCustomSections(TestFormatterBase): + note_bodies = { + 'note1': { + 'prelude': 'This is the prelude.', + }, + 'note2': { + 'features': [ + 'This is the first feature.', + ], + 'api': [ + 'This is the API change for the first feature.', + ], + }, + 'note3': { + 'api': [ + 'This is the API change for the second feature.', + ], + 'features': [ + 'This is the second feature.', + ], + }, + } + + def setUp(self): + super(TestFormatterCustomSections, self).setUp() + self.c.override(sections=[ + ['api', 'API Changes'], + ['features', 'New Features'], + ]) + + def test_custom_section_order(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title=None, + ) + prelude_pos = result.index('This is the prelude.') + api_pos = result.index('API Changes') + features_pos = result.index('New Features') + expected = [prelude_pos, api_pos, features_pos] + actual = list(sorted([prelude_pos, features_pos, api_pos])) + self.assertEqual(expected, actual) -- GitLab From 961f40c06830a76c11a661b44efaf6d917706b3a Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Tue, 21 Feb 2017 14:02:22 -0500 Subject: [PATCH 126/257] trim Newton history to avoid duplication Don't list changes before 1.9.0 in the release notes for the Newton series, since they are already displayed in the mainline history. Change-Id: Id1cbf3a5d111e6e3cfbe6635929bd45b5d13345b --- doc/source/history.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/history.rst b/doc/source/history.rst index d5f14e8..c226052 100644 --- a/doc/source/history.rst +++ b/doc/source/history.rst @@ -9,3 +9,4 @@ .. release-notes:: Newton Series :branch: origin/stable/newton + :earliest-version: 1.9.0 -- GitLab From ce925cca93ad51c77d175fcde11e7f02019ff074 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Tue, 21 Feb 2017 11:19:08 -0500 Subject: [PATCH 127/257] clarify automatic inclusion of prelude section As suggested in the previous review: https://review.openstack.org/#/c/436639/2/reno/config.py@146 it is worth clarifying that the prelude section will always be included. Change-Id: Iaae88434e02df5c7eb38a639c3dcb0ead6464c6c --- reno/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reno/config.py b/reno/config.py index 2dd6105..b64f95c 100644 --- a/reno/config.py +++ b/reno/config.py @@ -143,7 +143,8 @@ class Config(object): # The identifiers and names of permitted sections in the # release notes, in the order in which the final report will - # be generated. + # be generated. A prelude section will always be automatically + # inserted before the first element of this list. 'sections': [ ['features', 'New Features'], ['issues', 'Known Issues'], -- GitLab From 65a82c37d2669578e15dc57291cfb6f362b63335 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 1 Mar 2017 10:59:20 -0500 Subject: [PATCH 128/257] uncap pbr dependency also fix the hacking dependency to allow a version that does not cap pbr Change-Id: I742e116e9749209d5126a5a2b14b7f5b27ff8b03 Signed-off-by: Doug Hellmann --- requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7e2b9e9..5fb4f35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -pbr<2.0,>=1.4 +pbr>=1.4 Babel>=1.3 PyYAML>=3.1.0 six>=1.9.0 diff --git a/test-requirements.txt b/test-requirements.txt index ce26e18..2f939a5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking<0.11,>=0.10.0 +hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0 mock>=1.2 -- GitLab From 2e9cd7cfe53ae2a7c8b81dcc99a67114d410e382 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 2 Mar 2017 09:27:44 -0500 Subject: [PATCH 129/257] allow tracking branch names when the branches only exist on origin Fix a problem with branch references so that it is now possible to use a local tracking branch name when the branch only exists on the 'origin' remote. For example, this allows references to 'stable/ocata' when there is no local branch with that name but there is an 'origin/stable/ocata' branch. Change-Id: If11a0ff10ad3b1dfc82f99e6f68684ec9268af26 Signed-off-by: Doug Hellmann --- ...w-short-branch-names-61a35be55f04cea4.yaml | 8 +++++ reno/scanner.py | 4 +++ reno/tests/test_scanner.py | 36 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 releasenotes/notes/allow-short-branch-names-61a35be55f04cea4.yaml diff --git a/releasenotes/notes/allow-short-branch-names-61a35be55f04cea4.yaml b/releasenotes/notes/allow-short-branch-names-61a35be55f04cea4.yaml new file mode 100644 index 0000000..280ce62 --- /dev/null +++ b/releasenotes/notes/allow-short-branch-names-61a35be55f04cea4.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fix a problem with branch references so that it is now possible to + use a local tracking branch name when the branch only exists on + the 'origin' remote. For example, this allows references to + 'stable/ocata' when there is no local branch with that name but + there is an 'origin/stable/ocata' branch. \ No newline at end of file diff --git a/reno/scanner.py b/reno/scanner.py index 73df536..ef7dade 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -487,6 +487,10 @@ class Scanner(object): 'refs/tags/' + name, # If a stable branch was removed, look for its EOL tag. 'refs/tags/' + (name.rpartition('/')[-1] + '-eol'), + # If someone is using the "short" name for a branch + # without a local tracking branch, look to see if the + # name exists on the 'origin' remote. + 'refs/remotes/origin/' + name, ] for ref in candidates: key = ref.encode('utf-8') diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 6fe08dd..2d5f923 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1416,6 +1416,42 @@ class BranchTest(Base): self.assertIsNotNone(head2) self.assertEqual(head1, head2) + def test_remote_branch_without_prefix(self): + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'stable/2') + self.repo.git('checkout', 'master') + scanner1 = scanner.Scanner(self.c) + head1 = scanner1._get_ref('stable/2') + self.assertIsNotNone(head1) + print('head1', head1) + # Create a second repository by cloning the first. + print(utils.check_output( + ['git', 'clone', self.reporoot, 'reporoot2'], + cwd=self.temp_dir, + )) + reporoot2 = os.path.join(self.temp_dir, 'reporoot2') + print(utils.check_output( + ['git', 'remote', 'update'], + cwd=reporoot2, + )) + print(utils.check_output( + ['git', 'remote', '-v'], + cwd=reporoot2, + )) + print(utils.check_output( + ['find', '.git/refs'], + cwd=reporoot2, + )) + print(utils.check_output( + ['git', 'branch', '-a'], + cwd=reporoot2, + )) + c2 = config.Config(reporoot2) + scanner2 = scanner.Scanner(c2) + head2 = scanner2._get_ref('stable/2') + self.assertIsNotNone(head2) + self.assertEqual(head1, head2) + class ScanStopPointPrereleaseVersionsTest(Base): -- GitLab From 3387cfb3a69657a8a7e2e40eabbb56c514c797d4 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 2 Mar 2017 14:28:06 -0500 Subject: [PATCH 130/257] fix sphinxext scanner when it has a list of versions to include Only stop at the branch base if we have not been told explicitly which versions to include. Change-Id: I2488f614e6d2fec0cb7babce1258ae0c64e77e2d Signed-off-by: Doug Hellmann --- .../notes/fix-sphinxext-scanner-0aa012ada66db773.yaml | 8 ++++++++ reno/sphinxext.py | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/fix-sphinxext-scanner-0aa012ada66db773.yaml diff --git a/releasenotes/notes/fix-sphinxext-scanner-0aa012ada66db773.yaml b/releasenotes/notes/fix-sphinxext-scanner-0aa012ada66db773.yaml new file mode 100644 index 0000000..097c097 --- /dev/null +++ b/releasenotes/notes/fix-sphinxext-scanner-0aa012ada66db773.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fixes a problem with the sphinx extension that caused it to + produce an error if it had a list of versions to include that were + not within the set that seemed to be on the branch because of the + branch-base detection logic. Now if a list of versions is + included, the scan always includes the full history. \ No newline at end of file diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 3079f70..9269b19 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -66,7 +66,9 @@ class ReleaseNotesDirective(rst.Directive): # out how Sphinx passes a "false" flag later. # 'collapse-pre-releases' in self.options opt_overrides['collapse_pre_releases'] = True - opt_overrides['stop_at_branch_base'] = True + # Only stop at the branch base if we have not been told + # explicitly which versions to include. + opt_overrides['stop_at_branch_base'] = (version_opt is None) if 'earliest-version' in self.options: opt_overrides['earliest_version'] = self.options.get( 'earliest-version') -- GitLab From 6d67626e2b72b58860b8c52e34ad9f2c79c51ad2 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Fri, 3 Mar 2017 19:47:57 +0100 Subject: [PATCH 131/257] Update to Sphinx 1.5, tread warnings as errors Update to newer sphinx, handle warnings in doc building as errors with setting warning-is-error. Allow non-local image paths to import the badge. Change-Id: I57ee9972dbab2cdcf2ef6589cbcd1bc12fbcacb8 --- doc/source/conf.py | 3 +++ setup.cfg | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index c0ce8ab..d881af5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -51,6 +51,9 @@ add_module_names = True # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' +# Do not warn about non-local image URI +suppress_warnings = ['image.nonlocal_uri'] + # -- Options for HTML output -------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with diff --git a/setup.cfg b/setup.cfg index b50edb0..4362a22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,13 +29,14 @@ console_scripts = [extras] sphinx = - sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 - docutils>=0.11,!=0.13.1 # OSI-Approved Open Source, Public Domain + sphinx>=1.5.1 # BSD + docutils>=0.11 # OSI-Approved Open Source, Public Domain [build_sphinx] source-dir = doc/source build-dir = doc/build all_files = 1 +warning-is-error = 1 [upload_sphinx] upload-dir = doc/build/html -- GitLab From c2d0fd866c0ce007d17c800496de3f52b52201b9 Mon Sep 17 00:00:00 2001 From: gecong1973 Date: Thu, 9 Mar 2017 11:06:49 +0800 Subject: [PATCH 132/257] Using fixtures.MockPatch instead of mockpatch.Patch This module has been deprecated in favor of fixtures.MockPatch. Change-Id: I353ec2a42263401b42a6bda77fbb52fd5033cf18 --- reno/tests/test_cache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reno/tests/test_cache.py b/reno/tests/test_cache.py index ef7d120..5051bc4 100644 --- a/reno/tests/test_cache.py +++ b/reno/tests/test_cache.py @@ -12,10 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. +import fixtures import textwrap import mock -from oslotest import mockpatch from reno import cache from reno import config @@ -51,8 +51,8 @@ class TestCache(base.TestCase): def setUp(self): super(TestCache, self).setUp() self.useFixture( - mockpatch.Patch('reno.scanner.Scanner.get_file_at_commit', - new=self._get_note_body) + fixtures.MockPatch('reno.scanner.Scanner.get_file_at_commit', + new=self._get_note_body) ) self.c = config.Config('.') -- GitLab From d12c211172e1d721fa1f571fcbe82435bf77a360 Mon Sep 17 00:00:00 2001 From: Akihiro Motoki Date: Fri, 10 Mar 2017 00:26:16 +0900 Subject: [PATCH 133/257] sphinxext: Include branch information in source name It is nice if POT files generated by sphinx gettext contains branch information in source name as translators can easily know which string comes from which release. Currently, all strings rendered by reno have "../../../:1" as source name. This patch adds branch information to this. Change-Id: Ie48d6c869aa7768dc08be62aeea1404a5104f2ef --- reno/sphinxext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 9269b19..650eaaa 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -96,7 +96,7 @@ class ReleaseNotesDirective(rst.Directive): versions, title=title, ) - source_name = '<' + __name__ + '>' + source_name = '<%s %s>' % (__name__, branch or 'current branch') result = statemachine.ViewList() for line in text.splitlines(): result.append(line, source_name) -- GitLab From f6dbe9490779bd09d995f471a8203b0cc9bdf7ea Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 15 Mar 2017 10:56:24 -0400 Subject: [PATCH 134/257] fix bytes/str handling when looking for file content The tests were passing unicode but in some cases the real code was passing bytes, so accept both and do the encoding ourselves if we need to. Change-Id: I0f64d301b36207b1ffa6503e8c27f679cdf6516f Signed-off-by: Doug Hellmann --- reno/scanner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reno/scanner.py b/reno/scanner.py index ef7dade..aa33006 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -446,7 +446,9 @@ class RenoRepo(repo.Repo): # the one with the path matching the filename. Take the # associated SHA from the tree and get the file contents from # the repository. - commit = self[sha.encode('ascii')] + if hasattr(sha, 'encode'): + sha = sha.encode('ascii') + commit = self[sha] tree = self[commit.tree] try: mode, blob_sha = tree.lookup_path(self.get_object, -- GitLab From 33b135fe9a04dbaddc82f27f21f5955cbbefac02 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 15 Mar 2017 10:58:49 -0400 Subject: [PATCH 135/257] add --no-show-source option to report command Add a ``--no-show-source`` option to the report command to skip including the note reference file names and SHA information in comments in the output. This restores the previous format of the output for cases where it is meant to be read by people directly, not just converted to HTML. Change-Id: Ie284b8a8e60d5a5f958d229c5972e8d9bf697d44 Signed-off-by: Doug Hellmann --- .../notes/no-show-source-option-ee02766b26fe53be.yaml | 8 ++++++++ reno/formatter.py | 9 ++++++--- reno/main.py | 7 +++++++ reno/report.py | 1 + 4 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/no-show-source-option-ee02766b26fe53be.yaml diff --git a/releasenotes/notes/no-show-source-option-ee02766b26fe53be.yaml b/releasenotes/notes/no-show-source-option-ee02766b26fe53be.yaml new file mode 100644 index 0000000..7074ad3 --- /dev/null +++ b/releasenotes/notes/no-show-source-option-ee02766b26fe53be.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Add a ``--no-show-source`` option to the report command to skip + including the note reference file names and SHA information + in comments in the output. This restores the previous format of + the output for cases where it is meant to be read by people directly, + not just converted to HTML. diff --git a/reno/formatter.py b/reno/formatter.py index b6d43ec..646d68e 100644 --- a/reno/formatter.py +++ b/reno/formatter.py @@ -25,7 +25,8 @@ def _indent_for_list(text, prefix=' '): ]) + '\n' -def format_report(loader, config, versions_to_include, title=None): +def format_report(loader, config, versions_to_include, title=None, + show_source=True): report = [] if title: report.append('=' * len(title)) @@ -49,7 +50,8 @@ def format_report(loader, config, versions_to_include, title=None): notefiles = loader[version] for n, sha in notefiles: if 'prelude' in file_contents[n]: - report.append('.. %s @ %s\n' % (n, sha)) + if show_source: + report.append('.. %s @ %s\n' % (n, sha)) report.append(file_contents[n]['prelude']) report.append('') @@ -65,7 +67,8 @@ def format_report(loader, config, versions_to_include, title=None): report.append('-' * len(section_title)) report.append('') for n, fn, sha in notes: - report.append('.. %s @ %s\n' % (fn, sha)) + if show_source: + report.append('.. %s @ %s\n' % (fn, sha)) report.append('- %s' % _indent_for_list(n)) report.append('') diff --git a/reno/main.py b/reno/main.py index 79f8589..b8e2a8d 100644 --- a/reno/main.py +++ b/reno/main.py @@ -138,6 +138,13 @@ def main(argv=sys.argv[1:]): default=None, help='output filename, defaults to stdout', ) + do_report.add_argument( + '--no-show-source', + dest='show_source', + default=True, + action='store_false', + help='do not show the source for notes', + ) _build_query_arg_group(do_report) do_report.set_defaults(func=report.report_cmd) diff --git a/reno/report.py b/reno/report.py index 2a59410..fc2dda0 100644 --- a/reno/report.py +++ b/reno/report.py @@ -28,6 +28,7 @@ def report_cmd(args, conf): conf, versions, title='Release Notes', + show_source=args.show_source, ) if args.output: with open(args.output, 'w') as f: -- GitLab From 371fb0ff768668624c93c4ae135f63854fdf6e2a Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 15 Mar 2017 11:17:30 -0400 Subject: [PATCH 136/257] add a --title option to the report command Change-Id: I0ed8bbed995e637ef5526ae3fc8d7c3ac4214fb8 Signed-off-by: Doug Hellmann --- releasenotes/notes/report-title-option-f0875bfdbc54dd7b.yaml | 4 ++++ reno/main.py | 5 +++++ reno/report.py | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/report-title-option-f0875bfdbc54dd7b.yaml diff --git a/releasenotes/notes/report-title-option-f0875bfdbc54dd7b.yaml b/releasenotes/notes/report-title-option-f0875bfdbc54dd7b.yaml new file mode 100644 index 0000000..0948c26 --- /dev/null +++ b/releasenotes/notes/report-title-option-f0875bfdbc54dd7b.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Add a ``--title`` option to the report command. diff --git a/reno/main.py b/reno/main.py index b8e2a8d..dd062fe 100644 --- a/reno/main.py +++ b/reno/main.py @@ -145,6 +145,11 @@ def main(argv=sys.argv[1:]): action='store_false', help='do not show the source for notes', ) + do_report.add_argument( + '--title', + default='Release Notes', + help='set the main title of the generated report', + ) _build_query_arg_group(do_report) do_report.set_defaults(func=report.report_cmd) diff --git a/reno/report.py b/reno/report.py index fc2dda0..aab6be0 100644 --- a/reno/report.py +++ b/reno/report.py @@ -27,7 +27,7 @@ def report_cmd(args, conf): ldr, conf, versions, - title='Release Notes', + title=args.title, show_source=args.show_source, ) if args.output: -- GitLab From 378b305ca8d6c5c952b6d3b7601f14da050b5748 Mon Sep 17 00:00:00 2001 From: gecong1973 Date: Thu, 23 Mar 2017 15:46:22 +0800 Subject: [PATCH 137/257] Remove support for py34 The gating on python 3.4 is restricted to <= Mitaka. This is due to the change from Ubuntu Trusty to Xenial, where only python3.5 is available. There is no need to continue to keep these settings. Change-Id: Ia410cd56a57725aba1aad5e397e7915f1f773b6e --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 4362a22..19bd043 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,6 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 [files] -- GitLab From 252b48f171de8c9eb7a600fc825fbea4955c9b92 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Fri, 24 Mar 2017 10:47:01 +0000 Subject: [PATCH 138/257] doc: Document the available configuration options They're documented in the source. Let's document them in the actual documentation. Change-Id: Ia320808630281009c1700cb1ad25340761af86a0 --- doc/source/usage.rst | 72 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 53fb7a3..5299777 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -201,6 +201,78 @@ using command-line switches. For example: - ``--ignore-cache`` - ``--stop-at-branch-base``/``--no-stop-at-branch-base`` +The following options are configurable: + +`notesdir` + + The notes subdirectory within the `relnotesdir` where the notes live. + + Defaults to ``notes``. + +`collapse_pre_releases` + + Should pre-release versions be merged into the final release of the same + number (`1.0.0.0a1` notes appear under `1.0.0`). + + Defaults to ``True``. + +`stop_at_branch_base` + + Should the scanner stop at the base of a branch (True) or go ahead and scan + the entire history (False)? + + Defaults to ``True``. + +`branch` + + The git branch to scan. + + Defaults to the "current" branch checked out. + +`earliest_version` + + The earliest version to be included. This is usually the lowest version + number, and is meant to be the oldest version. If unset, all versions will be + scanned. + + Defaults to ``None``. + +`template` + + The template used by reno new to create a note. + +`release_tag_re` + + The regex pattern used to match the repo tags representing a valid release + version. The pattern is compiled with the verbose and unicode flags enabled. + + Defaults to ``((?:[\d.ab]|rc)+)``. + +`pre_release_tag_re` + + The regex pattern used to check if a valid release version tag is also a + valid pre-release version. The pattern is compiled with the verbose and + unicode flags enabled. The pattern must define a group called `pre_release` + that matches the pre-release part of the tag and any separator, e.g for + pre-release version `12.0.0.0rc1` the default RE pattern will identify + `.0rc1` as the value of the group 'pre_release'. + + Defaults to ``(?P\.\d+(?:[ab]|rc)+\d*)$``. + +`branch_name_re` + + The pattern for names for branches that are relevant when scanning history to + determine where to stop, to find the "base" of a branch. Other branches are + ignored. + + Defaults to ``stable/.+``. + +`sections` + + The identifiers and names of permitted sections in the release notes, in the + order in which the final report will be generated. A prelude section will + always be automatically inserted before the first element of this list. + Debugging ========= -- GitLab From 079cfc0f74fccfdeb6e2578ab2d85482c42103b8 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 13 Apr 2017 12:02:11 -0400 Subject: [PATCH 139/257] comment out openstack governance badges Comment out the governance badge link to avoid the warning from using a remote image. This can be reverted when a version of pbr with the fix is released. Addresses-Bug: #1682467 Change-Id: I9caf5b22d075b4ff3f6be71e23a98f3b5044747c Depends-On: If47e3ca6519cc9f70d62cd887707321fe9199f81 Signed-off-by: Doug Hellmann --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index bdeaf08..48651d2 100644 --- a/README.rst +++ b/README.rst @@ -48,7 +48,7 @@ from future documentation builds. Project Meta-data ================= -.. image:: http://governance.openstack.org/badges/reno.svg +.. .. image:: http://governance.openstack.org/badges/reno.svg :target: http://governance.openstack.org/reference/tags/index.html * Free software: Apache license -- GitLab From 3f02991ce7c26aaed146cf28c28e63159ceb084e Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 3 May 2017 11:59:03 -0400 Subject: [PATCH 140/257] deal with non-unique UIDs When we find multiple files added with the same UID in one patch, ignore them and emit a warning. That's bad input data, and we don't want to use them. Allow multiple files with the same UID to be deleted in a patch to support cleaning up existing situations where the add case was not caught properly. Related-Bug: #1688042 Change-Id: I37fee0660ff541677d26770818764f7de2a2d863 Signed-off-by: Doug Hellmann --- reno/scanner.py | 17 ++++++++++++++ reno/tests/test_scanner.py | 48 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/reno/scanner.py b/reno/scanner.py index aa33006..980b382 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -201,6 +201,23 @@ def _aggregate_changes(walk_entry, changes, notesdir): # different commits. for c in changes: results.append((uid, diff_tree.CHANGE_MODIFY, c[1], sha)) + elif types == set([diff_tree.CHANGE_DELETE]): + # There were multiple files in one commit using the + # same UID but different slugs. Treat them as + # different files and allow them to be deleted. + results.extend( + (uid, diff_tree.CHANGE_DELETE, c[1], sha) + for c in changes + ) + elif types == set([diff_tree.CHANGE_ADD]): + # There were multiple files in one commit using the + # same UID but different slugs. Warn the user about + # this case and then ignore the files. We allow delete + # (see above) to ensure they can be cleaned up. + LOG.warning( + ('%s: found several files in one commit (%s)' + ' with the same UID, ignoring them: %s'), + uid, sha, [c[1] for c in changes]) else: raise ValueError('Unrecognized changes: {!r}'.format(changes)) return results diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 2d5f923..fd3df34 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1794,6 +1794,29 @@ class AggregateChangesTest(Base): results, ) + def test_add_multiple(self): + # Adding multiple files in one commit using the same UID but + # different slug causes the files to be ignored. + entry = mock.Mock() + n = self.get_note_num() + changes = [] + for i in range(2): + name = 'prefix/add%d-%016x.yaml' % (i, n) + entry.commit.id = 'commit-id' + changes.append( + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ) + ) + ) + results = list(scanner._aggregate_changes(entry, changes, 'prefix')) + self.assertEqual([], results) + def test_delete(self): entry = mock.Mock() n = self.get_note_num() @@ -1816,6 +1839,31 @@ class AggregateChangesTest(Base): results, ) + def test_delete_multiple(self): + # Delete multiple files in one commit using the same UID but + # different slug. + entry = mock.Mock() + n = self.get_note_num() + changes = [] + expected = [] + for i in range(2): + name = 'prefix/delete%d-%016x.yaml' % (i, n) + entry.commit.id = 'commit-id' + changes.append( + diff_tree.TreeChange( + type=diff_tree.CHANGE_DELETE, + old=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ), + new=objects.TreeEntry(path=None, mode=None, sha=None), + ) + ) + expected.append(('%016x' % n, 'delete', name, 'commit-id')) + results = list(scanner._aggregate_changes(entry, changes, 'prefix')) + self.assertEqual(expected, results) + def test_change(self): entry = mock.Mock() n = self.get_note_num() -- GitLab From dd5487e5d3665198ee9e5b5b2c3d5582db931dec Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 3 May 2017 12:39:50 -0400 Subject: [PATCH 141/257] modify the change aggregation api Move _aggregate_changes() to a class so we can track state. Related-Bug: #1688042 Change-Id: I23a7add3a65c65e74b8e5b7378031346fe2a75b3 Signed-off-by: Doug Hellmann --- reno/scanner.py | 166 ++++++++++++++++++++----------------- reno/tests/test_scanner.py | 22 +++-- 2 files changed, 101 insertions(+), 87 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 980b382..95ceff6 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -121,7 +121,7 @@ def _changes_in_subdir(repo, walk_entry, subdir): return changes_func(store, parent_subtree, commit_subtree) -def _aggregate_changes(walk_entry, changes, notesdir): +class _ChangeAggregator(object): """Collapse a series of changes based on uniqueness for file uids. The list of TreeChange instances describe changes between the old @@ -140,87 +140,95 @@ def _aggregate_changes(walk_entry, changes, notesdir): The SHA values returned are for the commit, rather than the blob values in the TreeChange objects. - The path values in the change entries are encoded, so we decode - them to compare them against the notesdir and file pattern in - _note_file() and then return the decoded values to make consuming - them easier. + The path values in the change entries are encoded, so we return + the decoded values to make consuming them easier. """ - sha = walk_entry.commit.id - by_uid = collections.defaultdict(list) - for ec in changes: - if not isinstance(ec, list): - ec = [ec] - else: - ec = ec - for c in ec: - LOG.debug('change %r', c) - if c.type == diff_tree.CHANGE_ADD: - path = c.new.path.decode('utf-8') if c.new.path else None - if _note_file(path): - uid = _get_unique_id(path) - by_uid[uid].append((c.type, path, sha)) - else: - LOG.debug('ignoring') - elif c.type == diff_tree.CHANGE_DELETE: - path = c.old.path.decode('utf-8') if c.old.path else None - if _note_file(path): - uid = _get_unique_id(path) - by_uid[uid].append((c.type, path)) - else: - LOG.debug('ignoring') - elif c.type == diff_tree.CHANGE_MODIFY: - path = c.new.path.decode('utf-8') if c.new.path else None - if _note_file(path): - uid = _get_unique_id(path) - by_uid[uid].append((c.type, path, sha)) - else: - LOG.debug('ignoring') + + _rename_op = set([diff_tree.CHANGE_ADD, diff_tree.CHANGE_DELETE]) + _modify_op = set([diff_tree.CHANGE_MODIFY]) + _delete_op = set([diff_tree.CHANGE_DELETE]) + _add_op = set([diff_tree.CHANGE_ADD]) + + def aggregate_changes(self, walk_entry, changes): + sha = walk_entry.commit.id + by_uid = collections.defaultdict(list) + for ec in changes: + if not isinstance(ec, list): + ec = [ec] else: - raise ValueError('unhandled change type: {!r}'.format(c)) + ec = ec + for c in ec: + LOG.debug('change %r', c) + if c.type == diff_tree.CHANGE_ADD: + path = c.new.path.decode('utf-8') if c.new.path else None + if _note_file(path): + uid = _get_unique_id(path) + by_uid[uid].append((c.type, path, sha)) + else: + LOG.debug('ignoring') + elif c.type == diff_tree.CHANGE_DELETE: + path = c.old.path.decode('utf-8') if c.old.path else None + if _note_file(path): + uid = _get_unique_id(path) + by_uid[uid].append((c.type, path)) + else: + LOG.debug('ignoring') + elif c.type == diff_tree.CHANGE_MODIFY: + path = c.new.path.decode('utf-8') if c.new.path else None + if _note_file(path): + uid = _get_unique_id(path) + by_uid[uid].append((c.type, path, sha)) + else: + LOG.debug('ignoring') + else: + raise ValueError('unhandled change type: {!r}'.format(c)) - results = [] - for uid, changes in sorted(by_uid.items()): - if len(changes) == 1: - results.append((uid,) + changes[0]) - else: - types = set(c[0] for c in changes) - if types == set([diff_tree.CHANGE_ADD, diff_tree.CHANGE_DELETE]): - # A rename, combine the data from the add and delete entries. - added = [ - c for c in changes if c[0] == diff_tree.CHANGE_ADD - ][0] - deled = [ - c for c in changes if c[0] == diff_tree.CHANGE_DELETE - ][0] - results.append( - (uid, diff_tree.CHANGE_RENAME, deled[1]) + added[1:] - ) - elif types == set([diff_tree.CHANGE_MODIFY]): - # Merge commit with modifications to the same files in - # different commits. - for c in changes: - results.append((uid, diff_tree.CHANGE_MODIFY, c[1], sha)) - elif types == set([diff_tree.CHANGE_DELETE]): - # There were multiple files in one commit using the - # same UID but different slugs. Treat them as - # different files and allow them to be deleted. - results.extend( - (uid, diff_tree.CHANGE_DELETE, c[1], sha) - for c in changes - ) - elif types == set([diff_tree.CHANGE_ADD]): - # There were multiple files in one commit using the - # same UID but different slugs. Warn the user about - # this case and then ignore the files. We allow delete - # (see above) to ensure they can be cleaned up. - LOG.warning( - ('%s: found several files in one commit (%s)' - ' with the same UID, ignoring them: %s'), - uid, sha, [c[1] for c in changes]) + results = [] + for uid, changes in sorted(by_uid.items()): + if len(changes) == 1: + results.append((uid,) + changes[0]) else: - raise ValueError('Unrecognized changes: {!r}'.format(changes)) - return results + types = set(c[0] for c in changes) + if types == self._rename_op: + # A rename, combine the data from the add and + # delete entries. + added = [ + c for c in changes if c[0] == diff_tree.CHANGE_ADD + ][0] + deled = [ + c for c in changes if c[0] == diff_tree.CHANGE_DELETE + ][0] + results.append( + (uid, diff_tree.CHANGE_RENAME, deled[1]) + added[1:] + ) + elif types == self._modify_op: + # Merge commit with modifications to the same files in + # different commits. + for c in changes: + results.append((uid, diff_tree.CHANGE_MODIFY, + c[1], sha)) + elif types == self._delete_op: + # There were multiple files in one commit using the + # same UID but different slugs. Treat them as + # different files and allow them to be deleted. + results.extend( + (uid, diff_tree.CHANGE_DELETE, c[1], sha) + for c in changes + ) + elif types == self._add_op: + # There were multiple files in one commit using the + # same UID but different slugs. Warn the user about + # this case and then ignore the files. We allow delete + # (see above) to ensure they can be cleaned up. + msg = ('%s: found several files in one commit (%s)' + ' with the same UID, ignoring them: %s' % + (uid, sha, [c[1] for c in changes])) + LOG.warning(msg) + else: + raise ValueError('Unrecognized changes: {!r}'.format( + changes)) + return results class _ChangeTracker(object): @@ -951,6 +959,8 @@ class Scanner(object): if fname.startswith(prefix) and _note_file(fname): tracker.delete(fname, None, '*working-copy*') + aggregator = _ChangeAggregator() + # Process the git commit history. for counter, entry in enumerate(self._topo_traversal(branch), 1): @@ -974,7 +984,7 @@ class Scanner(object): # need to prefix that with the notesdir before giving it # to the tracker. changes = _changes_in_subdir(self._repo, entry, notesdir) - for change in _aggregate_changes(entry, changes, notesdir): + for change in aggregator.aggregate_changes(entry, changes): uniqueid = change[0] c_type = change[1] diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index fd3df34..7e1a750 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1750,6 +1750,10 @@ class VersionTest(Base): class AggregateChangesTest(Base): + def setUp(self): + super(AggregateChangesTest, self).setUp() + self.aggregator = scanner._ChangeAggregator() + def test_ignore(self): entry = mock.Mock() n = self.get_note_num() @@ -1766,7 +1770,7 @@ class AggregateChangesTest(Base): ) ) ] - results = scanner._aggregate_changes(entry, changes, 'prefix') + results = self.aggregator.aggregate_changes(entry, changes) self.assertEqual( [], results, @@ -1788,7 +1792,7 @@ class AggregateChangesTest(Base): ) ) ] - results = list(scanner._aggregate_changes(entry, changes, 'prefix')) + results = list(self.aggregator.aggregate_changes(entry, changes)) self.assertEqual( [('%016x' % n, 'add', name, 'commit-id')], results, @@ -1814,7 +1818,7 @@ class AggregateChangesTest(Base): ) ) ) - results = list(scanner._aggregate_changes(entry, changes, 'prefix')) + results = list(self.aggregator.aggregate_changes(entry, changes)) self.assertEqual([], results) def test_delete(self): @@ -1833,7 +1837,7 @@ class AggregateChangesTest(Base): new=objects.TreeEntry(path=None, mode=None, sha=None) ) ] - results = list(scanner._aggregate_changes(entry, changes, 'prefix')) + results = list(self.aggregator.aggregate_changes(entry, changes)) self.assertEqual( [('%016x' % n, 'delete', name)], results, @@ -1861,7 +1865,7 @@ class AggregateChangesTest(Base): ) ) expected.append(('%016x' % n, 'delete', name, 'commit-id')) - results = list(scanner._aggregate_changes(entry, changes, 'prefix')) + results = list(self.aggregator.aggregate_changes(entry, changes)) self.assertEqual(expected, results) def test_change(self): @@ -1884,7 +1888,7 @@ class AggregateChangesTest(Base): ), ) ] - results = list(scanner._aggregate_changes(entry, changes, 'prefix')) + results = list(self.aggregator.aggregate_changes(entry, changes)) self.assertEqual( [('%016x' % n, 'modify', name, 'commit-id')], results, @@ -1916,7 +1920,7 @@ class AggregateChangesTest(Base): new=objects.TreeEntry(path=None, mode=None, sha=None) ) ] - results = list(scanner._aggregate_changes(entry, changes, 'prefix')) + results = list(self.aggregator.aggregate_changes(entry, changes)) self.assertEqual( [('%016x' % n, 'rename', old_name, new_name, 'commit-id')], results, @@ -1948,7 +1952,7 @@ class AggregateChangesTest(Base): ) ), ] - results = list(scanner._aggregate_changes(entry, changes, 'prefix')) + results = list(self.aggregator.aggregate_changes(entry, changes)) self.assertEqual( [('%016x' % n, 'rename', old_name, new_name, 'commit-id')], results, @@ -1994,7 +1998,7 @@ class AggregateChangesTest(Base): ), ), ]] - results = list(scanner._aggregate_changes(entry, changes, 'prefix')) + results = list(self.aggregator.aggregate_changes(entry, changes)) self.assertEqual( [('%016x' % n, 'modify', old_name, 'commit-id'), ('%016x' % n, 'modify', old_name, 'commit-id')], -- GitLab From 8b1a3c652747f2d70c2136642ad5e1875971a870 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 3 May 2017 12:54:27 -0400 Subject: [PATCH 142/257] do not allow multiple files with the same UID Prevent someone from adding multiple files with the same UID in the same commit. Ignore any existing commits with the problem as long as there was a later commit to delete the files. Closes-Bug: #1688042 Change-Id: Id62361f3aba195417b369293e411c36172d27229 Signed-off-by: Doug Hellmann --- .../avoid-clashing-uids-e84ffe8132ce996d.yaml | 8 ++++ reno/scanner.py | 13 ++++- reno/tests/test_scanner.py | 48 +++++++++++++++++-- 3 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/avoid-clashing-uids-e84ffe8132ce996d.yaml diff --git a/releasenotes/notes/avoid-clashing-uids-e84ffe8132ce996d.yaml b/releasenotes/notes/avoid-clashing-uids-e84ffe8132ce996d.yaml new file mode 100644 index 0000000..4ea18c6 --- /dev/null +++ b/releasenotes/notes/avoid-clashing-uids-e84ffe8132ce996d.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fix a problem caused by failing to process multiple files with the + same UID portion of the filename. Ignore existing cases as long as + there is a corrective patch to remove them. Prevent new cases from + being introduced. See https://bugs.launchpad.net/reno/+bug/1688042 + for details. \ No newline at end of file diff --git a/reno/scanner.py b/reno/scanner.py index 95ceff6..e64fb97 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -150,6 +150,11 @@ class _ChangeAggregator(object): _delete_op = set([diff_tree.CHANGE_DELETE]) _add_op = set([diff_tree.CHANGE_ADD]) + def __init__(self): + # Track UIDs that had a duplication issue but have been + # deleted so we know not to throw an error for them. + self._deleted_bad_uids = set() + def aggregate_changes(self, walk_entry, changes): sha = walk_entry.commit.id by_uid = collections.defaultdict(list) @@ -216,15 +221,19 @@ class _ChangeAggregator(object): (uid, diff_tree.CHANGE_DELETE, c[1], sha) for c in changes ) + self._deleted_bad_uids.add(uid) elif types == self._add_op: # There were multiple files in one commit using the # same UID but different slugs. Warn the user about # this case and then ignore the files. We allow delete # (see above) to ensure they can be cleaned up. msg = ('%s: found several files in one commit (%s)' - ' with the same UID, ignoring them: %s' % + ' with the same UID: %s' % (uid, sha, [c[1] for c in changes])) - LOG.warning(msg) + if uid not in self._deleted_bad_uids: + raise ValueError(msg) + else: + LOG.warning(msg) else: raise ValueError('Unrecognized changes: {!r}'.format( changes)) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 7e1a750..1e6b9bc 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1798,14 +1798,16 @@ class AggregateChangesTest(Base): results, ) - def test_add_multiple(self): + def test_add_multiple_after_delete(self): # Adding multiple files in one commit using the same UID but - # different slug causes the files to be ignored. + # different slug after we have seen a delete for the same UID + # causes the files to be ignored. entry = mock.Mock() n = self.get_note_num() + uid = '%016x' % n changes = [] for i in range(2): - name = 'prefix/add%d-%016x.yaml' % (i, n) + name = 'prefix/add%d-%s.yaml' % (i, uid) entry.commit.id = 'commit-id' changes.append( diff_tree.TreeChange( @@ -1818,9 +1820,49 @@ class AggregateChangesTest(Base): ) ) ) + # Set up the aggregator as though it had already seen a delete + # operation. Since the scan happens in reverse chronological + # order, the delete would have happened after the add, and we + # can ignore the files because the error has been corrected in + # a later patch. + self.aggregator._deleted_bad_uids.add(uid) results = list(self.aggregator.aggregate_changes(entry, changes)) self.assertEqual([], results) + def test_add_multiple_without_delete(self): + # Adding multiple files in one commit using the same UID but + # different slug without a delete operation causes an + # exception. + entry = mock.Mock() + n = self.get_note_num() + uid = '%016x' % n + changes = [] + for i in range(2): + name = 'prefix/add%d-%s.yaml' % (i, uid) + entry.commit.id = 'commit-id' + changes.append( + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ) + ) + ) + + # aggregate_changes() is a generator, so we have to wrap it in + # list() to process the data, so we need a little temporary + # function to do that and pass to assertRaises(). + def get_results(): + return list(self.aggregator.aggregate_changes(entry, changes)) + + self.assertRaises( + ValueError, + get_results, + ) + def test_delete(self): entry = mock.Mock() n = self.get_note_num() -- GitLab From 06d6574d46091d48b9c78878cac04f639aec39cc Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 3 May 2017 15:35:27 -0400 Subject: [PATCH 143/257] add a lint command Provide a tool for doing some basic input validation. Related-Bug: #1688042 Change-Id: I850b57153c5286e19f4ac3af899b3d798aebd7d4 Signed-off-by: Doug Hellmann --- doc/source/usage.rst | 9 ++++ .../notes/add-linter-ce0a861ade64baf2.yaml | 5 ++ reno/linter.py | 54 +++++++++++++++++++ reno/main.py | 13 +++++ tox.ini | 4 +- 5 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/add-linter-ce0a861ade64baf2.yaml create mode 100644 reno/linter.py diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 5299777..445e252 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -162,6 +162,15 @@ Notes are output in the order they are found when scanning the git history of the branch using topological ordering. This is deterministic, but not necessarily predictable or mutable. +Checking Notes +============== + +Run ``reno lint `` to test the existing +release notes files against some rules for catching common +mistakes. The command exits with an error code if there are any +mistakes, so it can be used in a build pipeline to force some +correctness. + Configuring Reno ================ diff --git a/releasenotes/notes/add-linter-ce0a861ade64baf2.yaml b/releasenotes/notes/add-linter-ce0a861ade64baf2.yaml new file mode 100644 index 0000000..b36983a --- /dev/null +++ b/releasenotes/notes/add-linter-ce0a861ade64baf2.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add a ``lint`` command for checking the contents and names of the + release notes files against some basic validation rules. \ No newline at end of file diff --git a/reno/linter.py b/reno/linter.py new file mode 100644 index 0000000..5071842 --- /dev/null +++ b/reno/linter.py @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import glob +import logging +import os.path + +from reno import loader +from reno import scanner + +LOG = logging.getLogger(__name__) + + +def lint_cmd(args, conf): + "Check some common mistakes" + LOG.debug('starting lint') + notesdir = os.path.join(conf.reporoot, conf.notespath) + notes = glob.glob(os.path.join(notesdir, '*.yaml')) + + error = 0 + load = loader.Loader(conf, ignore_cache=True) + + allowed_section_names = ['prelude'] + [s[0] for s in conf.sections] + + uids = {} + for f in notes: + LOG.debug('examining %s', f) + uid = scanner._get_unique_id(f) + uids.setdefault(uid, []).append(f) + + content = load.parse_note_file(f, None) + for section_name in content.keys(): + if section_name not in allowed_section_names: + LOG.warning('unrecognized section name %s in %s', + section_name, f) + error = 1 + + for uid, names in sorted(uids.items()): + if len(names) > 1: + LOG.warning('UID collision: %s', names) + error = 1 + + return error diff --git a/reno/main.py b/reno/main.py index dd062fe..3237661 100644 --- a/reno/main.py +++ b/reno/main.py @@ -18,6 +18,7 @@ from reno import cache from reno import config from reno import create from reno import defaults +from reno import linter from reno import lister from reno import report @@ -173,6 +174,18 @@ def main(argv=sys.argv[1:]): _build_query_arg_group(do_cache) do_cache.set_defaults(func=cache.cache_cmd) + do_linter = subparsers.add_parser( + 'lint', + help='check some common mistakes', + ) + do_linter.add_argument( + 'reporoot', + default='.', + nargs='?', + help='root of the git repository', + ) + do_linter.set_defaults(func=linter.lint_cmd) + args = parser.parse_args(argv) conf = config.Config(args.reporoot, args.relnotesdir) conf.override_from_parsed_args(args) diff --git a/tox.ini b/tox.ini index 7b45ac9..88a0101 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,9 @@ commands = coverage report --show-missing [testenv:pep8] -commands = flake8 +commands = + flake8 + reno -q lint [testenv:venv] commands = {posargs} -- GitLab From c985f88a530c6b56c4b69271e50de128e80da702 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 17 May 2017 14:42:23 -0400 Subject: [PATCH 144/257] do not use sphinx 1.6.1 Change-Id: I91b606b8f46037b3f75ccbc51d921f060e055f09 Related-Bug: #465135 Signed-off-by: Doug Hellmann --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 19bd043..2389972 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ console_scripts = [extras] sphinx = - sphinx>=1.5.1 # BSD + sphinx>=1.5.1,!=1.6.1 # BSD docutils>=0.11 # OSI-Approved Open Source, Public Domain [build_sphinx] -- GitLab From 65a69398b8c7b8452303ad360d51c489b022ab00 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 16 May 2017 14:13:03 -0400 Subject: [PATCH 145/257] lower the log level for an error message Sphinx 1.6.1 now interprets error and warning log messages as reasons to abort the build when strict mode is enabled. Change the log level for some calls that weren't really errors to begin with. Change-Id: I688ee8b57e839ba6146633365be9ba8f92e3c7df Closes-Bug: #1691224 Signed-off-by: Doug Hellmann --- reno/scanner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index e64fb97..6296a74 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -64,7 +64,7 @@ def _note_file(name): if fnmatch.fnmatch(name, '*.yaml'): return True else: - LOG.warning('found and ignored extra file %s', name) + LOG.info('found and ignored extra file %s', name) return False @@ -233,7 +233,7 @@ class _ChangeAggregator(object): if uid not in self._deleted_bad_uids: raise ValueError(msg) else: - LOG.warning(msg) + LOG.info(msg) else: raise ValueError('Unrecognized changes: {!r}'.format( changes)) @@ -634,7 +634,7 @@ class Scanner(object): return tags[-1] else: # Naughty, naughty, branching without tagging. - LOG.error( + LOG.info( ('There is no tag on commit %s at the base of %s. ' 'Branch scan short-cutting is disabled.'), c.commit.sha().hexdigest(), branch) -- GitLab From 2d0d05d3019376af6377f0d47e06ac5bea88c31e Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 18 May 2017 11:41:53 -0400 Subject: [PATCH 146/257] add release note for log level fix Change-Id: I7dff8aa13a414b7b625636c1793774da4638bd04 Signed-off-by: Doug Hellmann --- .../notes/log-levels-and-sphinx-161-6efe0d291718a657.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 releasenotes/notes/log-levels-and-sphinx-161-6efe0d291718a657.yaml diff --git a/releasenotes/notes/log-levels-and-sphinx-161-6efe0d291718a657.yaml b/releasenotes/notes/log-levels-and-sphinx-161-6efe0d291718a657.yaml new file mode 100644 index 0000000..f54a20e --- /dev/null +++ b/releasenotes/notes/log-levels-and-sphinx-161-6efe0d291718a657.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Sphinx 1.6.1 now interprets error and warning log messages as + reasons to abort the build when strict mode is enabled. This + release changes the log level for some calls that weren't really + errors to begin with to avoid having Sphinx abort the build + unnecessarily. \ No newline at end of file -- GitLab From e2d60c07931f9095d5e4d9a45535f5ce1a7e0dd3 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 24 May 2017 15:44:20 -0400 Subject: [PATCH 147/257] fix the logic for deciding what to show as the "current" series Stop at the first version after the most recent branch was created, instead of always only showing the most current version. Change-Id: I58326c9e30349d2d7c473558b9aa2e8f7294c652 Closes-Bug: #1682147 Signed-off-by: Doug Hellmann --- reno/scanner.py | 11 ++++++++++- reno/tests/test_scanner.py | 4 +++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 6296a74..2df1166 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -887,7 +887,16 @@ class Scanner(object): LOG.debug('looking at base of %s to stop scanning master', branches[-1]) scan_stop_tag = self._get_branch_base(branches[-1]) - earliest_version = current_version + # If there is a tag on this branch after the point + # where the earlier branch was created, then use that + # tag as the earliest version to show in the current + # "series". If there is no such tag, then go all the + # way to the base of that earlier branch. + try: + idx = versions_by_date.index(scan_stop_tag) + 1 + earliest_version = versions_by_date[idx] + except IndexError: + earliest_version = scan_stop_tag elif branch and stop_at_branch_base and not earliest_version: # If branch is set and is not "master", # then we want to stop at the version before the tag at the diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 1e6b9bc..486d1a0 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -612,7 +612,7 @@ class BasicTest(Base): self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') self._add_notes_file() self.repo.git('tag', '-s', '-m', 'middle tag', '2.0.0') - self._add_notes_file() + f3 = self._add_notes_file() self.repo.git('tag', '-s', '-m', 'last tag', '3.0.0') self.repo.git('branch', 'stable/a') f4 = self._add_notes_file() @@ -627,6 +627,7 @@ class BasicTest(Base): } self.assertEqual( {'3.0.0-1': [f4], + '3.0.0': [f3], }, results, ) @@ -1112,6 +1113,7 @@ class BranchTest(Base): self.assertEqual( { '2.0.0-1': [f21], + '2.0.0': [self.f2], }, results, ) -- GitLab From 320736bdbf7ad4626f352247c6aa9f1109d5527c Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Thu, 25 May 2017 07:05:31 +0200 Subject: [PATCH 148/257] Remove oslotest from test-requirements.txt It is not used. Change-Id: I50604b19e603c4508ef61fdd32091a10235f1e5a --- test-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2f939a5..9680a52 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,7 +9,6 @@ mock>=1.2 coverage>=3.6 python-subunit>=0.0.18 oslosphinx>=2.5.0 # Apache-2.0 -oslotest>=1.10.0 # Apache-2.0 testrepository>=0.0.18 testscenarios>=0.4 testtools>=1.4.0 -- GitLab From e3dcbdd582b950504a17147b60e02904f3a5e8c8 Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Mon, 29 May 2017 19:10:59 +0200 Subject: [PATCH 149/257] Make oslosphinx requirement optional There is a build cycle between reno and oslosphinx. Now the documentation can be build without oslosphinx installed. Change-Id: Iae6abbe5b2991123f56bf2f4e852c57cd9ca6c11 --- doc/source/conf.py | 14 +++++++++++++- .../optional-oslosphinx-55843a7f80a14e58.yaml | 5 +++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/optional-oslosphinx-55843a7f80a14e58.yaml diff --git a/doc/source/conf.py b/doc/source/conf.py index d881af5..494bd24 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -15,6 +15,16 @@ import os import sys +# oslosphinx uses reno and reno uses oslosphinx. Make oslosphinx for +# reno optional to break the build cycle +try: + import oslosphinx +except: + has_oslosphinx = False +else: + has_oslosphinx = True + + sys.path.insert(0, os.path.abspath('../..')) # -- General configuration ---------------------------------------------------- @@ -23,10 +33,12 @@ sys.path.insert(0, os.path.abspath('../..')) extensions = [ 'sphinx.ext.autodoc', #'sphinx.ext.intersphinx', - 'oslosphinx', 'reno.sphinxext', ] +if has_oslosphinx: + extensions.append('oslosphinx') + # autodoc generation is a bit aggressive and a nuisance when doing heavy # text edit cycles. # execute "export SPHINX_DEBUG=1" in your terminal to disable diff --git a/releasenotes/notes/optional-oslosphinx-55843a7f80a14e58.yaml b/releasenotes/notes/optional-oslosphinx-55843a7f80a14e58.yaml new file mode 100644 index 0000000..ee0252f --- /dev/null +++ b/releasenotes/notes/optional-oslosphinx-55843a7f80a14e58.yaml @@ -0,0 +1,5 @@ +--- +other: + - The oslosphinx dependency for building documentation + is now optional. This breaks a build cycle between + oslosphinx and reno. -- GitLab From 2f5ad2897f36aba724d599941f4fdd05715118f8 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 1 Jun 2017 09:53:51 +0200 Subject: [PATCH 150/257] Remove Babel from setup.cfg and requirements It's not imported by reno at runtime and no translation has been setup at all. Change-Id: I0f36fc6b8d9112752b13028ba5b07f2a05504f5b --- babel.cfg | 2 -- requirements.txt | 1 - setup.cfg | 5 ----- 3 files changed, 8 deletions(-) delete mode 100644 babel.cfg diff --git a/babel.cfg b/babel.cfg deleted file mode 100644 index 15cd6cb..0000000 --- a/babel.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[python: **.py] - diff --git a/requirements.txt b/requirements.txt index 5fb4f35..2f2a16e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ # process, which may cause wedges in the gate later. pbr>=1.4 -Babel>=1.3 PyYAML>=3.1.0 six>=1.9.0 dulwich>=0.15.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 2389972..a45143c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,8 +48,3 @@ domain = reno domain = reno output_dir = reno/locale input_file = reno/locale/reno.pot - -[extract_messages] -keywords = _ gettext ngettext l_ lazy_gettext -mapping_file = babel.cfg -output_file = reno/locale/reno.pot -- GitLab From 5cefb37405522445e27cb5a396626c3bb4aa680d Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 1 Jun 2017 10:36:29 -0400 Subject: [PATCH 151/257] fix the way we handle deleted notes The aggregated change information for a deleted note includes the SHA as well as the filename. Extract both from the tuple separately so the path value is set correctly to a filename. Change-Id: I41fb4a7a0f6d24af47c8f059945b9bf45f859f15 Signed-off-by: Doug Hellmann --- .../notes/fix-delete-handling-55232c50b647aa57.yaml | 6 ++++++ reno/scanner.py | 4 ++-- reno/tests/test_scanner.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/fix-delete-handling-55232c50b647aa57.yaml diff --git a/releasenotes/notes/fix-delete-handling-55232c50b647aa57.yaml b/releasenotes/notes/fix-delete-handling-55232c50b647aa57.yaml new file mode 100644 index 0000000..eaca402 --- /dev/null +++ b/releasenotes/notes/fix-delete-handling-55232c50b647aa57.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Correct a problem with handling deleted release notes that + triggered a TypeError with a message like "Can't mix strings and + bytes in path components" \ No newline at end of file diff --git a/reno/scanner.py b/reno/scanner.py index 2df1166..2f1587a 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -176,7 +176,7 @@ class _ChangeAggregator(object): path = c.old.path.decode('utf-8') if c.old.path else None if _note_file(path): uid = _get_unique_id(path) - by_uid[uid].append((c.type, path)) + by_uid[uid].append((c.type, path, sha)) else: LOG.debug('ignoring') elif c.type == diff_tree.CHANGE_MODIFY: @@ -1013,7 +1013,7 @@ class Scanner(object): tracker.add(fullpath, sha, current_version) elif c_type == diff_tree.CHANGE_DELETE: - path = change[-1] + path, blob_sha = change[-2:] fullpath = os.path.join(notesdir, path) tracker.delete(fullpath, sha, current_version) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 486d1a0..a9f8a36 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1883,7 +1883,7 @@ class AggregateChangesTest(Base): ] results = list(self.aggregator.aggregate_changes(entry, changes)) self.assertEqual( - [('%016x' % n, 'delete', name)], + [('%016x' % n, 'delete', name, entry.commit.id)], results, ) -- GitLab From ecca68b147c44c96d17a95995897746b542984ec Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 1 Jun 2017 14:12:02 -0400 Subject: [PATCH 152/257] do not assume the current branch is the most recent Look through the other branches to find the previous branch and determine where to stop scanning, instead of assuming the current branch is the most recent and trying to artificially compute the previous branch. Change-Id: If905575a47c828ebe43e79a6c0f363eaa3226f6e Signed-off-by: Doug Hellmann --- reno/scanner.py | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 2f1587a..9230107 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -884,19 +884,35 @@ class Scanner(object): LOG.debug('working on current branch without earliest_version') branches = self._get_series_branches() if branches: - LOG.debug('looking at base of %s to stop scanning master', - branches[-1]) - scan_stop_tag = self._get_branch_base(branches[-1]) - # If there is a tag on this branch after the point - # where the earlier branch was created, then use that - # tag as the earliest version to show in the current - # "series". If there is no such tag, then go all the - # way to the base of that earlier branch. - try: - idx = versions_by_date.index(scan_stop_tag) + 1 - earliest_version = versions_by_date[idx] - except IndexError: - earliest_version = scan_stop_tag + for earlier_branch in reversed(branches): + LOG.debug('checking if current branch is later than %s', + earlier_branch) + scan_stop_tag = self._get_branch_base(earlier_branch) + if scan_stop_tag in versions_by_date: + LOG.info( + 'looking at %s at base of %s to ' + 'stop scanning the current branch', + scan_stop_tag, earlier_branch + ) + break + else: + LOG.info('unable to find the previous branch base') + scan_stop_tag = None + if scan_stop_tag: + # If there is a tag on this branch after the point + # where the earlier branch was created, then use that + # tag as the earliest version to show in the current + # "series". If there is no such tag, then go all the + # way to the base of that earlier branch. + try: + idx = versions_by_date.index(scan_stop_tag) + 1 + earliest_version = versions_by_date[idx] + except ValueError: + # The scan_stop_tag is not in versions_by_date. + earliest_version = None + except IndexError: + # The idx is not in versions_by_date. + earliest_version = scan_stop_tag elif branch and stop_at_branch_base and not earliest_version: # If branch is set and is not "master", # then we want to stop at the version before the tag at the -- GitLab From a26e9820dda7aa57b5d96ee6c75dff8d961dbbf0 Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Thu, 1 Jun 2017 14:30:43 -0400 Subject: [PATCH 153/257] Document how reno handles eol branches in config This comes up from time to time so we should probably document it. Change-Id: I58fef297612328ee1db61377c29e1cfe0ec6f5ab --- doc/source/usage.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 445e252..51eb650 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -234,7 +234,9 @@ The following options are configurable: `branch` - The git branch to scan. + The git branch to scan. If a stable branch is specified but does not exist, + reno attempts to automatically convert that to an "end-of-life" tag. For + example, ``origin/stable/liberty`` would be converted to ``liberty-eol``. Defaults to the "current" branch checked out. -- GitLab From bd6fecc8587ee919eba78b9fd70a17e6a5ad510a Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 1 Jun 2017 17:32:24 -0400 Subject: [PATCH 154/257] ignore null-merges OpenStack used to use null-merges to bring final release tags from stable branches back into the master branch. This confuses the regular traversal because it makes that stable branch appear to be part of master and/or the later stable branch. Update the scanner so that when it hit one of those merge commits, it skips it and take the first parent so it continues to traverse the branch being scanned. Change-Id: I90722a3946f691e8f58a52e68ee455d6530f047a Closes-Bug: #1695057 Signed-off-by: Doug Hellmann --- doc/source/usage.rst | 16 ++++ .../ignore-null-merges-56b7a8ed9b20859e.yaml | 18 +++++ reno/config.py | 12 +++ reno/scanner.py | 46 +++++++++++ reno/tests/test_scanner.py | 77 +++++++++++++++++++ 5 files changed, 169 insertions(+) create mode 100644 releasenotes/notes/ignore-null-merges-56b7a8ed9b20859e.yaml diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 445e252..3564f87 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -282,6 +282,22 @@ The following options are configurable: order in which the final report will be generated. A prelude section will always be automatically inserted before the first element of this list. +`ignore_null_merges` + + OpenStack used to use null-merges to bring final release tags from + stable branches back into the master branch. This confuses the + regular traversal because it makes that stable branch appear to be + part of master and/or the later stable branch. This option allows us + to ignore those. + + When this option is set to True, any merge commits with no changes + and in which the second or later parent is tagged are considered + "null-merges" that bring the tag information into the current branch + but nothing else. + + Defaults to ``True``. + + Debugging ========= diff --git a/releasenotes/notes/ignore-null-merges-56b7a8ed9b20859e.yaml b/releasenotes/notes/ignore-null-merges-56b7a8ed9b20859e.yaml new file mode 100644 index 0000000..03f3da6 --- /dev/null +++ b/releasenotes/notes/ignore-null-merges-56b7a8ed9b20859e.yaml @@ -0,0 +1,18 @@ +--- +features: + - | + By default, reno now ignores "null" merge commits that bring in + tags from other threads. The new configuration option + ``ignore_null_merges`` controls this behavior. Setting the flag to + False restores the previous behavior in which the null-merge + commits were traversed like any other merge commit. +upgrade: + - | + The new configuration option ``ignore_null_merges`` causes the + scanner to ignore merge commits with no changes when one of the + parents being merged in has a release tag on it. +fixes: + - | + This release fixes a problem with the scanner that may have caused + it to stop scanning a branch prematurely when the tag from another + branch had been merged into the history. diff --git a/reno/config.py b/reno/config.py index b64f95c..6a67515 100644 --- a/reno/config.py +++ b/reno/config.py @@ -155,6 +155,18 @@ class Config(object): ['fixes', 'Bug Fixes'], ['other', 'Other Notes'], ], + + # When this option is set to True, any merge commits with no + # changes and in which the second or later parent is tagged + # are considered "null-merges" that bring the tag information + # into the current branch but nothing else. + # + # OpenStack used to use null-merges to bring final release + # tags from stable branches back into the master branch. This + # confuses the regular traversal because it makes that stable + # branch appear to be part of master and/or the later stable + # branch. This option allows us to ignore those. + 'ignore_null_merges': True, } @classmethod diff --git a/reno/scanner.py b/reno/scanner.py index 9230107..36fc33b 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -685,9 +685,55 @@ class Scanner(object): todo = collections.deque() todo.appendleft(head) + ignore_null_merges = self.conf.ignore_null_merges + if ignore_null_merges: + LOG.debug('ignoring null-merge commits') + while todo: sha = todo.popleft() entry = all[sha] + null_merge = False + + # OpenStack used to use null-merges to bring final release + # tags from stable branches back into the master + # branch. This confuses the regular traversal because it + # makes that stable branch appear to be part of master + # and/or the later stable branch. When we hit one of those + # tags, skip it and take the first parent. + if ignore_null_merges and len(entry.commit.parents) > 1: + # Look for tags on the 2nd and later parents. The + # first parent is part of the branch we were + # originally trying to traverse, and any tags on it + # need to be kept. + for p in entry.commit.parents[1:]: + t = self._get_valid_tags_on_commit(p) + # If we have a tag being merged in, we need to + # include a check to verify that this is actually + # a null-merge (there are no changes). + if t and not entry.changes(): + LOG.debug( + 'treating %s as a null-merge because ' + 'parent %s has tag(s) %s', + sha, p, t, + ) + null_merge = True + break + if null_merge: + # Make it look like the parent entries that we're + # going to skip have been emitted so the + # bookkeeping for children works properly and we + # can continue past the merge. + emitted.update(set(entry.commit.parents[1:])) + # Make it look like the current entry was emitted + # so the bookkeeping for children works properly + # and we can continue past the merge. + emitted.add(sha) + # Now set up the first parent so it is processed + # later. + first_parent = entry.commit.parents[0] + if first_parent not in todo: + todo.appendleft(first_parent) + continue # If a node has multiple children, it is the start point # for a branch that was merged back into the rest of the diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index a9f8a36..f2db5d6 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1010,6 +1010,83 @@ class MergeCommitTest(Base): ) +class NullMergeTest(Base): + + def setUp(self): + super(NullMergeTest, self).setUp() + self.repo.add_file('ignore-0.txt') + self.n1 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + + # Create a branch, add a note, and tag it. + self.repo.git('checkout', '-b', 'test_ignore_null_merge') + self.n2 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.0') + + # Move back to master and advance it. + self.repo.git('checkout', 'master') + self.repo.add_file('ignore-1.txt') + self.n3 = self._add_notes_file() + + # Merge only the tag from the first branch back into master. + self.repo.git( + 'merge', '--no-ff', '--strategy', 'ours', '2.0.0', + ) + + # Add another note file. + self.n4 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'third tag', '3.0.0') + + self.repo.git('log', '--decorate', '--oneline', '--graph', '--all') + # The results should look like: + # + # * afea344 (HEAD -> master, tag: 3.0.0) add slug-0000000000000004.yaml + # * 7bb295c Merge tag '2.0.0' + # |\ + # | * 260c80b (tag: 2.0.0, test_ignore_null_merge) add slug-0000000000000002.yaml # noqa + # * | 5981ae3 add slug-0000000000000003.yaml + # * | 00f9376 add ignore-1.txt + # |/ + # * d24faf9 (tag: 1.0.0) add slug-0000000000000001.yaml + # * 6c221cd add ignore-0.txt + + def test_ignore(self): + # The scanner should skip over the null-merge and include the + # notes that come before the version being merged in, up to + # the base of the previous branch. + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0': [self.n1], + '3.0.0': [self.n3, self.n4]}, + results, + ) + + def test_follow(self): + # The scanner should not skip over the null-merge. The output + # should include the 2.0.0 tag that was merged in, as well as + # the earlier 1.0.0 version. + self.c.override( + ignore_null_merges=False, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0': [self.n1], + '2.0.0': [self.n2, self.n3], + '3.0.0': [self.n4]}, + results, + ) + + class UniqueIdTest(Base): def test_legacy(self): -- GitLab From bc3d1241dd842dcfb8797747b4083ba93ffd33cb Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 5 Jun 2017 15:33:16 -0400 Subject: [PATCH 155/257] allow release notes sections to be single strings Release notes entries may now be made up of single strings. This simplifies formatting for smaller notes, and eliminates a class of errors associated with escaping reStructuredText inside YAML lists. Change-Id: I7f2fb2d2fd16f49e7ee061582df7bcdd4116f215 Signed-off-by: Doug Hellmann --- .../add-complex-example-6b5927c246456896.yaml | 3 +++ .../flexible-formatting-31c8de2599d3637d.yaml | 5 +++++ reno/loader.py | 14 +++++++++++--- reno/tests/test_loader.py | 6 +++--- 4 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/flexible-formatting-31c8de2599d3637d.yaml diff --git a/examples/notes/add-complex-example-6b5927c246456896.yaml b/examples/notes/add-complex-example-6b5927c246456896.yaml index dfd9a0c..a104287 100644 --- a/examples/notes/add-complex-example-6b5927c246456896.yaml +++ b/examples/notes/add-complex-example-6b5927c246456896.yaml @@ -26,3 +26,6 @@ other: This example is also rendered correctly on multiple lines as a pre-formatted block. +features: + This note is a simple string, and does not retain its + formatting when it is rendered in HTML. diff --git a/releasenotes/notes/flexible-formatting-31c8de2599d3637d.yaml b/releasenotes/notes/flexible-formatting-31c8de2599d3637d.yaml new file mode 100644 index 0000000..a761130 --- /dev/null +++ b/releasenotes/notes/flexible-formatting-31c8de2599d3637d.yaml @@ -0,0 +1,5 @@ +--- +features: + Release notes entries may now be made up of single strings. This + simplifies formatting for smaller notes, and eliminates a class of + errors associated with escaping reStructuredText inside YAML lists. diff --git a/reno/loader.py b/reno/loader.py index f11196e..3df4988 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -101,6 +101,8 @@ class Loader(object): body = self._scanner.get_file_at_commit(filename, sha) content = yaml.safe_load(body) + cleaned_content = {} + for section_name, section_content in content.items(): if section_name == 'prelude': if not isinstance(section_content, six.string_types): @@ -111,10 +113,15 @@ class Loader(object): filename, ) else: - if not isinstance(section_content, list): + if isinstance(section_content, six.string_types): + # A single string is OK, but wrap it with a list + # so the rest of the code can treat the data model + # consistently. + section_content = [section_content] + elif not isinstance(section_content, list): LOG.warning( ('The %s section of %s ' - 'does not parse as a list of strings. ' + 'does not parse as a string or list of strings. ' 'Is the YAML input escaped properly?') % ( section_name, filename), ) @@ -128,5 +135,6 @@ class Loader(object): ) % (item, section_name, filename, type(item)), ) + cleaned_content[section_name] = section_content - return content + return cleaned_content diff --git a/reno/tests/test_loader.py b/reno/tests/test_loader.py index 8ca9cf8..33b6d28 100644 --- a/reno/tests/test_loader.py +++ b/reno/tests/test_loader.py @@ -67,7 +67,7 @@ class TestValidate(base.TestCase): ldr.parse_note_file('note1', None) self.assertIn('prelude', self.logger.output) - def test_non_prelude_single_string(self): + def test_non_prelude_single_string_converted_to_list(self): note_bodies = yaml.safe_load(textwrap.dedent(''' issues: | This is a single string. @@ -75,8 +75,8 @@ class TestValidate(base.TestCase): print(type(note_bodies['issues'])) self.assertIsInstance(note_bodies['issues'], six.string_types) ldr = self._make_loader(note_bodies) - ldr.parse_note_file('note1', None) - self.assertIn('list of strings', self.logger.output) + parse_results = ldr.parse_note_file('note1', None) + self.assertIsInstance(parse_results['issues'], list) def test_note_with_colon_as_dict(self): note_bodies = yaml.safe_load(textwrap.dedent(''' -- GitLab From 8666d22065dc0b2d80d3b48b3310c20b3d601f1e Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 5 Jun 2017 15:51:06 -0400 Subject: [PATCH 156/257] expand examples in documentation Add examples showing how sections that take lists of strings can also include a single string, and expand on why the escaped rst formatting works. Change-Id: I26be8c3027aebbfb1bf4a6f17c6f995dc44aac1a Signed-off-by: Doug Hellmann --- doc/source/examples.rst | 17 ++++++++++++++++- .../add-complex-example-6b5927c246456896.yaml | 16 +++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/doc/source/examples.rst b/doc/source/examples.rst index f71619f..872b620 100644 --- a/doc/source/examples.rst +++ b/doc/source/examples.rst @@ -1,2 +1,17 @@ -.. release-notes:: Examples +========== + Examples +========== + +Input file +========== + +.. literalinclude:: ../../examples/notes/add-complex-example-6b5927c246456896.yaml + :caption: examples/notes/add-complex-example-6b5927c246456896.yaml + :language: yaml + +Rendered +======== + +.. release-notes:: :relnotessubdir: examples + :earliest-version: 1.0.0 diff --git a/examples/notes/add-complex-example-6b5927c246456896.yaml b/examples/notes/add-complex-example-6b5927c246456896.yaml index a104287..24c31e1 100644 --- a/examples/notes/add-complex-example-6b5927c246456896.yaml +++ b/examples/notes/add-complex-example-6b5927c246456896.yaml @@ -14,18 +14,24 @@ prelude: | | with | so the reStructuredText | parser will retain | the line breaks. +features: + This note is a simple string, and does not retain its + formatting when it is rendered in HTML. rst markup here + may break the YAML parser, since the string is not escaped. +fixes: + - Use YAML lists to add multiple items to the same section. + - Another fix could be listed here. other: - | - This bullet item includes a paragraph and a nested list. + This bullet item includes a paragraph and a nested list, + which works because the content of the YAML list item + is an escaped string block with reStructuredText formatting. * list item 1 * list item 2 - :: + .. code-block:: text This example is also rendered correctly on multiple lines as a pre-formatted block. -features: - This note is a simple string, and does not retain its - formatting when it is rendered in HTML. -- GitLab From f957e74ff96038e69f2ffaee69b1a5e3f0727380 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 6 Jun 2017 10:40:58 -0400 Subject: [PATCH 157/257] add option for ignoring some notes files Make it easier to produce clean release notes by ignoring files mistakenly edited on the wrong branch. Change-Id: I74fb9e6c74af0b9de8cfe0d9c07ecfbd09cae925 Signed-off-by: Doug Hellmann --- doc/source/sphinxext.rst | 7 ++ doc/source/usage.rst | 13 +++ .../ignore-notes-option-9d0bde540fbcdf22.yaml | 8 ++ reno/config.py | 6 ++ reno/scanner.py | 9 ++ reno/sphinxext.py | 7 ++ reno/tests/test_scanner.py | 82 +++++++++++++++++++ 7 files changed, 132 insertions(+) create mode 100644 releasenotes/notes/ignore-notes-option-9d0bde540fbcdf22.yaml diff --git a/doc/source/sphinxext.rst b/doc/source/sphinxext.rst index 1ccf4d7..fb2c5f9 100644 --- a/doc/source/sphinxext.rst +++ b/doc/source/sphinxext.rst @@ -59,6 +59,13 @@ Enable the extension by adding ``'reno.sphinxext'`` to the typically set to the version used to create the branch to limit the output to only versions on that branch. + *ignore-notes* + + A string containing a comma-delimited list of filenames or UIDs + for notes that should be ignored by the scanner. It is most + useful to set this when a note is edited on the wrong branch, + making it appear to be part of a release that it is not. + Examples ======== diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 420c729..5459af4 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -299,6 +299,19 @@ The following options are configurable: Defaults to ``True``. +`ignore_notes` + + A list of filenames or UIDs for notes that should be ignored by the + reno scanner. It is most useful to set this when a note is edited on + the wrong branch, making it appear to be part of a release that it + is not. + + .. warning:: + + Setting the option in the main configuration file makes it apply + to all branches. To ignore a note in the HTML build, use the + ``ignore-notes`` parameter to the ``release-notes`` sphinx + directive. Debugging ========= diff --git a/releasenotes/notes/ignore-notes-option-9d0bde540fbcdf22.yaml b/releasenotes/notes/ignore-notes-option-9d0bde540fbcdf22.yaml new file mode 100644 index 0000000..6fd41e4 --- /dev/null +++ b/releasenotes/notes/ignore-notes-option-9d0bde540fbcdf22.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Add a new configuration option ``ignore_notes``. Setting the value + to a list of filenames or UIDs for notes causes the reno scanner + to ignore them. It is most useful to set this when a note is + edited on the wrong branch, making it appear to be part of a + release that it is not. diff --git a/reno/config.py b/reno/config.py index 6a67515..03d0362 100644 --- a/reno/config.py +++ b/reno/config.py @@ -167,6 +167,12 @@ class Config(object): # branch appear to be part of master and/or the later stable # branch. This option allows us to ignore those. 'ignore_null_merges': True, + + # Note files to be ignored. It's useful to be able to ignore a + # file if it is edited on the wrong branch. Notes should be + # specified by their filename or UID. Setting the value in the + # configuration file makes it apply to all branches. + 'ignore_notes': [], } @classmethod diff --git a/reno/scanner.py b/reno/scanner.py index 36fc33b..61929b0 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -514,6 +514,10 @@ class Scanner(object): self.conf.branch_name_re, flags=re.VERBOSE | re.UNICODE, ) + self._ignore_uids = set( + _get_unique_id(fn) + for fn in self.conf.ignore_notes + ) def _get_ref(self, name): if name: @@ -1067,6 +1071,11 @@ class Scanner(object): for change in aggregator.aggregate_changes(entry, changes): uniqueid = change[0] + if uniqueid in self._ignore_uids: + LOG.info('ignoring %s based on configuration setting', + uniqueid) + continue + c_type = change[1] if c_type == diff_tree.CHANGE_ADD: diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 650eaaa..45e57ef 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -39,6 +39,7 @@ class ReleaseNotesDirective(rst.Directive): 'collapse-pre-releases': directives.flag, 'earliest-version': directives.unchanged, 'stop-at-branch-base': directives.flag, + 'ignore-notes': directives.unchanged, } def run(self): @@ -57,6 +58,10 @@ class ReleaseNotesDirective(rst.Directive): reporoot = repo.Repo.discover(reporoot).path relnotessubdir = self.options.get('relnotessubdir', defaults.RELEASE_NOTES_SUBDIR) + ignore_notes = [ + name.strip() + for name in self.options.get('ignore-notes', '').split(',') + ] conf = config.Config(reporoot, relnotessubdir) opt_overrides = {} if 'notesdir' in self.options: @@ -74,6 +79,8 @@ class ReleaseNotesDirective(rst.Directive): 'earliest-version') if branch: opt_overrides['branch'] = branch + if ignore_notes: + opt_overrides['ignore_notes'] = ignore_notes conf.override(**opt_overrides) notesdir = os.path.join(relnotessubdir, conf.notesdir) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index f2db5d6..f8d52fa 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -660,6 +660,88 @@ class BasicTest(Base): ) +class IgnoreTest(Base): + + def test_by_fullname(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file() + f2 = self._add_notes_file() + self.c.override( + ignore_notes=[f1], + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0-2': [f2]}, + results, + ) + + def test_by_basename(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file() + f2 = self._add_notes_file() + self.c.override( + ignore_notes=[os.path.basename(f1)], + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0-2': [f2]}, + results, + ) + + def test_by_uid(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file() + f2 = self._add_notes_file() + self.c.override( + ignore_notes=[scanner._get_unique_id(f1)], + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0-2': [f2]}, + results, + ) + + def test_by_multiples(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file() + f2 = self._add_notes_file() + self.c.override( + ignore_notes=[ + scanner._get_unique_id(f1), + scanner._get_unique_id(f2), + ], + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {}, + results, + ) + + class FileContentsTest(Base): def test_basic_file(self): -- GitLab From a42a617350e36c0f09859c95ba89c64aa38009d2 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 14 Jun 2017 08:02:38 -0400 Subject: [PATCH 158/257] fix an infinite loop in the topo traversal algorithm When skipping null-merges, do not go back to the first parent node if we have already processed it. Fix a similar potential issue when handling parent nodes during regular processing. Change-Id: I10e531cdf3b203ca2e9249d89a37b61f79091311 Signed-off-by: Doug Hellmann --- .../notes/null-merge-infinite-loop-670367094ad83e19.yaml | 5 +++++ reno/scanner.py | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/null-merge-infinite-loop-670367094ad83e19.yaml diff --git a/releasenotes/notes/null-merge-infinite-loop-670367094ad83e19.yaml b/releasenotes/notes/null-merge-infinite-loop-670367094ad83e19.yaml new file mode 100644 index 0000000..ed7ae8a --- /dev/null +++ b/releasenotes/notes/null-merge-infinite-loop-670367094ad83e19.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Remove an infinite loop in the traversal algorithm caused by some + null-merge skip situations. diff --git a/reno/scanner.py b/reno/scanner.py index 36fc33b..6a9280e 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -729,9 +729,11 @@ class Scanner(object): # and we can continue past the merge. emitted.add(sha) # Now set up the first parent so it is processed - # later. + # later, as long as we haven't already processed + # it. first_parent = entry.commit.parents[0] - if first_parent not in todo: + if (first_parent not in todo and + first_parent not in emitted): todo.appendleft(first_parent) continue @@ -771,7 +773,7 @@ class Scanner(object): # to grow very large, but it's not clear the output # will be produced in the right order. for p in entry.commit.parents: - if p not in todo: + if p not in todo and p not in emitted: todo.appendleft(p) else: -- GitLab From 18df043b5a0d75baf311e4f4bb3c64a6f0de7373 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 15 Jun 2017 14:33:18 +0100 Subject: [PATCH 159/257] Move notesdir default to 'defaults' module We're going to use this shortly. Change-Id: I7382ee143c7fb19b19e52ac6c37960242fde8628 --- reno/config.py | 2 +- reno/defaults.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/reno/config.py b/reno/config.py index 03d0362..4783568 100644 --- a/reno/config.py +++ b/reno/config.py @@ -95,7 +95,7 @@ class Config(object): _OPTS = { # The notes subdirectory within the relnotesdir where the # notes live. - 'notesdir': 'notes', + 'notesdir': defaults.NOTES_SUBDIR, # Should pre-release versions be merged into the final release # of the same number (1.0.0.0a1 notes appear under 1.0.0). diff --git a/reno/defaults.py b/reno/defaults.py index 1b2cf00..b30a18d 100644 --- a/reno/defaults.py +++ b/reno/defaults.py @@ -11,3 +11,4 @@ # under the License. RELEASE_NOTES_SUBDIR = 'releasenotes' +NOTES_SUBDIR = 'notes' -- GitLab From 0d45aee7dceba7e1ddafdfb995d1b6a45e252a6a Mon Sep 17 00:00:00 2001 From: liuxiaoyang Date: Mon, 19 Jun 2017 08:56:29 +0800 Subject: [PATCH 160/257] Replace http with https The use of https and some of them are http. Use https instead of http to ensure the safety without containing our account/password information. e.g. https://review.openstack.org/#/c/462890/ Change-Id: I545833e4d7ede4435e4f50bed792a60847e9a813 --- CONTRIBUTING.rst | 4 ++-- HACKING.rst | 2 +- README.rst | 10 +++++----- doc/source/usage.rst | 2 +- setup.cfg | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index aafbd53..9a5eea2 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,14 +1,14 @@ If you would like to contribute to the development of OpenStack, you must follow the steps in this page: - http://docs.openstack.org/infra/manual/developers.html + https://docs.openstack.org/infra/manual/developers.html If you already have a good understanding of how the system works and your OpenStack accounts are set up, you can skip to the development workflow section of this documentation to learn how changes to OpenStack should be submitted for review via the Gerrit tool: - http://docs.openstack.org/infra/manual/developers.html#development-workflow + https://docs.openstack.org/infra/manual/developers.html#development-workflow Pull requests submitted through GitHub will be ignored. diff --git a/HACKING.rst b/HACKING.rst index f6627a0..ace48c9 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -1,4 +1,4 @@ reno Style Commandments =============================================== -Read the OpenStack Style Commandments http://docs.openstack.org/developer/hacking/ +Read the OpenStack Style Commandments https://docs.openstack.org/developer/hacking/ diff --git a/README.rst b/README.rst index 48651d2..8223bf9 100644 --- a/README.rst +++ b/README.rst @@ -48,11 +48,11 @@ from future documentation builds. Project Meta-data ================= -.. .. image:: http://governance.openstack.org/badges/reno.svg - :target: http://governance.openstack.org/reference/tags/index.html +.. .. image:: https://governance.openstack.org/badges/reno.svg + :target: https://governance.openstack.org/reference/tags/index.html * Free software: Apache license -* Documentation: http://docs.openstack.org/developer/reno -* Source: http://git.openstack.org/cgit/openstack/reno -* Bugs: http://bugs.launchpad.net/reno +* Documentation: https://docs.openstack.org/developer/reno +* Source: https://git.openstack.org/cgit/openstack/reno +* Bugs: https://bugs.launchpad.net/reno * IRC: #openstack-release on freenode diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 5459af4..86d99c7 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -331,5 +331,5 @@ Within OpenStack The OpenStack project maintains separate instructions for configuring the CI jobs and other project-specific settings used for reno. Refer to the `Managing Release Notes -`__ +`__ section of the Project Team Guide for details. diff --git a/setup.cfg b/setup.cfg index a45143c..0005f95 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ description-file = README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org -home-page = http://docs.openstack.org/developer/reno/ +home-page = https://docs.openstack.org/developer/reno/ classifier = Environment :: OpenStack Intended Audience :: Information Technology -- GitLab From ebfdb94a20ffcb31773c88701e6d89f6182f27f6 Mon Sep 17 00:00:00 2001 From: liuxiaoyang Date: Tue, 20 Jun 2017 09:45:54 +0800 Subject: [PATCH 161/257] Block comment should start with '# ' Each line of a block comment starts with a # and a single space. Change-Id: I9a36ecfa47d683a9cff43f54a04739686cf51740 --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 494bd24..762f726 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -32,7 +32,7 @@ sys.path.insert(0, os.path.abspath('../..')) # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.autodoc', - #'sphinx.ext.intersphinx', + # 'sphinx.ext.intersphinx', 'reno.sphinxext', ] @@ -88,4 +88,4 @@ latex_documents = [ ] # Example configuration for intersphinx: refer to the Python standard library. -#intersphinx_mapping = {'http://docs.python.org/': None} +# intersphinx_mapping = {'http://docs.python.org/': None} -- GitLab From f4e2d83b009fecee1f9fc7ee89083898f6858ce6 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 28 Jun 2017 14:43:44 -0400 Subject: [PATCH 162/257] rearrange the existing docs to follow the new standard layout Change-Id: Ia033a9243a55f4a7dfd70d5e0515d6d3372d6e08 Signed-off-by: Doug Hellmann --- doc/source/contributing.rst | 4 ---- doc/source/contributor/index.rst | 5 +++++ doc/source/index.rst | 11 ++++------- doc/source/{installation.rst => install/index.rst} | 0 doc/source/{history.rst => releasenotes/index.rst} | 0 doc/source/{ => user}/design.rst | 0 doc/source/{ => user}/examples.rst | 2 +- doc/source/user/index.rst | 10 ++++++++++ doc/source/{ => user}/sphinxext.rst | 0 doc/source/{ => user}/usage.rst | 2 +- 10 files changed, 21 insertions(+), 13 deletions(-) delete mode 100644 doc/source/contributing.rst create mode 100644 doc/source/contributor/index.rst rename doc/source/{installation.rst => install/index.rst} (100%) rename doc/source/{history.rst => releasenotes/index.rst} (100%) rename doc/source/{ => user}/design.rst (100%) rename doc/source/{ => user}/examples.rst (73%) create mode 100644 doc/source/user/index.rst rename doc/source/{ => user}/sphinxext.rst (100%) rename doc/source/{ => user}/usage.rst (99%) diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst deleted file mode 100644 index 1728a61..0000000 --- a/doc/source/contributing.rst +++ /dev/null @@ -1,4 +0,0 @@ -============ -Contributing -============ -.. include:: ../../CONTRIBUTING.rst diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst new file mode 100644 index 0000000..3d4ceb4 --- /dev/null +++ b/doc/source/contributor/index.rst @@ -0,0 +1,5 @@ +============ +Contributing +============ + +.. include:: ../../../CONTRIBUTING.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index c77378c..e5bbf0f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -6,10 +6,7 @@ Contents .. toctree:: :maxdepth: 2 - design - installation - usage - sphinxext - contributing - history - examples + user/index + install/index + contributor/index + releasenotes/index diff --git a/doc/source/installation.rst b/doc/source/install/index.rst similarity index 100% rename from doc/source/installation.rst rename to doc/source/install/index.rst diff --git a/doc/source/history.rst b/doc/source/releasenotes/index.rst similarity index 100% rename from doc/source/history.rst rename to doc/source/releasenotes/index.rst diff --git a/doc/source/design.rst b/doc/source/user/design.rst similarity index 100% rename from doc/source/design.rst rename to doc/source/user/design.rst diff --git a/doc/source/examples.rst b/doc/source/user/examples.rst similarity index 73% rename from doc/source/examples.rst rename to doc/source/user/examples.rst index 872b620..0119177 100644 --- a/doc/source/examples.rst +++ b/doc/source/user/examples.rst @@ -5,7 +5,7 @@ Input file ========== -.. literalinclude:: ../../examples/notes/add-complex-example-6b5927c246456896.yaml +.. literalinclude:: ../../../examples/notes/add-complex-example-6b5927c246456896.yaml :caption: examples/notes/add-complex-example-6b5927c246456896.yaml :language: yaml diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst new file mode 100644 index 0000000..86c517d --- /dev/null +++ b/doc/source/user/index.rst @@ -0,0 +1,10 @@ +================= + reno User Guide +================= + +.. toctree:: + + design + usage + sphinxext + examples diff --git a/doc/source/sphinxext.rst b/doc/source/user/sphinxext.rst similarity index 100% rename from doc/source/sphinxext.rst rename to doc/source/user/sphinxext.rst diff --git a/doc/source/usage.rst b/doc/source/user/usage.rst similarity index 99% rename from doc/source/usage.rst rename to doc/source/user/usage.rst index 86d99c7..9bec262 100644 --- a/doc/source/usage.rst +++ b/doc/source/user/usage.rst @@ -141,7 +141,7 @@ not needed for the bullet items in the other sections of the template. To escape the text of any section and *retain* the newlines, prefix the value with ``|``. For example: -.. include:: ../../examples/notes/add-complex-example-6b5927c246456896.yaml +.. include:: ../../../examples/notes/add-complex-example-6b5927c246456896.yaml :literal: See :doc:`examples` for the rendered version of the note. -- GitLab From b6832f7a81a9027a46acd7a2067e858dd4d71ff9 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 28 Jun 2017 14:46:09 -0400 Subject: [PATCH 163/257] switch from oslosphinx to openstackdocstheme Change-Id: Iba29f6743f5f12e7cb9624c499010f262e34fd01 Signed-off-by: Doug Hellmann --- doc/source/conf.py | 17 ++++++++++++----- test-requirements.txt | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 762f726..5c7b0bc 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -18,11 +18,11 @@ import sys # oslosphinx uses reno and reno uses oslosphinx. Make oslosphinx for # reno optional to break the build cycle try: - import oslosphinx + import openstackdocstheme except: - has_oslosphinx = False + has_theme = False else: - has_oslosphinx = True + has_theme = True sys.path.insert(0, os.path.abspath('../..')) @@ -36,8 +36,15 @@ extensions = [ 'reno.sphinxext', ] -if has_oslosphinx: - extensions.append('oslosphinx') +if has_theme: + extensions.append('openstackdocstheme') + html_theme = 'openstackdocs' + +# openstackdocstheme options +repository_name = 'openstack/reno' +bug_project = 'reno' +bug_tag = '' +html_last_updated_fmt = '%Y-%m-%d %H:%M' # autodoc generation is a bit aggressive and a nuisance when doing heavy # text edit cycles. diff --git a/test-requirements.txt b/test-requirements.txt index 9680a52..203542f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ mock>=1.2 coverage>=3.6 python-subunit>=0.0.18 -oslosphinx>=2.5.0 # Apache-2.0 +openstackdocstheme>=1.11.0 # Apache-2.0 testrepository>=0.0.18 testscenarios>=0.4 testtools>=1.4.0 -- GitLab From ecd1a171bae4f101bfe956d8a22bc023fb0cc9d3 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Tue, 21 Mar 2017 12:25:03 +0000 Subject: [PATCH 164/257] Support repodir config files Not everyone wants to use build release notes separately from their main documentation. For these users, having a 'notes' directory inside the 'releasenotes' directory is unnecessary. However, this also means we must be able to move the config file out of the 'releasesnotes' directory to avoid it being picked up as a release note. Make this possible by adding support for a 'reno.yaml' file in the root directory of the project. Sadly it is not possible to apply this change to reno itself - doing so would cause the files to be picked up as belonging to the current release - but other projects can benefit from this. Change-Id: Ie96103b85d70592dd766e5174784b992fe7782c5 --- doc/source/user/usage.rst | 14 +++++----- .../repodir-config-file-b6b8edc2975964fc.yaml | 7 +++++ reno/config.py | 27 ++++++++++--------- reno/tests/test_config.py | 16 +++++++---- 4 files changed, 39 insertions(+), 25 deletions(-) create mode 100644 releasenotes/notes/repodir-config-file-b6b8edc2975964fc.yaml diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst index 9bec262..d6e6cdb 100644 --- a/doc/source/user/usage.rst +++ b/doc/source/user/usage.rst @@ -174,13 +174,13 @@ correctness. Configuring Reno ================ -Reno looks for an optional ``config.yaml`` file in the release notes -directory. If the values in the configuration file do not apply to -the command being run, they are ignored. For example, some reno -commands take inputs controlling the branch, earliest revision, and -other common parameters that control which notes are included in the -output. Because they are commonly set options, a configuration file -may be the most convenient way to manage the values consistently. +Reno looks for an optional config file, either ``config.yaml`` in the release +notes directory or ``reno.yaml`` in the root directory. If the values in the +configuration file do not apply to the command being run, they are ignored. For +example, some reno commands take inputs controlling the branch, earliest +revision, and other common parameters that control which notes are included in +the output. Because they are commonly set options, a configuration file may be +the most convenient way to manage the values consistently. .. code-block:: yaml diff --git a/releasenotes/notes/repodir-config-file-b6b8edc2975964fc.yaml b/releasenotes/notes/repodir-config-file-b6b8edc2975964fc.yaml new file mode 100644 index 0000000..aedb19e --- /dev/null +++ b/releasenotes/notes/repodir-config-file-b6b8edc2975964fc.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + reno will now scan for a ``reno.yaml`` file in the root repo directory if a + ``config.yaml`` file does not exist in the releasenotes directory. This + allows users to do away with the unnecessary ``notes`` subdirectory in the + releasenotes directory. diff --git a/reno/config.py b/reno/config.py index 4783568..470e7d0 100644 --- a/reno/config.py +++ b/reno/config.py @@ -9,7 +9,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import errno + import logging import os.path @@ -90,8 +90,6 @@ other: class Config(object): - _FILENAME = 'config.yaml' - _OPTS = { # The notes subdirectory within the relnotesdir where the # notes live. @@ -191,7 +189,6 @@ class Config(object): :param str relnotesdir: The directory containing release notes. Defaults to 'releasenotes'. - """ self.reporoot = reporoot if relnotesdir is None: @@ -200,22 +197,26 @@ class Config(object): # Initialize attributes from the defaults. self.override(**self._OPTS) - self._filename = os.path.join(self.reporoot, relnotesdir, - self._FILENAME) self._contents = {} self._load_file() def _load_file(self): + filenames = [ + os.path.join(self.reporoot, self.relnotesdir, 'config.yaml'), + os.path.join(self.reporoot, 'reno.yaml')] + + for filename in filenames: + if os.path.isfile(filename): + break + else: + LOG.info('no configuration file in: %s', ', '.join(filenames)) + return + try: - with open(self._filename, 'r') as fd: + with open(filename, 'r') as fd: self._contents = yaml.safe_load(fd) except IOError as err: - if err.errno == errno.ENOENT: - LOG.info('no configuration file in %s', - self._filename) - else: - LOG.warning('did not load config file %s: %s', - self._filename, err) + LOG.warning('did not load config file %s: %s', filename, err) else: self.override(**self._contents) diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py index c2e0237..3a0ea62 100644 --- a/reno/tests/test_config.py +++ b/reno/tests/test_config.py @@ -77,17 +77,23 @@ collapse_pre_releases: false config.Config(self.tempdir.path) self.assertEqual(1, logger.call_count) - def test_load_file(self): - rn_path = self.tempdir.join('releasenotes') - os.mkdir(rn_path) - config_path = self.tempdir.join('releasenotes/' + - config.Config._FILENAME) + def _test_load_file(self, config_path): with open(config_path, 'w') as fd: fd.write(self.EXAMPLE_CONFIG) self.addCleanup(os.unlink, config_path) c = config.Config(self.tempdir.path) self.assertEqual(False, c.collapse_pre_releases) + def test_load_file_in_releasenotesdir(self): + rn_path = self.tempdir.join('releasenotes') + os.mkdir(rn_path) + config_path = self.tempdir.join('releasenotes/config.yaml') + self._test_load_file(config_path) + + def test_load_file_in_repodir(self): + config_path = self.tempdir.join('reno.yaml') + self._test_load_file(config_path) + def test_get_default(self): d = config.Config.get_default('notesdir') self.assertEqual('notes', d) -- GitLab From b2aadef72636bbc78bfdb8c34b65b360e96605e9 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Mon, 19 Jun 2017 15:32:13 +0100 Subject: [PATCH 165/257] loader: Extract cache filename from config object The only places that this is called, we already have a reno.config.Config instance available. Simply use that instead. We also include a note of the 'conf.notespath' property, because the odd behavior of this caught me out for a bit, and call 'os.path.normpath', because the default path included redundant up-level references. Change-Id: I58948cd8fad55d29bd30c65653630bd466259cdc --- reno/cache.py | 2 +- reno/config.py | 8 +++++++- reno/loader.py | 8 ++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/reno/cache.py b/reno/cache.py index ad852db..20c2a8e 100644 --- a/reno/cache.py +++ b/reno/cache.py @@ -74,7 +74,7 @@ def write_cache_db(conf, versions_to_include, stream = open(outfilename, 'w') close_stream = True else: - outfilename = loader.get_cache_filename(conf.reporoot, conf.notespath) + outfilename = loader.get_cache_filename(conf) if not os.path.exists(os.path.dirname(outfilename)): os.makedirs(os.path.dirname(outfilename)) stream = open(outfilename, 'w') diff --git a/reno/config.py b/reno/config.py index 4783568..6254d69 100644 --- a/reno/config.py +++ b/reno/config.py @@ -259,7 +259,13 @@ class Config(object): @property def notespath(self): - "The path in the repo where notes are kept." + """The path in the repo where notes are kept. + + .. important:: + + This does not take ``reporoot`` into account. You need to add this + manually if required. + """ return os.path.join(self.relnotesdir, self.notesdir) # def parse_config_into(parsed_arguments): diff --git a/reno/loader.py b/reno/loader.py index 3df4988..f39bd04 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -21,8 +21,9 @@ from reno import scanner LOG = logging.getLogger(__name__) -def get_cache_filename(reporoot, notesdir): - return os.path.join(reporoot, notesdir, 'reno.cache') +def get_cache_filename(conf): + return os.path.normpath(os.path.join( + conf.reporoot, conf.notespath, 'reno.cache')) class Loader(object): @@ -54,8 +55,7 @@ class Loader(object): self._cache = None self._scanner = None self._scanner_output = None - self._cache_filename = get_cache_filename(self._reporoot, - self._notespath) + self._cache_filename = get_cache_filename(conf) self._load_data() -- GitLab From 4003fc13210132e56040a90968fae653be3c88b3 Mon Sep 17 00:00:00 2001 From: Rajath Agasthya Date: Fri, 16 Jun 2017 02:14:29 -0700 Subject: [PATCH 166/257] Allow users to change prelude section name Convert note template to a format string. Also include prelude section in the report generated and update docs. Closes-Bug: #1698203 Change-Id: I7bef68bfb518dd8554d56cb200f2844e7d395fc8 --- doc/source/user/usage.rst | 16 ++++++- reno/config.py | 96 +++++++++++---------------------------- reno/defaults.py | 69 ++++++++++++++++++++++++++++ reno/formatter.py | 20 +++++--- reno/linter.py | 4 +- reno/loader.py | 6 +-- reno/tests/test_config.py | 39 +++++++++------- 7 files changed, 151 insertions(+), 99 deletions(-) diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst index 9bec262..3d19bf0 100644 --- a/doc/source/user/usage.rst +++ b/doc/source/user/usage.rst @@ -56,8 +56,8 @@ list: prelude - General comments about the release. The prelude from all notes in a - section are combined, in note order, to produce a single prelude + General comments about the release. Prelude sections from all notes in a + release are combined, in note order, to produce a single prelude introducing that release. This section is always included, regardless of what sections are configured. @@ -197,6 +197,9 @@ may be the most convenient way to manage the values consistently. - [api, API Changes] - [security, Security Issues] - [fixes, Bug Fixes] + # Change prelude_section_name to 'release_summary' from default value + # 'prelude'. + prelude_section_name: release_summary template: | ... @@ -284,6 +287,15 @@ The following options are configurable: order in which the final report will be generated. A prelude section will always be automatically inserted before the first element of this list. +`prelude_section_name` + + The name of the prelude section in the note template. Note that the + value for this must be a single word, but can have underscores. The + value is displayed in titlecase in the report after replacing + underscores with spaces. + + Defaults to ``prelude`` + `ignore_null_merges` OpenStack used to use null-merges to bring final release tags from diff --git a/reno/config.py b/reno/config.py index 6254d69..2c4d36c 100644 --- a/reno/config.py +++ b/reno/config.py @@ -19,74 +19,6 @@ from reno import defaults LOG = logging.getLogger(__name__) -_TEMPLATE = """\ ---- -prelude: > - Replace this text with content to appear at the top of the section for this - release. All of the prelude content is merged together and then rendered - separately from the items listed in other parts of the file, so the text - needs to be worded so that both the prelude and the other items make sense - when read independently. This may mean repeating some details. Not every - release note requires a prelude. Usually only notes describing major - features or adding release theme details should have a prelude. -features: - - | - List new features here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -issues: - - | - List known issues here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -upgrade: - - | - List upgrade notes here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -deprecations: - - | - List deprecations notes here, or remove this section. All of the list - items in this section are combined when the release notes are rendered, so - the text needs to be worded so that it does not depend on any information - only available in another section, such as the prelude. This may mean - repeating some details. -critical: - - | - Add critical notes here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -security: - - | - Add security notes here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -fixes: - - | - Add normal bug fixes here, or remove this section. All of the list items - in this section are combined when the release notes are rendered, so the - text needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -other: - - | - Add other notes here, or remove this section. All of the list items in - this section are combined when the release notes are rendered, so the text - needs to be worded so that it does not depend on any information only - available in another section, such as the prelude. This may mean repeating - some details. -""" - class Config(object): @@ -115,7 +47,7 @@ class Config(object): 'earliest_version': None, # The template used by reno new to create a note. - 'template': _TEMPLATE, + 'template': defaults.TEMPLATE.format(defaults.PRELUDE_SECTION_NAME), # The RE pattern used to match the repo tags representing a valid # release version. The pattern is compiled with the verbose and unicode @@ -156,6 +88,13 @@ class Config(object): ['other', 'Other Notes'], ], + # The name of the prelude section in the note template. This + # allows users to rename the section to, for example, + # 'release_summary' or 'project_wide_general_announcements', + # which is displayed in titlecase in the report after + # replacing underscores with spaces. + 'prelude_section_name': defaults.PRELUDE_SECTION_NAME, + # When this option is set to True, any merge commits with no # changes and in which the second or later parent is tagged # are considered "null-merges" that bring the tag information @@ -219,6 +158,13 @@ class Config(object): else: self.override(**self._contents) + def _rename_prelude_section(self, **kwargs): + key = 'prelude_section_name' + if key in kwargs and kwargs[key] != self._OPTS[key]: + new_prelude_name = kwargs[key] + + self.template = defaults.TEMPLATE.format(new_prelude_name) + def override(self, **kwds): """Set the values of the named configuration options. @@ -227,6 +173,9 @@ class Config(object): present. """ + # Replace prelude section name if it has been changed. + self._rename_prelude_section(**kwds) + for n, v in kwds.items(): if n not in self._OPTS: LOG.warning('ignoring unknown configuration value %r = %r', @@ -268,6 +217,15 @@ class Config(object): """ return os.path.join(self.relnotesdir, self.notesdir) + @property + def options(self): + """Get all configuration options as a dict. + + Returns the actual configuration options after overrides. + """ + options = {o: getattr(self, o) for o in self._OPTS} + return options + # def parse_config_into(parsed_arguments): # """Parse the user config onto the namespace arguments. diff --git a/reno/defaults.py b/reno/defaults.py index b30a18d..927656d 100644 --- a/reno/defaults.py +++ b/reno/defaults.py @@ -12,3 +12,72 @@ RELEASE_NOTES_SUBDIR = 'releasenotes' NOTES_SUBDIR = 'notes' +PRELUDE_SECTION_NAME = 'prelude' +# This is a format string, so it needs to be formatted wherever it is used. +TEMPLATE = """\ +--- +{0}: > + Replace this text with content to appear at the top of the section for this + release. All of the prelude content is merged together and then rendered + separately from the items listed in other parts of the file, so the text + needs to be worded so that both the prelude and the other items make sense + when read independently. This may mean repeating some details. Not every + release note requires a prelude. Usually only notes describing major + features or adding release theme details should have a prelude. +features: + - | + List new features here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +issues: + - | + List known issues here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +upgrade: + - | + List upgrade notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +deprecations: + - | + List deprecations notes here, or remove this section. All of the list + items in this section are combined when the release notes are rendered, so + the text needs to be worded so that it does not depend on any information + only available in another section, such as the prelude. This may mean + repeating some details. +critical: + - | + Add critical notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +security: + - | + Add security notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +fixes: + - | + Add normal bug fixes here, or remove this section. All of the list items + in this section are combined when the release notes are rendered, so the + text needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +other: + - | + Add other notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +""" diff --git a/reno/formatter.py b/reno/formatter.py index 646d68e..4092a8d 100644 --- a/reno/formatter.py +++ b/reno/formatter.py @@ -48,13 +48,21 @@ def format_report(loader, config, versions_to_include, title=None, # Add the preludes. notefiles = loader[version] - for n, sha in notefiles: - if 'prelude' in file_contents[n]: - if show_source: - report.append('.. %s @ %s\n' % (n, sha)) - report.append(file_contents[n]['prelude']) - report.append('') + prelude_name = config.prelude_section_name + notefiles_with_prelude = [(n, sha) for n, sha in notefiles + if prelude_name in file_contents[n]] + if notefiles_with_prelude: + report.append(prelude_name.replace('_', ' ').title()) + report.append('-' * len(prelude_name)) + report.append('') + + for n, sha in notefiles_with_prelude: + if show_source: + report.append('.. %s @ %s\n' % (n, sha)) + report.append(file_contents[n][prelude_name]) + report.append('') + # Add other sections. for section_name, section_title in config.sections: notes = [ (n, fn, sha) diff --git a/reno/linter.py b/reno/linter.py index 5071842..9cbda99 100644 --- a/reno/linter.py +++ b/reno/linter.py @@ -30,8 +30,8 @@ def lint_cmd(args, conf): error = 0 load = loader.Loader(conf, ignore_cache=True) - - allowed_section_names = ['prelude'] + [s[0] for s in conf.sections] + allowed_section_names = [conf.prelude_section_name] + \ + [s[0] for s in conf.sections] uids = {} for f in notes: diff --git a/reno/loader.py b/reno/loader.py index f39bd04..6f8dcb0 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -104,13 +104,13 @@ class Loader(object): cleaned_content = {} for section_name, section_content in content.items(): - if section_name == 'prelude': + if section_name == self._config.prelude_section_name: if not isinstance(section_content, six.string_types): LOG.warning( - ('The prelude section of %s ' + ('The %s section of %s ' 'does not parse as a single string. ' 'Is the YAML input escaped properly?') % - filename, + (self._config.prelude_section_name, filename), ) else: if isinstance(section_content, six.string_types): diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py index c2e0237..1d41232 100644 --- a/reno/tests/test_config.py +++ b/reno/tests/test_config.py @@ -17,6 +17,7 @@ import os import fixtures from reno import config +from reno import defaults from reno import main from reno.tests import base @@ -35,10 +36,7 @@ collapse_pre_releases: false def test_defaults(self): c = config.Config(self.tempdir.path) - actual = { - o: getattr(c, o) - for o in config.Config._OPTS.keys() - } + actual = c.options self.assertEqual(config.Config._OPTS, actual) def test_override(self): @@ -46,10 +44,7 @@ collapse_pre_releases: false c.override( collapse_pre_releases=False, ) - actual = { - o: getattr(c, o) - for o in config.Config._OPTS.keys() - } + actual = c.options expected = {} expected.update(config.Config._OPTS) expected['collapse_pre_releases'] = False @@ -63,10 +58,7 @@ collapse_pre_releases: false c.override( notesdir='value2', ) - actual = { - o: getattr(c, o) - for o in config.Config._OPTS.keys() - } + actual = c.options expected = {} expected.update(config.Config._OPTS) expected['notesdir'] = 'value2' @@ -119,10 +111,7 @@ collapse_pre_releases: false c = self._run_override_from_parsed_args([ '--no-collapse-pre-releases', ]) - actual = { - o: getattr(c, o) - for o in config.Config._OPTS.keys() - } + actual = c.options expected = {} expected.update(config.Config._OPTS) expected['collapse_pre_releases'] = False @@ -158,6 +147,22 @@ class TestConfigProperties(base.TestCase): self.assertEqual('releasenotes/thenotes', self.c.notespath) def test_template(self): - self.assertEqual(config._TEMPLATE, self.c.template) + template = defaults.TEMPLATE.format(defaults.PRELUDE_SECTION_NAME) + self.assertEqual(template, self.c.template) self.c.override(template='i-am-a-template') self.assertEqual('i-am-a-template', self.c.template) + + def test_prelude_override(self): + template = defaults.TEMPLATE.format(defaults.PRELUDE_SECTION_NAME) + self.assertEqual(template, self.c.template) + self.c.override(prelude_section_name='fake_prelude_name') + expected_template = defaults.TEMPLATE.format('fake_prelude_name') + self.assertEqual(expected_template, self.c.template) + + def test_prelude_and_template_override(self): + template = defaults.TEMPLATE.format(defaults.PRELUDE_SECTION_NAME) + self.assertEqual(template, self.c.template) + self.c.override(prelude_section_name='fake_prelude_name', + template='i-am-a-template') + self.assertEqual('fake_prelude_name', self.c.prelude_section_name) + self.assertEqual('i-am-a-template', self.c.template) -- GitLab From ec2afa0b90ec57cf41c4b016bf867a0a6e7f6526 Mon Sep 17 00:00:00 2001 From: chenxing Date: Thu, 13 Jul 2017 15:22:28 +0000 Subject: [PATCH 167/257] Update documention link for doc migration Change-Id: I7bf4cf447a090657a8a9d6384fa858f1e261dbb8 --- README.rst | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 8223bf9..fa6c266 100644 --- a/README.rst +++ b/README.rst @@ -52,7 +52,7 @@ Project Meta-data :target: https://governance.openstack.org/reference/tags/index.html * Free software: Apache license -* Documentation: https://docs.openstack.org/developer/reno +* Documentation: https://docs.openstack.org/reno/latest/ * Source: https://git.openstack.org/cgit/openstack/reno * Bugs: https://bugs.launchpad.net/reno * IRC: #openstack-release on freenode diff --git a/setup.cfg b/setup.cfg index 0005f95..4f304ec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ description-file = README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org -home-page = https://docs.openstack.org/developer/reno/ +home-page = https://docs.openstack.org/reno/latest/ classifier = Environment :: OpenStack Intended Audience :: Information Technology -- GitLab From ff2ca65e5c3b5ad2f2b60c2b367b5297854537f7 Mon Sep 17 00:00:00 2001 From: Akihiro Motoki Date: Wed, 19 Jul 2017 09:39:38 +0000 Subject: [PATCH 168/257] Clean up rendered HTML with openstackdocstheme After migrating to openstackdocstheme, there are several points to be improved in the rendered HTML files. * Cleanup unnecessary vertical lines for quote blocks. Existing quote blocks are actually not intended and leading extra spaces cause this. This commit removes unnecessary leading spaces. * Some quote blocks are converted into definition lists to clean up vertical lines for quote blocks. * Use code-block for better code highlighting. * Specify maxdepth in user/index toctree. Change-Id: I9add5a317718e97abce15b5ddbfa3d1208a01570 --- CONTRIBUTING.rst | 9 +++------ doc/source/user/design.rst | 12 ++++++------ doc/source/user/index.rst | 1 + doc/source/user/sphinxext.rst | 14 +++----------- doc/source/user/usage.rst | 28 +++------------------------- 5 files changed, 16 insertions(+), 48 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9a5eea2..e716304 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,17 +1,14 @@ If you would like to contribute to the development of OpenStack, you must follow the steps in this page: - - https://docs.openstack.org/infra/manual/developers.html +https://docs.openstack.org/infra/manual/developers.html If you already have a good understanding of how the system works and your OpenStack accounts are set up, you can skip to the development workflow section of this documentation to learn how changes to OpenStack should be submitted for review via the Gerrit tool: - - https://docs.openstack.org/infra/manual/developers.html#development-workflow +https://docs.openstack.org/infra/manual/developers.html#development-workflow Pull requests submitted through GitHub will be ignored. Bugs should be filed on Launchpad, not GitHub: - - https://bugs.launchpad.net/reno +https://bugs.launchpad.net/reno diff --git a/doc/source/user/design.rst b/doc/source/user/design.rst index 4cd818c..2a1dd2a 100644 --- a/doc/source/user/design.rst +++ b/doc/source/user/design.rst @@ -39,12 +39,12 @@ We had several design inputs: containing a release note on a particular topic. * Release notes should be grouped by type in the output document. - 1. New features - 2. Known issues - 3. Upgrade notes - 4. Security fixes - 5. Bugs fixes - 6. Other + 1. New features + 2. Known issues + 3. Upgrade notes + 4. Security fixes + 5. Bugs fixes + 6. Other We want to eventually provide the ability to create a release notes file for a given release and add it to the source distribution for the diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 86c517d..3dc9ce2 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -3,6 +3,7 @@ ================= .. toctree:: + :maxdepth: 2 design usage diff --git a/doc/source/user/sphinxext.rst b/doc/source/user/sphinxext.rst index fb2c5f9..18a6d6d 100644 --- a/doc/source/user/sphinxext.rst +++ b/doc/source/user/sphinxext.rst @@ -23,44 +23,36 @@ Enable the extension by adding ``'reno.sphinxext'`` to the Options: *branch* - The name of the branch to scan. Defaults to the current branch. *reporoot* - The path to the repository root directory. Defaults to the directory where ``sphinx-build`` is being run. *relnotessubdir* - The path under ``reporoot`` where the release notes are. Defaults to ``releasenotes``. *notesdir* - The path under ``relnotessubdir`` where the release notes are. Defaults to ``notes``. *version* - A comma separated list of versions to include in the notes. The default is to include all versions found on ``branch``. *collapse-pre-releases* - A flag indicating that notes attached to pre-release versions should be incorporated into the notes for the final release, after the final release is tagged. *earliest-version* - A string containing the version number of the earliest version to be included. For example, when scanning a branch, this is typically set to the version used to create the branch to limit the output to only versions on that branch. *ignore-notes* - A string containing a comma-delimited list of filenames or UIDs for notes that should be ignored by the scanner. It is most useful to set this when a note is edited on the wrong branch, @@ -72,14 +64,14 @@ Examples The release notes for the "current" branch, with "Release Notes" as a title. -:: +.. code-block:: rest .. release-notes:: Release Notes The release notes for the "stable/liberty" branch, with a separate title. -:: +.. code-block:: rest ======================= Liberty Release Notes @@ -90,7 +82,7 @@ title. The release notes for version "1.0.0". -:: +.. code-block:: rest .. release-notes:: 1.0.0 Release Notes :version: 1.0.0 diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst index 8dc58e6..22180e4 100644 --- a/doc/source/user/usage.rst +++ b/doc/source/user/usage.rst @@ -55,55 +55,46 @@ sections are configurable (see below) but default to the following list: prelude - General comments about the release. Prelude sections from all notes in a release are combined, in note order, to produce a single prelude introducing that release. This section is always included, regardless of what sections are configured. features - A list of new major features in the release. issues - A list of known issues in the release. For example, if a new driver is experimental or known to not work in some cases, it should be mentioned here. upgrade - A list of upgrade notes in the release. For example, if a database schema alteration is needed. deprecations - A list of features, APIs, configuration options to be deprecated in the release. Deprecations should not be used for something that is removed in the release, use upgrade section instead. Deprecation should allow time for users to make necessary changes for the removal to happen in a future release. critical - A list of *fixed* critical bugs. security - A list of *fixed* security issues. fixes - A list of other *fixed* bugs. other - Other notes that are important but do not fall into any of the given categories. Any sections that would be blank should be left out of the note file entirely. -:: +.. code-block:: yaml --- prelude: > @@ -141,8 +132,8 @@ not needed for the bullet items in the other sections of the template. To escape the text of any section and *retain* the newlines, prefix the value with ``|``. For example: -.. include:: ../../../examples/notes/add-complex-example-6b5927c246456896.yaml - :literal: +.. literalinclude:: ../../../examples/notes/add-complex-example-6b5927c246456896.yaml + :language: yaml See :doc:`examples` for the rendered version of the note. @@ -216,27 +207,23 @@ using command-line switches. For example: The following options are configurable: `notesdir` - The notes subdirectory within the `relnotesdir` where the notes live. Defaults to ``notes``. `collapse_pre_releases` - Should pre-release versions be merged into the final release of the same number (`1.0.0.0a1` notes appear under `1.0.0`). Defaults to ``True``. `stop_at_branch_base` - Should the scanner stop at the base of a branch (True) or go ahead and scan the entire history (False)? Defaults to ``True``. `branch` - The git branch to scan. If a stable branch is specified but does not exist, reno attempts to automatically convert that to an "end-of-life" tag. For example, ``origin/stable/liberty`` would be converted to ``liberty-eol``. @@ -244,7 +231,6 @@ The following options are configurable: Defaults to the "current" branch checked out. `earliest_version` - The earliest version to be included. This is usually the lowest version number, and is meant to be the oldest version. If unset, all versions will be scanned. @@ -252,18 +238,15 @@ The following options are configurable: Defaults to ``None``. `template` - The template used by reno new to create a note. `release_tag_re` - The regex pattern used to match the repo tags representing a valid release version. The pattern is compiled with the verbose and unicode flags enabled. Defaults to ``((?:[\d.ab]|rc)+)``. `pre_release_tag_re` - The regex pattern used to check if a valid release version tag is also a valid pre-release version. The pattern is compiled with the verbose and unicode flags enabled. The pattern must define a group called `pre_release` @@ -274,7 +257,6 @@ The following options are configurable: Defaults to ``(?P\.\d+(?:[ab]|rc)+\d*)$``. `branch_name_re` - The pattern for names for branches that are relevant when scanning history to determine where to stop, to find the "base" of a branch. Other branches are ignored. @@ -282,13 +264,11 @@ The following options are configurable: Defaults to ``stable/.+``. `sections` - The identifiers and names of permitted sections in the release notes, in the order in which the final report will be generated. A prelude section will always be automatically inserted before the first element of this list. `prelude_section_name` - The name of the prelude section in the note template. Note that the value for this must be a single word, but can have underscores. The value is displayed in titlecase in the report after replacing @@ -297,7 +277,6 @@ The following options are configurable: Defaults to ``prelude`` `ignore_null_merges` - OpenStack used to use null-merges to bring final release tags from stable branches back into the master branch. This confuses the regular traversal because it makes that stable branch appear to be @@ -312,7 +291,6 @@ The following options are configurable: Defaults to ``True``. `ignore_notes` - A list of filenames or UIDs for notes that should be ignored by the reno scanner. It is most useful to set this when a note is edited on the wrong branch, making it appear to be part of a release that it -- GitLab From 142ba2db70fab7ca10145a2a8f226dd5b5361422 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 4 Aug 2017 20:36:06 +0200 Subject: [PATCH 169/257] Updating vcs fields. Signed-off-by: Daniel Baumann --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 51b6f13..829398b 100644 --- a/debian/control +++ b/debian/control @@ -33,8 +33,8 @@ Build-Depends-Indep: git, subunit, testrepository, Standards-Version: 3.9.8 -Vcs-Browser: https://anonscm.debian.org/cgit/openstack/python-reno.git/ -Vcs-Git: https://anonscm.debian.org/git/openstack/python-reno.git +Vcs-Browser: https://anonscm.debian.org/cgit/openstack/libs/python-reno.git +Vcs-Git: https://anonscm.debian.org/git/openstack/libs/python-reno.git Homepage: http://www.openstack.org/ Package: python-reno -- GitLab From 4b3fc15c736f5a04e50b13b865844408a167e1c9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 4 Aug 2017 21:03:10 +0200 Subject: [PATCH 170/257] Adding changelog message about updating vcs fields. Signed-off-by: Daniel Baumann --- debian/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian/changelog b/debian/changelog index f53417b..1075915 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,9 @@ python-reno (1.3.0-4) unstable; urgency=medium * Add gnupg as build-depends-indep (Closes: #834685). * Standards-Version: 3.9.8 (no change). + [ Daniel Baumann ] + * Updating vcs fields. + -- Thomas Goirand Fri, 09 Sep 2016 14:11:39 +0200 python-reno (1.3.0-3) unstable; urgency=medium -- GitLab From f1f283a5609217939c477a74519ea747052993c7 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 4 Aug 2017 21:14:32 +0200 Subject: [PATCH 171/257] Updating copyright format url. Signed-off-by: Daniel Baumann --- debian/changelog | 1 + debian/copyright | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 1075915..0460b87 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,7 @@ python-reno (1.3.0-4) unstable; urgency=medium [ Daniel Baumann ] * Updating vcs fields. + * Updating copyright format url. -- Thomas Goirand Fri, 09 Sep 2016 14:11:39 +0200 diff --git a/debian/copyright b/debian/copyright index 7adc8f2..3112857 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,4 +1,4 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: reno Source: http://www.openstack.org/ -- GitLab From cc5d3b61a87873572b391758376f96d25eab88ce Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 4 Aug 2017 21:31:11 +0200 Subject: [PATCH 172/257] Running wrap-and-sort -bast. Signed-off-by: Daniel Baumann --- debian/changelog | 1 + debian/control | 110 +++++++++++++++++++++++++---------------------- 2 files changed, 60 insertions(+), 51 deletions(-) diff --git a/debian/changelog b/debian/changelog index 0460b87..6173707 100644 --- a/debian/changelog +++ b/debian/changelog @@ -10,6 +10,7 @@ python-reno (1.3.0-4) unstable; urgency=medium [ Daniel Baumann ] * Updating vcs fields. * Updating copyright format url. + * Running wrap-and-sort -bast. -- Thomas Goirand Fri, 09 Sep 2016 14:11:39 +0200 diff --git a/debian/control b/debian/control index 829398b..bd27885 100644 --- a/debian/control +++ b/debian/control @@ -2,36 +2,39 @@ Source: python-reno Section: python Priority: optional Maintainer: PKG OpenStack -Uploaders: Ivan Udovichenko , - Thomas Goirand , -Build-Depends: debhelper (>= 9), - dh-python, - python-all, - python-pbr (>= 1.4), - python-setuptools, - python-sphinx, - python3-all, - python3-pbr (>= 1.4), - python3-setuptools, -Build-Depends-Indep: git, - python-babel, - python-coverage, - python-hacking, - python-mock (>= 1.3), - python-oslosphinx (>= 2.5.0), - python-oslotest (>= 1.10.0), - python-testscenarios, - python-testtools (>= 1.4.0), - python-yaml, - python3-babel, - python3-coverage (>= 3.6), - python3-mock (>= 1.3), - python3-oslotest (>= 1.10.0), - python3-subunit, - python3-testscenarios, - python3-testtools (>= 1.4.0), - subunit, - testrepository, +Uploaders: + Ivan Udovichenko , + Thomas Goirand , +Build-Depends: + debhelper (>= 9), + dh-python, + python-all, + python-pbr (>= 1.4), + python-setuptools, + python-sphinx, + python3-all, + python3-pbr (>= 1.4), + python3-setuptools, +Build-Depends-Indep: + git, + python-babel, + python-coverage, + python-hacking, + python-mock (>= 1.3), + python-oslosphinx (>= 2.5.0), + python-oslotest (>= 1.10.0), + python-testscenarios, + python-testtools (>= 1.4.0), + python-yaml, + python3-babel, + python3-coverage (>= 3.6), + python3-mock (>= 1.3), + python3-oslotest (>= 1.10.0), + python3-subunit, + python3-testscenarios, + python3-testtools (>= 1.4.0), + subunit, + testrepository, Standards-Version: 3.9.8 Vcs-Browser: https://anonscm.debian.org/cgit/openstack/libs/python-reno.git Vcs-Git: https://anonscm.debian.org/git/openstack/libs/python-reno.git @@ -39,39 +42,44 @@ Homepage: http://www.openstack.org/ Package: python-reno Architecture: all -Depends: git, - python-pbr (>= 1.4), - python-yaml, - ${misc:Depends}, - ${python:Depends}, -Suggests: python-reno-doc, +Depends: + git, + python-pbr (>= 1.4), + python-yaml, + ${misc:Depends}, + ${python:Depends}, +Suggests: + python-reno-doc, Description: RElease NOtes manager - Python 2.x Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. . This package contains the Python 2.x module. -Package: python3-reno +Package: python-reno-doc +Section: doc Architecture: all -Depends: git, - python3-pbr (>= 1.4), - python3-yaml, - ${misc:Depends}, - ${python3:Depends}, -Suggests: python-reno-doc, -Description: RElease NOtes manager - Python 3.x +Depends: + ${misc:Depends}, + ${sphinxdoc:Depends}, +Description: RElease NOtes manager - doc Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. . - This package contains the Python 3.x module. + This package contains the documentation. -Package: python-reno-doc -Section: doc +Package: python3-reno Architecture: all -Depends: ${misc:Depends}, - ${sphinxdoc:Depends}, -Description: RElease NOtes manager - doc +Depends: + git, + python3-pbr (>= 1.4), + python3-yaml, + ${misc:Depends}, + ${python3:Depends}, +Suggests: + python-reno-doc, +Description: RElease NOtes manager - Python 3.x Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. . - This package contains the documentation. + This package contains the Python 3.x module. -- GitLab From 3cd5daa18b31f293c4c7645397a24b3f274e6d6d Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 4 Aug 2017 21:50:21 +0200 Subject: [PATCH 173/257] Fixing changelog. Signed-off-by: Daniel Baumann --- debian/changelog | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/debian/changelog b/debian/changelog index 6173707..d63a269 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +python-reno (1.3.0-5) UNRELEASED; urgency=medium + + * Updating vcs fields. + * Updating copyright format url. + * Running wrap-and-sort -bast. + + -- Daniel Baumann Fri, 04 Aug 2017 21:44:18 +0200 + python-reno (1.3.0-4) unstable; urgency=medium [ Ondřej Nový ] @@ -7,11 +15,6 @@ python-reno (1.3.0-4) unstable; urgency=medium * Add gnupg as build-depends-indep (Closes: #834685). * Standards-Version: 3.9.8 (no change). - [ Daniel Baumann ] - * Updating vcs fields. - * Updating copyright format url. - * Running wrap-and-sort -bast. - -- Thomas Goirand Fri, 09 Sep 2016 14:11:39 +0200 python-reno (1.3.0-3) unstable; urgency=medium -- GitLab From 0b8f7c69ab9fcbb066705109e36f79ad76851627 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 4 Aug 2017 23:21:35 +0200 Subject: [PATCH 174/257] Updating maintainer field. Signed-off-by: Daniel Baumann --- debian/changelog | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index d63a269..8a19c7d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ python-reno (1.3.0-5) UNRELEASED; urgency=medium * Updating vcs fields. * Updating copyright format url. * Running wrap-and-sort -bast. + * Updating maintainer field. -- Daniel Baumann Fri, 04 Aug 2017 21:44:18 +0200 diff --git a/debian/control b/debian/control index bd27885..1f7e9ec 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: python-reno Section: python Priority: optional -Maintainer: PKG OpenStack +Maintainer: Debian OpenStack Uploaders: Ivan Udovichenko , Thomas Goirand , -- GitLab From ac97a2c5b8a1c933a1cd968ee10022af254988cd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 4 Aug 2017 23:59:06 +0200 Subject: [PATCH 175/257] Updating standards version to 4.0.0. Signed-off-by: Daniel Baumann --- debian/changelog | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 8a19c7d..2110587 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ python-reno (1.3.0-5) UNRELEASED; urgency=medium * Updating copyright format url. * Running wrap-and-sort -bast. * Updating maintainer field. + * Updating standards version to 4.0.0. -- Daniel Baumann Fri, 04 Aug 2017 21:44:18 +0200 diff --git a/debian/control b/debian/control index 1f7e9ec..dc8b0f5 100644 --- a/debian/control +++ b/debian/control @@ -35,7 +35,7 @@ Build-Depends-Indep: python3-testtools (>= 1.4.0), subunit, testrepository, -Standards-Version: 3.9.8 +Standards-Version: 4.0.0 Vcs-Browser: https://anonscm.debian.org/cgit/openstack/libs/python-reno.git Vcs-Git: https://anonscm.debian.org/git/openstack/libs/python-reno.git Homepage: http://www.openstack.org/ -- GitLab From 439fb116f2736cfd318a1292aa73123b96e3f2a2 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 5 Aug 2017 00:15:34 +0200 Subject: [PATCH 176/257] Removing gbp.conf, not used anymore or should be specified in the developers dotfiles. Signed-off-by: Daniel Baumann --- debian/changelog | 2 ++ debian/gbp.conf | 9 --------- 2 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 debian/gbp.conf diff --git a/debian/changelog b/debian/changelog index 2110587..1eb28c1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,8 @@ python-reno (1.3.0-5) UNRELEASED; urgency=medium * Running wrap-and-sort -bast. * Updating maintainer field. * Updating standards version to 4.0.0. + * Removing gbp.conf, not used anymore or should be specified in the + developers dotfiles. -- Daniel Baumann Fri, 04 Aug 2017 21:44:18 +0200 diff --git a/debian/gbp.conf b/debian/gbp.conf deleted file mode 100644 index 10f9500..0000000 --- a/debian/gbp.conf +++ /dev/null @@ -1,9 +0,0 @@ -[DEFAULT] -upstream-branch = master -debian-branch = debian/unstable -upstream-tag = %(version)s -compression = xz - -[buildpackage] -export-dir = ../build-area/ - -- GitLab From b129f9be996e2f1727c11047a7be11611c5bbbd7 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 5 Aug 2017 00:47:50 +0200 Subject: [PATCH 177/257] Correcting permissions in debian packaging files. Signed-off-by: Daniel Baumann --- debian/changelog | 1 + debian/python-reno.postinst | 0 debian/python-reno.postrm | 0 debian/python-reno.prerm | 0 debian/python3-reno.postinst | 0 debian/python3-reno.postrm | 0 debian/python3-reno.prerm | 0 7 files changed, 1 insertion(+) mode change 100644 => 100755 debian/python-reno.postinst mode change 100644 => 100755 debian/python-reno.postrm mode change 100644 => 100755 debian/python-reno.prerm mode change 100644 => 100755 debian/python3-reno.postinst mode change 100644 => 100755 debian/python3-reno.postrm mode change 100644 => 100755 debian/python3-reno.prerm diff --git a/debian/changelog b/debian/changelog index 1eb28c1..3a1ad8d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,7 @@ python-reno (1.3.0-5) UNRELEASED; urgency=medium * Updating standards version to 4.0.0. * Removing gbp.conf, not used anymore or should be specified in the developers dotfiles. + * Correcting permissions in debian packaging files. -- Daniel Baumann Fri, 04 Aug 2017 21:44:18 +0200 diff --git a/debian/python-reno.postinst b/debian/python-reno.postinst old mode 100644 new mode 100755 diff --git a/debian/python-reno.postrm b/debian/python-reno.postrm old mode 100644 new mode 100755 diff --git a/debian/python-reno.prerm b/debian/python-reno.prerm old mode 100644 new mode 100755 diff --git a/debian/python3-reno.postinst b/debian/python3-reno.postinst old mode 100644 new mode 100755 diff --git a/debian/python3-reno.postrm b/debian/python3-reno.postrm old mode 100644 new mode 100755 diff --git a/debian/python3-reno.prerm b/debian/python3-reno.prerm old mode 100644 new mode 100755 -- GitLab From a49aadd078565d3f34750938eabe9ec733de1593 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 6 Aug 2017 13:48:47 +0200 Subject: [PATCH 178/257] Updating standards version to 4.0.1. Signed-off-by: Daniel Baumann --- debian/changelog | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 3a1ad8d..54ba873 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ python-reno (1.3.0-5) UNRELEASED; urgency=medium * Removing gbp.conf, not used anymore or should be specified in the developers dotfiles. * Correcting permissions in debian packaging files. + * Updating standards version to 4.0.1. -- Daniel Baumann Fri, 04 Aug 2017 21:44:18 +0200 diff --git a/debian/control b/debian/control index dc8b0f5..617d58d 100644 --- a/debian/control +++ b/debian/control @@ -35,7 +35,7 @@ Build-Depends-Indep: python3-testtools (>= 1.4.0), subunit, testrepository, -Standards-Version: 4.0.0 +Standards-Version: 4.0.1 Vcs-Browser: https://anonscm.debian.org/cgit/openstack/libs/python-reno.git Vcs-Git: https://anonscm.debian.org/git/openstack/libs/python-reno.git Homepage: http://www.openstack.org/ -- GitLab From 90986c6fb266cacf97f325a046803048dd3bce4a Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Tue, 22 Aug 2017 18:48:49 +0200 Subject: [PATCH 179/257] Updating standards version to 4.1.0. Signed-off-by: Daniel Baumann --- debian/changelog | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 54ba873..846dc8c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,7 @@ python-reno (1.3.0-5) UNRELEASED; urgency=medium developers dotfiles. * Correcting permissions in debian packaging files. * Updating standards version to 4.0.1. + * Updating standards version to 4.1.0. -- Daniel Baumann Fri, 04 Aug 2017 21:44:18 +0200 diff --git a/debian/control b/debian/control index 617d58d..ddc4bde 100644 --- a/debian/control +++ b/debian/control @@ -35,7 +35,7 @@ Build-Depends-Indep: python3-testtools (>= 1.4.0), subunit, testrepository, -Standards-Version: 4.0.1 +Standards-Version: 4.1.0 Vcs-Browser: https://anonscm.debian.org/cgit/openstack/libs/python-reno.git Vcs-Git: https://anonscm.debian.org/git/openstack/libs/python-reno.git Homepage: http://www.openstack.org/ -- GitLab From 05ada00466aa7274b26ce0bc1412893dfecd15ee Mon Sep 17 00:00:00 2001 From: Sean McGinnis Date: Tue, 22 Aug 2017 15:01:06 -0500 Subject: [PATCH 180/257] Add user details for editing stable branch notes Add some details on how to correctly handle cases where release notes on stable branches need to be updated and what to do for notes that already have been modified. Change-Id: I90d899f0067b163b05ad81939ed52e0a8462741c --- doc/source/user/usage.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst index 22180e4..71835f4 100644 --- a/doc/source/user/usage.rst +++ b/doc/source/user/usage.rst @@ -315,6 +315,41 @@ sphinx document directly for debugging, use the ``report`` command. $ reno report . +Updating Stable Branch Release Notes +==================================== + +Occasionally it is necessary to update release notes for past releases +due to URLs changing or errors not being noticed until after they have +been released. In cases like these, it is important to note that any +updates to these release notes should be proposed directly to the stable +branch where they were introduced. + +.. note:: + + Due to the way reno scans release notes, if a note is updated on a + later branch instead of its original branch, it will then show up + in the release notes for the later release. + +If a note is accidentally modified in a later branch causing it to show +up in the wrong release's notes, the ``ignore-notes`` directive may be +used to manually exclude it from the generated output: + +:: + + =========================== + Pike Series Release Notes + =========================== + + .. release-notes:: + :branch: stable/pike + :ignore-notes: + mistake-note-1-ee6274467572906b.yaml, + mistake-note-2-dd6274467572906b.yaml + + +Even though the note will be parsed in the newer release, it will be +excluded from the output for that release. + Within OpenStack ================ -- GitLab From c29aef5f17a1a3232f84f83e8da318b616913199 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 24 Aug 2017 12:02:48 +0100 Subject: [PATCH 181/257] requirements: Stop requiring a specific pbr version This is no longer recommended. Change-Id: Ic9fefc2a9e115ccb77c29dd51015a9232a6f12b9 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2f2a16e..2cc558f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -pbr>=1.4 +pbr PyYAML>=3.1.0 six>=1.9.0 dulwich>=0.15.0 # Apache-2.0 -- GitLab From 4ce16307775439a188ac48d3146847b473d150bf Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 28 Sep 2017 17:52:17 -0400 Subject: [PATCH 182/257] update ref search logic for zuulv3 CI layout If the reference points explicitly to the origin remote, but that remote isn't present (as it won't be when zuul configures the repo in CI), then we want the shortened form of the reference. We put this option last in the list because we want the more explicit name to be used when someone is running reno locally with a more standard git configuration. Change-Id: I1fef83e95b0a0605db7e59676561bf1396e799e6 Signed-off-by: Doug Hellmann --- reno/scanner.py | 12 ++++++++++++ reno/tests/test_scanner.py | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/reno/scanner.py b/reno/scanner.py index 69942d2..227da2d 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -532,7 +532,17 @@ class Scanner(object): # name exists on the 'origin' remote. 'refs/remotes/origin/' + name, ] + # If the reference points explicitly to the origin remote, + # but that remote isn't present (as it won't be when zuul + # configures the repo in CI), then we want the shortened + # form of the reference. We put this option last in the + # list because we want the more explicit name to be used + # when someone is running reno locally with a more + # standard git configuration. + if name.startswith('origin/'): + candidates.append('refs/heads/' + name.partition('/')[-1]) for ref in candidates: + LOG.debug('looking for ref {!r} as {!r}'.format(name, ref)) key = ref.encode('utf-8') if key in self._repo.refs: sha = self._repo.refs[key] @@ -542,6 +552,8 @@ class Scanner(object): # signed tags point to the signature and we # need to dereference it to get to the commit. sha = o.object[1] + LOG.info('found ref {!r} as {!r} at {}'.format( + name, ref, sha)) return sha # If we end up here we didn't find any of the candidates. raise ValueError('Unknown reference {!r}'.format(name)) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index f8d52fa..261f090 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1794,6 +1794,26 @@ class GetRefTest(Base): expected = self.scanner._repo.head() self.assertEqual(expected, ref) + def test_stable_branch(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('stable/foo') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_stable_branch_with_origin_prefix(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('origin/stable/foo') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_no_such_value(self): + self.scanner = scanner.Scanner(self.c) + self.assertRaises( + ValueError, + self.scanner._get_ref, + 'missing/remote', + ) + class TagsTest(Base): -- GitLab From bbe3543f7855d8dab9ac2c445530d7a782bc1e6e Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 29 Sep 2017 10:14:09 -0400 Subject: [PATCH 183/257] release note for zuulv3 fix Change-Id: Ia09ced6805a4a694505b85166e797562a20a6ab1 Signed-off-by: Doug Hellmann --- .../reference-name-mangling-3c845ebf88af6944.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 releasenotes/notes/reference-name-mangling-3c845ebf88af6944.yaml diff --git a/releasenotes/notes/reference-name-mangling-3c845ebf88af6944.yaml b/releasenotes/notes/reference-name-mangling-3c845ebf88af6944.yaml new file mode 100644 index 0000000..fd72815 --- /dev/null +++ b/releasenotes/notes/reference-name-mangling-3c845ebf88af6944.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + The automatic branch name handling is updated so that if the + reference points explicitly to the origin remote, but that remote + isn't present (as it won't be when zuul configures the repo in + CI), then the shortened form of the reference without the prefix + is used instead. This allows explicit references to + ``origin/stable/name`` to be translated to ``stable/name`` and + find the expected branch. -- GitLab From 93a063978e07caacdbc36a500a565599ac022887 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Robles Date: Fri, 13 Oct 2017 09:13:57 +0300 Subject: [PATCH 184/257] Add option to create release note from user-provided template This is useful when automating the release note creation (when automating commit creation). Change-Id: I11793aaa3232b0f2c44521a998d466f427eecd93 --- doc/source/user/usage.rst | 14 ++++++++ ...-using-tempalte-file-be734d8698309409.yaml | 6 ++++ reno/create.py | 16 ++++++++- reno/main.py | 4 +++ reno/tests/test_create.py | 34 +++++++++++++++++++ 5 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/Enable-using-tempalte-file-be734d8698309409.yaml diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst index 71835f4..3aa69ed 100644 --- a/doc/source/user/usage.rst +++ b/doc/source/user/usage.rst @@ -40,6 +40,20 @@ The ``--edit`` option opens the new note in a text editor. ... Opens the editor set in the EDITOR environment variable, editing the new file ... Created new notes file in releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml +The ``--from-template`` option allows you to use a pre-defined file and use +that as the release note. + +:: + + $ reno new slug-goes-here --from-template my-file.yaml + ... Creates a release note using the provided file my-file.yaml ... + Created new notes file in releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml + +.. note:: + + You can also combine the flags ``--edit`` and ``--from-template`` + to create a release note from a specified file and immediately start an + editor to modify the new file. By default, the new note is created under ``./releasenotes/notes``. The ``--rel-notes-dir`` command-line flag changes the parent directory diff --git a/releasenotes/notes/Enable-using-tempalte-file-be734d8698309409.yaml b/releasenotes/notes/Enable-using-tempalte-file-be734d8698309409.yaml new file mode 100644 index 0000000..68a82ff --- /dev/null +++ b/releasenotes/notes/Enable-using-tempalte-file-be734d8698309409.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + The --from-template flag was added to the release note creation command. + This enables one to create a release note from a pre-defined template, + which is useful when automating the creation of commits. diff --git a/reno/create.py b/reno/create.py index 0b5e0ea..703e641 100644 --- a/reno/create.py +++ b/reno/create.py @@ -47,6 +47,16 @@ def _edit_file(filename): return True +def _get_user_template(template_file): + if not os.path.exists(template_file): + raise ValueError( + 'The provided template file %s doesn\'t ' + 'exist' % template_file, + ) + with open(template_file, 'r') as f: + return f.read() + + def create_cmd(args, conf): "Create a new release note file from the template." # NOTE(dhellmann): There is a short race window where we might try @@ -57,7 +67,11 @@ def create_cmd(args, conf): # concern. slug = args.slug.replace(' ', '-') filename = _pick_note_file_name(conf.notespath, slug) - _make_note_file(filename, conf.template) + if args.from_template: + template = _get_user_template(args.from_template) + else: + template = conf.template + _make_note_file(filename, template) if args.edit and not _edit_file(filename): print('Was unable to edit the new note. EDITOR environment variable ' 'is missing!') diff --git a/reno/main.py b/reno/main.py index 3237661..51e40b9 100644 --- a/reno/main.py +++ b/reno/main.py @@ -99,6 +99,10 @@ def main(argv=sys.argv[1:]): action='store_true', help='Edit note after its creation (require EDITOR env variable)', ) + do_new.add_argument( + '--from-template', + help='Template to get the release note from.', + ) do_new.add_argument( 'slug', help='descriptive title of note (keep it short)', diff --git a/reno/tests/test_create.py b/reno/tests/test_create.py index 2ad374a..46cb9b0 100644 --- a/reno/tests/test_create.py +++ b/reno/tests/test_create.py @@ -13,6 +13,7 @@ # under the License. import fixtures +import io import mock from reno import create @@ -45,6 +46,16 @@ class TestCreate(base.TestCase): super(TestCreate, self).setUp() self.tmpdir = self.useFixture(fixtures.TempDir()).path + def _create_user_template(self, contents): + filename = create._pick_note_file_name(self.tmpdir, 'usertemplate') + with open(filename, 'w') as f: + f.write(contents) + return filename + + def _get_file_path_from_output(self, output): + # Get the last consecutive word from the output and remove the newline + return output[output.rfind(" ") + 1:-1] + def test_create_from_template(self): filename = create._pick_note_file_name(self.tmpdir, 'theslug') create._make_note_file(filename, 'i-am-a-template') @@ -52,6 +63,29 @@ class TestCreate(base.TestCase): body = f.read() self.assertEqual('i-am-a-template', body) + def test_create_from_user_template(self): + args = mock.Mock() + args.from_template = self._create_user_template('i-am-a-user-template') + args.slug = 'theslug' + args.edit = False + conf = mock.Mock() + conf.notespath = self.tmpdir + with mock.patch('sys.stdout', new=io.StringIO()) as fake_out: + create.create_cmd(args, conf) + filename = self._get_file_path_from_output(fake_out.getvalue()) + with open(filename, 'r') as f: + body = f.read() + self.assertEqual('i-am-a-user-template', body) + + def test_create_from_user_template_fails_because_unexistent_file(self): + args = mock.Mock() + args.from_template = 'some-unexistent-file.yaml' + args.slug = 'theslug' + args.edit = False + conf = mock.Mock() + conf.notespath = self.tmpdir + self.assertRaises(ValueError, create.create_cmd, args, conf) + def test_edit(self): self.useFixture(fixtures.EnvironmentVariable('EDITOR', 'myeditor')) with mock.patch('subprocess.call') as call_mock: -- GitLab From fbc8ecc613483d00ba06357ecc792c3455a6b414 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 5 Nov 2017 21:19:40 +0000 Subject: [PATCH 185/257] Reimported debian/newton packaging. --- debian/changelog | 31 +++-- debian/compat | 2 +- debian/control | 120 +++++++++--------- debian/copyright | 2 +- debian/patches/series | 1 - .../use_less_entropy_in_unit_tests.patch | 55 -------- debian/python-reno.postinst | 0 debian/python-reno.postrm | 0 debian/python-reno.prerm | 0 debian/python3-reno.postinst | 0 debian/python3-reno.postrm | 0 debian/python3-reno.prerm | 0 debian/rules | 21 +-- debian/source/options | 1 + 14 files changed, 78 insertions(+), 155 deletions(-) delete mode 100644 debian/patches/series delete mode 100644 debian/patches/use_less_entropy_in_unit_tests.patch mode change 100755 => 100644 debian/python-reno.postinst mode change 100755 => 100644 debian/python-reno.postrm mode change 100755 => 100644 debian/python-reno.prerm mode change 100755 => 100644 debian/python3-reno.postinst mode change 100755 => 100644 debian/python3-reno.postrm mode change 100755 => 100644 debian/python3-reno.prerm diff --git a/debian/changelog b/debian/changelog index 846dc8c..b0074a6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,28 +1,27 @@ -python-reno (1.3.0-5) UNRELEASED; urgency=medium +python-reno (1.3.0-7) UNRELEASED; urgency=medium - * Updating vcs fields. - * Updating copyright format url. - * Running wrap-and-sort -bast. - * Updating maintainer field. - * Updating standards version to 4.0.0. - * Removing gbp.conf, not used anymore or should be specified in the - developers dotfiles. - * Correcting permissions in debian packaging files. - * Updating standards version to 4.0.1. - * Updating standards version to 4.1.0. + * Bumped debhelper compat version to 10 - -- Daniel Baumann Fri, 04 Aug 2017 21:44:18 +0200 + -- Ondřej Nový Thu, 24 Nov 2016 00:10:54 +0100 -python-reno (1.3.0-4) unstable; urgency=medium +python-reno (1.3.0-6) unstable; urgency=medium + + * Realy added gnupg as build-depends (Closes: #834685). + * Using pkgos-dh_auto_install from openstack-pkg-tools >= 52~. + + -- Thomas Goirand Thu, 13 Oct 2016 13:59:52 +0200 + +python-reno (1.3.0-5) unstable; urgency=medium [ Ondřej Nový ] * d/rules: Changed UPSTREAM_GIT protocol to https + * d/s/options: extend-diff-ignore of .gitreview + * d/control: Using OpenStack's Gerrit as VCS URLs. [ Thomas Goirand ] - * Add gnupg as build-depends-indep (Closes: #834685). - * Standards-Version: 3.9.8 (no change). + * Added gnupg as build-depends (Closes: #834685). - -- Thomas Goirand Fri, 09 Sep 2016 14:11:39 +0200 + -- Thomas Goirand Tue, 04 Oct 2016 14:30:48 +0200 python-reno (1.3.0-3) unstable; urgency=medium diff --git a/debian/compat b/debian/compat index ec63514..f599e28 100644 --- a/debian/compat +++ b/debian/compat @@ -1 +1 @@ -9 +10 diff --git a/debian/control b/debian/control index ddc4bde..58e8a28 100644 --- a/debian/control +++ b/debian/control @@ -1,85 +1,79 @@ Source: python-reno Section: python Priority: optional -Maintainer: Debian OpenStack -Uploaders: - Ivan Udovichenko , - Thomas Goirand , -Build-Depends: - debhelper (>= 9), - dh-python, - python-all, - python-pbr (>= 1.4), - python-setuptools, - python-sphinx, - python3-all, - python3-pbr (>= 1.4), - python3-setuptools, -Build-Depends-Indep: - git, - python-babel, - python-coverage, - python-hacking, - python-mock (>= 1.3), - python-oslosphinx (>= 2.5.0), - python-oslotest (>= 1.10.0), - python-testscenarios, - python-testtools (>= 1.4.0), - python-yaml, - python3-babel, - python3-coverage (>= 3.6), - python3-mock (>= 1.3), - python3-oslotest (>= 1.10.0), - python3-subunit, - python3-testscenarios, - python3-testtools (>= 1.4.0), - subunit, - testrepository, -Standards-Version: 4.1.0 -Vcs-Browser: https://anonscm.debian.org/cgit/openstack/libs/python-reno.git -Vcs-Git: https://anonscm.debian.org/git/openstack/libs/python-reno.git +Maintainer: PKG OpenStack +Uploaders: Ivan Udovichenko , + Thomas Goirand , +Build-Depends: debhelper (>= 10), + dh-python, + openstack-pkg-tools (>= 52~), + gnupg, + python-all, + python-pbr (>= 1.4), + python-setuptools, + python-sphinx, + python3-all, + python3-pbr (>= 1.4), + python3-setuptools, +Build-Depends-Indep: git, + python-babel, + python-coverage, + python-hacking, + python-mock (>= 1.3), + python-oslosphinx (>= 2.5.0), + python-oslotest (>= 1:1.10.0), + python-testscenarios, + python-testtools (>= 1.4.0), + python-yaml, + python3-babel, + python3-coverage (>= 3.6), + python3-mock (>= 1.3), + python3-oslotest (>= 1:1.10.0), + python3-subunit, + python3-testscenarios, + python3-testtools (>= 1.4.0), + subunit, + testrepository, +Standards-Version: 3.9.6 +Vcs-Browser: https://git.openstack.org/cgit/openstack/deb-python-reno?h=debian%2Fnewton +Vcs-Git: https://git.openstack.org/openstack/deb-python-reno -b debian/newton Homepage: http://www.openstack.org/ Package: python-reno Architecture: all -Depends: - git, - python-pbr (>= 1.4), - python-yaml, - ${misc:Depends}, - ${python:Depends}, -Suggests: - python-reno-doc, +Depends: git, + python-pbr (>= 1.4), + python-yaml, + ${misc:Depends}, + ${python:Depends}, +Suggests: python-reno-doc, Description: RElease NOtes manager - Python 2.x Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. . This package contains the Python 2.x module. -Package: python-reno-doc -Section: doc +Package: python3-reno Architecture: all -Depends: - ${misc:Depends}, - ${sphinxdoc:Depends}, -Description: RElease NOtes manager - doc +Depends: git, + python3-pbr (>= 1.4), + python3-yaml, + ${misc:Depends}, + ${python3:Depends}, +Suggests: python-reno-doc, +Description: RElease NOtes manager - Python 3.x Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. . - This package contains the documentation. + This package contains the Python 3.x module. -Package: python3-reno +Package: python-reno-doc +Section: doc Architecture: all -Depends: - git, - python3-pbr (>= 1.4), - python3-yaml, - ${misc:Depends}, - ${python3:Depends}, -Suggests: - python-reno-doc, -Description: RElease NOtes manager - Python 3.x +Depends: ${misc:Depends}, + ${sphinxdoc:Depends}, +Description: RElease NOtes manager - doc Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. . - This package contains the Python 3.x module. + This package contains the documentation. diff --git a/debian/copyright b/debian/copyright index 3112857..7adc8f2 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,4 +1,4 @@ -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: reno Source: http://www.openstack.org/ diff --git a/debian/patches/series b/debian/patches/series deleted file mode 100644 index cc26a2e..0000000 --- a/debian/patches/series +++ /dev/null @@ -1 +0,0 @@ -use_less_entropy_in_unit_tests.patch diff --git a/debian/patches/use_less_entropy_in_unit_tests.patch b/debian/patches/use_less_entropy_in_unit_tests.patch deleted file mode 100644 index 3cb8ef5..0000000 --- a/debian/patches/use_less_entropy_in_unit_tests.patch +++ /dev/null @@ -1,55 +0,0 @@ -Description: use less entropy in unit tests - Fix the logic for dealing with entropy in the unit tests so we consume - less. -Author: Doug Hellmann -Date: Sun, 28 Feb 2016 10:39:21 -0500 -Change-Id: I1faebfd5de0b9ae150bc2298df5d797fbbf83c07 -Signed-off-by: Doug Hellmann -Origin: upstream, https://review.openstack.org/#/c/285812 -Last-Update: 2016-03-01 - -diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py -index 7e5e3ef..4e6d0bb 100644 ---- a/reno/tests/test_scanner.py -+++ b/reno/tests/test_scanner.py -@@ -65,7 +65,7 @@ class GPGKeyFixture(fixtures.Fixture): - gnupg_version_re = re.compile('^gpg\s.*\s([\d+])\.([\d+])\.([\d+])') - gnupg_version = utils.check_output(['gpg', '--version'], - cwd=tempdir.path) -- for line in gnupg_version[0].split('\n'): -+ for line in gnupg_version.split('\n'): - gnupg_version = gnupg_version_re.match(line) - if gnupg_version: - gnupg_version = (int(gnupg_version.group(1)), -@@ -98,17 +98,17 @@ class GPGKeyFixture(fixtures.Fixture): - # Note that --quick-random (--debug-quick-random in GnuPG 2.x) - # does not have a corresponding preferences file setting and - # must be passed explicitly on the command line instead -- # if gnupg_version[0] == 1: -- # gnupg_random = '--quick-random' -- # elif gnupg_version[0] >= 2: -- # gnupg_random = '--debug-quick-random' -- # else: -- # gnupg_random = '' -- subprocess.check_call( -- ['gpg', '--gen-key', '--batch', -- # gnupg_random, -- config_file], -- cwd=tempdir.path) -+ if gnupg_version[0] == 1: -+ gnupg_random = '--quick-random' -+ elif gnupg_version[0] >= 2: -+ gnupg_random = '--debug-quick-random' -+ else: -+ gnupg_random = '' -+ cmd = ['gpg', '--gen-key', '--batch'] -+ if gnupg_random: -+ cmd.append(gnupg_random) -+ cmd.append(config_file) -+ subprocess.check_call(cmd, cwd=tempdir.path) - - - class Base(base.TestCase): --- -1.9.1 - diff --git a/debian/python-reno.postinst b/debian/python-reno.postinst old mode 100755 new mode 100644 diff --git a/debian/python-reno.postrm b/debian/python-reno.postrm old mode 100755 new mode 100644 diff --git a/debian/python-reno.prerm b/debian/python-reno.prerm old mode 100755 new mode 100644 diff --git a/debian/python3-reno.postinst b/debian/python3-reno.postinst old mode 100755 new mode 100644 diff --git a/debian/python3-reno.postrm b/debian/python3-reno.postrm old mode 100755 new mode 100644 diff --git a/debian/python3-reno.prerm b/debian/python3-reno.prerm old mode 100755 new mode 100644 diff --git a/debian/rules b/debian/rules index f318bf2..51d7498 100755 --- a/debian/rules +++ b/debian/rules @@ -1,28 +1,13 @@ #!/usr/bin/make -f -PYTHONS:=$(shell pyversions -vr) -PYTHON3S:=$(shell py3versions -vr) - UPSTREAM_GIT := https://github.com/openstack/reno.git --include /usr/share/openstack-pkg-tools/pkgos.make - -export OSLO_PACKAGE_VERSION=$(shell dpkg-parsechangelog | grep Version: | cut -d' ' -f2 | sed -e 's/^[[:digit:]]*://' -e 's/[-].*//' -e 's/~/.0/' | head -n 1) +include /usr/share/openstack-pkg-tools/pkgos.make %: dh $@ --buildsystem=python_distutils --with python2,python3,sphinxdoc -override_dh_install: - set -e ; for pyvers in $(PYTHONS); do \ - python$$pyvers setup.py install --install-layout=deb \ - --root $(CURDIR)/debian/python-reno; \ - done - set -e ; for pyvers in $(PYTHON3S); do \ - python$$pyvers setup.py install --install-layout=deb \ - --root $(CURDIR)/debian/python3-reno; \ - done - rm -rf $(CURDIR)/debian/python*-reno/usr/lib/python*/dist-packages/*.pth - mv $(CURDIR)/debian/python-reno/usr/bin/reno $(CURDIR)/debian/python-reno/usr/bin/python2-reno - mv $(CURDIR)/debian/python3-reno/usr/bin/reno $(CURDIR)/debian/python3-reno/usr/bin/python3-reno +override_dh_auto_install: + pkgos-dh_auto_install override_dh_auto_test: ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) diff --git a/debian/source/options b/debian/source/options index cb61fa5..9122245 100644 --- a/debian/source/options +++ b/debian/source/options @@ -1 +1,2 @@ extend-diff-ignore = "^[^/]*[.]egg-info/" +extend-diff-ignore = "^[.]gitreview$" -- GitLab From 65fb6ec61d224bb5aaec965db13f9223136e3382 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 5 Nov 2017 21:21:18 +0000 Subject: [PATCH 186/257] Fixed VCS URLS. --- debian/changelog | 6 +++++- debian/control | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index b0074a6..d60177f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,12 @@ python-reno (1.3.0-7) UNRELEASED; urgency=medium + [ Ondřej Nový ] * Bumped debhelper compat version to 10 - -- Ondřej Nový Thu, 24 Nov 2016 00:10:54 +0100 + [ Thomas Goirand ] + * Fixed VCS URLS. + + -- Thomas Goirand Sun, 05 Nov 2017 21:21:02 +0000 python-reno (1.3.0-6) unstable; urgency=medium diff --git a/debian/control b/debian/control index 58e8a28..3b5daa3 100644 --- a/debian/control +++ b/debian/control @@ -35,8 +35,8 @@ Build-Depends-Indep: git, subunit, testrepository, Standards-Version: 3.9.6 -Vcs-Browser: https://git.openstack.org/cgit/openstack/deb-python-reno?h=debian%2Fnewton -Vcs-Git: https://git.openstack.org/openstack/deb-python-reno -b debian/newton +Vcs-Browser: https://anonscm.debian.org/cgit/openstack/libs/python-reno.git +Vcs-Git: https://anonscm.debian.org/git/openstack/libs/python-reno.git Homepage: http://www.openstack.org/ Package: python-reno -- GitLab From 3036ffea7db93e021c50b86748c50a061c37a2c5 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 5 Nov 2017 21:21:52 +0000 Subject: [PATCH 187/257] debian/copyright format using https. --- debian/changelog | 1 + debian/copyright | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index d60177f..88320c8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,7 @@ python-reno (1.3.0-7) UNRELEASED; urgency=medium [ Thomas Goirand ] * Fixed VCS URLS. + * debian/copyright format using https. -- Thomas Goirand Sun, 05 Nov 2017 21:21:02 +0000 diff --git a/debian/copyright b/debian/copyright index 7adc8f2..3112857 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,4 +1,4 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: reno Source: http://www.openstack.org/ -- GitLab From 9b4b4bca4f67fd1b930914ad75956fe551a86971 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 5 Nov 2017 21:22:14 +0000 Subject: [PATCH 188/257] Running wrap-and-sort -bast. --- debian/changelog | 1 + debian/control | 114 +++++++++++++++++++++++++---------------------- 2 files changed, 62 insertions(+), 53 deletions(-) diff --git a/debian/changelog b/debian/changelog index 88320c8..04a721a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,6 +6,7 @@ python-reno (1.3.0-7) UNRELEASED; urgency=medium [ Thomas Goirand ] * Fixed VCS URLS. * debian/copyright format using https. + * Running wrap-and-sort -bast. -- Thomas Goirand Sun, 05 Nov 2017 21:21:02 +0000 diff --git a/debian/control b/debian/control index 3b5daa3..07ecb52 100644 --- a/debian/control +++ b/debian/control @@ -2,38 +2,41 @@ Source: python-reno Section: python Priority: optional Maintainer: PKG OpenStack -Uploaders: Ivan Udovichenko , - Thomas Goirand , -Build-Depends: debhelper (>= 10), - dh-python, - openstack-pkg-tools (>= 52~), - gnupg, - python-all, - python-pbr (>= 1.4), - python-setuptools, - python-sphinx, - python3-all, - python3-pbr (>= 1.4), - python3-setuptools, -Build-Depends-Indep: git, - python-babel, - python-coverage, - python-hacking, - python-mock (>= 1.3), - python-oslosphinx (>= 2.5.0), - python-oslotest (>= 1:1.10.0), - python-testscenarios, - python-testtools (>= 1.4.0), - python-yaml, - python3-babel, - python3-coverage (>= 3.6), - python3-mock (>= 1.3), - python3-oslotest (>= 1:1.10.0), - python3-subunit, - python3-testscenarios, - python3-testtools (>= 1.4.0), - subunit, - testrepository, +Uploaders: + Ivan Udovichenko , + Thomas Goirand , +Build-Depends: + debhelper (>= 10), + dh-python, + gnupg, + openstack-pkg-tools (>= 52~), + python-all, + python-pbr (>= 1.4), + python-setuptools, + python-sphinx, + python3-all, + python3-pbr (>= 1.4), + python3-setuptools, +Build-Depends-Indep: + git, + python-babel, + python-coverage, + python-hacking, + python-mock (>= 1.3), + python-oslosphinx (>= 2.5.0), + python-oslotest (>= 1:1.10.0), + python-testscenarios, + python-testtools (>= 1.4.0), + python-yaml, + python3-babel, + python3-coverage (>= 3.6), + python3-mock (>= 1.3), + python3-oslotest (>= 1:1.10.0), + python3-subunit, + python3-testscenarios, + python3-testtools (>= 1.4.0), + subunit, + testrepository, Standards-Version: 3.9.6 Vcs-Browser: https://anonscm.debian.org/cgit/openstack/libs/python-reno.git Vcs-Git: https://anonscm.debian.org/git/openstack/libs/python-reno.git @@ -41,39 +44,44 @@ Homepage: http://www.openstack.org/ Package: python-reno Architecture: all -Depends: git, - python-pbr (>= 1.4), - python-yaml, - ${misc:Depends}, - ${python:Depends}, -Suggests: python-reno-doc, +Depends: + git, + python-pbr (>= 1.4), + python-yaml, + ${misc:Depends}, + ${python:Depends}, +Suggests: + python-reno-doc, Description: RElease NOtes manager - Python 2.x Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. . This package contains the Python 2.x module. -Package: python3-reno +Package: python-reno-doc +Section: doc Architecture: all -Depends: git, - python3-pbr (>= 1.4), - python3-yaml, - ${misc:Depends}, - ${python3:Depends}, -Suggests: python-reno-doc, -Description: RElease NOtes manager - Python 3.x +Depends: + ${misc:Depends}, + ${sphinxdoc:Depends}, +Description: RElease NOtes manager - doc Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. . - This package contains the Python 3.x module. + This package contains the documentation. -Package: python-reno-doc -Section: doc +Package: python3-reno Architecture: all -Depends: ${misc:Depends}, - ${sphinxdoc:Depends}, -Description: RElease NOtes manager - doc +Depends: + git, + python3-pbr (>= 1.4), + python3-yaml, + ${misc:Depends}, + ${python3:Depends}, +Suggests: + python-reno-doc, +Description: RElease NOtes manager - Python 3.x Reno is a release notes manager for storing release notes in a git repository and then building documentation from them. . - This package contains the documentation. + This package contains the Python 3.x module. -- GitLab From ea76e7a379c17abd39304001797ac8b873081aba Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 5 Nov 2017 21:22:36 +0000 Subject: [PATCH 189/257] Updating maintainer field. --- debian/changelog | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 04a721a..1825fae 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,7 @@ python-reno (1.3.0-7) UNRELEASED; urgency=medium * Fixed VCS URLS. * debian/copyright format using https. * Running wrap-and-sort -bast. + * Updating maintainer field. -- Thomas Goirand Sun, 05 Nov 2017 21:21:02 +0000 diff --git a/debian/control b/debian/control index 07ecb52..ea9a287 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: python-reno Section: python Priority: optional -Maintainer: PKG OpenStack +Maintainer: Debian OpenStack Uploaders: Ivan Udovichenko , Thomas Goirand , -- GitLab From 1a53b418819917fa6d7638ca82f6880bf5a7d3e2 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 5 Nov 2017 21:23:19 +0000 Subject: [PATCH 190/257] Standards-Version is now 4.1.1. --- debian/changelog | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 1825fae..2d2ded8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ python-reno (1.3.0-7) UNRELEASED; urgency=medium * debian/copyright format using https. * Running wrap-and-sort -bast. * Updating maintainer field. + * Standards-Version is now 4.1.1. -- Thomas Goirand Sun, 05 Nov 2017 21:21:02 +0000 diff --git a/debian/control b/debian/control index ea9a287..e8b5900 100644 --- a/debian/control +++ b/debian/control @@ -37,7 +37,7 @@ Build-Depends-Indep: python3-testtools (>= 1.4.0), subunit, testrepository, -Standards-Version: 3.9.6 +Standards-Version: 4.1.1 Vcs-Browser: https://anonscm.debian.org/cgit/openstack/libs/python-reno.git Vcs-Git: https://anonscm.debian.org/git/openstack/libs/python-reno.git Homepage: http://www.openstack.org/ -- GitLab From 139727bb121f16bb27d43980940cc7ee51e490e6 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 5 Nov 2017 21:24:38 +0000 Subject: [PATCH 191/257] Fixed python3 shebang to use python3, not python3.x. --- debian/changelog | 3 ++- debian/rules | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 2d2ded8..81ff97e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -python-reno (1.3.0-7) UNRELEASED; urgency=medium +python-reno (1.3.0-7) unstable; urgency=medium [ Ondřej Nový ] * Bumped debhelper compat version to 10 @@ -9,6 +9,7 @@ python-reno (1.3.0-7) UNRELEASED; urgency=medium * Running wrap-and-sort -bast. * Updating maintainer field. * Standards-Version is now 4.1.1. + * Fixed python3 shebang to use python3, not python3.x. -- Thomas Goirand Sun, 05 Nov 2017 21:21:02 +0000 diff --git a/debian/rules b/debian/rules index 51d7498..35ffe74 100755 --- a/debian/rules +++ b/debian/rules @@ -9,6 +9,9 @@ include /usr/share/openstack-pkg-tools/pkgos.make override_dh_auto_install: pkgos-dh_auto_install +override_dh_python3: + dh_python3 --shebang=/usr/bin/python3 + override_dh_auto_test: ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) @echo "===> Running tests" -- GitLab From d42315a8206d328c796812a3d15183bb5ef174d0 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 5 Nov 2017 21:29:47 +0000 Subject: [PATCH 192/257] Now packaging 2.5.0. --- debian/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 81ff97e..a6e80be 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -python-reno (1.3.0-7) unstable; urgency=medium +python-reno (2.5.0-1) unstable; urgency=medium [ Ondřej Nový ] * Bumped debhelper compat version to 10 @@ -10,6 +10,7 @@ python-reno (1.3.0-7) unstable; urgency=medium * Updating maintainer field. * Standards-Version is now 4.1.1. * Fixed python3 shebang to use python3, not python3.x. + * New upstream release. -- Thomas Goirand Sun, 05 Nov 2017 21:21:02 +0000 -- GitLab From dbfe1a767f4b17bd380b662753353c6cd94b2454 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 5 Nov 2017 21:33:38 +0000 Subject: [PATCH 193/257] Fixed (build-)depends for this release. --- debian/changelog | 1 + debian/control | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/debian/changelog b/debian/changelog index a6e80be..df855d2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,6 +11,7 @@ python-reno (2.5.0-1) unstable; urgency=medium * Standards-Version is now 4.1.1. * Fixed python3 shebang to use python3, not python3.x. * New upstream release. + * Fixed (build-)depends for this release. -- Thomas Goirand Sun, 05 Nov 2017 21:21:02 +0000 diff --git a/debian/control b/debian/control index e8b5900..86d33e1 100644 --- a/debian/control +++ b/debian/control @@ -9,32 +9,32 @@ Build-Depends: debhelper (>= 10), dh-python, gnupg, - openstack-pkg-tools (>= 52~), + openstack-pkg-tools, python-all, - python-pbr (>= 1.4), + python-pbr, python-setuptools, python-sphinx, python3-all, - python3-pbr (>= 1.4), + python3-pbr, python3-setuptools, Build-Depends-Indep: git, python-babel, python-coverage, + python-dulwich, python-hacking, - python-mock (>= 1.3), - python-oslosphinx (>= 2.5.0), - python-oslotest (>= 1:1.10.0), + python-mock, + python-openstackdocstheme (>= 1.11.0), python-testscenarios, - python-testtools (>= 1.4.0), + python-testtools, python-yaml, python3-babel, - python3-coverage (>= 3.6), - python3-mock (>= 1.3), - python3-oslotest (>= 1:1.10.0), + python3-coverage, + python3-dulwich, + python3-mock, python3-subunit, python3-testscenarios, - python3-testtools (>= 1.4.0), + python3-testtools, subunit, testrepository, Standards-Version: 4.1.1 @@ -46,6 +46,7 @@ Package: python-reno Architecture: all Depends: git, + python-dulwich, python-pbr (>= 1.4), python-yaml, ${misc:Depends}, @@ -74,6 +75,7 @@ Package: python3-reno Architecture: all Depends: git, + python3-dulwich, python3-pbr (>= 1.4), python3-yaml, ${misc:Depends}, -- GitLab From b578588d66e7a9fed27591019b3003a43b65976f Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 5 Nov 2017 21:42:34 +0000 Subject: [PATCH 194/257] Using pkgos-dh_auto_test and blacklisting test_build_cache_db(). --- debian/changelog | 1 + debian/rules | 13 +------------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/debian/changelog b/debian/changelog index df855d2..5e60d9e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -12,6 +12,7 @@ python-reno (2.5.0-1) unstable; urgency=medium * Fixed python3 shebang to use python3, not python3.x. * New upstream release. * Fixed (build-)depends for this release. + * Using pkgos-dh_auto_test and blacklisting test_build_cache_db(). -- Thomas Goirand Sun, 05 Nov 2017 21:21:02 +0000 diff --git a/debian/rules b/debian/rules index 35ffe74..fd937d1 100755 --- a/debian/rules +++ b/debian/rules @@ -14,18 +14,7 @@ override_dh_python3: override_dh_auto_test: ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) - @echo "===> Running tests" - set -e ; set -x ; for i in 2.7 $(PYTHON3S) ; do \ - PYMAJOR=`echo $$i | cut -d'.' -f1` ; \ - echo "===> Testing with python$$i (python$$PYMAJOR)" ; \ - rm -rf .testrepository ; \ - testr-python$$PYMAJOR init ; \ - TEMP_REZ=`mktemp -t` ; \ - PYTHONPATH=$(CURDIR) PYTHON=python$$i testr-python$$PYMAJOR run --subunit | tee $$TEMP_REZ | subunit2pyunit ; \ - cat $$TEMP_REZ | subunit-filter -s --no-passthrough | subunit-stats ; \ - rm -f $$TEMP_REZ ; \ - testr-python$$PYMAJOR slowest ; \ - done + pkgos-dh_auto_test 'reno\.tests(?!.*test_cache\.TestCache\.test_build_cache_db.*)' endif override_dh_sphinxdoc: -- GitLab From f67d8e21d03d0f0a8859e3f59c1875951795ba3d Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sun, 5 Nov 2017 21:51:45 +0000 Subject: [PATCH 195/257] Missing python3-yaml b-d --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 86d33e1..bd3aa24 100644 --- a/debian/control +++ b/debian/control @@ -32,9 +32,9 @@ Build-Depends-Indep: python3-coverage, python3-dulwich, python3-mock, - python3-subunit, python3-testscenarios, python3-testtools, + python3-yaml, subunit, testrepository, Standards-Version: 4.1.1 -- GitLab From 247f3afddfe5169b28154d1e86fb4e06c5d8b834 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 7 Nov 2017 02:00:21 -0500 Subject: [PATCH 196/257] fix release note markup Change-Id: I38bd51792cff3fcec0f67b83bee8b1536c04fe3f Signed-off-by: Doug Hellmann --- .../notes/Enable-using-tempalte-file-be734d8698309409.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasenotes/notes/Enable-using-tempalte-file-be734d8698309409.yaml b/releasenotes/notes/Enable-using-tempalte-file-be734d8698309409.yaml index 68a82ff..b152d9f 100644 --- a/releasenotes/notes/Enable-using-tempalte-file-be734d8698309409.yaml +++ b/releasenotes/notes/Enable-using-tempalte-file-be734d8698309409.yaml @@ -1,6 +1,6 @@ --- features: - | - The --from-template flag was added to the release note creation command. + The ``--from-template`` flag was added to the release note creation command. This enables one to create a release note from a pre-defined template, which is useful when automating the creation of commits. -- GitLab From bcc65c9e846c829158671ccfbcd0308284f2092a Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 9 Nov 2017 20:08:07 +1100 Subject: [PATCH 197/257] ignore changes until the file is added within the scanned range Ignore changes to a note file until the scanner encounters the git operation that adds the file. This ensures that if a file is modified on master when it should be modified on another branch the note is not incorporated into the notes for the next release from master. Change-Id: I4e41c482695e93ebb5a73866c432636201a7534f Fixes-Bug: #1682796 Signed-off-by: Doug Hellmann --- reno/scanner.py | 46 +++++++++-- reno/tests/test_scanner.py | 164 ++++++++++++++++++++++++++++++++++++- 2 files changed, 201 insertions(+), 9 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 227da2d..007a9a8 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -252,6 +252,10 @@ class _ChangeTracker(object): self.last_name_by_id = {} # Remember uniqueids that have had files deleted. self.uniqueids_deleted = set() + # Remember files that are changed but not explicitly added so + # when we do see an add we can use the more recent tracking + # info and if we don't see the add we know to ignore the file. + self.seen_but_not_added = {} def _common(self, uniqueid, sha, version): if version not in self.versions: @@ -284,12 +288,24 @@ class _ChangeTracker(object): ) return - # A note is being added in this commit. If we have - # not seen it before, it was added here and never - # changed. - if uniqueid not in self.last_name_by_id: + if uniqueid in self.seen_but_not_added: + # The note was seen for a modify operation already but + # this is where it was added. We want to remember where + # the modification happened, because that came earlier in + # the scan (and therefore later in the history). + filename, sha = self.seen_but_not_added[uniqueid] self.last_name_by_id[uniqueid] = (filename, sha) LOG.info( + '%s: copying data for %s from commit %s', + uniqueid, filename, sha, + ) + del self.seen_but_not_added[uniqueid] + elif uniqueid not in self.last_name_by_id: + # The note was added already and we want to keep that + # other reference because it came earlier in the scan (and + # therefore later in the history). + self.last_name_by_id[uniqueid] = (filename, sha) + LOG.debug( '%s: new %s in commit %s', uniqueid, filename, sha, ) @@ -313,12 +329,19 @@ class _ChangeTracker(object): ) return + if uniqueid in self.last_name_by_id: + LOG.debug('%s: already added', uniqueid) + to_update = self.last_name_by_id + else: + LOG.debug('%s: seen but not added', uniqueid) + to_update = self.seen_but_not_added + # The file is being renamed. We may have seen it # before, if there were subsequent modifications, # so only store the name information if it is not # there already. - if uniqueid not in self.last_name_by_id: - self.last_name_by_id[uniqueid] = (filename, sha) + if uniqueid not in to_update: + to_update[uniqueid] = (filename, sha) LOG.info( '%s: update to %s in commit %s', uniqueid, filename, sha, @@ -343,12 +366,19 @@ class _ChangeTracker(object): ) return + if uniqueid in self.last_name_by_id: + LOG.debug('%s: already added', uniqueid) + to_update = self.last_name_by_id + else: + LOG.debug('%s: seen but not added', uniqueid) + to_update = self.seen_but_not_added + # An existing file is being modified. We may have # seen it before, if there were subsequent # modifications, so only store the name # information if it is not there already. - if uniqueid not in self.last_name_by_id: - self.last_name_by_id[uniqueid] = (filename, sha) + if uniqueid not in to_update: + to_update[uniqueid] = (filename, sha) LOG.info( '%s: update to %s in commit %s', uniqueid, filename, sha, diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 261f090..09befa5 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -1254,7 +1254,7 @@ class BranchTest(Base): self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') self.f2 = self._add_notes_file('slug2') self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') - self._add_notes_file('slug3') + self.f3 = self._add_notes_file('slug3') self.repo.git('tag', '-s', '-m', 'first tag', '3.0.0') def test_files_current_branch(self): @@ -1613,6 +1613,33 @@ class BranchTest(Base): self.assertIsNotNone(head2) self.assertEqual(head1, head2) + def test_modify_old_branch_note_on_master(self): + # Modify a note from a stable branch on master and ensure that + # the note does not appear in the scanner output from master. + # This should replicate the problem described in bug #1682796 + self.repo.git('checkout', '2.0.0') + self.repo.git('branch', 'stable/2') + self.repo.git('checkout', 'master') + with open(os.path.join(self.reporoot, self.f1), 'w') as f: + f.write('new file contents') + self.repo.commit('update %s' % self.f1) + self.c.override( + earliest_version=None, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '2.0.0': [self.f2], + '3.0.0': [self.f3], + }, + results, + ) + class ScanStopPointPrereleaseVersionsTest(Base): @@ -2227,3 +2254,138 @@ class AggregateChangesTest(Base): ('%016x' % n, 'modify', old_name, 'commit-id')], results, ) + + +class ChangeTrackerTest(base.TestCase): + + def setUp(self): + super(ChangeTrackerTest, self).setUp() + self.changes = scanner._ChangeTracker() + basename = '%s-%016x.yaml' % ('slug', 1) + self.filename = os.path.join('releasenotes', 'notes', basename) + self.filename2 = self.filename.replace('slug', 'guls') + self.uniqueid = scanner._get_unique_id(self.filename) + self.fake_logger = self.useFixture( + fixtures.FakeLogger( + format='%(levelname)8s %(name)s %(message)s', + level=logging.DEBUG, + nuke_handlers=True, + ) + ) + + def test_add(self): + self.changes.add(self.filename, 'sha1', 'version') + self.assertEqual( + {}, + self.changes.seen_but_not_added, + ) + self.assertEqual( + ['version'], + self.changes.versions, + ) + self.assertEqual( + 'version', + self.changes.earliest_seen[self.uniqueid], + ) + self.assertEqual( + {self.uniqueid: (self.filename, 'sha1')}, + self.changes.last_name_by_id, + ) + self.assertEqual( + set(), + self.changes.uniqueids_deleted, + ) + + def test_modify_with_add(self): + self.changes.modify(self.filename, 'sha2', 'version2') + self.changes.add(self.filename, 'sha1', 'version1') + self.assertEqual( + {}, + self.changes.seen_but_not_added, + ) + self.assertEqual( + ['version2', 'version1'], + self.changes.versions, + ) + self.assertEqual( + 'version1', + self.changes.earliest_seen[self.uniqueid], + ) + self.assertEqual( + {self.uniqueid: (self.filename, 'sha2')}, + self.changes.last_name_by_id, + ) + self.assertEqual( + set(), + self.changes.uniqueids_deleted, + ) + + def test_modify_without_add(self): + self.changes.modify(self.filename, 'sha2', 'version2') + self.assertEqual( + {self.uniqueid: (self.filename, 'sha2')}, + self.changes.seen_but_not_added, + ) + self.assertEqual( + ['version2'], + self.changes.versions, + ) + self.assertEqual( + 'version2', + self.changes.earliest_seen[self.uniqueid], + ) + self.assertEqual( + {}, + self.changes.last_name_by_id, + ) + self.assertEqual( + set(), + self.changes.uniqueids_deleted, + ) + + def test_rename_with_add(self): + self.changes.rename(self.filename2, 'sha2', 'version2') + self.changes.add(self.filename, 'sha1', 'version1') + self.assertEqual( + {}, + self.changes.seen_but_not_added, + ) + self.assertEqual( + ['version2', 'version1'], + self.changes.versions, + ) + self.assertEqual( + 'version1', + self.changes.earliest_seen[self.uniqueid], + ) + self.assertEqual( + {self.uniqueid: (self.filename2, 'sha2')}, + self.changes.last_name_by_id, + ) + self.assertEqual( + set(), + self.changes.uniqueids_deleted, + ) + + def test_rename_without_add(self): + self.changes.rename(self.filename2, 'sha2', 'version2') + self.assertEqual( + {self.uniqueid: (self.filename2, 'sha2')}, + self.changes.seen_but_not_added, + ) + self.assertEqual( + ['version2'], + self.changes.versions, + ) + self.assertEqual( + 'version2', + self.changes.earliest_seen[self.uniqueid], + ) + self.assertEqual( + {}, + self.changes.last_name_by_id, + ) + self.assertEqual( + set(), + self.changes.uniqueids_deleted, + ) -- GitLab From 515823488b78dff9dd96c6d80f5637f582a91775 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 9 Nov 2017 20:09:59 +1100 Subject: [PATCH 198/257] remove some duplication in ChangeTracker The modify() and rename() methods are identical. Collapse them into a single method. Change-Id: Ie6a3d8bda2e8ca76dbeecf267628ee0e1781e9f8 Signed-off-by: Doug Hellmann --- reno/scanner.py | 43 ++++++------------------------------------- 1 file changed, 6 insertions(+), 37 deletions(-) diff --git a/reno/scanner.py b/reno/scanner.py index 007a9a8..7a784e1 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -315,7 +315,7 @@ class _ChangeTracker(object): uniqueid, ) - def rename(self, filename, sha, version): + def _change(self, filename, sha, version): uniqueid = _get_unique_id(filename) self._common(uniqueid, sha, version) @@ -348,46 +348,15 @@ class _ChangeTracker(object): ) else: LOG.debug( - '%s: renamed file already known with the new name', - uniqueid, - ) - - def modify(self, filename, sha, version): - uniqueid = _get_unique_id(filename) - self._common(uniqueid, sha, version) - - # If we have recorded that a UID was deleted, that - # means that was the last change made to the file and - # we can ignore it. - if uniqueid in self.uniqueids_deleted: - LOG.debug( - '%s: has already been deleted, ignoring this change', + '%s: modified file already known', uniqueid, ) - return - if uniqueid in self.last_name_by_id: - LOG.debug('%s: already added', uniqueid) - to_update = self.last_name_by_id - else: - LOG.debug('%s: seen but not added', uniqueid) - to_update = self.seen_but_not_added + def rename(self, filename, sha, version): + self._change(filename, sha, version) - # An existing file is being modified. We may have - # seen it before, if there were subsequent - # modifications, so only store the name - # information if it is not there already. - if uniqueid not in to_update: - to_update[uniqueid] = (filename, sha) - LOG.info( - '%s: update to %s in commit %s', - uniqueid, filename, sha, - ) - else: - LOG.debug( - '%s: modified file already known', - uniqueid, - ) + def modify(self, filename, sha, version): + self._change(filename, sha, version) def delete(self, filename, sha, version): uniqueid = _get_unique_id(filename) -- GitLab From bd054fd5dce536494737afc4c499cf314ea3cb81 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 7 Nov 2017 02:15:31 -0500 Subject: [PATCH 199/257] update bindep list Add a bindep.txt file so CI does not use the defaults and we are explicit about the python-dev dependency. Add a note to the manual installation instructions explaining the need for the Python source headers. Change-Id: Ida4b897da4bdc3284a131166fdfa17bc9a4b1abd Signed-off-by: Doug Hellmann --- bindep.txt | 5 +++++ doc/source/install/index.rst | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 bindep.txt diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..359b0ae --- /dev/null +++ b/bindep.txt @@ -0,0 +1,5 @@ +gcc [platform:rpm test] +python-dev [platform:dpkg test] +python-devel [platform:rpm test] +python3-devel [platform:fedora platform:suse] +python3 [platform:suse] diff --git a/doc/source/install/index.rst b/doc/source/install/index.rst index 3ed7b92..9941e9f 100644 --- a/doc/source/install/index.rst +++ b/doc/source/install/index.rst @@ -6,6 +6,11 @@ At the command line:: $ pip install reno +.. note:: + + Reno's dependencies include C extension modules, which in turn + depend on having the Python source header files installed. + Sphinx Extension ================ -- GitLab From 9d058ae097e6cfac079fdbabadfc4270c6297e7f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 15 Nov 2017 16:02:37 -0500 Subject: [PATCH 200/257] add release note for scanner fix See I4e41c482695e93ebb5a73866c432636201a7534f for the code change. Change-Id: Ia881488d459cd9b86f73986b016ff9f1eff50699 Signed-off-by: Doug Hellmann --- .../notes/scanner-change-96682cb04fc66c0b.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 releasenotes/notes/scanner-change-96682cb04fc66c0b.yaml diff --git a/releasenotes/notes/scanner-change-96682cb04fc66c0b.yaml b/releasenotes/notes/scanner-change-96682cb04fc66c0b.yaml new file mode 100644 index 0000000..900ab23 --- /dev/null +++ b/releasenotes/notes/scanner-change-96682cb04fc66c0b.yaml @@ -0,0 +1,11 @@ +--- +fixes: + - | + A fix is included to ignore changes to a note file until the + scanner encounters the git operation that adds the file. This + ensures that if a file is modified on master when it should be + modified on another branch the note is not erroneously + incorporated into the notes for the next release from master. + fixes `bug 1682796`_ + + .. _bug 1682796: https://bugs.launchpad.net/neutron/+bug/1682796 -- GitLab From fab39dfcc834143f306a43e4b1f3c2aaf8eb2e36 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 21 Nov 2017 14:46:57 -0500 Subject: [PATCH 201/257] define options with help text Expand the way options are defined to include help text so we can eventually include that in generated configuration files and in online documentation. Change-Id: I0636f5e2fb9b21519f6cdda25a1ac546a3ffe174 Signed-off-by: Doug Hellmann --- reno/config.py | 117 +++++++++++++++++++++++++------------- reno/tests/test_config.py | 34 +++++++---- 2 files changed, 102 insertions(+), 49 deletions(-) diff --git a/reno/config.py b/reno/config.py index d44c72c..9eb11c4 100644 --- a/reno/config.py +++ b/reno/config.py @@ -10,8 +10,10 @@ # License for the specific language governing permissions and limitations # under the License. +import collections import logging import os.path +import textwrap import yaml @@ -20,41 +22,61 @@ from reno import defaults LOG = logging.getLogger(__name__) -class Config(object): +Opt = collections.namedtuple('Opt', 'name default help') - _OPTS = { - # The notes subdirectory within the relnotesdir where the - # notes live. - 'notesdir': defaults.NOTES_SUBDIR, +_OPTIONS = [ + Opt('notesdir', defaults.NOTES_SUBDIR, + textwrap.dedent(""" + The notes subdirectory within the relnotesdir where the + notes live. + """)), - # Should pre-release versions be merged into the final release - # of the same number (1.0.0.0a1 notes appear under 1.0.0). - 'collapse_pre_releases': True, + Opt('collapse_pre_releases', True, + textwrap.dedent(""" + Should pre-release versions be merged into the final release + of the same number (1.0.0.0a1 notes appear under 1.0.0). + """)), - # Should the scanner stop at the base of a branch (True) or go - # ahead and scan the entire history (False)? - 'stop_at_branch_base': True, + Opt('stop_at_branch_base', True, + textwrap.dedent(""" + Should the scanner stop at the base of a branch (True) or go + ahead and scan the entire history (False)? + """)), + Opt('branch', None, + textwrap.dedent(""" # The git branch to scan. Defaults to the "current" branch # checked out. - 'branch': None, + """)), + Opt('earliest_version', None, + textwrap.dedent(""" # The earliest version to be included. This is usually the # lowest version number, and is meant to be the oldest # version. - 'earliest_version': None, + """)), + Opt('template', defaults.TEMPLATE.format(defaults.PRELUDE_SECTION_NAME), + textwrap.dedent(""" # The template used by reno new to create a note. - 'template': defaults.TEMPLATE.format(defaults.PRELUDE_SECTION_NAME), - + """)), + + Opt('release_tag_re', + textwrap.dedent(''' + ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and + # pre-releases + '''), + textwrap.dedent(""" # The RE pattern used to match the repo tags representing a valid # release version. The pattern is compiled with the verbose and unicode # flags enabled. - 'release_tag_re': ''' - ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and - # pre-releases - ''', + """)), + Opt('pre_release_tag_re', + textwrap.dedent(''' + (?P\.\d+(?:[ab]|rc)+\d*)$ + '''), + textwrap.dedent(""" # The RE pattern used to check if a valid release version tag is also a # valid pre-release version. The pattern is compiled with the verbose # and unicode flags enabled. The pattern must define a group called @@ -62,20 +84,17 @@ class Config(object): # separator, e.g for pre-release version '12.0.0.0rc1' the default RE # pattern will identify '.0rc1' as the value of the group # 'pre_release'. - 'pre_release_tag_re': ''' - (?P\.\d+(?:[ab]|rc)+\d*)$ - ''', + """)), + Opt('branch_name_re', 'stable/.+', + textwrap.dedent(""" # The pattern for names for branches that are relevant when # scanning history to determine where to stop, to find the # "base" of a branch. Other branches are ignored. - 'branch_name_re': 'stable/.+', + """)), - # The identifiers and names of permitted sections in the - # release notes, in the order in which the final report will - # be generated. A prelude section will always be automatically - # inserted before the first element of this list. - 'sections': [ + Opt('sections', + [ ['features', 'New Features'], ['issues', 'Known Issues'], ['upgrade', 'Upgrade Notes'], @@ -85,14 +104,24 @@ class Config(object): ['fixes', 'Bug Fixes'], ['other', 'Other Notes'], ], + textwrap.dedent(""" + # The identifiers and names of permitted sections in the + # release notes, in the order in which the final report will + # be generated. A prelude section will always be automatically + # inserted before the first element of this list. + """)), + Opt('prelude_section_name', defaults.PRELUDE_SECTION_NAME, + textwrap.dedent(""" # The name of the prelude section in the note template. This # allows users to rename the section to, for example, # 'release_summary' or 'project_wide_general_announcements', # which is displayed in titlecase in the report after # replacing underscores with spaces. - 'prelude_section_name': defaults.PRELUDE_SECTION_NAME, + """)), + Opt('ignore_null_merges', True, + textwrap.dedent(""" # When this option is set to True, any merge commits with no # changes and in which the second or later parent is tagged # are considered "null-merges" that bring the tag information @@ -103,20 +132,27 @@ class Config(object): # confuses the regular traversal because it makes that stable # branch appear to be part of master and/or the later stable # branch. This option allows us to ignore those. - 'ignore_null_merges': True, + """)), + Opt('ignore_notes', [], + textwrap.dedent(""" # Note files to be ignored. It's useful to be able to ignore a # file if it is edited on the wrong branch. Notes should be # specified by their filename or UID. Setting the value in the # configuration file makes it apply to all branches. - 'ignore_notes': [], - } + """)), +] + + +class Config(object): + + _OPTS = {o.name: o for o in _OPTIONS} @classmethod def get_default(cls, opt): "Return the default for an option." try: - return cls._OPTS[opt] + return cls._OPTS[opt].default except KeyError: raise ValueError('unknown option name %r' % (opt,)) @@ -134,7 +170,7 @@ class Config(object): relnotesdir = defaults.RELEASE_NOTES_SUBDIR self.relnotesdir = relnotesdir # Initialize attributes from the defaults. - self.override(**self._OPTS) + self.override(**{o.name: o.default for o in _OPTIONS}) self._contents = {} self._load_file() @@ -161,7 +197,7 @@ class Config(object): def _rename_prelude_section(self, **kwargs): key = 'prelude_section_name' - if key in kwargs and kwargs[key] != self._OPTS[key]: + if key in kwargs and kwargs[key] != self._OPTS[key].default: new_prelude_name = kwargs[key] self.template = defaults.TEMPLATE.format(new_prelude_name) @@ -192,9 +228,9 @@ class Config(object): """ arg_values = { - o: getattr(parsed_args, o) - for o in self._OPTS.keys() - if hasattr(parsed_args, o) + o.name: getattr(parsed_args, o.name) + for o in _OPTIONS + if hasattr(parsed_args, o.name) } self.override(**arg_values) @@ -224,7 +260,10 @@ class Config(object): Returns the actual configuration options after overrides. """ - options = {o: getattr(self, o) for o in self._OPTS} + options = { + o.name: getattr(self, o.name) + for o in _OPTIONS + } return options # def parse_config_into(parsed_arguments): diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py index 82a4446..39cf94e 100644 --- a/reno/tests/test_config.py +++ b/reno/tests/test_config.py @@ -37,7 +37,11 @@ collapse_pre_releases: false def test_defaults(self): c = config.Config(self.tempdir.path) actual = c.options - self.assertEqual(config.Config._OPTS, actual) + expected = { + o.name: o.default + for o in config._OPTIONS + } + self.assertEqual(expected, actual) def test_override(self): c = config.Config(self.tempdir.path) @@ -45,8 +49,10 @@ collapse_pre_releases: false collapse_pre_releases=False, ) actual = c.options - expected = {} - expected.update(config.Config._OPTS) + expected = { + o.name: o.default + for o in config._OPTIONS + } expected['collapse_pre_releases'] = False self.assertEqual(expected, actual) @@ -59,8 +65,10 @@ collapse_pre_releases: false notesdir='value2', ) actual = c.options - expected = {} - expected.update(config.Config._OPTS) + expected = { + o.name: o.default + for o in config._OPTIONS + } expected['notesdir'] = 'value2' self.assertEqual(expected, actual) @@ -108,18 +116,24 @@ collapse_pre_releases: false def test_override_from_parsed_args_empty(self): c = self._run_override_from_parsed_args([]) actual = { - o: getattr(c, o) - for o in config.Config._OPTS.keys() + o.name: getattr(c, o.name) + for o in config._OPTIONS } - self.assertEqual(config.Config._OPTS, actual) + expected = { + o.name: o.default + for o in config._OPTIONS + } + self.assertEqual(expected, actual) def test_override_from_parsed_args(self): c = self._run_override_from_parsed_args([ '--no-collapse-pre-releases', ]) actual = c.options - expected = {} - expected.update(config.Config._OPTS) + expected = { + o.name: o.default + for o in config._OPTIONS + } expected['collapse_pre_releases'] = False self.assertEqual(expected, actual) -- GitLab From 7d419d4df6d6d434a209364f339b2d325ca1e19f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 21 Nov 2017 15:36:11 -0500 Subject: [PATCH 202/257] add an internal sphinx extension to show the configuration defaults Rather than relying on contributors to add configuration data to the usage page by hand, extract the information from the help text for the configuration options. Merge some of the existing documentation text with the old comments that were turned into help text (and remove the comment markers left behind in the last patch). Change-Id: Ic89e6f0a083648d332ae81d20b2adbf59e4207ce Signed-off-by: Doug Hellmann --- doc/source/conf.py | 2 +- doc/source/user/usage.rst | 97 +------------------------ reno/_exts/__init__.py | 0 reno/_exts/show_reno_config.py | 75 ++++++++++++++++++++ reno/config.py | 121 ++++++++++++++++--------------- reno/tests/test_exts.py | 126 +++++++++++++++++++++++++++++++++ 6 files changed, 267 insertions(+), 154 deletions(-) create mode 100644 reno/_exts/__init__.py create mode 100644 reno/_exts/show_reno_config.py create mode 100644 reno/tests/test_exts.py diff --git a/doc/source/conf.py b/doc/source/conf.py index 5c7b0bc..67ed674 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -25,7 +25,6 @@ else: has_theme = True -sys.path.insert(0, os.path.abspath('../..')) # -- General configuration ---------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be @@ -34,6 +33,7 @@ extensions = [ 'sphinx.ext.autodoc', # 'sphinx.ext.intersphinx', 'reno.sphinxext', + 'reno._exts.show_reno_config', ] if has_theme: diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst index 3aa69ed..944a03a 100644 --- a/doc/source/user/usage.rst +++ b/doc/source/user/usage.rst @@ -220,102 +220,7 @@ using command-line switches. For example: The following options are configurable: -`notesdir` - The notes subdirectory within the `relnotesdir` where the notes live. - - Defaults to ``notes``. - -`collapse_pre_releases` - Should pre-release versions be merged into the final release of the same - number (`1.0.0.0a1` notes appear under `1.0.0`). - - Defaults to ``True``. - -`stop_at_branch_base` - Should the scanner stop at the base of a branch (True) or go ahead and scan - the entire history (False)? - - Defaults to ``True``. - -`branch` - The git branch to scan. If a stable branch is specified but does not exist, - reno attempts to automatically convert that to an "end-of-life" tag. For - example, ``origin/stable/liberty`` would be converted to ``liberty-eol``. - - Defaults to the "current" branch checked out. - -`earliest_version` - The earliest version to be included. This is usually the lowest version - number, and is meant to be the oldest version. If unset, all versions will be - scanned. - - Defaults to ``None``. - -`template` - The template used by reno new to create a note. - -`release_tag_re` - The regex pattern used to match the repo tags representing a valid release - version. The pattern is compiled with the verbose and unicode flags enabled. - - Defaults to ``((?:[\d.ab]|rc)+)``. - -`pre_release_tag_re` - The regex pattern used to check if a valid release version tag is also a - valid pre-release version. The pattern is compiled with the verbose and - unicode flags enabled. The pattern must define a group called `pre_release` - that matches the pre-release part of the tag and any separator, e.g for - pre-release version `12.0.0.0rc1` the default RE pattern will identify - `.0rc1` as the value of the group 'pre_release'. - - Defaults to ``(?P\.\d+(?:[ab]|rc)+\d*)$``. - -`branch_name_re` - The pattern for names for branches that are relevant when scanning history to - determine where to stop, to find the "base" of a branch. Other branches are - ignored. - - Defaults to ``stable/.+``. - -`sections` - The identifiers and names of permitted sections in the release notes, in the - order in which the final report will be generated. A prelude section will - always be automatically inserted before the first element of this list. - -`prelude_section_name` - The name of the prelude section in the note template. Note that the - value for this must be a single word, but can have underscores. The - value is displayed in titlecase in the report after replacing - underscores with spaces. - - Defaults to ``prelude`` - -`ignore_null_merges` - OpenStack used to use null-merges to bring final release tags from - stable branches back into the master branch. This confuses the - regular traversal because it makes that stable branch appear to be - part of master and/or the later stable branch. This option allows us - to ignore those. - - When this option is set to True, any merge commits with no changes - and in which the second or later parent is tagged are considered - "null-merges" that bring the tag information into the current branch - but nothing else. - - Defaults to ``True``. - -`ignore_notes` - A list of filenames or UIDs for notes that should be ignored by the - reno scanner. It is most useful to set this when a note is edited on - the wrong branch, making it appear to be part of a release that it - is not. - - .. warning:: - - Setting the option in the main configuration file makes it apply - to all branches. To ignore a note in the HTML build, use the - ``ignore-notes`` parameter to the ``release-notes`` sphinx - directive. +.. show-reno-config:: Debugging ========= diff --git a/reno/_exts/__init__.py b/reno/_exts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/reno/_exts/show_reno_config.py b/reno/_exts/show_reno_config.py new file mode 100644 index 0000000..29afd1b --- /dev/null +++ b/reno/_exts/show_reno_config.py @@ -0,0 +1,75 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from docutils import nodes +from docutils.parsers import rst +from docutils.statemachine import ViewList + +from sphinx.util.nodes import nested_parse_with_titles + +from reno import config +import six + + +def _multi_line_string(s, indent=''): + output_lines = s.splitlines() + if not output_lines[0].strip(): + output_lines = output_lines[1:] + for l in output_lines: + yield indent + l + + +def _format_option_help(options): + "Produce RST lines for the configuration options." + for opt in options: + yield '``{}``'.format(opt.name) + for l in _multi_line_string(opt.help, ' '): + yield l + yield '' + if isinstance(opt.default, six.string_types) and '\n' in opt.default: + # Multi-line string + yield ' Defaults to' + yield '' + yield ' ::' + yield '' + for l in _multi_line_string(opt.default, ' '): + yield l + else: + yield ' Defaults to ``{!r}``'.format(opt.default) + yield '' + + +class ShowConfigDirective(rst.Directive): + + option_spec = {} + + has_content = True + + def run(self): + env = self.state.document.settings.env + app = env.app + + result = ViewList() + source_name = '<' + __name__ + '>' + for line in _format_option_help(config._OPTIONS): + app.info(line) + result.append(line, source_name) + + node = nodes.section() + node.document = self.state.document + nested_parse_with_titles(self.state, result, node) + + return node.children + + +def setup(app): + app.add_directive('show-reno-config', ShowConfigDirective) diff --git a/reno/config.py b/reno/config.py index 9eb11c4..b45a03a 100644 --- a/reno/config.py +++ b/reno/config.py @@ -26,71 +26,74 @@ Opt = collections.namedtuple('Opt', 'name default help') _OPTIONS = [ Opt('notesdir', defaults.NOTES_SUBDIR, - textwrap.dedent(""" + textwrap.dedent("""\ The notes subdirectory within the relnotesdir where the notes live. """)), Opt('collapse_pre_releases', True, - textwrap.dedent(""" + textwrap.dedent("""\ Should pre-release versions be merged into the final release of the same number (1.0.0.0a1 notes appear under 1.0.0). """)), Opt('stop_at_branch_base', True, - textwrap.dedent(""" + textwrap.dedent("""\ Should the scanner stop at the base of a branch (True) or go ahead and scan the entire history (False)? """)), Opt('branch', None, - textwrap.dedent(""" - # The git branch to scan. Defaults to the "current" branch - # checked out. + textwrap.dedent("""\ + The git branch to scan. Defaults to the "current" branch + checked out. If a stable branch is specified but does not + exist, reno attempts to automatically convert that to an + "end-of-life" tag. For example, ``origin/stable/liberty`` + would be converted to ``liberty-eol``. """)), Opt('earliest_version', None, - textwrap.dedent(""" - # The earliest version to be included. This is usually the - # lowest version number, and is meant to be the oldest - # version. + textwrap.dedent("""\ + The earliest version to be included. This is usually the + lowest version number, and is meant to be the oldest + version. If unset, all versions will be scanned. """)), Opt('template', defaults.TEMPLATE.format(defaults.PRELUDE_SECTION_NAME), - textwrap.dedent(""" - # The template used by reno new to create a note. + textwrap.dedent("""\ + The template used by reno new to create a note. """)), Opt('release_tag_re', - textwrap.dedent(''' + textwrap.dedent('''\ ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and # pre-releases '''), - textwrap.dedent(""" - # The RE pattern used to match the repo tags representing a valid - # release version. The pattern is compiled with the verbose and unicode - # flags enabled. + textwrap.dedent("""\ + The regex pattern used to match the repo tags representing a + valid release version. The pattern is compiled with the + verbose and unicode flags enabled. """)), Opt('pre_release_tag_re', - textwrap.dedent(''' + textwrap.dedent('''\ (?P\.\d+(?:[ab]|rc)+\d*)$ '''), - textwrap.dedent(""" - # The RE pattern used to check if a valid release version tag is also a - # valid pre-release version. The pattern is compiled with the verbose - # and unicode flags enabled. The pattern must define a group called - # 'pre_release' that matches the pre-release part of the tag and any - # separator, e.g for pre-release version '12.0.0.0rc1' the default RE - # pattern will identify '.0rc1' as the value of the group - # 'pre_release'. + textwrap.dedent("""\ + The regex pattern used to check if a valid release version tag + is also a valid pre-release version. The pattern is compiled + with the verbose and unicode flags enabled. The pattern must + define a group called 'pre_release' that matches the + pre-release part of the tag and any separator, e.g for + pre-release version '12.0.0.0rc1' the default pattern will + identify '.0rc1' as the value of the group 'pre_release'. """)), Opt('branch_name_re', 'stable/.+', - textwrap.dedent(""" - # The pattern for names for branches that are relevant when - # scanning history to determine where to stop, to find the - # "base" of a branch. Other branches are ignored. + textwrap.dedent("""\ + The pattern for names for branches that are relevant when + scanning history to determine where to stop, to find the + "base" of a branch. Other branches are ignored. """)), Opt('sections', @@ -104,42 +107,46 @@ _OPTIONS = [ ['fixes', 'Bug Fixes'], ['other', 'Other Notes'], ], - textwrap.dedent(""" - # The identifiers and names of permitted sections in the - # release notes, in the order in which the final report will - # be generated. A prelude section will always be automatically - # inserted before the first element of this list. + textwrap.dedent("""\ + The identifiers and names of permitted sections in the + release notes, in the order in which the final report will + be generated. A prelude section will always be automatically + inserted before the first element of this list. """)), Opt('prelude_section_name', defaults.PRELUDE_SECTION_NAME, - textwrap.dedent(""" - # The name of the prelude section in the note template. This - # allows users to rename the section to, for example, - # 'release_summary' or 'project_wide_general_announcements', - # which is displayed in titlecase in the report after - # replacing underscores with spaces. + textwrap.dedent("""\ + The name of the prelude section in the note template. This + allows users to rename the section to, for example, + 'release_summary' or 'project_wide_general_announcements', + which is displayed in titlecase in the report after + replacing underscores with spaces. """)), Opt('ignore_null_merges', True, - textwrap.dedent(""" - # When this option is set to True, any merge commits with no - # changes and in which the second or later parent is tagged - # are considered "null-merges" that bring the tag information - # into the current branch but nothing else. - # - # OpenStack used to use null-merges to bring final release - # tags from stable branches back into the master branch. This - # confuses the regular traversal because it makes that stable - # branch appear to be part of master and/or the later stable - # branch. This option allows us to ignore those. + textwrap.dedent("""\ + When this option is set to True, any merge commits with no + changes and in which the second or later parent is tagged + are considered "null-merges" that bring the tag information + into the current branch but nothing else. + + OpenStack used to use null-merges to bring final release + tags from stable branches back into the master branch. This + confuses the regular traversal because it makes that stable + branch appear to be part of master and/or the later stable + branch. This option allows us to ignore those. """)), Opt('ignore_notes', [], - textwrap.dedent(""" - # Note files to be ignored. It's useful to be able to ignore a - # file if it is edited on the wrong branch. Notes should be - # specified by their filename or UID. Setting the value in the - # configuration file makes it apply to all branches. + textwrap.dedent("""\ + Note files to be ignored. It's useful to be able to ignore a + file if it is edited on the wrong branch. Notes should be + specified by their filename or UID. + + Setting the option in the main configuration file makes it + apply to all branches. To ignore a note in the HTML build, use + the ``ignore-notes`` parameter to the ``release-notes`` sphinx + directive. """)), ] diff --git a/reno/tests/test_exts.py b/reno/tests/test_exts.py new file mode 100644 index 0000000..85445ca --- /dev/null +++ b/reno/tests/test_exts.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import textwrap + +from reno._exts import show_reno_config +from reno import config +from reno.tests import base + + +class TestMultiLineString(base.TestCase): + + def test_no_indent(self): + input = textwrap.dedent("""\ + The notes subdirectory within the relnotesdir where the + notes live. + """) + expected = '\n'.join([ + 'The notes subdirectory within the relnotesdir where the', + 'notes live.', + ]) + actual = '\n'.join(show_reno_config._multi_line_string(input)) + self.assertEqual(expected, actual) + + def test_with_indent(self): + input = textwrap.dedent("""\ + The notes subdirectory within the relnotesdir where the + notes live. + """) + expected = '\n'.join([ + ' The notes subdirectory within the relnotesdir where the', + ' notes live.', + ]) + actual = '\n'.join(show_reno_config._multi_line_string(input, ' ')) + self.assertEqual(expected, actual) + + def test_first_line_blank(self): + input = textwrap.dedent(""" + The notes subdirectory within the relnotesdir where the + notes live. + """) + expected = '\n'.join([ + ' The notes subdirectory within the relnotesdir where the', + ' notes live.', + ]) + actual = '\n'.join(show_reno_config._multi_line_string(input, ' ')) + self.assertEqual(expected, actual) + + +class TestFormatOptionHelp(base.TestCase): + + def test_simple_default(self): + opt = config.Opt( + 'notesdir', 'path/to/notes', + textwrap.dedent("""\ + The notes subdirectory within the relnotesdir where the + notes live. + """), + ) + actual = '\n'.join(show_reno_config._format_option_help([opt])) + expected = textwrap.dedent("""\ + ``notesdir`` + The notes subdirectory within the relnotesdir where the + notes live. + + Defaults to ``'path/to/notes'`` + """) + self.assertEqual(expected, actual) + + def test_bool_default(self): + opt = config.Opt( + 'collapse_pre_releases', True, + textwrap.dedent("""\ + Should pre-release versions be merged into the final release + of the same number (1.0.0.0a1 notes appear under 1.0.0). + """), + ) + actual = '\n'.join(show_reno_config._format_option_help([opt])) + expected = textwrap.dedent("""\ + ``collapse_pre_releases`` + Should pre-release versions be merged into the final release + of the same number (1.0.0.0a1 notes appear under 1.0.0). + + Defaults to ``True`` + """) + self.assertEqual(expected, actual) + + def test_multiline_default(self): + opt = config.Opt( + 'release_tag_re', + textwrap.dedent('''\ + ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and + # pre-releases + '''), + textwrap.dedent("""\ + The regex pattern used to match the repo tags representing a + valid release version. The pattern is compiled with the + verbose and unicode flags enabled. + """), + ) + actual = '\n'.join(show_reno_config._format_option_help([opt])) + expected = textwrap.dedent("""\ + ``release_tag_re`` + The regex pattern used to match the repo tags representing a + valid release version. The pattern is compiled with the + verbose and unicode flags enabled. + + Defaults to + + :: + + ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and + # pre-releases + """) + self.assertEqual(expected, actual) -- GitLab From 32b5405d0e8bcbee7048aac1247fc95619a08f07 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sun, 14 Jan 2018 10:15:32 -0500 Subject: [PATCH 203/257] improve output messages Change the order of initialization so the logging module is configured before the configuration file is read. This lets us log debug messages from the config module. Add some information from the config module about which configuration file is loaded, and in verbose mode which files we're looking for. Add some information to the scanner output to show whether collapse_pre_releases is set to true. Change-Id: Ia78bc63e2c29de2ac8771b46397a63639012bb90 Addresses-Bug: #1743202 Signed-off-by: Doug Hellmann --- reno/config.py | 2 ++ reno/main.py | 5 +++-- reno/scanner.py | 11 ++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/reno/config.py b/reno/config.py index b45a03a..66ad353 100644 --- a/reno/config.py +++ b/reno/config.py @@ -188,6 +188,7 @@ class Config(object): os.path.join(self.reporoot, 'reno.yaml')] for filename in filenames: + LOG.debug('looking for configuration file %s', filename) if os.path.isfile(filename): break else: @@ -197,6 +198,7 @@ class Config(object): try: with open(filename, 'r') as fd: self._contents = yaml.safe_load(fd) + LOG.info('loaded configuration file %s', filename) except IOError as err: LOG.warning('did not load config file %s: %s', filename, err) else: diff --git a/reno/main.py b/reno/main.py index 51e40b9..eeaf1df 100644 --- a/reno/main.py +++ b/reno/main.py @@ -191,12 +191,13 @@ def main(argv=sys.argv[1:]): do_linter.set_defaults(func=linter.lint_cmd) args = parser.parse_args(argv) - conf = config.Config(args.reporoot, args.relnotesdir) - conf.override_from_parsed_args(args) logging.basicConfig( level=args.verbosity, format='%(message)s', ) + conf = config.Config(args.reporoot, args.relnotesdir) + conf.override_from_parsed_args(args) + return args.func(args, conf) diff --git a/reno/scanner.py b/reno/scanner.py index 7a784e1..7aae0a3 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -912,9 +912,14 @@ class Scanner(object): collapse_pre_releases = self.conf.collapse_pre_releases stop_at_branch_base = self.conf.stop_at_branch_base - LOG.info('scanning %s/%s (branch=%s earliest_version=%s)', - reporoot.rstrip('/'), notesdir.lstrip('/'), - branch or '*current*', earliest_version) + LOG.info( + ('scanning %s/%s ' + '(branch=%s earliest_version=%s collapse_pre_releases=%s)'), + reporoot.rstrip('/'), notesdir.lstrip('/'), + branch or '*current*', + earliest_version, + collapse_pre_releases, + ) # Determine the current version, which might be an unreleased or # dev version if there are unreleased commits at the head of the -- GitLab From b9cf9a7371eec7f20089f51bbd12e78963a10960 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 29 Jan 2018 14:38:18 -0500 Subject: [PATCH 204/257] support scanning closed stable branches The scanner was stopping too soon when reviewing the history of a branch for which the previous branch had been "closed" by deleting the branch and tagging it with an -eol tag. This fix treats closed -eol branches the same way as open stable branches, using new configuration options to allow projects that use different naming conventions to have the same benefits. Change-Id: I8024929a2a95e00df48ce56939d54c1569fe18c5 Fixes-Bug: #1746076 Signed-off-by: Doug Hellmann --- ...ranch-config-options-8773caf240e4653f.yaml | 19 +++++++ reno/config.py | 19 +++++++ reno/scanner.py | 20 ++++++++ reno/tests/test_scanner.py | 50 +++++++++++++++++++ 4 files changed, 108 insertions(+) create mode 100644 releasenotes/notes/add-closed-branch-config-options-8773caf240e4653f.yaml diff --git a/releasenotes/notes/add-closed-branch-config-options-8773caf240e4653f.yaml b/releasenotes/notes/add-closed-branch-config-options-8773caf240e4653f.yaml new file mode 100644 index 0000000..6deedcb --- /dev/null +++ b/releasenotes/notes/add-closed-branch-config-options-8773caf240e4653f.yaml @@ -0,0 +1,19 @@ +--- +features: + - | + Adds new configuration options ``closed_branch_tag_re`` (to + identify tags that replace branches that have been closed) and + ``branch_name_prefix`` (a value to be added back to the closed + branch tag to turn it into the original branch name. + + These options are used in OpenStack to support scanning the + history of a branch based on the previous series branch, even + after that previous series is closed by setting + ``closed_branch_tag_re`` to ``(.+)-eol`` so that the series name + in a value like ``"mitaka-eol"`` is extracted using the + group. With ``branch_name_prefix`` set to ``"stable/"`` the tag + ``mitaka-eol`` becomes the branch name ``stable/mitaka``. +fixes: + - | + Fixes bug 1746076 so that scanning stable branches correctly + includes the history of earlier closed stable branches. diff --git a/reno/config.py b/reno/config.py index 66ad353..5c806ce 100644 --- a/reno/config.py +++ b/reno/config.py @@ -96,6 +96,25 @@ _OPTIONS = [ "base" of a branch. Other branches are ignored. """)), + Opt('closed_branch_tag_re', '(.+)-eol', + textwrap.dedent("""\ + The pattern for names for tags that replace closed + branches that are relevant when scanning history to + determine where to stop, to find the "base" of a + branch. Other tags are ignored. + """)), + + Opt('branch_name_prefix', 'stable/', + textwrap.dedent("""\ + The prefix to add to tags for closed branches + to restore the old branch name to allow sorting + to place the tag in the proper place in history. + For example, OpenStack turns "mitaka-eol" into + "stable/mitaka" by removing the "-eol" suffix + via closed_branch_tag_re and setting the prefix + to "stable/". + """)), + Opt('sections', [ ['features', 'New Features'], diff --git a/reno/scanner.py b/reno/scanner.py index 7aae0a3..da7f88a 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -513,6 +513,11 @@ class Scanner(object): self.conf.branch_name_re, flags=re.VERBOSE | re.UNICODE, ) + self.branch_name_prefix = self.conf.branch_name_prefix + self.closed_branch_tag_re = re.compile( + self.conf.closed_branch_tag_re, + flags=re.VERBOSE | re.UNICODE, + ) self._ignore_uids = set( _get_unique_id(fn) for fn in self.conf.ignore_notes @@ -818,6 +823,18 @@ class Scanner(object): elif r.startswith('refs/heads/'): name = r[11:] if name and self.branch_name_re.search(name): + LOG.debug('branch name %s', name) + branch_names.add(name) + continue + if not r.startswith('refs/tags/'): + continue + # See if the ref is a closed branch tag. + name = r.rpartition('/')[-1] + match = self.closed_branch_tag_re.search(name) + if match: + name = self.branch_name_prefix + match.group(1) + LOG.debug('closed branch tag %s becomes %s', + r.rpartition('/')[-1], name) branch_names.add(name) return list(sorted(branch_names)) @@ -987,6 +1004,7 @@ class Scanner(object): # base of the branch, which involves a bit of searching. LOG.debug('determining earliest_version from branch') branch_base = self._get_branch_base(branch) + LOG.debug('branch base %s', branch_base) scan_stop_tag = self._find_scan_stop_point( branch_base, versions_by_date, collapse_pre_releases, branch) @@ -995,6 +1013,8 @@ class Scanner(object): else: idx = versions_by_date.index(scan_stop_tag) earliest_version = versions_by_date[idx - 1] + LOG.debug('using version before %s as scan stop point', + scan_stop_tag) if earliest_version and collapse_pre_releases: if self.pre_release_tag_re.search(earliest_version): # The earliest version won't actually be the pre-release diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 09befa5..d51fddc 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -2389,3 +2389,53 @@ class ChangeTrackerTest(base.TestCase): set(), self.changes.uniqueids_deleted, ) + + +class GetSeriesBranchesTest(Base): + + def setUp(self): + super(GetSeriesBranchesTest, self).setUp() + self.repo.add_file('test.txt') + + def test_none(self): + self.scanner = scanner.Scanner(self.c) + self.assertEqual( + [], + self.scanner._get_series_branches(), + ) + + def test_real_branches_sorted_names(self): + self.repo.git( + 'checkout', '-b', 'stable/a', + ) + self.repo.git( + 'checkout', '-b', 'stable/b', + ) + self.scanner = scanner.Scanner(self.c) + self.assertEqual( + ['stable/a', 'stable/b'], + self.scanner._get_series_branches(), + ) + + def test_eol_tag(self): + self.repo.git( + 'tag', '-s', '-m', 'closed branch', 'a-eol', + ) + self.scanner = scanner.Scanner(self.c) + self.assertEqual( + ['stable/a'], + self.scanner._get_series_branches(), + ) + + def test_mix_tag_and_branch(self): + self.repo.git( + 'tag', '-s', '-m', 'closed branch', 'a-eol', + ) + self.repo.git( + 'checkout', '-b', 'stable/b', + ) + self.scanner = scanner.Scanner(self.c) + self.assertEqual( + ['stable/a', 'stable/b'], + self.scanner._get_series_branches(), + ) -- GitLab From d2e2f3d781cf000104578cf3cc000c7a826f8f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nov=C3=BD?= Date: Mon, 12 Feb 2018 10:27:44 +0100 Subject: [PATCH 205/257] d/control: Set Vcs-* to salsa.debian.org --- debian/changelog | 6 ++++++ debian/control | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 5e60d9e..b6e0b91 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +python-reno (2.5.0-2) UNRELEASED; urgency=medium + + * d/control: Set Vcs-* to salsa.debian.org + + -- Ondřej Nový Mon, 12 Feb 2018 10:27:44 +0100 + python-reno (2.5.0-1) unstable; urgency=medium [ Ondřej Nový ] diff --git a/debian/control b/debian/control index bd3aa24..833ef85 100644 --- a/debian/control +++ b/debian/control @@ -38,8 +38,8 @@ Build-Depends-Indep: subunit, testrepository, Standards-Version: 4.1.1 -Vcs-Browser: https://anonscm.debian.org/cgit/openstack/libs/python-reno.git -Vcs-Git: https://anonscm.debian.org/git/openstack/libs/python-reno.git +Vcs-Browser: https://salsa.debian.org/openstack-team/libs/python-reno +Vcs-Git: https://salsa.debian.org/openstack-team/libs/python-reno.git Homepage: http://www.openstack.org/ Package: python-reno -- GitLab From 8973f576740fe50ad37ff5ffd6c0d8c4bfaa3670 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 19 Feb 2018 09:50:52 -0500 Subject: [PATCH 206/257] trivial change to contributing instructions We need a trivial change to trigger documentation publishing. This makes the wording of the contributor instructions a little more welcoming. Change-Id: If46d7679542ada430b7e67a29b0cbf299c9147d9 Signed-off-by: Doug Hellmann --- CONTRIBUTING.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e716304..a25c634 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,5 +1,5 @@ -If you would like to contribute to the development of OpenStack, you must -follow the steps in this page: +If you would like to contribute to the development of OpenStack, start by +following the steps in this page: https://docs.openstack.org/infra/manual/developers.html If you already have a good understanding of how the system works and your -- GitLab From 6f0ce883269e587143590e030e9e263568ea1540 Mon Sep 17 00:00:00 2001 From: melissaml Date: Mon, 26 Feb 2018 01:45:52 +0800 Subject: [PATCH 207/257] Update url in HACKING.rst Change-Id: I9be5017c8bcdc601608ce0fc9a2fe83d3575042e --- HACKING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HACKING.rst b/HACKING.rst index ace48c9..ec2bd24 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -1,4 +1,4 @@ reno Style Commandments =============================================== -Read the OpenStack Style Commandments https://docs.openstack.org/developer/hacking/ +Read the OpenStack Style Commandments https://docs.openstack.org/hacking/latest/ -- GitLab From 3a61cc8d58d9b6b287c64e9d4c598cfabd7f9b71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nov=C3=BD?= Date: Tue, 27 Feb 2018 16:40:14 +0100 Subject: [PATCH 208/257] d/control: Add trailing tilde to min version depend to allow backports --- debian/changelog | 2 ++ debian/control | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index b6e0b91..bfe4d65 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ python-reno (2.5.0-2) UNRELEASED; urgency=medium * d/control: Set Vcs-* to salsa.debian.org + * d/control: Add trailing tilde to min version depend to allow + backports -- Ondřej Nový Mon, 12 Feb 2018 10:27:44 +0100 diff --git a/debian/control b/debian/control index 833ef85..8dfae23 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,7 @@ Uploaders: Ivan Udovichenko , Thomas Goirand , Build-Depends: - debhelper (>= 10), + debhelper (>= 10~), dh-python, gnupg, openstack-pkg-tools, -- GitLab From dcaa7e5263048b47ee33489e43e4adda58ef2208 Mon Sep 17 00:00:00 2001 From: melissaml Date: Sun, 11 Mar 2018 02:20:58 +0800 Subject: [PATCH 209/257] Update links in README Change the outdated links to the latest links in README Change-Id: Ia216770445afb5cbcab7d6b42d774ea4a690043c --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fa6c266..298b3eb 100644 --- a/README.rst +++ b/README.rst @@ -48,8 +48,8 @@ from future documentation builds. Project Meta-data ================= -.. .. image:: https://governance.openstack.org/badges/reno.svg - :target: https://governance.openstack.org/reference/tags/index.html +.. .. image:: https://governance.openstack.org/tc/badges/reno.svg + :target: https://governance.openstack.org/tc/reference/tags/index.html * Free software: Apache license * Documentation: https://docs.openstack.org/reno/latest/ -- GitLab From 521bd47d9790db5e67da05105d2afc420f0a45d1 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 14 Mar 2018 10:52:29 -0400 Subject: [PATCH 210/257] update bug report URLs to use storyboard Change-Id: Id04dbed78b272720119bb996c84e9ea7ca8a511a Signed-off-by: Doug Hellmann --- CONTRIBUTING.rst | 4 ++-- README.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index a25c634..5dac14e 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -10,5 +10,5 @@ https://docs.openstack.org/infra/manual/developers.html#development-workflow Pull requests submitted through GitHub will be ignored. -Bugs should be filed on Launchpad, not GitHub: -https://bugs.launchpad.net/reno +Bugs should be filed on Storyboard, not GitHub: +https://storyboard.openstack.org/#!/project/933 diff --git a/README.rst b/README.rst index 298b3eb..314faec 100644 --- a/README.rst +++ b/README.rst @@ -54,5 +54,5 @@ Project Meta-data * Free software: Apache license * Documentation: https://docs.openstack.org/reno/latest/ * Source: https://git.openstack.org/cgit/openstack/reno -* Bugs: https://bugs.launchpad.net/reno +* Bugs: https://storyboard.openstack.org/#!/project/933 * IRC: #openstack-release on freenode -- GitLab From 13a116fac22f4d8025f4405c8b4f0b46a414e0b4 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Fri, 9 Feb 2018 10:07:04 +0000 Subject: [PATCH 211/257] doc: Note development workflows supported by reno Change-Id: Ia0eb80d6bb2927a806c228fd2ef5feb1b714860d Story: 1588309 --- doc/source/user/design.rst | 60 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/doc/source/user/design.rst b/doc/source/user/design.rst index 2a1dd2a..2e52955 100644 --- a/doc/source/user/design.rst +++ b/doc/source/user/design.rst @@ -50,3 +50,63 @@ We want to eventually provide the ability to create a release notes file for a given release and add it to the source distribution for the project. As a first step, we are going to settle for publishing release notes in the documentation for a project. + +Assumptions +----------- + +Based on the above, *reno* makes a couple of assumptions about the release +policy used for a given project. *reno* expects all development, including bug +fixes, to take place on a single branch, ``master``. If *stable* or *release* +branches are used to support an older release then development should not take +place on these branches. Instead, bug fixes should be backported or +cherry-picked from ``master`` to the given *stable* branch. This is commonly +referred to as a `trunk-based`__ development workflow. + +.. code-block:: none + :caption: Trunk-based development. This is what *reno* expects. + + * bc823f0 (HEAD -> master) Fix a bug + | + | * 9723350 (tag: 1.0.1, stable/1.0) Fix a bug + | * 49e2158 (tag: 1.0.0) Release 1.0 + * | ad13f52 Fix a bug on master + * | 81b6b41 doc: Handle multiple branches in release notes + |/ + * 0faba45 Integrate reno + * a7beb14 (tag: 0.1.0) Add documentation + * e23b0c8 Add gitignore + * ff980c7 Initial commit + +(where ``9723350`` is the backported version of ``bc823f0``). + +By comparison, *reno* does not currently support projects where development is +spread across multiple active branches. In these situations, bug fixes are +developed on the offending *stable* or *release* branch and this branch is +later merged back into ``master``. This is commonly referred to as a +`git-flow-based`__ development workflow. + +.. code-block:: none + :caption: git-flow-based development. This is not compatible with *reno*. + + * 7df1078 (HEAD -> master) Merge branch 'stable/1.0' + |\ + | * 9723350 (tag: 1.0.1, stable/1.0) Fix a bug on stable + | * 49e2158 (tag: 1.0.0) Release 1.0 + * | ad13f52 Fix a bug on master + * | 81b6b41 doc: Handle multiple branches in release notes + |/ + * 0faba45 Integrate reno + * a7beb14 (tag: 0.1.0) Add documentation + * e23b0c8 Add gitignore + * ff980c7 Initial commit + +When this happens, *reno* has no way to distinguish between changes that apply +to the given *stable* branch and those that apply to ``master``. This is +because *reno* is *branch-based*, rather than *release-based*. If your project +uses this workflow, *reno* might not be for you. + +More information is available `here`__. + +__ https://trunkbaseddevelopment.com/ +__ http://nvie.com/posts/a-successful-git-branching-model/ +__ https://bugs.launchpad.net/reno/+bug/1588309 -- GitLab From aa7c66c54dc4487e5321d78228ad6a695d1177dc Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 19 Mar 2018 09:52:29 -0400 Subject: [PATCH 212/257] cleanups for dev workflow descriptions Use named links and refer to storyboard instead of launchpad. Change-Id: I059859c809351dbc34ffbd8c5ce75676ea7d16e3 Story: 1588309 Signed-off-by: Doug Hellmann --- doc/source/user/design.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/user/design.rst b/doc/source/user/design.rst index 2e52955..3a5f1e9 100644 --- a/doc/source/user/design.rst +++ b/doc/source/user/design.rst @@ -60,7 +60,7 @@ fixes, to take place on a single branch, ``master``. If *stable* or *release* branches are used to support an older release then development should not take place on these branches. Instead, bug fixes should be backported or cherry-picked from ``master`` to the given *stable* branch. This is commonly -referred to as a `trunk-based`__ development workflow. +referred to as a `trunk-based`_ development workflow. .. code-block:: none :caption: Trunk-based development. This is what *reno* expects. @@ -83,7 +83,7 @@ By comparison, *reno* does not currently support projects where development is spread across multiple active branches. In these situations, bug fixes are developed on the offending *stable* or *release* branch and this branch is later merged back into ``master``. This is commonly referred to as a -`git-flow-based`__ development workflow. +`git-flow-based`_ development workflow. .. code-block:: none :caption: git-flow-based development. This is not compatible with *reno*. @@ -105,8 +105,8 @@ to the given *stable* branch and those that apply to ``master``. This is because *reno* is *branch-based*, rather than *release-based*. If your project uses this workflow, *reno* might not be for you. -More information is available `here`__. +More information is available `here`_. -__ https://trunkbaseddevelopment.com/ -__ http://nvie.com/posts/a-successful-git-branching-model/ -__ https://bugs.launchpad.net/reno/+bug/1588309 +.. _trunk-based: https://trunkbaseddevelopment.com/ +.. _git-flow-based: http://nvie.com/posts/a-successful-git-branching-model/ +.. _here: https://storyboard.openstack.org/#!/story/1588309 -- GitLab From e5b7dae85e38bf646009b16c48c387150d7da6e3 Mon Sep 17 00:00:00 2001 From: Markus Zoeller Date: Fri, 23 Mar 2018 09:26:16 +0100 Subject: [PATCH 213/257] Add usage with travis CI to docs This change adds a description how to use reno within a travis CI setup. It's neither a bug in Travis nor in reno. It's just a different set of expectations of these two. As reno gets used outside of OpenStack too, it makes sense to clarify the usage. Change-Id: I675402402d5eb3d9afd374c149b21c75977946c7 Related-bug: 1703603 --- doc/source/user/usage.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst index 944a03a..e660265 100644 --- a/doc/source/user/usage.rst +++ b/doc/source/user/usage.rst @@ -277,3 +277,29 @@ the CI jobs and other project-specific settings used for reno. Refer to the `Managing Release Notes `__ section of the Project Team Guide for details. + +Within Travis CI +================ + +The `Travis CI `_ uses shallow git clones, +which prevents reno from accessing the repo data it needs. You'll +see an error message like the one mentioned in +`Launchpad bug 1703603 `_. + +To use reno within a Travis CI job, the cloned repository needs to be +unshallowed in the ``.travis.yml`` control file with the command +``git fetch --unshallow --tags``, like in this example: + +.. code-block:: yaml + + --- + language: python + + python: + - 2.7 + + install: + - git fetch --unshallow --tags + + script: + - reno report . -- GitLab From 187d586d5fdaba42d4e6b720ffbfa3b5530d4939 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 3 Apr 2018 17:42:00 -0400 Subject: [PATCH 214/257] add unreleased_version_title configuration option Added configuration option ``unreleased_version_title`` with associated Sphinx directive argument to control whether to show the computed version number for changes that have not been tagged, or to show a static title string specified in the option value. Change-Id: I3069a008451e93cb62e5e43611cf62de5026952f Signed-off-by: Doug Hellmann --- doc/source/releasenotes/index.rst | 1 + ...leased-version-title-86751f52745fd3b7.yaml | 8 +++++ reno/config.py | 7 ++++ reno/formatter.py | 9 +++-- reno/sphinxext.py | 8 +++++ reno/tests/test_formatter.py | 35 +++++++++++++++++++ 6 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/unreleased-version-title-86751f52745fd3b7.yaml diff --git a/doc/source/releasenotes/index.rst b/doc/source/releasenotes/index.rst index c226052..33f2737 100644 --- a/doc/source/releasenotes/index.rst +++ b/doc/source/releasenotes/index.rst @@ -3,6 +3,7 @@ =============== .. release-notes:: Unreleased + :unreleased-version-title: In Development .. release-notes:: Mainline :branch: origin/master diff --git a/releasenotes/notes/unreleased-version-title-86751f52745fd3b7.yaml b/releasenotes/notes/unreleased-version-title-86751f52745fd3b7.yaml new file mode 100644 index 0000000..a094ca2 --- /dev/null +++ b/releasenotes/notes/unreleased-version-title-86751f52745fd3b7.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Added configuration option ``unreleased_version_title`` with + associated Sphinx directive argument to control whether to show + the computed version number for changes that have not been + tagged, or to show a static title string specified in the option + value. diff --git a/reno/config.py b/reno/config.py index 5c806ce..dfa8284 100644 --- a/reno/config.py +++ b/reno/config.py @@ -167,6 +167,13 @@ _OPTIONS = [ the ``ignore-notes`` parameter to the ``release-notes`` sphinx directive. """)), + + Opt('unreleased_version_title', '', + textwrap.dedent("""\ + The title to use for any notes that do not appear in a + released version. If this option is unset, the development + version number is used (for example, ``3.0.0-3``). + """)), ] diff --git a/reno/formatter.py b/reno/formatter.py index 4092a8d..3834142 100644 --- a/reno/formatter.py +++ b/reno/formatter.py @@ -42,8 +42,13 @@ def format_report(loader, config, versions_to_include, title=None, file_contents[filename] = body for version in versions_to_include: - report.append(version) - report.append('=' * len(version)) + if '-' in version: + # This looks like an "unreleased version". + title = config.unreleased_version_title or version + else: + title = version + report.append(title) + report.append('=' * len(title)) report.append('') # Add the preludes. diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 45e57ef..21584a5 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -30,6 +30,9 @@ class ReleaseNotesDirective(rst.Directive): has_content = True + # FIXME(dhellmann): We should be able to build this information + # from the configuration options so we don't have to edit it + # manually when we add new options. option_spec = { 'branch': directives.unchanged, 'reporoot': directives.unchanged, @@ -40,6 +43,7 @@ class ReleaseNotesDirective(rst.Directive): 'earliest-version': directives.unchanged, 'stop-at-branch-base': directives.flag, 'ignore-notes': directives.unchanged, + 'unreleased-version-title': directives.unchanged, } def run(self): @@ -77,6 +81,10 @@ class ReleaseNotesDirective(rst.Directive): if 'earliest-version' in self.options: opt_overrides['earliest_version'] = self.options.get( 'earliest-version') + if 'unreleased-version-title' in self.options: + opt_overrides['unreleased_version_title'] = self.options.get( + 'unreleased-version-title') + if branch: opt_overrides['branch'] = branch if ignore_notes: diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py index 23ed3e1..0d223a4 100644 --- a/reno/tests/test_formatter.py +++ b/reno/tests/test_formatter.py @@ -156,3 +156,38 @@ class TestFormatterCustomSections(TestFormatterBase): expected = [prelude_pos, api_pos, features_pos] actual = list(sorted([prelude_pos, features_pos, api_pos])) self.assertEqual(expected, actual) + + +class TestFormatterCustomUnreleaseTitle(TestFormatterBase): + + note_bodies = { + 'note1': { + 'prelude': 'This is the prelude.', + }, + } + + scanner_output = { + '0.1.0-1': [('note1', 'shaA')], + } + + versions = ['0.1.0-1'] + + def test_with_title(self): + self.c.override(unreleased_version_title='Not Released') + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title='This is the title', + ) + self.assertIn('Not Released', result) + self.assertNotIn('0.1.0-1', result) + + def test_without_title(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title='This is the title', + ) + self.assertIn('0.1.0-1', result) -- GitLab From 9e17ac9c0bb145fa4c7c2dae070bf2e82d50932e Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Mon, 9 Apr 2018 00:19:29 +0200 Subject: [PATCH 215/257] Enhance the travis hack unshallowing is now enough for supporting Pull Requests. Travis also detach the HEAD its branch, resulting in reno failing when not finding the ref 'master' Change-Id: Ic03d50f5eef399009d955be13a806c2628f6166a Signed-off-by: Gaetan Semet --- doc/source/user/usage.rst | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst index e660265..f9ad134 100644 --- a/doc/source/user/usage.rst +++ b/doc/source/user/usage.rst @@ -282,13 +282,14 @@ Within Travis CI ================ The `Travis CI `_ uses shallow git clones, -which prevents reno from accessing the repo data it needs. You'll -see an error message like the one mentioned in +and detached head, which prevents reno from accessing the repo data +it needs. +You'll see an error message like the one mentioned in `Launchpad bug 1703603 `_. To use reno within a Travis CI job, the cloned repository needs to be -unshallowed in the ``.travis.yml`` control file with the command -``git fetch --unshallow --tags``, like in this example: +unshallowed and checked out in the right branch from your ``.travis.yml``, +like in the following example: .. code-block:: yaml @@ -296,10 +297,16 @@ unshallowed in the ``.travis.yml`` control file with the command language: python python: - - 2.7 + - 3.5 install: - - git fetch --unshallow --tags + - | + # Force unshallow and checkout the current branch + # https://docs.openstack.org/reno/latest/user/usage.html#within-travis-ci + git config remote.origin.fetch +refs/heads/*:refs/remotes/origin/* + git fetch --unshallow --tags + git symbolic-ref --short HEAD || git checkout -b ${TRAVIS_BRANCH}-test $TRAVIS_BRANCH + # Ref: https://stackoverflow.com/questions/32580821/how-can-i-customize-override-the-git-clone-step-in-travis-ci script: - reno report . -- GitLab From b7bb0f1e087046fee9ca8bd147fddbb58d5b1aa2 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 24 Aug 2017 12:04:30 +0100 Subject: [PATCH 216/257] Integrate a setuptools command Add a 'build_reno' setuptools command. 'skipsdist' is removed from 'tox', because there's no reason not to run this step as part of tox. Change-Id: I49b659948f35381a44e2fb5f91dd6bf9e8ef619e --- .gitignore | 3 + doc/source/user/index.rst | 1 + doc/source/user/setuptools.rst | 60 ++++++++ doc/source/user/usage.rst | 2 + ...tuptools-integration-950bd8ab6d2970c7.yaml | 6 + reno/defaults.py | 9 ++ reno/setup_command.py | 138 ++++++++++++++++++ setup.cfg | 2 + tox.ini | 1 - 9 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 doc/source/user/setuptools.rst create mode 100644 releasenotes/notes/setuptools-integration-950bd8ab6d2970c7.yaml create mode 100644 reno/setup_command.py diff --git a/.gitignore b/.gitignore index 7c5611f..3b982a9 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,9 @@ doc/build AUTHORS ChangeLog +# reno generates these +RELEASENOTES.rst + # Editors *~ .*.swp diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 3dc9ce2..09416a4 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -8,4 +8,5 @@ design usage sphinxext + setuptools examples diff --git a/doc/source/user/setuptools.rst b/doc/source/user/setuptools.rst new file mode 100644 index 0000000..631f27c --- /dev/null +++ b/doc/source/user/setuptools.rst @@ -0,0 +1,60 @@ +============================== + Python Packaging Integration +============================== + +*reno* supports integration with `setuptools`_ and *setuptools* derivatives +like *pbr* through a custom command - ``build_reno``. + +.. _pbr: http://docs.openstack.org/developer/pbr/ +.. _setuptools: https://setuptools.readthedocs.io/en/latest/ + +Using setuptools integration +---------------------------- + +To enable the ``build_reno`` command, you simply need to install *reno*. Once +done, simply run: + +.. code-block:: shell + + python setup.py build_reno + +You can configure the command in ``setup.py`` or ``setup.cfg``. To configure it +from ``setup.py``, add a ``build_reno`` section to ``command_options`` like so: + +.. code-block:: python + + from setuptools import setup + + setup( + name='mypackage', + version='0.1', + ... + command_options={ + 'build_reno': { + 'output_file': ('setup.py', 'RELEASENOTES.txt'), + }, + }, + ) + +To configure the command from ``setup.cfg``, add a ``build_reno`` section. For +example: + +.. code-block:: ini + + [build_reno] + output-file = RELEASENOTES.txt + +Options for setuptools integration +---------------------------------- + +These options related to the *setuptools* integration only. For general +configuration of *reno*, refer to :ref:`configuration`. + +``repo-root`` + The root directory of the Git repository; defaults to ``.`` + +``rel-notes-dir`` + The parent directory; defaults to ``releasenotes`` + +``output-file`` + The filename of the release notes file; defaults to ``RELEASENOTES.rst`` diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst index f9ad134..01de922 100644 --- a/doc/source/user/usage.rst +++ b/doc/source/user/usage.rst @@ -176,6 +176,8 @@ mistakes. The command exits with an error code if there are any mistakes, so it can be used in a build pipeline to force some correctness. +.. _configuration: + Configuring Reno ================ diff --git a/releasenotes/notes/setuptools-integration-950bd8ab6d2970c7.yaml b/releasenotes/notes/setuptools-integration-950bd8ab6d2970c7.yaml new file mode 100644 index 0000000..29ae876 --- /dev/null +++ b/releasenotes/notes/setuptools-integration-950bd8ab6d2970c7.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add a ``build_reno`` setuptools command that allows users to generate a + release notes document and a reno cache file that can be used to build + release notes documents without the full Git history present. diff --git a/reno/defaults.py b/reno/defaults.py index 927656d..ded78f2 100644 --- a/reno/defaults.py +++ b/reno/defaults.py @@ -11,8 +11,11 @@ # under the License. RELEASE_NOTES_SUBDIR = 'releasenotes' + NOTES_SUBDIR = 'notes' + PRELUDE_SECTION_NAME = 'prelude' + # This is a format string, so it needs to be formatted wherever it is used. TEMPLATE = """\ --- @@ -81,3 +84,9 @@ other: available in another section, such as the prelude. This may mean repeating some details. """ + +# default filename of a release notes file generated by the setuptool extension +RELEASE_NOTES_FILENAME = 'RELEASENOTES.rst' + +# default path to the root of the repo, used by the setuptools extension +REPO_ROOT = '.' diff --git a/reno/setup_command.py b/reno/setup_command.py new file mode 100644 index 0000000..102ab25 --- /dev/null +++ b/reno/setup_command.py @@ -0,0 +1,138 @@ +# Copyright 2017, Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Custom distutils command. + +For more information, refer to the distutils and setuptools source: + +- https://github.com/python/cpython/blob/3.6/Lib/distutils/cmd.py +- https://github.com/pypa/setuptools/blob/v36.0.0/setuptools/command/sdist.py +""" + +from distutils import cmd +from distutils import errors +from distutils import log + +import six + +from reno import cache +from reno import config +from reno import defaults +from reno import formatter +from reno import loader + +COMMAND_NAME = 'build_reno' # duplicates what's found in setup.cfg + + +def load_config(distribution): + """Utility method to parse distutils/setuptools configuration. + + This is for use by other libraries to extract the command configuration. + + :param distribution: A :class:`distutils.dist.Distribution` object + :returns: A tuple of a :class:`reno.config.Config` object, the output path + of the human-readable release notes file, and the output file of the + reno cache file + """ + option_dict = distribution.get_option_dict(COMMAND_NAME) + + if option_dict.get('repo_root') is not None: + repo_root = option_dict.get('repo_root')[1] + else: + repo_root = defaults.REPO_ROOT + + if option_dict.get('rel_notes_dir') is not None: + rel_notes_dir = option_dict.get('rel_notes_dir')[1] + else: + rel_notes_dir = defaults.RELEASE_NOTES_SUBDIR + + if option_dict.get('output_file') is not None: + output_file = option_dict.get('output_file')[1] + else: + output_file = defaults.RELEASE_NOTES_FILENAME + + conf = config.Config(repo_root, rel_notes_dir) + cache_file = loader.get_cache_filename(conf) + + return (conf, output_file, cache_file) + + +class BuildReno(cmd.Command): + """Distutils command to build reno release notes. + + The release note build can be triggered from distutils, and some + configuration can be included in ``setup.py`` or ``setup.cfg`` instead of + being specified from the command-line. + """ + description = 'Build reno release notes' + user_options = [ + ('repo-root=', None, 'the root directory of the Git repository; ' + 'defaults to "."'), + ('rel-notes-dir=', None, 'the parent directory; defaults to ' + '"releasenotes"'), + ('output-file=', None, 'the filename of the release notes file'), + ] + + def initialize_options(self): + self.repo_root = None + self.rel_notes_dir = None + self.output_file = None + + def finalize_options(self): + if self.repo_root is None: + self.repo_root = defaults.REPO_ROOT + + if self.rel_notes_dir is None: + self.rel_notes_dir = defaults.RELEASE_NOTES_SUBDIR + + if self.output_file is None: + self.output_file = defaults.RELEASE_NOTES_FILENAME + + # Overriding distutils' Command._ensure_stringlike which doesn't support + # unicode, causing finalize_options to fail if invoked again. Workaround + # for http://bugs.python.org/issue19570 + def _ensure_stringlike(self, option, what, default=None): + # type: (unicode, unicode, Any) -> Any + val = getattr(self, option) + if val is None: + setattr(self, option, default) + return default + elif not isinstance(val, six.string_types): + raise errors.DistutilsOptionError("'%s' must be a %s (got `%s`)" + % (option, what, val)) + return val + + def run(self): + conf = config.Config(self.repo_root, self.rel_notes_dir) + + # Generate the cache using the configuration options found + # in the release notes directory and the default output + # filename. + cache_filename = cache.write_cache_db( + conf=conf, + versions_to_include=[], # include all versions + outfilename=None, # generate the default name + ) + log.info('wrote cache file to %s', cache_filename) + + ldr = loader.Loader(conf) + text = formatter.format_report( + ldr, + conf, + ldr.versions, + title=self.distribution.metadata.name, + ) + with open(self.output_file, 'w') as f: + f.write(text) + log.info('wrote release notes to %s', self.output_file) diff --git a/setup.cfg b/setup.cfg index 4f304ec..2646fd0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,8 @@ packages = [entry_points] console_scripts = reno = reno.main:main +distutils.commands = + build_reno = reno.setup_command:BuildReno [extras] sphinx = diff --git a/tox.ini b/tox.ini index 88a0101..b7c6c61 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,6 @@ [tox] minversion = 1.6 envlist = py35,py27,pep8 -skipsdist = True [testenv] usedevelop = True -- GitLab From 847f13a14abe5a1d7bd748ba39ea4d948dff150d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 13 Apr 2018 11:15:42 -0500 Subject: [PATCH 217/257] Make section titles have stable anchor links Currently some sections produce anchors like "#id1", which will change over time as new releases are made. Instead, put in explicit references so that each version gets an anchor based on the version number, and so that each heading inside of the version has one based on the section title and the release version. Each anchor needs to be prefixed either with the title, if given, or with another string as some people include reno inside of other sphinx documentation and these will produce global refererences for the sphinx build. Also, since it's one global set of anchors, each version needs to be prefixed by title (or 'relnotes') and the section title because otherwise in cases like reno's own docs where some versions are included in the output twice, sphinx will produce conflicts. Change-Id: Ia6bdaffa6d0ae286542fbb7ae12613be56bdb326 --- ...able-section-anchors-d99258b6df39c0fa.yaml | 5 ++ reno/formatter.py | 34 +++++++++++-- reno/tests/test_formatter.py | 48 +++++++++++++++++++ 3 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/stable-section-anchors-d99258b6df39c0fa.yaml diff --git a/releasenotes/notes/stable-section-anchors-d99258b6df39c0fa.yaml b/releasenotes/notes/stable-section-anchors-d99258b6df39c0fa.yaml new file mode 100644 index 0000000..52a27e1 --- /dev/null +++ b/releasenotes/notes/stable-section-anchors-d99258b6df39c0fa.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Added explicitly calculated anchors to ensure section links are both + unique and stable over time. diff --git a/reno/formatter.py b/reno/formatter.py index 3834142..dd93fa7 100644 --- a/reno/formatter.py +++ b/reno/formatter.py @@ -25,6 +25,21 @@ def _indent_for_list(text, prefix=' '): ]) + '\n' +def _anchor(version_title, title): + title = title or 'relnotes' + return '.. _{title}_{version_title}:'.format( + title=title, + version_title=version_title) + + +def _section_anchor(section_title, version_title, title): + # Get the title and remove the trailing : + title = _anchor(version_title, title)[:-1] + return "{title}_{section_title}:".format( + title=title, + section_title=section_title) + + def format_report(loader, config, versions_to_include, title=None, show_source=True): report = [] @@ -44,11 +59,13 @@ def format_report(loader, config, versions_to_include, title=None, for version in versions_to_include: if '-' in version: # This looks like an "unreleased version". - title = config.unreleased_version_title or version + version_title = config.unreleased_version_title or version else: - title = version - report.append(title) - report.append('=' * len(title)) + version_title = version + report.append(_anchor(version_title, title)) + report.append('') + report.append(version_title) + report.append('=' * len(version_title)) report.append('') # Add the preludes. @@ -57,7 +74,11 @@ def format_report(loader, config, versions_to_include, title=None, notefiles_with_prelude = [(n, sha) for n, sha in notefiles if prelude_name in file_contents[n]] if notefiles_with_prelude: - report.append(prelude_name.replace('_', ' ').title()) + prelude_title = prelude_name.replace('_', ' ').title() + report.append(_section_anchor( + prelude_title, version_title, title)) + report.append('') + report.append(prelude_title) report.append('-' * len(prelude_name)) report.append('') @@ -76,6 +97,9 @@ def format_report(loader, config, versions_to_include, title=None, for n in file_contents[fn].get(section_name, []) ] if notes: + report.append(_section_anchor( + section_title, version_title, title)) + report.append('') report.append(section_title) report.append('-' * len(section_title)) report.append('') diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py index 0d223a4..4aecf18 100644 --- a/reno/tests/test_formatter.py +++ b/reno/tests/test_formatter.py @@ -156,6 +156,7 @@ class TestFormatterCustomSections(TestFormatterBase): expected = [prelude_pos, api_pos, features_pos] actual = list(sorted([prelude_pos, features_pos, api_pos])) self.assertEqual(expected, actual) + self.assertIn('.. _relnotes_1.0.0_API Changes:', result) class TestFormatterCustomUnreleaseTitle(TestFormatterBase): @@ -182,6 +183,7 @@ class TestFormatterCustomUnreleaseTitle(TestFormatterBase): ) self.assertIn('Not Released', result) self.assertNotIn('0.1.0-1', result) + self.assertIn('.. _This is the title_Not Released:', result) def test_without_title(self): result = formatter.format_report( @@ -191,3 +193,49 @@ class TestFormatterCustomUnreleaseTitle(TestFormatterBase): title='This is the title', ) self.assertIn('0.1.0-1', result) + self.assertIn('.. _This is the title_0.1.0-1:', result) + + +class TestFormatterAnchors(TestFormatterBase): + + note_bodies = { + 'note1': { + 'prelude': 'This is the prelude.', + }, + 'note2': { + 'issues': [ + 'This is the first issue.', + 'This is the second issue.', + ], + }, + 'note3': { + 'features': [ + 'We added a feature!', + ], + 'upgrade': None, + }, + } + + def test_with_title(self): + self.c.override(unreleased_version_title='Not Released') + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title='This is the title', + ) + self.assertIn('.. _This is the title_0.0.0:', result) + self.assertIn('.. _This is the title_0.0.0_Prelude:', result) + self.assertIn('.. _This is the title_1.0.0:', result) + self.assertIn('.. _This is the title_1.0.0_Known Issues:', result) + + def test_without_title(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + ) + self.assertIn('.. _relnotes_0.0.0:', result) + self.assertIn('.. _relnotes_0.0.0_Prelude:', result) + self.assertIn('.. _relnotes_1.0.0:', result) + self.assertIn('.. _relnotes_1.0.0_Known Issues:', result) -- GitLab From 4773959c2c75f550c4a7e6221c36079e9ed63beb Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 14 Apr 2018 10:22:06 -0500 Subject: [PATCH 218/257] Collapse Unreleased and Mainline sections The current release notes produced have an 'Unreleased' section that includes the releases back through 2.0.0, then mainline that includes those release notes as well as 1.5.0 through 0.1.0. Squish the two so that there is just one copy of each of them. Remove the explicit mention of origin/master and the earliest-version and just let the normal reno logic apply. Change-Id: I02bb2c69b8dd9e866e4efa3437e9c09c33488914 --- doc/source/releasenotes/index.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/source/releasenotes/index.rst b/doc/source/releasenotes/index.rst index 33f2737..9cf0ca3 100644 --- a/doc/source/releasenotes/index.rst +++ b/doc/source/releasenotes/index.rst @@ -2,12 +2,8 @@ Release Notes =============== -.. release-notes:: Unreleased - :unreleased-version-title: In Development - .. release-notes:: Mainline - :branch: origin/master + :unreleased-version-title: In Development .. release-notes:: Newton Series :branch: origin/stable/newton - :earliest-version: 1.9.0 -- GitLab From 00b96961e305bbcd53e66a6a143413be5afc224b Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 14 Apr 2018 10:34:30 -0500 Subject: [PATCH 219/257] Streamline published release notes reno isn't really doing stable branches anymore, but its release notes are set up like it is. Remove 'mainline' title from the primary section. Then update the 'Newton Series' title to 'Newton and Earlier' to indicate it's not just capturing Newton itself, but really everything before 2.0. Change-Id: Idbc70e270b1a70ffdee1da843cdc234164df05d2 --- doc/source/releasenotes/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/releasenotes/index.rst b/doc/source/releasenotes/index.rst index 9cf0ca3..aaac03e 100644 --- a/doc/source/releasenotes/index.rst +++ b/doc/source/releasenotes/index.rst @@ -2,8 +2,8 @@ Release Notes =============== -.. release-notes:: Mainline +.. release-notes:: :unreleased-version-title: In Development -.. release-notes:: Newton Series +.. release-notes:: Newton and Earlier :branch: origin/stable/newton -- GitLab From 7d355ee6c5894ec78de8de9731308f12f1f37558 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 25 Apr 2018 14:49:24 -0400 Subject: [PATCH 220/257] report when loading data from the cache file Add debug output to show when we are loading data from the cache file instead of scanning the git history, to make debugging easier. Change-Id: I42e96679b8c9e0369a6bc463a7fc99f78d6d55f9 Signed-off-by: Doug Hellmann --- reno/loader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reno/loader.py b/reno/loader.py index 6f8dcb0..ab39f39 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -66,6 +66,7 @@ class Loader(object): LOG.debug('ignoring cache file %s', self._cache_filename) if (not self._ignore_cache) and cache_file_exists: + LOG.debug('loading cache file %s', self._cache_filename) with open(self._cache_filename, 'r') as f: self._cache = yaml.safe_load(f.read()) # Save the cached scanner output to the same attribute -- GitLab From 451d1ebcb9c9b3744c62f97bd4b63959e793de1b Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 27 Apr 2018 09:48:17 -0400 Subject: [PATCH 221/257] include the branch name in anchors to make them more unique If the same version number appears in multiple output pages, sphinx reports a warning because the anchors generated by the formatter are the same. This patch adds the branch name to the anchor to make it more unique, since we should only be scanning a given branch one time in each release notes build. See http://logs.openstack.org/05/564405/1/check/build-openstack-releasenotes/b5fad95/job-output.txt.gz#_2018-04-27_06_02_12_077967 for an example of a failed build. Change-Id: I0b02e7eb319c95f885fc494977b356af71370970 Signed-off-by: Doug Hellmann --- reno/formatter.py | 23 +++++++++++++---------- reno/report.py | 1 + reno/sphinxext.py | 1 + reno/tests/test_formatter.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/reno/formatter.py b/reno/formatter.py index dd93fa7..a4f8742 100644 --- a/reno/formatter.py +++ b/reno/formatter.py @@ -25,23 +25,26 @@ def _indent_for_list(text, prefix=' '): ]) + '\n' -def _anchor(version_title, title): +def _anchor(version_title, title, branch): title = title or 'relnotes' - return '.. _{title}_{version_title}:'.format( + return '.. _{title}_{version_title}{branch}:'.format( title=title, - version_title=version_title) + version_title=version_title, + branch=('_' + branch.replace('/', '_') if branch else ''), + ) -def _section_anchor(section_title, version_title, title): +def _section_anchor(section_title, version_title, title, branch): # Get the title and remove the trailing : - title = _anchor(version_title, title)[:-1] + title = _anchor(version_title, title, branch)[:-1] return "{title}_{section_title}:".format( title=title, - section_title=section_title) + section_title=section_title, + ) def format_report(loader, config, versions_to_include, title=None, - show_source=True): + show_source=True, branch=None): report = [] if title: report.append('=' * len(title)) @@ -62,7 +65,7 @@ def format_report(loader, config, versions_to_include, title=None, version_title = config.unreleased_version_title or version else: version_title = version - report.append(_anchor(version_title, title)) + report.append(_anchor(version_title, title, branch)) report.append('') report.append(version_title) report.append('=' * len(version_title)) @@ -76,7 +79,7 @@ def format_report(loader, config, versions_to_include, title=None, if notefiles_with_prelude: prelude_title = prelude_name.replace('_', ' ').title() report.append(_section_anchor( - prelude_title, version_title, title)) + prelude_title, version_title, title, branch)) report.append('') report.append(prelude_title) report.append('-' * len(prelude_name)) @@ -98,7 +101,7 @@ def format_report(loader, config, versions_to_include, title=None, ] if notes: report.append(_section_anchor( - section_title, version_title, title)) + section_title, version_title, title, branch)) report.append('') report.append(section_title) report.append('-' * len(section_title)) diff --git a/reno/report.py b/reno/report.py index aab6be0..a6b3e6a 100644 --- a/reno/report.py +++ b/reno/report.py @@ -29,6 +29,7 @@ def report_cmd(args, conf): versions, title=args.title, show_source=args.show_source, + branch=args.branch, ) if args.output: with open(args.output, 'w') as f: diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 21584a5..177280a 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -110,6 +110,7 @@ class ReleaseNotesDirective(rst.Directive): conf, versions, title=title, + branch=branch, ) source_name = '<%s %s>' % (__name__, branch or 'current branch') result = statemachine.ViewList() diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py index 4aecf18..c643f74 100644 --- a/reno/tests/test_formatter.py +++ b/reno/tests/test_formatter.py @@ -239,3 +239,32 @@ class TestFormatterAnchors(TestFormatterBase): self.assertIn('.. _relnotes_0.0.0_Prelude:', result) self.assertIn('.. _relnotes_1.0.0:', result) self.assertIn('.. _relnotes_1.0.0_Known Issues:', result) + + def test_with_branch_and_title(self): + self.c.override(unreleased_version_title='Not Released') + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title='This is the title', + branch='stable/queens', + ) + self.assertIn('.. _This is the title_0.0.0_stable_queens:', result) + self.assertIn('.. _This is the title_0.0.0_stable_queens_Prelude:', + result) + self.assertIn('.. _This is the title_1.0.0_stable_queens:', result) + self.assertIn( + '.. _This is the title_1.0.0_stable_queens_Known Issues:', + result) + + def test_with_branch(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + branch='stable/queens', + ) + self.assertIn('.. _relnotes_0.0.0_stable_queens:', result) + self.assertIn('.. _relnotes_0.0.0_stable_queens_Prelude:', result) + self.assertIn('.. _relnotes_1.0.0_stable_queens:', result) + self.assertIn('.. _relnotes_1.0.0_stable_queens_Known Issues:', result) -- GitLab From c18078355b0d0817d86762b0ee6f7f948b536f14 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 27 Apr 2018 15:47:13 -0400 Subject: [PATCH 222/257] preserve the order of tags when reading the cache file Use an OrderedDict to preserve the order of tags when reading the cache file because the Loader assumes the scanner and cache manage the order of values. Without this change building the release notes with the cache in place results in random ordering of the output. Story: #2001934 Task: #14464 Change-Id: I07d73e2e0c2f8bb2fd66f26fcbc088ef9a2a23a5 Signed-off-by: Doug Hellmann --- releasenotes/notes/cache-ordering-6c743f68e3f7107f.yaml | 6 ++++++ reno/loader.py | 7 ++++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/cache-ordering-6c743f68e3f7107f.yaml diff --git a/releasenotes/notes/cache-ordering-6c743f68e3f7107f.yaml b/releasenotes/notes/cache-ordering-6c743f68e3f7107f.yaml new file mode 100644 index 0000000..39e40ee --- /dev/null +++ b/releasenotes/notes/cache-ordering-6c743f68e3f7107f.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Correct a problem with version number ordering when reading data + from the cache file. See + https://storyboard.openstack.org/#!/story/2001934 for details. diff --git a/reno/loader.py b/reno/loader.py index 6f8dcb0..9c44bfc 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import collections import logging import os.path @@ -71,10 +72,10 @@ class Loader(object): # Save the cached scanner output to the same attribute # it would be in if we had loaded it "live". This # simplifies some of the logic in the other methods. - self._scanner_output = { - n['version']: n['files'] + self._scanner_output = collections.OrderedDict( + (n['version'], n['files']) for n in self._cache['notes'] - } + ) else: self._scanner = scanner.Scanner(self._config) self._scanner_output = self._scanner.get_notes_by_version() -- GitLab From 79bebc4d15bef06c77d5ad550776b1b866d2b43d Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 10 May 2018 10:35:41 +0100 Subject: [PATCH 223/257] tests: Use mock decorator instead of context manager We're going to introduce some additional mocks here shortly which would make the current pattern unwieldy. Change-Id: I72d2f9e6bf5ed5fa95880aa4434725e88059a89b --- reno/tests/test_cache.py | 56 +++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/reno/tests/test_cache.py b/reno/tests/test_cache.py index 5051bc4..abf19a2 100644 --- a/reno/tests/test_cache.py +++ b/reno/tests/test_cache.py @@ -56,31 +56,33 @@ class TestCache(base.TestCase): ) self.c = config.Config('.') - def test_build_cache_db(self): - with mock.patch('reno.scanner.Scanner.get_notes_by_version') as gnbv: - gnbv.return_value = self.scanner_output - db = cache.build_cache_db( - self.c, - versions_to_include=[], - ) - expected = { - 'notes': [ - {'version': k, 'files': v} - for k, v in self.scanner_output.items() - ], - 'file-contents': { - 'note1': { - 'prelude': 'This is the prelude.\n', - }, - 'note2': { - 'issues': [ - 'This is the first issue.', - 'This is the second issue.', - ], - }, - 'note3': { - 'features': ['We added a feature!'], - }, + @mock.patch('reno.scanner.Scanner.get_notes_by_version') + def test_build_cache_db(self, gnbv): + gnbv.return_value = self.scanner_output + expected = { + 'notes': [ + {'version': k, 'files': v} + for k, v in self.scanner_output.items() + ], + 'file-contents': { + 'note1': { + 'prelude': 'This is the prelude.\n', }, - } - self.assertEqual(expected, db) + 'note2': { + 'issues': [ + 'This is the first issue.', + 'This is the second issue.', + ], + }, + 'note3': { + 'features': ['We added a feature!'], + }, + }, + } + + db = cache.build_cache_db( + self.c, + versions_to_include=[], + ) + + self.assertEqual(expected, db) -- GitLab From 0217f4c08d7ca3a3b8e34af877e90a7e5b0a15ce Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 11 May 2018 17:12:04 -0400 Subject: [PATCH 224/257] report line numbers for generated content more accurately Show the line numbers for content generated by the formatter. Go ahead and dump that output to the log, too, so someone can use it to debug build errors. Change-Id: Ib088df597d3b02e9040b90fece9552864d8f1fb7 Signed-off-by: Doug Hellmann --- reno/sphinxext.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 177280a..3a60393 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -114,8 +114,9 @@ class ReleaseNotesDirective(rst.Directive): ) source_name = '<%s %s>' % (__name__, branch or 'current branch') result = statemachine.ViewList() - for line in text.splitlines(): - result.append(line, source_name) + for line_num, line in enumerate(text.splitlines(), 1): + info('{:>4d}: {}'.format(line_num, line)) + result.append(line, source_name, line_num) node = nodes.section() node.document = self.state.document -- GitLab From dffc3db32dfd7da7f18fb291b19c64e2bde9fbe9 Mon Sep 17 00:00:00 2001 From: "huang.zhiping" Date: Sat, 9 Jun 2018 22:47:40 +0800 Subject: [PATCH 225/257] fix tox python3 overrides We want to default to running all tox environments under python 3, so set the basepython value in each environment. We do not want to specify a minor version number, because we do not want to have to update the file every time we upgrade python. We do not want to set the override once in testenv, because that breaks the more specific versions used in default environments like py35 and py36. Change-Id: I11b311684849dfa9bf19d05eed0b4fa515176a13 --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index b7c6c61..a3e1c66 100644 --- a/tox.ini +++ b/tox.ini @@ -15,20 +15,25 @@ commands = coverage report --show-missing [testenv:pep8] +basepython = python3 commands = flake8 reno -q lint [testenv:venv] +basepython = python3 commands = {posargs} [testenv:cover] +basepython = python3 commands = python setup.py test --coverage --testr-args='{posargs}' [testenv:docs] +basepython = python3 commands = python setup.py build_sphinx [testenv:debug] +basepython = python3 commands = oslo_debug_helper {posargs} [flake8] -- GitLab From f5dc4be34a175b6b77922782d730845e76f038ed Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 11 Jun 2018 15:25:08 -0400 Subject: [PATCH 226/257] import zuul job settings from project-config Change-Id: I6ac51f9d169f6e8d067166efee01d88b0f6e8529 Signed-off-by: Doug Hellmann --- .zuul.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .zuul.yaml diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000..584fe18 --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,6 @@ +- project: + templates: + - openstack-python-jobs + - openstack-python35-jobs + - publish-openstack-sphinx-docs + - publish-to-pypi -- GitLab From 9caada6138bf8f5101d6c057fa362907494699c9 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 11 Jun 2018 15:25:31 -0400 Subject: [PATCH 227/257] switch doc and pypi jobs to use python3 Update to the project templates that use python3 for the jobs. Depends-On: https://review.openstack.org/574375 Change-Id: I7f0797033a64901bb7248fba876b08e99586b3f2 Signed-off-by: Doug Hellmann --- .zuul.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 584fe18..d329b4e 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -2,5 +2,5 @@ templates: - openstack-python-jobs - openstack-python35-jobs - - publish-openstack-sphinx-docs - - publish-to-pypi + - publish-openstack-sphinx-docs-python3 + - publish-to-pypi-python3 -- GitLab From e8bb93f673a210af45b3e697c7a1c3eeb202aaf7 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 21 Jun 2018 15:18:12 +0100 Subject: [PATCH 228/257] sphinxext: Use 'sphinx.util.logging' This resolves a deprecation warning. It also allows us to silence some of the noise that reno emits (unnecessarily, most of the time) during a doc build. Change-Id: Ifdc14f0c0ebb82981fc79f9a24f7673b38a3bedf Signed-off-by: Stephen Finucane --- reno/sphinxext.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 3a60393..4ceeef0 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -10,13 +10,13 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import os.path from docutils import nodes from docutils.parsers import rst from docutils.parsers.rst import directives from docutils import statemachine +from sphinx.util import logging from sphinx.util.nodes import nested_parse_with_titles from dulwich import repo @@ -25,6 +25,8 @@ from reno import defaults from reno import formatter from reno import loader +LOG = logging.getLogger(__name__) + class ReleaseNotesDirective(rst.Directive): @@ -47,12 +49,6 @@ class ReleaseNotesDirective(rst.Directive): } def run(self): - env = self.state.document.settings.env - app = env.app - - def info(msg): - app.info('[reno] %s' % (msg,)) - title = ' '.join(self.content) branch = self.options.get('branch') reporoot_opt = self.options.get('reporoot', '.') @@ -92,9 +88,9 @@ class ReleaseNotesDirective(rst.Directive): conf.override(**opt_overrides) notesdir = os.path.join(relnotessubdir, conf.notesdir) - info('scanning %s for %s release notes' % - (os.path.join(conf.reporoot, notesdir), - branch or 'current branch')) + LOG.info('scanning %s for %s release notes' % ( + os.path.join(conf.reporoot, notesdir), + branch or 'current branch')) ldr = loader.Loader(conf) if version_opt is not None: @@ -104,7 +100,7 @@ class ReleaseNotesDirective(rst.Directive): ] else: versions = ldr.versions - info('got versions %s' % (versions,)) + LOG.info('got versions %s' % (versions,)) text = formatter.format_report( ldr, conf, @@ -115,7 +111,7 @@ class ReleaseNotesDirective(rst.Directive): source_name = '<%s %s>' % (__name__, branch or 'current branch') result = statemachine.ViewList() for line_num, line in enumerate(text.splitlines(), 1): - info('{:>4d}: {}'.format(line_num, line)) + LOG.debug('{:>4d}: {}'.format(line_num, line)) result.append(line, source_name, line_num) node = nodes.section() @@ -126,8 +122,3 @@ class ReleaseNotesDirective(rst.Directive): def setup(app): app.add_directive('release-notes', ReleaseNotesDirective) - - logging.basicConfig( - level=logging.INFO, - format='[%(name)s] %(message)s', - ) -- GitLab From e0c6b840518d94c472a0355545593cff0684351d Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Wed, 27 Jun 2018 08:17:14 -0400 Subject: [PATCH 229/257] Fix traceback when no args are passed to reno When a user passes no args to reno, we'll now properly display the help message. $ reno Traceback (most recent call last): File "/home/pabelanger/git/openstack/reno/.tox/pep8/bin/reno", line 10, in sys.exit(main()) File "/home/pabelanger/git/openstack/reno/reno/main.py", line 200, in main conf = config.Config(args.reporoot, args.relnotesdir) AttributeError: 'Namespace' object has no attribute 'reporoot' Change-Id: I9b9fdd044f43a05ced299dbb941de60d2bbc7192 Signed-off-by: Paul Belanger --- reno/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/reno/main.py b/reno/main.py index eeaf1df..b9837d7 100644 --- a/reno/main.py +++ b/reno/main.py @@ -88,6 +88,9 @@ def main(argv=sys.argv[1:]): ) subparsers = parser.add_subparsers( title='commands', + description='valid commands', + dest='command', + help='additional help', ) do_new = subparsers.add_parser( @@ -191,6 +194,10 @@ def main(argv=sys.argv[1:]): do_linter.set_defaults(func=linter.lint_cmd) args = parser.parse_args(argv) + # no arguments, print help messaging, then exit with error(1) + if not args.command: + parser.print_help() + return 1 logging.basicConfig( level=args.verbosity, -- GitLab From f8e6cde079b5a2a611dd57515ea2bb8368284aba Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 10 Jul 2018 12:45:12 -0400 Subject: [PATCH 230/257] fix documentation project template Use the correct project template name for the documentation jobs. Change-Id: Ibe7f567fcd04a6d7169b5322a92e9ff7c2041e89 Signed-off-by: Doug Hellmann --- .zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index d329b4e..c6de6d5 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -2,5 +2,5 @@ templates: - openstack-python-jobs - openstack-python35-jobs - - publish-openstack-sphinx-docs-python3 + - publish-openstack-docs-pti - publish-to-pypi-python3 -- GitLab From f7bf78209bdd9a98355f31ce32e4badc8b0ef672 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 5 Jul 2018 16:58:50 -0400 Subject: [PATCH 231/257] move package publishing template back to project-config The publish-to-pypi-python3 template should never have been set here in the repo because we need to manage it in a place that does not have different branches. Change-Id: I308dd851667f21f01030118596841279280eb883 Depends-On: https://review.openstack.org/580507 Signed-off-by: Doug Hellmann --- .zuul.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index c6de6d5..cafb35d 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -3,4 +3,3 @@ - openstack-python-jobs - openstack-python35-jobs - publish-openstack-docs-pti - - publish-to-pypi-python3 -- GitLab From c356e5295ec5091cc7780e93f46f1cb73c9e79e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nov=C3=BD?= Date: Fri, 3 Aug 2018 06:03:16 +0200 Subject: [PATCH 232/257] d/control: Use team+openstack@tracker.debian.org as maintainer --- debian/changelog | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index bfe4d65..9b516f0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ python-reno (2.5.0-2) UNRELEASED; urgency=medium * d/control: Set Vcs-* to salsa.debian.org * d/control: Add trailing tilde to min version depend to allow backports + * d/control: Use team+openstack@tracker.debian.org as maintainer -- Ondřej Nový Mon, 12 Feb 2018 10:27:44 +0100 diff --git a/debian/control b/debian/control index 8dfae23..c711427 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: python-reno Section: python Priority: optional -Maintainer: Debian OpenStack +Maintainer: Debian OpenStack Uploaders: Ivan Udovichenko , Thomas Goirand , -- GitLab From 9b1c353bd05459a54293e06b3b4f5f0bb04e9ff8 Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Fri, 24 Aug 2018 12:17:09 -0400 Subject: [PATCH 233/257] Allow tags prefixed with v in default regex The version scheme v1.0.0 is not an uncommon way to tag versions and used to be part of the semver spec. This patch allows the optional v prefix in the default version scheme. ref: https://github.com/semver/semver/blob/master/semver.md#is-v123-a-semantic-version Change-Id: I4a99bda8b788ee5fd2a8aca3463eb548d07f1313 Signed-off-by: Thanh Ha --- .../notes/tag-format-bd5018a813c804fd.yaml | 4 +++ reno/config.py | 4 +-- reno/tests/test_scanner.py | 31 +++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/tag-format-bd5018a813c804fd.yaml diff --git a/releasenotes/notes/tag-format-bd5018a813c804fd.yaml b/releasenotes/notes/tag-format-bd5018a813c804fd.yaml new file mode 100644 index 0000000..dbc58f3 --- /dev/null +++ b/releasenotes/notes/tag-format-bd5018a813c804fd.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Allow optional 'v' prefix in the default version tag regex. diff --git a/reno/config.py b/reno/config.py index dfa8284..03ae26b 100644 --- a/reno/config.py +++ b/reno/config.py @@ -66,7 +66,7 @@ _OPTIONS = [ Opt('release_tag_re', textwrap.dedent('''\ - ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and + ((?:v?[\d.ab]|rc)+) # digits, a, b, and rc cover regular and # pre-releases '''), textwrap.dedent("""\ @@ -77,7 +77,7 @@ _OPTIONS = [ Opt('pre_release_tag_re', textwrap.dedent('''\ - (?P\.\d+(?:[ab]|rc)+\d*)$ + (?P\.v?\d+(?:[ab]|rc)+\d*)$ '''), textwrap.dedent("""\ The regex pattern used to check if a valid release version tag diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index d51fddc..5c7a794 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -273,6 +273,20 @@ class BasicTest(Base): results, ) + def test_tag_with_v_prefix(self): + filename = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'tag with v prefix', 'v1.0.0') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'v1.0.0': [filename]}, + results, + ) + def test_note_commit_after_tag(self): self._make_python_package() self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') @@ -876,6 +890,23 @@ class PreReleaseTest(Base): results, ) + def test_tag_with_v_prefix(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', 'v1.0.0.0a1') + f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', 'v1.0.0.0a2') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'v1.0.0.0a2': [f1], + }, + results, + ) + def test_collapse(self): files = [] self._make_python_package() -- GitLab From a847e91d6c214c43828e238beac5f63d37035544 Mon Sep 17 00:00:00 2001 From: zhulingjie Date: Thu, 30 Aug 2018 06:30:19 -0400 Subject: [PATCH 234/257] Migrate the link of bug report button to storyboard Change-Id: I7cbcd38a6e5db3fcc48b778b61d2a0136b75c00c --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 67ed674..12150e9 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -42,8 +42,8 @@ if has_theme: # openstackdocstheme options repository_name = 'openstack/reno' -bug_project = 'reno' -bug_tag = '' +bug_project = '933' +bug_tag = 'docs' html_last_updated_fmt = '%Y-%m-%d %H:%M' # autodoc generation is a bit aggressive and a nuisance when doing heavy -- GitLab From d853ad850fe965be9311f3247b3fd64d5db4fd9c Mon Sep 17 00:00:00 2001 From: David Rabel Date: Mon, 3 Sep 2018 17:19:03 +0200 Subject: [PATCH 235/257] d/control, d/copyright: adjust to new upstream version --- debian/control | 3 +++ debian/copyright | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index c711427..27cf99c 100644 --- a/debian/control +++ b/debian/control @@ -11,12 +11,15 @@ Build-Depends: gnupg, openstack-pkg-tools, python-all, + python-dev, python-pbr, python-setuptools, python-sphinx, python3-all, + python3-dev, python3-pbr, python3-setuptools, + python3-sphinx, Build-Depends-Indep: git, python-babel, diff --git a/debian/copyright b/debian/copyright index 3112857..a13c231 100644 --- a/debian/copyright +++ b/debian/copyright @@ -3,8 +3,9 @@ Upstream-Name: reno Source: http://www.openstack.org/ Files: * -Copyright: (c) 2010-2016, OpenStack Foundation +Copyright: (c) 2010-2018, OpenStack Foundation (c) 2013, Hewlett-Packard Development Company, L.P. + (c) 2018, Red Hat, Inc. License: Apache-2 Files: debian/* -- GitLab From 481839eb90ddfdc7581b35ecc4814fd96df5fc85 Mon Sep 17 00:00:00 2001 From: David Rabel Date: Mon, 3 Sep 2018 17:20:14 +0200 Subject: [PATCH 236/257] Release new version: 2.9.2-1 --- debian/changelog | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 9b516f0..442fe95 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,11 +1,17 @@ -python-reno (2.5.0-2) UNRELEASED; urgency=medium +python-reno (2.9.2-1) unstable; urgency=medium + * Team upload. + + [ Ondřej Nový ] * d/control: Set Vcs-* to salsa.debian.org * d/control: Add trailing tilde to min version depend to allow backports * d/control: Use team+openstack@tracker.debian.org as maintainer - -- Ondřej Nový Mon, 12 Feb 2018 10:27:44 +0100 + [ David Rabel ] + * New upstream version 2.9.2 + + -- David Rabel Mon, 03 Sep 2018 17:19:07 +0200 python-reno (2.5.0-1) unstable; urgency=medium -- GitLab From 399049a49eaae3419343a70ce07b6843504bb928 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 4 Sep 2018 14:55:19 -0400 Subject: [PATCH 237/257] add lower-constraints tox environment and job Update the minimum for PyYAML to a version that appears on PyPI. Change-Id: Icd8dbdeb4dcbf45b5e0d5efb5e66baf3ef9749f5 Signed-off-by: Doug Hellmann --- .zuul.yaml | 1 + lower-constraints.txt | 5 +++++ requirements.txt | 2 +- tox.ini | 6 ++++++ 4 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 lower-constraints.txt diff --git a/.zuul.yaml b/.zuul.yaml index cafb35d..547a6e7 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -3,3 +3,4 @@ - openstack-python-jobs - openstack-python35-jobs - publish-openstack-docs-pti + - openstack-lower-constraints-jobs diff --git a/lower-constraints.txt b/lower-constraints.txt new file mode 100644 index 0000000..b9c41a3 --- /dev/null +++ b/lower-constraints.txt @@ -0,0 +1,5 @@ +Sphinx==1.5.1 +docutils==0.11 +PyYAML==3.10.0 +six==1.9.0 +dulwich==0.15.0 diff --git a/requirements.txt b/requirements.txt index 2cc558f..079e796 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ # process, which may cause wedges in the gate later. pbr -PyYAML>=3.1.0 +PyYAML>=3.10 six>=1.9.0 dulwich>=0.15.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index a3e1c66..920da36 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,12 @@ commands = flake8 reno -q lint +[testenv:lower-constraints] +basepython = python3 +deps = + -c{toxinidir}/lower-constraints.txt + {[testenv]deps} + [testenv:venv] basepython = python3 commands = {posargs} -- GitLab From 93582756d4098d3caf328385d0a96b463b025a0e Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 4 Sep 2018 15:05:23 -0400 Subject: [PATCH 238/257] move sphinx flags to tox.ini Use sphinx-build directly instead of relying on pbr's integration. Change-Id: I3ed1c9f3507769d784103a4e087ec1bbb73ce532 Signed-off-by: Doug Hellmann --- setup.cfg | 9 --------- tox.ini | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/setup.cfg b/setup.cfg index 2646fd0..fa2f12c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,15 +33,6 @@ sphinx = sphinx>=1.5.1,!=1.6.1 # BSD docutils>=0.11 # OSI-Approved Open Source, Public Domain -[build_sphinx] -source-dir = doc/source -build-dir = doc/build -all_files = 1 -warning-is-error = 1 - -[upload_sphinx] -upload-dir = doc/build/html - [compile_catalog] directory = reno/locale domain = reno diff --git a/tox.ini b/tox.ini index 920da36..2f8a3e1 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ commands = python setup.py test --coverage --testr-args='{posargs}' [testenv:docs] basepython = python3 -commands = python setup.py build_sphinx +commands = sphinx-build -a -W -E -b html doc/source doc/build/html [testenv:debug] basepython = python3 -- GitLab From ba9eab17ad4dd12956f26c8f56b7e51394af8428 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 4 Sep 2018 15:00:43 -0400 Subject: [PATCH 239/257] update sphinx to at least 1.6.1 Change-Id: I23ad557ccf63458b64bda80ce9f443ba41dec734 Signed-off-by: Doug Hellmann --- lower-constraints.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index b9c41a3..abff126 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -1,4 +1,4 @@ -Sphinx==1.5.1 +Sphinx==1.6.1 docutils==0.11 PyYAML==3.10.0 six==1.9.0 diff --git a/setup.cfg b/setup.cfg index fa2f12c..0aae22e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ distutils.commands = [extras] sphinx = - sphinx>=1.5.1,!=1.6.1 # BSD + sphinx>=1.6.1 # BSD docutils>=0.11 # OSI-Approved Open Source, Public Domain [compile_catalog] -- GitLab From 6dec42836f8efa36a9761416e8f63572e60b9e1f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 4 Sep 2018 15:00:02 -0400 Subject: [PATCH 240/257] build our docs with the lower-constraints We aren't unit testing the sphinxext module yet so run sphinx using the lower-constraints list as an integration test to let us verify that things work with those settings. Change-Id: I7cd770a2267dfcda3e7765e2708534eb705dedbd Signed-off-by: Doug Hellmann --- tox.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tox.ini b/tox.ini index 2f8a3e1..d9d5714 100644 --- a/tox.ini +++ b/tox.ini @@ -35,6 +35,12 @@ basepython = python3 commands = python setup.py test --coverage --testr-args='{posargs}' [testenv:docs] +# NOTE(dhellmann): Build our own documentation using the +# lower-constraints list as a hacky way to test the sphinx extension +# module, since we don't have separate unit tests for it. +deps = + -c{toxinidir}/lower-constraints.txt + {[testenv]deps} basepython = python3 commands = sphinx-build -a -W -E -b html doc/source doc/build/html -- GitLab From 030785abe2b96730b87279305fe40839672d43ba Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 4 Sep 2018 19:41:10 -0400 Subject: [PATCH 241/257] link to the europython 2018 presentation about reno Change-Id: I8fe819be0eedf76c5c0a17c278a6a65bc930272c Signed-off-by: Doug Hellmann --- doc/source/index.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index e5bbf0f..d5b8c4e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,5 +1,14 @@ .. include:: ../../README.rst +EuroPython 2018 Presentation +============================ + +.. raw:: html + + + Contents ======== -- GitLab From 355e1674be8c37e67f59e8bcd39bf7d6a0aac792 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 11 Sep 2018 07:35:03 +1000 Subject: [PATCH 242/257] Use unicode for debug string On Python2, the "line" variable here is a unicode string, but the LOG.debug statement is trying to format it with a bytes string; this leads to a conversion error when the line has unicode characters in it. Ensure the logging format string is a unicode object to avoid this. Change-Id: I9948ffcea3fd10c4b6695593fa2a42731b44c02f --- reno/sphinxext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 4ceeef0..e805e7c 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -111,7 +111,7 @@ class ReleaseNotesDirective(rst.Directive): source_name = '<%s %s>' % (__name__, branch or 'current branch') result = statemachine.ViewList() for line_num, line in enumerate(text.splitlines(), 1): - LOG.debug('{:>4d}: {}'.format(line_num, line)) + LOG.debug(u'{:>4d}: {}'.format(line_num, line)) result.append(line, source_name, line_num) node = nodes.section() -- GitLab From 11bfc01d1ffd9a7e3ce76c0675135179e6a9e245 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 11 Sep 2018 10:07:21 +1000 Subject: [PATCH 243/257] sphinxext: Use unicode_literals docutils consistently uses unicode values, so for better python2 compatability use unicode_literals. Remove the now unnecessary unicode cast for the debug logging. Going through every line is also a good case for lazy binding of the arguments -- this way the string is only built when LOG.debug() is actually active. Change-Id: I50556a7efddaa168f4e5d96400d7388cecbb2033 --- reno/sphinxext.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index e805e7c..5b99993 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -9,6 +9,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +from __future__ import unicode_literals import os.path @@ -111,7 +112,7 @@ class ReleaseNotesDirective(rst.Directive): source_name = '<%s %s>' % (__name__, branch or 'current branch') result = statemachine.ViewList() for line_num, line in enumerate(text.splitlines(), 1): - LOG.debug(u'{:>4d}: {}'.format(line_num, line)) + LOG.debug('%4d: %s', line_num, line) result.append(line, source_name, line_num) node = nodes.section() -- GitLab From f95ed0e61e10f04f88cf5d48fef1dbcd2eba1dcc Mon Sep 17 00:00:00 2001 From: melissaml Date: Sun, 23 Sep 2018 17:43:32 +0800 Subject: [PATCH 244/257] update the oudated URL in doc Change-Id: Icc49ddbaafd17f9accee5161f295ced028e8438c --- doc/source/user/setuptools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/user/setuptools.rst b/doc/source/user/setuptools.rst index 631f27c..693b88a 100644 --- a/doc/source/user/setuptools.rst +++ b/doc/source/user/setuptools.rst @@ -5,7 +5,7 @@ *reno* supports integration with `setuptools`_ and *setuptools* derivatives like *pbr* through a custom command - ``build_reno``. -.. _pbr: http://docs.openstack.org/developer/pbr/ +.. _pbr: https://docs.openstack.org/pbr/latest/ .. _setuptools: https://setuptools.readthedocs.io/en/latest/ Using setuptools integration -- GitLab From e12ecf7c77babd935cb67f3db50e64c5df5611ab Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 2 Oct 2018 16:42:38 -0400 Subject: [PATCH 245/257] build universal wheels Change-Id: I649c9cecaf479b9e3e27cfc59f59c9228a9c92e3 Signed-off-by: Doug Hellmann --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 0aae22e..a5bfc41 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,3 +41,6 @@ domain = reno domain = reno output_dir = reno/locale input_file = reno/locale/reno.pot + +[wheel] +universal = 1 -- GitLab From caadd1d11c307be5fad520a992afb6e013bf94e9 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 10 Oct 2018 12:46:12 -0400 Subject: [PATCH 246/257] update test fixtures to capture log output Change-Id: I0a99a77bdcf02cae8d0ebaf0d3a25f3204a4161e Signed-off-by: Doug Hellmann --- reno/tests/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reno/tests/base.py b/reno/tests/base.py index c1f9829..cc95054 100644 --- a/reno/tests/base.py +++ b/reno/tests/base.py @@ -31,3 +31,4 @@ class TestCase(testtools.TestCase): self._stderr_fixture = fixtures.StringStream('stderr') self.stderr = self.useFixture(self._stderr_fixture).stream self.useFixture(fixtures.MonkeyPatch('sys.stderr', self.stderr)) + self.useFixture(fixtures.FakeLogger()) -- GitLab From 1fa03dc4951fe02c872e4ff6fd3ca5fb6720a177 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 10 Oct 2018 12:46:55 -0400 Subject: [PATCH 247/257] refactor handling of missing config files for better testing Rather than testing that we log a message, place the handling in its own method and verify that we call that. This allows us to change the logging in the configuration class without counting messages and updating the test. Change-Id: Ic3067d8ab6699ceb82db7dd64f892a757f5cc12f Signed-off-by: Doug Hellmann --- reno/config.py | 14 ++++++++++++-- reno/tests/test_config.py | 5 +++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/reno/config.py b/reno/config.py index 03ae26b..3aa6583 100644 --- a/reno/config.py +++ b/reno/config.py @@ -218,7 +218,7 @@ class Config(object): if os.path.isfile(filename): break else: - LOG.info('no configuration file in: %s', ', '.join(filenames)) + self._report_missing_config_files(filenames) return try: @@ -226,10 +226,20 @@ class Config(object): self._contents = yaml.safe_load(fd) LOG.info('loaded configuration file %s', filename) except IOError as err: - LOG.warning('did not load config file %s: %s', filename, err) + self._report_failure_config_file(filename, err) else: self.override(**self._contents) + def _report_missing_config_files(self, filenames): + # NOTE(dhellmann): This is extracted so we can mock it for + # testing. + LOG.info('no configuration file in: %s', ', '.join(filenames)) + + def _report_failure_config_file(self, filename, err): + # NOTE(dhellmann): This is extracted so we can mock it for + # testing. + LOG.warning('did not load config file %s: %s', filename, err) + def _rename_prelude_section(self, **kwargs): key = 'prelude_section_name' if key in kwargs and kwargs[key] != self._OPTS[key].default: diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py index 39cf94e..6251aa7 100644 --- a/reno/tests/test_config.py +++ b/reno/tests/test_config.py @@ -73,9 +73,10 @@ collapse_pre_releases: false self.assertEqual(expected, actual) def test_load_file_not_present(self): - with mock.patch.object(config.LOG, 'info') as logger: + missing = 'reno.config.Config._report_missing_config_files' + with mock.patch(missing) as error_handler: config.Config(self.tempdir.path) - self.assertEqual(1, logger.call_count) + self.assertEqual(1, error_handler.call_count) def _test_load_file(self, config_path): with open(config_path, 'w') as fd: -- GitLab From 9f866b35a16e01cea06deb73006fa2e55f30ea51 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 10 Oct 2018 12:48:37 -0400 Subject: [PATCH 248/257] only override config values from command line if they are actually set If we set defaults for the query configuration options and then pass the namespace created by parsing the command line options to the config object it cannot tell the difference between values set on the command line and defaults. That means we may override settings in the configuration file with defaults from the command line parser. This change sets all of the defaults for command line options to None as a sentinel value and then updates the Config class to ignore command line option values of None. It also adds tests to verify that we get the expected override behavior with boolean and string options. Change-Id: I1c9ce668b5e5c372d1c861bcae6e6de05a8ebc0c Signed-off-by: Doug Hellmann --- ...-cli-option-handling-a13652d14507f2d7.yaml | 7 +++++ reno/config.py | 6 +++-- reno/main.py | 6 ++--- reno/tests/test_config.py | 26 ++++++++++++++++++- 4 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/fix-cli-option-handling-a13652d14507f2d7.yaml diff --git a/releasenotes/notes/fix-cli-option-handling-a13652d14507f2d7.yaml b/releasenotes/notes/fix-cli-option-handling-a13652d14507f2d7.yaml new file mode 100644 index 0000000..8c84efe --- /dev/null +++ b/releasenotes/notes/fix-cli-option-handling-a13652d14507f2d7.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fix an issue with the way command line options and configuration + settings interact so that the settings in the configuration file + are used properly when command line arguments for those options + are not provided. diff --git a/reno/config.py b/reno/config.py index 3aa6583..5990e25 100644 --- a/reno/config.py +++ b/reno/config.py @@ -275,9 +275,11 @@ class Config(object): arg_values = { o.name: getattr(parsed_args, o.name) for o in _OPTIONS - if hasattr(parsed_args, o.name) + if getattr(parsed_args, o.name, None) is not None } - self.override(**arg_values) + if arg_values: + LOG.info('[config] updating from command line options') + self.override(**arg_values) @property def reporoot(self): diff --git a/reno/main.py b/reno/main.py index b9837d7..cb27712 100644 --- a/reno/main.py +++ b/reno/main.py @@ -32,7 +32,7 @@ _query_args = [ help='the branch to scan, defaults to the current')), (('--collapse-pre-releases',), dict(action='store_true', - default=config.Config.get_default('collapse_pre_releases'), + default=None, help='combine pre-releases with their final release')), (('--no-collapse-pre-releases',), dict(action='store_false', @@ -42,12 +42,12 @@ _query_args = [ dict(default=None, help='stop when this version is reached in the history')), (('--ignore-cache',), - dict(default=False, + dict(default=None, action='store_true', help='if there is a cache file present, do not use it')), (('--stop-at-branch-base',), dict(action='store_true', - default=True, + default=None, dest='stop_at_branch_base', help='stop scanning when the branch meets master')), (('--no-stop-at-branch-base',), diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py index 6251aa7..38e9635 100644 --- a/reno/tests/test_config.py +++ b/reno/tests/test_config.py @@ -126,7 +126,7 @@ collapse_pre_releases: false } self.assertEqual(expected, actual) - def test_override_from_parsed_args(self): + def test_override_from_parsed_args_boolean_false(self): c = self._run_override_from_parsed_args([ '--no-collapse-pre-releases', ]) @@ -138,6 +138,30 @@ collapse_pre_releases: false expected['collapse_pre_releases'] = False self.assertEqual(expected, actual) + def test_override_from_parsed_args_boolean_true(self): + c = self._run_override_from_parsed_args([ + '--collapse-pre-releases', + ]) + actual = c.options + expected = { + o.name: o.default + for o in config._OPTIONS + } + expected['collapse_pre_releases'] = True + self.assertEqual(expected, actual) + + def test_override_from_parsed_args_string(self): + c = self._run_override_from_parsed_args([ + '--earliest-version', '1.2.3', + ]) + actual = c.options + expected = { + o.name: o.default + for o in config._OPTIONS + } + expected['earliest_version'] = '1.2.3' + self.assertEqual(expected, actual) + def test_override_from_parsed_args_ignore_non_options(self): parser = argparse.ArgumentParser() main._build_query_arg_group(parser) -- GitLab From df69d2fd2002c931cb7b862a64f5cb632c517d96 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Tue, 26 Mar 2019 21:16:23 +0100 Subject: [PATCH 249/257] Now packaging 2.11.2. --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 442fe95..a8532f1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +python-reno (2.11.2-1) experimental; urgency=medium + + * New upstream release. + + -- Thomas Goirand Tue, 26 Mar 2019 21:16:04 +0100 + python-reno (2.9.2-1) unstable; urgency=medium * Team upload. -- GitLab From 09c91f4ab467d8ccb3081e6ab771293741c1620e Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Tue, 26 Mar 2019 21:19:49 +0100 Subject: [PATCH 250/257] Removed Python 2 support. --- debian/changelog | 1 + debian/control | 39 +++++------------------------------- debian/python-reno.postinst | 11 ---------- debian/python-reno.postrm | 11 ---------- debian/python-reno.prerm | 11 ---------- debian/python3-reno.postinst | 11 ---------- debian/rules | 35 ++++++++++---------------------- 7 files changed, 17 insertions(+), 102 deletions(-) delete mode 100644 debian/python-reno.postinst delete mode 100644 debian/python-reno.postrm delete mode 100644 debian/python-reno.prerm delete mode 100644 debian/python3-reno.postinst diff --git a/debian/changelog b/debian/changelog index a8532f1..6e6b25a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ python-reno (2.11.2-1) experimental; urgency=medium * New upstream release. + * Removed Python 2 support. -- Thomas Goirand Tue, 26 Mar 2019 21:16:04 +0100 diff --git a/debian/control b/debian/control index 27cf99c..1a085b3 100644 --- a/debian/control +++ b/debian/control @@ -6,15 +6,10 @@ Uploaders: Ivan Udovichenko , Thomas Goirand , Build-Depends: - debhelper (>= 10~), + debhelper (>= 10), dh-python, gnupg, openstack-pkg-tools, - python-all, - python-dev, - python-pbr, - python-setuptools, - python-sphinx, python3-all, python3-dev, python3-pbr, @@ -22,46 +17,22 @@ Build-Depends: python3-sphinx, Build-Depends-Indep: git, - python-babel, - python-coverage, - python-dulwich, - python-hacking, - python-mock, - python-openstackdocstheme (>= 1.11.0), - python-testscenarios, - python-testtools, - python-yaml, python3-babel, python3-coverage, python3-dulwich, + python3-hacking, python3-mock, + python3-openstackdocstheme, python3-testscenarios, python3-testtools, python3-yaml, subunit, testrepository, -Standards-Version: 4.1.1 +Standards-Version: 4.3.0 Vcs-Browser: https://salsa.debian.org/openstack-team/libs/python-reno Vcs-Git: https://salsa.debian.org/openstack-team/libs/python-reno.git Homepage: http://www.openstack.org/ -Package: python-reno -Architecture: all -Depends: - git, - python-dulwich, - python-pbr (>= 1.4), - python-yaml, - ${misc:Depends}, - ${python:Depends}, -Suggests: - python-reno-doc, -Description: RElease NOtes manager - Python 2.x - Reno is a release notes manager for storing release notes in a git - repository and then building documentation from them. - . - This package contains the Python 2.x module. - Package: python-reno-doc Section: doc Architecture: all @@ -79,7 +50,7 @@ Architecture: all Depends: git, python3-dulwich, - python3-pbr (>= 1.4), + python3-pbr, python3-yaml, ${misc:Depends}, ${python3:Depends}, diff --git a/debian/python-reno.postinst b/debian/python-reno.postinst deleted file mode 100644 index 341c60e..0000000 --- a/debian/python-reno.postinst +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -set -e - -if [ "$1" = "configure" ] ; then - update-alternatives --install /usr/bin/reno reno /usr/bin/python2-reno 300 -fi - -#DEBHELPER# - -exit 0 diff --git a/debian/python-reno.postrm b/debian/python-reno.postrm deleted file mode 100644 index 3bb38a0..0000000 --- a/debian/python-reno.postrm +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -set -e - -if [ "$1" = "remove" ] || [ "$1" = "disappear" ] ; then - update-alternatives --remove reno /usr/bin/python2-reno -fi - -#DEBHELPER# - -exit 0 diff --git a/debian/python-reno.prerm b/debian/python-reno.prerm deleted file mode 100644 index 1e512a6..0000000 --- a/debian/python-reno.prerm +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -set -e - -if [ "$1" = "remove" ] ; then - update-alternatives --remove reno /usr/bin/python2-reno -fi - -#DEBHELPER# - -exit 0 diff --git a/debian/python3-reno.postinst b/debian/python3-reno.postinst deleted file mode 100644 index d2689da..0000000 --- a/debian/python3-reno.postinst +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -set -e - -if [ "$1" = "configure" ] ; then - update-alternatives --install /usr/bin/reno reno /usr/bin/python3-reno 200 -fi - -#DEBHELPER# - -exit 0 diff --git a/debian/rules b/debian/rules index fd937d1..8f80441 100755 --- a/debian/rules +++ b/debian/rules @@ -4,17 +4,20 @@ UPSTREAM_GIT := https://github.com/openstack/reno.git include /usr/share/openstack-pkg-tools/pkgos.make %: - dh $@ --buildsystem=python_distutils --with python2,python3,sphinxdoc + dh $@ --buildsystem=python_distutils --with python3,sphinxdoc -override_dh_auto_install: - pkgos-dh_auto_install +override_dh_auto_clean: + rm -rf build -override_dh_python3: - dh_python3 --shebang=/usr/bin/python3 +override_dh_auto_build: + echo "Do nothing..." + +override_dh_auto_install: + pkgos-dh_auto_install --no-py2 override_dh_auto_test: ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) - pkgos-dh_auto_test 'reno\.tests(?!.*test_cache\.TestCache\.test_build_cache_db.*)' + pkgos-dh_auto_test --no-py2 'reno\.tests(?!.*test_cache\.TestCache\.test_build_cache_db.*)' endif override_dh_sphinxdoc: @@ -24,21 +27,5 @@ override_dh_sphinxdoc: #sphinx-build -b html doc/source debian/python-reno-doc/usr/share/doc/python-reno-doc/html #dh_sphinxdoc -O--buildsystem=python_distutils - -override_dh_clean: - dh_clean -O--buildsystem=python_distutils - rm -rf build - - -# Commands not to run -override_dh_installcatalogs: -override_dh_installemacsen override_dh_installifupdown: -override_dh_installinfo override_dh_installmenu: -override_dh_installmime override_dh_installmodules: -override_dh_installlogcheck override_dh_installpam: -override_dh_installppp override_dh_installudev: -override_dh_installwm override_dh_installxfonts: -override_dh_gconf override_dh_icons override_dh_perl: -override_dh_usrlocal override_dh_installcron: -override_dh_installdebconf override_dh_installlogrotate: -override_dh_installgsettings: +override_dh_python3: + dh_python3 --shebang=/usr/bin/python3 -- GitLab From d437c1714d9ee269b4302d76992f1dd4507b91b6 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Wed, 17 Jul 2019 00:17:54 +0200 Subject: [PATCH 251/257] Uploading to unstable. --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 6e6b25a..3728a80 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +python-reno (2.11.2-2) unstable; urgency=medium + + * Uploading to unstable. + + -- Thomas Goirand Wed, 17 Jul 2019 00:17:41 +0200 + python-reno (2.11.2-1) experimental; urgency=medium * New upstream release. -- GitLab From d3f618928faac69287b128c8700b71557ea81f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nov=C3=BD?= Date: Thu, 18 Jul 2019 16:38:40 +0200 Subject: [PATCH 252/257] Use debhelper-compat instead of debian/compat --- debian/changelog | 6 ++++++ debian/compat | 1 - debian/control | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 debian/compat diff --git a/debian/changelog b/debian/changelog index 3728a80..0a1484a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +python-reno (2.11.2-3) UNRELEASED; urgency=medium + + * Use debhelper-compat instead of debian/compat. + + -- Ondřej Nový Thu, 18 Jul 2019 16:38:40 +0200 + python-reno (2.11.2-2) unstable; urgency=medium * Uploading to unstable. diff --git a/debian/compat b/debian/compat deleted file mode 100644 index f599e28..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -10 diff --git a/debian/control b/debian/control index 1a085b3..a4f7b3c 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,7 @@ Uploaders: Ivan Udovichenko , Thomas Goirand , Build-Depends: - debhelper (>= 10), + debhelper-compat (= 10), dh-python, gnupg, openstack-pkg-tools, -- GitLab From 2ae6d986b54af5b426d1b082992d5c087646711b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nov=C3=BD?= Date: Fri, 19 Jul 2019 15:54:38 +0200 Subject: [PATCH 253/257] Bump Standards-Version to 4.4.0 --- debian/changelog | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 0a1484a..fd3fd2e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ python-reno (2.11.2-3) UNRELEASED; urgency=medium * Use debhelper-compat instead of debian/compat. + * Bump Standards-Version to 4.4.0. -- Ondřej Nový Thu, 18 Jul 2019 16:38:40 +0200 diff --git a/debian/control b/debian/control index a4f7b3c..419b054 100644 --- a/debian/control +++ b/debian/control @@ -28,7 +28,7 @@ Build-Depends-Indep: python3-yaml, subunit, testrepository, -Standards-Version: 4.3.0 +Standards-Version: 4.4.0 Vcs-Browser: https://salsa.debian.org/openstack-team/libs/python-reno Vcs-Git: https://salsa.debian.org/openstack-team/libs/python-reno.git Homepage: http://www.openstack.org/ -- GitLab From 892e524c96c9953615fd3cef4932e9f66580236e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nov=C3=BD?= Date: Fri, 18 Oct 2019 16:28:34 +0200 Subject: [PATCH 254/257] Bump Standards-Version to 4.4.1 --- debian/changelog | 2 +- debian/control | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index fd3fd2e..694dd2b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,7 @@ python-reno (2.11.2-3) UNRELEASED; urgency=medium * Use debhelper-compat instead of debian/compat. - * Bump Standards-Version to 4.4.0. + * Bump Standards-Version to 4.4.1. -- Ondřej Nový Thu, 18 Jul 2019 16:38:40 +0200 diff --git a/debian/control b/debian/control index 419b054..bae6e9c 100644 --- a/debian/control +++ b/debian/control @@ -28,7 +28,7 @@ Build-Depends-Indep: python3-yaml, subunit, testrepository, -Standards-Version: 4.4.0 +Standards-Version: 4.4.1 Vcs-Browser: https://salsa.debian.org/openstack-team/libs/python-reno Vcs-Git: https://salsa.debian.org/openstack-team/libs/python-reno.git Homepage: http://www.openstack.org/ -- GitLab From 7b9e6ba8d99052b5aba0658d39a86b7dcb099205 Mon Sep 17 00:00:00 2001 From: Debian Janitor Date: Sat, 11 Sep 2021 09:39:09 +0000 Subject: [PATCH 255/257] Bump debhelper from old 10 to 13. + Replace python_distutils buildsystem with pybuild. Changes-By: lintian-brush Fixes: lintian: package-uses-old-debhelper-compat-version See-also: https://lintian.debian.org/tags/package-uses-old-debhelper-compat-version.html --- debian/changelog | 5 +++++ debian/control | 2 +- debian/rules | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 694dd2b..12a1535 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,13 @@ python-reno (2.11.2-3) UNRELEASED; urgency=medium + [ Ondřej Nový ] * Use debhelper-compat instead of debian/compat. * Bump Standards-Version to 4.4.1. + [ Debian Janitor ] + * Bump debhelper from old 10 to 13. + + Replace python_distutils buildsystem with pybuild. + -- Ondřej Nový Thu, 18 Jul 2019 16:38:40 +0200 python-reno (2.11.2-2) unstable; urgency=medium diff --git a/debian/control b/debian/control index bae6e9c..08f395f 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,7 @@ Uploaders: Ivan Udovichenko , Thomas Goirand , Build-Depends: - debhelper-compat (= 10), + debhelper-compat (= 13), dh-python, gnupg, openstack-pkg-tools, diff --git a/debian/rules b/debian/rules index 8f80441..c8e5166 100755 --- a/debian/rules +++ b/debian/rules @@ -4,7 +4,7 @@ UPSTREAM_GIT := https://github.com/openstack/reno.git include /usr/share/openstack-pkg-tools/pkgos.make %: - dh $@ --buildsystem=python_distutils --with python3,sphinxdoc + dh $@ --buildsystem=pybuild --with python3,sphinxdoc override_dh_auto_clean: rm -rf build @@ -25,7 +25,7 @@ override_dh_sphinxdoc: mkdir -p debian/python-reno-doc/usr/share/doc/python-reno-doc/html cp -auxf doc/source/*.rst debian/python-reno-doc/usr/share/doc/python-reno-doc/html #sphinx-build -b html doc/source debian/python-reno-doc/usr/share/doc/python-reno-doc/html - #dh_sphinxdoc -O--buildsystem=python_distutils + #dh_sphinxdoc -O--buildsystem=pybuild override_dh_python3: dh_python3 --shebang=/usr/bin/python3 -- GitLab From 022f68be6cc8d72f53bcd08ed0309c61a5e5955d Mon Sep 17 00:00:00 2001 From: Debian Janitor Date: Sat, 11 Sep 2021 09:39:13 +0000 Subject: [PATCH 256/257] Set upstream metadata fields: Repository, Repository-Browse. Changes-By: lintian-brush Fixes: lintian: upstream-metadata-file-is-missing See-also: https://lintian.debian.org/tags/upstream-metadata-file-is-missing.html Fixes: lintian: upstream-metadata-missing-repository See-also: https://lintian.debian.org/tags/upstream-metadata-missing-repository.html --- debian/changelog | 1 + debian/upstream/metadata | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 debian/upstream/metadata diff --git a/debian/changelog b/debian/changelog index 12a1535..aa01794 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,7 @@ python-reno (2.11.2-3) UNRELEASED; urgency=medium [ Debian Janitor ] * Bump debhelper from old 10 to 13. + Replace python_distutils buildsystem with pybuild. + * Set upstream metadata fields: Repository, Repository-Browse. -- Ondřej Nový Thu, 18 Jul 2019 16:38:40 +0200 diff --git a/debian/upstream/metadata b/debian/upstream/metadata new file mode 100644 index 0000000..3b646c7 --- /dev/null +++ b/debian/upstream/metadata @@ -0,0 +1,3 @@ +--- +Repository: https://github.com/openstack/reno.git +Repository-Browse: https://github.com/openstack/reno -- GitLab From 5928842f7ac516dd2db7e686cfa65bbab4a87130 Mon Sep 17 00:00:00 2001 From: Debian Janitor Date: Sat, 11 Sep 2021 09:39:14 +0000 Subject: [PATCH 257/257] Update standards version to 4.5.1, no changes needed. Changes-By: lintian-brush Fixes: lintian: out-of-date-standards-version See-also: https://lintian.debian.org/tags/out-of-date-standards-version.html --- debian/changelog | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index aa01794..86071a4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ python-reno (2.11.2-3) UNRELEASED; urgency=medium * Bump debhelper from old 10 to 13. + Replace python_distutils buildsystem with pybuild. * Set upstream metadata fields: Repository, Repository-Browse. + * Update standards version to 4.5.1, no changes needed. -- Ondřej Nový Thu, 18 Jul 2019 16:38:40 +0200 diff --git a/debian/control b/debian/control index 08f395f..6e3f1d2 100644 --- a/debian/control +++ b/debian/control @@ -28,7 +28,7 @@ Build-Depends-Indep: python3-yaml, subunit, testrepository, -Standards-Version: 4.4.1 +Standards-Version: 4.5.1 Vcs-Browser: https://salsa.debian.org/openstack-team/libs/python-reno Vcs-Git: https://salsa.debian.org/openstack-team/libs/python-reno.git Homepage: http://www.openstack.org/ -- GitLab