diff --git a/.gitignore b/.gitignore index e6257802a9a502ee40f90c788eb6abe4428f311a..3b982a942abb32b18889427e36de12db0e84a5e2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ lib64 pip-log.txt # Unit test / coverage reports -.coverage +.coverage* .tox nosetests.xml .testrepository @@ -48,7 +48,12 @@ doc/build AUTHORS ChangeLog +# reno generates these +RELEASENOTES.rst + # Editors *~ .*.swp .*sw? +/cover/ +/releasenotes/notes/reno.cache diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..547a6e7d0d04fdb34e45ec091600f7fc9c7bf251 --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,6 @@ +- project: + templates: + - openstack-python-jobs + - openstack-python35-jobs + - publish-openstack-docs-pti + - openstack-lower-constraints-jobs diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index aafbd53b90b6a5103918e6d72cacc96812a4757f..5dac14ed50b3cf1338bac42d63953d5163c267fd 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,17 +1,14 @@ -If you would like to contribute to the development of OpenStack, you must -follow the steps in this page: - - http://docs.openstack.org/infra/manual/developers.html +If you would like to contribute to the development of OpenStack, start by +following the steps in this page: +https://docs.openstack.org/infra/manual/developers.html If you already have a good understanding of how the system works and your OpenStack accounts are set up, you can skip to the development workflow section of this documentation to learn how changes to OpenStack should be submitted for review via the Gerrit tool: - - http://docs.openstack.org/infra/manual/developers.html#development-workflow +https://docs.openstack.org/infra/manual/developers.html#development-workflow Pull requests submitted through GitHub will be ignored. -Bugs should be filed on Launchpad, not GitHub: - - https://bugs.launchpad.net/reno +Bugs should be filed on Storyboard, not GitHub: +https://storyboard.openstack.org/#!/project/933 diff --git a/HACKING.rst b/HACKING.rst index f6627a0c62d054725545a45a70c79065648bd060..ec2bd24d2664f69c856a52e6ee8e41369d8f791c 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -1,4 +1,4 @@ reno Style Commandments =============================================== -Read the OpenStack Style Commandments http://docs.openstack.org/developer/hacking/ +Read the OpenStack Style Commandments https://docs.openstack.org/hacking/latest/ diff --git a/README.rst b/README.rst index ba6bc88e414f6f49736615313a5c1f913e9fe74e..314faec0ac54fe5d34a6969743b8e089cf2cc4d4 100644 --- a/README.rst +++ b/README.rst @@ -1,19 +1,58 @@ -=============================== - reno -- Release Notes Manager -=============================== +========================================= + reno: A New Way to Manage Release Notes +========================================= -Reno is a release notes manager for storing release notes in a git -repository and then building documentation from them. +Reno is a release notes manager designed with high throughput in mind, +supporting fast distributed development teams without introducing +additional development processes. Our goal is to encourage detailed +and accurate release notes for every release. + +Reno uses git to store its data, along side the code being +described. This means release notes can be written when the code +changes are fresh, so no details are forgotten. It also means that +release notes can go through the same review process used for managing +code and other documentation changes. + +Reno stores each release note in a separate file to enable a large +number of developers to work on multiple patches simultaneously, all +targeting the same branch, without worrying about merge +conflicts. This cuts down on the need to rebase or otherwise manually +resolve conflicts, and keeps a development team moving quickly. + +Reno also supports multiple branches, allowing release notes to be +back-ported from master to maintenance branches together with the +code for bug fixes. + +Reno organizes notes into logical groups based on whether they +describe new features, bug fixes, known issues, or other topics of +interest to the user. Contributors categorize individual notes as they +are added, and reno combines them before publishing. + +Notes can be styled using reStructuredText directives, and reno's +Sphinx integration makes it easy to incorporate release notes into +automated documentation builds. + +Notes are automatically associated with the release version based on +the git tags applied to the repository, so it is not necessary to +track changes manually using a bug tracker or other tool, or to worry +that an important change will be missed when the release notes are +written by hand all at one time, just before a release. + +Modifications to notes are incorporated when the notes are shown in +their original location in the history. This feature makes it possible +to correct typos or otherwise fix a published release note after a +release is made, but have the new note content associated with the +original version number. Notes also can be deleted, eliminating them +from future documentation builds. Project Meta-data ================= -* Free software: Apache license -* Documentation: http://docs.openstack.org/developer/reno -* Source: http://git.openstack.org/cgit/openstack/reno -* Bugs: http://bugs.launchpad.net/reno +.. .. image:: https://governance.openstack.org/tc/badges/reno.svg + :target: https://governance.openstack.org/tc/reference/tags/index.html -Features -======== - -* TODO +* Free software: Apache license +* Documentation: https://docs.openstack.org/reno/latest/ +* Source: https://git.openstack.org/cgit/openstack/reno +* Bugs: https://storyboard.openstack.org/#!/project/933 +* IRC: #openstack-release on freenode diff --git a/babel.cfg b/babel.cfg deleted file mode 100644 index 15cd6cb76b93453343e70650a70db58cff98195b..0000000000000000000000000000000000000000 --- a/babel.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[python: **.py] - diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000000000000000000000000000000000000..359b0ae12c886f4965617b382730bb32f3b10c29 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,5 @@ +gcc [platform:rpm test] +python-dev [platform:dpkg test] +python-devel [platform:rpm test] +python3-devel [platform:fedora platform:suse] +python3 [platform:suse] diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000000000000000000000000000000000000..86071a49ecada0a4abf78fb595feca73a3968e55 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,101 @@ +python-reno (2.11.2-3) UNRELEASED; urgency=medium + + [ Ondřej Nový ] + * Use debhelper-compat instead of debian/compat. + * Bump Standards-Version to 4.4.1. + + [ Debian Janitor ] + * Bump debhelper from old 10 to 13. + + Replace python_distutils buildsystem with pybuild. + * Set upstream metadata fields: Repository, Repository-Browse. + * Update standards version to 4.5.1, no changes needed. + + -- Ondřej Nový Thu, 18 Jul 2019 16:38:40 +0200 + +python-reno (2.11.2-2) unstable; urgency=medium + + * Uploading to unstable. + + -- Thomas Goirand Wed, 17 Jul 2019 00:17:41 +0200 + +python-reno (2.11.2-1) experimental; urgency=medium + + * New upstream release. + * Removed Python 2 support. + + -- Thomas Goirand Tue, 26 Mar 2019 21:16:04 +0100 + +python-reno (2.9.2-1) unstable; urgency=medium + + * Team upload. + + [ Ondřej Nový ] + * d/control: Set Vcs-* to salsa.debian.org + * d/control: Add trailing tilde to min version depend to allow + backports + * d/control: Use team+openstack@tracker.debian.org as maintainer + + [ David Rabel ] + * New upstream version 2.9.2 + + -- David Rabel Mon, 03 Sep 2018 17:19:07 +0200 + +python-reno (2.5.0-1) unstable; urgency=medium + + [ Ondřej Nový ] + * Bumped debhelper compat version to 10 + + [ Thomas Goirand ] + * Fixed VCS URLS. + * debian/copyright format using https. + * Running wrap-and-sort -bast. + * Updating maintainer field. + * Standards-Version is now 4.1.1. + * Fixed python3 shebang to use python3, not python3.x. + * New upstream release. + * Fixed (build-)depends for this release. + * Using pkgos-dh_auto_test and blacklisting test_build_cache_db(). + + -- Thomas Goirand Sun, 05 Nov 2017 21:21:02 +0000 + +python-reno (1.3.0-6) unstable; urgency=medium + + * Realy added gnupg as build-depends (Closes: #834685). + * Using pkgos-dh_auto_install from openstack-pkg-tools >= 52~. + + -- Thomas Goirand Thu, 13 Oct 2016 13:59:52 +0200 + +python-reno (1.3.0-5) unstable; urgency=medium + + [ Ondřej Nový ] + * d/rules: Changed UPSTREAM_GIT protocol to https + * d/s/options: extend-diff-ignore of .gitreview + * d/control: Using OpenStack's Gerrit as VCS URLs. + + [ Thomas Goirand ] + * Added gnupg as build-depends (Closes: #834685). + + -- Thomas Goirand Tue, 04 Oct 2016 14:30:48 +0200 + +python-reno (1.3.0-3) unstable; urgency=medium + + * Added git as runtime and build depends (Closes: #824593). + + -- Thomas Goirand Wed, 18 May 2016 10:23:32 +0200 + +python-reno (1.3.0-2) unstable; urgency=medium + + [ Thomas Goirand ] + * Use patch to use less entropy in unit tests. + * Reactivated unit tests. + + [ Ondřej Nový ] + * Fixed VCS URLs (https). + + -- Thomas Goirand Mon, 29 Feb 2016 17:34:49 +0000 + +python-reno (1.3.0-1) unstable; urgency=medium + + * Initial release. (Closes: #811150) + + -- Thomas Goirand Sun, 17 Jan 2016 06:17:15 +0000 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000000000000000000000000000000000000..6e3f1d2d66b654cccfc75977f2f28ee61fa3e067 --- /dev/null +++ b/debian/control @@ -0,0 +1,63 @@ +Source: python-reno +Section: python +Priority: optional +Maintainer: Debian OpenStack +Uploaders: + Ivan Udovichenko , + Thomas Goirand , +Build-Depends: + debhelper-compat (= 13), + dh-python, + gnupg, + openstack-pkg-tools, + python3-all, + python3-dev, + python3-pbr, + python3-setuptools, + python3-sphinx, +Build-Depends-Indep: + git, + python3-babel, + python3-coverage, + python3-dulwich, + python3-hacking, + python3-mock, + python3-openstackdocstheme, + python3-testscenarios, + python3-testtools, + python3-yaml, + subunit, + testrepository, +Standards-Version: 4.5.1 +Vcs-Browser: https://salsa.debian.org/openstack-team/libs/python-reno +Vcs-Git: https://salsa.debian.org/openstack-team/libs/python-reno.git +Homepage: http://www.openstack.org/ + +Package: python-reno-doc +Section: doc +Architecture: all +Depends: + ${misc:Depends}, + ${sphinxdoc:Depends}, +Description: RElease NOtes manager - doc + Reno is a release notes manager for storing release notes in a git + repository and then building documentation from them. + . + This package contains the documentation. + +Package: python3-reno +Architecture: all +Depends: + git, + python3-dulwich, + python3-pbr, + python3-yaml, + ${misc:Depends}, + ${python3:Depends}, +Suggests: + python-reno-doc, +Description: RElease NOtes manager - Python 3.x + Reno is a release notes manager for storing release notes in a git + repository and then building documentation from them. + . + This package contains the Python 3.x module. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000000000000000000000000000000000000..a13c231722afd0104a551ebd3e3d3fab0e4b94ad --- /dev/null +++ b/debian/copyright @@ -0,0 +1,30 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: reno +Source: http://www.openstack.org/ + +Files: * +Copyright: (c) 2010-2018, OpenStack Foundation + (c) 2013, Hewlett-Packard Development Company, L.P. + (c) 2018, Red Hat, Inc. +License: Apache-2 + +Files: debian/* +Copyright: (c) 2015, Ivan Udovichenko + (c) 2016, Thomas Goirand +License: Apache-2 + +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian-based systems the full text of the Apache version 2.0 license + can be found in /usr/share/common-licenses/Apache-2.0. diff --git a/debian/python3-reno.postrm b/debian/python3-reno.postrm new file mode 100644 index 0000000000000000000000000000000000000000..7a3f7d6dcb220373ac5c2a610084ee2d10d9028b --- /dev/null +++ b/debian/python3-reno.postrm @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +if [ "$1" = "remove" ] || [ "$1" = "disappear" ] ; then + update-alternatives --remove reno /usr/bin/python3-reno +fi + +#DEBHELPER# + +exit 0 diff --git a/debian/python3-reno.prerm b/debian/python3-reno.prerm new file mode 100644 index 0000000000000000000000000000000000000000..b05f77830faeb377e080941d8d36833e7f9d3974 --- /dev/null +++ b/debian/python3-reno.prerm @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +if [ "$1" = "remove" ] ; then + update-alternatives --remove reno /usr/bin/python3-reno +fi + +#DEBHELPER# + +exit 0 diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000000000000000000000000000000000000..c8e51662344b1a73446556aca4cb19776149c2ed --- /dev/null +++ b/debian/rules @@ -0,0 +1,31 @@ +#!/usr/bin/make -f + +UPSTREAM_GIT := https://github.com/openstack/reno.git +include /usr/share/openstack-pkg-tools/pkgos.make + +%: + dh $@ --buildsystem=pybuild --with python3,sphinxdoc + +override_dh_auto_clean: + rm -rf build + +override_dh_auto_build: + echo "Do nothing..." + +override_dh_auto_install: + pkgos-dh_auto_install --no-py2 + +override_dh_auto_test: +ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) + pkgos-dh_auto_test --no-py2 'reno\.tests(?!.*test_cache\.TestCache\.test_build_cache_db.*)' +endif + +override_dh_sphinxdoc: + # Disabling sphinx doc generation as it fails if no git files are present + mkdir -p debian/python-reno-doc/usr/share/doc/python-reno-doc/html + cp -auxf doc/source/*.rst debian/python-reno-doc/usr/share/doc/python-reno-doc/html + #sphinx-build -b html doc/source debian/python-reno-doc/usr/share/doc/python-reno-doc/html + #dh_sphinxdoc -O--buildsystem=pybuild + +override_dh_python3: + dh_python3 --shebang=/usr/bin/python3 diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000000000000000000000000000000000000..163aaf8d82b6c54f23c45f32895dbdfdcc27b047 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/source/options b/debian/source/options new file mode 100644 index 0000000000000000000000000000000000000000..912224515e06f63403e6e177a39145c46c89344b --- /dev/null +++ b/debian/source/options @@ -0,0 +1,2 @@ +extend-diff-ignore = "^[^/]*[.]egg-info/" +extend-diff-ignore = "^[.]gitreview$" diff --git a/debian/upstream/metadata b/debian/upstream/metadata new file mode 100644 index 0000000000000000000000000000000000000000..3b646c730f64e78acd9965596211c41570cfd9ce --- /dev/null +++ b/debian/upstream/metadata @@ -0,0 +1,3 @@ +--- +Repository: https://github.com/openstack/reno.git +Repository-Browse: https://github.com/openstack/reno diff --git a/debian/watch b/debian/watch new file mode 100644 index 0000000000000000000000000000000000000000..59efb5eec5011ab2cc499419a8d13e7cc7aa918e --- /dev/null +++ b/debian/watch @@ -0,0 +1,4 @@ +version=3 +opts="uversionmangle=s/\.(b|rc)/~$1/" \ +https://github.com/openstack/reno/tags .*/(\d[\d\.]+)\.tar\.gz + diff --git a/doc/source/conf.py b/doc/source/conf.py old mode 100755 new mode 100644 index c0ce8ab6e9cca76707fe71bd9da34c4ff1375804..12150e91e695db3224a5488f0ed2ff36dbb75bd4 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -15,18 +15,37 @@ import os import sys -sys.path.insert(0, os.path.abspath('../..')) +# oslosphinx uses reno and reno uses oslosphinx. Make oslosphinx for +# reno optional to break the build cycle +try: + import openstackdocstheme +except: + has_theme = False +else: + has_theme = True + + # -- General configuration ---------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.autodoc', - #'sphinx.ext.intersphinx', - 'oslosphinx', + # 'sphinx.ext.intersphinx', 'reno.sphinxext', + 'reno._exts.show_reno_config', ] +if has_theme: + extensions.append('openstackdocstheme') + html_theme = 'openstackdocs' + +# openstackdocstheme options +repository_name = 'openstack/reno' +bug_project = '933' +bug_tag = 'docs' +html_last_updated_fmt = '%Y-%m-%d %H:%M' + # autodoc generation is a bit aggressive and a nuisance when doing heavy # text edit cycles. # execute "export SPHINX_DEBUG=1" in your terminal to disable @@ -51,6 +70,9 @@ add_module_names = True # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' +# Do not warn about non-local image URI +suppress_warnings = ['image.nonlocal_uri'] + # -- Options for HTML output -------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with @@ -73,4 +95,4 @@ latex_documents = [ ] # Example configuration for intersphinx: refer to the Python standard library. -#intersphinx_mapping = {'http://docs.python.org/': None} +# intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst deleted file mode 100644 index 1728a61ca25fd9c25280aa71414fa86b48c73444..0000000000000000000000000000000000000000 --- a/doc/source/contributing.rst +++ /dev/null @@ -1,4 +0,0 @@ -============ -Contributing -============ -.. include:: ../../CONTRIBUTING.rst diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..3d4ceb40a59b72ac0a7aa73f3a0e0d2cd0c06998 --- /dev/null +++ b/doc/source/contributor/index.rst @@ -0,0 +1,5 @@ +============ +Contributing +============ + +.. include:: ../../../CONTRIBUTING.rst diff --git a/doc/source/design.rst b/doc/source/design.rst deleted file mode 100644 index 4cd818ccefe8d2d184dbcc62711a8d64134eb055..0000000000000000000000000000000000000000 --- a/doc/source/design.rst +++ /dev/null @@ -1,52 +0,0 @@ -===================================== - Design Constraints and Requirements -===================================== - -Managing release notes for a complex project over a long period of -time with many releases can be time consuming and error prone. Reno -helps automate the hard parts by devising a way to store the notes -inside the git repository where they can be tagged as part of the -release. - -We had several design inputs: - -* Release notes should be part of the git history, so as fixes in - master are back-ported to older branches the notes can go with the - code change. -* Release notes may need to change over time, as typos are found, - logical errors or confusing language needs to be fixed, or as more - information becomes available (CVE numbers, etc.). -* Release notes should be peer-reviewed, as with other documentation - and code changes. -* Notes are mutable in that a clone today vs a clone tomorrow might - have different release notes about the same change. -* Notes are immutable in that for a given git hash/tag the release - notes will be the same. Tagging a commit will change the version - description but that is all. -* We want to avoid merge issues when shepherding in a lot of - release-note-worthy changes, which we expect to happen on stable - branches always, and at release times on master branches. -* We want writing a release note to be straight-forward. -* We do not want release notes to be custom ordered within a release, - but we do want the ordering to be predictable and consistent. -* We must be able to entirely remove a release note. -* We must not make things progressively slow down to a crawl over - years of usage. -* Release note authors shouldn't need to know any special values for - naming their notes files (i.e., no change id or SHA value that has - special meaning). -* It would be nice if it was somewhat easy to identify the file - containing a release note on a particular topic. -* Release notes should be grouped by type in the output document. - - 1. New features - 2. Known issues - 3. Upgrade notes - 4. Security fixes - 5. Bugs fixes - 6. Other - -We want to eventually provide the ability to create a release notes -file for a given release and add it to the source distribution for the -project. As a first step, we are going to settle for publishing -release notes in the documentation for a project. diff --git a/doc/source/examples.rst b/doc/source/examples.rst deleted file mode 100644 index f71619f7e1e8861d0d9f69c9cbc2c53688e02057..0000000000000000000000000000000000000000 --- a/doc/source/examples.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. release-notes:: Examples - :relnotessubdir: examples diff --git a/doc/source/history.rst b/doc/source/history.rst deleted file mode 100644 index ec77acc34ce1468a26a8357996d09734ea3a58c7..0000000000000000000000000000000000000000 --- a/doc/source/history.rst +++ /dev/null @@ -1 +0,0 @@ -.. release-notes:: Release Notes diff --git a/doc/source/index.rst b/doc/source/index.rst index f33586eb4b356d5c9406e04d32d43267fb981dbd..d5b8c4ecd8f40eade6e6a1fbbceb02256c67bae1 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,28 +1,21 @@ -.. reno documentation master file, created by - sphinx-quickstart on Tue Jul 9 22:26:36 2013. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. include:: ../../README.rst -Welcome to reno's documentation! -======================================================== +EuroPython 2018 Presentation +============================ -Contents: +.. raw:: html -.. toctree:: - :maxdepth: 2 - - design - installation - usage - sphinxext - contributing - history - examples + -Indices and tables -================== +Contents +======== -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` +.. toctree:: + :maxdepth: 2 + user/index + install/index + contributor/index + releasenotes/index diff --git a/doc/source/installation.rst b/doc/source/install/index.rst similarity index 63% rename from doc/source/installation.rst rename to doc/source/install/index.rst index 3ed7b92e2c83f2a03cd39a5bcd32e91087a33fd5..9941e9f7ec7062a35c594b1063531d76668480a7 100644 --- a/doc/source/installation.rst +++ b/doc/source/install/index.rst @@ -6,6 +6,11 @@ At the command line:: $ pip install reno +.. note:: + + Reno's dependencies include C extension modules, which in turn + depend on having the Python source header files installed. + Sphinx Extension ================ diff --git a/doc/source/readme.rst b/doc/source/readme.rst deleted file mode 100644 index a6210d3d8a7988fdee6556eeba665d6b58ee6db4..0000000000000000000000000000000000000000 --- a/doc/source/readme.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../README.rst diff --git a/doc/source/releasenotes/index.rst b/doc/source/releasenotes/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..aaac03e7572de881e2d20a49e01ec8bea034c984 --- /dev/null +++ b/doc/source/releasenotes/index.rst @@ -0,0 +1,9 @@ +=============== + Release Notes +=============== + +.. release-notes:: + :unreleased-version-title: In Development + +.. release-notes:: Newton and Earlier + :branch: origin/stable/newton diff --git a/doc/source/usage.rst b/doc/source/usage.rst deleted file mode 100644 index e9e12698940c52df72f38f5783679c3341f95003..0000000000000000000000000000000000000000 --- a/doc/source/usage.rst +++ /dev/null @@ -1,140 +0,0 @@ -======== - Usage -======== - -Creating New Release Notes -========================== - -The ``reno`` command line tool is used to create a new release note -file in the correct format and with a unique name. The ``new`` -subcommand combines a random suffix with a "slug" value to make the -new file with a unique name that is easy to identify again later. - -:: - - $ reno new slug-goes-here - Created new notes file in releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml - -Within OpenStack projects, ``reno`` is often run via tox instead of -being installed globally. For example - -:: - - $ tox -e venv -- reno new slug-goes-here - venv develop-inst-nodeps: /mnt/projects/release-notes-generation/reno - venv runtests: commands[0] | reno new slug-goes-here - Created new notes file in releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml - venv: commands succeeded - congratulations :) - $ git status - Untracked files: - (use "git add ..." to include in what will be committed) - - releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml - - -By default the new note is created under ``./releasenotes/notes``. Use -the ``--rel-notes-dir`` to change the parent directory (the ``notes`` -subdirectory is always appended). - -Editing a Release Note -====================== - -The note file is a YAML file with several sections. All of the text is -interpreted as having reStructuredText formatting. - -prelude - - General comments about the release. The prelude from all notes in a - section are combined, in note order, to produce a single prelude - introducing that release. - -features - - A list of new major features in the release. - -issues - - A list of known issues in the release. For example, if a new driver - is experimental or known to not work in some cases, it should be - mentioned here. - -upgrade - - A list of upgrade notes in the release. For example, if a database - schema alteration is needed. - -critical - - A list of *fixed* critical bugs. - -security - - A list of *fixed* security issues. - -fixes - - A list of other *fixed* bugs. - -other - - Other notes that are important but do not fall into any of the given - categories. - -Any sections that would be blank should be left out of the note file -entirely. - -:: - - --- - prelude: > - Replace this text with content to appear at the - top of the section for this release. - features: - - List new features here, or remove this section. - issues: - - List known issues here, or remove this section. - upgrade: - - List upgrade notes here, or remove this section. - critical: - - Add critical notes here, or remove this section. - security: - - Add security notes here, or remove this section. - fixes: - - Add normal bug fixes here, or remove this section. - other: - - Add other notes here, or remove this section. - -Formatting ----------- - -Release notes may include embedded reStructuredText, including simple -inline markup like emphasis and pre-formatted text as well as complex -body structures such as nested lists and tables. To use these -formatting features, the note must be escaped from the YAML parser. - -The default template sets up the ``prelude`` section to use ``>`` so -that line breaks in the text are removed. This escaping mechanism is -not needed for the bullet items in the other sections of the template. - -To escape the text of any section and *retain* the newlines, prefix -the value with ``|``. For example: - -.. include:: ../../examples/notes/add-complex-example-6b5927c246456896.yaml - :literal: - -See :doc:`examples` for the rendered version of the note. - -Generating a Report -=================== - -Run ``reno report `` to generate a report -containing the release notes. The ``--branch`` argument can be used to -generate a report for a specific branch (the default is the branch -that is checked out). To limit the report to a subset of the available -versions on the branch, use the ``--version`` option (it can be -repeated). - -Notes are output in the order they are found by ``git log`` looking -over the history of the branch. This is deterministic, but not -necessarily predictable or mutable. diff --git a/doc/source/user/design.rst b/doc/source/user/design.rst new file mode 100644 index 0000000000000000000000000000000000000000..3a5f1e9269358c54a90284dc17a09d9aaece54f0 --- /dev/null +++ b/doc/source/user/design.rst @@ -0,0 +1,112 @@ +===================================== + Design Constraints and Requirements +===================================== + +Managing release notes for a complex project over a long period of +time with many releases can be time consuming and error prone. Reno +helps automate the hard parts by devising a way to store the notes +inside the git repository where they can be tagged as part of the +release. + +We had several design inputs: + +* Release notes should be part of the git history, so as fixes in + master are back-ported to older branches the notes can go with the + code change. +* Release notes may need to change over time, as typos are found, + logical errors or confusing language needs to be fixed, or as more + information becomes available (CVE numbers, etc.). +* Release notes should be peer-reviewed, as with other documentation + and code changes. +* Notes are mutable in that a clone today vs a clone tomorrow might + have different release notes about the same change. +* Notes are immutable in that for a given git hash/tag the release + notes will be the same. Tagging a commit will change the version + description but that is all. +* We want to avoid merge issues when shepherding in a lot of + release-note-worthy changes, which we expect to happen on stable + branches always, and at release times on master branches. +* We want writing a release note to be straight-forward. +* We do not want release notes to be custom ordered within a release, + but we do want the ordering to be predictable and consistent. +* We must be able to entirely remove a release note. +* We must not make things progressively slow down to a crawl over + years of usage. +* Release note authors shouldn't need to know any special values for + naming their notes files (i.e., no change id or SHA value that has + special meaning). +* It would be nice if it was somewhat easy to identify the file + containing a release note on a particular topic. +* Release notes should be grouped by type in the output document. + + 1. New features + 2. Known issues + 3. Upgrade notes + 4. Security fixes + 5. Bugs fixes + 6. Other + +We want to eventually provide the ability to create a release notes +file for a given release and add it to the source distribution for the +project. As a first step, we are going to settle for publishing +release notes in the documentation for a project. + +Assumptions +----------- + +Based on the above, *reno* makes a couple of assumptions about the release +policy used for a given project. *reno* expects all development, including bug +fixes, to take place on a single branch, ``master``. If *stable* or *release* +branches are used to support an older release then development should not take +place on these branches. Instead, bug fixes should be backported or +cherry-picked from ``master`` to the given *stable* branch. This is commonly +referred to as a `trunk-based`_ development workflow. + +.. code-block:: none + :caption: Trunk-based development. This is what *reno* expects. + + * bc823f0 (HEAD -> master) Fix a bug + | + | * 9723350 (tag: 1.0.1, stable/1.0) Fix a bug + | * 49e2158 (tag: 1.0.0) Release 1.0 + * | ad13f52 Fix a bug on master + * | 81b6b41 doc: Handle multiple branches in release notes + |/ + * 0faba45 Integrate reno + * a7beb14 (tag: 0.1.0) Add documentation + * e23b0c8 Add gitignore + * ff980c7 Initial commit + +(where ``9723350`` is the backported version of ``bc823f0``). + +By comparison, *reno* does not currently support projects where development is +spread across multiple active branches. In these situations, bug fixes are +developed on the offending *stable* or *release* branch and this branch is +later merged back into ``master``. This is commonly referred to as a +`git-flow-based`_ development workflow. + +.. code-block:: none + :caption: git-flow-based development. This is not compatible with *reno*. + + * 7df1078 (HEAD -> master) Merge branch 'stable/1.0' + |\ + | * 9723350 (tag: 1.0.1, stable/1.0) Fix a bug on stable + | * 49e2158 (tag: 1.0.0) Release 1.0 + * | ad13f52 Fix a bug on master + * | 81b6b41 doc: Handle multiple branches in release notes + |/ + * 0faba45 Integrate reno + * a7beb14 (tag: 0.1.0) Add documentation + * e23b0c8 Add gitignore + * ff980c7 Initial commit + +When this happens, *reno* has no way to distinguish between changes that apply +to the given *stable* branch and those that apply to ``master``. This is +because *reno* is *branch-based*, rather than *release-based*. If your project +uses this workflow, *reno* might not be for you. + +More information is available `here`_. + +.. _trunk-based: https://trunkbaseddevelopment.com/ +.. _git-flow-based: http://nvie.com/posts/a-successful-git-branching-model/ +.. _here: https://storyboard.openstack.org/#!/story/1588309 diff --git a/doc/source/user/examples.rst b/doc/source/user/examples.rst new file mode 100644 index 0000000000000000000000000000000000000000..0119177405cc3673750f93385a1440b948b3d9e8 --- /dev/null +++ b/doc/source/user/examples.rst @@ -0,0 +1,17 @@ +========== + Examples +========== + +Input file +========== + +.. literalinclude:: ../../../examples/notes/add-complex-example-6b5927c246456896.yaml + :caption: examples/notes/add-complex-example-6b5927c246456896.yaml + :language: yaml + +Rendered +======== + +.. release-notes:: + :relnotessubdir: examples + :earliest-version: 1.0.0 diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..09416a400d36e820893a4becfe31c927cd3a1ebc --- /dev/null +++ b/doc/source/user/index.rst @@ -0,0 +1,12 @@ +================= + reno User Guide +================= + +.. toctree:: + :maxdepth: 2 + + design + usage + sphinxext + setuptools + examples diff --git a/doc/source/user/setuptools.rst b/doc/source/user/setuptools.rst new file mode 100644 index 0000000000000000000000000000000000000000..693b88a3818ce94bb53f79e2385b264079d6ef65 --- /dev/null +++ b/doc/source/user/setuptools.rst @@ -0,0 +1,60 @@ +============================== + Python Packaging Integration +============================== + +*reno* supports integration with `setuptools`_ and *setuptools* derivatives +like *pbr* through a custom command - ``build_reno``. + +.. _pbr: https://docs.openstack.org/pbr/latest/ +.. _setuptools: https://setuptools.readthedocs.io/en/latest/ + +Using setuptools integration +---------------------------- + +To enable the ``build_reno`` command, you simply need to install *reno*. Once +done, simply run: + +.. code-block:: shell + + python setup.py build_reno + +You can configure the command in ``setup.py`` or ``setup.cfg``. To configure it +from ``setup.py``, add a ``build_reno`` section to ``command_options`` like so: + +.. code-block:: python + + from setuptools import setup + + setup( + name='mypackage', + version='0.1', + ... + command_options={ + 'build_reno': { + 'output_file': ('setup.py', 'RELEASENOTES.txt'), + }, + }, + ) + +To configure the command from ``setup.cfg``, add a ``build_reno`` section. For +example: + +.. code-block:: ini + + [build_reno] + output-file = RELEASENOTES.txt + +Options for setuptools integration +---------------------------------- + +These options related to the *setuptools* integration only. For general +configuration of *reno*, refer to :ref:`configuration`. + +``repo-root`` + The root directory of the Git repository; defaults to ``.`` + +``rel-notes-dir`` + The parent directory; defaults to ``releasenotes`` + +``output-file`` + The filename of the release notes file; defaults to ``RELEASENOTES.rst`` diff --git a/doc/source/sphinxext.rst b/doc/source/user/sphinxext.rst similarity index 69% rename from doc/source/sphinxext.rst rename to doc/source/user/sphinxext.rst index b43407cd643b3404a3d45c22e62be7261e4d7be8..18a6d6d00843981ba97b220d99f1e1825c9fe254 100644 --- a/doc/source/sphinxext.rst +++ b/doc/source/user/sphinxext.rst @@ -23,43 +23,55 @@ Enable the extension by adding ``'reno.sphinxext'`` to the Options: *branch* - The name of the branch to scan. Defaults to the current branch. *reporoot* - The path to the repository root directory. Defaults to the directory where ``sphinx-build`` is being run. *relnotessubdir* - The path under ``reporoot`` where the release notes are. Defaults to ``releasenotes``. *notesdir* - The path under ``relnotessubdir`` where the release notes are. Defaults to ``notes``. *version* - A comma separated list of versions to include in the notes. The default is to include all versions found on ``branch``. + *collapse-pre-releases* + A flag indicating that notes attached to pre-release versions + should be incorporated into the notes for the final release, + after the final release is tagged. + + *earliest-version* + A string containing the version number of the earliest version to + be included. For example, when scanning a branch, this is + typically set to the version used to create the branch to limit + the output to only versions on that branch. + + *ignore-notes* + A string containing a comma-delimited list of filenames or UIDs + for notes that should be ignored by the scanner. It is most + useful to set this when a note is edited on the wrong branch, + making it appear to be part of a release that it is not. + Examples ======== The release notes for the "current" branch, with "Release Notes" as a title. -:: +.. code-block:: rest .. release-notes:: Release Notes The release notes for the "stable/liberty" branch, with a separate title. -:: +.. code-block:: rest ======================= Liberty Release Notes @@ -70,7 +82,7 @@ title. The release notes for version "1.0.0". -:: +.. code-block:: rest .. release-notes:: 1.0.0 Release Notes :version: 1.0.0 diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst new file mode 100644 index 0000000000000000000000000000000000000000..01de9221672ef8c4b3008b61c3590bba23ce423f --- /dev/null +++ b/doc/source/user/usage.rst @@ -0,0 +1,314 @@ +======== + Usage +======== + +Creating New Release Notes +========================== + +The ``reno`` command line tool is used to create a new release note +file in the correct format and with a unique name. The ``new`` +subcommand combines a random suffix with a "slug" value to create +the file with a unique name that is easy to identify again later. + +:: + + $ reno new slug-goes-here + Created new notes file in releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml + +Within OpenStack projects, ``reno`` is often run via tox instead of +being installed globally. For example + +:: + + $ tox -e venv -- reno new slug-goes-here + venv develop-inst-nodeps: /mnt/projects/release-notes-generation/reno + venv runtests: commands[0] | reno new slug-goes-here + Created new notes file in releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml + venv: commands succeeded + congratulations :) + $ git status + Untracked files: + (use "git add ..." to include in what will be committed) + + releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml + +The ``--edit`` option opens the new note in a text editor. + +:: + + $ reno new slug-goes-here --edit + ... Opens the editor set in the EDITOR environment variable, editing the new file ... + Created new notes file in releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml + +The ``--from-template`` option allows you to use a pre-defined file and use +that as the release note. + +:: + + $ reno new slug-goes-here --from-template my-file.yaml + ... Creates a release note using the provided file my-file.yaml ... + Created new notes file in releasenotes/notes/slug-goes-here-95915aaedd3c48d8.yaml + +.. note:: + + You can also combine the flags ``--edit`` and ``--from-template`` + to create a release note from a specified file and immediately start an + editor to modify the new file. + +By default, the new note is created under ``./releasenotes/notes``. +The ``--rel-notes-dir`` command-line flag changes the parent directory +(the ``notes`` subdirectory is always appended). It's also possible to +set a custom template to create notes (see `Configuring Reno`_ ). + +Editing a Release Note +====================== + +The note file is a YAML file with several sections. All of the text is +interpreted as having `reStructuredText`_ formatting. The permitted +sections are configurable (see below) but default to the following +list: + +prelude + General comments about the release. Prelude sections from all notes in a + release are combined, in note order, to produce a single prelude + introducing that release. This section is always included, regardless + of what sections are configured. + +features + A list of new major features in the release. + +issues + A list of known issues in the release. For example, if a new driver + is experimental or known to not work in some cases, it should be + mentioned here. + +upgrade + A list of upgrade notes in the release. For example, if a database + schema alteration is needed. + +deprecations + A list of features, APIs, configuration options to be deprecated in the + release. Deprecations should not be used for something that is removed in the + release, use upgrade section instead. Deprecation should allow time for users + to make necessary changes for the removal to happen in a future release. + +critical + A list of *fixed* critical bugs. + +security + A list of *fixed* security issues. + +fixes + A list of other *fixed* bugs. + +other + Other notes that are important but do not fall into any of the given + categories. + +Any sections that would be blank should be left out of the note file +entirely. + +.. code-block:: yaml + + --- + prelude: > + Replace this text with content to appear at the + top of the section for this release. + features: + - List new features here, or remove this section. + issues: + - List known issues here, or remove this section. + upgrade: + - List upgrade notes here, or remove this section. + deprecations: + - List deprecation notes here, or remove this section + critical: + - Add critical notes here, or remove this section. + security: + - Add security notes here, or remove this section. + fixes: + - Add normal bug fixes here, or remove this section. + other: + - Add other notes here, or remove this section. + +Note File Syntax +---------------- + +Release notes may include embedded `reStructuredText`_, including simple +inline markup like emphasis and pre-formatted text as well as complex +body structures such as nested lists and tables. To use these +formatting features, the note must be escaped from the YAML parser. + +The default template sets up the ``prelude`` section to use ``>`` so +that line breaks in the text are removed. This escaping mechanism is +not needed for the bullet items in the other sections of the template. + +To escape the text of any section and *retain* the newlines, prefix +the value with ``|``. For example: + +.. literalinclude:: ../../../examples/notes/add-complex-example-6b5927c246456896.yaml + :language: yaml + +See :doc:`examples` for the rendered version of the note. + +.. _reStructuredText: http://www.sphinx-doc.org/en/stable/rest.html + +Generating a Report +=================== + +Run ``reno report `` to generate a report +containing the release notes. The ``--branch`` argument can be used to +generate a report for a specific branch (the default is the branch +that is checked out). To limit the report to a subset of the available +versions on the branch, use the ``--version`` option (it can be +repeated). + +Notes are output in the order they are found when scanning the git +history of the branch using topological ordering. This is +deterministic, but not necessarily predictable or mutable. + +Checking Notes +============== + +Run ``reno lint `` to test the existing +release notes files against some rules for catching common +mistakes. The command exits with an error code if there are any +mistakes, so it can be used in a build pipeline to force some +correctness. + +.. _configuration: + +Configuring Reno +================ + +Reno looks for an optional config file, either ``config.yaml`` in the release +notes directory or ``reno.yaml`` in the root directory. If the values in the +configuration file do not apply to the command being run, they are ignored. For +example, some reno commands take inputs controlling the branch, earliest +revision, and other common parameters that control which notes are included in +the output. Because they are commonly set options, a configuration file may be +the most convenient way to manage the values consistently. + +.. code-block:: yaml + + --- + branch: master + earliest_version: 12.0.0 + collapse_pre_releases: false + stop_at_branch_base: true + sections: + # The prelude section is implicitly included. + - [features, New Features] + - [issues, Known Issues] + - [upgrade, Upgrade Notes] + - [api, API Changes] + - [security, Security Issues] + - [fixes, Bug Fixes] + # Change prelude_section_name to 'release_summary' from default value + # 'prelude'. + prelude_section_name: release_summary + template: | + + ... + +Many of the settings in the configuration file can be overridden by +using command-line switches. For example: + +- ``--branch`` +- ``--earliest-version`` +- ``--collapse-pre-releases``/``--no-collapse-pre-releases`` +- ``--ignore-cache`` +- ``--stop-at-branch-base``/``--no-stop-at-branch-base`` + +The following options are configurable: + +.. show-reno-config:: + +Debugging +========= + +The true location of formatting errors in release notes may be masked +because of the way release notes are included into sphinx documents. +To generate the release notes manually, so that they can be put into a +sphinx document directly for debugging, use the ``report`` command. + +.. code-block:: console + + $ reno report . + +Updating Stable Branch Release Notes +==================================== + +Occasionally it is necessary to update release notes for past releases +due to URLs changing or errors not being noticed until after they have +been released. In cases like these, it is important to note that any +updates to these release notes should be proposed directly to the stable +branch where they were introduced. + +.. note:: + + Due to the way reno scans release notes, if a note is updated on a + later branch instead of its original branch, it will then show up + in the release notes for the later release. + +If a note is accidentally modified in a later branch causing it to show +up in the wrong release's notes, the ``ignore-notes`` directive may be +used to manually exclude it from the generated output: + +:: + + =========================== + Pike Series Release Notes + =========================== + + .. release-notes:: + :branch: stable/pike + :ignore-notes: + mistake-note-1-ee6274467572906b.yaml, + mistake-note-2-dd6274467572906b.yaml + + +Even though the note will be parsed in the newer release, it will be +excluded from the output for that release. + +Within OpenStack +================ + +The OpenStack project maintains separate instructions for configuring +the CI jobs and other project-specific settings used for reno. Refer +to the `Managing Release Notes +`__ +section of the Project Team Guide for details. + +Within Travis CI +================ + +The `Travis CI `_ uses shallow git clones, +and detached head, which prevents reno from accessing the repo data +it needs. +You'll see an error message like the one mentioned in +`Launchpad bug 1703603 `_. + +To use reno within a Travis CI job, the cloned repository needs to be +unshallowed and checked out in the right branch from your ``.travis.yml``, +like in the following example: + +.. code-block:: yaml + + --- + language: python + + python: + - 3.5 + + install: + - | + # Force unshallow and checkout the current branch + # https://docs.openstack.org/reno/latest/user/usage.html#within-travis-ci + git config remote.origin.fetch +refs/heads/*:refs/remotes/origin/* + git fetch --unshallow --tags + git symbolic-ref --short HEAD || git checkout -b ${TRAVIS_BRANCH}-test $TRAVIS_BRANCH + # Ref: https://stackoverflow.com/questions/32580821/how-can-i-customize-override-the-git-clone-step-in-travis-ci + + script: + - reno report . diff --git a/examples/notes/add-complex-example-6b5927c246456896.yaml b/examples/notes/add-complex-example-6b5927c246456896.yaml index dfd9a0c8168b587f5caf36b14053e78018fcb811..24c31e1f05b6f272c7c516063c03ff0979c8034c 100644 --- a/examples/notes/add-complex-example-6b5927c246456896.yaml +++ b/examples/notes/add-complex-example-6b5927c246456896.yaml @@ -14,14 +14,23 @@ prelude: | | with | so the reStructuredText | parser will retain | the line breaks. +features: + This note is a simple string, and does not retain its + formatting when it is rendered in HTML. rst markup here + may break the YAML parser, since the string is not escaped. +fixes: + - Use YAML lists to add multiple items to the same section. + - Another fix could be listed here. other: - | - This bullet item includes a paragraph and a nested list. + This bullet item includes a paragraph and a nested list, + which works because the content of the YAML list item + is an escaped string block with reStructuredText formatting. * list item 1 * list item 2 - :: + .. code-block:: text This example is also rendered correctly on multiple lines diff --git a/lower-constraints.txt b/lower-constraints.txt new file mode 100644 index 0000000000000000000000000000000000000000..abff1269ff728515de66c6e6f05530d3af6082ab --- /dev/null +++ b/lower-constraints.txt @@ -0,0 +1,5 @@ +Sphinx==1.6.1 +docutils==0.11 +PyYAML==3.10.0 +six==1.9.0 +dulwich==0.15.0 diff --git a/openstack-common.conf b/openstack-common.conf deleted file mode 100644 index ebac8298eab384f6054b8ba4e7a23ae014e60356..0000000000000000000000000000000000000000 --- a/openstack-common.conf +++ /dev/null @@ -1,6 +0,0 @@ -[DEFAULT] - -# The list of modules to copy from oslo-incubator.git - -# The base module to hold the copy of openstack.common -base=reno diff --git a/releasenotes/notes/Enable-using-tempalte-file-be734d8698309409.yaml b/releasenotes/notes/Enable-using-tempalte-file-be734d8698309409.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b152d9ffdee6ed7d077a8a86205d85f06bd4572a --- /dev/null +++ b/releasenotes/notes/Enable-using-tempalte-file-be734d8698309409.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + The ``--from-template`` flag was added to the release note creation command. + This enables one to create a release note from a pre-defined template, + which is useful when automating the creation of commits. diff --git a/releasenotes/notes/add-closed-branch-config-options-8773caf240e4653f.yaml b/releasenotes/notes/add-closed-branch-config-options-8773caf240e4653f.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6deedcb8f45c3428ee76275fb034be9f85c6428a --- /dev/null +++ b/releasenotes/notes/add-closed-branch-config-options-8773caf240e4653f.yaml @@ -0,0 +1,19 @@ +--- +features: + - | + Adds new configuration options ``closed_branch_tag_re`` (to + identify tags that replace branches that have been closed) and + ``branch_name_prefix`` (a value to be added back to the closed + branch tag to turn it into the original branch name. + + These options are used in OpenStack to support scanning the + history of a branch based on the previous series branch, even + after that previous series is closed by setting + ``closed_branch_tag_re`` to ``(.+)-eol`` so that the series name + in a value like ``"mitaka-eol"`` is extracted using the + group. With ``branch_name_prefix`` set to ``"stable/"`` the tag + ``mitaka-eol`` becomes the branch name ``stable/mitaka``. +fixes: + - | + Fixes bug 1746076 so that scanning stable branches correctly + includes the history of earlier closed stable branches. diff --git a/releasenotes/notes/add-config-file-e77084792c1dc695.yaml b/releasenotes/notes/add-config-file-e77084792c1dc695.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32ec2041061f2d961a693ed9c44598d76b7099c9 --- /dev/null +++ b/releasenotes/notes/add-config-file-e77084792c1dc695.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Reno now supports having a ``config.yaml`` file in your release notes + directory. It will search for file in the directory specified by + ``--rel-notes-dir`` and parse it. It will apply whatever options are + valid for that particular command. If an option is not relevant to a + particular sub-command, it will not attempt to apply them. diff --git a/releasenotes/notes/add-earliest-version-6f3d634770e855d0.yaml b/releasenotes/notes/add-earliest-version-6f3d634770e855d0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5454d3f7fd897b6a042615a0b5309dc77f2f15b --- /dev/null +++ b/releasenotes/notes/add-earliest-version-6f3d634770e855d0.yaml @@ -0,0 +1,6 @@ +--- +features: + - Add the ability to limit queries by stopping at an "earliest + version". This is intended to be used when scanning a branch, for + example, to stop at a point when the branch was created and not + include all of the history from the parent branch. \ No newline at end of file diff --git a/releasenotes/notes/add-linter-ce0a861ade64baf2.yaml b/releasenotes/notes/add-linter-ce0a861ade64baf2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b36983abe821a6fede14b0544d992545da56c8c1 --- /dev/null +++ b/releasenotes/notes/add-linter-ce0a861ade64baf2.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add a ``lint`` command for checking the contents and names of the + release notes files against some basic validation rules. \ No newline at end of file diff --git a/releasenotes/notes/allow-short-branch-names-61a35be55f04cea4.yaml b/releasenotes/notes/allow-short-branch-names-61a35be55f04cea4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..280ce62027c597f114f58c1c0f08df9d572ae980 --- /dev/null +++ b/releasenotes/notes/allow-short-branch-names-61a35be55f04cea4.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fix a problem with branch references so that it is now possible to + use a local tracking branch name when the branch only exists on + the 'origin' remote. For example, this allows references to + 'stable/ocata' when there is no local branch with that name but + there is an 'origin/stable/ocata' branch. \ No newline at end of file diff --git a/releasenotes/notes/avoid-clashing-uids-e84ffe8132ce996d.yaml b/releasenotes/notes/avoid-clashing-uids-e84ffe8132ce996d.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ea18c6309ebf4f1b74af4f153d2492c0ed7cc63 --- /dev/null +++ b/releasenotes/notes/avoid-clashing-uids-e84ffe8132ce996d.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fix a problem caused by failing to process multiple files with the + same UID portion of the filename. Ignore existing cases as long as + there is a corrective patch to remove them. Prevent new cases from + being introduced. See https://bugs.launchpad.net/reno/+bug/1688042 + for details. \ No newline at end of file diff --git a/releasenotes/notes/branches-eol-bcafc2a007a1eb9f.yaml b/releasenotes/notes/branches-eol-bcafc2a007a1eb9f.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a852d9fcfe0a1b966e79df558077856c645c48d --- /dev/null +++ b/releasenotes/notes/branches-eol-bcafc2a007a1eb9f.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Explicitly allow reno to scan starting from a tag by specifying the + tag where a branch name would otherwise be used. + - | + Add logic to allow reno to detect a branch that has been marked as + end-of-life using the OpenStack community's process of tagging the + HEAD of a stable/foo branch foo-eol before deleting the + branch. This means that references to "stable/foo" are translated + to "foo-eol" when the branch does not exist, and that Sphinx + directives do not need to be manually updated. \ No newline at end of file diff --git a/releasenotes/notes/bug-1537451-f44591da125ba09d.yaml b/releasenotes/notes/bug-1537451-f44591da125ba09d.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0334660e46591f16d1ba131f503c91705d045ff --- /dev/null +++ b/releasenotes/notes/bug-1537451-f44591da125ba09d.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - Resolves `a bug `__ + with properly detecting pre-release versions in the existing + history of a repository that resulted in some release notes not + appearing in the report output. diff --git a/releasenotes/notes/cache-ordering-6c743f68e3f7107f.yaml b/releasenotes/notes/cache-ordering-6c743f68e3f7107f.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39e40ee03802ed34098a5abf72ade098ce064b00 --- /dev/null +++ b/releasenotes/notes/cache-ordering-6c743f68e3f7107f.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Correct a problem with version number ordering when reading data + from the cache file. See + https://storyboard.openstack.org/#!/story/2001934 for details. diff --git a/releasenotes/notes/collapse-pre-releases-0b24e0bab46d7cf1.yaml b/releasenotes/notes/collapse-pre-releases-0b24e0bab46d7cf1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cd1675c82017de0057a5c7045eac099c0afc633 --- /dev/null +++ b/releasenotes/notes/collapse-pre-releases-0b24e0bab46d7cf1.yaml @@ -0,0 +1,4 @@ +--- +features: + - Add a flag to collapse pre-release notes into their final release, + if the final release tag is present. \ No newline at end of file diff --git a/releasenotes/notes/config-option-branch-name-re-8ecfe93195b8824e.yaml b/releasenotes/notes/config-option-branch-name-re-8ecfe93195b8824e.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d776c0c279e0ae1cfb09d746a4894ed545ef6b35 --- /dev/null +++ b/releasenotes/notes/config-option-branch-name-re-8ecfe93195b8824e.yaml @@ -0,0 +1,15 @@ +--- +features: + - | + Add a configuration option ``branch_name_re`` to hold a regular expression + for choosing "interesting" branches when trying to automatically detect + how far back the scanner should look. The default is ``stable/.+``, which + works for the OpenStack practice of creating branches named after the + stable series of releases. +fixes: + - | + Fixes the logic for determining how far back in history to look when + scanning a given branch. Reno now looks for the base of the "previous" + branch, as determined by looking at branches matching ``branch_name_re`` + in lexical order. This may not work if branches are created using + version numbers as their names. diff --git a/releasenotes/notes/config-option-sections-9c68b070698e984a.yaml b/releasenotes/notes/config-option-sections-9c68b070698e984a.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73c77bc0ef6686f9c13081cbb073b02a339fc7a9 --- /dev/null +++ b/releasenotes/notes/config-option-sections-9c68b070698e984a.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Add a configuration option ``sections`` to hold the list of + permitted section identifiers and corresponding display names. + This also determines the order in which sections are collated. + diff --git a/releasenotes/notes/custom-tag-versions-d02028b6d35db967.yaml b/releasenotes/notes/custom-tag-versions-d02028b6d35db967.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1ce5433e8b63e793beaf05cbf4931f7bec83e3b --- /dev/null +++ b/releasenotes/notes/custom-tag-versions-d02028b6d35db967.yaml @@ -0,0 +1,15 @@ +--- +features: + - | + Add the ability to specify regular expressions to a define a + customised versioning scheme for release tags and pre-release tags. + + By default this change supports the current versioning scheme used by + OpenStack. + + To customise, update the config.yaml file with the appropriate values. + For example, for tags with versions like ``v1.0.0`` and pre-release + versions like ``v1.0.0rc1`` the following could be added to config.yaml:: + + release_tag_re: 'v\d\.\d\.\d(rc\d+)?' + pre_release_tag_re: '(?Prc\d+$)' diff --git a/releasenotes/notes/default-repository-root-cli-85d23034bef81619.yaml b/releasenotes/notes/default-repository-root-cli-85d23034bef81619.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebd74fc521dbed266fcde3cee74a4e3a3df8df45 --- /dev/null +++ b/releasenotes/notes/default-repository-root-cli-85d23034bef81619.yaml @@ -0,0 +1,5 @@ +--- +features: + - Set the default value of the reporoot argument + for all command line programs to "." and make + it an optional parameter. \ No newline at end of file diff --git a/releasenotes/notes/dulwich-rewrite-3a5377162d97402b.yaml b/releasenotes/notes/dulwich-rewrite-3a5377162d97402b.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d29bd26b69aa7336a62c8e102a6686cf068094b7 --- /dev/null +++ b/releasenotes/notes/dulwich-rewrite-3a5377162d97402b.yaml @@ -0,0 +1,5 @@ +--- +prelude: > + This release includes a significant rewrite of the internal logic of + reno to access git data through the dulwich library instead of the + git command line porcelain. diff --git a/releasenotes/notes/fix-branch-base-detection-95300805f26a0c15.yaml b/releasenotes/notes/fix-branch-base-detection-95300805f26a0c15.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffbc8a02eeed718169b141f4852d27ae35bbdd6a --- /dev/null +++ b/releasenotes/notes/fix-branch-base-detection-95300805f26a0c15.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fix a problem with the way reno automatically detects the initial + version in a branch that prevented it from including all of the + notes associated with a release, especially if the branch was + created at a pre-release version number. + `Bug #1652092 `__ diff --git a/releasenotes/notes/fix-cli-option-handling-a13652d14507f2d7.yaml b/releasenotes/notes/fix-cli-option-handling-a13652d14507f2d7.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c84efee0ab109648a8cb8d704f467990b3eaf38 --- /dev/null +++ b/releasenotes/notes/fix-cli-option-handling-a13652d14507f2d7.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fix an issue with the way command line options and configuration + settings interact so that the settings in the configuration file + are used properly when command line arguments for those options + are not provided. diff --git a/releasenotes/notes/fix-delete-handling-55232c50b647aa57.yaml b/releasenotes/notes/fix-delete-handling-55232c50b647aa57.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eaca402f0aac23c48c4d9483cecefb05bfed32c7 --- /dev/null +++ b/releasenotes/notes/fix-delete-handling-55232c50b647aa57.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Correct a problem with handling deleted release notes that + triggered a TypeError with a message like "Can't mix strings and + bytes in path components" \ No newline at end of file diff --git a/releasenotes/notes/fix-sphinxext-scanner-0aa012ada66db773.yaml b/releasenotes/notes/fix-sphinxext-scanner-0aa012ada66db773.yaml new file mode 100644 index 0000000000000000000000000000000000000000..097c0977baed9fd8114e30132b3abdb6e339576a --- /dev/null +++ b/releasenotes/notes/fix-sphinxext-scanner-0aa012ada66db773.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fixes a problem with the sphinx extension that caused it to + produce an error if it had a list of versions to include that were + not within the set that seemed to be on the branch because of the + branch-base detection logic. Now if a list of versions is + included, the scan always includes the full history. \ No newline at end of file diff --git a/releasenotes/notes/flexible-formatting-31c8de2599d3637d.yaml b/releasenotes/notes/flexible-formatting-31c8de2599d3637d.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7611303756abc3275069f5730b8ede25bf5530f --- /dev/null +++ b/releasenotes/notes/flexible-formatting-31c8de2599d3637d.yaml @@ -0,0 +1,5 @@ +--- +features: + Release notes entries may now be made up of single strings. This + simplifies formatting for smaller notes, and eliminates a class of + errors associated with escaping reStructuredText inside YAML lists. diff --git a/releasenotes/notes/ignore-notes-option-9d0bde540fbcdf22.yaml b/releasenotes/notes/ignore-notes-option-9d0bde540fbcdf22.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fd41e457e21c408efa6c3eb06d80584880122ae --- /dev/null +++ b/releasenotes/notes/ignore-notes-option-9d0bde540fbcdf22.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Add a new configuration option ``ignore_notes``. Setting the value + to a list of filenames or UIDs for notes causes the reno scanner + to ignore them. It is most useful to set this when a note is + edited on the wrong branch, making it appear to be part of a + release that it is not. diff --git a/releasenotes/notes/ignore-null-merges-56b7a8ed9b20859e.yaml b/releasenotes/notes/ignore-null-merges-56b7a8ed9b20859e.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03f3da6a9f8640416f8e288df06ef9758a3f052f --- /dev/null +++ b/releasenotes/notes/ignore-null-merges-56b7a8ed9b20859e.yaml @@ -0,0 +1,18 @@ +--- +features: + - | + By default, reno now ignores "null" merge commits that bring in + tags from other threads. The new configuration option + ``ignore_null_merges`` controls this behavior. Setting the flag to + False restores the previous behavior in which the null-merge + commits were traversed like any other merge commit. +upgrade: + - | + The new configuration option ``ignore_null_merges`` causes the + scanner to ignore merge commits with no changes when one of the + parents being merged in has a release tag on it. +fixes: + - | + This release fixes a problem with the scanner that may have caused + it to stop scanning a branch prematurely when the tag from another + branch had been merged into the history. diff --git a/releasenotes/notes/include-working-copy-d0aed2e77bb095e6.yaml b/releasenotes/notes/include-working-copy-d0aed2e77bb095e6.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a02790d10be9e4bae1963c2c44b96b06e5668c8 --- /dev/null +++ b/releasenotes/notes/include-working-copy-d0aed2e77bb095e6.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Include the local working copy when scanning the history of the + current branch. Notes files must at least be staged to indicate + that they will eventually be part of the history, but subsequent + changes to the file do not need to also be staged to be seen. diff --git a/releasenotes/notes/log-levels-and-sphinx-161-6efe0d291718a657.yaml b/releasenotes/notes/log-levels-and-sphinx-161-6efe0d291718a657.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f54a20e49e26cc39a746f57411116e054fc73632 --- /dev/null +++ b/releasenotes/notes/log-levels-and-sphinx-161-6efe0d291718a657.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Sphinx 1.6.1 now interprets error and warning log messages as + reasons to abort the build when strict mode is enabled. This + release changes the log level for some calls that weren't really + errors to begin with to avoid having Sphinx abort the build + unnecessarily. \ No newline at end of file diff --git a/releasenotes/notes/no-show-source-option-ee02766b26fe53be.yaml b/releasenotes/notes/no-show-source-option-ee02766b26fe53be.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7074ad37beba5f8655ca0e3665959e5f7252921b --- /dev/null +++ b/releasenotes/notes/no-show-source-option-ee02766b26fe53be.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Add a ``--no-show-source`` option to the report command to skip + including the note reference file names and SHA information + in comments in the output. This restores the previous format of + the output for cases where it is meant to be read by people directly, + not just converted to HTML. diff --git a/releasenotes/notes/null-merge-infinite-loop-670367094ad83e19.yaml b/releasenotes/notes/null-merge-infinite-loop-670367094ad83e19.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed7ae8a4cc24ca58a0534937813b7886ea4bcdca --- /dev/null +++ b/releasenotes/notes/null-merge-infinite-loop-670367094ad83e19.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Remove an infinite loop in the traversal algorithm caused by some + null-merge skip situations. diff --git a/releasenotes/notes/optional-oslosphinx-55843a7f80a14e58.yaml b/releasenotes/notes/optional-oslosphinx-55843a7f80a14e58.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee0252fca8a8cd3343ecfb70a6e9034eed923a83 --- /dev/null +++ b/releasenotes/notes/optional-oslosphinx-55843a7f80a14e58.yaml @@ -0,0 +1,5 @@ +--- +other: + - The oslosphinx dependency for building documentation + is now optional. This breaks a build cycle between + oslosphinx and reno. diff --git a/releasenotes/notes/reference-name-mangling-3c845ebf88af6944.yaml b/releasenotes/notes/reference-name-mangling-3c845ebf88af6944.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd7281596cb1d4d0e24c8d0be2246088a1510e9c --- /dev/null +++ b/releasenotes/notes/reference-name-mangling-3c845ebf88af6944.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + The automatic branch name handling is updated so that if the + reference points explicitly to the origin remote, but that remote + isn't present (as it won't be when zuul configures the repo in + CI), then the shortened form of the reference without the prefix + is used instead. This allows explicit references to + ``origin/stable/name`` to be translated to ``stable/name`` and + find the expected branch. diff --git a/releasenotes/notes/repodir-config-file-b6b8edc2975964fc.yaml b/releasenotes/notes/repodir-config-file-b6b8edc2975964fc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aedb19e3352060d5e95904bb04968466d44cf0f7 --- /dev/null +++ b/releasenotes/notes/repodir-config-file-b6b8edc2975964fc.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + reno will now scan for a ``reno.yaml`` file in the root repo directory if a + ``config.yaml`` file does not exist in the releasenotes directory. This + allows users to do away with the unnecessary ``notes`` subdirectory in the + releasenotes directory. diff --git a/releasenotes/notes/report-title-option-f0875bfdbc54dd7b.yaml b/releasenotes/notes/report-title-option-f0875bfdbc54dd7b.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0948c2685ec3a5948da73206c1d13ac83695a317 --- /dev/null +++ b/releasenotes/notes/report-title-option-f0875bfdbc54dd7b.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Add a ``--title`` option to the report command. diff --git a/releasenotes/notes/scanner-change-96682cb04fc66c0b.yaml b/releasenotes/notes/scanner-change-96682cb04fc66c0b.yaml new file mode 100644 index 0000000000000000000000000000000000000000..900ab23f19ac8f14a1ee6bf8505c96c97c8d6b3d --- /dev/null +++ b/releasenotes/notes/scanner-change-96682cb04fc66c0b.yaml @@ -0,0 +1,11 @@ +--- +fixes: + - | + A fix is included to ignore changes to a note file until the + scanner encounters the git operation that adds the file. This + ensures that if a file is modified on master when it should be + modified on another branch the note is not erroneously + incorporated into the notes for the next release from master. + fixes `bug 1682796`_ + + .. _bug 1682796: https://bugs.launchpad.net/neutron/+bug/1682796 diff --git a/releasenotes/notes/setuptools-integration-950bd8ab6d2970c7.yaml b/releasenotes/notes/setuptools-integration-950bd8ab6d2970c7.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29ae876761dad352f3ba7b257b1571014a7bfce5 --- /dev/null +++ b/releasenotes/notes/setuptools-integration-950bd8ab6d2970c7.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add a ``build_reno`` setuptools command that allows users to generate a + release notes document and a reno cache file that can be used to build + release notes documents without the full Git history present. diff --git a/releasenotes/notes/show-less-unreleased-802781a1a3bf110e.yaml b/releasenotes/notes/show-less-unreleased-802781a1a3bf110e.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20673e0864b980a3ecec69f7adae01e8a31d3a73 --- /dev/null +++ b/releasenotes/notes/show-less-unreleased-802781a1a3bf110e.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + The scanner for the "current" branch (usually ``master``) now stops + when it encounters the base of an earlier branch matching the + ``branch_name_re`` config option. This results in less history + appearing on the unreleased pages, while still actually showing + the current series and any unreleased notes. diff --git a/releasenotes/notes/show-note-filename-in-report-a1118c917588b58d.yaml b/releasenotes/notes/show-note-filename-in-report-a1118c917588b58d.yaml new file mode 100644 index 0000000000000000000000000000000000000000..286b754a4e268ff2ebd1e29b2f2dd84830f6f737 --- /dev/null +++ b/releasenotes/notes/show-note-filename-in-report-a1118c917588b58d.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + The report output now includes debugging details with the filename + and sha for the version of the content used to indicate where the + content is from to assist with debugging formatting or content + issues. diff --git a/releasenotes/notes/stable-section-anchors-d99258b6df39c0fa.yaml b/releasenotes/notes/stable-section-anchors-d99258b6df39c0fa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52a27e1a2f0d38387a4e752fdf31b89c5b2c6063 --- /dev/null +++ b/releasenotes/notes/stable-section-anchors-d99258b6df39c0fa.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Added explicitly calculated anchors to ensure section links are both + unique and stable over time. diff --git a/releasenotes/notes/stop-scanning-branch-e5a8937c248acc99.yaml b/releasenotes/notes/stop-scanning-branch-e5a8937c248acc99.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3caccc5648714f2d5d2347f492f76e0854ee97a6 --- /dev/null +++ b/releasenotes/notes/stop-scanning-branch-e5a8937c248acc99.yaml @@ -0,0 +1,7 @@ +--- +features: + - Automatically stop scanning branches at the point where they + diverge from master. This avoids having release notes from older + versions, that appear on master before the branch, from showing up + in the versions from the branch. This logic is only applied to + branches created from master. diff --git a/releasenotes/notes/stop-scanning-branch-option-6a0156b183814d7f.yaml b/releasenotes/notes/stop-scanning-branch-option-6a0156b183814d7f.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fbba24bdcb91489b05a67b48af65b178f3bdd2c9 --- /dev/null +++ b/releasenotes/notes/stop-scanning-branch-option-6a0156b183814d7f.yaml @@ -0,0 +1,9 @@ +--- +features: + - Add a new configuration option, stop_at_branch_base, to control + whether or not the scanner stops looking for changes at the point + where a branch diverges from master. The default is True, meaning + that the scanner does stop. A false value means that versions that + appear on master from a point earlier than when the branch was + created will be included when scanning the branch for release + notes. diff --git a/releasenotes/notes/support-custom-template-0534a2199cfec44c.yaml b/releasenotes/notes/support-custom-template-0534a2199cfec44c.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb3bed84b30cb6e9d640cc0038d37481315ec63d --- /dev/null +++ b/releasenotes/notes/support-custom-template-0534a2199cfec44c.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Reno now supports to set through ``template`` attribute in + ``config.yaml`` a custom template which will be used by reno new + to create notes. diff --git a/releasenotes/notes/support-edit-ec5c01ad6144815a.yaml b/releasenotes/notes/support-edit-ec5c01ad6144815a.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2997e1ee499e218390cc5f7fded07d95330d344d --- /dev/null +++ b/releasenotes/notes/support-edit-ec5c01ad6144815a.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Reno now enables with reno new ``--edit`` to create a note and edit it with + your editor (defined with EDITOR environment variable). diff --git a/releasenotes/notes/tag-format-bd5018a813c804fd.yaml b/releasenotes/notes/tag-format-bd5018a813c804fd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbc58f31b9edacf33515375c24d0bad58d8e6131 --- /dev/null +++ b/releasenotes/notes/tag-format-bd5018a813c804fd.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Allow optional 'v' prefix in the default version tag regex. diff --git a/releasenotes/notes/unreleased-version-title-86751f52745fd3b7.yaml b/releasenotes/notes/unreleased-version-title-86751f52745fd3b7.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a094ca2f1deea19c4b3562e23dcd94e7d2fdb8aa --- /dev/null +++ b/releasenotes/notes/unreleased-version-title-86751f52745fd3b7.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Added configuration option ``unreleased_version_title`` with + associated Sphinx directive argument to control whether to show + the computed version number for changes that have not been + tagged, or to show a static title string specified in the option + value. diff --git a/reno/__init__.py b/reno/__init__.py index a61b2d075200b76b4785b6b4cf59489174234f60..ce87e7279abceab357ccb2ca15a0683c6fb55e51 100644 --- a/reno/__init__.py +++ b/reno/__init__.py @@ -12,8 +12,14 @@ # License for the specific language governing permissions and limitations # under the License. +import logging + import pbr.version __version__ = pbr.version.VersionInfo( 'reno').version_string() + +# Configure a null logger so that if reno is used as a library by an +# application that does not configure logging there are no warnings. +logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/reno/_exts/__init__.py b/reno/_exts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/reno/_exts/show_reno_config.py b/reno/_exts/show_reno_config.py new file mode 100644 index 0000000000000000000000000000000000000000..29afd1b8482cd53c73333a6d9e71c2195f9ed797 --- /dev/null +++ b/reno/_exts/show_reno_config.py @@ -0,0 +1,75 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from docutils import nodes +from docutils.parsers import rst +from docutils.statemachine import ViewList + +from sphinx.util.nodes import nested_parse_with_titles + +from reno import config +import six + + +def _multi_line_string(s, indent=''): + output_lines = s.splitlines() + if not output_lines[0].strip(): + output_lines = output_lines[1:] + for l in output_lines: + yield indent + l + + +def _format_option_help(options): + "Produce RST lines for the configuration options." + for opt in options: + yield '``{}``'.format(opt.name) + for l in _multi_line_string(opt.help, ' '): + yield l + yield '' + if isinstance(opt.default, six.string_types) and '\n' in opt.default: + # Multi-line string + yield ' Defaults to' + yield '' + yield ' ::' + yield '' + for l in _multi_line_string(opt.default, ' '): + yield l + else: + yield ' Defaults to ``{!r}``'.format(opt.default) + yield '' + + +class ShowConfigDirective(rst.Directive): + + option_spec = {} + + has_content = True + + def run(self): + env = self.state.document.settings.env + app = env.app + + result = ViewList() + source_name = '<' + __name__ + '>' + for line in _format_option_help(config._OPTIONS): + app.info(line) + result.append(line, source_name) + + node = nodes.section() + node.document = self.state.document + nested_parse_with_titles(self.state, result, node) + + return node.children + + +def setup(app): + app.add_directive('show-reno-config', ShowConfigDirective) diff --git a/reno/cache.py b/reno/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..20c2a8ed48d21b4ed84ab22e6c36481cb7e8fc5f --- /dev/null +++ b/reno/cache.py @@ -0,0 +1,107 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +import yaml + +from reno import loader +from reno import scanner + + +def build_cache_db(conf, versions_to_include): + s = scanner.Scanner(conf) + notes = s.get_notes_by_version() + + # Default to including all versions returned by the scanner. + if not versions_to_include: + versions_to_include = list(notes.keys()) + + # Build a cache data structure including the file contents as well + # as the basic data returned by the scanner. + file_contents = {} + for version in versions_to_include: + for filename, sha in notes[version]: + body = s.get_file_at_commit(filename, sha) + # We want to save the contents of the file, which is YAML, + # inside another YAML file. That looks terribly ugly with + # all of the escapes needed to format it properly as + # embedded YAML, so parse the input and convert it to a + # data structure that can be serialized cleanly. + y = yaml.safe_load(body) + file_contents[filename] = y + + cache = { + 'notes': [ + {'version': k, 'files': v} + for k, v in notes.items() + ], + 'file-contents': file_contents, + } + return cache + + +def write_cache_db(conf, versions_to_include, + outfilename=None): + """Create a cache database file for the release notes data. + + Build the cache database from scanning the project history and + write it to a file within the project. + + By default, the data is written to the same file the scanner will + try to read when it cannot look at the git history. If outfilename + is given and is '-' the data is written to stdout + instead. Otherwise, if outfilename is given, the data overwrites + the named file. + + Return the name of the file created, if any. + + """ + if outfilename == '-': + stream = sys.stdout + close_stream = False + elif outfilename: + stream = open(outfilename, 'w') + close_stream = True + else: + outfilename = loader.get_cache_filename(conf) + if not os.path.exists(os.path.dirname(outfilename)): + os.makedirs(os.path.dirname(outfilename)) + stream = open(outfilename, 'w') + close_stream = True + try: + cache = build_cache_db( + conf, + versions_to_include=versions_to_include, + ) + yaml.safe_dump( + cache, + stream, + allow_unicode=True, + explicit_start=True, + encoding='utf-8', + ) + finally: + if close_stream: + stream.close() + return outfilename + + +def cache_cmd(args, conf): + "Generates a release notes cache" + write_cache_db( + conf=conf, + versions_to_include=args.version, + outfilename=args.output, + ) + return diff --git a/reno/config.py b/reno/config.py new file mode 100644 index 0000000000000000000000000000000000000000..5990e2598988cf3514c0a8af28a2301fcfa5bd26 --- /dev/null +++ b/reno/config.py @@ -0,0 +1,337 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import logging +import os.path +import textwrap + +import yaml + +from reno import defaults + +LOG = logging.getLogger(__name__) + + +Opt = collections.namedtuple('Opt', 'name default help') + +_OPTIONS = [ + Opt('notesdir', defaults.NOTES_SUBDIR, + textwrap.dedent("""\ + The notes subdirectory within the relnotesdir where the + notes live. + """)), + + Opt('collapse_pre_releases', True, + textwrap.dedent("""\ + Should pre-release versions be merged into the final release + of the same number (1.0.0.0a1 notes appear under 1.0.0). + """)), + + Opt('stop_at_branch_base', True, + textwrap.dedent("""\ + Should the scanner stop at the base of a branch (True) or go + ahead and scan the entire history (False)? + """)), + + Opt('branch', None, + textwrap.dedent("""\ + The git branch to scan. Defaults to the "current" branch + checked out. If a stable branch is specified but does not + exist, reno attempts to automatically convert that to an + "end-of-life" tag. For example, ``origin/stable/liberty`` + would be converted to ``liberty-eol``. + """)), + + Opt('earliest_version', None, + textwrap.dedent("""\ + The earliest version to be included. This is usually the + lowest version number, and is meant to be the oldest + version. If unset, all versions will be scanned. + """)), + + Opt('template', defaults.TEMPLATE.format(defaults.PRELUDE_SECTION_NAME), + textwrap.dedent("""\ + The template used by reno new to create a note. + """)), + + Opt('release_tag_re', + textwrap.dedent('''\ + ((?:v?[\d.ab]|rc)+) # digits, a, b, and rc cover regular and + # pre-releases + '''), + textwrap.dedent("""\ + The regex pattern used to match the repo tags representing a + valid release version. The pattern is compiled with the + verbose and unicode flags enabled. + """)), + + Opt('pre_release_tag_re', + textwrap.dedent('''\ + (?P\.v?\d+(?:[ab]|rc)+\d*)$ + '''), + textwrap.dedent("""\ + The regex pattern used to check if a valid release version tag + is also a valid pre-release version. The pattern is compiled + with the verbose and unicode flags enabled. The pattern must + define a group called 'pre_release' that matches the + pre-release part of the tag and any separator, e.g for + pre-release version '12.0.0.0rc1' the default pattern will + identify '.0rc1' as the value of the group 'pre_release'. + """)), + + Opt('branch_name_re', 'stable/.+', + textwrap.dedent("""\ + The pattern for names for branches that are relevant when + scanning history to determine where to stop, to find the + "base" of a branch. Other branches are ignored. + """)), + + Opt('closed_branch_tag_re', '(.+)-eol', + textwrap.dedent("""\ + The pattern for names for tags that replace closed + branches that are relevant when scanning history to + determine where to stop, to find the "base" of a + branch. Other tags are ignored. + """)), + + Opt('branch_name_prefix', 'stable/', + textwrap.dedent("""\ + The prefix to add to tags for closed branches + to restore the old branch name to allow sorting + to place the tag in the proper place in history. + For example, OpenStack turns "mitaka-eol" into + "stable/mitaka" by removing the "-eol" suffix + via closed_branch_tag_re and setting the prefix + to "stable/". + """)), + + Opt('sections', + [ + ['features', 'New Features'], + ['issues', 'Known Issues'], + ['upgrade', 'Upgrade Notes'], + ['deprecations', 'Deprecation Notes'], + ['critical', 'Critical Issues'], + ['security', 'Security Issues'], + ['fixes', 'Bug Fixes'], + ['other', 'Other Notes'], + ], + textwrap.dedent("""\ + The identifiers and names of permitted sections in the + release notes, in the order in which the final report will + be generated. A prelude section will always be automatically + inserted before the first element of this list. + """)), + + Opt('prelude_section_name', defaults.PRELUDE_SECTION_NAME, + textwrap.dedent("""\ + The name of the prelude section in the note template. This + allows users to rename the section to, for example, + 'release_summary' or 'project_wide_general_announcements', + which is displayed in titlecase in the report after + replacing underscores with spaces. + """)), + + Opt('ignore_null_merges', True, + textwrap.dedent("""\ + When this option is set to True, any merge commits with no + changes and in which the second or later parent is tagged + are considered "null-merges" that bring the tag information + into the current branch but nothing else. + + OpenStack used to use null-merges to bring final release + tags from stable branches back into the master branch. This + confuses the regular traversal because it makes that stable + branch appear to be part of master and/or the later stable + branch. This option allows us to ignore those. + """)), + + Opt('ignore_notes', [], + textwrap.dedent("""\ + Note files to be ignored. It's useful to be able to ignore a + file if it is edited on the wrong branch. Notes should be + specified by their filename or UID. + + Setting the option in the main configuration file makes it + apply to all branches. To ignore a note in the HTML build, use + the ``ignore-notes`` parameter to the ``release-notes`` sphinx + directive. + """)), + + Opt('unreleased_version_title', '', + textwrap.dedent("""\ + The title to use for any notes that do not appear in a + released version. If this option is unset, the development + version number is used (for example, ``3.0.0-3``). + """)), +] + + +class Config(object): + + _OPTS = {o.name: o for o in _OPTIONS} + + @classmethod + def get_default(cls, opt): + "Return the default for an option." + try: + return cls._OPTS[opt].default + except KeyError: + raise ValueError('unknown option name %r' % (opt,)) + + def __init__(self, reporoot, relnotesdir=None): + """Instantiate a Config object + + :param str reporoot: + The root directory of the repository. + :param str relnotesdir: + The directory containing release notes. Defaults to + 'releasenotes'. + """ + self.reporoot = reporoot + if relnotesdir is None: + relnotesdir = defaults.RELEASE_NOTES_SUBDIR + self.relnotesdir = relnotesdir + # Initialize attributes from the defaults. + self.override(**{o.name: o.default for o in _OPTIONS}) + + self._contents = {} + self._load_file() + + def _load_file(self): + filenames = [ + os.path.join(self.reporoot, self.relnotesdir, 'config.yaml'), + os.path.join(self.reporoot, 'reno.yaml')] + + for filename in filenames: + LOG.debug('looking for configuration file %s', filename) + if os.path.isfile(filename): + break + else: + self._report_missing_config_files(filenames) + return + + try: + with open(filename, 'r') as fd: + self._contents = yaml.safe_load(fd) + LOG.info('loaded configuration file %s', filename) + except IOError as err: + self._report_failure_config_file(filename, err) + else: + self.override(**self._contents) + + def _report_missing_config_files(self, filenames): + # NOTE(dhellmann): This is extracted so we can mock it for + # testing. + LOG.info('no configuration file in: %s', ', '.join(filenames)) + + def _report_failure_config_file(self, filename, err): + # NOTE(dhellmann): This is extracted so we can mock it for + # testing. + LOG.warning('did not load config file %s: %s', filename, err) + + def _rename_prelude_section(self, **kwargs): + key = 'prelude_section_name' + if key in kwargs and kwargs[key] != self._OPTS[key].default: + new_prelude_name = kwargs[key] + + self.template = defaults.TEMPLATE.format(new_prelude_name) + + def override(self, **kwds): + """Set the values of the named configuration options. + + Take the values of the keyword arguments as the current value + of the same option, regardless of whether a value is already + present. + + """ + # Replace prelude section name if it has been changed. + self._rename_prelude_section(**kwds) + + for n, v in kwds.items(): + if n not in self._OPTS: + LOG.warning('ignoring unknown configuration value %r = %r', + n, v) + else: + setattr(self, n, v) + + def override_from_parsed_args(self, parsed_args): + """Set the values of the configuration options from parsed CLI args. + + This method assumes that the DEST values for the command line + arguments are named the same as the configuration options. + + """ + arg_values = { + o.name: getattr(parsed_args, o.name) + for o in _OPTIONS + if getattr(parsed_args, o.name, None) is not None + } + if arg_values: + LOG.info('[config] updating from command line options') + self.override(**arg_values) + + @property + def reporoot(self): + return self._reporoot + + # Ensure that the 'reporoot' value always only ends in one '/'. + @reporoot.setter + def reporoot(self, value): + self._reporoot = value.rstrip('/') + '/' + + @property + def notespath(self): + """The path in the repo where notes are kept. + + .. important:: + + This does not take ``reporoot`` into account. You need to add this + manually if required. + """ + return os.path.join(self.relnotesdir, self.notesdir) + + @property + def options(self): + """Get all configuration options as a dict. + + Returns the actual configuration options after overrides. + """ + options = { + o.name: getattr(self, o.name) + for o in _OPTIONS + } + return options + +# def parse_config_into(parsed_arguments): + +# """Parse the user config onto the namespace arguments. + +# :param parsed_arguments: +# The result of calling :meth:`argparse.ArgumentParser.parse_args`. +# :type parsed_arguments: +# argparse.Namespace +# """ +# config_path = get_config_path(parsed_arguments.relnotesdir) +# config_values = read_config(config_path) + +# for key in config_values.keys(): +# try: +# getattr(parsed_arguments, key) +# except AttributeError: +# LOG.info('Option "%s" does not apply to this particular command.' +# '. Ignoring...', key) +# continue +# setattr(parsed_arguments, key, config_values[key]) + +# parsed_arguments._config = config_values diff --git a/reno/create.py b/reno/create.py index b37ee9f98cb3279ecbce34bb8b0923e4ec150963..703e641e83ef10a6e2fb4cd51c700403f0930814 100644 --- a/reno/create.py +++ b/reno/create.py @@ -13,83 +13,11 @@ from __future__ import print_function import os +import subprocess from reno import utils -_TEMPLATE = """\ ---- -prelude: > - Replace this text with content to appear at the - top of the section for this release. All of the - prelude content is merged together and then rendered - separately from the items listed in other parts of - the file, so the text needs to be worded so that - both the prelude and the other items make sense - when read independently. This may mean repeating - some details. Not every release note - requires a prelude. Usually only notes describing - major features or adding release theme details should - have a prelude. -features: - - List new features here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. -issues: - - List known issues here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. -upgrade: - - List upgrade notes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. -deprecations: - - List deprecations notes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. -critical: - - Add critical notes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. -security: - - Add security notes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. -fixes: - - Add normal bug fixes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. -other: - - Add other notes here, or remove this section. - All of the list items in this section are combined - when the release notes are rendered, so the text - needs to be worded so that it does not depend on any - information only available in another section, such - as the prelude. This may mean repeating some details. -""" - - def _pick_note_file_name(notesdir, slug): "Pick a unique name in notesdir." for i in range(50): @@ -104,17 +32,33 @@ def _pick_note_file_name(notesdir, slug): ) -def _make_note_file(filename): +def _make_note_file(filename, template): notesdir = os.path.dirname(filename) if not os.path.exists(notesdir): os.makedirs(notesdir) with open(filename, 'w') as f: - f.write(_TEMPLATE) + f.write(template) + +def _edit_file(filename): + if 'EDITOR' not in os.environ: + return False + subprocess.call([os.environ['EDITOR'], filename]) + return True -def create_cmd(args): + +def _get_user_template(template_file): + if not os.path.exists(template_file): + raise ValueError( + 'The provided template file %s doesn\'t ' + 'exist' % template_file, + ) + with open(template_file, 'r') as f: + return f.read() + + +def create_cmd(args, conf): "Create a new release note file from the template." - notesdir = utils.get_notes_dir(args) # NOTE(dhellmann): There is a short race window where we might try # to pick a name that does not exist, then overwrite the file if # it is created before we try to write it. This isn't a problem @@ -122,7 +66,14 @@ def create_cmd(args): # their local git tree, and so there should not be any concurrency # concern. slug = args.slug.replace(' ', '-') - filename = _pick_note_file_name(notesdir, slug) - _make_note_file(filename) + filename = _pick_note_file_name(conf.notespath, slug) + if args.from_template: + template = _get_user_template(args.from_template) + else: + template = conf.template + _make_note_file(filename, template) + if args.edit and not _edit_file(filename): + print('Was unable to edit the new note. EDITOR environment variable ' + 'is missing!') print('Created new notes file in %s' % filename) return diff --git a/reno/defaults.py b/reno/defaults.py index b30a18d5a390d31ce739b5d16a838baff97ac4cd..ded78f2a9e1cef89916ccea17fae6ed592cb7a17 100644 --- a/reno/defaults.py +++ b/reno/defaults.py @@ -11,4 +11,82 @@ # under the License. RELEASE_NOTES_SUBDIR = 'releasenotes' + NOTES_SUBDIR = 'notes' + +PRELUDE_SECTION_NAME = 'prelude' + +# This is a format string, so it needs to be formatted wherever it is used. +TEMPLATE = """\ +--- +{0}: > + Replace this text with content to appear at the top of the section for this + release. All of the prelude content is merged together and then rendered + separately from the items listed in other parts of the file, so the text + needs to be worded so that both the prelude and the other items make sense + when read independently. This may mean repeating some details. Not every + release note requires a prelude. Usually only notes describing major + features or adding release theme details should have a prelude. +features: + - | + List new features here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +issues: + - | + List known issues here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +upgrade: + - | + List upgrade notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +deprecations: + - | + List deprecations notes here, or remove this section. All of the list + items in this section are combined when the release notes are rendered, so + the text needs to be worded so that it does not depend on any information + only available in another section, such as the prelude. This may mean + repeating some details. +critical: + - | + Add critical notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +security: + - | + Add security notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +fixes: + - | + Add normal bug fixes here, or remove this section. All of the list items + in this section are combined when the release notes are rendered, so the + text needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +other: + - | + Add other notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +""" + +# default filename of a release notes file generated by the setuptool extension +RELEASE_NOTES_FILENAME = 'RELEASENOTES.rst' + +# default path to the root of the repo, used by the setuptools extension +REPO_ROOT = '.' diff --git a/reno/formatter.py b/reno/formatter.py index 10112f0bf82e583383ebf29d9b53deb206df1011..a4f87420898b43de49b0e5f0b183a3cbc53c12c3 100644 --- a/reno/formatter.py +++ b/reno/formatter.py @@ -12,22 +12,6 @@ from __future__ import print_function -from reno import scanner - -import yaml - - -_SECTION_ORDER = [ - ('features', 'New Features'), - ('issues', 'Known Issues'), - ('upgrade', 'Upgrade Notes'), - ('deprecations', 'Deprecation Notes'), - ('critical', 'Critical Issues'), - ('security', 'Security Issues'), - ('fixes', 'Bug Fixes'), - ('other', 'Other Notes'), -] - def _indent_for_list(text, prefix=' '): """Indent some text to make it work as a list entry. @@ -41,7 +25,26 @@ def _indent_for_list(text, prefix=' '): ]) + '\n' -def format_report(reporoot, scanner_output, versions_to_include, title=None): +def _anchor(version_title, title, branch): + title = title or 'relnotes' + return '.. _{title}_{version_title}{branch}:'.format( + title=title, + version_title=version_title, + branch=('_' + branch.replace('/', '_') if branch else ''), + ) + + +def _section_anchor(section_title, version_title, title, branch): + # Get the title and remove the trailing : + title = _anchor(version_title, title, branch)[:-1] + return "{title}_{section_title}:".format( + title=title, + section_title=section_title, + ) + + +def format_report(loader, config, versions_to_include, title=None, + show_source=True, branch=None): report = [] if title: report.append('=' * len(title)) @@ -52,38 +55,60 @@ def format_report(reporoot, scanner_output, versions_to_include, title=None): # Read all of the notes files. file_contents = {} for version in versions_to_include: - for filename, sha in scanner_output[version]: - body = scanner.get_file_at_commit( - reporoot, - filename, - sha, - ) - y = yaml.safe_load(body) - file_contents[filename] = y + for filename, sha in loader[version]: + body = loader.parse_note_file(filename, sha) + file_contents[filename] = body for version in versions_to_include: - report.append(version) - report.append('=' * len(version)) + if '-' in version: + # This looks like an "unreleased version". + version_title = config.unreleased_version_title or version + else: + version_title = version + report.append(_anchor(version_title, title, branch)) + report.append('') + report.append(version_title) + report.append('=' * len(version_title)) report.append('') # Add the preludes. - notefiles = scanner_output[version] - for n, sha in notefiles: - if 'prelude' in file_contents[n]: - report.append(file_contents[n]['prelude']) - report.append('') + notefiles = loader[version] + prelude_name = config.prelude_section_name + notefiles_with_prelude = [(n, sha) for n, sha in notefiles + if prelude_name in file_contents[n]] + if notefiles_with_prelude: + prelude_title = prelude_name.replace('_', ' ').title() + report.append(_section_anchor( + prelude_title, version_title, title, branch)) + report.append('') + report.append(prelude_title) + report.append('-' * len(prelude_name)) + report.append('') + + for n, sha in notefiles_with_prelude: + if show_source: + report.append('.. %s @ %s\n' % (n, sha)) + report.append(file_contents[n][prelude_name]) + report.append('') - for section_name, section_title in _SECTION_ORDER: + # Add other sections. + for section_name, section_title in config.sections: notes = [ - n + (n, fn, sha) for fn, sha in notefiles + if file_contents[fn].get(section_name) for n in file_contents[fn].get(section_name, []) ] if notes: + report.append(_section_anchor( + section_title, version_title, title, branch)) + report.append('') report.append(section_title) report.append('-' * len(section_title)) report.append('') - for n in notes: + for n, fn, sha in notes: + if show_source: + report.append('.. %s @ %s\n' % (fn, sha)) report.append('- %s' % _indent_for_list(n)) report.append('') diff --git a/reno/linter.py b/reno/linter.py new file mode 100644 index 0000000000000000000000000000000000000000..9cbda996e0a9971da511fd787097f0314823c70b --- /dev/null +++ b/reno/linter.py @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import glob +import logging +import os.path + +from reno import loader +from reno import scanner + +LOG = logging.getLogger(__name__) + + +def lint_cmd(args, conf): + "Check some common mistakes" + LOG.debug('starting lint') + notesdir = os.path.join(conf.reporoot, conf.notespath) + notes = glob.glob(os.path.join(notesdir, '*.yaml')) + + error = 0 + load = loader.Loader(conf, ignore_cache=True) + allowed_section_names = [conf.prelude_section_name] + \ + [s[0] for s in conf.sections] + + uids = {} + for f in notes: + LOG.debug('examining %s', f) + uid = scanner._get_unique_id(f) + uids.setdefault(uid, []).append(f) + + content = load.parse_note_file(f, None) + for section_name in content.keys(): + if section_name not in allowed_section_names: + LOG.warning('unrecognized section name %s in %s', + section_name, f) + error = 1 + + for uid, names in sorted(uids.items()): + if len(names) > 1: + LOG.warning('UID collision: %s', names) + error = 1 + + return error diff --git a/reno/lister.py b/reno/lister.py index b60267aef1fdbe67dfd7dc9fb8718c279576cc36..aa1ec4d17a484bd21f573f8508c444f7a33760d3 100644 --- a/reno/lister.py +++ b/reno/lister.py @@ -14,24 +14,22 @@ from __future__ import print_function import logging -from reno import scanner -from reno import utils +from reno import loader LOG = logging.getLogger(__name__) -def list_cmd(args): +def list_cmd(args, conf): "List notes files based on query arguments" LOG.debug('starting list') - reporoot = args.reporoot.rstrip('/') + '/' - notesdir = utils.get_notes_dir(args) - notes = scanner.get_notes_by_version(reporoot, notesdir, args.branch) + reporoot = conf.reporoot + ldr = loader.Loader(conf) if args.version: versions = args.version else: - versions = notes.keys() + versions = ldr.versions for version in versions: - notefiles = notes[version] + notefiles = ldr[version] print(version) for n, sha in notefiles: if n.startswith(reporoot): diff --git a/reno/loader.py b/reno/loader.py new file mode 100644 index 0000000000000000000000000000000000000000..b0b2a03c5e9e8783dd474c90086274ddb60c4227 --- /dev/null +++ b/reno/loader.py @@ -0,0 +1,142 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import logging +import os.path + +import six +import yaml + +from reno import scanner + +LOG = logging.getLogger(__name__) + + +def get_cache_filename(conf): + return os.path.normpath(os.path.join( + conf.reporoot, conf.notespath, 'reno.cache')) + + +class Loader(object): + "Load the release notes for a given repository." + + def __init__(self, conf, + ignore_cache=False): + """Initialize a Loader. + + The versions are presented in reverse chronological order. + + Notes files are associated with the earliest version for which + they were available, regardless of whether they changed later. + + :param conf: Parsed configuration from file + :type conf: reno.config.Config + :param ignore_cache: Do not load a cache file if it is present. + :type ignore_cache: bool + """ + self._config = conf + self._ignore_cache = ignore_cache + + self._reporoot = conf.reporoot + self._notespath = conf.notespath + self._branch = conf.branch + self._collapse_pre_releases = conf.collapse_pre_releases + self._earliest_version = conf.earliest_version + + self._cache = None + self._scanner = None + self._scanner_output = None + self._cache_filename = get_cache_filename(conf) + + self._load_data() + + def _load_data(self): + cache_file_exists = os.path.exists(self._cache_filename) + + if self._ignore_cache and cache_file_exists: + LOG.debug('ignoring cache file %s', self._cache_filename) + + if (not self._ignore_cache) and cache_file_exists: + LOG.debug('loading cache file %s', self._cache_filename) + with open(self._cache_filename, 'r') as f: + self._cache = yaml.safe_load(f.read()) + # Save the cached scanner output to the same attribute + # it would be in if we had loaded it "live". This + # simplifies some of the logic in the other methods. + self._scanner_output = collections.OrderedDict( + (n['version'], n['files']) + for n in self._cache['notes'] + ) + else: + self._scanner = scanner.Scanner(self._config) + self._scanner_output = self._scanner.get_notes_by_version() + + @property + def versions(self): + "A list of all of the versions found." + return list(self._scanner_output.keys()) + + def __getitem__(self, version): + "Return data about the files that should go into a given version." + return self._scanner_output[version] + + def parse_note_file(self, filename, sha): + """Return the data structure encoded in the note file. + + Emit warnings for content that does not look valid in some + way, but return it anyway for backwards-compatibility. + + """ + if self._cache: + content = self._cache['file-contents'][filename] + else: + body = self._scanner.get_file_at_commit(filename, sha) + content = yaml.safe_load(body) + + cleaned_content = {} + + for section_name, section_content in content.items(): + if section_name == self._config.prelude_section_name: + if not isinstance(section_content, six.string_types): + LOG.warning( + ('The %s section of %s ' + 'does not parse as a single string. ' + 'Is the YAML input escaped properly?') % + (self._config.prelude_section_name, filename), + ) + else: + if isinstance(section_content, six.string_types): + # A single string is OK, but wrap it with a list + # so the rest of the code can treat the data model + # consistently. + section_content = [section_content] + elif not isinstance(section_content, list): + LOG.warning( + ('The %s section of %s ' + 'does not parse as a string or list of strings. ' + 'Is the YAML input escaped properly?') % ( + section_name, filename), + ) + else: + for item in section_content: + if not isinstance(item, six.string_types): + LOG.warning( + ('The item %r in the %s section of %s ' + 'parses as a %s instead of a string. ' + 'Is the YAML input escaped properly?' + ) % (item, section_name, + filename, type(item)), + ) + cleaned_content[section_name] = section_content + + return cleaned_content diff --git a/reno/main.py b/reno/main.py index 7f4cfb5c1de1defc83dca29a59d6b1d0e5345518..cb2771289e10dfa88c478b62f522b782c88001b2 100644 --- a/reno/main.py +++ b/reno/main.py @@ -14,11 +14,54 @@ import argparse import logging import sys +from reno import cache +from reno import config from reno import create from reno import defaults +from reno import linter from reno import lister from reno import report +_query_args = [ + (('--version',), + dict(default=[], + action='append', + help='the version(s) to include, defaults to all')), + (('--branch',), + dict(default=config.Config.get_default('branch'), + help='the branch to scan, defaults to the current')), + (('--collapse-pre-releases',), + dict(action='store_true', + default=None, + help='combine pre-releases with their final release')), + (('--no-collapse-pre-releases',), + dict(action='store_false', + dest='collapse_pre_releases', + help='show pre-releases separately')), + (('--earliest-version',), + dict(default=None, + help='stop when this version is reached in the history')), + (('--ignore-cache',), + dict(default=None, + action='store_true', + help='if there is a cache file present, do not use it')), + (('--stop-at-branch-base',), + dict(action='store_true', + default=None, + dest='stop_at_branch_base', + help='stop scanning when the branch meets master')), + (('--no-stop-at-branch-base',), + dict(action='store_false', + dest='stop_at_branch_base', + help='do not stop scanning when the branch meets master')), +] + + +def _build_query_arg_group(parser): + group = parser.add_argument_group('query') + for args, kwds in _query_args: + group.add_argument(*args, **kwds) + def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser() @@ -45,37 +88,47 @@ def main(argv=sys.argv[1:]): ) subparsers = parser.add_subparsers( title='commands', + description='valid commands', + dest='command', + help='additional help', ) do_new = subparsers.add_parser( 'new', help='create a new note', ) + do_new.add_argument( + '--edit', + action='store_true', + help='Edit note after its creation (require EDITOR env variable)', + ) + do_new.add_argument( + '--from-template', + help='Template to get the release note from.', + ) do_new.add_argument( 'slug', help='descriptive title of note (keep it short)', ) + do_new.add_argument( + 'reporoot', + default='.', + nargs='?', + help='root of the git repository', + ) do_new.set_defaults(func=create.create_cmd) do_list = subparsers.add_parser( 'list', help='list notes files based on query arguments', ) + _build_query_arg_group(do_list) do_list.add_argument( 'reporoot', + default='.', + nargs='?', help='root of the git repository', ) - do_list.add_argument( - '--version', - default=[], - action='append', - help='the version(s) to include, defaults to all', - ) - do_list.add_argument( - '--branch', - default=None, - help='the branch to scan, defaults to the current', - ) do_list.set_defaults(func=lister.list_cmd) do_report = subparsers.add_parser( @@ -84,6 +137,8 @@ def main(argv=sys.argv[1:]): ) do_report.add_argument( 'reporoot', + default='.', + nargs='?', help='root of the git repository', ) do_report.add_argument( @@ -92,23 +147,64 @@ def main(argv=sys.argv[1:]): help='output filename, defaults to stdout', ) do_report.add_argument( - '--branch', - default=None, - help='the branch to scan, defaults to the current', + '--no-show-source', + dest='show_source', + default=True, + action='store_false', + help='do not show the source for notes', ) do_report.add_argument( - '--version', - default=[], - action='append', - help='the version(s) to include, defaults to all', + '--title', + default='Release Notes', + help='set the main title of the generated report', ) + _build_query_arg_group(do_report) do_report.set_defaults(func=report.report_cmd) - args = parser.parse_args() + do_cache = subparsers.add_parser( + 'cache', + help='generate release notes cache', + ) + do_cache.add_argument( + 'reporoot', + default='.', + nargs='?', + help='root of the git repository', + ) + do_cache.add_argument( + '--output', '-o', + default=None, + help=('output filename, ' + 'defaults to the cache file within the notesdir, ' + 'use "-" for stdout'), + ) + _build_query_arg_group(do_cache) + do_cache.set_defaults(func=cache.cache_cmd) + + do_linter = subparsers.add_parser( + 'lint', + help='check some common mistakes', + ) + do_linter.add_argument( + 'reporoot', + default='.', + nargs='?', + help='root of the git repository', + ) + do_linter.set_defaults(func=linter.lint_cmd) + + args = parser.parse_args(argv) + # no arguments, print help messaging, then exit with error(1) + if not args.command: + parser.print_help() + return 1 logging.basicConfig( level=args.verbosity, format='%(message)s', ) - return args.func(args) + conf = config.Config(args.reporoot, args.relnotesdir) + conf.override_from_parsed_args(args) + + return args.func(args, conf) diff --git a/reno/report.py b/reno/report.py index 8f37c09e0bfa4841c22c3a4bc6fedc41d16ed0b2..a6b3e6af157db83b70c6c93ea99358d4cd500928 100644 --- a/reno/report.py +++ b/reno/report.py @@ -13,24 +13,23 @@ from __future__ import print_function from reno import formatter -from reno import scanner -from reno import utils +from reno import loader -def report_cmd(args): +def report_cmd(args, conf): "Generates a release notes report" - reporoot = args.reporoot.rstrip('/') + '/' - notesdir = utils.get_notes_dir(args) - notes = scanner.get_notes_by_version(reporoot, notesdir, args.branch) + ldr = loader.Loader(conf) if args.version: versions = args.version else: - versions = notes.keys() + versions = ldr.versions text = formatter.format_report( - reporoot, - notes, + ldr, + conf, versions, - title='Release Notes', + title=args.title, + show_source=args.show_source, + branch=args.branch, ) if args.output: with open(args.output, 'w') as f: diff --git a/reno/scanner.py b/reno/scanner.py index e8593da8e5adef7837703b87f5b55d2b9142d37c..da7f88aadf18cedee4534b776265a19e95460af6 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -17,57 +17,28 @@ import fnmatch import logging import os.path import re -import subprocess import sys -from reno import utils +from dulwich import diff_tree +from dulwich import index as d_index +from dulwich import objects +from dulwich import porcelain +from dulwich import repo -_TAG_PAT = re.compile('tag: ([\d\.]+)') LOG = logging.getLogger(__name__) -def _get_current_version(reporoot, branch=None): - """Return the current version of the repository. - - If the repo appears to contain a python project, use setup.py to - get the version so pbr (if used) can do its thing. Otherwise, use - git describe. - - """ - cmd = ['git', 'describe', '--tags'] - if branch is not None: - cmd.append(branch) - try: - result = utils.check_output(cmd, cwd=reporoot).strip() - if '-' in result: - # Descriptions that come after a commit look like - # 2.0.0-1-abcde, and we want to remove the SHA value from - # the end since we only care about the version number - # itself, but we need to recognize that the change is - # unreleased so keep the -1 part. - result, dash, ignore = result.rpartition('-') - except subprocess.CalledProcessError: - # This probably means there are no tags. - result = '0.0.0' +def _parse_version(v): + parts = v.split('.') + ['0', '0', '0'] + result = [] + for p in parts[:3]: + try: + result.append(int(p)) + except ValueError: + result.append(p) return result -def get_file_at_commit(reporoot, filename, sha): - "Return the contents of the file if it exists at the commit, or None." - try: - return utils.check_output( - ['git', 'show', '%s:%s' % (sha, filename)], - cwd=reporoot, - ) - except subprocess.CalledProcessError: - return None - - -def _file_exists_at_commit(reporoot, filename, sha): - "Return true if the file exists at the given commit." - return bool(get_file_at_commit(reporoot, filename, sha)) - - def _get_unique_id(filename): base = os.path.basename(filename) root, ext = os.path.splitext(base) @@ -79,216 +50,1183 @@ def _get_unique_id(filename): return uniqueid -# The git log output from _get_tags_on_branch() looks like this sample -# from the openstack/nova repository for git 1.9.1: -# -# git log --simplify-by-decoration --pretty="%d" -# (HEAD, origin/master, origin/HEAD, gerrit/master, master) -# (apu/master) -# (tag: 13.0.0.0b1) -# (tag: 12.0.0.0rc3, tag: 12.0.0) -# -# And this for git 1.7.1 (RHEL): -# -# $ git log --simplify-by-decoration --pretty="%d" -# (HEAD, origin/master, origin/HEAD, master) -# (tag: 13.0.0.0b1) -# (tag: 12.0.0.0rc3, tag: 12.0.0) -# (tag: 12.0.0.0rc2) -# (tag: 2015.1.0rc3, tag: 2015.1.0) -# ... -# (tag: folsom-2) -# (tag: folsom-1) -# (essex-1) -# (diablo-2) -# (diablo-1) -# (2011.2) -# -# The difference in the tags with "tag:" and without appears to be -# caused by some being annotated and others not. -# -# So we might have multiple tags on a given commit, and we might have -# lines that have no tags or are completely blank, and we might have -# "tag:" or not. This pattern is used to find the tag entries on each -# line, ignoring tags that don't look like version numbers. -TAG_RE = re.compile('(?:[(]|tag: )([\d.ab]+)[,)]') +def _note_file(name): + """Return bool indicating if the filename looks like a note file. + This is used to filter the files in changes based on the notes + directory we were given. We cannot do this in the walker directly + because it means we end up skipping some of the tags if the + commits being tagged don't include any release note files. -def _get_version_tags_on_branch(reporoot, branch): - """Return tags from the branch, in date order. + """ + if not name: + return False + if fnmatch.fnmatch(name, '*.yaml'): + return True + else: + LOG.info('found and ignored extra file %s', name) + return False - Need to get the list of tags in the right order, because the topo - search breaks the date ordering. Use git to ask for the tags in - order, rather than trying to sort them, because many repositories - have "non-standard" tags or have renumbered projects (from - date-based to SemVer), for which sorting would require complex - logic. + +def _changes_in_subdir(repo, walk_entry, subdir): + """Iterator producing changes of interest to reno. + + The default changes() method of a WalkEntry computes all of the + changes in the entire repo at that point. We only care about + changes in a subdirectory, so this reimplements + WalkeEntry.changes() with that filter in place. + + The alternative, passing paths to the TreeWalker, does not work + because we need all of the commits in sequence so we can tell when + the tag changes. We have to look at every commit to see if it + either has a tag, a note file, or both. + + NOTE(dhellmann): The TreeChange entries returned as a result of + the manipulation done by this function have the subdir prefix + stripped. """ - tags = [] - tag_cmd = [ - 'git', 'log', - '--simplify-by-decoration', - '--pretty="%d"', - ] - if branch: - tag_cmd.append(branch) - LOG.debug('running %s' % ' '.join(tag_cmd)) - tag_results = utils.check_output(tag_cmd, cwd=reporoot) - LOG.debug(tag_results) - for line in tag_results.splitlines(): - LOG.debug('line %r' % line) - for match in TAG_RE.findall(line): - tags.append(match) - return tags - - -def get_notes_by_version(reporoot, notesdir, branch=None): - """Return an OrderedDict mapping versions to lists of notes files. - - The versions are presented in reverse chronological order. - - Notes files are associated with the earliest version for which - they were available, regardless of whether they changed later. - """ + commit = walk_entry.commit + store = repo.object_store + + parents = walk_entry._get_parents(commit) - LOG.debug('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch)) - - # Determine all of the tags known on the branch, in their date - # order. We scan the commit history in topological order to ensure - # we have the commits in the right version, so we might encounter - # the tags in a different order during that phase. - versions_by_date = _get_version_tags_on_branch(reporoot, branch) - LOG.debug('versions by date %r' % (versions_by_date,)) - versions = [] - earliest_seen = collections.OrderedDict() - - # Determine the current version, which might be an unreleased or - # dev version if there are unreleased commits at the head of the - # branch in question. Since the version may not already be known, - # make sure it is in the list of versions by date. And since it is - # the most recent version, go ahead and insert it at the front of - # the list. - current_version = _get_current_version(reporoot, branch) - LOG.debug('current repository version: %s' % current_version) - if current_version not in versions_by_date: - LOG.debug('adding %s to versions by date' % current_version) - versions_by_date.insert(0, current_version) - - # Remember the most current filename for each id, to allow for - # renames. - last_name_by_id = {} - - # FIXME(dhellmann): This might need to be more line-oriented for - # longer histories. - log_cmd = [ - 'git', 'log', - '--topo-order', # force traversal order rather than date order - '--pretty=%x00%H %d', # output contents in parsable format - '--name-only' # only include the names of the files in the patch - ] - if branch is not None: - log_cmd.append(branch) - LOG.debug('running %s' % ' '.join(log_cmd)) - history_results = utils.check_output(log_cmd, cwd=reporoot) - history = history_results.split('\x00') - current_version = current_version - for h in history: - h = h.strip() - if not h: - continue - # print(h) - - hlines = h.splitlines() - - # The first line of the block will include the SHA and may - # include tags, the other lines are filenames. - sha = hlines[0].split(' ')[0] - tags = _TAG_PAT.findall(hlines[0]) - # Filter the files based on the notes directory we were - # given. We cannot do this in the git log command directly - # because it means we end up skipping some of the tags if the - # commits being tagged don't include any release note - # files. Even if this list ends up empty, we continue doing - # the other processing so that we record all of the known - # versions. - filenames = [ - f - for f in hlines[2:] - if fnmatch.fnmatch(f, notesdir + '/*.yaml') + if not parents: + changes_func = diff_tree.tree_changes + parent_subtree = None + elif len(parents) == 1: + changes_func = diff_tree.tree_changes + parent_tree = repo[repo[parents[0]].tree] + parent_subtree = repo._get_subtree(parent_tree, subdir) + if parent_subtree: + parent_subtree = parent_subtree.sha().hexdigest().encode('ascii') + else: + changes_func = diff_tree.tree_changes_for_merge + parent_subtree = [ + repo._get_subtree(repo[repo[p].tree], subdir) + for p in parents + ] + parent_subtree = [ + p.sha().hexdigest().encode('ascii') + for p in parent_subtree + if p ] + subdir_tree = repo._get_subtree(repo[commit.tree], subdir) + if subdir_tree: + commit_subtree = subdir_tree.sha().hexdigest().encode('ascii') + else: + commit_subtree = None + if parent_subtree == commit_subtree: + return [] + return changes_func(store, parent_subtree, commit_subtree) + + +class _ChangeAggregator(object): + """Collapse a series of changes based on uniqueness for file uids. + + The list of TreeChange instances describe changes between the old + and new repository trees. The change has a type, and new and old + paths and shas. + + Simple add, delete, and change operations are handled directly. + + There is a rename type, but detection of renamed files is + incomplete so we handle that ourselves based on the UID value + built into the filenames (under the assumption that if someone + changes that part of the filename they want it treated as a + different file for some reason). If we see both an add and a + delete for a given UID treat that as a rename. + + The SHA values returned are for the commit, rather than the blob + values in the TreeChange objects. + + The path values in the change entries are encoded, so we return + the decoded values to make consuming them easier. + + """ + + _rename_op = set([diff_tree.CHANGE_ADD, diff_tree.CHANGE_DELETE]) + _modify_op = set([diff_tree.CHANGE_MODIFY]) + _delete_op = set([diff_tree.CHANGE_DELETE]) + _add_op = set([diff_tree.CHANGE_ADD]) + + def __init__(self): + # Track UIDs that had a duplication issue but have been + # deleted so we know not to throw an error for them. + self._deleted_bad_uids = set() + + def aggregate_changes(self, walk_entry, changes): + sha = walk_entry.commit.id + by_uid = collections.defaultdict(list) + for ec in changes: + if not isinstance(ec, list): + ec = [ec] + else: + ec = ec + for c in ec: + LOG.debug('change %r', c) + if c.type == diff_tree.CHANGE_ADD: + path = c.new.path.decode('utf-8') if c.new.path else None + if _note_file(path): + uid = _get_unique_id(path) + by_uid[uid].append((c.type, path, sha)) + else: + LOG.debug('ignoring') + elif c.type == diff_tree.CHANGE_DELETE: + path = c.old.path.decode('utf-8') if c.old.path else None + if _note_file(path): + uid = _get_unique_id(path) + by_uid[uid].append((c.type, path, sha)) + else: + LOG.debug('ignoring') + elif c.type == diff_tree.CHANGE_MODIFY: + path = c.new.path.decode('utf-8') if c.new.path else None + if _note_file(path): + uid = _get_unique_id(path) + by_uid[uid].append((c.type, path, sha)) + else: + LOG.debug('ignoring') + else: + raise ValueError('unhandled change type: {!r}'.format(c)) + + results = [] + for uid, changes in sorted(by_uid.items()): + if len(changes) == 1: + results.append((uid,) + changes[0]) + else: + types = set(c[0] for c in changes) + if types == self._rename_op: + # A rename, combine the data from the add and + # delete entries. + added = [ + c for c in changes if c[0] == diff_tree.CHANGE_ADD + ][0] + deled = [ + c for c in changes if c[0] == diff_tree.CHANGE_DELETE + ][0] + results.append( + (uid, diff_tree.CHANGE_RENAME, deled[1]) + added[1:] + ) + elif types == self._modify_op: + # Merge commit with modifications to the same files in + # different commits. + for c in changes: + results.append((uid, diff_tree.CHANGE_MODIFY, + c[1], sha)) + elif types == self._delete_op: + # There were multiple files in one commit using the + # same UID but different slugs. Treat them as + # different files and allow them to be deleted. + results.extend( + (uid, diff_tree.CHANGE_DELETE, c[1], sha) + for c in changes + ) + self._deleted_bad_uids.add(uid) + elif types == self._add_op: + # There were multiple files in one commit using the + # same UID but different slugs. Warn the user about + # this case and then ignore the files. We allow delete + # (see above) to ensure they can be cleaned up. + msg = ('%s: found several files in one commit (%s)' + ' with the same UID: %s' % + (uid, sha, [c[1] for c in changes])) + if uid not in self._deleted_bad_uids: + raise ValueError(msg) + else: + LOG.info(msg) + else: + raise ValueError('Unrecognized changes: {!r}'.format( + changes)) + return results + + +class _ChangeTracker(object): + + def __init__(self): + # Track the versions we have seen and the earliest version for + # which we have seen a given note's unique id. + self.versions = [] + self.earliest_seen = collections.OrderedDict() + # Remember the most current filename for each id, to allow for + # renames. + self.last_name_by_id = {} + # Remember uniqueids that have had files deleted. + self.uniqueids_deleted = set() + # Remember files that are changed but not explicitly added so + # when we do see an add we can use the more recent tracking + # info and if we don't see the add we know to ignore the file. + self.seen_but_not_added = {} - # If there are no tags in this block, assume the most recently - # seen version. - if not tags: - tags = [current_version] + def _common(self, uniqueid, sha, version): + if version not in self.versions: + self.versions.append(version) + # Update the "earliest" version where a UID appears + # every time we see it, because we are scanning the + # history in reverse order so "early" items come + # later. + if uniqueid in self.earliest_seen: + LOG.debug('%s: resetting earliest reference from %s to %s for %s', + uniqueid, self.earliest_seen[uniqueid], version, sha) else: - current_version = tags[0] - LOG.debug('%s has tags, updating current version to %s' % - (sha, current_version)) - - # Remember each version we have seen. - if current_version not in versions: - LOG.debug('%s is a new version' % current_version) - versions.append(current_version) - - LOG.debug('%s contains files %s' % (sha, filenames)) - - # Remember the files seen, using their UUID suffix as a unique id. - for f in filenames: - # Updated as older tags are found, handling edits to release - # notes. - LOG.debug('setting earliest reference to %s to %s' % - (f, tags[0])) - uniqueid = _get_unique_id(f) - earliest_seen[uniqueid] = tags[0] - if uniqueid in last_name_by_id: - # We already have a filename for this id from a - # new commit, so use that one in case the name has - # changed. - LOG.debug('%s was seen before' % f) - continue - if _file_exists_at_commit(reporoot, f, sha): - # Remember this filename as the most recent version of - # the unique id we have seen, in case the name - # changed from an older commit. - last_name_by_id[uniqueid] = (f, sha) - LOG.debug('remembering %s as filename for %s' % (f, uniqueid)) - - # Invert earliest_seen to make a list of notes files for each - # version. - files_and_tags = collections.OrderedDict() - for v in versions: - files_and_tags[v] = [] - # Produce a list of the actual files present in the repository. If - # a note is removed, this step should let us ignore it. - for uniqueid, version in earliest_seen.items(): + LOG.debug('%s: setting earliest reference to %s for %s', + uniqueid, version, sha) + self.earliest_seen[uniqueid] = version + + def add(self, filename, sha, version): + uniqueid = _get_unique_id(filename) + self._common(uniqueid, sha, version) + LOG.info('%s: adding %s from %s', + uniqueid, filename, version) + + # If we have recorded that a UID was deleted, that + # means that was the last change made to the file and + # we can ignore it. + if uniqueid in self.uniqueids_deleted: + LOG.debug( + '%s: has already been deleted, ignoring this change', + uniqueid, + ) + return + + if uniqueid in self.seen_but_not_added: + # The note was seen for a modify operation already but + # this is where it was added. We want to remember where + # the modification happened, because that came earlier in + # the scan (and therefore later in the history). + filename, sha = self.seen_but_not_added[uniqueid] + self.last_name_by_id[uniqueid] = (filename, sha) + LOG.info( + '%s: copying data for %s from commit %s', + uniqueid, filename, sha, + ) + del self.seen_but_not_added[uniqueid] + elif uniqueid not in self.last_name_by_id: + # The note was added already and we want to keep that + # other reference because it came earlier in the scan (and + # therefore later in the history). + self.last_name_by_id[uniqueid] = (filename, sha) + LOG.debug( + '%s: new %s in commit %s', + uniqueid, filename, sha, + ) + else: + LOG.debug( + '%s: add for file we have already seen', + uniqueid, + ) + + def _change(self, filename, sha, version): + uniqueid = _get_unique_id(filename) + self._common(uniqueid, sha, version) + + # If we have recorded that a UID was deleted, that + # means that was the last change made to the file and + # we can ignore it. + if uniqueid in self.uniqueids_deleted: + LOG.debug( + '%s: has already been deleted, ignoring this change', + uniqueid, + ) + return + + if uniqueid in self.last_name_by_id: + LOG.debug('%s: already added', uniqueid) + to_update = self.last_name_by_id + else: + LOG.debug('%s: seen but not added', uniqueid) + to_update = self.seen_but_not_added + + # The file is being renamed. We may have seen it + # before, if there were subsequent modifications, + # so only store the name information if it is not + # there already. + if uniqueid not in to_update: + to_update[uniqueid] = (filename, sha) + LOG.info( + '%s: update to %s in commit %s', + uniqueid, filename, sha, + ) + else: + LOG.debug( + '%s: modified file already known', + uniqueid, + ) + + def rename(self, filename, sha, version): + self._change(filename, sha, version) + + def modify(self, filename, sha, version): + self._change(filename, sha, version) + + def delete(self, filename, sha, version): + uniqueid = _get_unique_id(filename) + self._common(uniqueid, sha, version) + # This file is being deleted without a rename. If + # we have already seen the UID before, that means + # that after the file was deleted another file + # with the same UID was added back. In that case + # we do not want to treat it as deleted. + # + # Never store deleted files in last_name_by_id so + # we can safely use all of those entries to build + # the history data. + if uniqueid not in self.last_name_by_id: + self.uniqueids_deleted.add(uniqueid) + LOG.info( + '%s: note deleted in %s', + uniqueid, sha, + ) + else: + LOG.debug( + '%s: delete for file re-added after the delete', + uniqueid, + ) + + +class RenoRepo(repo.Repo): + + # Populated by _load_tags(). + _all_tags = None + _shas_to_tags = None + + def _get_commit_from_tag(self, tag, tag_sha): + """Return the commit referenced by the tag and when it was tagged.""" + tag_obj = self[tag_sha] + + if isinstance(tag_obj, objects.Tag): + # A signed tag has its own SHA, but the tag refers to + # the commit and that's the SHA we'll see when we scan + # commits on a branch. + git_obj = tag_obj + while True: + # Tags can point to other tags, in such cases follow the chain + # of tags until there are no more. + child_obj = self[git_obj.object[1]] + if isinstance(child_obj, objects.Tag): + git_obj = child_obj + else: + break + + tagged_sha = git_obj.object[1] + date = tag_obj.tag_time + elif isinstance(tag_obj, objects.Commit): + # Unsigned tags refer directly to commits. This seems + # to especially happen when the tag definition moves + # to the packed-refs list instead of being represented + # by its own file. + tagged_sha = tag_obj.id + date = tag_obj.commit_time + else: + raise ValueError( + ('Unrecognized tag object {!r} with ' + 'tag {} and SHA {!r}: {}').format( + tag_obj, tag, tag_sha, type(tag_obj)) + ) + return tagged_sha, date + + def _load_tags(self): + self._all_tags = { + k.partition(b'/tags/')[-1].decode('utf-8'): v + for k, v in self.get_refs().items() + if k.startswith(b'refs/tags/') + } + self._shas_to_tags = {} + for tag, tag_sha in self._all_tags.items(): + tagged_sha, date = self._get_commit_from_tag(tag, tag_sha) + self._shas_to_tags.setdefault(tagged_sha, []).append((tag, date)) + + def get_tags_on_commit(self, sha): + "Return the tag(s) on a commit, in application order." + if self._all_tags is None: + self._load_tags() + tags_and_dates = self._shas_to_tags.get(sha, []) + tags_and_dates.sort(key=lambda x: x[1]) + return [t[0] for t in tags_and_dates] + + def _get_subtree(self, tree, path): + "Given a tree SHA and a path, return the SHA of the subtree." try: - base, sha = last_name_by_id[uniqueid] - files_and_tags[version].append((base, sha)) + mode, tree_sha = tree.lookup_path(self.get_object, + path.encode('utf-8')) except KeyError: - # Unable to find the file again, skip it to avoid breaking - # the build. - msg = ('[reno] unable to find file associated ' - 'with unique id %r, skipping') % uniqueid - LOG.debug(msg) - print(msg, file=sys.stderr) - - # Only return the parts of files_and_tags that actually have - # filenames associated with the versions. - trimmed = collections.OrderedDict() - for ov in versions_by_date: - if not files_and_tags.get(ov): - continue - # Sort the notes associated with the version so they are in a - # deterministic order, to avoid having the same data result in - # different output depending on random factors. Earlier - # versions of the scanner assumed the notes were recorded in - # chronological order based on the commit date, but with the - # change to use topological sorting that is no longer - # necessarily true. We want the notes to always show up in the - # same order, but it doesn't really matter what order that is, - # so just sort based on the unique id. - trimmed[ov] = sorted(files_and_tags[ov]) - - return trimmed + # Some part of the path wasn't found, so the subtree is + # not present. Return the sentinel value. + return None + else: + tree = self[tree_sha] + return tree + + def get_file_at_commit(self, filename, sha): + """Return the contents of the file. + + If sha is None, return the working copy of the file. If the + file cannot be read from the working dir, return None. + + If the sha is not None and the file exists at the commit, + return the data from the stored blob. If the file does not + exist at the commit, return None. + + """ + if sha is None: + # Get the copy from the working directory. + try: + with open(os.path.join(self.path, filename), 'r') as f: + return f.read() + except IOError: + return None + # Get the tree associated with the commit identified by the + # input SHA, then look through the items in the tree to find + # the one with the path matching the filename. Take the + # associated SHA from the tree and get the file contents from + # the repository. + if hasattr(sha, 'encode'): + sha = sha.encode('ascii') + commit = self[sha] + tree = self[commit.tree] + try: + mode, blob_sha = tree.lookup_path(self.get_object, + filename.encode('utf-8')) + except KeyError: + # Some part of the filename wasn't found, so the file is + # not present. Return the sentinel value. + return None + else: + blob = self[blob_sha] + return blob.data + + +class Scanner(object): + + def __init__(self, conf): + self.conf = conf + self.reporoot = self.conf.reporoot + self._repo = RenoRepo(self.reporoot) + self.release_tag_re = re.compile( + self.conf.release_tag_re, + flags=re.VERBOSE | re.UNICODE, + ) + self.pre_release_tag_re = re.compile( + self.conf.pre_release_tag_re, + flags=re.VERBOSE | re.UNICODE, + ) + self.branch_name_re = re.compile( + self.conf.branch_name_re, + flags=re.VERBOSE | re.UNICODE, + ) + self.branch_name_prefix = self.conf.branch_name_prefix + self.closed_branch_tag_re = re.compile( + self.conf.closed_branch_tag_re, + flags=re.VERBOSE | re.UNICODE, + ) + self._ignore_uids = set( + _get_unique_id(fn) + for fn in self.conf.ignore_notes + ) + + def _get_ref(self, name): + if name: + candidates = [ + 'refs/heads/' + name, + 'refs/remotes/' + name, + 'refs/tags/' + name, + # If a stable branch was removed, look for its EOL tag. + 'refs/tags/' + (name.rpartition('/')[-1] + '-eol'), + # If someone is using the "short" name for a branch + # without a local tracking branch, look to see if the + # name exists on the 'origin' remote. + 'refs/remotes/origin/' + name, + ] + # If the reference points explicitly to the origin remote, + # but that remote isn't present (as it won't be when zuul + # configures the repo in CI), then we want the shortened + # form of the reference. We put this option last in the + # list because we want the more explicit name to be used + # when someone is running reno locally with a more + # standard git configuration. + if name.startswith('origin/'): + candidates.append('refs/heads/' + name.partition('/')[-1]) + for ref in candidates: + LOG.debug('looking for ref {!r} as {!r}'.format(name, ref)) + key = ref.encode('utf-8') + if key in self._repo.refs: + sha = self._repo.refs[key] + o = self._repo[sha] + if isinstance(o, objects.Tag): + # Branches point directly to commits, but + # signed tags point to the signature and we + # need to dereference it to get to the commit. + sha = o.object[1] + LOG.info('found ref {!r} as {!r} at {}'.format( + name, ref, sha)) + return sha + # If we end up here we didn't find any of the candidates. + raise ValueError('Unknown reference {!r}'.format(name)) + return self._repo.refs[b'HEAD'] + + def _get_walker_for_branch(self, branch): + branch_head = self._get_ref(branch) + return self._repo.get_walker(branch_head) + + def _get_valid_tags_on_commit(self, sha): + return [tag for tag in self._repo.get_tags_on_commit(sha) + if self.release_tag_re.match(tag)] + + def _get_tags_on_branch(self, branch): + "Return a list of tag names on the given branch." + results = [] + for c in self._get_walker_for_branch(branch): + # shas_to_tags has encoded versions of the shas + # but the commit object gives us a decoded version + sha = c.commit.sha().hexdigest().encode('ascii') + tags = self._get_valid_tags_on_commit(sha) + results.extend(tags) + return results + + def _get_current_version(self, branch=None): + "Return the current version of the repository, like git describe." + # This is similar to _get_tags_on_branch() except that it + # counts up to where the tag appears and it returns when it + # finds the first tagged commit (there is no need to scan the + # rest of the branch). + commit = self._repo[self._get_ref(branch)] + count = 0 + while commit: + # shas_to_tags has encoded versions of the shas + # but the commit object gives us a decoded version + sha = commit.sha().hexdigest().encode('ascii') + tags = self._get_valid_tags_on_commit(sha) + if tags: + if count: + val = '{}-{}'.format(tags[-1], count) + else: + val = tags[-1] + return val + if commit.parents: + # Only traverse the first parent of each node. + commit = self._repo[commit.parents[0]] + count += 1 + else: + commit = None + return '0.0.0' + + def _strip_pre_release(self, tag): + """Return tag with pre-release identifier removed if present.""" + pre_release_match = self.pre_release_tag_re.search(tag) + if pre_release_match: + try: + start = pre_release_match.start('pre_release') + end = pre_release_match.end('pre_release') + except IndexError: + raise ValueError( + ("The pre-release tag regular expression, {!r}, is missing" + " a group named 'pre_release'.").format( + self.pre_release_tag_re.pattern + ) + ) + else: + stripped_tag = tag[:start] + tag[end:] + else: + stripped_tag = tag + + return stripped_tag + + def _get_branch_base(self, branch): + "Return the tag at base of the branch." + # Based on + # http://stackoverflow.com/questions/1527234/finding-a-branch-point-with-git + # git rev-list $(git rev-list --first-parent \ + # ^origin/stable/newton master | tail -n1)^^! + # + # Build the set of all commits that appear on the master + # branch, then scan the commits that appear on the specified + # branch until we find something that is on both. + master_commits = set( + c.commit.sha().hexdigest() + for c in self._get_walker_for_branch('master') + ) + for c in self._get_walker_for_branch(branch): + if c.commit.sha().hexdigest() in master_commits: + # We got to this commit via the branch, but it is also + # on master, so this is the base. + tags = self._get_valid_tags_on_commit( + c.commit.sha().hexdigest().encode('ascii')) + if tags: + return tags[-1] + else: + # Naughty, naughty, branching without tagging. + LOG.info( + ('There is no tag on commit %s at the base of %s. ' + 'Branch scan short-cutting is disabled.'), + c.commit.sha().hexdigest(), branch) + return None + return None + + def _topo_traversal(self, branch): + """Generator that yields the branch entries in topological order. + + The topo ordering in dulwich does not match the git command line + output, so we have our own that follows the branch being merged + into the mainline before following the mainline. This ensures that + tags on the mainline appear in the right place relative to the + merge points, regardless of the commit date on the entry. + + # * d1239b6 (HEAD -> master) Merge branch 'new-branch' + # |\ + # | * 9478612 (new-branch) one commit on branch + # * | 303e21d second commit on master + # * | 0ba5186 first commit on master + # |/ + # * a7f573d original commit on master + + """ + head = self._get_ref(branch) + + # Map SHA values to Entry objects, because we will be traversing + # commits not entries. + all = {} + + children = {} + + # Populate all and children structures by traversing the + # entire graph once. It doesn't matter what order we do this + # the first time, since we're just recording the relationships + # of the nodes. + for e in self._repo.get_walker(head): + all[e.commit.id] = e + for p in e.commit.parents: + children.setdefault(p, set()).add(e.commit.id) + + # Track what we have already emitted. + emitted = set() + + # Use a deque as a stack with the nodes left to process. This + # lets us avoid recursion, since we have no idea how deep some + # branches might be. + todo = collections.deque() + todo.appendleft(head) + + ignore_null_merges = self.conf.ignore_null_merges + if ignore_null_merges: + LOG.debug('ignoring null-merge commits') + + while todo: + sha = todo.popleft() + entry = all[sha] + null_merge = False + + # OpenStack used to use null-merges to bring final release + # tags from stable branches back into the master + # branch. This confuses the regular traversal because it + # makes that stable branch appear to be part of master + # and/or the later stable branch. When we hit one of those + # tags, skip it and take the first parent. + if ignore_null_merges and len(entry.commit.parents) > 1: + # Look for tags on the 2nd and later parents. The + # first parent is part of the branch we were + # originally trying to traverse, and any tags on it + # need to be kept. + for p in entry.commit.parents[1:]: + t = self._get_valid_tags_on_commit(p) + # If we have a tag being merged in, we need to + # include a check to verify that this is actually + # a null-merge (there are no changes). + if t and not entry.changes(): + LOG.debug( + 'treating %s as a null-merge because ' + 'parent %s has tag(s) %s', + sha, p, t, + ) + null_merge = True + break + if null_merge: + # Make it look like the parent entries that we're + # going to skip have been emitted so the + # bookkeeping for children works properly and we + # can continue past the merge. + emitted.update(set(entry.commit.parents[1:])) + # Make it look like the current entry was emitted + # so the bookkeeping for children works properly + # and we can continue past the merge. + emitted.add(sha) + # Now set up the first parent so it is processed + # later, as long as we haven't already processed + # it. + first_parent = entry.commit.parents[0] + if (first_parent not in todo and + first_parent not in emitted): + todo.appendleft(first_parent) + continue + + # If a node has multiple children, it is the start point + # for a branch that was merged back into the rest of the + # tree. We will have already processed the merge commit + # and are traversing either the branch that was merged in + # or the base into which it was merged. We want to stop + # traversing the branch that was merged in at the point + # where the branch was created, because we are trying to + # linearize the history. At that point, we go back to the + # merge node and take the other parent node, which should + # lead us back to the origin of the branch through the + # mainline. + unprocessed_children = [ + c + for c in children.get(sha, set()) + if c not in emitted + ] + + if not unprocessed_children: + # All children have been processed. Remember that we have + # processed this node and then emit the entry. + emitted.add(sha) + yield entry + + # Now put the parents on the stack from left to right + # so they are processed right to left. If the node is + # already on the stack, leave it to be processed in + # the original order where it was added. + # + # NOTE(dhellmann): It's not clear if this is the right + # solution, or if we should re-stack and then ignore + # duplicate emissions at the top of this + # loop. Checking if the item is already on the todo + # stack isn't very expensive, since we don't expect it + # to grow very large, but it's not clear the output + # will be produced in the right order. + for p in entry.commit.parents: + if p not in todo and p not in emitted: + todo.appendleft(p) + + else: + # Has unprocessed children. Do not emit, and do not + # restack, since when we get to the other child they will + # stack it. + pass + + def get_file_at_commit(self, filename, sha): + "Return the contents of the file if it exists at the commit, or None." + return self._repo.get_file_at_commit(filename, sha) + + def _file_exists_at_commit(self, filename, sha): + "Return true if the file exists at the given commit." + return bool(self.get_file_at_commit(filename, sha)) + + def _get_series_branches(self): + "Get branches matching the branch_name_re config option." + refs = self._repo.get_refs() + LOG.debug('refs %s', list(refs.keys())) + branch_names = set() + for r in refs.keys(): + name = None + r = r.decode('utf-8') + if r.startswith('refs/remotes/origin/'): + name = r[20:] + elif r.startswith('refs/heads/'): + name = r[11:] + if name and self.branch_name_re.search(name): + LOG.debug('branch name %s', name) + branch_names.add(name) + continue + if not r.startswith('refs/tags/'): + continue + # See if the ref is a closed branch tag. + name = r.rpartition('/')[-1] + match = self.closed_branch_tag_re.search(name) + if match: + name = self.branch_name_prefix + match.group(1) + LOG.debug('closed branch tag %s becomes %s', + r.rpartition('/')[-1], name) + branch_names.add(name) + return list(sorted(branch_names)) + + def _get_earlier_branch(self, branch): + "Return the name of the branch created before the given branch." + # FIXME(dhellmann): Assumes branches come in order based on + # name. That may not be true for projects that branch based on + # version numbers instead of names. + if branch.startswith('origin/'): + branch = branch[7:] + LOG.debug('looking for the branch before %s', branch) + branch_names = self._get_series_branches() + if branch not in branch_names: + LOG.debug('Could not find branch %r among %s', + branch, branch_names) + return None + LOG.debug('found branches %s', branch_names) + current = branch_names.index(branch) + if current == 0: + # This is the first branch. + LOG.debug('%s appears to be the first branch', branch) + return None + previous = branch_names[current - 1] + LOG.debug('found earlier branch %s', previous) + return previous + + def _find_scan_stop_point(self, earliest_version, versions_by_date, + collapse_pre_releases, branch): + """Return the version to use to stop the scan. + + Use the list of versions_by_date to get the tag with a + different version created *before* the branch to ensure that + we include notes that go with that version that *is* in the + branch. + + :param earliest_version: Version string of the earliest + version to be included in the output. + :param versions_by_date: List of version strings in reverse + chronological order. + :param collapse_pre_releases: Boolean indicating whether we are + collapsing pre-releases or not. If false, the next tag + is used, regardless of its version. + :param branch: The name of the branch we are scanning. + + """ + if not earliest_version: + return None + earliest_parts = _parse_version(earliest_version) + try: + idx = versions_by_date.index(earliest_version) + 1 + except ValueError: + # The version we were given is not present, use a full + # scan. + return None + # We need to look for the previous branch's root. + if branch and branch != 'master': + previous_branch = self._get_earlier_branch(branch) + if not previous_branch: + # This was the first branch, so scan the whole + # history. + return None + previous_base = self._get_branch_base(previous_branch) + return previous_base + is_pre_release = bool(self.pre_release_tag_re.search(earliest_version)) + if is_pre_release and not collapse_pre_releases: + # We just take the next tag. + return versions_by_date[idx] + # We need to look for a different version. + for candidate in versions_by_date[idx:]: + parts = _parse_version(candidate) + if parts != earliest_parts: + # The candidate is a different version, use it. + return candidate + return None + + def get_notes_by_version(self): + """Return an OrderedDict mapping versions to lists of notes files. + + The versions are presented in reverse chronological order. + + Notes files are associated with the earliest version for which + they were available, regardless of whether they changed later. + + :param reporoot: Path to the root of the git repository. + :type reporoot: str + """ + + reporoot = self.reporoot + notesdir = self.conf.notespath + branch = self.conf.branch + earliest_version = self.conf.earliest_version + collapse_pre_releases = self.conf.collapse_pre_releases + stop_at_branch_base = self.conf.stop_at_branch_base + + LOG.info( + ('scanning %s/%s ' + '(branch=%s earliest_version=%s collapse_pre_releases=%s)'), + reporoot.rstrip('/'), notesdir.lstrip('/'), + branch or '*current*', + earliest_version, + collapse_pre_releases, + ) + + # Determine the current version, which might be an unreleased or + # dev version if there are unreleased commits at the head of the + # branch in question. + current_version = self._get_current_version(branch) + LOG.debug('current repository version: %s' % current_version) + + # Determine all of the tags known on the branch, in their date + # order. We scan the commit history in topological order to ensure + # we have the commits in the right version, so we might encounter + # the tags in a different order during that phase. + versions_by_date = self._get_tags_on_branch(branch) + LOG.debug('versions by date %r' % (versions_by_date,)) + if earliest_version and earliest_version not in versions_by_date: + raise ValueError( + 'earliest-version set to unknown revision {!r}'.format( + earliest_version)) + + # If the user has told us where to stop, use that as the + # default. + scan_stop_tag = self._find_scan_stop_point( + earliest_version, versions_by_date, + collapse_pre_releases, branch) + + # If the user has not told us where to stop, try to work it + # out for ourselves. + if not branch and not earliest_version and stop_at_branch_base: + # On the current branch, stop at the point where the most + # recent branch was created, if we can find one. + LOG.debug('working on current branch without earliest_version') + branches = self._get_series_branches() + if branches: + for earlier_branch in reversed(branches): + LOG.debug('checking if current branch is later than %s', + earlier_branch) + scan_stop_tag = self._get_branch_base(earlier_branch) + if scan_stop_tag in versions_by_date: + LOG.info( + 'looking at %s at base of %s to ' + 'stop scanning the current branch', + scan_stop_tag, earlier_branch + ) + break + else: + LOG.info('unable to find the previous branch base') + scan_stop_tag = None + if scan_stop_tag: + # If there is a tag on this branch after the point + # where the earlier branch was created, then use that + # tag as the earliest version to show in the current + # "series". If there is no such tag, then go all the + # way to the base of that earlier branch. + try: + idx = versions_by_date.index(scan_stop_tag) + 1 + earliest_version = versions_by_date[idx] + except ValueError: + # The scan_stop_tag is not in versions_by_date. + earliest_version = None + except IndexError: + # The idx is not in versions_by_date. + earliest_version = scan_stop_tag + elif branch and stop_at_branch_base and not earliest_version: + # If branch is set and is not "master", + # then we want to stop at the version before the tag at the + # base of the branch, which involves a bit of searching. + LOG.debug('determining earliest_version from branch') + branch_base = self._get_branch_base(branch) + LOG.debug('branch base %s', branch_base) + scan_stop_tag = self._find_scan_stop_point( + branch_base, versions_by_date, + collapse_pre_releases, branch) + if not scan_stop_tag: + earliest_version = branch_base + else: + idx = versions_by_date.index(scan_stop_tag) + earliest_version = versions_by_date[idx - 1] + LOG.debug('using version before %s as scan stop point', + scan_stop_tag) + if earliest_version and collapse_pre_releases: + if self.pre_release_tag_re.search(earliest_version): + # The earliest version won't actually be the pre-release + # that might have been tagged when the branch was created, + # but the final version. Strip the pre-release portion of + # the version number. + earliest_version = self._strip_pre_release( + earliest_version + ) + if earliest_version: + LOG.info('earliest version to include is %s', earliest_version) + else: + LOG.info('including entire branch history') + if scan_stop_tag: + LOG.info('stopping scan at %s', scan_stop_tag) + + # Since the version may not already be known, make sure it is + # in the list of versions by date. And since it is the most + # recent version, go ahead and insert it at the front of the + # list. + if current_version not in versions_by_date: + versions_by_date.insert(0, current_version) + versions_by_date.insert(0, '*working-copy*') + + # Track the versions we have seen and the earliest version for + # which we have seen a given note's unique id. + tracker = _ChangeTracker() + + # Process the local index, if we are scanning the current + # branch. + if not branch: + prefix = notesdir.rstrip('/') + '/' + index = self._repo.open_index() + + # Pretend anything known to the repo and changed but not + # staged is part of the fake version '*working-copy*'. + LOG.debug('scanning unstaged changes') + for fname in d_index.get_unstaged_changes(index, self.reporoot): + fname = fname.decode('utf-8') + LOG.debug('found unstaged file %s', fname) + if fname.startswith(prefix) and _note_file(fname): + fullpath = os.path.join(self.reporoot, fname) + if os.path.exists(fullpath): + LOG.debug('found file %s', fullpath) + tracker.add(fname, None, '*working-copy*') + else: + LOG.debug('deleted file %s', fullpath) + tracker.delete(fname, None, '*working-copy*') + + # Pretend anything in the index is part of the fake + # version "*working-copy*". + LOG.debug('scanning staged schanges') + changes = porcelain.get_tree_changes(self._repo) + for fname in changes['add']: + fname = fname.decode('utf-8') + if fname.startswith(prefix) and _note_file(fname): + tracker.add(fname, None, '*working-copy*') + for fname in changes['modify']: + fname = fname.decode('utf-8') + if fname.startswith(prefix) and _note_file(fname): + tracker.modify(fname, None, '*working-copy*') + for fname in changes['delete']: + fname = fname.decode('utf-8') + if fname.startswith(prefix) and _note_file(fname): + tracker.delete(fname, None, '*working-copy*') + + aggregator = _ChangeAggregator() + + # Process the git commit history. + for counter, entry in enumerate(self._topo_traversal(branch), 1): + + sha = entry.commit.id + tags_on_commit = self._get_valid_tags_on_commit(sha) + + LOG.debug('%06d %s %s', counter, sha, tags_on_commit) + + # If there are no tags in this block, assume the most recently + # seen version. + tags = tags_on_commit + if not tags: + tags = [current_version] + else: + current_version = tags_on_commit[-1] + LOG.info('%06d %s updating current version to %s', + counter, sha, current_version) + + # Look for changes to notes files in this commit. The + # change has only the basename of the path file, so we + # need to prefix that with the notesdir before giving it + # to the tracker. + changes = _changes_in_subdir(self._repo, entry, notesdir) + for change in aggregator.aggregate_changes(entry, changes): + uniqueid = change[0] + + if uniqueid in self._ignore_uids: + LOG.info('ignoring %s based on configuration setting', + uniqueid) + continue + + c_type = change[1] + + if c_type == diff_tree.CHANGE_ADD: + path, blob_sha = change[-2:] + fullpath = os.path.join(notesdir, path) + tracker.add(fullpath, sha, current_version) + + elif c_type == diff_tree.CHANGE_DELETE: + path, blob_sha = change[-2:] + fullpath = os.path.join(notesdir, path) + tracker.delete(fullpath, sha, current_version) + + elif c_type == diff_tree.CHANGE_RENAME: + path, blob_sha = change[-2:] + fullpath = os.path.join(notesdir, path) + tracker.rename(fullpath, sha, current_version) + + elif c_type == diff_tree.CHANGE_MODIFY: + path, blob_sha = change[-2:] + fullpath = os.path.join(notesdir, path) + tracker.modify(fullpath, sha, current_version) + + else: + raise ValueError( + 'unknown change instructions {!r}'.format(change) + ) + + if scan_stop_tag and scan_stop_tag in tags: + LOG.info( + ('reached end of branch after %d commits at %s ' + 'with tags %s'), + counter, sha, tags) + break + + # Invert earliest_seen to make a list of notes files for each + # version. + files_and_tags = collections.OrderedDict() + for v in tracker.versions: + files_and_tags[v] = [] + # Produce a list of the actual files present in the repository. If + # a note is removed, this step should let us ignore it. + for uniqueid, version in tracker.earliest_seen.items(): + try: + base, sha = tracker.last_name_by_id[uniqueid] + LOG.debug('%s: sorting %s into version %s', + uniqueid, base, version) + files_and_tags[version].append((base, sha)) + except KeyError: + # Unable to find the file again, skip it to avoid breaking + # the build. + msg = ('unable to find release notes file associated ' + 'with unique id %r, skipping') % uniqueid + LOG.debug(msg) + print(msg, file=sys.stderr) + + # Combine pre-releases into the final release, if we are told to + # and the final release exists. + if collapse_pre_releases: + LOG.debug('collapsing pre-release versions into final releases') + collapsing = files_and_tags + files_and_tags = collections.OrderedDict() + for ov in versions_by_date: + if ov not in collapsing: + # We don't need to collapse this one because there are + # no notes attached to it. + continue + pre_release_match = self.pre_release_tag_re.search(ov) + LOG.debug('checking %r', ov) + if pre_release_match: + # Remove the trailing pre-release part of the version + # from the string. + canonical_ver = self._strip_pre_release(ov) + if canonical_ver not in versions_by_date: + # This canonical version was never tagged, so we + # do not want to collapse the pre-releases. Reset + # to the original version. + canonical_ver = ov + else: + LOG.debug('combining into %r', canonical_ver) + else: + canonical_ver = ov + if canonical_ver not in files_and_tags: + files_and_tags[canonical_ver] = [] + files_and_tags[canonical_ver].extend(collapsing[ov]) + + LOG.debug('files_and_tags %s', + {k: len(v) for k, v in files_and_tags.items()}) + # Only return the parts of files_and_tags that actually have + # filenames associated with the versions. + LOG.debug('trimming') + trimmed = collections.OrderedDict() + for ov in versions_by_date: + if not files_and_tags.get(ov): + continue + LOG.debug('keeping %s', ov) + # Sort the notes associated with the version so they are in a + # deterministic order, to avoid having the same data result in + # different output depending on random factors. Earlier + # versions of the scanner assumed the notes were recorded in + # chronological order based on the commit date, but with the + # change to use topological sorting that is no longer + # necessarily true. We want the notes to always show up in the + # same order, but it doesn't really matter what order that is, + # so just sort based on the unique id. + trimmed[ov] = sorted(files_and_tags[ov]) + # If we have been told to stop at a version, we can do that + # now. + if earliest_version and ov == earliest_version: + LOG.debug('stopping trimming at %s', earliest_version) + break + + LOG.debug( + 'found %d versions and %d files', + len(trimmed.keys()), sum(len(ov) for ov in trimmed.values()), + ) + return trimmed diff --git a/reno/setup_command.py b/reno/setup_command.py new file mode 100644 index 0000000000000000000000000000000000000000..102ab255181d1a4e159012ccaa134bab0433688b --- /dev/null +++ b/reno/setup_command.py @@ -0,0 +1,138 @@ +# Copyright 2017, Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Custom distutils command. + +For more information, refer to the distutils and setuptools source: + +- https://github.com/python/cpython/blob/3.6/Lib/distutils/cmd.py +- https://github.com/pypa/setuptools/blob/v36.0.0/setuptools/command/sdist.py +""" + +from distutils import cmd +from distutils import errors +from distutils import log + +import six + +from reno import cache +from reno import config +from reno import defaults +from reno import formatter +from reno import loader + +COMMAND_NAME = 'build_reno' # duplicates what's found in setup.cfg + + +def load_config(distribution): + """Utility method to parse distutils/setuptools configuration. + + This is for use by other libraries to extract the command configuration. + + :param distribution: A :class:`distutils.dist.Distribution` object + :returns: A tuple of a :class:`reno.config.Config` object, the output path + of the human-readable release notes file, and the output file of the + reno cache file + """ + option_dict = distribution.get_option_dict(COMMAND_NAME) + + if option_dict.get('repo_root') is not None: + repo_root = option_dict.get('repo_root')[1] + else: + repo_root = defaults.REPO_ROOT + + if option_dict.get('rel_notes_dir') is not None: + rel_notes_dir = option_dict.get('rel_notes_dir')[1] + else: + rel_notes_dir = defaults.RELEASE_NOTES_SUBDIR + + if option_dict.get('output_file') is not None: + output_file = option_dict.get('output_file')[1] + else: + output_file = defaults.RELEASE_NOTES_FILENAME + + conf = config.Config(repo_root, rel_notes_dir) + cache_file = loader.get_cache_filename(conf) + + return (conf, output_file, cache_file) + + +class BuildReno(cmd.Command): + """Distutils command to build reno release notes. + + The release note build can be triggered from distutils, and some + configuration can be included in ``setup.py`` or ``setup.cfg`` instead of + being specified from the command-line. + """ + description = 'Build reno release notes' + user_options = [ + ('repo-root=', None, 'the root directory of the Git repository; ' + 'defaults to "."'), + ('rel-notes-dir=', None, 'the parent directory; defaults to ' + '"releasenotes"'), + ('output-file=', None, 'the filename of the release notes file'), + ] + + def initialize_options(self): + self.repo_root = None + self.rel_notes_dir = None + self.output_file = None + + def finalize_options(self): + if self.repo_root is None: + self.repo_root = defaults.REPO_ROOT + + if self.rel_notes_dir is None: + self.rel_notes_dir = defaults.RELEASE_NOTES_SUBDIR + + if self.output_file is None: + self.output_file = defaults.RELEASE_NOTES_FILENAME + + # Overriding distutils' Command._ensure_stringlike which doesn't support + # unicode, causing finalize_options to fail if invoked again. Workaround + # for http://bugs.python.org/issue19570 + def _ensure_stringlike(self, option, what, default=None): + # type: (unicode, unicode, Any) -> Any + val = getattr(self, option) + if val is None: + setattr(self, option, default) + return default + elif not isinstance(val, six.string_types): + raise errors.DistutilsOptionError("'%s' must be a %s (got `%s`)" + % (option, what, val)) + return val + + def run(self): + conf = config.Config(self.repo_root, self.rel_notes_dir) + + # Generate the cache using the configuration options found + # in the release notes directory and the default output + # filename. + cache_filename = cache.write_cache_db( + conf=conf, + versions_to_include=[], # include all versions + outfilename=None, # generate the default name + ) + log.info('wrote cache file to %s', cache_filename) + + ldr = loader.Loader(conf) + text = formatter.format_report( + ldr, + conf, + ldr.versions, + title=self.distribution.metadata.name, + ) + with open(self.output_file, 'w') as f: + f.write(text) + log.info('wrote release notes to %s', self.output_file) diff --git a/reno/sphinxext.py b/reno/sphinxext.py index 07359e574d1bc4cbcf073f0f8d9de39742569dca..5b99993cd810400db000ada3d79ab94b0e0b3bbb 100644 --- a/reno/sphinxext.py +++ b/reno/sphinxext.py @@ -9,70 +9,111 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +from __future__ import unicode_literals import os.path -from reno import defaults -from reno import formatter -from reno import scanner - from docutils import nodes from docutils.parsers import rst from docutils.parsers.rst import directives -from docutils.statemachine import ViewList +from docutils import statemachine +from sphinx.util import logging from sphinx.util.nodes import nested_parse_with_titles +from dulwich import repo +from reno import config +from reno import defaults +from reno import formatter +from reno import loader + +LOG = logging.getLogger(__name__) + class ReleaseNotesDirective(rst.Directive): has_content = True + # FIXME(dhellmann): We should be able to build this information + # from the configuration options so we don't have to edit it + # manually when we add new options. option_spec = { 'branch': directives.unchanged, 'reporoot': directives.unchanged, 'relnotessubdir': directives.unchanged, 'notesdir': directives.unchanged, 'version': directives.unchanged, + 'collapse-pre-releases': directives.flag, + 'earliest-version': directives.unchanged, + 'stop-at-branch-base': directives.flag, + 'ignore-notes': directives.unchanged, + 'unreleased-version-title': directives.unchanged, } def run(self): - env = self.state.document.settings.env - app = env.app - - def info(msg): - app.info('[reno] %s' % (msg,)) - title = ' '.join(self.content) branch = self.options.get('branch') reporoot_opt = self.options.get('reporoot', '.') reporoot = os.path.abspath(reporoot_opt) + # When building on RTD.org the root directory may not be + # the current directory, so look for it. + reporoot = repo.Repo.discover(reporoot).path relnotessubdir = self.options.get('relnotessubdir', defaults.RELEASE_NOTES_SUBDIR) - notessubdir = self.options.get('notesdir', defaults.NOTES_SUBDIR) + ignore_notes = [ + name.strip() + for name in self.options.get('ignore-notes', '').split(',') + ] + conf = config.Config(reporoot, relnotessubdir) + opt_overrides = {} + if 'notesdir' in self.options: + opt_overrides['notesdir'] = self.options.get('notesdir') version_opt = self.options.get('version') + # FIXME(dhellmann): Force these flags True for now and figure + # out how Sphinx passes a "false" flag later. + # 'collapse-pre-releases' in self.options + opt_overrides['collapse_pre_releases'] = True + # Only stop at the branch base if we have not been told + # explicitly which versions to include. + opt_overrides['stop_at_branch_base'] = (version_opt is None) + if 'earliest-version' in self.options: + opt_overrides['earliest_version'] = self.options.get( + 'earliest-version') + if 'unreleased-version-title' in self.options: + opt_overrides['unreleased_version_title'] = self.options.get( + 'unreleased-version-title') + + if branch: + opt_overrides['branch'] = branch + if ignore_notes: + opt_overrides['ignore_notes'] = ignore_notes + conf.override(**opt_overrides) - notesdir = os.path.join(relnotessubdir, notessubdir) - info('scanning %s for %s release notes' % - (os.path.join(reporoot, notesdir), branch or 'current branch')) + notesdir = os.path.join(relnotessubdir, conf.notesdir) + LOG.info('scanning %s for %s release notes' % ( + os.path.join(conf.reporoot, notesdir), + branch or 'current branch')) - notes = scanner.get_notes_by_version(reporoot, notesdir, branch) + ldr = loader.Loader(conf) if version_opt is not None: versions = [ v.strip() for v in version_opt.split(',') ] else: - versions = notes.keys() + versions = ldr.versions + LOG.info('got versions %s' % (versions,)) text = formatter.format_report( - reporoot, - notes, + ldr, + conf, versions, title=title, + branch=branch, ) - source_name = '<' + __name__ + '>' - result = ViewList() - for line in text.splitlines(): - result.append(line, source_name) + source_name = '<%s %s>' % (__name__, branch or 'current branch') + result = statemachine.ViewList() + for line_num, line in enumerate(text.splitlines(), 1): + LOG.debug('%4d: %s', line_num, line) + result.append(line, source_name, line_num) node = nodes.section() node.document = self.state.document diff --git a/reno/tests/base.py b/reno/tests/base.py index 1c30cdb56ec9c8a721b9245384ea415dc675cf89..cc95054f636b255e2458f1983452ef21212a8c02 100644 --- a/reno/tests/base.py +++ b/reno/tests/base.py @@ -15,9 +15,20 @@ # License for the specific language governing permissions and limitations # under the License. -from oslotest import base +import fixtures +import testtools -class TestCase(base.BaseTestCase): +class TestCase(testtools.TestCase): """Test case base class for all unit tests.""" + + def setUp(self): + super(TestCase, self).setUp() + self._stdout_fixture = fixtures.StringStream('stdout') + self.stdout = self.useFixture(self._stdout_fixture).stream + self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.stdout)) + self._stderr_fixture = fixtures.StringStream('stderr') + self.stderr = self.useFixture(self._stderr_fixture).stream + self.useFixture(fixtures.MonkeyPatch('sys.stderr', self.stderr)) + self.useFixture(fixtures.FakeLogger()) diff --git a/reno/tests/test_cache.py b/reno/tests/test_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..abf19a2bf7a79604908a2431d7cbba1d92c59db5 --- /dev/null +++ b/reno/tests/test_cache.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures +import textwrap + +import mock + +from reno import cache +from reno import config +from reno.tests import base + + +class TestCache(base.TestCase): + + scanner_output = { + '0.0.0': [('note1', 'shaA')], + '1.0.0': [('note2', 'shaB'), ('note3', 'shaC')], + } + + note_bodies = { + 'note1': textwrap.dedent(""" + prelude: > + This is the prelude. + """), + 'note2': textwrap.dedent(""" + issues: + - This is the first issue. + - This is the second issue. + """), + 'note3': textwrap.dedent(""" + features: + - We added a feature! + """) + } + + def _get_note_body(self, filename, sha): + return self.note_bodies.get(filename, '') + + def setUp(self): + super(TestCache, self).setUp() + self.useFixture( + fixtures.MockPatch('reno.scanner.Scanner.get_file_at_commit', + new=self._get_note_body) + ) + self.c = config.Config('.') + + @mock.patch('reno.scanner.Scanner.get_notes_by_version') + def test_build_cache_db(self, gnbv): + gnbv.return_value = self.scanner_output + expected = { + 'notes': [ + {'version': k, 'files': v} + for k, v in self.scanner_output.items() + ], + 'file-contents': { + 'note1': { + 'prelude': 'This is the prelude.\n', + }, + 'note2': { + 'issues': [ + 'This is the first issue.', + 'This is the second issue.', + ], + }, + 'note3': { + 'features': ['We added a feature!'], + }, + }, + } + + db = cache.build_cache_db( + self.c, + versions_to_include=[], + ) + + self.assertEqual(expected, db) diff --git a/reno/tests/test_config.py b/reno/tests/test_config.py new file mode 100644 index 0000000000000000000000000000000000000000..38e96355d42dc4da8c2e2eeb2ea9ce7e7309dba3 --- /dev/null +++ b/reno/tests/test_config.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import argparse +import os + +import fixtures + +from reno import config +from reno import defaults +from reno import main +from reno.tests import base + +import mock + + +class TestConfig(base.TestCase): + EXAMPLE_CONFIG = """ +collapse_pre_releases: false +""" + + def setUp(self): + super(TestConfig, self).setUp() + # Temporary directory to store our config + self.tempdir = self.useFixture(fixtures.TempDir()) + + def test_defaults(self): + c = config.Config(self.tempdir.path) + actual = c.options + expected = { + o.name: o.default + for o in config._OPTIONS + } + self.assertEqual(expected, actual) + + def test_override(self): + c = config.Config(self.tempdir.path) + c.override( + collapse_pre_releases=False, + ) + actual = c.options + expected = { + o.name: o.default + for o in config._OPTIONS + } + expected['collapse_pre_releases'] = False + self.assertEqual(expected, actual) + + def test_override_multiple(self): + c = config.Config(self.tempdir.path) + c.override( + notesdir='value1', + ) + c.override( + notesdir='value2', + ) + actual = c.options + expected = { + o.name: o.default + for o in config._OPTIONS + } + expected['notesdir'] = 'value2' + self.assertEqual(expected, actual) + + def test_load_file_not_present(self): + missing = 'reno.config.Config._report_missing_config_files' + with mock.patch(missing) as error_handler: + config.Config(self.tempdir.path) + self.assertEqual(1, error_handler.call_count) + + def _test_load_file(self, config_path): + with open(config_path, 'w') as fd: + fd.write(self.EXAMPLE_CONFIG) + self.addCleanup(os.unlink, config_path) + c = config.Config(self.tempdir.path) + self.assertEqual(False, c.collapse_pre_releases) + + def test_load_file_in_releasenotesdir(self): + rn_path = self.tempdir.join('releasenotes') + os.mkdir(rn_path) + config_path = self.tempdir.join('releasenotes/config.yaml') + self._test_load_file(config_path) + + def test_load_file_in_repodir(self): + config_path = self.tempdir.join('reno.yaml') + self._test_load_file(config_path) + + def test_get_default(self): + d = config.Config.get_default('notesdir') + self.assertEqual('notes', d) + + def test_get_default_unknown(self): + self.assertRaises( + ValueError, + config.Config.get_default, + 'unknownopt', + ) + + def _run_override_from_parsed_args(self, argv): + parser = argparse.ArgumentParser() + main._build_query_arg_group(parser) + args = parser.parse_args(argv) + c = config.Config(self.tempdir.path) + c.override_from_parsed_args(args) + return c + + def test_override_from_parsed_args_empty(self): + c = self._run_override_from_parsed_args([]) + actual = { + o.name: getattr(c, o.name) + for o in config._OPTIONS + } + expected = { + o.name: o.default + for o in config._OPTIONS + } + self.assertEqual(expected, actual) + + def test_override_from_parsed_args_boolean_false(self): + c = self._run_override_from_parsed_args([ + '--no-collapse-pre-releases', + ]) + actual = c.options + expected = { + o.name: o.default + for o in config._OPTIONS + } + expected['collapse_pre_releases'] = False + self.assertEqual(expected, actual) + + def test_override_from_parsed_args_boolean_true(self): + c = self._run_override_from_parsed_args([ + '--collapse-pre-releases', + ]) + actual = c.options + expected = { + o.name: o.default + for o in config._OPTIONS + } + expected['collapse_pre_releases'] = True + self.assertEqual(expected, actual) + + def test_override_from_parsed_args_string(self): + c = self._run_override_from_parsed_args([ + '--earliest-version', '1.2.3', + ]) + actual = c.options + expected = { + o.name: o.default + for o in config._OPTIONS + } + expected['earliest_version'] = '1.2.3' + self.assertEqual(expected, actual) + + def test_override_from_parsed_args_ignore_non_options(self): + parser = argparse.ArgumentParser() + main._build_query_arg_group(parser) + parser.add_argument('not_a_config_option') + args = parser.parse_args(['value']) + c = config.Config(self.tempdir.path) + c.override_from_parsed_args(args) + self.assertFalse(hasattr(c, 'not_a_config_option')) + + +class TestConfigProperties(base.TestCase): + + def setUp(self): + super(TestConfigProperties, self).setUp() + # Temporary directory to store our config + self.tempdir = self.useFixture(fixtures.TempDir()) + self.c = config.Config('releasenotes') + + def test_reporoot(self): + self.c.reporoot = 'blah//' + self.assertEqual('blah/', self.c.reporoot) + self.c.reporoot = 'blah' + self.assertEqual('blah/', self.c.reporoot) + + def test_notespath(self): + self.assertEqual('releasenotes/notes', self.c.notespath) + self.c.override(notesdir='thenotes') + self.assertEqual('releasenotes/thenotes', self.c.notespath) + + def test_template(self): + template = defaults.TEMPLATE.format(defaults.PRELUDE_SECTION_NAME) + self.assertEqual(template, self.c.template) + self.c.override(template='i-am-a-template') + self.assertEqual('i-am-a-template', self.c.template) + + def test_prelude_override(self): + template = defaults.TEMPLATE.format(defaults.PRELUDE_SECTION_NAME) + self.assertEqual(template, self.c.template) + self.c.override(prelude_section_name='fake_prelude_name') + expected_template = defaults.TEMPLATE.format('fake_prelude_name') + self.assertEqual(expected_template, self.c.template) + + def test_prelude_and_template_override(self): + template = defaults.TEMPLATE.format(defaults.PRELUDE_SECTION_NAME) + self.assertEqual(template, self.c.template) + self.c.override(prelude_section_name='fake_prelude_name', + template='i-am-a-template') + self.assertEqual('fake_prelude_name', self.c.prelude_section_name) + self.assertEqual('i-am-a-template', self.c.template) diff --git a/reno/tests/test_create.py b/reno/tests/test_create.py index 6663dd83c6764430ad7b9fa5a1dc513cba46f99b..46cb9b04790064ebb81c16d2f9355b59452c1723 100644 --- a/reno/tests/test_create.py +++ b/reno/tests/test_create.py @@ -12,11 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. +import fixtures +import io +import mock + from reno import create from reno.tests import base -import mock - class TestPickFileName(base.TestCase): @@ -29,3 +31,69 @@ class TestPickFileName(base.TestCase): 'somepath', 'someslug', ) + + @mock.patch('os.path.exists') + def test_random_enough(self, exists): + exists.return_value = False + result = create._pick_note_file_name('somepath', 'someslug') + self.assertIn('somepath', result) + self.assertIn('someslug', result) + + +class TestCreate(base.TestCase): + + def setUp(self): + super(TestCreate, self).setUp() + self.tmpdir = self.useFixture(fixtures.TempDir()).path + + def _create_user_template(self, contents): + filename = create._pick_note_file_name(self.tmpdir, 'usertemplate') + with open(filename, 'w') as f: + f.write(contents) + return filename + + def _get_file_path_from_output(self, output): + # Get the last consecutive word from the output and remove the newline + return output[output.rfind(" ") + 1:-1] + + def test_create_from_template(self): + filename = create._pick_note_file_name(self.tmpdir, 'theslug') + create._make_note_file(filename, 'i-am-a-template') + with open(filename, 'r') as f: + body = f.read() + self.assertEqual('i-am-a-template', body) + + def test_create_from_user_template(self): + args = mock.Mock() + args.from_template = self._create_user_template('i-am-a-user-template') + args.slug = 'theslug' + args.edit = False + conf = mock.Mock() + conf.notespath = self.tmpdir + with mock.patch('sys.stdout', new=io.StringIO()) as fake_out: + create.create_cmd(args, conf) + filename = self._get_file_path_from_output(fake_out.getvalue()) + with open(filename, 'r') as f: + body = f.read() + self.assertEqual('i-am-a-user-template', body) + + def test_create_from_user_template_fails_because_unexistent_file(self): + args = mock.Mock() + args.from_template = 'some-unexistent-file.yaml' + args.slug = 'theslug' + args.edit = False + conf = mock.Mock() + conf.notespath = self.tmpdir + self.assertRaises(ValueError, create.create_cmd, args, conf) + + def test_edit(self): + self.useFixture(fixtures.EnvironmentVariable('EDITOR', 'myeditor')) + with mock.patch('subprocess.call') as call_mock: + self.assertTrue(create._edit_file('somepath')) + call_mock.assert_called_once_with(['myeditor', 'somepath']) + + def test_edit_without_editor_env_var(self): + self.useFixture(fixtures.EnvironmentVariable('EDITOR')) + with mock.patch('subprocess.call') as call_mock: + self.assertFalse(create._edit_file('somepath')) + call_mock.assert_not_called() diff --git a/reno/tests/test_exts.py b/reno/tests/test_exts.py new file mode 100644 index 0000000000000000000000000000000000000000..85445ca1f7a90911dee8bdcee544b2245d8ba53f --- /dev/null +++ b/reno/tests/test_exts.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import textwrap + +from reno._exts import show_reno_config +from reno import config +from reno.tests import base + + +class TestMultiLineString(base.TestCase): + + def test_no_indent(self): + input = textwrap.dedent("""\ + The notes subdirectory within the relnotesdir where the + notes live. + """) + expected = '\n'.join([ + 'The notes subdirectory within the relnotesdir where the', + 'notes live.', + ]) + actual = '\n'.join(show_reno_config._multi_line_string(input)) + self.assertEqual(expected, actual) + + def test_with_indent(self): + input = textwrap.dedent("""\ + The notes subdirectory within the relnotesdir where the + notes live. + """) + expected = '\n'.join([ + ' The notes subdirectory within the relnotesdir where the', + ' notes live.', + ]) + actual = '\n'.join(show_reno_config._multi_line_string(input, ' ')) + self.assertEqual(expected, actual) + + def test_first_line_blank(self): + input = textwrap.dedent(""" + The notes subdirectory within the relnotesdir where the + notes live. + """) + expected = '\n'.join([ + ' The notes subdirectory within the relnotesdir where the', + ' notes live.', + ]) + actual = '\n'.join(show_reno_config._multi_line_string(input, ' ')) + self.assertEqual(expected, actual) + + +class TestFormatOptionHelp(base.TestCase): + + def test_simple_default(self): + opt = config.Opt( + 'notesdir', 'path/to/notes', + textwrap.dedent("""\ + The notes subdirectory within the relnotesdir where the + notes live. + """), + ) + actual = '\n'.join(show_reno_config._format_option_help([opt])) + expected = textwrap.dedent("""\ + ``notesdir`` + The notes subdirectory within the relnotesdir where the + notes live. + + Defaults to ``'path/to/notes'`` + """) + self.assertEqual(expected, actual) + + def test_bool_default(self): + opt = config.Opt( + 'collapse_pre_releases', True, + textwrap.dedent("""\ + Should pre-release versions be merged into the final release + of the same number (1.0.0.0a1 notes appear under 1.0.0). + """), + ) + actual = '\n'.join(show_reno_config._format_option_help([opt])) + expected = textwrap.dedent("""\ + ``collapse_pre_releases`` + Should pre-release versions be merged into the final release + of the same number (1.0.0.0a1 notes appear under 1.0.0). + + Defaults to ``True`` + """) + self.assertEqual(expected, actual) + + def test_multiline_default(self): + opt = config.Opt( + 'release_tag_re', + textwrap.dedent('''\ + ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and + # pre-releases + '''), + textwrap.dedent("""\ + The regex pattern used to match the repo tags representing a + valid release version. The pattern is compiled with the + verbose and unicode flags enabled. + """), + ) + actual = '\n'.join(show_reno_config._format_option_help([opt])) + expected = textwrap.dedent("""\ + ``release_tag_re`` + The regex pattern used to match the repo tags representing a + valid release version. The pattern is compiled with the + verbose and unicode flags enabled. + + Defaults to + + :: + + ((?:[\d.ab]|rc)+) # digits, a, b, and rc cover regular and + # pre-releases + """) + self.assertEqual(expected, actual) diff --git a/reno/tests/test_formatter.py b/reno/tests/test_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..c643f74109b9c861b86881600bf5add0385c72ba --- /dev/null +++ b/reno/tests/test_formatter.py @@ -0,0 +1,270 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from reno import config +from reno import formatter +from reno import loader +from reno.tests import base + + +class TestFormatterBase(base.TestCase): + + scanner_output = { + '0.0.0': [('note1', 'shaA')], + '1.0.0': [('note2', 'shaB'), ('note3', 'shaC')], + } + + versions = ['0.0.0', '1.0.0'] + + def _get_note_body(self, reporoot, filename, sha): + return self.note_bodies.get(filename, '') + + def setUp(self): + super(TestFormatterBase, self).setUp() + + def _load(ldr): + ldr._scanner_output = self.scanner_output + ldr._cache = { + 'file-contents': self.note_bodies + } + + self.c = config.Config('reporoot') + + with mock.patch('reno.loader.Loader._load_data', _load): + self.ldr = loader.Loader( + self.c, + ignore_cache=False, + ) + + +class TestFormatter(TestFormatterBase): + + note_bodies = { + 'note1': { + 'prelude': 'This is the prelude.', + }, + 'note2': { + 'issues': [ + 'This is the first issue.', + 'This is the second issue.', + ], + }, + 'note3': { + 'features': [ + 'We added a feature!', + ], + 'upgrade': None, + }, + } + + def test_with_title(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title='This is the title', + ) + self.assertIn('This is the title', result) + + def test_versions(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title='This is the title', + ) + self.assertIn('0.0.0\n=====', result) + self.assertIn('1.0.0\n=====', result) + + def test_without_title(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title=None, + ) + self.assertNotIn('This is the title', result) + + def test_default_section_order(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title=None, + ) + prelude_pos = result.index('This is the prelude.') + issues_pos = result.index('This is the first issue.') + features_pos = result.index('We added a feature!') + expected = [prelude_pos, features_pos, issues_pos] + actual = list(sorted([prelude_pos, features_pos, issues_pos])) + self.assertEqual(expected, actual) + + +class TestFormatterCustomSections(TestFormatterBase): + note_bodies = { + 'note1': { + 'prelude': 'This is the prelude.', + }, + 'note2': { + 'features': [ + 'This is the first feature.', + ], + 'api': [ + 'This is the API change for the first feature.', + ], + }, + 'note3': { + 'api': [ + 'This is the API change for the second feature.', + ], + 'features': [ + 'This is the second feature.', + ], + }, + } + + def setUp(self): + super(TestFormatterCustomSections, self).setUp() + self.c.override(sections=[ + ['api', 'API Changes'], + ['features', 'New Features'], + ]) + + def test_custom_section_order(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title=None, + ) + prelude_pos = result.index('This is the prelude.') + api_pos = result.index('API Changes') + features_pos = result.index('New Features') + expected = [prelude_pos, api_pos, features_pos] + actual = list(sorted([prelude_pos, features_pos, api_pos])) + self.assertEqual(expected, actual) + self.assertIn('.. _relnotes_1.0.0_API Changes:', result) + + +class TestFormatterCustomUnreleaseTitle(TestFormatterBase): + + note_bodies = { + 'note1': { + 'prelude': 'This is the prelude.', + }, + } + + scanner_output = { + '0.1.0-1': [('note1', 'shaA')], + } + + versions = ['0.1.0-1'] + + def test_with_title(self): + self.c.override(unreleased_version_title='Not Released') + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title='This is the title', + ) + self.assertIn('Not Released', result) + self.assertNotIn('0.1.0-1', result) + self.assertIn('.. _This is the title_Not Released:', result) + + def test_without_title(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title='This is the title', + ) + self.assertIn('0.1.0-1', result) + self.assertIn('.. _This is the title_0.1.0-1:', result) + + +class TestFormatterAnchors(TestFormatterBase): + + note_bodies = { + 'note1': { + 'prelude': 'This is the prelude.', + }, + 'note2': { + 'issues': [ + 'This is the first issue.', + 'This is the second issue.', + ], + }, + 'note3': { + 'features': [ + 'We added a feature!', + ], + 'upgrade': None, + }, + } + + def test_with_title(self): + self.c.override(unreleased_version_title='Not Released') + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title='This is the title', + ) + self.assertIn('.. _This is the title_0.0.0:', result) + self.assertIn('.. _This is the title_0.0.0_Prelude:', result) + self.assertIn('.. _This is the title_1.0.0:', result) + self.assertIn('.. _This is the title_1.0.0_Known Issues:', result) + + def test_without_title(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + ) + self.assertIn('.. _relnotes_0.0.0:', result) + self.assertIn('.. _relnotes_0.0.0_Prelude:', result) + self.assertIn('.. _relnotes_1.0.0:', result) + self.assertIn('.. _relnotes_1.0.0_Known Issues:', result) + + def test_with_branch_and_title(self): + self.c.override(unreleased_version_title='Not Released') + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + title='This is the title', + branch='stable/queens', + ) + self.assertIn('.. _This is the title_0.0.0_stable_queens:', result) + self.assertIn('.. _This is the title_0.0.0_stable_queens_Prelude:', + result) + self.assertIn('.. _This is the title_1.0.0_stable_queens:', result) + self.assertIn( + '.. _This is the title_1.0.0_stable_queens_Known Issues:', + result) + + def test_with_branch(self): + result = formatter.format_report( + loader=self.ldr, + config=self.c, + versions_to_include=self.versions, + branch='stable/queens', + ) + self.assertIn('.. _relnotes_0.0.0_stable_queens:', result) + self.assertIn('.. _relnotes_0.0.0_stable_queens_Prelude:', result) + self.assertIn('.. _relnotes_1.0.0_stable_queens:', result) + self.assertIn('.. _relnotes_1.0.0_stable_queens_Known Issues:', result) diff --git a/reno/tests/test_loader.py b/reno/tests/test_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..33b6d284f0b8f1bca10e336a87a81fc746709ffc --- /dev/null +++ b/reno/tests/test_loader.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import textwrap + +import fixtures +import mock +import six +import yaml + +from reno import config +from reno import loader +from reno.tests import base + + +class TestValidate(base.TestCase): + + scanner_output = { + '0.0.0': [('note', 'shaA')], + } + + versions = ['0.0.0'] + + def setUp(self): + super(TestValidate, self).setUp() + self.logger = self.useFixture( + fixtures.FakeLogger( + format='%(message)s', + level=logging.WARNING, + ) + ) + self.c = config.Config('reporoot') + + def _make_loader(self, note_bodies): + def _load(ldr): + ldr._scanner_output = self.scanner_output + ldr._cache = { + 'file-contents': {'note1': note_bodies}, + } + + with mock.patch('reno.loader.Loader._load_data', _load): + return loader.Loader( + self.c, + ignore_cache=False, + ) + + def test_prelude_list(self): + note_bodies = yaml.safe_load(textwrap.dedent(''' + prelude: + - This is the first comment. + - This is a second. + ''')) + self.assertIsInstance(note_bodies['prelude'], list) + ldr = self._make_loader(note_bodies) + ldr.parse_note_file('note1', None) + self.assertIn('prelude', self.logger.output) + + def test_non_prelude_single_string_converted_to_list(self): + note_bodies = yaml.safe_load(textwrap.dedent(''' + issues: | + This is a single string. + ''')) + print(type(note_bodies['issues'])) + self.assertIsInstance(note_bodies['issues'], six.string_types) + ldr = self._make_loader(note_bodies) + parse_results = ldr.parse_note_file('note1', None) + self.assertIsInstance(parse_results['issues'], list) + + def test_note_with_colon_as_dict(self): + note_bodies = yaml.safe_load(textwrap.dedent(''' + issues: + - This is the first issue. + - dict: This is parsed as a dictionary. + ''')) + self.assertIsInstance(note_bodies['issues'][-1], dict) + ldr = self._make_loader(note_bodies) + ldr.parse_note_file('note1', None) + self.assertIn('dict', self.logger.output) diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 1a334a2505382defba4a5e467e43898796824d3b..5c7a79497c4a3e12bf953effa7115c5a1377c275 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -12,22 +12,28 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import unicode_literals + import itertools import logging import os.path import re import subprocess -import textwrap +import time +import unittest + +from dulwich import diff_tree +from dulwich import objects +import fixtures +import mock +from testtools.content import text_content +from reno import config from reno import create from reno import scanner from reno.tests import base from reno import utils -import fixtures -import mock -from testtools.content import text_content - _SETUP_TEMPLATE = """ import setuptools @@ -65,7 +71,7 @@ class GPGKeyFixture(fixtures.Fixture): gnupg_version_re = re.compile('^gpg\s.*\s([\d+])\.([\d+])\.([\d+])') gnupg_version = utils.check_output(['gpg', '--version'], cwd=tempdir.path) - for line in gnupg_version[0].split('\n'): + for line in gnupg_version.split('\n'): gnupg_version = gnupg_version_re.match(line) if gnupg_version: gnupg_version = (int(gnupg_version.group(1)), @@ -98,45 +104,70 @@ class GPGKeyFixture(fixtures.Fixture): # Note that --quick-random (--debug-quick-random in GnuPG 2.x) # does not have a corresponding preferences file setting and # must be passed explicitly on the command line instead - # if gnupg_version[0] == 1: - # gnupg_random = '--quick-random' - # elif gnupg_version[0] >= 2: - # gnupg_random = '--debug-quick-random' - # else: - # gnupg_random = '' + if gnupg_version[0] == 1: + gnupg_random = '--quick-random' + elif gnupg_version[0] >= 2: + gnupg_random = '--debug-quick-random' + else: + gnupg_random = '' + cmd = ['gpg', '--gen-key', '--batch'] + if gnupg_random: + cmd.append(gnupg_random) + cmd.append(config_file) subprocess.check_call( - ['gpg', '--gen-key', '--batch', - # gnupg_random, - config_file], - cwd=tempdir.path) + cmd, + cwd=tempdir.path, + # Direct stderr to its own pipe, from which we don't read, + # to quiet the commands. + stderr=subprocess.PIPE, + ) -class Base(base.TestCase): +class GitRepoFixture(fixtures.Fixture): + + logger = logging.getLogger('git') + + def __init__(self, reporoot): + self.reporoot = reporoot + super(GitRepoFixture, self).__init__() + + def setUp(self): + super(GitRepoFixture, self).setUp() + self.useFixture(GPGKeyFixture()) + os.makedirs(self.reporoot) + self.git('init', '.') + self.git('config', '--local', 'user.email', 'example@example.com') + self.git('config', '--local', 'user.name', 'reno developer') + self.git('config', '--local', 'user.signingkey', + 'example@example.com') - def _run_git(self, *args): - return utils.check_output( + def git(self, *args): + self.logger.debug('$ git %s', ' '.join(args)) + output = utils.check_output( ['git'] + list(args), cwd=self.reporoot, ) + self.logger.debug(output) + return output - def _git_setup(self): - os.makedirs(self.reporoot) - self._run_git('init', '.') - self._run_git('config', '--local', 'user.email', 'example@example.com') - self._run_git('config', '--local', 'user.name', 'reno developer') - self._run_git('config', '--local', 'user.signingkey', - 'example@example.com') - - def _git_commit(self, message='commit message'): - self._run_git('add', '.') - self._run_git('commit', '-m', message) + def commit(self, message='commit message'): + self.git('add', '.') + self.git('commit', '-m', message) + self.git('show', '--pretty=format:%H') + time.sleep(0.1) # force a delay between commits - def _add_other_file(self, name): + def add_file(self, name): with open(os.path.join(self.reporoot, name), 'w') as f: f.write('adding %s\n' % name) - self._git_commit('add %s' % name) + self.commit('add %s' % name) + + +class Base(base.TestCase): + + logger = logging.getLogger('test') - def _add_notes_file(self, slug='slug', commit=True, legacy=False): + def _add_notes_file(self, slug='slug', commit=True, legacy=False, + contents='i-am-also-a-template'): n = self.get_note_num() if legacy: basename = '%016x-%s.yaml' % (n, slug) @@ -144,8 +175,8 @@ class Base(base.TestCase): basename = '%s-%016x.yaml' % (slug, n) filename = os.path.join(self.reporoot, 'releasenotes', 'notes', basename) - create._make_note_file(filename) - self._git_commit('add %s' % basename) + create._make_note_file(filename, contents) + self.repo.commit('add %s' % basename) return os.path.join('releasenotes', 'notes', basename) def _make_python_package(self): @@ -160,11 +191,11 @@ class Base(base.TestCase): init = os.path.join(pkgdir, '__init__.py') with open(init, 'w') as f: f.write("Test package") - self._git_commit('add test package') + self.repo.commit('add test package') def setUp(self): super(Base, self).setUp() - self.logger = self.useFixture( + self.fake_logger = self.useFixture( fixtures.FakeLogger( format='%(levelname)8s %(name)s %(message)s', level=logging.DEBUG, @@ -175,15 +206,11 @@ class Base(base.TestCase): # directory to permit using git config --global without stepping on # developer configuration. self.useFixture(fixtures.TempHomeDir()) - self.useFixture(GPGKeyFixture()) self.useFixture(fixtures.NestedTempfile()) self.temp_dir = self.useFixture(fixtures.TempDir()).path self.reporoot = os.path.join(self.temp_dir, 'reporoot') - self.notesdir = os.path.join(self.reporoot, - 'releasenotes', - 'notes', - ) - self._git_setup() + self.repo = self.useFixture(GitRepoFixture(self.reporoot)) + self.c = config.Config(self.reporoot) self._counter = itertools.count(1) self.get_note_num = lambda: next(self._counter) @@ -192,10 +219,8 @@ class BasicTest(Base): def test_non_python_no_tags(self): filename = self._add_notes_file() - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -208,10 +233,8 @@ class BasicTest(Base): def test_python_no_tags(self): self._make_python_package() filename = self._add_notes_file() - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -223,12 +246,10 @@ class BasicTest(Base): def test_note_before_tag(self): filename = self._add_notes_file() - self._add_other_file('not-a-release-note.txt') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.repo.add_file('not-a-release-note.txt') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -240,11 +261,9 @@ class BasicTest(Base): def test_note_commit_tagged(self): filename = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -254,14 +273,26 @@ class BasicTest(Base): results, ) + def test_tag_with_v_prefix(self): + filename = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'tag with v prefix', 'v1.0.0') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'v1.0.0': [filename]}, + results, + ) + def test_note_commit_after_tag(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') filename = self._add_notes_file() - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -273,13 +304,11 @@ class BasicTest(Base): def test_other_commit_after_tag(self): filename = self._add_notes_file() - self._add_other_file('ignore-1.txt') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - self._add_other_file('ignore-2.txt') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.repo.add_file('ignore-1.txt') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.add_file('ignore-2.txt') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -291,13 +320,11 @@ class BasicTest(Base): def test_multiple_notes_after_tag(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file() f2 = self._add_notes_file() - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -311,11 +338,9 @@ class BasicTest(Base): self._make_python_package() f1 = self._add_notes_file(commit=False) f2 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -327,14 +352,12 @@ class BasicTest(Base): def test_multiple_tags(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = self._add_notes_file() - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -348,16 +371,14 @@ class BasicTest(Base): def test_rename_file(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = f1.replace('slug1', 'slug2') - self._run_git('mv', f1, f2) - self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.repo.git('mv', f1, f2) + self.repo.commit('rename note file') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -370,16 +391,14 @@ class BasicTest(Base): def test_rename_file_sort_earlier(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = f1.replace('slug1', 'slug0') - self._run_git('mv', f1, f2) - self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.repo.git('mv', f1, f2) + self.repo.commit('rename note file') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -392,16 +411,14 @@ class BasicTest(Base): def test_edit_file(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') with open(os.path.join(self.reporoot, f1), 'w') as f: f.write('---\npreamble: new contents for file') - self._git_commit('edit note file') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.repo.commit('edit note file') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -414,16 +431,14 @@ class BasicTest(Base): def test_legacy_file(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1', legacy=True) - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = f1.replace('slug1', 'slug2') - self._run_git('mv', f1, f2) - self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.repo.git('mv', f1, f2) + self.repo.commit('rename note file') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -436,19 +451,17 @@ class BasicTest(Base): def test_rename_legacy_file_to_new(self): self._make_python_package() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file('slug1', legacy=True) - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') # Rename the file with the new convention of placing the UUID # after the slug instead of before. f2 = f1.replace('0000000000000001-slug1', 'slug1-0000000000000001') - self._run_git('mv', f1, f2) - self._git_commit('rename note file') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - ) + self.repo.git('mv', f1, f2) + self.repo.commit('rename note file') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() @@ -459,380 +472,2001 @@ class BasicTest(Base): results, ) + def test_limit_by_earliest_version(self): + self._make_python_package() + self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f2 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'middle tag', '2.0.0') + f3 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'last tag', '3.0.0') + self.c.override( + earliest_version='2.0.0', + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'2.0.0': [f2], + '3.0.0': [f3], + }, + results, + ) -class MergeCommitTest(Base): - - def test_1(self): - # Create changes on master and in the branch - # in order so the history is "normal" - n1 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - self._run_git('checkout', '-b', 'test_merge_commit') - n2 = self._add_notes_file() - self._run_git('checkout', 'master') - self._add_other_file('ignore-1.txt') - self._run_git('merge', '--no-ff', 'test_merge_commit') - self._add_other_file('ignore-2.txt') - self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', + def test_delete_file(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file('slug1') + f2 = self._add_notes_file('slug2') + self.repo.git('rm', f1) + self.repo.commit('remove note file') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'2.0.0': [f2], + }, + results, ) + + def test_rename_then_delete_file(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file('slug1') + f2 = f1.replace('slug1', 'slug2') + self.repo.git('mv', f1, f2) + self.repo.git('status') + self.repo.commit('rename note file') + self.repo.git('rm', f2) + self.repo.commit('remove note file') + f3 = self._add_notes_file('slug3') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') + log_results = self.repo.git('log', '--topo-order', + '--pretty=%H %d', + '--name-only') + self.addDetail('git log', text_content(log_results)) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() } self.assertEqual( - {'1.0.0': [n1], - '2.0.0': [n2]}, + {'2.0.0': [f3], + }, results, ) + + def test_staged_file(self): + # Prove that we can get a file we have staged. + # Start with a standard commit and tag + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + # Now stage a release note + n = self.get_note_num() + basename = 'staged-note-%016x.yaml' % n + filename = os.path.join(self.reporoot, 'releasenotes', 'notes', + basename) + create._make_note_file(filename, 'staged note') + self.repo.git('add', filename) + status_results = self.repo.git('status') + self.addDetail('git status', text_content(status_results)) + # Now run the scanner + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() self.assertEqual( - ['2.0.0', '1.0.0'], - list(raw_results.keys()), + {'*working-copy*': [ + (os.path.join('releasenotes', 'notes', basename), + None)], + }, + raw_results, ) - def test_2(self): - # Create changes on the branch before the tag into which it is - # actually merged. - self._add_other_file('ignore-0.txt') - self._run_git('checkout', '-b', 'test_merge_commit') - n1 = self._add_notes_file() - self._run_git('checkout', 'master') - n2 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - self._add_other_file('ignore-1.txt') - self._run_git('merge', '--no-ff', 'test_merge_commit') - self._add_other_file('ignore-2.txt') - self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', + @unittest.skip('dulwich does not know how to identify new files') + def test_added_tagged_not_staged(self): + # Prove that we can get a file we have created but not staged. + # Start with a standard commit and tag + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + # Now create a note without staging it + n = self.get_note_num() + basename = 'staged-note-%016x.yaml' % n + filename = os.path.join(self.reporoot, 'releasenotes', 'notes', + basename) + create._make_note_file(filename, 'staged note') + status_results = self.repo.git('status') + self.addDetail('git status', text_content(status_results)) + # Now run the scanner + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + # Take the staged version of the file, but associate it with + # tagged version 1.0.0 because the file was added before that + # version. + self.assertEqual( + {'1.0.0': [(os.path.join('releasenotes', 'notes', basename), + None)], + }, + raw_results, + ) + + def test_modified_tagged_not_staged(self): + # Prove that we can get a file we have changed but not staged. + # Start with a standard commit and tag + self._make_python_package() + f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + # Now modify the note + fullpath = os.path.join(self.repo.reporoot, f1) + with open(fullpath, 'w') as f: + f.write('modified first note') + status_results = self.repo.git('status') + self.addDetail('git status', text_content(status_results)) + # Now run the scanner + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + # Take the staged version of the file, but associate it with + # tagged version 1.0.0 because the file was added before that + # version. + self.assertEqual( + {'1.0.0': [(f1, None)], + }, + raw_results, + ) + + def test_stop_on_master_with_other_branch(self): + self._make_python_package() + self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'middle tag', '2.0.0') + f3 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'last tag', '3.0.0') + self.repo.git('branch', 'stable/a') + f4 = self._add_notes_file() + self.c.override( + earliest_version=None, ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() } self.assertEqual( - {'1.0.0': [n2], - '2.0.0': [n1]}, + {'3.0.0-1': [f4], + '3.0.0': [f3], + }, results, ) + + def test_stop_on_master_without_limits_or_branches(self): + self._make_python_package() + f1 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f2 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'middle tag', '2.0.0') + f3 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'last tag', '3.0.0') + f4 = self._add_notes_file() + self.c.override( + earliest_version=None, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } self.assertEqual( - ['2.0.0', '1.0.0'], - list(raw_results.keys()), + {'3.0.0-1': [f4], + '3.0.0': [f3], + '2.0.0': [f2], + '1.0.0': [f1], + }, + results, ) - def test_3(self): - # Create changes on the branch before the tag into which it is - # actually merged, with another tag in between the time of the - # commit and the time of the merge. This should reflect the - # order of events described in bug #1522153. - self._add_other_file('ignore-0.txt') - self._run_git('checkout', '-b', 'test_merge_commit') - n1 = self._add_notes_file() - self._run_git('checkout', 'master') - n2 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - self._add_other_file('ignore-1.txt') - self._run_git('tag', '-s', '-m', 'second tag', '1.1.0') - self._run_git('merge', '--no-ff', 'test_merge_commit') - self._add_other_file('ignore-2.txt') - self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') - self._add_other_file('ignore-3.txt') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', + +class IgnoreTest(Base): + + def test_by_fullname(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file() + f2 = self._add_notes_file() + self.c.override( + ignore_notes=[f1], ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() } - # Since the 1.1.0 tag has no notes files, it does not appear - # in the output. It's only there to trigger the bug as it was - # originally reported. self.assertEqual( - {'1.0.0': [n2], - '2.0.0': [n1]}, + {'1.0.0-2': [f2]}, results, ) + + def test_by_basename(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file() + f2 = self._add_notes_file() + self.c.override( + ignore_notes=[os.path.basename(f1)], + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } self.assertEqual( - ['2.0.0', '1.0.0'], - list(raw_results.keys()), + {'1.0.0-2': [f2]}, + results, ) - def test_4(self): - # Create changes on the branch before the tag into which it is - # actually merged, with another tag in between the time of the - # commit and the time of the merge. This should reflect the - # order of events described in bug #1522153. - self._add_other_file('ignore-0.txt') - self._run_git('checkout', '-b', 'test_merge_commit') - n1 = self._add_notes_file() - self._run_git('checkout', 'master') - n2 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - self._add_other_file('ignore-1.txt') - n3 = self._add_notes_file() - self._run_git('tag', '-s', '-m', 'second tag', '1.1.0') - self._run_git('merge', '--no-ff', 'test_merge_commit') - self._add_other_file('ignore-2.txt') - self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') - self._add_other_file('ignore-3.txt') - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', + def test_by_uid(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file() + f2 = self._add_notes_file() + self.c.override( + ignore_notes=[scanner._get_unique_id(f1)], ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() } self.assertEqual( - {'1.0.0': [n2], - '1.1.0': [n3], - '2.0.0': [n1]}, + {'1.0.0-2': [f2]}, results, ) + + def test_by_multiples(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + f1 = self._add_notes_file() + f2 = self._add_notes_file() + self.c.override( + ignore_notes=[ + scanner._get_unique_id(f1), + scanner._get_unique_id(f2), + ], + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } self.assertEqual( - ['2.0.0', '1.1.0', '1.0.0'], - list(raw_results.keys()), + {}, + results, ) -class UniqueIdTest(Base): +class FileContentsTest(Base): - def test_legacy(self): - uid = scanner._get_unique_id( - 'releasenotes/notes/0000000000000001-slug1.yaml' + def test_basic_file(self): + # Prove that we can get a file we have committed. + f1 = self._add_notes_file(contents='well-known-contents') + r = scanner.RenoRepo(self.reporoot) + contents = r.get_file_at_commit(f1, 'HEAD') + self.assertEqual( + b'well-known-contents', + contents, ) - self.assertEqual('0000000000000001', uid) - def test_modern(self): - uid = scanner._get_unique_id( - 'releasenotes/notes/slug1-0000000000000001.yaml' + def test_no_such_file(self): + # Returns None when the file does not exist at all. + # (we have to commit something, otherwise there is no HEAD) + self._add_notes_file(contents='well-known-contents') + r = scanner.RenoRepo(self.reporoot) + contents = r.get_file_at_commit('no-such-dir/no-such-file', 'HEAD') + self.assertEqual( + None, + contents, ) - self.assertEqual('0000000000000001', uid) + def test_edit_file_and_commit(self): + # Prove that we can edit a file and see the changes. + f1 = self._add_notes_file(contents='initial-contents') + with open(os.path.join(self.reporoot, f1), 'w') as f: + f.write('new contents for file') + self.repo.commit('edit note file') + r = scanner.RenoRepo(self.reporoot) + contents = r.get_file_at_commit(f1, 'HEAD') + self.assertEqual( + b'new contents for file', + contents, + ) -class BranchTest(Base): + def test_earlier_version_of_edited_file(self): + # Prove that we are not always just returning the most current + # version of a file. + f1 = self._add_notes_file(contents='initial-contents') + with open(os.path.join(self.reporoot, f1), 'w') as f: + f.write('new contents for file') + self.repo.commit('edit note file') + self.scanner = scanner.Scanner(self.c) + r = scanner.RenoRepo(self.reporoot) + head = r.head() + parent = r.get_parents(head)[0] + parent = parent.decode('ascii') + contents = r.get_file_at_commit(f1, parent) + self.assertEqual( + b'initial-contents', + contents, + ) - def setUp(self): - super(BranchTest, self).setUp() - self._make_python_package() - self.f1 = self._add_notes_file('slug1') - self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') - self.f2 = self._add_notes_file('slug2') - self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') - self._add_notes_file('slug3') - self._run_git('tag', '-s', '-m', 'first tag', '3.0.0') + def test_edit_file_without_commit(self): + # Prove we are not picking up the contents from the local + # filesystem outside of the git history. + f1 = self._add_notes_file(contents='initial-contents') + with open(os.path.join(self.reporoot, f1), 'w') as f: + f.write('new contents for file') + r = scanner.RenoRepo(self.reporoot) + contents = r.get_file_at_commit(f1, 'HEAD') + self.assertEqual( + b'initial-contents', + contents, + ) - def test_files_current_branch(self): - self._run_git('checkout', '2.0.0') - self._run_git('checkout', '-b', 'stable/2') - f21 = self._add_notes_file('slug21') - log_text = self._run_git('log') - self.addDetail('git log', text_content(log_text)) - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', + def test_staged_file(self): + # Prove we are not picking up the contents from the local + # filesystem outside of the git history. + f1 = self._add_notes_file(contents='initial-contents') + with open(os.path.join(self.reporoot, f1), 'w') as f: + f.write('new contents for file') + r = scanner.RenoRepo(self.reporoot) + contents = r.get_file_at_commit(f1, None) + self.assertEqual( + 'new contents for file', + contents, ) + + +class PreReleaseTest(Base): + + def test_alpha(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0a1') + f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0a2') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() } self.assertEqual( - { - '1.0.0': [self.f1], - '2.0.0': [self.f2], - '2.0.0-1': [f21], - }, + {'1.0.0.0a2': [f1], + }, results, ) - def test_files_stable_from_master(self): - self._run_git('checkout', '2.0.0') - self._run_git('checkout', '-b', 'stable/2') - f21 = self._add_notes_file('slug21') - self._run_git('checkout', 'master') - log_text = self._run_git('log', '--pretty=%x00%H %d', '--name-only', - 'stable/2') - self.addDetail('git log', text_content(log_text)) - raw_results = scanner.get_notes_by_version( - self.reporoot, - 'releasenotes/notes', - 'stable/2', - ) + def test_beta(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0b1') + f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0b2') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] for (k, v) in raw_results.items() } self.assertEqual( - { - '1.0.0': [self.f1], - '2.0.0': [self.f2], - '2.0.0-1': [f21], - }, + {'1.0.0.0b2': [f1], + }, results, ) + def test_release_candidate(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0rc1') + f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0.0rc2') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0.0rc2': [f1], + }, + results, + ) -class GetTagsParseTest(base.TestCase): - - EXPECTED = [ - '2.0.0', - '1.8.1', - '1.8.0', - '1.7.1', - '1.7.0', - '1.6.0', - '1.5.0', - '1.4.0', - '1.3.0', - '1.2.0', - '1.1.0', - '1.0.0', - '0.11.2', - '0.11.1', - '0.11.0', - '0.10.1', - '0.10.0', - '0.9.0', - '0.8.0', - '0.7.1', - '0.7.0', - '0.6.0', - '0.5.1', - '0.5.0', - '0.4.2', - '0.4.1', - '0.4.0', - '0.3.2', - '0.3.1', - '0.3.0', - '0.2.5', - '0.2.4', - '0.2.3', - '0.2.2', - '0.2.1', - '0.2.0', - '0.1.3', - '0.1.2', - '0.1.1', - '0.1.0', - ] - - def test_keystoneclient_ubuntu_1_9_1(self): - # git 1.9.1 as it produces output on ubuntu for python-keystoneclient - # git log --simplify-by-decoration --pretty="%d" - tag_list_output = textwrap.dedent(""" - (HEAD, origin/master, origin/HEAD, gerrit/master, master) - (apu/master) - (tag: 2.0.0) - (tag: 1.8.1) - (tag: 1.8.0) - (tag: 1.7.1) - (tag: 1.7.0) - (tag: 1.6.0) - (tag: 1.5.0) - (tag: 1.4.0) - (uncap-requirements) - (tag: 1.3.0) - (tag: 1.2.0) - (tag: 1.1.0) - (tag: 1.0.0) - (tag: 0.11.2) - (tag: 0.11.1) - (tag: 0.11.0) - (tag: 0.10.1) - (tag: 0.10.0) - (tag: 0.9.0) - (tag: 0.8.0) - (tag: 0.7.1) - (tag: 0.7.0) - (tag: 0.6.0) - (tag: 0.5.1) - (tag: 0.5.0) - (tag: 0.4.2) - (tag: 0.4.1) - (tag: 0.4.0) - (tag: 0.3.2) - (tag: 0.3.1) - (tag: 0.3.0) - (tag: 0.2.5) - (tag: 0.2.4) - (tag: 0.2.3) - (tag: 0.2.2) - (tag: 0.2.1) - (tag: 0.2.0) - - (origin/feature/keystone-v3, gerrit/feature/keystone-v3) - (tag: 0.1.3) - (tag: 0.1.2) - (tag: 0.1.1) - (tag: 0.1.0) - (tag: folsom-1) - (tag: essex-rc1) - (tag: essex-4) - (tag: essex-3) - """) - with mock.patch('reno.utils.check_output') as co: - co.return_value = tag_list_output - actual = scanner._get_version_tags_on_branch('reporoot', - branch=None) - self.assertEqual(self.EXPECTED, actual) - - def test_keystoneclient_rhel_1_7_1(self): - # git 1.7.1 as it produces output on RHEL 6 for python-keystoneclient - # git log --simplify-by-decoration --pretty="%d" - tag_list_output = textwrap.dedent(""" - (HEAD, origin/master, origin/HEAD, master) - (tag: 2.0.0) - (tag: 1.8.1) - (tag: 1.8.0) - (tag: 1.7.1) - (tag: 1.7.0) - (tag: 1.6.0) - (tag: 1.5.0) - (tag: 1.4.0) - (tag: 1.3.0) - (tag: 1.2.0) - (tag: 1.1.0) - (tag: 1.0.0) - (tag: 0.11.2) - (tag: 0.11.1) - (tag: 0.11.0) - (tag: 0.10.1) - (tag: 0.10.0) - (tag: 0.9.0) - (tag: 0.8.0) - (tag: 0.7.1) - (tag: 0.7.0) - (tag: 0.6.0) - (tag: 0.5.1) - (tag: 0.5.0) - (tag: 0.4.2) - (tag: 0.4.1) - (tag: 0.4.0) - (tag: 0.3.2) - (tag: 0.3.1) - (tag: 0.3.0) - (tag: 0.2.5) - (tag: 0.2.4) - (tag: 0.2.3) - (tag: 0.2.2) - (tag: 0.2.1) - (tag: 0.2.0) - (tag: 0.1.3) - (0.1.2) - (tag: 0.1.1) - (0.1.0) - (tag: folsom-1) - (tag: essex-rc1) - (essex-4) - (essex-3) - """) - with mock.patch('reno.utils.check_output') as co: - co.return_value = tag_list_output - actual = scanner._get_version_tags_on_branch('reporoot', - branch=None) - self.assertEqual(self.EXPECTED, actual) + def test_tag_with_v_prefix(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'first tag', 'v1.0.0.0a1') + f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', 'v1.0.0.0a2') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'v1.0.0.0a2': [f1], + }, + results, + ) + + def test_collapse(self): + files = [] + self._make_python_package() + files.append(self._add_notes_file('slug1')) + self.repo.git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') + files.append(self._add_notes_file('slug2')) + self.repo.git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') + files.append(self._add_notes_file('slug3')) + self.repo.git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') + files.append(self._add_notes_file('slug4')) + self.repo.git('tag', '-s', '-m', 'full release tag', '1.0.0') + self.c.override( + collapse_pre_releases=True, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0': files, + }, + results, + ) + + def test_collapse_without_full_release(self): + self._make_python_package() + f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') + f2 = self._add_notes_file('slug2') + self.repo.git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') + f3 = self._add_notes_file('slug3') + self.repo.git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') + self.c.override( + collapse_pre_releases=True, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0.0a1': [f1], + '1.0.0.0b1': [f2], + '1.0.0.0rc1': [f3], + }, + results, + ) + + def test_collapse_without_notes(self): + self._make_python_package() + self.repo.git('tag', '-s', '-m', 'earlier tag', '0.1.0') + f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'alpha tag', '1.0.0.0a1') + f2 = self._add_notes_file('slug2') + self.repo.git('tag', '-s', '-m', 'beta tag', '1.0.0.0b1') + f3 = self._add_notes_file('slug3') + self.repo.git('tag', '-s', '-m', 'release candidate tag', '1.0.0.0rc1') + self.c.override( + collapse_pre_releases=True, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0.0a1': [f1], + '1.0.0.0b1': [f2], + '1.0.0.0rc1': [f3], + }, + results, + ) + + +class MergeCommitTest(Base): + + def test_1(self): + # Create changes on master and in the branch + # in order so the history is "normal" + n1 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('checkout', '-b', 'test_merge_commit') + n2 = self._add_notes_file() + self.repo.git('checkout', 'master') + self.repo.add_file('ignore-1.txt') + # Merge the branch into master. + self.repo.git('merge', '--no-ff', 'test_merge_commit') + time.sleep(0.1) # force a delay between commits + self.repo.add_file('ignore-2.txt') + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.0') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0': [n1], + '2.0.0': [n2]}, + results, + ) + self.assertEqual( + ['2.0.0', '1.0.0'], + list(raw_results.keys()), + ) + + def test_2(self): + # Create changes on the branch before the tag into which it is + # actually merged. + self.repo.add_file('ignore-0.txt') + self.repo.git('checkout', '-b', 'test_merge_commit') + n1 = self._add_notes_file() + self.repo.git('checkout', 'master') + n2 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.add_file('ignore-1.txt') + # Merge the branch into master. + self.repo.git('merge', '--no-ff', 'test_merge_commit') + time.sleep(0.1) # force a delay between commits + self.repo.git('show') + self.repo.add_file('ignore-2.txt') + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.0') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0': [n2], + '2.0.0': [n1]}, + results, + ) + self.assertEqual( + ['2.0.0', '1.0.0'], + list(raw_results.keys()), + ) + + def test_3(self): + # Create changes on the branch before the tag into which it is + # actually merged, with another tag in between the time of the + # commit and the time of the merge. This should reflect the + # order of events described in bug #1522153. + self.repo.add_file('ignore-0.txt') + self.repo.git('checkout', '-b', 'test_merge_commit') + n1 = self._add_notes_file() + self.repo.git('checkout', 'master') + n2 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.add_file('ignore-1.txt') + self.repo.git('tag', '-s', '-m', 'second tag', '1.1.0') + self.repo.git('merge', '--no-ff', 'test_merge_commit') + time.sleep(0.1) # force a delay between commits + self.repo.add_file('ignore-2.txt') + self.repo.git('tag', '-s', '-m', 'third tag', '2.0.0') + self.repo.add_file('ignore-3.txt') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + # Since the 1.1.0 tag has no notes files, it does not appear + # in the output. It's only there to trigger the bug as it was + # originally reported. + self.assertEqual( + {'1.0.0': [n2], + '2.0.0': [n1]}, + results, + ) + self.assertEqual( + ['2.0.0', '1.0.0'], + list(raw_results.keys()), + ) + + def test_4(self): + # Create changes on the branch before the tag into which it is + # actually merged, with another tag in between the time of the + # commit and the time of the merge. This should reflect the + # order of events described in bug #1522153. + self.repo.add_file('ignore-0.txt') + self.repo.git('checkout', '-b', 'test_merge_commit') + n1 = self._add_notes_file() + self.repo.git('checkout', 'master') + n2 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.add_file('ignore-1.txt') + n3 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'second tag', '1.1.0') + self.repo.git('merge', '--no-ff', 'test_merge_commit') + time.sleep(0.1) # force a delay between commits + self.repo.add_file('ignore-2.txt') + self.repo.git('tag', '-s', '-m', 'third tag', '2.0.0') + self.repo.add_file('ignore-3.txt') + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0': [n2], + '1.1.0': [n3], + '2.0.0': [n1]}, + results, + ) + self.assertEqual( + ['2.0.0', '1.1.0', '1.0.0'], + list(raw_results.keys()), + ) + + +class NullMergeTest(Base): + + def setUp(self): + super(NullMergeTest, self).setUp() + self.repo.add_file('ignore-0.txt') + self.n1 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + + # Create a branch, add a note, and tag it. + self.repo.git('checkout', '-b', 'test_ignore_null_merge') + self.n2 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.0') + + # Move back to master and advance it. + self.repo.git('checkout', 'master') + self.repo.add_file('ignore-1.txt') + self.n3 = self._add_notes_file() + + # Merge only the tag from the first branch back into master. + self.repo.git( + 'merge', '--no-ff', '--strategy', 'ours', '2.0.0', + ) + + # Add another note file. + self.n4 = self._add_notes_file() + self.repo.git('tag', '-s', '-m', 'third tag', '3.0.0') + + self.repo.git('log', '--decorate', '--oneline', '--graph', '--all') + # The results should look like: + # + # * afea344 (HEAD -> master, tag: 3.0.0) add slug-0000000000000004.yaml + # * 7bb295c Merge tag '2.0.0' + # |\ + # | * 260c80b (tag: 2.0.0, test_ignore_null_merge) add slug-0000000000000002.yaml # noqa + # * | 5981ae3 add slug-0000000000000003.yaml + # * | 00f9376 add ignore-1.txt + # |/ + # * d24faf9 (tag: 1.0.0) add slug-0000000000000001.yaml + # * 6c221cd add ignore-0.txt + + def test_ignore(self): + # The scanner should skip over the null-merge and include the + # notes that come before the version being merged in, up to + # the base of the previous branch. + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0': [self.n1], + '3.0.0': [self.n3, self.n4]}, + results, + ) + + def test_follow(self): + # The scanner should not skip over the null-merge. The output + # should include the 2.0.0 tag that was merged in, as well as + # the earlier 1.0.0 version. + self.c.override( + ignore_null_merges=False, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + {'1.0.0': [self.n1], + '2.0.0': [self.n2, self.n3], + '3.0.0': [self.n4]}, + results, + ) + + +class UniqueIdTest(Base): + + def test_legacy(self): + uid = scanner._get_unique_id( + 'releasenotes/notes/0000000000000001-slug1.yaml' + ) + self.assertEqual('0000000000000001', uid) + + def test_modern(self): + uid = scanner._get_unique_id( + 'releasenotes/notes/slug1-0000000000000001.yaml' + ) + self.assertEqual('0000000000000001', uid) + + +class BranchBaseTest(Base): + + def setUp(self): + super(BranchBaseTest, self).setUp() + self._make_python_package() + self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self._add_notes_file('slug2') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') + self._add_notes_file('slug3') + self.repo.git('tag', '-s', '-m', 'first tag', '3.0.0') + self.repo.git('checkout', '2.0.0') + self.repo.git('branch', 'not-master') + self.repo.git('checkout', 'master') + self.scanner = scanner.Scanner(self.c) + + def test_current_branch_no_extra_commits(self): + # checkout the branch and then ask for its base + self.repo.git('checkout', 'not-master') + self.assertEqual( + '2.0.0', + self.scanner._get_branch_base('not-master'), + ) + + def test_current_branch_extra_commit(self): + # checkout the branch and then ask for its base + self.repo.git('checkout', 'not-master') + self._add_notes_file('slug4') + self.assertEqual( + '2.0.0', + self.scanner._get_branch_base('not-master'), + ) + + def test_alternate_branch_no_extra_commits(self): + # checkout master and then ask for the alternate branch base + self.repo.git('checkout', 'master') + self.assertEqual( + '2.0.0', + self.scanner._get_branch_base('not-master'), + ) + + def test_alternate_branch_extra_commit(self): + # checkout master and then ask for the alternate branch base + self.repo.git('checkout', 'not-master') + self._add_notes_file('slug4') + self.repo.git('checkout', 'master') + self.assertEqual( + '2.0.0', + self.scanner._get_branch_base('not-master'), + ) + + def test_no_tag_at_base(self): + # remove the tag at the branch point + self.repo.git('tag', '-d', '2.0.0') + self._add_notes_file('slug4') + self.repo.git('checkout', 'master') + self.assertIsNone( + self.scanner._get_branch_base('not-master') + ) + + +class BranchTest(Base): + + def setUp(self): + super(BranchTest, self).setUp() + self._make_python_package() + self.f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.f2 = self._add_notes_file('slug2') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') + self.f3 = self._add_notes_file('slug3') + self.repo.git('tag', '-s', '-m', 'first tag', '3.0.0') + + def test_files_current_branch(self): + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'stable/2') + f21 = self._add_notes_file('slug21') + log_text = self.repo.git('log', '--decorate') + self.addDetail('git log', text_content(log_text)) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '2.0.0-1': [f21], + '2.0.0': [self.f2], + }, + results, + ) + + def test_files_stable_from_master(self): + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'stable/2') + f21 = self._add_notes_file('slug21') + self.repo.git('checkout', 'master') + log_text = self.repo.git('log', '--pretty=%x00%H %d', '--name-only', + 'stable/2') + self.addDetail('git log', text_content(log_text)) + self.c.override( + branch='stable/2', + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '2.0.0': [self.f2], + '2.0.0-1': [f21], + }, + results, + ) + + def test_files_stable_from_master_no_stop_base(self): + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'stable/2') + f21 = self._add_notes_file('slug21') + self.repo.git('checkout', 'master') + log_text = self.repo.git('log', '--pretty=%x00%H %d', '--name-only', + 'stable/2') + self.addDetail('git log', text_content(log_text)) + self.c.override( + branch='stable/2', + ) + self.c.override( + stop_at_branch_base=False, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '1.0.0': [self.f1], + '2.0.0': [self.f2], + '2.0.0-1': [f21], + }, + results, + ) + + def test_pre_release_branch_no_collapse(self): + f4 = self._add_notes_file('slug4') + self.repo.git('tag', '-s', '-m', 'pre-release', '4.0.0.0rc1') + # Add a commit on master after the tag + self._add_notes_file('slug5') + # Move back to the tag and create the branch + self.repo.git('checkout', '4.0.0.0rc1') + self.repo.git('checkout', '-b', 'stable/4') + # Create a commit on the branch + f41 = self._add_notes_file('slug41') + log_text = self.repo.git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self.repo.git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + collapse_pre_releases=False, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0.0rc1': [f4], + '4.0.0.0rc1-1': [f41], + }, + results, + ) + + def test_pre_release_branch_collapse(self): + f4 = self._add_notes_file('slug4') + self.repo.git('tag', '-s', '-m', 'pre-release', '4.0.0.0rc1') + # Add a commit on master after the tag + self._add_notes_file('slug5') + # Move back to the tag and create the branch + self.repo.git('checkout', '4.0.0.0rc1') + self.repo.git('checkout', '-b', 'stable/4') + # Create a commit on the branch + f41 = self._add_notes_file('slug41') + self.repo.git('tag', '-s', '-m', 'release', '4.0.0') + log_text = self.repo.git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self.repo.git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + collapse_pre_releases=True, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0': [f4, f41], + }, + results, + ) + + def test_pre_release_note_before_branch(self): + f4 = self._add_notes_file('slug4') + self.repo.git('tag', '-s', '-m', 'beta', '4.0.0.0b1') + self.repo.add_file('not-a-release-note.txt') + self.repo.git('tag', '-s', '-m', 'pre-release', '4.0.0.0rc1') + # Add a commit on master after the tag + self._add_notes_file('slug5') + # Move back to the tag and create the branch + self.repo.git('checkout', '4.0.0.0rc1') + self.repo.git('checkout', '-b', 'stable/4') + # Create a commit on the branch + f41 = self._add_notes_file('slug41') + self.repo.git('tag', '-s', '-m', 'release', '4.0.0') + log_text = self.repo.git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self.repo.git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + collapse_pre_releases=True, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0': [f4, f41], + }, + results, + ) + + def test_full_release_branch(self): + f4 = self._add_notes_file('slug4') + self.repo.git('tag', '-s', '-m', 'release', '4.0.0') + # Add a commit on master after the tag + self._add_notes_file('slug5') + # Move back to the tag and create the branch + self.repo.git('checkout', '4.0.0') + self.repo.git('checkout', '-b', 'stable/4') + # Create a commit on the branch + f41 = self._add_notes_file('slug41') + log_text = self.repo.git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self.repo.git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0': [f4], + '4.0.0-1': [f41], + }, + results, + ) + + def test_branch_tip_of_master(self): + # We have branched from master, but not added any commits to + # master. + f4 = self._add_notes_file('slug4') + self.repo.git('tag', '-s', '-m', 'release', '4.0.0') + self.repo.git('checkout', '-b', 'stable/4') + # Create a commit on the branch + f41 = self._add_notes_file('slug41') + f42 = self._add_notes_file('slug42') + log_text = self.repo.git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self.repo.git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0': [f4], + '4.0.0-2': [f41, f42], + }, + results, + ) + + def test_branch_no_more_commits(self): + # We have branched from master, but not added any commits to + # our branch or to master. + f4 = self._add_notes_file('slug4') + self.repo.git('tag', '-s', '-m', 'release', '4.0.0') + self.repo.git('checkout', '-b', 'stable/4') + # Create a commit on the branch + log_text = self.repo.git( + 'log', '--pretty=%x00%H %d', '--name-only', '--graph', + '--all', '--decorate', + ) + self.addDetail('git log', text_content(log_text)) + rev_list = self.repo.git('rev-list', '--first-parent', + '^stable/4', 'master') + self.addDetail('rev-list', text_content(rev_list)) + self.c.override( + branch='stable/4', + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '4.0.0': [f4], + }, + results, + ) + + def test_remote_branches(self): + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'stable/2') + self.repo.git('checkout', 'master') + scanner1 = scanner.Scanner(self.c) + head1 = scanner1._get_ref('stable/2') + self.assertIsNotNone(head1) + print('head1', head1) + # Create a second repository by cloning the first. + print(utils.check_output( + ['git', 'clone', self.reporoot, 'reporoot2'], + cwd=self.temp_dir, + )) + reporoot2 = os.path.join(self.temp_dir, 'reporoot2') + print(utils.check_output( + ['git', 'remote', 'update'], + cwd=reporoot2, + )) + print(utils.check_output( + ['git', 'remote', '-v'], + cwd=reporoot2, + )) + print(utils.check_output( + ['find', '.git/refs'], + cwd=reporoot2, + )) + print(utils.check_output( + ['git', 'branch', '-a'], + cwd=reporoot2, + )) + c2 = config.Config(reporoot2) + scanner2 = scanner.Scanner(c2) + head2 = scanner2._get_ref('origin/stable/2') + self.assertIsNotNone(head2) + self.assertEqual(head1, head2) + + def test_remote_branch_without_prefix(self): + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'stable/2') + self.repo.git('checkout', 'master') + scanner1 = scanner.Scanner(self.c) + head1 = scanner1._get_ref('stable/2') + self.assertIsNotNone(head1) + print('head1', head1) + # Create a second repository by cloning the first. + print(utils.check_output( + ['git', 'clone', self.reporoot, 'reporoot2'], + cwd=self.temp_dir, + )) + reporoot2 = os.path.join(self.temp_dir, 'reporoot2') + print(utils.check_output( + ['git', 'remote', 'update'], + cwd=reporoot2, + )) + print(utils.check_output( + ['git', 'remote', '-v'], + cwd=reporoot2, + )) + print(utils.check_output( + ['find', '.git/refs'], + cwd=reporoot2, + )) + print(utils.check_output( + ['git', 'branch', '-a'], + cwd=reporoot2, + )) + c2 = config.Config(reporoot2) + scanner2 = scanner.Scanner(c2) + head2 = scanner2._get_ref('stable/2') + self.assertIsNotNone(head2) + self.assertEqual(head1, head2) + + def test_modify_old_branch_note_on_master(self): + # Modify a note from a stable branch on master and ensure that + # the note does not appear in the scanner output from master. + # This should replicate the problem described in bug #1682796 + self.repo.git('checkout', '2.0.0') + self.repo.git('branch', 'stable/2') + self.repo.git('checkout', 'master') + with open(os.path.join(self.reporoot, self.f1), 'w') as f: + f.write('new file contents') + self.repo.commit('update %s' % self.f1) + self.c.override( + earliest_version=None, + ) + self.scanner = scanner.Scanner(self.c) + raw_results = self.scanner.get_notes_by_version() + results = { + k: [f for (f, n) in v] + for (k, v) in raw_results.items() + } + self.assertEqual( + { + '2.0.0': [self.f2], + '3.0.0': [self.f3], + }, + results, + ) + + +class ScanStopPointPrereleaseVersionsTest(Base): + + def setUp(self): + super(ScanStopPointPrereleaseVersionsTest, self).setUp() + self.scanner = scanner.Scanner(self.c) + self._make_python_package() + self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first series', '1.0.0.0rc1') + self.repo.git('checkout', '-b', 'stable/a') + self._add_notes_file('slug2') + self._add_notes_file('slug3') + self.repo.git('tag', '-s', '-m', 'second tag', '1.0.0') + self.repo.git('checkout', 'master') + self._add_notes_file('slug4') + self._add_notes_file('slug5') + self.repo.git('tag', '-s', '-m', 'second series', '2.0.0.0b3') + self._add_notes_file('slug6') + self._add_notes_file('slug7') + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.0.0rc1') + self.repo.git('checkout', '-b', 'stable/b') + self._add_notes_file('slug8') + self._add_notes_file('slug9') + self.repo.git('tag', '-s', '-m', 'third tag', '2.0.0') + self.repo.git('checkout', 'master') + + def test_beta_collapse(self): + self.assertEqual( + '1.0.0.0rc1', + self.scanner._find_scan_stop_point( + '2.0.0.0b3', ['2.0.0.0b3', '1.0.0.0rc1'], + True, 'master'), + ) + + def test_rc_collapse_master(self): + self.assertEqual( + '1.0.0.0rc1', + self.scanner._find_scan_stop_point( + '2.0.0.0rc1', ['2.0.0.0rc1', '2.0.0.0b3', '1.0.0.0rc1'], + True, 'master'), + ) + + def test_rc_collapse_branch(self): + self.assertEqual( + '1.0.0.0rc1', + self.scanner._find_scan_stop_point( + '2.0.0.0rc1', ['2.0.0.0rc1', '2.0.0.0b3', '1.0.0.0rc1'], + True, 'stable/b'), + ) + + def test_rc_no_collapse(self): + self.assertEqual( + '2.0.0.0b3', + self.scanner._find_scan_stop_point( + '2.0.0.0rc1', ['2.0.0.0rc1', '2.0.0.0b3', '1.0.0.0rc1'], + False, 'master'), + ) + + def test_stable_branch_with_collapse(self): + self.assertEqual( + '1.0.0.0rc1', + self.scanner._find_scan_stop_point( + '2.0.0', ['2.0.0', '2.0.0.0rc1', '2.0.0.0b3', '1.0.0.0rc1'], + True, 'stable/b'), + ) + + # def test_nova_newton(self): + # self.assertEqual( + # '13.0.0.0rc3', + # self.scanner._find_scan_stop_point( + # '14.0.0', + # [u'14.0.3', u'14.0.2', u'14.0.1', u'14.0.0.0rc2', + # u'14.0.0', u'14.0.0.0rc1', u'14.0.0.0b3', u'14.0.0.0b2', + # u'14.0.0.0b1', u'13.0.0.0rc3', u'13.0.0', u'13.0.0.0rc2', + # u'13.0.0.0rc1', u'13.0.0.0b3', u'13.0.0.0b2', u'13.0.0.0b1', + # u'12.0.0.0rc3', u'12.0.0', u'12.0.0.0rc2', u'12.0.0.0rc1', + # u'12.0.0.0b3', u'12.0.0.0b2', u'12.0.0.0b1', u'12.0.0a0', + # u'2015.1.0rc3', u'2015.1.0', u'2015.1.0rc2', u'2015.1.0rc1', + # u'2015.1.0b3', u'2015.1.0b2', u'2015.1.0b1', u'2014.2.rc2', + # u'2014.2', u'2014.2.rc1', u'2014.2.b3', u'2014.2.b2', + # u'2014.2.b1', u'2014.1.rc1', u'2014.1.b3', u'2014.1.b2', + # u'2014.1.b1', u'2013.2.rc1', u'2013.2.b3', u'2013.1.rc1', + # u'folsom-2', u'folsom-1', u'essex-1', u'diablo-2', + # u'diablo-1', u'2011.2', u'2011.2rc1', u'2011.2gamma1', + # u'2011.1rc1', u'0.9.0'], + # True), + # ) + + +class ScanStopPointRegularVersionsTest(Base): + + def setUp(self): + super(ScanStopPointRegularVersionsTest, self).setUp() + self.scanner = scanner.Scanner(self.c) + self._make_python_package() + self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first series', '1.0.0') + self.repo.git('checkout', '-b', 'stable/a') + self._add_notes_file('slug2') + self._add_notes_file('slug3') + self.repo.git('tag', '-s', '-m', 'second tag', '1.0.1') + self.repo.git('checkout', 'master') + self._add_notes_file('slug4') + self._add_notes_file('slug5') + self.repo.git('tag', '-s', '-m', 'second series', '2.0.0') + self._add_notes_file('slug6') + self._add_notes_file('slug7') + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.1') + self.repo.git('checkout', '-b', 'stable/b') + self._add_notes_file('slug8') + self._add_notes_file('slug9') + self.repo.git('tag', '-s', '-m', 'third tag', '2.0.2') + self.repo.git('checkout', 'master') + + def test_invalid_earliest_version(self): + self.assertIsNone( + self.scanner._find_scan_stop_point( + 'not.a.numeric.version', [], True, 'stable/b'), + ) + + def test_none(self): + self.assertIsNone( + self.scanner._find_scan_stop_point( + None, [], True, 'stable/b'), + ) + + def test_unknown_version(self): + self.assertIsNone( + self.scanner._find_scan_stop_point( + '2.0.2', [], True, 'stable/b'), + ) + + def test_only_version(self): + self.assertIsNone( + self.scanner._find_scan_stop_point( + '2.0.2', ['1.0.0'], True, 'stable/b'), + ) + + def test_find_prior_branch(self): + self.assertEqual( + '1.0.0', + self.scanner._find_scan_stop_point( + '2.0.2', ['2.0.2', '2.0.1', '2.0.0', '1.0.0'], + True, 'stable/b'), + ) + + +class GetRefTest(Base): + + def setUp(self): + super(GetRefTest, self).setUp() + self._make_python_package() + self.f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.repo.git('branch', 'stable/foo') + self.repo.git('tag', 'bar-eol') + + def test_signed_tag(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('1.0.0') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_unsigned_tag(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('bar-eol') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_eol_tag_from_branch(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('stable/bar') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_head(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref(None) + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_stable_branch(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('stable/foo') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_stable_branch_with_origin_prefix(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('origin/stable/foo') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_no_such_value(self): + self.scanner = scanner.Scanner(self.c) + self.assertRaises( + ValueError, + self.scanner._get_ref, + 'missing/remote', + ) + + +class TagsTest(Base): + + def setUp(self): + super(TagsTest, self).setUp() + self._make_python_package() + self.f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.f2 = self._add_notes_file('slug2') + self.repo.git('tag', '-s', '-m', 'first tag', '2.0.0') + self._add_notes_file('slug3') + self.repo.git('tag', '-s', '-m', 'first tag', '3.0.0') + + def test_master(self): + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch(None) + self.assertEqual( + ['3.0.0', '2.0.0', '1.0.0'], + results, + ) + + def test_get_ref(self): + self.scanner = scanner.Scanner(self.c) + ref = self.scanner._get_ref('3.0.0') + expected = self.scanner._repo.head() + self.assertEqual(expected, ref) + + def test_not_master(self): + self.repo.git('checkout', '2.0.0') + self.repo.git('checkout', '-b', 'not-master') + self._add_notes_file('slug4') + self.repo.git('tag', '-s', '-m', 'not on master', '2.0.1') + self.repo.git('checkout', 'master') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch('not-master') + self.assertEqual( + ['2.0.1', '2.0.0', '1.0.0'], + results, + ) + + def test_unsigned(self): + self._add_notes_file('slug4') + self.repo.git('tag', '-m', 'first tag', '4.0.0') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch(None) + self.assertEqual( + ['4.0.0', '3.0.0', '2.0.0', '1.0.0'], + results, + ) + + def test_tagged_tag_annotated(self): + time.sleep(1) + self.repo.git('tag', '-s', '-m', 'fourth tag', '4.0.0', '3.0.0') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch(None) + self.assertEqual( + ['3.0.0', '4.0.0', '2.0.0', '1.0.0'], + results, + ) + + def test_tagged_tag_lightweight(self): + time.sleep(1) + self.repo.git('tag', '-m', 'fourth tag', '4.0.0', '3.0.0') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch(None) + self.assertEqual( + ['3.0.0', '4.0.0', '2.0.0', '1.0.0'], + results, + ) + + +class VersionTest(Base): + + def setUp(self): + super(VersionTest, self).setUp() + self._make_python_package() + self.f1 = self._add_notes_file('slug1') + self.repo.git('tag', '-s', '-m', 'first tag', '1.0.0') + self.f2 = self._add_notes_file('slug2') + self.repo.git('tag', '-s', '-m', 'second tag', '2.0.0') + self._add_notes_file('slug3') + self.repo.git('tag', '-s', '-m', 'third tag', '3.0.0') + + def test_tagged_head(self): + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_current_version(None) + self.assertEqual( + '3.0.0', + results, + ) + + def test_head_after_tag(self): + self._add_notes_file('slug4') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_current_version(None) + self.assertEqual( + '3.0.0-1', + results, + ) + + def test_multiple_tags(self): + # The timestamp resolution appears to be 1 second, so sleep to + # ensure distinct timestamps for the 2 tags. In practice it is + # unlikely that anything could apply 2 signed tags within a + # single second (certainly not a person). + time.sleep(1) + self.repo.git('tag', '-s', '-m', 'fourth tag', '4.0.0') + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_current_version(None) + self.assertEqual( + '4.0.0', + results, + ) + + +class AggregateChangesTest(Base): + + def setUp(self): + super(AggregateChangesTest, self).setUp() + self.aggregator = scanner._ChangeAggregator() + + def test_ignore(self): + entry = mock.Mock() + n = self.get_note_num() + name = 'prefix/add-%016x' % n # no .yaml extension + entry.commit.id = 'commit-id' + changes = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ) + ) + ] + results = self.aggregator.aggregate_changes(entry, changes) + self.assertEqual( + [], + results, + ) + + def test_add(self): + entry = mock.Mock() + n = self.get_note_num() + name = 'prefix/add-%016x.yaml' % n + entry.commit.id = 'commit-id' + changes = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ) + ) + ] + results = list(self.aggregator.aggregate_changes(entry, changes)) + self.assertEqual( + [('%016x' % n, 'add', name, 'commit-id')], + results, + ) + + def test_add_multiple_after_delete(self): + # Adding multiple files in one commit using the same UID but + # different slug after we have seen a delete for the same UID + # causes the files to be ignored. + entry = mock.Mock() + n = self.get_note_num() + uid = '%016x' % n + changes = [] + for i in range(2): + name = 'prefix/add%d-%s.yaml' % (i, uid) + entry.commit.id = 'commit-id' + changes.append( + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ) + ) + ) + # Set up the aggregator as though it had already seen a delete + # operation. Since the scan happens in reverse chronological + # order, the delete would have happened after the add, and we + # can ignore the files because the error has been corrected in + # a later patch. + self.aggregator._deleted_bad_uids.add(uid) + results = list(self.aggregator.aggregate_changes(entry, changes)) + self.assertEqual([], results) + + def test_add_multiple_without_delete(self): + # Adding multiple files in one commit using the same UID but + # different slug without a delete operation causes an + # exception. + entry = mock.Mock() + n = self.get_note_num() + uid = '%016x' % n + changes = [] + for i in range(2): + name = 'prefix/add%d-%s.yaml' % (i, uid) + entry.commit.id = 'commit-id' + changes.append( + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ) + ) + ) + + # aggregate_changes() is a generator, so we have to wrap it in + # list() to process the data, so we need a little temporary + # function to do that and pass to assertRaises(). + def get_results(): + return list(self.aggregator.aggregate_changes(entry, changes)) + + self.assertRaises( + ValueError, + get_results, + ) + + def test_delete(self): + entry = mock.Mock() + n = self.get_note_num() + name = 'prefix/delete-%016x.yaml' % n + entry.commit.id = 'commit-id' + changes = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_DELETE, + old=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ), + new=objects.TreeEntry(path=None, mode=None, sha=None) + ) + ] + results = list(self.aggregator.aggregate_changes(entry, changes)) + self.assertEqual( + [('%016x' % n, 'delete', name, entry.commit.id)], + results, + ) + + def test_delete_multiple(self): + # Delete multiple files in one commit using the same UID but + # different slug. + entry = mock.Mock() + n = self.get_note_num() + changes = [] + expected = [] + for i in range(2): + name = 'prefix/delete%d-%016x.yaml' % (i, n) + entry.commit.id = 'commit-id' + changes.append( + diff_tree.TreeChange( + type=diff_tree.CHANGE_DELETE, + old=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='not-a-hash', + ), + new=objects.TreeEntry(path=None, mode=None, sha=None), + ) + ) + expected.append(('%016x' % n, 'delete', name, 'commit-id')) + results = list(self.aggregator.aggregate_changes(entry, changes)) + self.assertEqual(expected, results) + + def test_change(self): + entry = mock.Mock() + n = self.get_note_num() + name = 'prefix/change-%016x.yaml' % n + entry.commit.id = 'commit-id' + changes = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_MODIFY, + old=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='old-sha', + ), + new=objects.TreeEntry( + path=name.encode('utf-8'), + mode='0222', + sha='new-sha', + ), + ) + ] + results = list(self.aggregator.aggregate_changes(entry, changes)) + self.assertEqual( + [('%016x' % n, 'modify', name, 'commit-id')], + results, + ) + + def test_add_then_delete(self): + entry = mock.Mock() + n = self.get_note_num() + new_name = 'prefix/new-%016x.yaml' % n + old_name = 'prefix/old-%016x.yaml' % n + entry.commit.id = 'commit-id' + changes = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=new_name.encode('utf-8'), + mode='0222', + sha='new-hash', + ) + ), + diff_tree.TreeChange( + type=diff_tree.CHANGE_DELETE, + old=objects.TreeEntry( + path=old_name.encode('utf-8'), + mode='0222', + sha='old-hash', + ), + new=objects.TreeEntry(path=None, mode=None, sha=None) + ) + ] + results = list(self.aggregator.aggregate_changes(entry, changes)) + self.assertEqual( + [('%016x' % n, 'rename', old_name, new_name, 'commit-id')], + results, + ) + + def test_delete_then_add(self): + entry = mock.Mock() + n = self.get_note_num() + new_name = 'prefix/new-%016x.yaml' % n + old_name = 'prefix/old-%016x.yaml' % n + entry.commit.id = 'commit-id' + changes = [ + diff_tree.TreeChange( + type=diff_tree.CHANGE_DELETE, + old=objects.TreeEntry( + path=old_name.encode('utf-8'), + mode='0222', + sha='old-hash', + ), + new=objects.TreeEntry(path=None, mode=None, sha=None) + ), + diff_tree.TreeChange( + type=diff_tree.CHANGE_ADD, + old=objects.TreeEntry(path=None, mode=None, sha=None), + new=objects.TreeEntry( + path=new_name.encode('utf-8'), + mode='0222', + sha='new-hash', + ) + ), + ] + results = list(self.aggregator.aggregate_changes(entry, changes)) + self.assertEqual( + [('%016x' % n, 'rename', old_name, new_name, 'commit-id')], + results, + ) + + def test_tree_changes(self): + # Under some conditions when dulwich sees merge commits, + # changes() returns a list with nested lists. See commit + # cc11da6dcfb1dbaa015e9804b6a23f7872380c1b in this repo for an + # example. + entry = mock.Mock() + n = self.get_note_num() + # The files modified by the commit are actually + # reno/scanner.py, but the fake names are used in this test to + # comply with the rest of the configuration for the scanner. + old_name = 'prefix/old-%016x.yaml' % n + entry.commit.id = 'commit-id' + changes = [[ + diff_tree.TreeChange( + type='modify', + old=diff_tree.TreeEntry( + path=old_name.encode('utf-8'), + mode=33188, + sha=b'8247dfdd116fd0e3cc4ba32328e4a3eafd227de6', + ), + new=diff_tree.TreeEntry( + path=old_name.encode('utf-8'), + mode=33188, + sha=b'611f3663f54afb1f018a6a8680b6488da50ac340', + ), + ), + diff_tree.TreeChange( + type='modify', + old=diff_tree.TreeEntry( + path=old_name.encode('utf-8'), + mode=33188, + sha=b'ecb7788066eefa9dc8f110b56360efe7b1140b84', + ), + new=diff_tree.TreeEntry( + path=old_name.encode('utf-8'), + mode=33188, + sha=b'611f3663f54afb1f018a6a8680b6488da50ac340', + ), + ), + ]] + results = list(self.aggregator.aggregate_changes(entry, changes)) + self.assertEqual( + [('%016x' % n, 'modify', old_name, 'commit-id'), + ('%016x' % n, 'modify', old_name, 'commit-id')], + results, + ) + + +class ChangeTrackerTest(base.TestCase): + + def setUp(self): + super(ChangeTrackerTest, self).setUp() + self.changes = scanner._ChangeTracker() + basename = '%s-%016x.yaml' % ('slug', 1) + self.filename = os.path.join('releasenotes', 'notes', basename) + self.filename2 = self.filename.replace('slug', 'guls') + self.uniqueid = scanner._get_unique_id(self.filename) + self.fake_logger = self.useFixture( + fixtures.FakeLogger( + format='%(levelname)8s %(name)s %(message)s', + level=logging.DEBUG, + nuke_handlers=True, + ) + ) + + def test_add(self): + self.changes.add(self.filename, 'sha1', 'version') + self.assertEqual( + {}, + self.changes.seen_but_not_added, + ) + self.assertEqual( + ['version'], + self.changes.versions, + ) + self.assertEqual( + 'version', + self.changes.earliest_seen[self.uniqueid], + ) + self.assertEqual( + {self.uniqueid: (self.filename, 'sha1')}, + self.changes.last_name_by_id, + ) + self.assertEqual( + set(), + self.changes.uniqueids_deleted, + ) + + def test_modify_with_add(self): + self.changes.modify(self.filename, 'sha2', 'version2') + self.changes.add(self.filename, 'sha1', 'version1') + self.assertEqual( + {}, + self.changes.seen_but_not_added, + ) + self.assertEqual( + ['version2', 'version1'], + self.changes.versions, + ) + self.assertEqual( + 'version1', + self.changes.earliest_seen[self.uniqueid], + ) + self.assertEqual( + {self.uniqueid: (self.filename, 'sha2')}, + self.changes.last_name_by_id, + ) + self.assertEqual( + set(), + self.changes.uniqueids_deleted, + ) + + def test_modify_without_add(self): + self.changes.modify(self.filename, 'sha2', 'version2') + self.assertEqual( + {self.uniqueid: (self.filename, 'sha2')}, + self.changes.seen_but_not_added, + ) + self.assertEqual( + ['version2'], + self.changes.versions, + ) + self.assertEqual( + 'version2', + self.changes.earliest_seen[self.uniqueid], + ) + self.assertEqual( + {}, + self.changes.last_name_by_id, + ) + self.assertEqual( + set(), + self.changes.uniqueids_deleted, + ) + + def test_rename_with_add(self): + self.changes.rename(self.filename2, 'sha2', 'version2') + self.changes.add(self.filename, 'sha1', 'version1') + self.assertEqual( + {}, + self.changes.seen_but_not_added, + ) + self.assertEqual( + ['version2', 'version1'], + self.changes.versions, + ) + self.assertEqual( + 'version1', + self.changes.earliest_seen[self.uniqueid], + ) + self.assertEqual( + {self.uniqueid: (self.filename2, 'sha2')}, + self.changes.last_name_by_id, + ) + self.assertEqual( + set(), + self.changes.uniqueids_deleted, + ) + + def test_rename_without_add(self): + self.changes.rename(self.filename2, 'sha2', 'version2') + self.assertEqual( + {self.uniqueid: (self.filename2, 'sha2')}, + self.changes.seen_but_not_added, + ) + self.assertEqual( + ['version2'], + self.changes.versions, + ) + self.assertEqual( + 'version2', + self.changes.earliest_seen[self.uniqueid], + ) + self.assertEqual( + {}, + self.changes.last_name_by_id, + ) + self.assertEqual( + set(), + self.changes.uniqueids_deleted, + ) + + +class GetSeriesBranchesTest(Base): + + def setUp(self): + super(GetSeriesBranchesTest, self).setUp() + self.repo.add_file('test.txt') + + def test_none(self): + self.scanner = scanner.Scanner(self.c) + self.assertEqual( + [], + self.scanner._get_series_branches(), + ) + + def test_real_branches_sorted_names(self): + self.repo.git( + 'checkout', '-b', 'stable/a', + ) + self.repo.git( + 'checkout', '-b', 'stable/b', + ) + self.scanner = scanner.Scanner(self.c) + self.assertEqual( + ['stable/a', 'stable/b'], + self.scanner._get_series_branches(), + ) + + def test_eol_tag(self): + self.repo.git( + 'tag', '-s', '-m', 'closed branch', 'a-eol', + ) + self.scanner = scanner.Scanner(self.c) + self.assertEqual( + ['stable/a'], + self.scanner._get_series_branches(), + ) + + def test_mix_tag_and_branch(self): + self.repo.git( + 'tag', '-s', '-m', 'closed branch', 'a-eol', + ) + self.repo.git( + 'checkout', '-b', 'stable/b', + ) + self.scanner = scanner.Scanner(self.c) + self.assertEqual( + ['stable/a', 'stable/b'], + self.scanner._get_series_branches(), + ) diff --git a/reno/tests/test_utils.py b/reno/tests/test_utils.py index 8e31100ec8e592649fe3a2ce83463b20c25f957a..cb76cc76c61ec44ea1e73f14ebc351ee9e204966 100644 --- a/reno/tests/test_utils.py +++ b/reno/tests/test_utils.py @@ -12,12 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -from reno.tests import base -from reno import utils - import mock import six +from reno.tests import base +from reno import utils + class TestGetRandomString(base.TestCase): diff --git a/reno/utils.py b/reno/utils.py index fce17726f607f09911a51bcab2d21560217957e4..88cd098c8bbbade5896859aa0f9049d9d28f4149 100644 --- a/reno/utils.py +++ b/reno/utils.py @@ -11,16 +11,13 @@ # under the License. import binascii +import logging +import os import os.path import random import subprocess -from reno import defaults - - -def get_notes_dir(args): - """Return the path to the release notes directory.""" - return os.path.join(args.relnotesdir, defaults.NOTES_SUBDIR) +LOG = logging.getLogger(__name__) def get_random_string(nbytes=8): @@ -41,5 +38,17 @@ def get_random_string(nbytes=8): def check_output(*args, **kwds): """Unicode-aware wrapper for subprocess.check_output""" - raw = subprocess.check_output(*args, **kwds) - return raw.decode('utf-8') + process = subprocess.Popen(stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + *args, **kwds) + output, errors = process.communicate() + retcode = process.poll() + if errors: + LOG.debug('ran: %s', ' '.join(*args)) + LOG.debug('returned: %s', retcode) + LOG.debug('error output: %s', errors.rstrip()) + LOG.debug('regular output: %s', output.rstrip()) + if retcode: + LOG.debug('raising error') + raise subprocess.CalledProcessError(retcode, args, output=output) + return output.decode('utf-8') diff --git a/requirements.txt b/requirements.txt index 2dc6330ee72fce831383aa07e8b6a1f7e8095bed..079e796ee234079841df590c5eb7159f0eeaad2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -pbr<2.0,>=1.4 -Babel>=1.3 -PyYAML>=3.1.0 +pbr +PyYAML>=3.10 +six>=1.9.0 +dulwich>=0.15.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 1d8a008c8b6329a4ec03ec7c120d912d3dac9eee..a5bfc41e49c2dd0c9ed3c4a6407fee4e6be8982f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ description-file = README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org -home-page = http://www.openstack.org/ +home-page = https://docs.openstack.org/reno/latest/ classifier = Environment :: OpenStack Intended Audience :: Information Technology @@ -16,8 +16,7 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 - Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 [files] packages = @@ -26,18 +25,13 @@ packages = [entry_points] console_scripts = reno = reno.main:main +distutils.commands = + build_reno = reno.setup_command:BuildReno [extras] sphinx = - sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 - -[build_sphinx] -source-dir = doc/source -build-dir = doc/build -all_files = 1 - -[upload_sphinx] -upload-dir = doc/build/html + sphinx>=1.6.1 # BSD + docutils>=0.11 # OSI-Approved Open Source, Public Domain [compile_catalog] directory = reno/locale @@ -48,7 +42,5 @@ domain = reno output_dir = reno/locale input_file = reno/locale/reno.pot -[extract_messages] -keywords = _ gettext ngettext l_ lazy_gettext -mapping_file = babel.cfg -output_file = reno/locale/reno.pot +[wheel] +universal = 1 diff --git a/test-requirements.txt b/test-requirements.txt index c352436432599c5d181848c56582efc6bcaaa171..203542f8efd3a7c58172f5b925788cda14e915c8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,15 +2,13 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking<0.11,>=0.10.0 +hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0 mock>=1.2 coverage>=3.6 -discover python-subunit>=0.0.18 -oslosphinx>=2.5.0 # Apache-2.0 -oslotest>=1.10.0 # Apache-2.0 +openstackdocstheme>=1.11.0 # Apache-2.0 testrepository>=0.0.18 testscenarios>=0.4 testtools>=1.4.0 diff --git a/tox.ini b/tox.ini index 884c7729d667fbdc012660765c871191e8d644cd..d9d5714829f639f40764f2c43f33b5b004675a34 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,6 @@ [tox] minversion = 1.6 -envlist = py34,py27,pep8 -skipsdist = True +envlist = py35,py27,pep8 [testenv] usedevelop = True @@ -11,21 +10,42 @@ setenv = deps = -r{toxinidir}/test-requirements.txt .[sphinx] -commands = python setup.py test --slowest --testr-args='{posargs}' +commands = + python setup.py test --slowest --coverage --coverage-package-name=reno --testr-args='{posargs}' + coverage report --show-missing [testenv:pep8] -commands = flake8 +basepython = python3 +commands = + flake8 + reno -q lint + +[testenv:lower-constraints] +basepython = python3 +deps = + -c{toxinidir}/lower-constraints.txt + {[testenv]deps} [testenv:venv] +basepython = python3 commands = {posargs} [testenv:cover] +basepython = python3 commands = python setup.py test --coverage --testr-args='{posargs}' [testenv:docs] -commands = python setup.py build_sphinx +# NOTE(dhellmann): Build our own documentation using the +# lower-constraints list as a hacky way to test the sphinx extension +# module, since we don't have separate unit tests for it. +deps = + -c{toxinidir}/lower-constraints.txt + {[testenv]deps} +basepython = python3 +commands = sphinx-build -a -W -E -b html doc/source doc/build/html [testenv:debug] +basepython = python3 commands = oslo_debug_helper {posargs} [flake8] @@ -34,4 +54,4 @@ commands = oslo_debug_helper {posargs} show-source = True ignore = E123,E125 builtins = _ -exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build