From 85abe44233bee73148a895b21c4e965d310f3e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Tue, 21 Jul 2020 13:37:18 +0200 Subject: [PATCH 1/6] Adding pre-commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduced changes: - pre-commit config and rules - Add pre-commit to pep8 gate, Flake8 is covered in the pre-commit hooks. - Applying fixes for pre-commit compliance in all code. Also commit hash will be used instead of version tags in pre-commit to prevend arbitrary code from running in developer's machines. pre-commit will be used to: - trailing whitespace; - Replaces or checks mixed line ending (mixed-line-ending); - Forbid files which have a UTF-8 byte-order marker (check-byte-order-marker); - Checks that non-binary executables have a proper shebang (check-executables-have-shebangs); - Check for files that contain merge conflict strings (check-merge-conflict); - Check for debugger imports and py37+ breakpoint() calls in python source (debug-statements); - Attempts to load all yaml files to verify syntax (check-yaml); - Run flake8 checks (flake8) (local) For further details about tests please refer to: https://github.com/pre-commit/pre-commit-hooks Change-Id: I8d5290d62b12a7850fc932c3778cc37951dd67ed Co-authored-by: Moisés Guimarães de Medeiros --- .pre-commit-config.yaml | 35 +++++++++++++++++++++++++++++++++++ doc/source/conf.py | 13 +++++++++++++ releasenotes/source/conf.py | 13 +++++++++++++ test-requirements.txt | 1 + tox.ini | 2 +- 5 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 .pre-commit-config.yaml mode change 100755 => 100644 doc/source/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..9d94556 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +# We from the Oslo project decided to pin repos based on the +# commit hash instead of the version tag to prevend arbitrary +# code from running in developer's machines. To update to a +# newer version, run `pre-commit autoupdate` and then replace +# the newer versions with their commit hash. + +default_language_version: + python: python3 + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: ebc15addedad713c86ef18ae9632c88e187dd0af # v3.1.0 + hooks: + - id: trailing-whitespace + # Replaces or checks mixed line ending + - id: mixed-line-ending + args: ['--fix', 'lf'] + exclude: '.*\.(svg)$' + # Forbid files which have a UTF-8 byte-order marker + - id: check-byte-order-marker + # Checks that non-binary executables have a proper shebang + - id: check-executables-have-shebangs + # Check for files that contain merge conflict strings. + - id: check-merge-conflict + # Check for debugger imports and py37+ breakpoint() + # calls in python source + - id: debug-statements + - id: check-yaml + files: .*\.(yaml|yml)$ + - repo: https://gitlab.com/pycqa/flake8 + rev: 181bb46098dddf7e2d45319ea654b4b4d58c2840 # 3.8.3 + hooks: + - id: flake8 + additional_dependencies: + - hacking>=3.0.1,<3.1.0 diff --git a/doc/source/conf.py b/doc/source/conf.py old mode 100755 new mode 100644 index 4a50d47..51e290e --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,17 @@ # -*- coding: utf-8 -*- +# Copyright (C) 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # # Configuration file for the Sphinx documentation builder. # diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index c93fca6..3060903 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -1,4 +1,17 @@ # -*- coding: utf-8 -*- +# Copyright (C) 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # # Configuration file for the Sphinx documentation builder. # diff --git a/test-requirements.txt b/test-requirements.txt index 24bf608..fdc4461 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,3 +5,4 @@ hacking>=3.0,<3.1.0 # Apache-2.0 oslotest>=1.5.1 stestr>=2.0.0 +pre-commit>=2.6.0 # MIT diff --git a/tox.ini b/tox.ini index 285e284..98cf387 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ deps = commands = stestr run --slowest {posargs} [testenv:pep8] -commands = flake8 {posargs} +commands = pre-commit run -a [testenv:venv] commands = {posargs} -- GitLab From 0a09ac14fc5369b3abc330f31e1f51efb7545bbb Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 11 Sep 2020 21:01:29 +0000 Subject: [PATCH 2/6] Update master for stable/victoria Add file to the reno documentation build to show release notes for stable/victoria. Use pbr instruction to increment the minor version number automatically so that master versions are higher than the versions on stable/victoria. Change-Id: Icf5ce9d7fa96131cd2ae66f8a3b0c57af58eb453 Sem-Ver: feature --- releasenotes/source/index.rst | 1 + releasenotes/source/victoria.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/victoria.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 100c70b..e888e66 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -5,6 +5,7 @@ oslo.upgradecheck Release Notes :maxdepth: 1 unreleased + victoria ussuri train stein diff --git a/releasenotes/source/victoria.rst b/releasenotes/source/victoria.rst new file mode 100644 index 0000000..4efc7b6 --- /dev/null +++ b/releasenotes/source/victoria.rst @@ -0,0 +1,6 @@ +============================= +Victoria Series Release Notes +============================= + +.. release-notes:: + :branch: stable/victoria -- GitLab From 741df12ff499c6f5f38d68c67f8edf5257c13b3e Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 11 Sep 2020 21:01:31 +0000 Subject: [PATCH 3/6] Add Python3 wallaby unit tests This is an automatically generated patch to ensure unit testing is in place for all the of the tested runtimes for wallaby. See also the PTI in governance [1]. [1]: https://governance.openstack.org/tc/reference/project-testing-interface.html Change-Id: Ib5ee719e065a5be67a64982dafc7c4cb14ff8f3e --- .zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index 8e3db27..b45d2b6 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,7 +1,7 @@ - project: templates: - publish-openstack-docs-pti - - openstack-python3-victoria-jobs + - openstack-python3-wallaby-jobs - check-requirements - release-notes-jobs-python3 - periodic-stable-jobs -- GitLab From c70506727800d99fbbe8133f0bf0ccb1e70d690a Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Thu, 19 Nov 2020 18:27:32 -0600 Subject: [PATCH 4/6] Add common checks for json to yaml policy file migration As part of community goal[1] for migrating the policy file from JSON format to YAML we need to add upgrade check to detect any JSON formatted file. That check needs to be done in each service side which are deprecating the JSON format. So to avoid duplicate code, this commit add that check in oslo_upgradechecks and that can be enabled from service side during they migrate to YAML format. Service will enable it like below: _upgrade_checks = ( (_('Policy File JSON to YAML Migration'), (common_checks.check_policy_json, {'conf': CONF})), ) Also modify the check() method to accept the **kwargs for checks. [1] https://governance.openstack.org/tc/goals/selected/wallaby/migrate-policy-format-from-json-to-yaml.html Change-Id: Ifa71c20879c964f794a40b981744d95db4ab1602 --- lower-constraints.txt | 2 + oslo_upgradecheck/common_checks.py | 36 ++++++ oslo_upgradecheck/tests/test_common_checks.py | 103 ++++++++++++++++++ oslo_upgradecheck/upgradecheck.py | 6 +- requirements.txt | 1 + test-requirements.txt | 1 + 6 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 oslo_upgradecheck/common_checks.py create mode 100644 oslo_upgradecheck/tests/test_common_checks.py diff --git a/lower-constraints.txt b/lower-constraints.txt index 6224498..5ded84f 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -4,3 +4,5 @@ enum34==1.0.4 PrettyTable==0.7.1 oslotest==1.5.1 stestr==2.0.0 +oslo.serialization==2.21.1 +oslo.utils==4.5.0 diff --git a/oslo_upgradecheck/common_checks.py b/oslo_upgradecheck/common_checks.py new file mode 100644 index 0000000..1881980 --- /dev/null +++ b/oslo_upgradecheck/common_checks.py @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_utils import fileutils + +from oslo_upgradecheck import upgradecheck + +""" +Common checks which can be used by multiple services. +""" + + +def check_policy_json(self, conf): + "Checks to see if policy file is JSON-formatted policy file." + msg = ("Your policy file is JSON-formatted which is " + "deprecated. You need to switch to YAML-formatted file. " + "Use the ``oslopolicy-convert-json-to-yaml`` " + "tool to convert the existing JSON-formatted files to " + "YAML in a backwards-compatible manner: " + "https://docs.openstack.org/oslo.policy/" + "latest/cli/oslopolicy-convert-json-to-yaml.html.") + status = upgradecheck.Result(upgradecheck.Code.SUCCESS) + # Check if policy file exist and is JSON-formatted. + policy_path = conf.find_file(conf.oslo_policy.policy_file) + if policy_path and fileutils.is_json(policy_path): + status = upgradecheck.Result(upgradecheck.Code.FAILURE, msg) + return status diff --git a/oslo_upgradecheck/tests/test_common_checks.py b/oslo_upgradecheck/tests/test_common_checks.py new file mode 100644 index 0000000..c5c54a2 --- /dev/null +++ b/oslo_upgradecheck/tests/test_common_checks.py @@ -0,0 +1,103 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures +import os.path +import tempfile +import yaml + +from oslo_config import cfg +from oslo_config import fixture as config +from oslo_serialization import jsonutils +from oslotest import base + +from oslo_upgradecheck import common_checks +from oslo_upgradecheck import upgradecheck + + +class TestUpgradeCheckPolicyJSON(base.BaseTestCase): + + def setUp(self): + super(TestUpgradeCheckPolicyJSON, self).setUp() + conf_fixture = self.useFixture(config.Config()) + conf_fixture.load_raw_values() + self.conf = conf_fixture.conf + policy_file_opt = cfg.StrOpt('policy_file', + default='policy.json', + help='policy file') + self.conf.register_opt(policy_file_opt, group="oslo_policy") + + self.cmd = upgradecheck.UpgradeCommands() + self.cmd._upgrade_checks = (('Policy File JSON to YAML Migration', + (common_checks.check_policy_json, + {'conf': self.conf})),) + + self.data = { + 'rule_admin': 'True', + 'rule_admin2': 'is_admin:True' + } + self.temp_dir = self.useFixture(fixtures.TempDir()) + fd, self.json_file = tempfile.mkstemp(dir=self.temp_dir.path) + fd, self.yaml_file = tempfile.mkstemp(dir=self.temp_dir.path) + with open(self.json_file, 'w') as fh: + jsonutils.dump(self.data, fh) + with open(self.yaml_file, 'w') as fh: + yaml.dump(self.data, fh) + + original_search_dirs = cfg._search_dirs + + def fake_search_dirs(dirs, name): + dirs.append(self.temp_dir.path) + return original_search_dirs(dirs, name) + + mock_search_dir = self.useFixture( + fixtures.MockPatch('oslo_config.cfg._search_dirs')).mock + mock_search_dir.side_effect = fake_search_dirs + + def test_policy_json_file_fail_upgrade(self): + # Test with policy json file full path set in config. + self.conf.set_override('policy_file', self.json_file, + group="oslo_policy") + self.assertEqual(upgradecheck.Code.FAILURE, + self.cmd.check()) + + def test_policy_yaml_file_pass_upgrade(self): + # Test with full policy yaml file path set in config. + self.conf.set_override('policy_file', self.yaml_file, + group="oslo_policy") + self.assertEqual(upgradecheck.Code.SUCCESS, + self.cmd.check()) + + def test_no_policy_file_pass_upgrade(self): + # Test with no policy file exist, means use policy from code. + self.conf.set_override('policy_file', 'non_exist_file', + group="oslo_policy") + self.assertEqual(upgradecheck.Code.SUCCESS, + self.cmd.check()) + + def test_default_policy_yaml_file_pass_upgrade(self): + self.conf.set_override('policy_file', 'policy.yaml', + group="oslo_policy") + tmpfilename = os.path.join(self.temp_dir.path, 'policy.yaml') + with open(tmpfilename, 'w') as fh: + yaml.dump(self.data, fh) + self.assertEqual(upgradecheck.Code.SUCCESS, + self.cmd.check()) + + def test_old_default_policy_json_file_fail_upgrade(self): + self.conf.set_override('policy_file', 'policy.json', + group="oslo_policy") + tmpfilename = os.path.join(self.temp_dir.path, 'policy.json') + with open(tmpfilename, 'w') as fh: + jsonutils.dump(self.data, fh) + self.assertEqual(upgradecheck.Code.FAILURE, + self.cmd.check()) diff --git a/oslo_upgradecheck/upgradecheck.py b/oslo_upgradecheck/upgradecheck.py index 744413d..62c9a68 100644 --- a/oslo_upgradecheck/upgradecheck.py +++ b/oslo_upgradecheck/upgradecheck.py @@ -99,7 +99,11 @@ class UpgradeCommands(object): # This is a list if 2-item tuples for the check name and it's results. check_results = [] for name, func in self._upgrade_checks: - result = func(self) + if isinstance(func, tuple): + func_name, kwargs = func + result = func_name(self, **kwargs) + else: + result = func(self) # store the result of the check for the summary table check_results.append((name, result)) # we want to end up with the highest level code of all checks diff --git a/requirements.txt b/requirements.txt index a843587..4b13697 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ oslo.config>=5.2.0 # Apache-2.0 oslo.i18n>=3.15.3 # Apache-2.0 PrettyTable<0.8,>=0.7.1 # BSD +oslo.utils>=4.5.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index fdc4461..ccb9645 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,3 +6,4 @@ hacking>=3.0,<3.1.0 # Apache-2.0 oslotest>=1.5.1 stestr>=2.0.0 pre-commit>=2.6.0 # MIT +oslo.serialization>=2.21.1 # Apache-2.0 -- GitLab From 9d3bb03fc844027ac180b796b5df4476811fe275 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Mon, 7 Dec 2020 14:24:37 -0600 Subject: [PATCH 5/6] Fix lower-constraints job oslotest 1.5.1 is restricted to pbr <1.0 which conflict with other deps lower constarint for pbr. Error: The conflict is caused by: oslo-i18n 3.15.3 depends on pbr!=2.1.0 and >=2.0.0 oslo-serialization 2.21.1 depends on pbr!=2.1.0 and >=2.0.0 oslo-utils 4.5.0 depends on pbr!=2.1.0 and >=2.0.0 oslotest 1.5.1 depends on pbr!=0.7, <1.0 and >=0.6 https: //zuul.opendev.org/t/openstack/build/cb2bce3b4e004601958b764dfd0737b8 Change-Id: I513262b85a57ce9f81a63bd778fdd7f7be27e485 --- lower-constraints.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index 5ded84f..071c7ea 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -2,7 +2,7 @@ oslo.config==5.2.0 oslo.i18n==3.15.3 enum34==1.0.4 PrettyTable==0.7.1 -oslotest==1.5.1 +oslotest==3.5.0 stestr==2.0.0 oslo.serialization==2.21.1 oslo.utils==4.5.0 diff --git a/test-requirements.txt b/test-requirements.txt index ccb9645..ade8eef 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,7 +3,7 @@ # process, which may cause wedges in the gate later. hacking>=3.0,<3.1.0 # Apache-2.0 -oslotest>=1.5.1 +oslotest>=3.5.0 stestr>=2.0.0 pre-commit>=2.6.0 # MIT oslo.serialization>=2.21.1 # Apache-2.0 -- GitLab From 9f95a6e19bbf17369cf9917480a9fe8b07f245ec Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Fri, 4 Dec 2020 17:00:29 -0600 Subject: [PATCH 6/6] Register oslo policy opts explicitly for check_policy_json check_policy_json() common check need [oslo_policy].policy_file config value and it is better to register those options explicitly in check_policy_json() in case they are not register by services. Currently it work as oslo.policy opts are registered via some import in service side code path for upgradechecks which are not so safe when these code import path change. Change-Id: I16fcce119698c7375e4b58558fdce89e78a79a9b --- lower-constraints.txt | 1 + oslo_upgradecheck/common_checks.py | 8 ++++++++ oslo_upgradecheck/tests/test_common_checks.py | 7 +++---- requirements.txt | 1 + 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index 071c7ea..c30a967 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -6,3 +6,4 @@ oslotest==3.5.0 stestr==2.0.0 oslo.serialization==2.21.1 oslo.utils==4.5.0 +oslo.policy==2.0.0 diff --git a/oslo_upgradecheck/common_checks.py b/oslo_upgradecheck/common_checks.py index 1881980..6085070 100644 --- a/oslo_upgradecheck/common_checks.py +++ b/oslo_upgradecheck/common_checks.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_policy import opts as policy_opts from oslo_utils import fileutils from oslo_upgradecheck import upgradecheck @@ -21,6 +22,13 @@ Common checks which can be used by multiple services. def check_policy_json(self, conf): "Checks to see if policy file is JSON-formatted policy file." + + # NOTE(gmann): This method need [oslo_policy].policy_file + # config value so register those options in case they + # are not register by services. + conf.register_opts(policy_opts._options, + group=policy_opts._option_group) + msg = ("Your policy file is JSON-formatted which is " "deprecated. You need to switch to YAML-formatted file. " "Use the ``oslopolicy-convert-json-to-yaml`` " diff --git a/oslo_upgradecheck/tests/test_common_checks.py b/oslo_upgradecheck/tests/test_common_checks.py index c5c54a2..da9ec24 100644 --- a/oslo_upgradecheck/tests/test_common_checks.py +++ b/oslo_upgradecheck/tests/test_common_checks.py @@ -17,6 +17,7 @@ import yaml from oslo_config import cfg from oslo_config import fixture as config +from oslo_policy import opts as policy_opts from oslo_serialization import jsonutils from oslotest import base @@ -31,10 +32,8 @@ class TestUpgradeCheckPolicyJSON(base.BaseTestCase): conf_fixture = self.useFixture(config.Config()) conf_fixture.load_raw_values() self.conf = conf_fixture.conf - policy_file_opt = cfg.StrOpt('policy_file', - default='policy.json', - help='policy file') - self.conf.register_opt(policy_file_opt, group="oslo_policy") + self.conf.register_opts(policy_opts._options, + group=policy_opts._option_group) self.cmd = upgradecheck.UpgradeCommands() self.cmd._upgrade_checks = (('Policy File JSON to YAML Migration', diff --git a/requirements.txt b/requirements.txt index 4b13697..2f5f024 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ oslo.config>=5.2.0 # Apache-2.0 oslo.i18n>=3.15.3 # Apache-2.0 PrettyTable<0.8,>=0.7.1 # BSD oslo.utils>=4.5.0 # Apache-2.0 +oslo.policy>=2.0.0 # Apache-2.0 -- GitLab