From b8289101887ca5864190c95bb4ff292763006288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Tue, 21 Jul 2020 13:36:16 +0200 Subject: [PATCH 1/8] Adding pre-commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduced changes: - pre-commit config and rules. - Add pre-commit to pep8 gate, Flake8 is covered in the pre-commit hooks. - Applying fixes for pre-commit compliance in all code. Also commit hash will be used instead of version tags in pre-commit to prevend arbitrary code from running in developer's machines. pre-commit will be used to: - trailing whitespace; - Replaces or checks mixed line ending (mixed-line-ending); - Forbid files which have a UTF-8 byte-order marker (check-byte-order-marker); - Checks that non-binary executables have a proper shebang (check-executables-have-shebangs); - Check for files that contain merge conflict strings (check-merge-conflict); - Check for debugger imports and py37+ breakpoint() calls in python source (debug-statements); - Attempts to load all yaml files to verify syntax (check-yaml); - Run flake8 checks (flake8) (local) For further details about tests please refer to: https://github.com/pre-commit/pre-commit-hooks Change-Id: I8c993d6ffaf869f67713bbbde8632a0143ed0c72 Signed-off-by: Moisés Guimarães de Medeiros --- .pre-commit-config.yaml | 35 +++++++++++++++++++++++++++++++++++ doc/source/conf.py | 6 ++---- releasenotes/source/conf.py | 2 ++ test-requirements.txt | 2 ++ tox.ini | 2 +- 5 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..9d94556 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +# We from the Oslo project decided to pin repos based on the +# commit hash instead of the version tag to prevend arbitrary +# code from running in developer's machines. To update to a +# newer version, run `pre-commit autoupdate` and then replace +# the newer versions with their commit hash. + +default_language_version: + python: python3 + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: ebc15addedad713c86ef18ae9632c88e187dd0af # v3.1.0 + hooks: + - id: trailing-whitespace + # Replaces or checks mixed line ending + - id: mixed-line-ending + args: ['--fix', 'lf'] + exclude: '.*\.(svg)$' + # Forbid files which have a UTF-8 byte-order marker + - id: check-byte-order-marker + # Checks that non-binary executables have a proper shebang + - id: check-executables-have-shebangs + # Check for files that contain merge conflict strings. + - id: check-merge-conflict + # Check for debugger imports and py37+ breakpoint() + # calls in python source + - id: debug-statements + - id: check-yaml + files: .*\.(yaml|yml)$ + - repo: https://gitlab.com/pycqa/flake8 + rev: 181bb46098dddf7e2d45319ea654b4b4d58c2840 # 3.8.3 + hooks: + - id: flake8 + additional_dependencies: + - hacking>=3.0.1,<3.1.0 diff --git a/doc/source/conf.py b/doc/source/conf.py index 6870813..5180160 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# Copyright (C) 2020 Red Hat, Inc. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -22,7 +24,6 @@ sys.path.insert(0, os.path.abspath('../..')) # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.autodoc', - #'sphinx.ext.intersphinx', 'openstackdocstheme' ] @@ -76,6 +77,3 @@ latex_documents = [ u'%s Documentation' % project, u'OpenStack Foundation', 'manual'), ] - -# Example configuration for intersphinx: refer to the Python standard library. -#intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index af9b55f..36186ab 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# Copyright (C) 2020 Red Hat, Inc. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/test-requirements.txt b/test-requirements.txt index 5dd6fe0..49c8816 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,3 +11,5 @@ coverage!=4.4,>=4.0 # Apache-2.0 # Bandit security code scanner bandit>=1.6.0,<1.7.0 # Apache-2.0 + +pre-commit>=2.6.0 # MIT diff --git a/tox.ini b/tox.ini index adbe63f..0973c33 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ commands = stestr run --slowest {posargs} [testenv:pep8] commands = - flake8 + pre-commit run -a # Run security linter bandit -r oslo_serialization tests -n5 -- GitLab From be517b4af66423d10243f9ee6a8a051ea9c3b055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Tue, 8 Sep 2020 12:34:48 +0200 Subject: [PATCH 2/8] ignore reno generated artifacts Change-Id: I81068c9d9c3aa542684c6f9f8bbb5113ddbfe05a --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index adf36be..402479b 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,5 @@ ChangeLog # reno build releasenotes/build +RELEASENOTES.rst +releasenotes/notes/reno.cache -- GitLab From 7c2c0716125498baa8f44386ac112ef94f3b8d8e Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 11 Sep 2020 21:00:50 +0000 Subject: [PATCH 3/8] Update master for stable/victoria Add file to the reno documentation build to show release notes for stable/victoria. Use pbr instruction to increment the minor version number automatically so that master versions are higher than the versions on stable/victoria. Change-Id: Ib07c45377a60fce27ccf91c503fbc9691d0022b8 Sem-Ver: feature --- releasenotes/source/index.rst | 1 + releasenotes/source/victoria.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/victoria.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 82a1326..8e1b9fe 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 unreleased + victoria ussuri train stein diff --git a/releasenotes/source/victoria.rst b/releasenotes/source/victoria.rst new file mode 100644 index 0000000..4efc7b6 --- /dev/null +++ b/releasenotes/source/victoria.rst @@ -0,0 +1,6 @@ +============================= +Victoria Series Release Notes +============================= + +.. release-notes:: + :branch: stable/victoria -- GitLab From 128a747332494e674d95235483976046fbd53a5c Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 11 Sep 2020 21:00:52 +0000 Subject: [PATCH 4/8] Add Python3 wallaby unit tests This is an automatically generated patch to ensure unit testing is in place for all the of the tested runtimes for wallaby. See also the PTI in governance [1]. [1]: https://governance.openstack.org/tc/reference/project-testing-interface.html Change-Id: Idc70da67ee74c4ac873dba25472f682f4311f38c --- .zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index 2c16442..1155b74 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -3,7 +3,7 @@ - check-requirements - lib-forward-testing-python3 - openstack-lower-constraints-jobs - - openstack-python3-victoria-jobs + - openstack-python3-wallaby-jobs - periodic-stable-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 -- GitLab From a2b52b1309026749940142c1b9a39ae23aeeff9e Mon Sep 17 00:00:00 2001 From: zhoulinhui Date: Fri, 18 Sep 2020 00:26:35 +0800 Subject: [PATCH 5/8] Fix hacking min version to 3.0.1 flake8 new release 3.8.0 added new checks and gate pep8 job start failing. hacking 3.0.1 fix the pinning of flake8 to avoid bringing in a new version with new checks. Though it is fixed in latest hacking but 2.0 and 3.0 has cap for flake8 as <4.0.0 which mean flake8 new version 3.9.0 can also break the pep8 job if new check are added. To avoid similar gate break in future, we need to bump the hacking min version. - http://lists.openstack.org/pipermail/openstack-discuss/2020-May/014828.html Change-Id: Idc5d70e753953aeb006caeb3ba8f78f5ceeafdf3 --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5dd6fe0..b7fbcb9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,7 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=3.0,<3.1.0 # Apache-2.0 +hacking>=3.0.1,<3.1.0 # Apache-2.0 netaddr>=0.7.18 # BSD stestr>=2.0.0 # Apache-2.0 -- GitLab From 523fa45f64dcfe960ab1751fe72b1bed360f6591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Wed, 4 Nov 2020 10:04:41 +0100 Subject: [PATCH 6/8] Use py3 as the default runtime for tox Moving on py3 as the default runtime for tox to avoid to update this at each new cycle. Wallaby support officially the following runtimes [1]: - Python 3.6 - Python 3.8 During Victoria Python 3.7 was used as the default runtime [2] however this version isn't longer officially supported. [1] https://governance.openstack.org/tc/reference/runtimes/wallaby.html#python-runtimes-for-wallaby [2] https://governance.openstack.org/tc/reference/runtimes/victoria.html#python-runtimes-for-victoria Change-Id: Iaa02d12b545597a3b064f320ee33253300a5de1b --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0973c33..929b8dc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 3.2.0 -envlist = py38,pep8 +envlist = py3,pep8 basepython = python3 ignore_basepython_conflict = true -- GitLab From 3799d180978c4354a18815f93ebdceb3104d8fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Fri, 18 Dec 2020 15:12:22 +0100 Subject: [PATCH 7/8] Dropping lower constraints testing We facing errors related to the new pip resolver, this topic was discussed on the ML and QA team proposed to to test lower-constraints [1]. I propose to drop this test because the complexity and recurring pain needed to maintain that now exceeds the benefits provided by this mechanismes. [1] http://lists.openstack.org/pipermail/openstack-discuss/2020-December/019390.html Change-Id: Ie53b809ab031525958b58730a12cb37d0bdf5c82 --- .zuul.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index 1155b74..8425030 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -2,7 +2,6 @@ templates: - check-requirements - lib-forward-testing-python3 - - openstack-lower-constraints-jobs - openstack-python3-wallaby-jobs - periodic-stable-jobs - publish-openstack-docs-pti -- GitLab From 02037330d863ecbe2471b63bff5461dee6a3c024 Mon Sep 17 00:00:00 2001 From: Sofia Enriquez Date: Thu, 7 Jan 2021 18:07:02 +0000 Subject: [PATCH 8/8] Fix json to_primitive when using IO OBjects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, using Cinder's backup service with RBD the backup-create operation gets stuck when logging ('use_json=True' must be set in the config file). The oslo.log JSONFormatter gets stuck when passing an RBDVolumeIOWrapper from os-brick. This happens via os-brick's utils.trace() method which passes a connector containing {'path': RBDVolumeIOWrapper}. The oslo.log JSONFormatter format() method calls oslo_serialization's jsonutils.to_primitive and passes in this RBDVolumeIOWrapper object.   Therefore the to_primitive method eventually calls RBDVolumeIOWrapper.read(). In order to fix this the current path avoids mapping io.IOBase objects and fallback the wrapper RBD volume object. Co-authored-by: Eric Harney Closes-Bug: #1908607 Change-Id: I3c416e855cb5f0dc32d14b2749ba92aba8964574 --- oslo_serialization/jsonutils.py | 3 ++- oslo_serialization/tests/test_jsonutils.py | 10 ++++++++++ ...-json-to_primitive-IO-OBjects-04faff4a1b5cf48f.yaml | 5 +++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/bug-1908607-fix-json-to_primitive-IO-OBjects-04faff4a1b5cf48f.yaml diff --git a/oslo_serialization/jsonutils.py b/oslo_serialization/jsonutils.py index 0e310b1..a2a700b 100644 --- a/oslo_serialization/jsonutils.py +++ b/oslo_serialization/jsonutils.py @@ -33,6 +33,7 @@ import codecs import datetime import functools import inspect +import io import itertools import json import uuid @@ -161,7 +162,7 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, # Python 3 does not have iteritems elif hasattr(value, 'items'): return recursive(dict(value.items()), level=level + 1) - elif hasattr(value, '__iter__'): + elif hasattr(value, '__iter__') and not isinstance(value, io.IOBase): return list(map(recursive, value)) elif convert_instances and hasattr(value, '__dict__'): # Likely an instance of something. Watch for cycles. diff --git a/oslo_serialization/tests/test_jsonutils.py b/oslo_serialization/tests/test_jsonutils.py index e04b9dc..f076822 100644 --- a/oslo_serialization/tests/test_jsonutils.py +++ b/oslo_serialization/tests/test_jsonutils.py @@ -401,6 +401,16 @@ class ToPrimitiveTestCase(test_base.BaseTestCase): ret = jsonutils.to_primitive(obj, fallback=lambda _: 'fallback') self.assertEqual('fallback', ret) + def test_fallback_typeerror_IO_object(self): + # IO Objects are not callable, cause a TypeError in to_primitive() + obj = io.IOBase + + ret = jsonutils.to_primitive(obj) + self.assertEqual(str(obj), ret) + + ret = jsonutils.to_primitive(obj, fallback=lambda _: 'fallback') + self.assertEqual('fallback', ret) + def test_exception(self): self.assertIn(jsonutils.to_primitive(ValueError("an exception")), ["ValueError('an exception',)", diff --git a/releasenotes/notes/bug-1908607-fix-json-to_primitive-IO-OBjects-04faff4a1b5cf48f.yaml b/releasenotes/notes/bug-1908607-fix-json-to_primitive-IO-OBjects-04faff4a1b5cf48f.yaml new file mode 100644 index 0000000..0246ef7 --- /dev/null +++ b/releasenotes/notes/bug-1908607-fix-json-to_primitive-IO-OBjects-04faff4a1b5cf48f.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + `Bug #1908607 `_: Fix + json to_primitive when using IO OBjects. -- GitLab