From fcb2642c552e5762ca642959c89bda0de6220abb Mon Sep 17 00:00:00 2001 From: Baptiste Beauplat Date: Fri, 15 Oct 2021 21:40:58 +0200 Subject: [PATCH 1/4] Add regression test for encoding issue when processing GPG keys with UTF-8 uid --- tests/functional/accounts/test_profile.py | 59 +++++++++++++++-------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/tests/functional/accounts/test_profile.py b/tests/functional/accounts/test_profile.py index 6e564167..b8b77080 100644 --- a/tests/functional/accounts/test_profile.py +++ b/tests/functional/accounts/test_profile.py @@ -27,6 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. from tests import TransactionTestController +from locale import getlocale, setlocale, LC_CTYPE from django.urls import reverse # from debexpo.lib import constants # from debexpo.model import meta @@ -153,25 +154,29 @@ xOwJ1heEnfmgPkuiz7jFCAo= _GPGKEY_2 = """-----BEGIN PGP PUBLIC KEY BLOCK----- mDMEW/GBqhYJKwYBBAHaRw8BAQdA+6hBA4PcdcPwgMsKGQXrqwbJemLBgS1PkKZg -RFlKdKi0IHByaW1hcnkgaWQgPHByaW1hcnlAZXhhbXBsZS5vcmc+iJAEExYIADgC -GwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQRVkwbu4cjBst0cc7HENHgc6HHz -3wUCXyxWUwAKCRDENHgc6HHz3zxDAQCB9zEqs0mWmriFqhXtRSwjhLhbprWxpAqk -WTat6AU6XgD+MDVYYgKEHeLuKqJ1MiR+x53f5ypxtA5eHJZdbs5OEA+0HVRlc3Qg -dXNlciA8ZW1haWxAZXhhbXBsZS5jb20+iJMEExYIADsCGwMFCwkIBwIGFQoJCAsC -BBYCAwECHgECF4AWIQRVkwbu4cjBst0cc7HENHgc6HHz3wUCXyxWUwIZAQAKCRDE -NHgc6HHz30lxAP9Zvb7ssZ0bg94u2y1G3zbh8+5svSmGp3HThxMooIHvcwEA8jB3 -s5fVTZBIXagHBxACGSG5EhxlA8KlmaOSDGvl9w+4OARb8YGqEgorBgEEAZdVAQUB -AQdANrk3qq/eP1TEWfFZqhR0vcz7YB9c5+OnvMV+xO4W3nQDAQgHiHgEGBYIACAW -IQRVkwbu4cjBst0cc7HENHgc6HHz3wUCW/GBqgIbDAAKCRDENHgc6HHz3/CHAP0c -hxes4Ebtg7N8B/BoMYwmUVvmMVmoV+ef/vqYvfm6sgEA6fKzMSXllw57UJ90Unyn -xOwJ1heEnfmgPkuiz7jFCAq4MwReCQ2QFgkrBgEEAdpHDwEBB0A+v2Y8n88j+WwI -Q3hChPR7xa49prRSyKRnGBD/AXhJfYjvBBgWCgAgFiEEVZMG7uHIwbLdHHOxxDR4 -HOhx898FAl4JDZACGwIAgQkQxDR4HOhx8992IAQZFgoAHRYhBLPPezP4B2M420+o -DoeRkoMRdTvXBQJeCQ2QAAoJEIeRkoMRdTvX0AcA/i8tjP8ihM2nJHRXwBnrh/iH -v0eSEi3sH+j0fwy9OBLJAP9ne01k9LkCXplS8ys+0u0e4545IIbiw8D4ToupD25q -CiIIAP4hwNooM6bAlg2HDYTUxJl4LA0qlJS66qnXv94Q8p4VngD/Y5O0AJw06BCw -Xcgnuh6Rlywt6uiaFIGYnGefYPGXRAA= -=kRLH +RFlKdKi0HVRlc3QgdXNlciA8ZW1haWxAZXhhbXBsZS5jb20+iJMEExYIADsCGwMF +CwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQRVkwbu4cjBst0cc7HENHgc6HHz3wUC +XyxWUwIZAQAKCRDENHgc6HHz30lxAP9Zvb7ssZ0bg94u2y1G3zbh8+5svSmGp3HT +hxMooIHvcwEA8jB3s5fVTZBIXagHBxACGSG5EhxlA8KlmaOSDGvl9w+0IHByaW1h +cnkgaWQgPHByaW1hcnlAZXhhbXBsZS5vcmc+iJAEExYIADgCGwMFCwkIBwIGFQoJ +CAsCBBYCAwECHgECF4AWIQRVkwbu4cjBst0cc7HENHgc6HHz3wUCXyxWUwAKCRDE +NHgc6HHz3zxDAQCB9zEqs0mWmriFqhXtRSwjhLhbprWxpAqkWTat6AU6XgD+MDVY +YgKEHeLuKqJ1MiR+x53f5ypxtA5eHJZdbs5OEA+0IlVURi04IMO8aWQgPHNlY29u +ZGFyeUBleGFtcGxlLm9yZz6IkAQTFggAOBYhBFWTBu7hyMGy3RxzscQ0eBzocfPf +BQJhaHZSAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEMQ0eBzocfPfnboA ++wZMsDxIterEz3NJq/8DL9M+zGkw+m+a1i7vIoujsJOQAP9wQwRRDOE16vTjlj5B +cATF0DFQTkv7Efmh8pveItzzCbg4BFvxgaoSCisGAQQBl1UBBQEBB0A2uTeqr94/ +VMRZ8VmqFHS9zPtgH1zn46e8xX7E7hbedAMBCAeIeAQYFggAIBYhBFWTBu7hyMGy +3RxzscQ0eBzocfPfBQJb8YGqAhsMAAoJEMQ0eBzocfPf8IcA/RyHF6zgRu2Ds3wH +8GgxjCZRW+YxWahX55/++pi9+bqyAQDp8rMxJeWXDntQn3RSfKfE7AnWF4Sd+aA+ +S6LPuMUICrgzBF4JDZAWCSsGAQQB2kcPAQEHQD6/ZjyfzyP5bAhDeEKE9HvFrj2m +tFLIpGcYEP8BeEl9iO8EGBYKACAWIQRVkwbu4cjBst0cc7HENHgc6HHz3wUCXgkN +kAIbAgCBCRDENHgc6HHz33YgBBkWCgAdFiEEs897M/gHYzjbT6gOh5GSgxF1O9cF +Al4JDZAACgkQh5GSgxF1O9fQBwD+Ly2M/yKEzackdFfAGeuH+Ie/R5ISLewf6PR/ +DL04EskA/2d7TWT0uQJemVLzKz7S7R7jnjkghuLDwPhOi6kPbmoKIggA/iHA2igz +psCWDYcNhNTEmXgsDSqUlLrqqde/3hDynhWeAP9jk7QAnDToELBdyCe6HpGXLC3q +6JoUgZicZ59g8ZdEAA== +=/JDz -----END PGP PUBLIC KEY BLOCK-----""" # def _setup_gpg_env(self): @@ -183,12 +188,14 @@ Xcgnuh6Rlywt6uiaFIGYnGefYPGXRAA= # shutil.rmtree(self.homedir) # def setUp(self): + self.locale = getlocale(LC_CTYPE) self._setup_example_user() # self._setup_gpg_env() # self._setup_models() # self._setup_example_countries() def tearDown(self): + setlocale(LC_CTYPE, self.locale) self._remove_example_user() # self._remove_example_countries() # self._cleanup_gpg_env() @@ -246,6 +253,20 @@ Xcgnuh6Rlywt6uiaFIGYnGefYPGXRAA= self.assertEquals(response.status_code, 302) self.assertIn(reverse('login'), response.url) + def test_gpg_encoding(self): + setlocale(LC_CTYPE, 'C') + + self.client.post(reverse('login'), self._AUTHDATA) + response = self.client.post(reverse('profile'), { + 'key': self._GPGKEY_2, + 'commit_gpg': 'submit' + }) + self.assertEquals(response.status_code, 200) + self.assertNotIn('errorlist', str(response.content)) + + user = User.objects.get(email='email@example.com') + self.assertEquals(user.key.key, self._GPGKEY_2) + def test__gpg(self): # Anonymous access to the form is denined response = self.client.post(reverse('profile'), {'form': 'gpg'}) -- GitLab From e85d0d894280c9f80f2cd2f837dfd9cd36bdcc6e Mon Sep 17 00:00:00 2001 From: Baptiste Beauplat Date: Fri, 15 Oct 2021 20:50:18 +0200 Subject: [PATCH 2/4] Use text=True as default for debexpo_exec --- debexpo/base/views.py | 3 +-- debexpo/plugins/diffclean.py | 3 +-- debexpo/plugins/lintian.py | 3 +-- debexpo/plugins/watch-file.py | 3 +-- debexpo/tools/debian/source.py | 3 +-- debexpo/tools/gnupg.py | 2 +- debexpo/tools/proc.py | 5 ++++- 7 files changed, 10 insertions(+), 12 deletions(-) diff --git a/debexpo/base/views.py b/debexpo/base/views.py index 9726afbb..c8906975 100644 --- a/debexpo/base/views.py +++ b/debexpo/base/views.py @@ -63,8 +63,7 @@ def get_debexpo_version(): # currently are in a git reposirtory. Hence, this is tested manually. try: output = debexpo_exec(command, args, - cwd=dirname(abspath(__file__)), - text=True) + cwd=dirname(abspath(__file__))) except FileNotFoundError: # pragma: no cover log.debug('git not found, skip revision detection.') except CalledProcessError: # pragma: no cover diff --git a/debexpo/plugins/diffclean.py b/debexpo/plugins/diffclean.py index af8c92fb..0197756a 100644 --- a/debexpo/plugins/diffclean.py +++ b/debexpo/plugins/diffclean.py @@ -43,8 +43,7 @@ class PluginDiffClean(BasePlugin): def _run_diffstat(self, diff_file): try: output = debexpo_exec("diffstat", ["-p1", diff_file], - cwd=dirname(diff_file), - text=True) + cwd=dirname(diff_file)) except FileNotFoundError: # pragma: no cover self.failed('diffstat not found') # Looking at diffstat code, it only exit with a return code different diff --git a/debexpo/plugins/lintian.py b/debexpo/plugins/lintian.py index 774104b5..a7d37c0d 100644 --- a/debexpo/plugins/lintian.py +++ b/debexpo/plugins/lintian.py @@ -68,8 +68,7 @@ class PluginLintian(BasePlugin): # run as root in CI. "--allow-root", str(changes)], - cwd=dirname(changes.filename), - text=True) + cwd=dirname(changes.filename)) except FileNotFoundError: # pragma: no cover self.failed('lintian not found') except TimeoutExpired: diff --git a/debexpo/plugins/watch-file.py b/debexpo/plugins/watch-file.py index aa05a5c1..4902a499 100644 --- a/debexpo/plugins/watch-file.py +++ b/debexpo/plugins/watch-file.py @@ -48,8 +48,7 @@ class PluginWatchFile(BasePlugin): def _run_uscan(self, source): try: output = debexpo_exec("uscan", ["--dehs", '--report'], - cwd=source.get_source_dir(), - text=True) + cwd=source.get_source_dir()) except FileNotFoundError: # pragma: no cover self.failed('uscan not found') except CalledProcessError as e: diff --git a/debexpo/tools/debian/source.py b/debexpo/tools/debian/source.py index 49f6f84e..2c86719d 100644 --- a/debexpo/tools/debian/source.py +++ b/debexpo/tools/debian/source.py @@ -57,8 +57,7 @@ class Source(): try: debexpo_exec('dpkg-source', args, stderr=STDOUT, - cwd=dirname(self.dsc.filename), - text=True) + cwd=dirname(self.dsc.filename)) except FileNotFoundError: # pragma: no cover log.error('dpkg-source not found') raise ExceptionSource('Internal error. Please contact debexpo ' diff --git a/debexpo/tools/gnupg.py b/debexpo/tools/gnupg.py index 824ced96..41395f17 100644 --- a/debexpo/tools/gnupg.py +++ b/debexpo/tools/gnupg.py @@ -225,7 +225,7 @@ class GnuPG(): try: output = debexpo_exec(self.gpg_path, cmd, env=env, stderr=subprocess.STDOUT, - input=str(stdin), text=True) + input=str(stdin)) except subprocess.CalledProcessError as e: return (e.output, e.returncode) except subprocess.TimeoutExpired: diff --git a/debexpo/tools/proc.py b/debexpo/tools/proc.py index e25fc148..7b2b7719 100644 --- a/debexpo/tools/proc.py +++ b/debexpo/tools/proc.py @@ -38,4 +38,7 @@ def debexpo_exec(command, args, **kwargs): f'{basename(command).replace("-", "_").upper()}', getattr(settings, 'SUBPROCESS_TIMEOUT', None)) - return check_output([command] + args, timeout=timeout, **kwargs) + return check_output([command] + args, + timeout=timeout, + text=True, + **kwargs) -- GitLab From 805c37aa30a043f922b5718effe9612e1a6c2723 Mon Sep 17 00:00:00 2001 From: Baptiste Beauplat Date: Fri, 15 Oct 2021 20:50:45 +0200 Subject: [PATCH 3/4] Force utf-8 decoding when running an external command --- debexpo/tools/proc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debexpo/tools/proc.py b/debexpo/tools/proc.py index 7b2b7719..c6a992bd 100644 --- a/debexpo/tools/proc.py +++ b/debexpo/tools/proc.py @@ -40,5 +40,6 @@ def debexpo_exec(command, args, **kwargs): return check_output([command] + args, timeout=timeout, + encoding='utf-8', text=True, **kwargs) -- GitLab From 2e881f6ca3cc90f0239692c3980f10cf7389d212 Mon Sep 17 00:00:00 2001 From: Baptiste Beauplat Date: Fri, 15 Oct 2021 22:04:28 +0200 Subject: [PATCH 4/4] Add locales-all as dependency to gitlab CI --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c2a239f9..bc3880ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,7 @@ before_script: python3-django-filters python3-django-redis ca-certificates + locales-all libjs-jquery libjs-jquery-throttle-debounce libjs-jquery-isonscreen -- GitLab