Upgrading to GitLab 12.5.4.

Commit 0b0e6dc2 authored by Ximin Luo's avatar Ximin Luo

Fix the logic of some tests, and don't vary fileordering on Debian buildds

Rename "variation" to "capture" in mock_build.py to be clearer what happens:
we're not varying those things, we're capturing them in the build output.

Capture the whole of `locale`; python getlocale() sometimes isn't enough.
parent fe49513c
...@@ -11,14 +11,13 @@ Build-Depends: debhelper (>= 9), ...@@ -11,14 +11,13 @@ Build-Depends: debhelper (>= 9),
# tests.py uses debuild from devscripts # tests.py uses debuild from devscripts
devscripts <!nocheck>, devscripts <!nocheck>,
diffoscope <!nocheck>, diffoscope <!nocheck>,
# test this variation as well
disorderfs <!nocheck>,
python3-coverage <!nocheck>, python3-coverage <!nocheck>,
python3-pytest <!nocheck>, python3-pytest <!nocheck>,
tox <!nocheck>, tox <!nocheck>,
# these below helps diffoscope produce nicer output in tests # these below helps diffoscope produce nicer output in tests
python3-tlsh <!nocheck>, python3-tlsh <!nocheck>,
unzip <!nocheck> unzip <!nocheck>,
xxd <!nocheck>
Vcs-Git: https://anonscm.debian.org/git/reproducible/reprotest.git Vcs-Git: https://anonscm.debian.org/git/reproducible/reprotest.git
Vcs-Browser: https://anonscm.debian.org/git/reproducible/reprotest.git Vcs-Browser: https://anonscm.debian.org/git/reproducible/reprotest.git
X-Python3-Version: >= 3.5 X-Python3-Version: >= 3.5
......
...@@ -6,12 +6,15 @@ export PYBUILD_NAME = reprotest ...@@ -6,12 +6,15 @@ export PYBUILD_NAME = reprotest
%: %:
dh $@ --with python3 --buildsystem=pybuild dh $@ --with python3 --buildsystem=pybuild
# It's hard to make sure disorderfs works on buildds, so drop it.
# The build user needs fuse access and this is not easily arranged.
export REPROTEST_TEST_DONTVARY = fileordering
override_dh_auto_test: override_dh_auto_test:
ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
# FIXME: The patch from #838663 is also required for this to work on pbuilder
VIRTUALENV_DOWNLOAD=no \ VIRTUALENV_DOWNLOAD=no \
http_proxy=http://127.0.9.1:9 \ http_proxy=http://127.0.9.1:9 \
https_proxy=https://127.0.9.1:9 \ https_proxy=https://127.0.9.1:9 \
TOX_TESTENV_PASSENV=PYTHONIOENCODING PYTHONIOENCODING=utf-8 \ TOX_TESTENV_PASSENV=PYTHONIOENCODING PYTHONIOENCODING=utf-8 \
REPROTEST_TEST_SERVERS=null tox -r --sitepackages -- -s tox -r --sitepackages -- -s
endif endif
...@@ -15,18 +15,18 @@ if __name__ == '__main__': ...@@ -15,18 +15,18 @@ if __name__ == '__main__':
arg_parser = argparse.ArgumentParser( arg_parser = argparse.ArgumentParser(
description='Create binaries for testing reproducibility.', description='Create binaries for testing reproducibility.',
formatter_class=argparse.RawDescriptionHelpFormatter) formatter_class=argparse.RawDescriptionHelpFormatter)
arg_parser.add_argument('variations', nargs='*', arg_parser.add_argument('captures', nargs='*',
help='Reproducibility properties.') help='Reproducibility properties.')
variations = set(arg_parser.parse_args().variations) captures = set(arg_parser.parse_args().captures)
output = [b''] output = [b'']
# This test can theoretically fail by producing the same # This test can theoretically fail by producing the same
# random bits in both runs, but it is extremely unlikely. # random bits in both runs, but it is extremely unlikely.
if 'irreproducible' in variations: if 'irreproducible' in captures:
output.append(os.urandom(1024)) output.append(os.urandom(1024))
# Like the above test, this test can theoretically fail by # Like the above test, this test can theoretically fail by
# producing the same file order, but this is unlikely, if not # producing the same file order, but this is unlikely, if not
# as unlikely as in the above test. # as unlikely as in the above test.
if 'fileordering' in variations: if 'fileordering' in captures:
# Ensure this temporary directory is created in the disorders # Ensure this temporary directory is created in the disorders
# mount point by passing the dir argument. # mount point by passing the dir argument.
with tempfile.TemporaryDirectory(dir=str(pathlib.Path.cwd())) as temp: with tempfile.TemporaryDirectory(dir=str(pathlib.Path.cwd())) as temp:
...@@ -34,20 +34,23 @@ if __name__ == '__main__': ...@@ -34,20 +34,23 @@ if __name__ == '__main__':
for i in range(20): for i in range(20):
str((test_file_order/str(i)).touch()) str((test_file_order/str(i)).touch())
output.extend(p.name.encode('ascii') for p in test_file_order.iterdir()) output.extend(p.name.encode('ascii') for p in test_file_order.iterdir())
if 'home' in variations: if 'home' in captures:
output.append(os.path.expanduser('~').encode('ascii')) output.append(os.path.expanduser('~').encode('ascii'))
if 'kernel' in variations: if 'kernel' in captures:
output.append(subprocess.check_output(['uname', '-r'])) output.append(subprocess.check_output(['uname', '-r']))
if 'locales' in variations: if 'locales' in captures:
output.extend(l.encode('ascii') if l else b'(None)' for l in locale.getlocale()) output.extend(l.encode('ascii') if l else b'(None)' for l in locale.getlocale())
if 'path' in variations: output.append(subprocess.check_output(['locale']))
if 'path' in captures:
output.extend(p.encode('ascii') for p in os.get_exec_path()) output.extend(p.encode('ascii') for p in os.get_exec_path())
if 'timezone' in variations: if 'timezone' in captures:
output.append(str(time.timezone).encode('ascii')) output.append(str(time.timezone).encode('ascii'))
if 'umask' in variations: if 'umask' in captures:
with tempfile.TemporaryDirectory(dir=str(pathlib.Path.cwd())) as temp: with tempfile.TemporaryDirectory(dir=str(pathlib.Path.cwd())) as temp:
test_permissions = pathlib.Path(temp)/'test_permissions' test_permissions = pathlib.Path(temp)/'test_permissions'
test_permissions.touch() test_permissions.touch()
output.append(stat.filemode(test_permissions.stat().st_mode).encode('ascii')) output.append(stat.filemode(test_permissions.stat().st_mode).encode('ascii'))
else:
os.umask(0o0022) # otherwise open() will capture the surrounding one in its file metadata
with open('artifact', 'wb') as artifact: with open('artifact', 'wb') as artifact:
artifact.write(b''.join(output)) artifact.write(b''.join(output))
...@@ -9,15 +9,25 @@ import pytest ...@@ -9,15 +9,25 @@ import pytest
import reprotest import reprotest
REPROTEST = [sys.executable, "-m", "reprotest"] REPROTEST = [sys.executable, "-m", "reprotest"]
REPROTEST_TEST_SERVERS = os.getenv("REPROTEST_TEST_SERVERS", "null").split(",")
REPROTEST_TEST_DONTVARY = os.getenv("REPROTEST_TEST_DONTVARY", "").split(",")
if REPROTEST_TEST_DONTVARY:
REPROTEST += ["--dont-vary", ",".join(REPROTEST_TEST_DONTVARY)]
TEST_VARIATIONS = frozenset(reprotest.VARIATIONS.keys()) - frozenset(REPROTEST_TEST_DONTVARY)
def check_return_code(command, virtual_server, code): def check_return_code(command, virtual_server, code):
try: try:
retcode = reprotest.check(command, 'artifact', virtual_server, 'tests') retcode = reprotest.check(command, 'artifact', virtual_server, 'tests', variations=TEST_VARIATIONS)
assert(code == retcode)
except SystemExit as system_exit: except SystemExit as system_exit:
assert(system_exit.args[0] == code) retcode = system_exit.args[0]
finally:
if isinstance(code, int):
assert(retcode == code)
else:
assert(retcode in code)
REPROTEST_TEST_SERVERS = os.getenv("REPROTEST_TEST_SERVERS", "null").split(",")
@pytest.fixture(scope='module', params=REPROTEST_TEST_SERVERS) @pytest.fixture(scope='module', params=REPROTEST_TEST_SERVERS)
def virtual_server(request): def virtual_server(request):
if request.param == 'null': if request.param == 'null':
...@@ -30,20 +40,21 @@ def virtual_server(request): ...@@ -30,20 +40,21 @@ def virtual_server(request):
raise ValueError(request.param) raise ValueError(request.param)
def test_simple_builds(virtual_server): def test_simple_builds(virtual_server):
# mock_build is not expected to reproduce when disorderfs is active, though check_return_code('python3 mock_build.py', virtual_server, 0)
# we should probably change "1" to int(is_disorderfs_active)
check_return_code('python3 mock_build.py', virtual_server, 1)
check_return_code('python3 mock_failure.py', virtual_server, 2) check_return_code('python3 mock_failure.py', virtual_server, 2)
check_return_code('python3 mock_build.py irreproducible', virtual_server, 1) check_return_code('python3 mock_build.py irreproducible', virtual_server, 1)
@pytest.mark.parametrize('variation', ['fileordering', 'home', 'kernel', 'locales', 'path', 'timezone', 'umask']) @pytest.mark.parametrize('captures', ['fileordering', 'home', 'kernel', 'locales', 'path', 'timezone', 'umask'])
def test_variations(virtual_server, variation): def test_variations(virtual_server, captures):
check_return_code('python3 mock_build.py ' + variation, virtual_server, 1) expected = 1 if captures in TEST_VARIATIONS else 0
check_return_code('python3 mock_build.py ' + captures, virtual_server, expected)
def test_self_build(virtual_server): def test_self_build(virtual_server):
assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist', 'dist/*.tar.gz'] + virtual_server))
# at time of writing (2016-09-23) these are not expected to reproduce; # at time of writing (2016-09-23) these are not expected to reproduce;
# strip-nondeterminism normalises them for Debian # if these start failing then you should change 1 == to 0 == but please
# figure out which version of setuptools made things reproduce and add a
# versioned dependency on that one
assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist', 'dist/*.tar.gz'] + virtual_server))
assert(1 == subprocess.call(REPROTEST + ['python3 setup.py sdist 2>/dev/null', 'dist/*.tar.gz'] + virtual_server)) assert(1 == subprocess.call(REPROTEST + ['python3 setup.py sdist 2>/dev/null', 'dist/*.tar.gz'] + virtual_server))
assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist_wheel', 'dist/*.whl'] + virtual_server)) assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist_wheel', 'dist/*.whl'] + virtual_server))
......
...@@ -15,7 +15,7 @@ commands = ...@@ -15,7 +15,7 @@ commands =
{envpython} -m coverage html {envpython} -m coverage html
[testenv] [testenv]
passenv = REPROTEST_TEST_SERVERS VIRTUALENV_DOWNLOAD *_proxy passenv = REPROTEST_TEST_* VIRTUALENV_DOWNLOAD *_proxy
# usedevelop = True # usedevelop = True
deps = deps =
coverage coverage
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment