Commit 0b0e6dc2 authored by Ximin Luo's avatar Ximin Luo

Fix the logic of some tests, and don't vary fileordering on Debian buildds

Rename "variation" to "capture" in mock_build.py to be clearer what happens:
we're not varying those things, we're capturing them in the build output.

Capture the whole of `locale`; python getlocale() sometimes isn't enough.
parent fe49513c
......@@ -11,14 +11,13 @@ Build-Depends: debhelper (>= 9),
# tests.py uses debuild from devscripts
devscripts <!nocheck>,
diffoscope <!nocheck>,
# test this variation as well
disorderfs <!nocheck>,
python3-coverage <!nocheck>,
python3-pytest <!nocheck>,
tox <!nocheck>,
# these below helps diffoscope produce nicer output in tests
python3-tlsh <!nocheck>,
unzip <!nocheck>
unzip <!nocheck>,
xxd <!nocheck>
Vcs-Git: https://anonscm.debian.org/git/reproducible/reprotest.git
Vcs-Browser: https://anonscm.debian.org/git/reproducible/reprotest.git
X-Python3-Version: >= 3.5
......
......@@ -6,12 +6,15 @@ export PYBUILD_NAME = reprotest
%:
dh $@ --with python3 --buildsystem=pybuild
# It's hard to make sure disorderfs works on buildds, so drop it.
# The build user needs fuse access and this is not easily arranged.
export REPROTEST_TEST_DONTVARY = fileordering
override_dh_auto_test:
ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
# FIXME: The patch from #838663 is also required for this to work on pbuilder
VIRTUALENV_DOWNLOAD=no \
http_proxy=http://127.0.9.1:9 \
https_proxy=https://127.0.9.1:9 \
TOX_TESTENV_PASSENV=PYTHONIOENCODING PYTHONIOENCODING=utf-8 \
REPROTEST_TEST_SERVERS=null tox -r --sitepackages -- -s
tox -r --sitepackages -- -s
endif
......@@ -15,18 +15,18 @@ if __name__ == '__main__':
arg_parser = argparse.ArgumentParser(
description='Create binaries for testing reproducibility.',
formatter_class=argparse.RawDescriptionHelpFormatter)
arg_parser.add_argument('variations', nargs='*',
arg_parser.add_argument('captures', nargs='*',
help='Reproducibility properties.')
variations = set(arg_parser.parse_args().variations)
captures = set(arg_parser.parse_args().captures)
output = [b'']
# This test can theoretically fail by producing the same
# random bits in both runs, but it is extremely unlikely.
if 'irreproducible' in variations:
if 'irreproducible' in captures:
output.append(os.urandom(1024))
# Like the above test, this test can theoretically fail by
# producing the same file order, but this is unlikely, if not
# as unlikely as in the above test.
if 'fileordering' in variations:
if 'fileordering' in captures:
# Ensure this temporary directory is created in the disorders
# mount point by passing the dir argument.
with tempfile.TemporaryDirectory(dir=str(pathlib.Path.cwd())) as temp:
......@@ -34,20 +34,23 @@ if __name__ == '__main__':
for i in range(20):
str((test_file_order/str(i)).touch())
output.extend(p.name.encode('ascii') for p in test_file_order.iterdir())
if 'home' in variations:
if 'home' in captures:
output.append(os.path.expanduser('~').encode('ascii'))
if 'kernel' in variations:
if 'kernel' in captures:
output.append(subprocess.check_output(['uname', '-r']))
if 'locales' in variations:
if 'locales' in captures:
output.extend(l.encode('ascii') if l else b'(None)' for l in locale.getlocale())
if 'path' in variations:
output.append(subprocess.check_output(['locale']))
if 'path' in captures:
output.extend(p.encode('ascii') for p in os.get_exec_path())
if 'timezone' in variations:
if 'timezone' in captures:
output.append(str(time.timezone).encode('ascii'))
if 'umask' in variations:
if 'umask' in captures:
with tempfile.TemporaryDirectory(dir=str(pathlib.Path.cwd())) as temp:
test_permissions = pathlib.Path(temp)/'test_permissions'
test_permissions.touch()
output.append(stat.filemode(test_permissions.stat().st_mode).encode('ascii'))
else:
os.umask(0o0022) # otherwise open() will capture the surrounding one in its file metadata
with open('artifact', 'wb') as artifact:
artifact.write(b''.join(output))
......@@ -9,15 +9,25 @@ import pytest
import reprotest
REPROTEST = [sys.executable, "-m", "reprotest"]
REPROTEST_TEST_SERVERS = os.getenv("REPROTEST_TEST_SERVERS", "null").split(",")
REPROTEST_TEST_DONTVARY = os.getenv("REPROTEST_TEST_DONTVARY", "").split(",")
if REPROTEST_TEST_DONTVARY:
REPROTEST += ["--dont-vary", ",".join(REPROTEST_TEST_DONTVARY)]
TEST_VARIATIONS = frozenset(reprotest.VARIATIONS.keys()) - frozenset(REPROTEST_TEST_DONTVARY)
def check_return_code(command, virtual_server, code):
try:
retcode = reprotest.check(command, 'artifact', virtual_server, 'tests')
assert(code == retcode)
retcode = reprotest.check(command, 'artifact', virtual_server, 'tests', variations=TEST_VARIATIONS)
except SystemExit as system_exit:
assert(system_exit.args[0] == code)
retcode = system_exit.args[0]
finally:
if isinstance(code, int):
assert(retcode == code)
else:
assert(retcode in code)
REPROTEST_TEST_SERVERS = os.getenv("REPROTEST_TEST_SERVERS", "null").split(",")
@pytest.fixture(scope='module', params=REPROTEST_TEST_SERVERS)
def virtual_server(request):
if request.param == 'null':
......@@ -30,20 +40,21 @@ def virtual_server(request):
raise ValueError(request.param)
def test_simple_builds(virtual_server):
# mock_build is not expected to reproduce when disorderfs is active, though
# we should probably change "1" to int(is_disorderfs_active)
check_return_code('python3 mock_build.py', virtual_server, 1)
check_return_code('python3 mock_build.py', virtual_server, 0)
check_return_code('python3 mock_failure.py', virtual_server, 2)
check_return_code('python3 mock_build.py irreproducible', virtual_server, 1)
@pytest.mark.parametrize('variation', ['fileordering', 'home', 'kernel', 'locales', 'path', 'timezone', 'umask'])
def test_variations(virtual_server, variation):
check_return_code('python3 mock_build.py ' + variation, virtual_server, 1)
@pytest.mark.parametrize('captures', ['fileordering', 'home', 'kernel', 'locales', 'path', 'timezone', 'umask'])
def test_variations(virtual_server, captures):
expected = 1 if captures in TEST_VARIATIONS else 0
check_return_code('python3 mock_build.py ' + captures, virtual_server, expected)
def test_self_build(virtual_server):
assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist', 'dist/*.tar.gz'] + virtual_server))
# at time of writing (2016-09-23) these are not expected to reproduce;
# strip-nondeterminism normalises them for Debian
# if these start failing then you should change 1 == to 0 == but please
# figure out which version of setuptools made things reproduce and add a
# versioned dependency on that one
assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist', 'dist/*.tar.gz'] + virtual_server))
assert(1 == subprocess.call(REPROTEST + ['python3 setup.py sdist 2>/dev/null', 'dist/*.tar.gz'] + virtual_server))
assert(1 == subprocess.call(REPROTEST + ['python3 setup.py bdist_wheel', 'dist/*.whl'] + virtual_server))
......
......@@ -15,7 +15,7 @@ commands =
{envpython} -m coverage html
[testenv]
passenv = REPROTEST_TEST_SERVERS VIRTUALENV_DOWNLOAD *_proxy
passenv = REPROTEST_TEST_* VIRTUALENV_DOWNLOAD *_proxy
# usedevelop = True
deps =
coverage
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment