Commit 6f393354 authored by Tristan Seligmann's avatar Tristan Seligmann

Imported Debian patch 0.0.20-1

......@@ -6,7 +6,7 @@ Run time dependencies
* Python2.4 or newer.
* subunit
* subunit (0.0.18 or newer).
* fixtures (https://launchpad.net/python-fixtures, or
http://pypi.python.org/pypi/fixtures/).
......
......@@ -5,6 +5,57 @@ testrepository release notes
NEXT (In development)
+++++++++++++++++++++
0.0.20
++++++
IMPROVEMENTS
------------
* Tests will be reliably tagged with worker-%d. The previous tagging logic
had an implicit race - the tag id was looked up via a closure which gets
the state of the pos variable at the position the overall loop has advanced
too, not the position when the closure was created.
(Robert Collins, #1316858)
0.0.19
++++++
CHANGES
-------
* Passing --subunit to all testr commands will now consistently output subunit
v2. Previously it would output v1 for stored streams and v2 for live
streams. (Robert Collins)
* ``run`` was outputting bad MIME types - test/plain, not text/plain.
(Robert Collins)
* Test filtering was failing under python3 and would only apply the
filters to the first test listed by discover. (Clark Boylan, #1317607)
* Tests that are enumerated but not executed will no longer reset the test
timing data. Enumeration was incorrectly recording a 0 timestamp for
enumerated tests. This leads to poor scheduling after an interrupted test
run. (Robert Collins, #1322763)
* Version 0.0.18 of subunit is now a hard dependency - the v2 protocol solves
key issues in concurrency and stream handling. Users that cannot use subunit
v2 can run an older testrepository, or contact upstream to work through
whatever issue is blocking them. (Robert Collins)
* When list-tests encounters an error, a much clearer response will
now be shown. (Robert Collins, #1271133)
INTERNALS
---------
* The ``get_subunit_stream`` methods now return subunit v2 streams rather
than v1 streams, preparing the way for storage of native v2 streams in
the repository. (Robert Collins)
* ``UI.output_stream`` is now tested for handling of non-utf8 bytestreams.
(Robert Collins)
0.0.18
++++++
......
Metadata-Version: 1.1
Name: testrepository
Version: 0.0.18
Version: 0.0.20
Summary: A repository of test results.
Home-page: https://launchpad.net/testrepository
Author: Robert Collins
......
testrepository (0.0.20-1) unstable; urgency=medium
* New upstream release.
* Fix ordering of paragraphs in debian/copyright.
-- Tristan Seligmann <mithrandi@debian.org> Sat, 13 Sep 2014 19:57:12 +0200
testrepository (0.0.18-5) unstable; urgency=medium
* Correct license information in debian/copyright; testrepository is
......
......@@ -3,16 +3,16 @@ Upstream-Name: testrepository
Upstream-Contact: Robert Collins <robertc@robertcollins.net>
Source: https://github.com/testing-cabal/testrepository
Files: *
Copyright: © 2009-2013, Robert Collins <robertc@robertcollins.net>
License: Apache-2.0-or-BSD-3-clauses
Files: debian/*
Copyright: © 2010-2011, Robert Collins <robertc@robertcollins.net>
© 2011, Jakub Wilk <jwilk@debian.org>
© 2013, Thomas Goirand <zigo@debian.org>
License: Apache-2.0-or-BSD-3-clauses
Files: *
Copyright: © 2009-2013, Robert Collins <robertc@robertcollins.net>
License: Apache-2.0-or-BSD-3-clauses
License: Apache-2.0-or-BSD-3-clauses
Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
license at the users choice. A copy of both licenses are available in the
......
......@@ -52,3 +52,4 @@ Releasing
Update NEWS and testrepository/__init__.py version numbers. Release to pypi.
Pivot the next milestone on LP to version, and make a new next milestone.
Make a new tag and push that to github.
......@@ -95,7 +95,7 @@ setup(name='testrepository',
],
install_requires=[
'fixtures',
'python-subunit >= 0.0.10',
'python-subunit >= 0.0.18',
'testtools >= 0.9.30',
],
extras_require = dict(
......
Metadata-Version: 1.1
Name: testrepository
Version: 0.0.18
Version: 0.0.20
Summary: A repository of test results.
Home-page: https://launchpad.net/testrepository
Author: Robert Collins
......
fixtures
python-subunit >= 0.0.10
python-subunit >= 0.0.18
testtools >= 0.9.30
[test]
......
......@@ -33,4 +33,4 @@ The tests package contains tests and test specific support code.
# established at this point, and setup.py will use a version of next-$(revno).
# If the releaselevel is 'final', then the tarball will be major.minor.micro.
# Otherwise it is major.minor.micro~$(revno).
__version__ = (0, 0, 18, 'final', 0)
__version__ = (0, 0, 20, 'final', 0)
......@@ -100,6 +100,8 @@ class load(Command):
streams = map(opener, self.ui.arguments['streams'])
else:
streams = self.ui.iter_streams('subunit')
mktagger = lambda pos, result:testtools.StreamTagger(
[result], add=['worker-%d' % pos])
def make_tests():
for pos, stream in enumerate(streams):
if v2_avail:
......@@ -121,9 +123,8 @@ class load(Command):
case = testtools.DecorateTestCaseResult(case, wrap_result,
methodcaller('startTestRun'),
methodcaller('stopTestRun'))
case = testtools.DecorateTestCaseResult(case,
lambda result:testtools.StreamTagger(
[result], add=['worker-%d' % pos]))
decorate = partial(mktagger, pos)
case = testtools.DecorateTestCaseResult(case, decorate)
yield (case, str(pos))
case = testtools.ConcurrentStreamTestSuite(make_tests)
# One unmodified copy of the stream to repository storage
......
......@@ -79,7 +79,7 @@ class ReturnCodeToSubunit(object):
if v2_avail:
stream = subunit.StreamResultToBytes(self.source)
stream.status(test_id='process-returncode', test_status='fail',
file_name='traceback', mime_type='test/plain;charset=utf8',
file_name='traceback', mime_type='text/plain;charset=utf8',
file_bytes=('returncode %d' % returncode).encode('utf8'))
else:
self.source.write(_b('test: process-returncode\n'
......
......@@ -25,7 +25,7 @@ import os.path
import sys
import tempfile
import subunit
import subunit.v2
from subunit import TestProtocolClient
import testtools
from testtools.compat import _b
......@@ -188,10 +188,23 @@ class _DiskRun(AbstractTestRun):
return self._run_id
def get_subunit_stream(self):
return BytesIO(self._content)
# Transcode - we want V2.
v1_stream = BytesIO(self._content)
v1_case = subunit.ProtocolTestCase(v1_stream)
output = BytesIO()
output_stream = subunit.v2.StreamResultToBytes(output)
output_stream = testtools.ExtendedToStreamDecorator(output_stream)
output_stream.startTestRun()
try:
v1_case.run(output_stream)
finally:
output_stream.stopTestRun()
output.seek(0)
return output
def get_test(self):
case = subunit.ProtocolTestCase(self.get_subunit_stream())
#case = subunit.ProtocolTestCase(self.get_subunit_stream())
case = subunit.ProtocolTestCase(BytesIO(self._content))
def wrap_result(result):
# Wrap in a router to mask out startTestRun/stopTestRun from the
# ExtendedToStreamDecorator.
......@@ -227,7 +240,7 @@ class _SafeInserter(object):
def _handle_test(self, test_dict):
start, stop = test_dict['timestamps']
if None in (start, stop):
if test_dict['status'] == 'exists' or None in (start, stop):
return
self._times[test_dict['id']] = str(timedelta_to_seconds(stop - start))
......
......@@ -102,8 +102,13 @@ class _Failures(AbstractTestRun):
def get_subunit_stream(self):
result = BytesIO()
serialiser = subunit.TestProtocolClient(result)
self.run(serialiser)
serialiser = subunit.v2.StreamResultToBytes(result)
serialiser = testtools.ExtendedToStreamDecorator(serialiser)
serialiser.startTestRun()
try:
self.run(serialiser)
finally:
serialiser.stopTestRun()
result.seek(0)
return result
......@@ -120,7 +125,7 @@ class _Failures(AbstractTestRun):
methodcaller('stopTestRun'))
def run(self, result):
# Speaks original.
# Speaks original V1 protocol.
for case in self._repository._failing.values():
case.run(result)
......@@ -132,13 +137,12 @@ class _Inserter(AbstractTestRun):
self._repository = repository
self._partial = partial
self._tests = []
# Subunit V1 stream for get_subunit_stream
# Subunit V2 stream for get_subunit_stream
self._subunit = None
def startTestRun(self):
self._subunit = BytesIO()
serialiser = subunit.TestProtocolClient(self._subunit)
serialiser = testtools.StreamToExtendedDecorator(serialiser)
serialiser = subunit.v2.StreamResultToBytes(self._subunit)
self._hook = testtools.CopyStreamResult([
testtools.StreamToDict(self._handle_test),
serialiser])
......@@ -147,7 +151,7 @@ class _Inserter(AbstractTestRun):
def _handle_test(self, test_dict):
self._tests.append(test_dict)
start, stop = test_dict['timestamps']
if None in (start, stop):
if test_dict['status'] == 'exists' or None in (start, stop):
return
duration_delta = stop - start
duration_seconds = ((duration_delta.microseconds +
......
......@@ -13,7 +13,11 @@
# limitations under that license.
from testtools import StreamSummary
import subunit
from testtools import (
StreamSummary,
StreamResult,
)
from testrepository.utils import timedelta_to_seconds
......@@ -47,3 +51,23 @@ class SummarizingResult(StreamSummary):
if None in (self._last_time, self._first_time):
return None
return timedelta_to_seconds(self._last_time - self._first_time)
#XXX: Should be in testtools.
class CatFiles(StreamResult):
"""Cat file attachments received to a stream."""
def __init__(self, byte_stream):
self.stream = subunit.make_stream_binary(byte_stream)
self.last_file = None
def status(self, test_id=None, test_status=None, test_tags=None,
runnable=True, file_name=None, file_bytes=None, eof=False,
mime_type=None, route_code=None, timestamp=None):
if file_name is None:
return
if self.last_file != file_name:
self.stream.write(("--- %s ---\n" % file_name).encode('utf8'))
self.last_file = file_name
self.stream.write(file_bytes)
self.stream.flush()
......@@ -14,10 +14,14 @@
"""The test command that test repository knows how to run."""
from extras import try_imports
from extras import (
try_import,
try_imports,
)
from collections import defaultdict
ConfigParser = try_imports(['ConfigParser', 'configparser'])
import io
import itertools
import operator
import os.path
......@@ -29,7 +33,9 @@ import multiprocessing
from textwrap import dedent
from fixtures import Fixture
v2 = try_import('subunit.v2')
from testrepository import results
from testrepository.testlist import (
parse_enumeration,
write_list,
......@@ -273,7 +279,7 @@ class TestListingFixture(Fixture):
"""
if self.test_filters is None:
return test_ids
filters = map(re.compile, self.test_filters)
filters = list(map(re.compile, self.test_filters))
def include(test_id):
for pred in filters:
if pred.search(test_id):
......@@ -294,9 +300,16 @@ class TestListingFixture(Fixture):
stdout=subprocess.PIPE, stdin=subprocess.PIPE)
out, err = run_proc.communicate()
if run_proc.returncode != 0:
if v2 is not None:
new_out = io.BytesIO()
v2.ByteStreamToStreamResult(io.BytesIO(out), 'stdout').run(
results.CatFiles(new_out))
out = new_out.getvalue()
self.ui.output_stream(io.BytesIO(out))
self.ui.output_stream(io.BytesIO(err))
raise ValueError(
"Non-zero exit code (%d) from test listing."
" stdout=%r, stderr=%r" % (run_proc.returncode, out, err))
% (run_proc.returncode))
ids = parse_enumeration(out)
return ids
finally:
......@@ -443,8 +456,8 @@ class TestListingFixture(Fixture):
out, err = run_proc.communicate()
if run_proc.returncode:
raise ValueError(
"test_run_concurrency failed: exit code %d, stderr=%r" % (
run_proc.returncode, err))
"test_run_concurrency failed: exit code %d, stderr='%s'" % (
run_proc.returncode, err.decode('utf8', 'replace')))
return int(out.strip())
def local_concurrency(self):
......
......@@ -15,13 +15,16 @@
"""Tests for the failing command."""
import doctest
from io import BytesIO
from subunit.v2 import ByteStreamToStreamResult
import testtools
from testtools.compat import _b
from testtools.matchers import (
DocTestMatches,
Equals,
)
from testtools.testresult.doubles import StreamResult
from testrepository.commands import failing
from testrepository.ui.model import UI
......@@ -78,9 +81,23 @@ class TestCommand(ResourcedTestCase):
self.assertEqual(0, cmd.execute())
self.assertEqual(1, len(ui.outputs))
self.assertEqual('stream', ui.outputs[0][0])
self.assertThat(ui.outputs[0][1].decode('utf8'),
DocTestMatches("""...test: ...failing
...failure: ...failing...""", doctest.ELLIPSIS))
as_subunit = BytesIO(ui.outputs[0][1])
stream = ByteStreamToStreamResult(as_subunit)
log = StreamResult()
log.startTestRun()
try:
stream.run(log)
finally:
log.stopTestRun()
self.assertEqual(
log._events, [
('startTestRun',),
('status', 'failing', 'inprogress', None, True, None, None, False,
None, None, Wildcard),
('status', 'failing', 'fail', None, True, None, None, False, None,
None, Wildcard),
('stopTestRun',)
])
def test_with_subunit_no_failures_exit_0(self):
ui, cmd = self.get_test_ui_and_cmd(options=[('subunit', True)])
......
......@@ -14,9 +14,12 @@
"""Tests for the last command."""
from io import BytesIO
from subunit.v2 import ByteStreamToStreamResult
import testtools
from testtools.compat import _b
from testtools.matchers import Equals
from testtools.testresult.doubles import StreamResult
from testrepository.commands import last
from testrepository.ui.model import UI
......@@ -104,11 +107,20 @@ class TestCommand(ResourcedTestCase):
self.assertEqual([
('stream', Wildcard),
], ui.outputs)
self.assertThat(ui.outputs[0][1], Equals(_b("""\
test: failing
failure: failing [ multipart
]
test: ok
successful: ok [ multipart
]
""")))
as_subunit = BytesIO(ui.outputs[0][1])
stream = ByteStreamToStreamResult(as_subunit)
log = StreamResult()
log.startTestRun()
try:
stream.run(log)
finally:
log.stopTestRun()
self.assertEqual(
log._events, [
('startTestRun',),
('status', 'failing', 'fail', None, True, None, None, False,
None, None, None),
('status', 'ok', 'success', None, True, None, None, False, None,
None, None),
('stopTestRun',)
])
......@@ -532,7 +532,7 @@ class TestReturnCodeToSubunit(ResourcedTestCase):
buffer.write(b'foo\nbar\n')
stream = subunit.StreamResultToBytes(buffer)
stream.status(test_id='process-returncode', test_status='fail',
file_name='traceback', mime_type='test/plain;charset=utf8',
file_name='traceback', mime_type='text/plain;charset=utf8',
file_bytes=b'returncode 1')
expected_content = buffer.getvalue()
else:
......
......@@ -20,7 +20,10 @@ from datetime import (
)
import doctest
from subunit import iso8601
from subunit import (
iso8601,
v2,
)
from testresources import TestResource
from testtools import (
......@@ -29,7 +32,10 @@ from testtools import (
)
import testtools
from testtools.compat import _b
from testtools.testresult.doubles import ExtendedTestResult
from testtools.testresult.doubles import (
ExtendedTestResult,
StreamResult,
)
from testtools.matchers import DocTestMatches, raises
from testrepository import repository
......@@ -113,12 +119,18 @@ def make_test(id, should_pass):
return clone_test_with_new_id(case, id)
def run_timed(id, duration, result):
"""Make and run a test taking duration seconds."""
def run_timed(id, duration, result, enumeration=False):
"""Make and run a test taking duration seconds.
:param enumeration: If True, don't run, just enumerate.
"""
start = datetime.now(tz=iso8601.Utc())
result.status(test_id=id, test_status='inprogress', timestamp=start)
result.status(test_id=id, test_status='success',
timestamp=start + timedelta(seconds=duration))
if enumeration:
result.status(test_id=id, test_status='exists', timestamp=start)
else:
result.status(test_id=id, test_status='inprogress', timestamp=start)
result.status(test_id=id, test_status='success',
timestamp=start + timedelta(seconds=duration))
class TestRepositoryErrors(ResourcedTestCase):
......@@ -374,6 +386,61 @@ class TestRepositoryContract(ResourcedTestCase):
run = repo.get_failing()
self.assertEqual(None, run.get_id())
def test_get_failing_get_subunit_stream(self):
repo = self.repo_impl.initialise(self.sample_url)
result = repo.get_inserter()
legacy_result = testtools.ExtendedToStreamDecorator(result)
legacy_result.startTestRun()
make_test('testrepository.tests.test_repository.Case.method', False).run(legacy_result)
legacy_result.stopTestRun()
run = repo.get_failing()
as_subunit = run.get_subunit_stream()
stream = v2.ByteStreamToStreamResult(as_subunit)
log = StreamResult()
log.startTestRun()
try:
stream.run(log)
finally:
log.stopTestRun()
self.assertEqual(
log._events, [
('startTestRun',),
('status',
'testrepository.tests.test_repository.Case.method',
'inprogress',
None,
True,
None,
None,
False,
None,
None,
Wildcard),
('status',
'testrepository.tests.test_repository.Case.method',
None,
None,
True,
'traceback',
Wildcard,
True,
Wildcard,
None,
Wildcard),
('status',
'testrepository.tests.test_repository.Case.method',
'fail',
None,
True,
None,
None,
False,
None,
None,
Wildcard),
('stopTestRun',)
])
def test_get_subunit_from_test_run(self):
repo = self.repo_impl.initialise(self.sample_url)
result = repo.get_inserter()
......@@ -384,10 +451,41 @@ class TestRepositoryContract(ResourcedTestCase):
inserted = result.get_id()
run = repo.get_test_run(inserted)
as_subunit = run.get_subunit_stream()
self.assertThat(as_subunit.read().decode('utf8'),
DocTestMatches("""...test: testrepository.tests.test_repository.Case.method...
successful: testrepository.tests.test_repository.Case.method...
""", doctest.ELLIPSIS))
stream = v2.ByteStreamToStreamResult(as_subunit)
log = StreamResult()
log.startTestRun()
try:
stream.run(log)
finally:
log.stopTestRun()
self.assertEqual(
log._events,
[
('startTestRun',),
('status',
'testrepository.tests.test_repository.Case.method',
'inprogress',
None,
True,
None,
None,
False,
None,
None,
Wildcard),
('status',
'testrepository.tests.test_repository.Case.method',
'success',
None,
True,
None,
None,
False,
None,
None,
Wildcard),
('stopTestRun',)
])
def test_get_test_from_test_run(self):
repo = self.repo_impl.initialise(self.sample_url)
......@@ -423,6 +521,22 @@ successful: testrepository.tests.test_repository.Case.method...
self.assertEqual({test_name: 0.1},
repo.get_test_times([test_name])['known'])
def test_inserted_exists_no_impact_on_test_times(self):
repo = self.repo_impl.initialise(self.sample_url)
result = repo.get_inserter()
legacy_result = testtools.ExtendedToStreamDecorator(result)
legacy_result.startTestRun()
test_name = 'testrepository.tests.test_repository.Case.method'
run_timed(test_name, 0.1, legacy_result)
legacy_result.stopTestRun()
result = repo.get_inserter()
result.startTestRun()
test_name = 'testrepository.tests.test_repository.Case.method'
run_timed(test_name, 0.2, result, True)
result.stopTestRun()
self.assertEqual({test_name: 0.1},
repo.get_test_times([test_name])['known'])
def test_get_test_ids(self):
repo = self.repo_impl.initialise(self.sample_url)
inserter = repo.get_inserter()
......
......@@ -35,7 +35,7 @@ class TestCanSetup(TestCase):
proc = subprocess.Popen([sys.executable, path, 'bdist'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, universal_newlines=True)
output, _ = proc.communicate()
output, err = proc.communicate()
self.assertThat(output, MatchesAny(
# win32
DocTestMatches("""...
......@@ -48,4 +48,5 @@ adding '...testr'
...bin/testr ...
""", doctest.ELLIPSIS)
))
self.assertEqual(0, proc.returncode)
self.assertEqual(0, proc.returncode,
"Setup failed out=%r err=%r" % (output, err))
......@@ -575,3 +575,14 @@ class TestTestCommand(ResourcedTestCase):
fixture = self.useFixture(command.get_run_command(
test_ids=['return', 'of', 'the', 'king'], test_filters=filters))
self.assertEqual(['return'], fixture.test_ids)
def test_filter_tests_by_regex_supplied_ids_multi_match(self):
ui, command = self.get_test_ui_and_cmd()
ui.proc_outputs = [_b('returned\nids\n')]
self.set_config(
'[DEFAULT]\ntest_command=foo $LISTOPT $IDLIST\ntest_id_list_default=whoo yea\n'
'test_list_option=--list\n')
filters = ['return']
fixture = self.useFixture(command.get_run_command(
test_ids=['return', 'of', 'the', 'king', 'thereisnoreturn'], test_filters=filters))
self.assertEqual(['return', 'thereisnoreturn'], fixture.test_ids)
......@@ -119,6 +119,11 @@ class TestUIContract(ResourcedTestCase):
ui = self.get_test_ui()
ui.output_stream(BytesIO())
def test_output_stream_non_utf8(self):
# When the stream has non-utf8 bytes it still outputs correctly.
ui = self.get_test_ui()
ui.output_stream(BytesIO(_b('\xfa')))
def test_output_table(self):
# output_table shows a table.
ui = self.get_test_ui()
......
......@@ -109,7 +109,8 @@ class TestCLIUI(ResourcedTestCase):
except Exception:
err_tuple = sys.exc_info()
expected = str(err_tuple[1]) + '\n'
stdout = StringIO()
bytestream = BytesIO()
stdout = TextIOWrapper(bytestream, 'utf8', line_buffering=True)
stdin = StringIO()
stderr = StringIO()
ui = cli.UI([], stdin, stdout, stderr)
......@@ -128,6 +129,9 @@ class TestCLIUI(ResourcedTestCase):
<BLANKLINE>
fooo
""")
# This should be a BytesIO + Textwrapper, but pdb on 2.7 writes bytes
# - this code is the most pragmatic to test on 2.6 and up, and 3.2 and
# up.
stdout = StringIO()
stdin = StringIO(_u('c\n'))
stderr = StringIO()
......@@ -196,7 +200,8 @@ AssertionError: quux...
ui._stdout.buffer.getvalue())
def test_parse_error_goes_to_stderr(self):
stdout = StringIO()
bytestream = BytesIO()
stdout = TextIOWrapper(bytestream, 'utf8', line_buffering=True)
stdin = StringIO()
stderr = StringIO()
ui = cli.UI(['one'], stdin, stdout, stderr)
......@@ -206,7 +211,8 @@ AssertionError: quux...
self.assertEqual("Could not find command 'one'.\n", stderr.getvalue())
def test_parse_excess_goes_to_stderr(self):