🚀 Socket Launch Week Day 5:Introducing Repository Access Permissions and Custom Roles.Learn more →
Sign In

python-subunit

Package Overview
Dependencies
Maintainers
3
Versions
28
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

python-subunit - pypi Package Compare versions

Comparing version
1.4.5
 to
1.4.6
+38
python/subunit/filter_scripts/gojson2subunit.py
#!/usr/bin/env python3
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2026 Jelmer Vernooij <jelmer@samba.org>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""A filter that reads a `go test -json` stream and outputs a subunit stream.
Pipe Go's structured test output into this script:
go test -json ./... | gojson2subunit
The conversion preserves per-test elapsed time (via paired ``inprogress`` /
terminal subunit packets) and folds captured stdout/stderr lines into a
single ``text/plain`` attachment on each terminal packet.
"""
import sys
from subunit import GoJSON2SubUnit
def main():
sys.exit(GoJSON2SubUnit(sys.stdin, sys.stdout.buffer))
if __name__ == "__main__":
main()
+100
python/subunit/filter_scripts/junitxml2subunit.py
#!/usr/bin/env python3
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2026 Jelmer Vernooij <jelmer@samba.org>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""A filter that reads JUnit XML test reports and emits a subunit v2 stream.
JUnit XML is the de-facto interchange format for JVM test runners (Maven
Surefire, Gradle, Ant) and many other ecosystems. Maven and Gradle write
one XML file per test class into a reports directory, so this script
accepts directories as well as individual files.
Typical use with Maven::
mvn clean test ; junitxml2subunit -d target/surefire-reports
Typical use with Gradle::
gradle clean test ; junitxml2subunit -d build/test-results/test
"""
import argparse
import os
import sys
from subunit import JUnitXML2SubUnit
def parse_args(argv):
parser = argparse.ArgumentParser(
description=(
"Convert JUnit XML test reports to a subunit v2 stream on stdout. "
"Pass individual files as positional arguments or use -d/--dir to "
"walk a reports directory for *.xml files."
),
)
parser.add_argument(
"-d",
"--dir",
dest="dirs",
action="append",
default=[],
metavar="DIR",
help=(
"Directory to walk for *.xml report files. May be repeated. "
"Files inside the directory are converted in lexical order so "
"the output is deterministic across runs."
),
)
parser.add_argument(
"files",
nargs="*",
help="Individual JUnit XML report files to convert.",
)
return parser.parse_args(argv)
def collect_files(dirs, files):
"""Combine `--dir DIR` walks with explicit FILE arguments.
Within each directory we sort by filename so the resulting subunit
stream is reproducible. Across directories we preserve the user's
argv order (some workflows feed multiple module-specific report
directories and care about the suite ordering).
"""
out = []
for d in dirs:
if not os.path.isdir(d):
sys.stderr.write("junitxml2subunit: not a directory: {}\n".format(d))
continue
for root, _dirs, names in sorted(os.walk(d)):
for name in sorted(names):
if name.endswith(".xml"):
out.append(os.path.join(root, name))
out.extend(files)
return out
def main(argv=None):
args = parse_args(argv if argv is not None else sys.argv[1:])
inputs = collect_files(args.dirs, args.files)
if not inputs:
sys.stderr.write("junitxml2subunit: no input files found (pass FILE arguments or use -d DIR)\n")
return 2
return JUnitXML2SubUnit(inputs, sys.stdout.buffer)
if __name__ == "__main__":
sys.exit(main())
+337
python/subunit/filter_scripts/subunit_combine.py
#!/usr/bin/env python3
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2026 Jelmer Vernooij <jelmer@jelmer.uk>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Run multiple commands producing subunit v2 output and merge their streams.
The commands to run are described in a YAML configuration file. Each command
may have a test id ``prefix`` prepended to all test ids it emits, which makes
it possible to combine output from, for example, a Python and a Rust test
suite without test id collisions.
Example configuration::
commands:
- prefix: "python/"
argv: ["python", "-m", "subunit.run", "discover", ".", "$LISTOPT", "$IDOPTION"]
list_option: "--list"
id_option: "--load-list $IDFILE"
- prefix: "rust/"
argv: ["cargo", "test", "--", "--format=subunit"]
cwd: "rust"
The combined subunit v2 stream is written to stdout. The exit code is 0 if
all commands exited 0, 1 otherwise.
testr-style substitutions
-------------------------
Each command's ``argv``, ``list_option`` and ``id_option`` entries may contain
the following placeholders, which are expanded per invocation:
* ``$LISTOPT`` -- expands to ``list_option`` when ``subunit-combine --list``
is used, empty otherwise.
* ``$IDLIST`` -- space-separated list of (prefix-stripped) test ids to run
for this command. Empty if no ids were requested or none match.
* ``$IDFILE`` -- path to a temporary file containing one test id per line.
* ``$IDOPTION`` -- expands to ``id_option`` (with ``$IDFILE`` further
substituted) when ids are being supplied, empty otherwise.
Test ids can be passed as positional arguments after the config file or via
``--load-list FILE``. Ids that start with a command's ``prefix`` are routed
(with the prefix stripped) to that command; ids that don't match any prefix
are ignored for that command.
"""
import os
import re
import subprocess
import sys
import tempfile
from argparse import ArgumentParser
from typing import Optional
import yaml
from subunit import ByteStreamToStreamResult, StreamResultToBytes
_VARIABLE_RE = re.compile(r"\$(IDOPTION|IDFILE|IDLIST|LISTOPT)")
class _PrefixingStreamResult:
"""Forward StreamResult events, prepending ``prefix`` to every test_id."""
def __init__(self, target, prefix: str):
self._target = target
self._prefix = prefix
def startTestRun(self):
self._target.startTestRun()
def stopTestRun(self):
self._target.stopTestRun()
def status(self, test_id=None, **kwargs):
if test_id is not None:
test_id = self._prefix + test_id
self._target.status(test_id=test_id, **kwargs)
def load_config(path: str) -> list[dict]:
"""Load and validate a combine configuration file.
Returns a list of command dictionaries.
"""
with open(path) as f:
data = yaml.safe_load(f)
if not isinstance(data, dict):
raise ValueError(f"{path}: top-level configuration must be a mapping")
commands = data.get("commands")
if not isinstance(commands, list) or not commands:
raise ValueError(f"{path}: 'commands' must be a non-empty list")
for i, cmd in enumerate(commands):
if not isinstance(cmd, dict):
raise ValueError(f"{path}: commands[{i}] must be a mapping")
argv = cmd.get("argv")
if not isinstance(argv, list) or not argv or not all(isinstance(a, str) for a in argv):
raise ValueError(f"{path}: commands[{i}].argv must be a non-empty list of strings")
prefix = cmd.get("prefix", "")
if not isinstance(prefix, str):
raise ValueError(f"{path}: commands[{i}].prefix must be a string")
for key in ("list_option", "id_option"):
value = cmd.get(key)
if value is not None and not isinstance(value, str):
raise ValueError(f"{path}: commands[{i}].{key} must be a string")
return commands
def _substitute(template: str, variables: dict[str, str]) -> list[str]:
"""Substitute $VAR placeholders in ``template`` and return a shell-split list.
Empty values are expanded to the empty string; the resulting string is then
split on whitespace so that e.g. an empty ``$LISTOPT`` disappears rather
than leaving an empty argument behind.
"""
def repl(match: re.Match) -> str:
return variables.get(match.group(1), "")
return re.sub(_VARIABLE_RE, repl, template).split()
def _expand_argv(
cmd: dict,
*,
list_mode: bool,
test_ids: Optional[list[str]],
idfile_path: Optional[str],
) -> list[str]:
"""Expand testr-style placeholders in ``cmd['argv']``."""
list_option = cmd.get("list_option", "") if list_mode else ""
id_option_template = cmd.get("id_option", "")
if test_ids is None or not id_option_template:
id_option = ""
else:
id_option = re.sub(
_VARIABLE_RE,
lambda m: {"IDFILE": idfile_path or "", "IDLIST": " ".join(test_ids)}.get(m.group(1), ""),
id_option_template,
)
variables = {
"LISTOPT": list_option,
"IDOPTION": id_option,
"IDFILE": idfile_path or "",
"IDLIST": " ".join(test_ids) if test_ids else "",
}
expanded: list[str] = []
for piece in cmd["argv"]:
if _VARIABLE_RE.search(piece):
expanded.extend(_substitute(piece, variables))
else:
expanded.append(piece)
return expanded
def _select_ids_for_command(cmd: dict, test_ids: Optional[list[str]]) -> Optional[list[str]]:
"""Return the ids that belong to ``cmd`` (with the prefix stripped).
Returns None when no filtering should be applied (no ids were requested
globally).
"""
if test_ids is None:
return None
prefix = cmd.get("prefix", "")
if not prefix:
return list(test_ids)
return [tid[len(prefix) :] for tid in test_ids if tid.startswith(prefix)]
def _write_idfile(test_ids: list[str]) -> str:
fd, path = tempfile.mkstemp(prefix="subunit-combine-", suffix=".list")
with os.fdopen(fd, "w") as f:
for tid in test_ids:
f.write(tid + "\n")
return path
def run_command(
cmd: dict,
output,
*,
list_mode: bool = False,
test_ids: Optional[list[str]] = None,
) -> int:
"""Run a single command and forward its subunit v2 output.
The child's stdout is parsed as subunit v2 and re-emitted to ``output``
(a :class:`StreamResultToBytes`) with each test_id prefixed with
``cmd['prefix']`` (if any).
:param list_mode: If True, ``$LISTOPT`` is substituted with the command's
``list_option`` so the child lists tests rather than running them.
:param test_ids: If not None, the list of (prefix-stripped) test ids to
supply to the child via ``$IDLIST`` / ``$IDOPTION`` / ``$IDFILE``.
:return: The exit code of the child process.
"""
prefix = cmd.get("prefix", "")
cwd = cmd.get("cwd")
env = os.environ.copy()
extra_env = cmd.get("env")
if extra_env:
env.update(extra_env)
idfile_path: Optional[str] = None
if test_ids is not None and test_ids:
idfile_path = _write_idfile(test_ids)
try:
argv = _expand_argv(
cmd,
list_mode=list_mode,
test_ids=test_ids,
idfile_path=idfile_path,
)
proc = subprocess.Popen(argv, stdout=subprocess.PIPE, cwd=cwd, env=env)
try:
assert proc.stdout is not None
result = _PrefixingStreamResult(output, prefix) if prefix else output
ByteStreamToStreamResult(proc.stdout, non_subunit_name="stdout").run(result)
finally:
returncode = proc.wait()
finally:
if idfile_path is not None:
try:
os.unlink(idfile_path)
except OSError:
pass
return returncode
def combine(
commands: list[dict],
output_stream,
*,
list_mode: bool = False,
test_ids: Optional[list[str]] = None,
) -> int:
"""Run ``commands`` and merge their subunit streams into ``output_stream``.
:param list_mode: Run each command in listing mode (``$LISTOPT`` expanded).
:param test_ids: Optional list of test ids to restrict execution to.
Each command only sees ids whose prefix matches; commands with no
matching ids are skipped entirely.
:return: 0 if every command exited 0, 1 otherwise.
"""
output = StreamResultToBytes(output_stream)
output.startTestRun()
failed = False
try:
for cmd in commands:
cmd_ids = _select_ids_for_command(cmd, test_ids)
if test_ids is not None and not cmd_ids:
# Ids were requested, but none match this command.
continue
rc = run_command(cmd, output, list_mode=list_mode, test_ids=cmd_ids)
if rc != 0:
failed = True
finally:
output.stopTestRun()
return 1 if failed else 0
def _read_id_list(path: str) -> list[str]:
"""Read test ids from a file, one per line; blank lines and # comments are skipped."""
ids = []
with open(path) as f:
for line in f:
line = line.split("#", 1)[0].strip()
if line:
ids.append(line)
return ids
def make_parser() -> ArgumentParser:
parser = ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else None)
parser.add_argument(
"config",
help="Path to a YAML configuration file describing the commands to run.",
)
parser.add_argument(
"test_ids",
nargs="*",
help="Optional test ids to restrict execution to. Ids whose prefix "
"matches a command are routed to that command.",
)
parser.add_argument(
"--list",
dest="list_mode",
action="store_true",
help="List tests that would be run instead of running them. Each "
"command is invoked with its configured list_option substituted for "
"$LISTOPT.",
)
parser.add_argument(
"--load-list",
dest="load_list",
metavar="FILE",
help="Read test ids (one per line) from FILE to restrict execution.",
)
return parser
def main(argv: Optional[list[str]] = None) -> None:
parser = make_parser()
options = parser.parse_args(argv)
commands = load_config(options.config)
test_ids: Optional[list[str]] = None
if options.load_list:
test_ids = _read_id_list(options.load_list)
if options.test_ids:
test_ids = (test_ids or []) + options.test_ids
sys.exit(
combine(
commands,
sys.stdout,
list_mode=options.list_mode,
test_ids=test_ids,
)
)
if __name__ == "__main__":
main()
+63
python/tests/__init__.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
from unittest import TestLoader
from testscenarios import generate_scenarios
# Before the test module imports to avoid circularity.
# For testing: different pythons have different str() implementations.
_remote_exception_repr = "testtools.testresult.real._StringException"
_remote_exception_repr_chunked = "34\r\n" + _remote_exception_repr + ": boo qux\n0\r\n"
_remote_exception_str = "Traceback (most recent call last):\ntesttools.testresult.real._StringException"
_remote_exception_str_chunked = "57\r\n" + _remote_exception_str + ": boo qux\n0\r\n"
from . import ( # noqa: E402
test_chunked,
test_details,
test_filter_to_disk,
test_filters,
test_output_filter,
test_progress_model,
test_run,
test_subunit_filter,
test_subunit_stats,
test_subunit_tags,
test_tap2subunit,
test_test_protocol,
test_test_protocol2,
test_test_results,
)
def test_suite():
loader = TestLoader()
result = loader.loadTestsFromModule(test_chunked)
result.addTest(loader.loadTestsFromModule(test_details))
result.addTest(loader.loadTestsFromModule(test_filters))
result.addTest(loader.loadTestsFromModule(test_progress_model))
result.addTest(loader.loadTestsFromModule(test_test_results))
result.addTest(loader.loadTestsFromModule(test_test_protocol))
result.addTest(loader.loadTestsFromModule(test_test_protocol2))
result.addTest(loader.loadTestsFromModule(test_tap2subunit))
result.addTest(loader.loadTestsFromModule(test_filter_to_disk))
result.addTest(loader.loadTestsFromModule(test_subunit_filter))
result.addTest(loader.loadTestsFromModule(test_subunit_tags))
result.addTest(loader.loadTestsFromModule(test_subunit_stats))
result.addTest(loader.loadTestsFromModule(test_run))
result.addTests(generate_scenarios(loader.loadTestsFromModule(test_output_filter)))
return result
+24
python/tests/sample-script.py
#!/usr/bin/env python3
import sys
if sys.platform == "win32":
import msvcrt
import os
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
if len(sys.argv) == 2:
# tests.test_test_protocol.TestExecTestCase.test_sample_method_args
# uses this code path to be sure that the arguments were passed to
# sample-script.py
print("test fail")
print("error fail")
sys.exit(0)
print("test old mcdonald")
print("success old mcdonald")
print("test bing crosby")
print("failure bing crosby [")
print("foo.c:53:ERROR invalid state")
print("]")
print("test an error")
print("error an error")
sys.exit(0)
+8
python/tests/sample-two-script.py
#!/usr/bin/env python3
import sys
print("test old mcdonald")
print("success old mcdonald")
print("test bing crosby")
print("success bing crosby")
sys.exit(0)
+140
python/tests/test_chunked.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
# Copyright (C) 2011 Martin Pool <mbp@sourcefrog.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import unittest
from io import BytesIO
import subunit.chunked
class TestDecode(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.output = BytesIO()
self.decoder = subunit.chunked.Decoder(self.output)
def test_close_read_length_short_errors(self):
self.assertRaises(ValueError, self.decoder.close)
def test_close_body_short_errors(self):
self.assertEqual(None, self.decoder.write(b"2\r\na"))
self.assertRaises(ValueError, self.decoder.close)
def test_close_body_buffered_data_errors(self):
self.assertEqual(None, self.decoder.write(b"2\r"))
self.assertRaises(ValueError, self.decoder.close)
def test_close_after_finished_stream_safe(self):
self.assertEqual(None, self.decoder.write(b"2\r\nab"))
self.assertEqual(b"", self.decoder.write(b"0\r\n"))
self.decoder.close()
def test_decode_nothing(self):
self.assertEqual(b"", self.decoder.write(b"0\r\n"))
self.assertEqual(b"", self.output.getvalue())
def test_decode_serialised_form(self):
self.assertEqual(None, self.decoder.write(b"F\r\n"))
self.assertEqual(None, self.decoder.write(b"serialised\n"))
self.assertEqual(b"", self.decoder.write(b"form0\r\n"))
def test_decode_short(self):
self.assertEqual(b"", self.decoder.write(b"3\r\nabc0\r\n"))
self.assertEqual(b"abc", self.output.getvalue())
def test_decode_combines_short(self):
self.assertEqual(b"", self.decoder.write(b"6\r\nabcdef0\r\n"))
self.assertEqual(b"abcdef", self.output.getvalue())
def test_decode_excess_bytes_from_write(self):
self.assertEqual(b"1234", self.decoder.write(b"3\r\nabc0\r\n1234"))
self.assertEqual(b"abc", self.output.getvalue())
def test_decode_write_after_finished_errors(self):
self.assertEqual(b"1234", self.decoder.write(b"3\r\nabc0\r\n1234"))
self.assertRaises(ValueError, self.decoder.write, b"")
def test_decode_hex(self):
self.assertEqual(b"", self.decoder.write(b"A\r\n12345678900\r\n"))
self.assertEqual(b"1234567890", self.output.getvalue())
def test_decode_long_ranges(self):
self.assertEqual(None, self.decoder.write(b"10000\r\n"))
self.assertEqual(None, self.decoder.write(b"1" * 65536))
self.assertEqual(None, self.decoder.write(b"10000\r\n"))
self.assertEqual(None, self.decoder.write(b"2" * 65536))
self.assertEqual(b"", self.decoder.write(b"0\r\n"))
self.assertEqual(b"1" * 65536 + b"2" * 65536, self.output.getvalue())
def test_decode_newline_nonstrict(self):
"""Tolerate chunk markers with no CR character."""
# From <http://pad.lv/505078>
self.decoder = subunit.chunked.Decoder(self.output, strict=False)
self.assertEqual(None, self.decoder.write(b"a\n"))
self.assertEqual(None, self.decoder.write(b"abcdeabcde"))
self.assertEqual(b"", self.decoder.write(b"0\n"))
self.assertEqual(b"abcdeabcde", self.output.getvalue())
def test_decode_strict_newline_only(self):
"""Reject chunk markers with no CR character in strict mode."""
# From <http://pad.lv/505078>
self.assertRaises(ValueError, self.decoder.write, b"a\n")
def test_decode_strict_multiple_crs(self):
self.assertRaises(ValueError, self.decoder.write, b"a\r\r\n")
def test_decode_short_header(self):
self.assertRaises(ValueError, self.decoder.write, b"\n")
class TestEncode(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.output = BytesIO()
self.encoder = subunit.chunked.Encoder(self.output)
def test_encode_nothing(self):
self.encoder.close()
self.assertEqual(b"0\r\n", self.output.getvalue())
def test_encode_empty(self):
self.encoder.write(b"")
self.encoder.close()
self.assertEqual(b"0\r\n", self.output.getvalue())
def test_encode_short(self):
self.encoder.write(b"abc")
self.encoder.close()
self.assertEqual(b"3\r\nabc0\r\n", self.output.getvalue())
def test_encode_combines_short(self):
self.encoder.write(b"abc")
self.encoder.write(b"def")
self.encoder.close()
self.assertEqual(b"6\r\nabcdef0\r\n", self.output.getvalue())
def test_encode_over_9_is_in_hex(self):
self.encoder.write(b"1234567890")
self.encoder.close()
self.assertEqual(b"A\r\n12345678900\r\n", self.output.getvalue())
def test_encode_long_ranges_not_combined(self):
self.encoder.write(b"1" * 65536)
self.encoder.write(b"2" * 65536)
self.encoder.close()
self.assertEqual(b"10000\r\n" + b"1" * 65536 + b"10000\r\n" + b"2" * 65536 + b"0\r\n", self.output.getvalue())
+90
python/tests/test_details.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import unittest
from subunit import content, content_type, details
class TestSimpleDetails(unittest.TestCase):
def test_lineReceived(self):
parser = details.SimpleDetailsParser(None)
parser.lineReceived(b"foo\n")
parser.lineReceived(b"bar\n")
self.assertEqual(b"foo\nbar\n", parser._message)
def test_lineReceived_escaped_bracket(self):
parser = details.SimpleDetailsParser(None)
parser.lineReceived(b"foo\n")
parser.lineReceived(b" ]are\n")
parser.lineReceived(b"bar\n")
self.assertEqual(b"foo\n]are\nbar\n", parser._message)
def test_get_message(self):
parser = details.SimpleDetailsParser(None)
self.assertEqual(b"", parser.get_message())
def test_get_details(self):
parser = details.SimpleDetailsParser(None)
expected = {}
expected["traceback"] = content.Content(
content_type.ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b""]
)
found = parser.get_details()
self.assertEqual(expected.keys(), found.keys())
self.assertEqual(expected["traceback"].content_type, found["traceback"].content_type)
self.assertEqual(b"".join(expected["traceback"].iter_bytes()), b"".join(found["traceback"].iter_bytes()))
def test_get_details_skip(self):
parser = details.SimpleDetailsParser(None)
expected = {}
expected["reason"] = content.Content(content_type.ContentType("text", "plain"), lambda: [b""])
found = parser.get_details("skip")
self.assertEqual(expected, found)
def test_get_details_success(self):
parser = details.SimpleDetailsParser(None)
expected = {}
expected["message"] = content.Content(content_type.ContentType("text", "plain"), lambda: [b""])
found = parser.get_details("success")
self.assertEqual(expected, found)
class TestMultipartDetails(unittest.TestCase):
def test_get_message_is_None(self):
parser = details.MultipartDetailsParser(None)
self.assertEqual(None, parser.get_message())
def test_get_details(self):
parser = details.MultipartDetailsParser(None)
self.assertEqual({}, parser.get_details())
def test_parts(self):
parser = details.MultipartDetailsParser(None)
parser.lineReceived(b"Content-Type: text/plain\n")
parser.lineReceived(b"something\n")
parser.lineReceived(b"F\r\n")
parser.lineReceived(b"serialised\n")
parser.lineReceived(b"form0\r\n")
expected = {}
expected["something"] = content.Content(
content_type.ContentType("text", "plain"), lambda: [b"serialised\nform"]
)
found = parser.get_details()
self.assertEqual(expected.keys(), found.keys())
self.assertEqual(expected["something"].content_type, found["something"].content_type)
self.assertEqual(b"".join(expected["something"].iter_bytes()), b"".join(found["something"].iter_bytes()))
+46
python/tests/test_filter_to_disk.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2013 Subunit Contributors
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
import io
import os.path
from fixtures import TempDir
from testtools import TestCase
from testtools.matchers import FileContains
from subunit import _to_disk
from subunit.v2 import StreamResultToBytes
class SmokeTest(TestCase):
def test_smoke(self):
output = os.path.join(self.useFixture(TempDir()).path, "output")
stdin = io.BytesIO()
stdout = io.StringIO()
writer = StreamResultToBytes(stdin)
writer.startTestRun()
writer.status(
"foo", "success", {"tag"}, file_name="fred", file_bytes=b"abcdefg", eof=True, mime_type="text/plain"
)
writer.stopTestRun()
stdin.seek(0)
_to_disk.to_disk(["-d", output], stdin=stdin, stdout=stdout)
self.expectThat(
os.path.join(output, "foo/test.json"),
FileContains(
'{"details": ["fred"], "id": "foo", "start": null, "status": "success", "stop": null, "tags": ["tag"]}'
),
)
self.expectThat(os.path.join(output, "foo/fred"), FileContains("abcdefg"))
+42
python/tests/test_filters.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2013 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
from tempfile import NamedTemporaryFile
from testtools import TestCase
from subunit import read_test_list
from subunit.filters import find_stream
class TestReadTestList(TestCase):
def test_read_list(self):
with NamedTemporaryFile() as f:
f.write(b"foo\nbar\n# comment\nother # comment\n")
f.flush()
self.assertEqual(read_test_list(f.name), ["foo", "bar", "other"])
class TestFindStream(TestCase):
def test_no_argv(self):
self.assertEqual("foo", find_stream("foo", []))
def test_opens_file(self):
f = NamedTemporaryFile()
f.write(b"foo")
f.flush()
stream = find_stream("bar", [f.name])
self.assertEqual(b"foo", stream.read())
+229
python/tests/test_gojson2subunit.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2026 Jelmer Vernooij <jelmer@samba.org>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Tests for GoJSON2SubUnit."""
import json
from io import BytesIO, StringIO
from testtools import TestCase
from testtools.testresult.doubles import StreamResult
import subunit
UTF8_TEXT = "text/plain; charset=UTF8"
def _ndjson(*events):
"""Render an iterable of dicts as one JSON object per line."""
return "\n".join(json.dumps(e) for e in events) + "\n"
class TestGoJSON2SubUnit(TestCase):
"""Behavioural tests for `GoJSON2SubUnit`.
Each test feeds a synthetic `go test -json` event stream in, decodes
the resulting subunit bytes back into events via `StreamResult`, and
asserts on the (status, test_id, test_status, ...) tuples.
"""
def setUp(self):
super().setUp()
self.gojson = StringIO()
self.subunit = BytesIO()
def _events(self):
self.subunit.seek(0)
sink = StreamResult()
subunit.ByteStreamToStreamResult(self.subunit).run(sink)
return sink._events
def _statuses(self, events):
# Strip the verbose tuple down to (test_id, test_status) for clearer
# assertions when timing/file content isn't what's being tested.
return [(e[1], e[2]) for e in events if e[0] == "status"]
def test_pass_emits_inprogress_then_success(self):
self.gojson.write(
_ndjson(
{"Action": "run", "Package": "pkg/a", "Test": "TestFoo"},
{"Action": "pass", "Package": "pkg/a", "Test": "TestFoo", "Elapsed": 0.01},
)
)
self.gojson.seek(0)
rc = subunit.GoJSON2SubUnit(self.gojson, self.subunit)
self.assertEqual(0, rc)
self.assertEqual(
[("pkg/a.TestFoo", "inprogress"), ("pkg/a.TestFoo", "success")],
self._statuses(self._events()),
)
def test_fail_returns_nonzero(self):
self.gojson.write(
_ndjson(
{"Action": "run", "Package": "pkg/a", "Test": "TestBad"},
{"Action": "run", "Package": "pkg/a", "Test": "TestGood"},
{"Action": "pass", "Package": "pkg/a", "Test": "TestGood", "Elapsed": 0.01},
{"Action": "fail", "Package": "pkg/a", "Test": "TestBad", "Elapsed": 0.02},
)
)
self.gojson.seek(0)
rc = subunit.GoJSON2SubUnit(self.gojson, self.subunit)
self.assertEqual(1, rc)
# Each test produces an inprogress packet and a terminal packet;
# check the terminal status (last one wins in dict()).
statuses = dict(self._statuses(self._events()))
self.assertEqual("fail", statuses["pkg/a.TestBad"])
self.assertEqual("success", statuses["pkg/a.TestGood"])
def test_skip_status(self):
self.gojson.write(
_ndjson(
{"Action": "run", "Package": "pkg/a", "Test": "TestSkipped"},
{"Action": "skip", "Package": "pkg/a", "Test": "TestSkipped", "Elapsed": 0.0},
)
)
self.gojson.seek(0)
rc = subunit.GoJSON2SubUnit(self.gojson, self.subunit)
self.assertEqual(0, rc)
self.assertIn(("pkg/a.TestSkipped", "skip"), self._statuses(self._events()))
def test_subtest_keeps_slash_separator(self):
# Go subtests are reported as "Parent/Sub"; the resulting test ID
# should be "<package>.Parent/Sub" so it round-trips through
# `go test -run '^Parent$/^Sub$'`.
self.gojson.write(
_ndjson(
{"Action": "run", "Package": "pkg/a", "Test": "TestParent/sub_one"},
{"Action": "pass", "Package": "pkg/a", "Test": "TestParent/sub_one", "Elapsed": 0.01},
)
)
self.gojson.seek(0)
subunit.GoJSON2SubUnit(self.gojson, self.subunit)
self.assertIn(
("pkg/a.TestParent/sub_one", "success"),
self._statuses(self._events()),
)
def test_output_attached_to_terminal_packet(self):
self.gojson.write(
_ndjson(
{"Action": "run", "Package": "pkg/a", "Test": "TestNoisy"},
{"Action": "output", "Package": "pkg/a", "Test": "TestNoisy", "Output": "hello\n"},
{"Action": "output", "Package": "pkg/a", "Test": "TestNoisy", "Output": "world\n"},
{"Action": "fail", "Package": "pkg/a", "Test": "TestNoisy", "Elapsed": 0.01},
)
)
self.gojson.seek(0)
subunit.GoJSON2SubUnit(self.gojson, self.subunit)
events = self._events()
# Find the terminal packet for TestNoisy and confirm both lines
# were folded into one attachment.
terminal = [e for e in events if e[0] == "status" and e[1] == "pkg/a.TestNoisy" and e[2] == "fail"]
self.assertEqual(1, len(terminal))
# Tuple shape from StreamResult:
# ("status", test_id, test_status, test_tags, runnable, file_name,
# file_bytes, eof, mime_type, route_code, timestamp)
ev = terminal[0]
self.assertEqual("go test output", ev[5])
self.assertEqual(b"hello\nworld\n", ev[6])
self.assertEqual(UTF8_TEXT, ev[8])
def test_package_level_build_failure_synthesises_test(self):
# When `go test` can't build a package it emits a `fail` event with
# no `Test` field, preceded by `output` events scoped to the package.
self.gojson.write(
_ndjson(
{"Action": "output", "Package": "pkg/broken", "Output": "./x.go:1:1: syntax error\n"},
{"Action": "fail", "Package": "pkg/broken", "Elapsed": 0.0},
)
)
self.gojson.seek(0)
rc = subunit.GoJSON2SubUnit(self.gojson, self.subunit)
self.assertEqual(1, rc)
events = self._events()
terminal = [e for e in events if e[0] == "status" and e[1] == "pkg/broken [build]" and e[2] == "fail"]
self.assertEqual(1, len(terminal))
self.assertEqual(b"./x.go:1:1: syntax error\n", terminal[0][6])
def test_garbage_lines_are_skipped(self):
# `go test -json` occasionally interleaves a non-JSON banner on
# certain failure paths; a junk line shouldn't abort the stream.
self.gojson.write("not json at all\n")
self.gojson.write(
_ndjson(
{"Action": "run", "Package": "pkg/a", "Test": "TestFoo"},
{"Action": "pass", "Package": "pkg/a", "Test": "TestFoo", "Elapsed": 0.0},
)
)
self.gojson.seek(0)
rc = subunit.GoJSON2SubUnit(self.gojson, self.subunit)
self.assertEqual(0, rc)
self.assertIn(("pkg/a.TestFoo", "success"), self._statuses(self._events()))
def test_blank_lines_are_skipped(self):
self.gojson.write("\n\n")
self.gojson.write(_ndjson({"Action": "run", "Package": "pkg/a", "Test": "TestFoo"}))
self.gojson.write("\n")
self.gojson.write(_ndjson({"Action": "pass", "Package": "pkg/a", "Test": "TestFoo", "Elapsed": 0.0}))
self.gojson.seek(0)
rc = subunit.GoJSON2SubUnit(self.gojson, self.subunit)
self.assertEqual(0, rc)
self.assertIn(("pkg/a.TestFoo", "success"), self._statuses(self._events()))
def test_unfinished_test_at_eof_is_failed(self):
# A test that started but never reached a terminal action — the
# runner died mid-test. Surface as a failure rather than dropping it.
self.gojson.write(
_ndjson(
{"Action": "run", "Package": "pkg/a", "Test": "TestStuck"},
{"Action": "output", "Package": "pkg/a", "Test": "TestStuck", "Output": "panic\n"},
)
)
self.gojson.seek(0)
rc = subunit.GoJSON2SubUnit(self.gojson, self.subunit)
self.assertEqual(1, rc)
events = self._events()
terminal = [e for e in events if e[0] == "status" and e[1] == "pkg/a.TestStuck" and e[2] == "fail"]
self.assertEqual(1, len(terminal))
self.assertEqual(b"panic\n", terminal[0][6])
def test_timestamp_is_propagated(self):
# The `Time` on each event should land on the matching subunit packet.
self.gojson.write(
_ndjson(
{
"Time": "2026-01-02T03:04:05.000000Z",
"Action": "run",
"Package": "pkg/a",
"Test": "TestTimed",
},
{
"Time": "2026-01-02T03:04:06.000000Z",
"Action": "pass",
"Package": "pkg/a",
"Test": "TestTimed",
"Elapsed": 1.0,
},
)
)
self.gojson.seek(0)
subunit.GoJSON2SubUnit(self.gojson, self.subunit)
# The terminal packet's timestamp (last tuple element) must be set,
# which gives consumers a basis for computing duration.
terminal = [e for e in self._events() if e[0] == "status" and e[1] == "pkg/a.TestTimed" and e[2] == "success"]
self.assertEqual(1, len(terminal))
self.assertIsNotNone(terminal[0][-1])
+312
python/tests/test_junitxml2subunit.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2026 Jelmer Vernooij <jelmer@samba.org>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Tests for JUnitXML2SubUnit."""
import io
import os
import tempfile
from unittest import mock
from testtools import TestCase
from testtools.testresult.doubles import StreamResult
import subunit
from subunit.filter_scripts import junitxml2subunit
def _write(tmpdir, name, content):
path = os.path.join(tmpdir, name)
with open(path, "w", encoding="utf-8") as fh:
fh.write(content)
return path
class TestJUnitXML2SubUnit(TestCase):
"""Behavioural tests for `JUnitXML2SubUnit`.
Each test writes a synthetic JUnit XML doc to disk, runs the converter,
decodes the resulting subunit bytes back into events via
`StreamResult`, and asserts on the (test_id, test_status) tuples.
"""
def setUp(self):
super().setUp()
self.tmp = tempfile.mkdtemp(prefix="junitxml2subunit-test-")
self.addCleanup(self._rmtree, self.tmp)
self.subunit = io.BytesIO()
def _rmtree(self, path):
import shutil
shutil.rmtree(path, ignore_errors=True)
def _events(self):
self.subunit.seek(0)
sink = StreamResult()
subunit.ByteStreamToStreamResult(self.subunit).run(sink)
return sink._events
def _statuses(self, events):
return [(e[1], e[2]) for e in events if e[0] == "status"]
def test_passing_testcase(self):
path = _write(
self.tmp,
"TEST-FooTest.xml",
"""<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="com.example.FooTest" tests="1" failures="0" errors="0" skipped="0" time="0.123">
<testcase classname="com.example.FooTest" name="testBar" time="0.05"/>
</testsuite>
""",
)
rc = subunit.JUnitXML2SubUnit([path], self.subunit)
self.assertEqual(0, rc)
self.assertEqual(
[
("com.example.FooTest::testBar", "inprogress"),
("com.example.FooTest::testBar", "success"),
],
self._statuses(self._events()),
)
def test_failure_marks_fail_and_returns_nonzero(self):
path = _write(
self.tmp,
"TEST-FooTest.xml",
"""<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="com.example.FooTest" tests="1" failures="1">
<testcase classname="com.example.FooTest" name="testBar" time="0.01">
<failure type="java.lang.AssertionError" message="expected x but was y">at line 42</failure>
</testcase>
</testsuite>
""",
)
rc = subunit.JUnitXML2SubUnit([path], self.subunit)
self.assertEqual(1, rc)
events = self._events()
terminal = [e for e in events if e[0] == "status" and e[1] == "com.example.FooTest::testBar" and e[2] == "fail"]
self.assertEqual(1, len(terminal))
# The failure detail (type + message + body) is folded into a
# single attachment so consumers see everything in one place.
ev = terminal[0]
self.assertEqual("junit detail", ev[5])
# The decoder returns the attachment as a memoryview; coerce to
# bytes for substring searching.
body = bytes(ev[6])
self.assertIn(b"java.lang.AssertionError", body)
self.assertIn(b"expected x but was y", body)
self.assertIn(b"at line 42", body)
def test_error_is_treated_as_fail(self):
# An <error> in JUnit terms is "an unexpected exception". subunit
# has no separate "error" status that maps cleanly, and from a
# consumer's perspective both mean "did not pass" — so it lands
# on `fail` like a failure does.
path = _write(
self.tmp,
"TEST-FooTest.xml",
"""<?xml version="1.0"?>
<testsuite>
<testcase classname="com.example.FooTest" name="testBar">
<error type="java.lang.NullPointerException">stack trace</error>
</testcase>
</testsuite>
""",
)
rc = subunit.JUnitXML2SubUnit([path], self.subunit)
self.assertEqual(1, rc)
statuses = self._statuses(self._events())
self.assertIn(("com.example.FooTest::testBar", "fail"), statuses)
def test_skipped(self):
path = _write(
self.tmp,
"TEST-FooTest.xml",
"""<?xml version="1.0"?>
<testsuite>
<testcase classname="com.example.FooTest" name="testBar">
<skipped message="ignored for now"/>
</testcase>
</testsuite>
""",
)
rc = subunit.JUnitXML2SubUnit([path], self.subunit)
self.assertEqual(0, rc)
statuses = self._statuses(self._events())
self.assertIn(("com.example.FooTest::testBar", "skip"), statuses)
def test_testsuites_wrapper_is_unwrapped(self):
# Some emitters (Gradle, Ant) wrap multiple <testsuite> in a
# <testsuites> root; both shapes need to work.
path = _write(
self.tmp,
"report.xml",
"""<?xml version="1.0"?>
<testsuites>
<testsuite name="A">
<testcase classname="A" name="testOne"/>
</testsuite>
<testsuite name="B">
<testcase classname="B" name="testTwo"/>
</testsuite>
</testsuites>
""",
)
rc = subunit.JUnitXML2SubUnit([path], self.subunit)
self.assertEqual(0, rc)
statuses = self._statuses(self._events())
self.assertIn(("A::testOne", "success"), statuses)
self.assertIn(("B::testTwo", "success"), statuses)
def test_multiple_files_concatenate(self):
a = _write(
self.tmp,
"TEST-A.xml",
"""<?xml version="1.0"?>
<testsuite>
<testcase classname="A" name="testOne"/>
</testsuite>
""",
)
b = _write(
self.tmp,
"TEST-B.xml",
"""<?xml version="1.0"?>
<testsuite>
<testcase classname="B" name="testTwo"/>
</testsuite>
""",
)
rc = subunit.JUnitXML2SubUnit([a, b], self.subunit)
self.assertEqual(0, rc)
statuses = self._statuses(self._events())
self.assertIn(("A::testOne", "success"), statuses)
self.assertIn(("B::testTwo", "success"), statuses)
def test_missing_classname_falls_back_to_name(self):
# `classname` is technically optional; without it the test ID
# is just the bare method name rather than emitting "::name".
path = _write(
self.tmp,
"report.xml",
"""<?xml version="1.0"?>
<testsuite>
<testcase name="bareName"/>
</testsuite>
""",
)
rc = subunit.JUnitXML2SubUnit([path], self.subunit)
self.assertEqual(0, rc)
statuses = self._statuses(self._events())
self.assertIn(("bareName", "success"), statuses)
def test_unparseable_xml_counts_as_failure(self):
# A broken file is loud (stderr warning + non-zero exit) rather
# than silently dropping the suite — broken XML in a CI report
# almost always means a test runner crash.
path = _write(self.tmp, "broken.xml", "<<not really xml>>")
with mock.patch("sys.stderr", new=io.StringIO()) as stderr:
rc = subunit.JUnitXML2SubUnit([path], self.subunit)
self.assertEqual(1, rc)
self.assertIn("failed to parse", stderr.getvalue())
def test_time_attribute_advances_synthetic_clock(self):
# Each testcase's `time` attribute should determine the gap
# between its inprogress and terminal packets, so the consumer
# can recover the duration. Use distinct values per test to
# confirm both make it through.
path = _write(
self.tmp,
"report.xml",
"""<?xml version="1.0"?>
<testsuite>
<testcase classname="A" name="testOne" time="0.5"/>
<testcase classname="A" name="testTwo" time="1.5"/>
</testsuite>
""",
)
subunit.JUnitXML2SubUnit([path], self.subunit)
events = self._events()
# Pull the per-test (inprogress, terminal) timestamp pairs. The
# ByteStreamToStreamResult emits a "time" event before each
# status event when the packet carries a timestamp.
# Easier: walk the events and pair them up by test_id.
timestamps = {}
for e in events:
if e[0] != "status":
continue
test_id = e[1]
ts = e[-1]
if ts is None:
continue
timestamps.setdefault(test_id, []).append(ts)
one = timestamps["A::testOne"]
two = timestamps["A::testTwo"]
self.assertEqual(2, len(one))
self.assertEqual(2, len(two))
# testOne spans 0.5s
self.assertAlmostEqual(0.5, (one[1] - one[0]).total_seconds(), places=3)
# testTwo spans 1.5s
self.assertAlmostEqual(1.5, (two[1] - two[0]).total_seconds(), places=3)
class TestCollectFiles(TestCase):
"""Tests for the `--dir` walking logic in the script entrypoint."""
def setUp(self):
super().setUp()
self.tmp = tempfile.mkdtemp(prefix="junitxml2subunit-collect-")
self.addCleanup(self._rmtree, self.tmp)
def _rmtree(self, path):
import shutil
shutil.rmtree(path, ignore_errors=True)
def test_dir_walks_xml_files_only(self):
a = _write(self.tmp, "TEST-A.xml", "")
_write(self.tmp, "ignored.txt", "")
b = _write(self.tmp, "TEST-B.xml", "")
out = junitxml2subunit.collect_files([self.tmp], [])
self.assertEqual({a, b}, set(out))
def test_dir_results_are_sorted_for_deterministic_output(self):
# Lexical sort within a directory so the subunit stream is
# reproducible across runs (and across filesystems with
# different readdir order).
b = _write(self.tmp, "TEST-B.xml", "")
a = _write(self.tmp, "TEST-A.xml", "")
out = junitxml2subunit.collect_files([self.tmp], [])
self.assertEqual([a, b], out)
def test_explicit_files_appended_after_dir_walks(self):
a = _write(self.tmp, "TEST-A.xml", "")
# Build an explicit file outside the walked dir to confirm it's
# appended after the discovered files rather than re-walked.
extra_dir = tempfile.mkdtemp(prefix="junitxml2subunit-extra-")
self.addCleanup(self._rmtree, extra_dir)
explicit = _write(extra_dir, "extra.xml", "")
out = junitxml2subunit.collect_files([self.tmp], [explicit])
self.assertEqual([a, explicit], out)
def test_missing_dir_warned_and_skipped(self):
with mock.patch("sys.stderr", new=io.StringIO()) as stderr:
out = junitxml2subunit.collect_files(["/nonexistent/junit/dir"], [])
self.assertEqual([], out)
self.assertIn("not a directory", stderr.getvalue())
+580
python/tests/test_output_filter.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2013 Subunit Contributors
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import datetime
import optparse
from contextlib import contextmanager
from functools import partial
from io import BytesIO, TextIOWrapper
from tempfile import NamedTemporaryFile
from iso8601 import UTC
from testtools import TestCase
from testtools.matchers import Equals, Matcher, MatchesAny, MatchesListwise, Mismatch, raises
from testtools.testresult.doubles import StreamResult
import subunit._output as _o
from subunit._output import _ALL_ACTIONS, _FINAL_ACTIONS, generate_stream_results, parse_arguments
class SafeOptionParser(optparse.OptionParser):
"""An ArgumentParser class that doesn't call sys.exit."""
def exit(self, status=0, message=""):
raise RuntimeError(message)
def error(self, message):
raise RuntimeError(message)
safe_parse_arguments = partial(parse_arguments, ParserClass=SafeOptionParser)
class TestStatusArgParserTests(TestCase):
scenarios = [(cmd, dict(command=cmd, option="--" + cmd)) for cmd in _ALL_ACTIONS]
def test_can_parse_all_commands_with_test_id(self):
test_id = self.getUniqueString()
args = safe_parse_arguments(args=[self.option, test_id])
self.assertThat(args.action, Equals(self.command))
self.assertThat(args.test_id, Equals(test_id))
def test_all_commands_parse_file_attachment(self):
with NamedTemporaryFile() as tmp_file:
args = safe_parse_arguments(args=[self.option, "foo", "--attach-file", tmp_file.name])
self.assertThat(args.attach_file.name, Equals(tmp_file.name))
def test_all_commands_accept_mimetype_argument(self):
with NamedTemporaryFile() as tmp_file:
args = safe_parse_arguments(
args=[self.option, "foo", "--attach-file", tmp_file.name, "--mimetype", "text/plain"]
)
self.assertThat(args.mimetype, Equals("text/plain"))
def test_all_commands_accept_file_name_argument(self):
with NamedTemporaryFile() as tmp_file:
args = safe_parse_arguments(args=[self.option, "foo", "--attach-file", tmp_file.name, "--file-name", "foo"])
self.assertThat(args.file_name, Equals("foo"))
def test_all_commands_accept_tags_argument(self):
args = safe_parse_arguments(args=[self.option, "foo", "--tag", "foo", "--tag", "bar", "--tag", "baz"])
self.assertThat(args.tags, Equals(["foo", "bar", "baz"]))
def test_attach_file_with_hyphen_opens_stdin(self):
self.patch(_o.sys, "stdin", TextIOWrapper(BytesIO(b"Hello")))
args = safe_parse_arguments(args=[self.option, "foo", "--attach-file", "-"])
self.assertThat(args.attach_file.read(), Equals(b"Hello"))
def test_attach_file_with_hyphen_sets_filename_to_stdin(self):
args = safe_parse_arguments(args=[self.option, "foo", "--attach-file", "-"])
self.assertThat(args.file_name, Equals("stdin"))
def test_can_override_stdin_filename(self):
args = safe_parse_arguments(args=[self.option, "foo", "--attach-file", "-", "--file-name", "foo"])
self.assertThat(args.file_name, Equals("foo"))
def test_requires_test_id(self):
def fn():
return safe_parse_arguments(args=[self.option])
self.assertThat(fn, raises(RuntimeError("argument %s: must specify a single TEST_ID." % self.option)))
class ArgParserTests(TestCase):
def test_can_parse_attach_file_without_test_id(self):
with NamedTemporaryFile() as tmp_file:
args = safe_parse_arguments(args=["--attach-file", tmp_file.name])
self.assertThat(args.attach_file.name, Equals(tmp_file.name))
def test_can_run_without_args(self):
safe_parse_arguments([])
def test_cannot_specify_more_than_one_status_command(self):
def fn():
return safe_parse_arguments(["--fail", "foo", "--skip", "bar"])
self.assertThat(fn, raises(RuntimeError("argument --skip: Only one status may be specified at once.")))
def test_cannot_specify_mimetype_without_attach_file(self):
def fn():
return safe_parse_arguments(["--mimetype", "foo"])
self.assertThat(fn, raises(RuntimeError("Cannot specify --mimetype without --attach-file")))
def test_cannot_specify_filename_without_attach_file(self):
def fn():
return safe_parse_arguments(["--file-name", "foo"])
self.assertThat(fn, raises(RuntimeError("Cannot specify --file-name without --attach-file")))
def test_can_specify_tags_without_status_command(self):
args = safe_parse_arguments(["--tag", "foo"])
self.assertEqual(["foo"], args.tags)
def test_must_specify_tags_with_tags_options(self):
def fn():
return safe_parse_arguments(["--fail", "foo", "--tag"])
self.assertThat(
fn,
MatchesAny(
raises(RuntimeError("--tag option requires 1 argument")),
raises(RuntimeError("--tag option requires an argument")),
),
)
def get_result_for(commands):
"""Get a result object from *commands.
Runs the 'generate_stream_results' function from subunit._output after
parsing *commands as if they were specified on the command line. The
resulting bytestream is then converted back into a result object and
returned.
"""
result = StreamResult()
args = safe_parse_arguments(commands)
generate_stream_results(args, result)
return result
@contextmanager
def temp_file_contents(data):
"""Create a temporary file on disk containing 'data'."""
with NamedTemporaryFile() as f:
f.write(data)
f.seek(0)
yield f
class StatusStreamResultTests(TestCase):
scenarios = [(s, dict(status=s, option="--" + s)) for s in _ALL_ACTIONS]
_dummy_timestamp = datetime.datetime(2013, 1, 1, 0, 0, 0, 0, UTC)
def setUp(self):
super().setUp()
self.patch(_o, "create_timestamp", lambda: self._dummy_timestamp)
self.test_id = self.getUniqueString()
def test_only_one_packet_is_generated(self):
result = get_result_for([self.option, self.test_id])
self.assertThat(
len(result._events),
Equals(3), # startTestRun and stopTestRun are also called, making 3 total.
)
def test_correct_status_is_generated(self):
result = get_result_for([self.option, self.test_id])
self.assertThat(result._events[1], MatchesStatusCall(test_status=self.status))
def test_all_commands_generate_tags(self):
result = get_result_for([self.option, self.test_id, "--tag", "hello", "--tag", "world"])
self.assertThat(result._events[1], MatchesStatusCall(test_tags={"hello", "world"}))
def test_all_commands_generate_timestamp(self):
result = get_result_for([self.option, self.test_id])
self.assertThat(result._events[1], MatchesStatusCall(timestamp=self._dummy_timestamp))
def test_all_commands_generate_correct_test_id(self):
result = get_result_for([self.option, self.test_id])
self.assertThat(result._events[1], MatchesStatusCall(test_id=self.test_id))
def test_file_is_sent_in_single_packet(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_can_read_binary_files(self):
with temp_file_contents(b"\xde\xad\xbe\xef") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_bytes=b"\xde\xad\xbe\xef", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_can_read_empty_files(self):
with temp_file_contents(b"") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_bytes=b"", file_name=f.name, eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_can_read_stdin(self):
self.patch(_o.sys, "stdin", TextIOWrapper(BytesIO(b"\xfe\xed\xfa\xce")))
result = get_result_for([self.option, self.test_id, "--attach-file", "-"])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_bytes=b"\xfe\xed\xfa\xce", file_name="stdin", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_is_sent_with_test_id(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_is_sent_with_test_status(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_status=self.status, file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_chunk_size_is_honored(self):
with temp_file_contents(b"Hello") as f:
self.patch(_o, "_CHUNK_SIZE", 1)
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"H", eof=False),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"e", eof=False),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"l", eof=False),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"l", eof=False),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"o", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_mimetype_specified_once_only(self):
with temp_file_contents(b"Hi") as f:
self.patch(_o, "_CHUNK_SIZE", 1)
result = get_result_for(
[
self.option,
self.test_id,
"--attach-file",
f.name,
"--mimetype",
"text/plain",
]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=self.test_id, mime_type="text/plain", file_bytes=b"H", eof=False),
MatchesStatusCall(test_id=self.test_id, mime_type=None, file_bytes=b"i", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_tags_specified_once_only(self):
with temp_file_contents(b"Hi") as f:
self.patch(_o, "_CHUNK_SIZE", 1)
result = get_result_for(
[
self.option,
self.test_id,
"--attach-file",
f.name,
"--tag",
"foo",
"--tag",
"bar",
]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=self.test_id, test_tags={"foo", "bar"}),
MatchesStatusCall(test_id=self.test_id, test_tags=None),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_timestamp_specified_once_only(self):
with temp_file_contents(b"Hi") as f:
self.patch(_o, "_CHUNK_SIZE", 1)
result = get_result_for(
[
self.option,
self.test_id,
"--attach-file",
f.name,
]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=self.test_id, timestamp=self._dummy_timestamp),
MatchesStatusCall(test_id=self.test_id, timestamp=None),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_test_status_specified_once_only(self):
with temp_file_contents(b"Hi") as f:
self.patch(_o, "_CHUNK_SIZE", 1)
result = get_result_for(
[
self.option,
self.test_id,
"--attach-file",
f.name,
]
)
# 'inprogress' status should be on the first packet only, all other
# statuses should be on the last packet.
if self.status in _FINAL_ACTIONS:
first_call = MatchesStatusCall(test_id=self.test_id, test_status=None)
last_call = MatchesStatusCall(test_id=self.test_id, test_status=self.status)
else:
first_call = MatchesStatusCall(test_id=self.test_id, test_status=self.status)
last_call = MatchesStatusCall(test_id=self.test_id, test_status=None)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
first_call,
last_call,
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_filename_can_be_overridden(self):
with temp_file_contents(b"Hello") as f:
specified_file_name = self.getUniqueString()
result = get_result_for(
[self.option, self.test_id, "--attach-file", f.name, "--file-name", specified_file_name]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_name=specified_file_name, file_bytes=b"Hello"),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_name_is_used_by_default(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_name=f.name, file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
class FileDataTests(TestCase):
def test_can_attach_file_without_test_id(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for(["--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=None, file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_name_is_used_by_default(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for(["--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_name=f.name, file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_filename_can_be_overridden(self):
with temp_file_contents(b"Hello") as f:
specified_file_name = self.getUniqueString()
result = get_result_for(["--attach-file", f.name, "--file-name", specified_file_name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_name=specified_file_name, file_bytes=b"Hello"),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_files_have_timestamp(self):
_dummy_timestamp = datetime.datetime(2013, 1, 1, 0, 0, 0, 0, UTC)
self.patch(_o, "create_timestamp", lambda: _dummy_timestamp)
with temp_file_contents(b"Hello") as f:
self.getUniqueString()
result = get_result_for(
[
"--attach-file",
f.name,
]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_bytes=b"Hello", timestamp=_dummy_timestamp),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_can_specify_tags_without_test_status(self):
result = get_result_for(
[
"--tag",
"foo",
]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_tags={"foo"}),
MatchesStatusCall(call="stopTestRun"),
]
),
)
class MatchesStatusCall(Matcher):
_position_lookup = {
"call": 0,
"test_id": 1,
"test_status": 2,
"test_tags": 3,
"runnable": 4,
"file_name": 5,
"file_bytes": 6,
"eof": 7,
"mime_type": 8,
"route_code": 9,
"timestamp": 10,
}
def __init__(self, **kwargs):
unknown_kwargs = list(filter(lambda k: k not in self._position_lookup, kwargs))
if unknown_kwargs:
raise ValueError("Unknown keywords: %s" % ",".join(unknown_kwargs))
self._filters = kwargs
def match(self, call_tuple):
for k, v in self._filters.items():
try:
pos = self._position_lookup[k]
if call_tuple[pos] != v:
return Mismatch("Value for key is {!r}, not {!r}".format(call_tuple[pos], v))
except IndexError:
return Mismatch("Key %s is not present." % k)
def __str__(self):
return "<MatchesStatusCall %r>" % self._filters
+110
python/tests/test_progress_model.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2009 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import unittest
from subunit.progress_model import ProgressModel
class TestProgressModel(unittest.TestCase):
def assertProgressSummary(self, pos, total, progress):
"""Assert that a progress model has reached a particular point."""
self.assertEqual(pos, progress.pos())
self.assertEqual(total, progress.width())
def test_new_progress_0_0(self):
progress = ProgressModel()
self.assertProgressSummary(0, 0, progress)
def test_advance_0_0(self):
progress = ProgressModel()
progress.advance()
self.assertProgressSummary(1, 0, progress)
def test_advance_1_0(self):
progress = ProgressModel()
progress.advance()
self.assertProgressSummary(1, 0, progress)
def test_set_width_absolute(self):
progress = ProgressModel()
progress.set_width(10)
self.assertProgressSummary(0, 10, progress)
def test_set_width_absolute_preserves_pos(self):
progress = ProgressModel()
progress.advance()
progress.set_width(2)
self.assertProgressSummary(1, 2, progress)
def test_adjust_width(self):
progress = ProgressModel()
progress.adjust_width(10)
self.assertProgressSummary(0, 10, progress)
progress.adjust_width(-10)
self.assertProgressSummary(0, 0, progress)
def test_adjust_width_preserves_pos(self):
progress = ProgressModel()
progress.advance()
progress.adjust_width(10)
self.assertProgressSummary(1, 10, progress)
progress.adjust_width(-10)
self.assertProgressSummary(1, 0, progress)
def test_push_preserves_progress(self):
progress = ProgressModel()
progress.adjust_width(3)
progress.advance()
progress.push()
self.assertProgressSummary(1, 3, progress)
def test_advance_advances_substack(self):
progress = ProgressModel()
progress.adjust_width(3)
progress.advance()
progress.push()
progress.adjust_width(1)
progress.advance()
self.assertProgressSummary(2, 3, progress)
def test_adjust_width_adjusts_substack(self):
progress = ProgressModel()
progress.adjust_width(3)
progress.advance()
progress.push()
progress.adjust_width(2)
progress.advance()
self.assertProgressSummary(3, 6, progress)
def test_set_width_adjusts_substack(self):
progress = ProgressModel()
progress.adjust_width(3)
progress.advance()
progress.push()
progress.set_width(2)
progress.advance()
self.assertProgressSummary(3, 6, progress)
def test_pop_restores_progress(self):
progress = ProgressModel()
progress.adjust_width(3)
progress.advance()
progress.push()
progress.adjust_width(1)
progress.advance()
progress.pop()
self.assertProgressSummary(1, 3, progress)
+115
python/tests/test_run.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2011 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import io
import unittest
from testtools import PlaceHolder, TestCase
from testtools.matchers import StartsWith
from testtools.testresult.doubles import StreamResult
import subunit
from subunit import run
from subunit.run import SubunitTestRunner
class TestSubunitTestRunner(TestCase):
def test_includes_timing_output(self):
bytestream = io.BytesIO()
runner = SubunitTestRunner(stream=bytestream)
test = PlaceHolder("name")
runner.run(test)
bytestream.seek(0)
eventstream = StreamResult()
subunit.ByteStreamToStreamResult(bytestream).run(eventstream)
timestamps = [event[-1] for event in eventstream._events if event is not None]
self.assertNotEqual([], timestamps)
def test_enumerates_tests_before_run(self):
bytestream = io.BytesIO()
runner = SubunitTestRunner(stream=bytestream)
test1 = PlaceHolder("name1")
test2 = PlaceHolder("name2")
case = unittest.TestSuite([test1, test2])
runner.run(case)
bytestream.seek(0)
eventstream = StreamResult()
subunit.ByteStreamToStreamResult(bytestream).run(eventstream)
self.assertEqual(
[
("status", "name1", "exists"),
("status", "name2", "exists"),
],
[event[:3] for event in eventstream._events[:2]],
)
def test_list_errors_if_errors_from_list_test(self):
bytestream = io.BytesIO()
runner = SubunitTestRunner(stream=bytestream)
def list_test(test):
return [], ["failed import"]
self.patch(run, "list_test", list_test)
exc = self.assertRaises(SystemExit, runner.list, None)
self.assertEqual((2,), exc.args)
def test_list_includes_loader_errors(self):
bytestream = io.BytesIO()
runner = SubunitTestRunner(stream=bytestream)
def list_test(test):
return [], []
class Loader:
errors = ["failed import"]
loader = Loader()
self.patch(run, "list_test", list_test)
exc = self.assertRaises(SystemExit, runner.list, None, loader=loader)
self.assertEqual((2,), exc.args)
class FailingTest(TestCase):
def test_fail(self):
1 / 0
def test_exits_zero_when_tests_fail(self):
bytestream = io.BytesIO()
stream = io.TextIOWrapper(bytestream, encoding="utf8")
try:
self.assertEqual(
None,
run.main(argv=["progName", "tests.test_run.TestSubunitTestRunner.FailingTest"], stdout=stream),
)
except SystemExit:
self.fail("SystemExit raised")
self.assertThat(bytestream.getvalue(), StartsWith(b"\xb3"))
class ExitingTest(TestCase):
def test_exit(self):
raise SystemExit(0)
def test_exits_nonzero_when_execution_errors(self):
bytestream = io.BytesIO()
stream = io.TextIOWrapper(bytestream, encoding="utf8")
exc = self.assertRaises(
SystemExit,
run.main,
argv=["progName", "tests.test_run.TestSubunitTestRunner.ExitingTest"],
stdout=stream,
)
self.assertEqual(0, exc.args[0])
+475
python/tests/test_subunit_combine.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2026 Jelmer Vernooij <jelmer@jelmer.uk>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Tests for subunit.filter_scripts.subunit_combine."""
import os
import subprocess
import sys
import tempfile
from io import BytesIO
from testtools import TestCase
from testtools.testresult.doubles import StreamResult
from subunit import ByteStreamToStreamResult, StreamResultToBytes
from subunit.filter_scripts.subunit_combine import (
_PrefixingStreamResult,
_expand_argv,
_read_id_list,
_select_ids_for_command,
combine,
load_config,
)
def _stream_with_tests(test_ids):
"""Build a subunit v2 byte stream with an inprogress+success pair per id."""
buf = BytesIO()
out = StreamResultToBytes(buf)
for tid in test_ids:
out.status(test_id=tid, test_status="inprogress")
out.status(test_id=tid, test_status="success")
return buf.getvalue()
def _parse(data):
"""Parse a subunit v2 byte stream into a list of (test_id, test_status)."""
events = StreamResult()
ByteStreamToStreamResult(BytesIO(data)).run(events)
return [(ev[1], ev[2]) for ev in events._events if ev[0] == "status"]
class TestPrefixingStreamResult(TestCase):
def test_prefixes_test_id(self):
target = StreamResult()
result = _PrefixingStreamResult(target, "py/")
result.status(test_id="foo", test_status="success")
self.assertEqual([("status", "py/foo", "success")], [ev[:3] for ev in target._events])
def test_passes_through_none_test_id(self):
target = StreamResult()
result = _PrefixingStreamResult(target, "py/")
result.status(file_name="stdout", file_bytes=b"hi")
self.assertEqual([("status", None, None)], [ev[:3] for ev in target._events])
class TestLoadConfig(TestCase):
def _write(self, text):
fd, path = tempfile.mkstemp(suffix=".yaml")
self.addCleanup(os.unlink, path)
with os.fdopen(fd, "w") as f:
f.write(text)
return path
def test_basic(self):
path = self._write("commands:\n - prefix: 'a/'\n argv: ['echo', 'hi']\n - argv: ['true']\n")
self.assertEqual(
[{"prefix": "a/", "argv": ["echo", "hi"]}, {"argv": ["true"]}],
load_config(path),
)
def test_rejects_non_mapping(self):
path = self._write("- 1\n- 2\n")
self.assertRaises(ValueError, load_config, path)
def test_rejects_empty_commands(self):
path = self._write("commands: []\n")
self.assertRaises(ValueError, load_config, path)
def test_rejects_missing_argv(self):
path = self._write("commands:\n - prefix: 'a/'\n")
self.assertRaises(ValueError, load_config, path)
def test_rejects_non_string_prefix(self):
path = self._write("commands:\n - argv: ['x']\n prefix: 42\n")
self.assertRaises(ValueError, load_config, path)
class TestCombine(TestCase):
def _cat_cmd(self, data):
"""A command that writes the given bytes to stdout and exits 0.
We use python -c so this works on all platforms and without relying on
shell quoting.
"""
src = "import sys; sys.stdout.buffer.write({!r})".format(data)
return [sys.executable, "-c", src]
def test_merges_streams_with_prefixes(self):
stream_a = _stream_with_tests(["one", "two"])
stream_b = _stream_with_tests(["alpha"])
commands = [
{"prefix": "py/", "argv": self._cat_cmd(stream_a)},
{"prefix": "rs/", "argv": self._cat_cmd(stream_b)},
]
output = BytesIO()
rc = combine(commands, output)
self.assertEqual(0, rc)
self.assertEqual(
[
("py/one", "inprogress"),
("py/one", "success"),
("py/two", "inprogress"),
("py/two", "success"),
("rs/alpha", "inprogress"),
("rs/alpha", "success"),
],
_parse(output.getvalue()),
)
def test_no_prefix(self):
stream = _stream_with_tests(["solo"])
commands = [{"argv": self._cat_cmd(stream)}]
output = BytesIO()
rc = combine(commands, output)
self.assertEqual(0, rc)
self.assertEqual(
[("solo", "inprogress"), ("solo", "success")],
_parse(output.getvalue()),
)
def test_nonzero_exit_propagates(self):
stream = _stream_with_tests(["only"])
src = "import sys; sys.stdout.buffer.write({!r}); sys.exit(3)".format(stream)
commands = [{"argv": [sys.executable, "-c", src]}]
output = BytesIO()
rc = combine(commands, output)
self.assertEqual(1, rc)
class TestCombineCommand(TestCase):
def test_end_to_end(self):
stream_a = _stream_with_tests(["a"])
stream_b = _stream_with_tests(["b"])
def cmd_for(data):
src = "import sys; sys.stdout.buffer.write({!r})".format(data)
return [sys.executable, "-c", src]
import yaml
config = {
"commands": [
{"prefix": "py/", "argv": cmd_for(stream_a)},
{"prefix": "rs/", "argv": cmd_for(stream_b)},
]
}
fd, path = tempfile.mkstemp(suffix=".yaml")
self.addCleanup(os.unlink, path)
with os.fdopen(fd, "w") as f:
yaml.safe_dump(config, f)
ps = subprocess.Popen(
[sys.executable, "-m", "subunit.filter_scripts.subunit_combine", path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, err = ps.communicate()
self.assertEqual(0, ps.returncode, err)
self.assertEqual(
[
("py/a", "inprogress"),
("py/a", "success"),
("rs/b", "inprogress"),
("rs/b", "success"),
],
_parse(out),
)
class TestExpandArgv(TestCase):
def test_listopt_expanded_in_list_mode(self):
cmd = {"argv": ["runner", "$LISTOPT", "$IDOPTION"], "list_option": "--list"}
self.assertEqual(
["runner", "--list"],
_expand_argv(cmd, list_mode=True, test_ids=None, idfile_path=None),
)
def test_listopt_empty_when_not_listing(self):
cmd = {"argv": ["runner", "$LISTOPT", "$IDOPTION"], "list_option": "--list"}
self.assertEqual(
["runner"],
_expand_argv(cmd, list_mode=False, test_ids=None, idfile_path=None),
)
def test_idoption_expanded_with_idfile(self):
cmd = {
"argv": ["runner", "$IDOPTION"],
"id_option": "--load-list $IDFILE",
}
self.assertEqual(
["runner", "--load-list", "/tmp/ids"],
_expand_argv(cmd, list_mode=False, test_ids=["a", "b"], idfile_path="/tmp/ids"),
)
def test_idlist_expanded(self):
cmd = {"argv": ["runner", "$IDLIST"]}
self.assertEqual(
["runner", "a", "b", "c"],
_expand_argv(cmd, list_mode=False, test_ids=["a", "b", "c"], idfile_path=None),
)
def test_idoption_empty_without_ids(self):
cmd = {
"argv": ["runner", "$IDOPTION"],
"id_option": "--load-list $IDFILE",
}
self.assertEqual(
["runner"],
_expand_argv(cmd, list_mode=False, test_ids=None, idfile_path=None),
)
def test_literal_argv_entries_untouched(self):
cmd = {"argv": ["runner", "--tag=$literal", "$IDLIST"]}
self.assertEqual(
["runner", "--tag=$literal", "x"],
_expand_argv(cmd, list_mode=False, test_ids=["x"], idfile_path=None),
)
class TestSelectIdsForCommand(TestCase):
def test_none_returns_none(self):
self.assertIsNone(_select_ids_for_command({"prefix": "py/"}, None))
def test_filters_by_prefix_and_strips(self):
ids = ["py/one", "rs/two", "py/three"]
self.assertEqual(["one", "three"], _select_ids_for_command({"prefix": "py/"}, ids))
def test_no_prefix_returns_all(self):
ids = ["one", "two"]
self.assertEqual(ids, _select_ids_for_command({}, ids))
self.assertIsNot(ids, _select_ids_for_command({}, ids)) # must copy
def test_no_matches_returns_empty(self):
self.assertEqual([], _select_ids_for_command({"prefix": "py/"}, ["rs/a"]))
class TestReadIdList(TestCase):
def test_skips_blank_and_comments(self):
fd, path = tempfile.mkstemp()
self.addCleanup(os.unlink, path)
with os.fdopen(fd, "w") as f:
f.write("one\n\n# comment\ntwo # trailing\nthree\n")
self.assertEqual(["one", "two", "three"], _read_id_list(path))
class TestCombineListAndFilter(TestCase):
def _cmd_for(self, data):
src = "import sys; sys.stdout.buffer.write({!r})".format(data)
return [sys.executable, "-c", src]
def _exists_stream(self, test_ids):
buf = BytesIO()
out = StreamResultToBytes(buf)
for tid in test_ids:
out.status(test_id=tid, test_status="exists")
return buf.getvalue()
def test_list_mode_substitutes_listopt(self):
# Child emits ids only when --list is present: we simulate this by
# embedding the argument directly in the command and asserting it's
# what gets executed.
list_stream = self._exists_stream(["foo", "bar"])
src = (
"import sys; args = sys.argv[1:]; assert args == ['--list'], args; sys.stdout.buffer.write({!r})"
).format(list_stream)
commands = [
{
"prefix": "py/",
"argv": [sys.executable, "-c", src, "$LISTOPT"],
"list_option": "--list",
}
]
output = BytesIO()
rc = combine(commands, output, list_mode=True)
self.assertEqual(0, rc)
self.assertEqual(
[("py/foo", "exists"), ("py/bar", "exists")],
_parse(output.getvalue()),
)
def test_listopt_absent_when_not_listing(self):
# When list_mode is False, $LISTOPT should expand to nothing.
run_stream = _stream_with_tests(["x"])
src = ("import sys; args = sys.argv[1:]; assert args == [], args; sys.stdout.buffer.write({!r})").format(
run_stream
)
commands = [
{
"argv": [sys.executable, "-c", src, "$LISTOPT"],
"list_option": "--list",
}
]
output = BytesIO()
rc = combine(commands, output)
self.assertEqual(0, rc)
def test_idoption_and_idfile_substituted(self):
# The child asserts it received --load-list <file> with two ids in it.
run_stream = _stream_with_tests(["one", "two"])
src = (
"import sys, pathlib; "
"args = sys.argv[1:]; "
"assert args[0] == '--load-list', args; "
"contents = pathlib.Path(args[1]).read_text().splitlines(); "
"assert contents == ['one', 'two'], contents; "
"sys.stdout.buffer.write({!r})"
).format(run_stream)
commands = [
{
"prefix": "py/",
"argv": [sys.executable, "-c", src, "$IDOPTION"],
"id_option": "--load-list $IDFILE",
}
]
output = BytesIO()
rc = combine(commands, output, test_ids=["py/one", "py/two"])
self.assertEqual(0, rc)
def test_idlist_substituted(self):
run_stream = _stream_with_tests(["a"])
src = (
"import sys; args = sys.argv[1:]; assert args == ['a', 'b'], args; sys.stdout.buffer.write({!r})"
).format(run_stream)
commands = [
{
"prefix": "py/",
"argv": [sys.executable, "-c", src, "$IDLIST"],
}
]
output = BytesIO()
rc = combine(commands, output, test_ids=["py/a", "py/b"])
self.assertEqual(0, rc)
def test_command_with_no_matching_ids_is_skipped(self):
# 'py/' receives ids; 'rs/' has no matching ids and must not run.
py_stream = _stream_with_tests(["one"])
py_src = "import sys; sys.stdout.buffer.write({!r})".format(py_stream)
rs_src = "import sys; sys.exit('must not run')"
commands = [
{"prefix": "py/", "argv": [sys.executable, "-c", py_src]},
{"prefix": "rs/", "argv": [sys.executable, "-c", rs_src]},
]
output = BytesIO()
rc = combine(commands, output, test_ids=["py/one"])
self.assertEqual(0, rc)
self.assertEqual(
[("py/one", "inprogress"), ("py/one", "success")],
_parse(output.getvalue()),
)
def test_cli_list_flag(self):
list_stream = self._exists_stream(["foo"])
src = (
"import sys; args = sys.argv[1:]; assert args == ['--list'], args; sys.stdout.buffer.write({!r})"
).format(list_stream)
config = {
"commands": [
{
"prefix": "py/",
"argv": [sys.executable, "-c", src, "$LISTOPT"],
"list_option": "--list",
}
]
}
fd, path = tempfile.mkstemp(suffix=".yaml")
self.addCleanup(os.unlink, path)
with os.fdopen(fd, "w") as f:
import yaml
yaml.safe_dump(config, f)
ps = subprocess.Popen(
[sys.executable, "-m", "subunit.filter_scripts.subunit_combine", "--list", path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, err = ps.communicate()
self.assertEqual(0, ps.returncode, err)
self.assertEqual([("py/foo", "exists")], _parse(out))
def test_cli_positional_ids(self):
run_stream = _stream_with_tests(["a"])
src = ("import sys; args = sys.argv[1:]; assert args == ['a'], args; sys.stdout.buffer.write({!r})").format(
run_stream
)
config = {
"commands": [
{
"prefix": "py/",
"argv": [sys.executable, "-c", src, "$IDLIST"],
}
]
}
fd, path = tempfile.mkstemp(suffix=".yaml")
self.addCleanup(os.unlink, path)
with os.fdopen(fd, "w") as f:
import yaml
yaml.safe_dump(config, f)
ps = subprocess.Popen(
[sys.executable, "-m", "subunit.filter_scripts.subunit_combine", path, "py/a"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, err = ps.communicate()
self.assertEqual(0, ps.returncode, err)
self.assertEqual(
[("py/a", "inprogress"), ("py/a", "success")],
_parse(out),
)
def test_cli_load_list(self):
run_stream = _stream_with_tests(["one"])
src = ("import sys; args = sys.argv[1:]; assert args == ['one'], args; sys.stdout.buffer.write({!r})").format(
run_stream
)
config = {
"commands": [
{
"prefix": "py/",
"argv": [sys.executable, "-c", src, "$IDLIST"],
}
]
}
cfg_fd, cfg_path = tempfile.mkstemp(suffix=".yaml")
self.addCleanup(os.unlink, cfg_path)
with os.fdopen(cfg_fd, "w") as f:
import yaml
yaml.safe_dump(config, f)
list_fd, list_path = tempfile.mkstemp(suffix=".list")
self.addCleanup(os.unlink, list_path)
with os.fdopen(list_fd, "w") as f:
f.write("py/one\n")
ps = subprocess.Popen(
[
sys.executable,
"-m",
"subunit.filter_scripts.subunit_combine",
"--load-list",
list_path,
cfg_path,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, err = ps.communicate()
self.assertEqual(0, ps.returncode, err)
self.assertEqual(
[("py/one", "inprogress"), ("py/one", "success")],
_parse(out),
)
+322
python/tests/test_subunit_filter.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Tests for subunit.TestResultFilter."""
import subprocess
import sys
import unittest
from datetime import datetime
from io import BytesIO
from testtools import TestCase
from testtools.testresult.doubles import ExtendedTestResult, StreamResult
import iso8601
import subunit
from subunit.test_results import make_tag_filter, TestResultFilter
from subunit import ByteStreamToStreamResult, StreamResultToBytes
class TestTestResultFilter(TestCase):
"""Test for TestResultFilter, a TestResult object which filters tests."""
# While TestResultFilter works on python objects, using a subunit stream
# is an easy pithy way of getting a series of test objects to call into
# the TestResult, and as TestResultFilter is intended for use with subunit
# also has the benefit of detecting any interface skew issues.
example_subunit_stream = b"""\
tags: global
test passed
success passed
test failed
tags: local
failure failed
test error
error error [
error details
]
test skipped
skip skipped
test todo
xfail todo
"""
def run_tests(self, result_filter, input_stream=None):
"""Run tests through the given filter.
:param result_filter: A filtering TestResult object.
:param input_stream: Bytes of subunit stream data. If not provided,
uses TestTestResultFilter.example_subunit_stream.
"""
if input_stream is None:
input_stream = self.example_subunit_stream
test = subunit.ProtocolTestCase(BytesIO(input_stream))
test.run(result_filter)
def test_default(self):
"""The default is to exclude success and include everything else."""
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result)
self.run_tests(result_filter)
# skips are seen as success by default python TestResult.
self.assertEqual(["error"], [error[0].id() for error in filtered_result.errors])
self.assertEqual(["failed"], [failure[0].id() for failure in filtered_result.failures])
self.assertEqual(4, filtered_result.testsRun)
def test_tag_filter(self):
tag_filter = make_tag_filter(["global"], ["local"])
result = ExtendedTestResult()
result_filter = TestResultFilter(result, filter_success=False, filter_predicate=tag_filter)
self.run_tests(result_filter)
tests_included = [event[1] for event in result._events if event[0] == "startTest"]
tests_expected = list(map(subunit.RemotedTestCase, ["passed", "error", "skipped", "todo"]))
self.assertEqual(tests_expected, tests_included)
def test_tags_tracked_correctly(self):
tag_filter = make_tag_filter(["a"], [])
result = ExtendedTestResult()
result_filter = TestResultFilter(result, filter_success=False, filter_predicate=tag_filter)
input_stream = b"test: foo\ntags: a\nsuccessful: foo\ntest: bar\nsuccessful: bar\n"
self.run_tests(result_filter, input_stream)
foo = subunit.RemotedTestCase("foo")
self.assertEqual(
[
("startTest", foo),
("tags", {"a"}, set()),
("addSuccess", foo),
("stopTest", foo),
],
result._events,
)
def test_exclude_errors(self):
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, filter_error=True)
self.run_tests(result_filter)
# skips are seen as errors by default python TestResult.
self.assertEqual([], filtered_result.errors)
self.assertEqual(["failed"], [failure[0].id() for failure in filtered_result.failures])
self.assertEqual(3, filtered_result.testsRun)
def test_fixup_expected_failures(self):
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, fixup_expected_failures={"failed"})
self.run_tests(result_filter)
self.assertEqual(["failed", "todo"], [failure[0].id() for failure in filtered_result.expectedFailures])
self.assertEqual([], filtered_result.failures)
self.assertEqual(4, filtered_result.testsRun)
def test_fixup_expected_errors(self):
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, fixup_expected_failures={"error"})
self.run_tests(result_filter)
self.assertEqual(["error", "todo"], [failure[0].id() for failure in filtered_result.expectedFailures])
self.assertEqual([], filtered_result.errors)
self.assertEqual(4, filtered_result.testsRun)
def test_fixup_unexpected_success(self):
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, filter_success=False, fixup_expected_failures={"passed"})
self.run_tests(result_filter)
self.assertEqual(["passed"], [passed.id() for passed in filtered_result.unexpectedSuccesses])
self.assertEqual(5, filtered_result.testsRun)
def test_exclude_failure(self):
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, filter_failure=True)
self.run_tests(result_filter)
self.assertEqual(["error"], [error[0].id() for error in filtered_result.errors])
self.assertEqual([], [failure[0].id() for failure in filtered_result.failures])
self.assertEqual(3, filtered_result.testsRun)
def test_exclude_skips(self):
filtered_result = subunit.TestResultStats(None)
result_filter = TestResultFilter(filtered_result, filter_skip=True)
self.run_tests(result_filter)
self.assertEqual(0, filtered_result.skipped_tests)
self.assertEqual(2, filtered_result.failed_tests)
self.assertEqual(3, filtered_result.testsRun)
def test_include_success(self):
"""Successes can be included if requested."""
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, filter_success=False)
self.run_tests(result_filter)
self.assertEqual(["error"], [error[0].id() for error in filtered_result.errors])
self.assertEqual(["failed"], [failure[0].id() for failure in filtered_result.failures])
self.assertEqual(5, filtered_result.testsRun)
def test_filter_predicate(self):
"""You can filter by predicate callbacks"""
# 0.0.7 and earlier did not support the 'tags' parameter, so we need
# to test that we still support behaviour without it.
filtered_result = unittest.TestResult()
def filter_cb(test, outcome, err, details):
return outcome == "success"
result_filter = TestResultFilter(filtered_result, filter_predicate=filter_cb, filter_success=False)
self.run_tests(result_filter)
# Only success should pass
self.assertEqual(1, filtered_result.testsRun)
def test_filter_predicate_with_tags(self):
"""You can filter by predicate callbacks that accept tags"""
filtered_result = unittest.TestResult()
def filter_cb(test, outcome, err, details, tags):
return outcome == "success"
result_filter = TestResultFilter(filtered_result, filter_predicate=filter_cb, filter_success=False)
self.run_tests(result_filter)
# Only success should pass
self.assertEqual(1, filtered_result.testsRun)
def test_time_ordering_preserved(self):
# Passing a subunit stream through TestResultFilter preserves the
# relative ordering of 'time' directives and any other subunit
# directives that are still included.
date_a = datetime(year=2000, month=1, day=1, tzinfo=iso8601.UTC)
date_b = datetime(year=2000, month=1, day=2, tzinfo=iso8601.UTC)
date_c = datetime(year=2000, month=1, day=3, tzinfo=iso8601.UTC)
subunit_stream = (
"\n".join(["time: %s", "test: foo", "time: %s", "error: foo", "time: %s", ""]) % (date_a, date_b, date_c)
).encode()
result = ExtendedTestResult()
result_filter = TestResultFilter(result)
self.run_tests(result_filter, subunit_stream)
foo = subunit.RemotedTestCase("foo")
self.maxDiff = None
self.assertEqual(
[
("time", date_a),
("time", date_b),
("startTest", foo),
("addError", foo, {}),
("stopTest", foo),
("time", date_c),
],
result._events,
)
def test_time_passes_through_filtered_tests(self):
# Passing a subunit stream through TestResultFilter preserves 'time'
# directives even if a specific test is filtered out.
date_a = datetime(year=2000, month=1, day=1, tzinfo=iso8601.UTC)
date_b = datetime(year=2000, month=1, day=2, tzinfo=iso8601.UTC)
date_c = datetime(year=2000, month=1, day=3, tzinfo=iso8601.UTC)
subunit_stream = (
"\n".join(["time: %s", "test: foo", "time: %s", "success: foo", "time: %s", ""]) % (date_a, date_b, date_c)
).encode()
result = ExtendedTestResult()
result_filter = TestResultFilter(result)
result_filter.startTestRun()
self.run_tests(result_filter, subunit_stream)
result_filter.stopTestRun()
subunit.RemotedTestCase("foo")
self.maxDiff = None
self.assertEqual(
[
("startTestRun",),
("time", date_a),
("time", date_c),
("stopTestRun",),
],
result._events,
)
def test_skip_preserved(self):
subunit_stream = "\n".join(["test: foo", "skip: foo", ""]).encode()
result = ExtendedTestResult()
result_filter = TestResultFilter(result)
self.run_tests(result_filter, subunit_stream)
foo = subunit.RemotedTestCase("foo")
self.assertEqual(
[
("startTest", foo),
("addSkip", foo, {}),
("stopTest", foo),
],
result._events,
)
def test_renames(self):
def rename(name):
return name + " - renamed"
result = ExtendedTestResult()
result_filter = TestResultFilter(result, filter_success=False, rename=rename)
input_stream = b"test: foo\nsuccessful: foo\n"
self.run_tests(result_filter, input_stream)
self.assertEqual(
[("startTest", "foo - renamed"), ("addSuccess", "foo - renamed"), ("stopTest", "foo - renamed")],
[(ev[0], ev[1].id()) for ev in result._events],
)
class TestFilterCommand(TestCase):
def run_command(self, args, stream):
command = [sys.executable, "-m", "subunit.filter_scripts.subunit_filter"] + list(args)
ps = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = ps.communicate(stream)
if ps.returncode != 0:
raise RuntimeError("{} failed: {}".format(command, err))
return out
def test_default(self):
byte_stream = BytesIO()
stream = StreamResultToBytes(byte_stream)
stream.status(test_id="foo", test_status="inprogress")
stream.status(test_id="foo", test_status="skip")
output = self.run_command([], byte_stream.getvalue())
events = StreamResult()
ByteStreamToStreamResult(BytesIO(output)).run(events)
{event[1] for event in events._events}
self.assertEqual(
[
("status", "foo", "inprogress"),
("status", "foo", "skip"),
],
[event[:3] for event in events._events],
)
def test_tags(self):
byte_stream = BytesIO()
stream = StreamResultToBytes(byte_stream)
stream.status(test_id="foo", test_status="inprogress", test_tags={"a"})
stream.status(test_id="foo", test_status="success", test_tags={"a"})
stream.status(test_id="bar", test_status="inprogress")
stream.status(test_id="bar", test_status="inprogress")
stream.status(test_id="baz", test_status="inprogress", test_tags={"a"})
stream.status(test_id="baz", test_status="success", test_tags={"a"})
output = self.run_command(["-s", "--with-tag", "a"], byte_stream.getvalue())
events = StreamResult()
ByteStreamToStreamResult(BytesIO(output)).run(events)
ids = {event[1] for event in events._events}
self.assertEqual({"foo", "baz"}, ids)
def test_no_passthrough(self):
output = self.run_command(["--no-passthrough"], b"hi thar")
self.assertEqual(b"", output)
def test_passthrough(self):
output = self.run_command([], b"hi thar")
byte_stream = BytesIO()
stream = StreamResultToBytes(byte_stream)
stream.status(file_name="stdout", file_bytes=b"hi thar")
self.assertEqual(byte_stream.getvalue(), output)
+82
python/tests/test_subunit_stats.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Tests for subunit.TestResultStats."""
import unittest
from io import BytesIO, StringIO
import subunit
class TestTestResultStats(unittest.TestCase):
"""Test for TestResultStats, a TestResult object that generates stats."""
def setUp(self):
self.output = StringIO()
self.result = subunit.TestResultStats(self.output)
self.input_stream = BytesIO()
self.test = subunit.ProtocolTestCase(self.input_stream)
def test_stats_empty(self):
self.test.run(self.result)
self.assertEqual(0, self.result.total_tests)
self.assertEqual(0, self.result.passed_tests)
self.assertEqual(0, self.result.failed_tests)
self.assertEqual(set(), self.result.seen_tags)
def setUpUsedStream(self):
self.input_stream.write(
b"""tags: global
test passed
success passed
test failed
tags: local
failure failed
test error
error error
test skipped
skip skipped
test todo
xfail todo
"""
)
self.input_stream.seek(0)
self.test.run(self.result)
def test_stats_smoke_everything(self):
# Statistics are calculated usefully.
self.setUpUsedStream()
self.assertEqual(5, self.result.total_tests)
self.assertEqual(2, self.result.passed_tests)
self.assertEqual(2, self.result.failed_tests)
self.assertEqual(1, self.result.skipped_tests)
self.assertEqual({"global", "local"}, self.result.seen_tags)
def test_stat_formatting(self):
expected = (
"""
Total tests: 5
Passed tests: 2
Failed tests: 2
Skipped tests: 1
Seen tags: global, local
"""
)[1:]
self.setUpUsedStream()
self.result.formatStats()
self.assertEqual(expected, self.output.getvalue())
+80
python/tests/test_subunit_tags.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Tests for subunit.tag_stream."""
from io import BytesIO
import testtools
from testtools.matchers import Contains
import subunit
import subunit.test_results
class TestSubUnitTags(testtools.TestCase):
def setUp(self):
super().setUp()
self.original = BytesIO()
self.filtered = BytesIO()
def test_add_tag(self):
# Literal values to avoid set sort-order dependencies. Python code show
# derivation.
# reference = BytesIO()
# stream = subunit.StreamResultToBytes(reference)
# stream.status(
# test_id='test', test_status='inprogress', test_tags=set(['quux', 'foo']))
# stream.status(
# test_id='test', test_status='success', test_tags=set(['bar', 'quux', 'foo']))
reference = [
b"\xb3)\x82\x17\x04test\x02\x04quux\x03foo\x05\x97n\x86"
b"\xb3)\x83\x1b\x04test\x03\x03bar\x04quux\x03fooqn\xab)",
b"\xb3)\x82\x17\x04test\x02\x04quux\x03foo\x05\x97n\x86"
b"\xb3)\x83\x1b\x04test\x03\x04quux\x03foo\x03bar\xaf\xbd\x9d\xd6",
b"\xb3)\x82\x17\x04test\x02\x04quux\x03foo\x05\x97n\x86"
b"\xb3)\x83\x1b\x04test\x03\x04quux\x03bar\x03foo\x03\x04b\r",
b"\xb3)\x82\x17\x04test\x02\x04quux\x03foo\x05\x97n\x86"
b"\xb3)\x83\x1b\x04test\x03\x03bar\x03foo\x04quux\xd2\x18\x1bC",
b"\xb3)\x82\x17\x04test\x02\x04quux\x03foo\x05\x97n\x86"
b"\xb3)\x83\x1b\x04test\x03\x03foo\x04quux\x03bar\x08\xc2X\x83",
b"\xb3)\x82\x17\x04test\x02\x03foo\x04quux\xa6\xe1\xde\xec"
b"\xb3)\x83\x1b\x04test\x03\x03foo\x04quux\x03bar\x08\xc2X\x83",
b"\xb3)\x82\x17\x04test\x02\x03foo\x04quux\xa6\xe1\xde\xec"
b"\xb3)\x83\x1b\x04test\x03\x03bar\x03foo\x04quux\xd2\x18\x1bC",
b"\xb3)\x82\x17\x04test\x02\x03foo\x04quux\xa6\xe1\xde\xec"
b"\xb3)\x83\x1b\x04test\x03\x03foo\x03bar\x04quux:\x05e\x80",
b"\xb3)\x82\x17\x04test\x02\x03foo\x04quux\xa6\xe1\xde\xec"
b"\xb3)\x83\x1b\x04test\x03\x04quux\x03foo\x03bar\xaf\xbd\x9d\xd6",
]
stream = subunit.StreamResultToBytes(self.original)
stream.status(test_id="test", test_status="inprogress", test_tags={"foo"})
stream.status(test_id="test", test_status="success", test_tags={"foo", "bar"})
self.original.seek(0)
self.assertEqual(0, subunit.tag_stream(self.original, self.filtered, ["quux"]))
self.assertThat(reference, Contains(bytes(self.filtered.getvalue())))
def test_remove_tag(self):
reference = BytesIO()
stream = subunit.StreamResultToBytes(reference)
stream.status(test_id="test", test_status="inprogress", test_tags={"foo"})
stream.status(test_id="test", test_status="success", test_tags={"foo"})
stream = subunit.StreamResultToBytes(self.original)
stream.status(test_id="test", test_status="inprogress", test_tags={"foo"})
stream.status(test_id="test", test_status="success", test_tags={"foo", "bar"})
self.original.seek(0)
self.assertEqual(0, subunit.tag_stream(self.original, self.filtered, ["-bar"]))
self.assertEqual(reference.getvalue(), self.filtered.getvalue())
+570
python/tests/test_tap2subunit.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Tests for TAP2SubUnit."""
from io import BytesIO, StringIO
from testtools import TestCase
from testtools.testresult.doubles import StreamResult
import subunit
UTF8_TEXT = "text/plain; charset=UTF8"
class TestTAP2SubUnit(TestCase):
"""Tests for TAP2SubUnit.
These tests test TAP string data in, and subunit string data out.
This is ok because the subunit protocol is intended to be stable,
but it might be easier/pithier to write tests against TAP string in,
parsed subunit objects out (by hooking the subunit stream to a subunit
protocol server.
"""
def setUp(self):
super().setUp()
self.tap = StringIO()
self.subunit = BytesIO()
def test_skip_entire_file(self):
# A file
# 1..- # Skipped: comment
# results in a single skipped test.
self.tap.write("1..0 # Skipped: entire file skipped\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"file skip",
"skip",
None,
True,
"tap comment",
b"Skipped: entire file skipped",
True,
None,
None,
None,
)
]
)
def test_ok_test_pass(self):
# A file
# ok
# results in a passed test with name 'test 1' (a synthetic name as tap
# does not require named fixtures - it is the first test in the tap
# stream).
self.tap.write("ok\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events([("status", "test 1", "success", None, False, None, None, True, None, None, None)])
def test_ok_test_number_pass(self):
# A file
# ok 1
# results in a passed test with name 'test 1'
self.tap.write("ok 1\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events([("status", "test 1", "success", None, False, None, None, True, None, None, None)])
def test_ok_test_number_description_pass(self):
# A file
# ok 1 - There is a description
# results in a passed test with name 'test 1 - There is a description'
self.tap.write("ok 1 - There is a description\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[("status", "test 1 - There is a description", "success", None, False, None, None, True, None, None, None)]
)
def test_ok_test_description_pass(self):
# A file
# ok There is a description
# results in a passed test with name 'test 1 There is a description'
self.tap.write("ok There is a description\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[("status", "test 1 There is a description", "success", None, False, None, None, True, None, None, None)]
)
def test_ok_SKIP_skip(self):
# A file
# ok # SKIP
# results in a skkip test with name 'test 1'
self.tap.write("ok # SKIP\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events([("status", "test 1", "skip", None, False, None, None, True, None, None, None)])
def test_ok_skip_number_comment_lowercase(self):
self.tap.write("ok 1 # skip no samba environment available, skipping compilation\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1",
"skip",
None,
False,
"tap comment",
b"no samba environment available, skipping compilation",
True,
"text/plain; charset=UTF8",
None,
None,
)
]
)
def test_ok_number_description_SKIP_skip_comment(self):
# A file
# ok 1 foo # SKIP Not done yet
# results in a skip test with name 'test 1 foo' and a log of
# Not done yet
self.tap.write("ok 1 foo # SKIP Not done yet\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1 foo",
"skip",
None,
False,
"tap comment",
b"Not done yet",
True,
"text/plain; charset=UTF8",
None,
None,
)
]
)
def test_ok_SKIP_skip_comment(self):
# A file
# ok # SKIP Not done yet
# results in a skip test with name 'test 1' and a log of Not done yet
self.tap.write("ok # SKIP Not done yet\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1",
"skip",
None,
False,
"tap comment",
b"Not done yet",
True,
"text/plain; charset=UTF8",
None,
None,
)
]
)
def test_ok_TODO_xfail(self):
# A file
# ok # TODO
# results in a xfail test with name 'test 1'
self.tap.write("ok # TODO\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events([("status", "test 1", "xfail", None, False, None, None, True, None, None, None)])
def test_ok_TODO_xfail_comment(self):
# A file
# ok # TODO Not done yet
# results in a xfail test with name 'test 1' and a log of Not done yet
self.tap.write("ok # TODO Not done yet\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1",
"xfail",
None,
False,
"tap comment",
b"Not done yet",
True,
"text/plain; charset=UTF8",
None,
None,
)
]
)
def test_bail_out_errors(self):
# A file with line in it
# Bail out! COMMENT
# is treated as an error
self.tap.write("ok 1 foo\n")
self.tap.write("Bail out! Lifejacket engaged\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
("status", "test 1 foo", "success", None, False, None, None, True, None, None, None),
("status", "Bail out! Lifejacket engaged", "fail", None, False, None, None, True, None, None, None),
]
)
def test_missing_test_at_end_with_plan_adds_error(self):
# A file
# 1..3
# ok first test
# not ok third test
# results in three tests, with the third being created
self.tap.write("1..3\n")
self.tap.write("ok first test\n")
self.tap.write("not ok second test\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
("status", "test 1 first test", "success", None, False, None, None, True, None, None, None),
("status", "test 2 second test", "fail", None, False, None, None, True, None, None, None),
(
"status",
"test 3",
"fail",
None,
False,
"tap meta",
b"test missing from TAP output",
True,
"text/plain; charset=UTF8",
None,
None,
),
]
)
def test_missing_test_with_plan_adds_error(self):
# A file
# 1..3
# ok first test
# not ok 3 third test
# results in three tests, with the second being created
self.tap.write("1..3\n")
self.tap.write("ok first test\n")
self.tap.write("not ok 3 third test\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
("status", "test 1 first test", "success", None, False, None, None, True, None, None, None),
(
"status",
"test 2",
"fail",
None,
False,
"tap meta",
b"test missing from TAP output",
True,
"text/plain; charset=UTF8",
None,
None,
),
("status", "test 3 third test", "fail", None, False, None, None, True, None, None, None),
]
)
def test_missing_test_no_plan_adds_error(self):
# A file
# ok first test
# not ok 3 third test
# results in three tests, with the second being created
self.tap.write("ok first test\n")
self.tap.write("not ok 3 third test\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
("status", "test 1 first test", "success", None, False, None, None, True, None, None, None),
(
"status",
"test 2",
"fail",
None,
False,
"tap meta",
b"test missing from TAP output",
True,
"text/plain; charset=UTF8",
None,
None,
),
("status", "test 3 third test", "fail", None, False, None, None, True, None, None, None),
]
)
def test_four_tests_in_a_row_trailing_plan(self):
# A file
# ok 1 - first test in a script with no plan at all
# not ok 2 - second
# ok 3 - third
# not ok 4 - fourth
# 1..4
# results in four tests numbered and named
self.tap.write("ok 1 - first test in a script with trailing plan\n")
self.tap.write("not ok 2 - second\n")
self.tap.write("ok 3 - third\n")
self.tap.write("not ok 4 - fourth\n")
self.tap.write("1..4\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1 - first test in a script with trailing plan",
"success",
None,
False,
None,
None,
True,
None,
None,
None,
),
("status", "test 2 - second", "fail", None, False, None, None, True, None, None, None),
("status", "test 3 - third", "success", None, False, None, None, True, None, None, None),
("status", "test 4 - fourth", "fail", None, False, None, None, True, None, None, None),
]
)
def test_four_tests_in_a_row_with_plan(self):
# A file
# 1..4
# ok 1 - first test in a script with no plan at all
# not ok 2 - second
# ok 3 - third
# not ok 4 - fourth
# results in four tests numbered and named
self.tap.write("1..4\n")
self.tap.write("ok 1 - first test in a script with a plan\n")
self.tap.write("not ok 2 - second\n")
self.tap.write("ok 3 - third\n")
self.tap.write("not ok 4 - fourth\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1 - first test in a script with a plan",
"success",
None,
False,
None,
None,
True,
None,
None,
None,
),
("status", "test 2 - second", "fail", None, False, None, None, True, None, None, None),
("status", "test 3 - third", "success", None, False, None, None, True, None, None, None),
("status", "test 4 - fourth", "fail", None, False, None, None, True, None, None, None),
]
)
def test_four_tests_in_a_row_no_plan(self):
# A file
# ok 1 - first test in a script with no plan at all
# not ok 2 - second
# ok 3 - third
# not ok 4 - fourth
# results in four tests numbered and named
self.tap.write("ok 1 - first test in a script with no plan at all\n")
self.tap.write("not ok 2 - second\n")
self.tap.write("ok 3 - third\n")
self.tap.write("not ok 4 - fourth\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1 - first test in a script with no plan at all",
"success",
None,
False,
None,
None,
True,
None,
None,
None,
),
("status", "test 2 - second", "fail", None, False, None, None, True, None, None, None),
("status", "test 3 - third", "success", None, False, None, None, True, None, None, None),
("status", "test 4 - fourth", "fail", None, False, None, None, True, None, None, None),
]
)
def test_todo_and_skip(self):
# A file
# not ok 1 - a fail but # TODO but is TODO
# not ok 2 - another fail # SKIP instead
# results in two tests, numbered and commented.
self.tap.write("not ok 1 - a fail but # TODO but is TODO\n")
self.tap.write("not ok 2 - another fail # SKIP instead\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.subunit.seek(0)
events = StreamResult()
subunit.ByteStreamToStreamResult(self.subunit).run(events)
self.check_events(
[
(
"status",
"test 1 - a fail but",
"xfail",
None,
False,
"tap comment",
b"but is TODO",
True,
"text/plain; charset=UTF8",
None,
None,
),
(
"status",
"test 2 - another fail",
"skip",
None,
False,
"tap comment",
b"instead",
True,
"text/plain; charset=UTF8",
None,
None,
),
]
)
def test_leading_comments_add_to_next_test_log(self):
# A file
# # comment
# ok
# ok
# results in a single test with the comment included
# in the first test and not the second.
self.tap.write("# comment\n")
self.tap.write("ok\n")
self.tap.write("ok\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1",
"success",
None,
False,
"tap comment",
b"# comment",
True,
"text/plain; charset=UTF8",
None,
None,
),
("status", "test 2", "success", None, False, None, None, True, None, None, None),
]
)
def test_trailing_comments_are_included_in_last_test_log(self):
# A file
# ok foo
# ok foo
# # comment
# results in a two tests, with the second having the comment
# attached to its log.
self.tap.write("ok\n")
self.tap.write("ok\n")
self.tap.write("# comment\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
("status", "test 1", "success", None, False, None, None, True, None, None, None),
(
"status",
"test 2",
"success",
None,
False,
"tap comment",
b"# comment",
True,
"text/plain; charset=UTF8",
None,
None,
),
]
)
def check_events(self, events):
self.subunit.seek(0)
eventstream = StreamResult()
subunit.ByteStreamToStreamResult(self.subunit).run(eventstream)
self.assertEqual(events, eventstream._events)
+1340
python/tests/test_test_protocol.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import datetime
import io
import os
import tempfile
import unittest
from io import BytesIO
from testtools import PlaceHolder, TestCase, TestResult, skipIf
from testtools.content import Content, TracebackContent, text_content
from testtools.content_type import ContentType
from testtools.testresult.doubles import ExtendedTestResult
from testtools.matchers import Contains, Equals, MatchesAny
import iso8601
import subunit
from . import (
_remote_exception_repr,
_remote_exception_repr_chunked,
_remote_exception_str,
_remote_exception_str_chunked,
)
tb_prelude = "Traceback (most recent call last):\n"
def details_to_str(details):
return TestResult()._err_details_to_string(None, details=details)
class TestHelpers(TestCase):
def test__unwrap_text_file_read_mode(self):
fd, file_path = tempfile.mkstemp()
self.addCleanup(os.remove, file_path)
fake_file = os.fdopen(fd, "r")
self.assertEqual(fake_file.buffer, subunit._unwrap_text(fake_file))
def test__unwrap_text_file_write_mode(self):
fd, file_path = tempfile.mkstemp()
self.addCleanup(os.remove, file_path)
fake_file = os.fdopen(fd, "w")
self.assertEqual(fake_file.buffer, subunit._unwrap_text(fake_file))
def test__unwrap_text_fileIO_read_mode(self):
fd, file_path = tempfile.mkstemp()
self.addCleanup(os.remove, file_path)
fake_file = io.FileIO(file_path, "r")
self.assertEqual(fake_file, subunit._unwrap_text(fake_file))
def test__unwrap_text_fileIO_write_mode(self):
fd, file_path = tempfile.mkstemp()
self.addCleanup(os.remove, file_path)
fake_file = io.FileIO(file_path, "w")
self.assertEqual(fake_file, subunit._unwrap_text(fake_file))
def test__unwrap_text_BytesIO(self):
fake_stream = io.BytesIO()
self.assertEqual(fake_stream, subunit._unwrap_text(fake_stream))
class TestTestImports(unittest.TestCase):
def test_imports(self):
from subunit import ( # noqa: F401
DiscardStream,
ExecTestCase,
IsolatedTestCase,
ProtocolTestCase,
RemotedTestCase,
RemoteError,
TestProtocolClient,
TestProtocolServer,
) # noqa: F401
class TestDiscardStream(unittest.TestCase):
def test_write(self):
subunit.DiscardStream().write("content")
class TestProtocolServerForward(unittest.TestCase):
def test_story(self):
client = unittest.TestResult()
out = BytesIO()
protocol = subunit.TestProtocolServer(client, forward_stream=out)
pipe = BytesIO(b"test old mcdonald\nsuccess old mcdonald\n")
protocol.readFrom(pipe)
self.assertEqual(client.testsRun, 1)
self.assertEqual(pipe.getvalue(), out.getvalue())
def test_not_command(self):
client = unittest.TestResult()
out = BytesIO()
protocol = subunit.TestProtocolServer(client, stream=subunit.DiscardStream(), forward_stream=out)
pipe = BytesIO(b"success old mcdonald\n")
protocol.readFrom(pipe)
self.assertEqual(client.testsRun, 0)
self.assertEqual(b"", out.getvalue())
class TestTestProtocolServerPipe(unittest.TestCase):
def test_story(self):
client = unittest.TestResult()
protocol = subunit.TestProtocolServer(client)
traceback = "foo.c:53:ERROR invalid state\n"
pipe = BytesIO(
b"test old mcdonald\n"
b"success old mcdonald\n"
b"test bing crosby\n"
b"failure bing crosby [\n" + traceback.encode() + b"]\n"
b"test an error\n"
b"error an error\n"
)
protocol.readFrom(pipe)
bing = subunit.RemotedTestCase("bing crosby")
an_error = subunit.RemotedTestCase("an error")
self.assertEqual(
client.errors,
[(an_error, _remote_exception_repr + "\n")],
)
self.assertEqual(
client.failures,
[(bing, _remote_exception_repr + ": " + details_to_str({"traceback": text_content(traceback)}) + "\n")],
)
self.assertEqual(client.testsRun, 3)
def test_non_test_characters_forwarded_immediately(self):
pass
class TestTestProtocolServerStartTest(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.stream = BytesIO()
self.protocol = subunit.TestProtocolServer(self.client, self.stream)
def test_start_test(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.assertEqual(self.client._events, [("startTest", subunit.RemotedTestCase("old mcdonald"))])
def test_start_testing(self):
self.protocol.lineReceived(b"testing old mcdonald\n")
self.assertEqual(self.client._events, [("startTest", subunit.RemotedTestCase("old mcdonald"))])
def test_start_test_colon(self):
self.protocol.lineReceived(b"test: old mcdonald\n")
self.assertEqual(self.client._events, [("startTest", subunit.RemotedTestCase("old mcdonald"))])
def test_indented_test_colon_ignored(self):
ignored_line = b" test: old mcdonald\n"
self.protocol.lineReceived(ignored_line)
self.assertEqual([], self.client._events)
self.assertEqual(self.stream.getvalue(), ignored_line)
def test_start_testing_colon(self):
self.protocol.lineReceived(b"testing: old mcdonald\n")
self.assertEqual(self.client._events, [("startTest", subunit.RemotedTestCase("old mcdonald"))])
class TestTestProtocolServerPassThrough(unittest.TestCase):
def setUp(self):
self.stdout = BytesIO()
self.test = subunit.RemotedTestCase("old mcdonald")
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client, self.stdout)
def keywords_before_test(self):
self.protocol.lineReceived(b"failure a\n")
self.protocol.lineReceived(b"failure: a\n")
self.protocol.lineReceived(b"error a\n")
self.protocol.lineReceived(b"error: a\n")
self.protocol.lineReceived(b"success a\n")
self.protocol.lineReceived(b"success: a\n")
self.protocol.lineReceived(b"successful a\n")
self.protocol.lineReceived(b"successful: a\n")
self.protocol.lineReceived(b"]\n")
self.assertEqual(
self.stdout.getvalue(),
b"failure a\nfailure: a\nerror a\nerror: a\nsuccess a\nsuccess: a\nsuccessful a\nsuccessful: a\n]\n",
)
def test_keywords_before_test(self):
self.keywords_before_test()
self.assertEqual(self.client._events, [])
def test_keywords_after_error(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"error old mcdonald\n")
self.keywords_before_test()
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, {}),
("stopTest", self.test),
],
self.client._events,
)
def test_keywords_after_failure(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"failure old mcdonald\n")
self.keywords_before_test()
self.assertEqual(
self.client._events,
[
("startTest", self.test),
("addFailure", self.test, {}),
("stopTest", self.test),
],
)
def test_keywords_after_success(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"success old mcdonald\n")
self.keywords_before_test()
self.assertEqual(
[
("startTest", self.test),
("addSuccess", self.test),
("stopTest", self.test),
],
self.client._events,
)
def test_keywords_after_test(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"failure a\n")
self.protocol.lineReceived(b"failure: a\n")
self.protocol.lineReceived(b"error a\n")
self.protocol.lineReceived(b"error: a\n")
self.protocol.lineReceived(b"success a\n")
self.protocol.lineReceived(b"success: a\n")
self.protocol.lineReceived(b"successful a\n")
self.protocol.lineReceived(b"successful: a\n")
self.protocol.lineReceived(b"]\n")
self.protocol.lineReceived(b"failure old mcdonald\n")
self.assertEqual(
self.stdout.getvalue(),
b"test old mcdonald\n"
b"failure a\n"
b"failure: a\n"
b"error a\n"
b"error: a\n"
b"success a\n"
b"success: a\n"
b"successful a\n"
b"successful: a\n"
b"]\n",
)
self.assertEqual(
self.client._events,
[
("startTest", self.test),
("addFailure", self.test, {}),
("stopTest", self.test),
],
)
def test_keywords_during_failure(self):
# A smoke test to make sure that the details parsers have control
# appropriately.
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"failure: old mcdonald [\n")
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"failure a\n")
self.protocol.lineReceived(b"failure: a\n")
self.protocol.lineReceived(b"error a\n")
self.protocol.lineReceived(b"error: a\n")
self.protocol.lineReceived(b"success a\n")
self.protocol.lineReceived(b"success: a\n")
self.protocol.lineReceived(b"successful a\n")
self.protocol.lineReceived(b"successful: a\n")
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.assertEqual(self.stdout.getvalue(), b"")
details = {}
details["traceback"] = Content(
ContentType("text", "x-traceback", {"charset": "utf8"}),
lambda: [
b"test old mcdonald\n"
b"failure a\n"
b"failure: a\n"
b"error a\n"
b"error: a\n"
b"success a\n"
b"success: a\n"
b"successful a\n"
b"successful: a\n"
b"]\n"
],
)
self.assertEqual(
self.client._events,
[
("startTest", self.test),
("addFailure", self.test, details),
("stopTest", self.test),
],
)
def test_stdout_passthrough(self):
"""Lines received which cannot be interpreted as any protocol action
should be passed through to sys.stdout.
"""
bytes = b"randombytes\n"
self.protocol.lineReceived(bytes)
self.assertEqual(self.stdout.getvalue(), bytes)
class TestTestProtocolServerLostConnection(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.test = subunit.RemotedTestCase("old mcdonald")
def test_lost_connection_no_input(self):
self.protocol.lostConnection()
self.assertEqual([], self.client._events)
def test_lost_connection_after_start(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lostConnection()
failure = subunit.RemoteError("lost connection during test 'old mcdonald'")
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, failure),
("stopTest", self.test),
],
self.client._events,
)
def test_lost_connected_after_error(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"error old mcdonald\n")
self.protocol.lostConnection()
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, {}),
("stopTest", self.test),
],
self.client._events,
)
def do_connection_lost(self, outcome, opening):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived("{} old mcdonald {}".format(outcome, opening).encode())
self.protocol.lostConnection()
failure = subunit.RemoteError("lost connection during %s report of test 'old mcdonald'" % outcome)
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, failure),
("stopTest", self.test),
],
self.client._events,
)
def test_lost_connection_during_error(self):
self.do_connection_lost("error", "[\n")
def test_lost_connection_during_error_details(self):
self.do_connection_lost("error", "[ multipart\n")
def test_lost_connected_after_failure(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"failure old mcdonald\n")
self.protocol.lostConnection()
self.assertEqual(
[
("startTest", self.test),
("addFailure", self.test, {}),
("stopTest", self.test),
],
self.client._events,
)
def test_lost_connection_during_failure(self):
self.do_connection_lost("failure", "[\n")
def test_lost_connection_during_failure_details(self):
self.do_connection_lost("failure", "[ multipart\n")
def test_lost_connection_after_success(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"success old mcdonald\n")
self.protocol.lostConnection()
self.assertEqual(
[
("startTest", self.test),
("addSuccess", self.test),
("stopTest", self.test),
],
self.client._events,
)
def test_lost_connection_during_success(self):
self.do_connection_lost("success", "[\n")
def test_lost_connection_during_success_details(self):
self.do_connection_lost("success", "[ multipart\n")
def test_lost_connection_during_skip(self):
self.do_connection_lost("skip", "[\n")
def test_lost_connection_during_skip_details(self):
self.do_connection_lost("skip", "[ multipart\n")
def test_lost_connection_during_xfail(self):
self.do_connection_lost("xfail", "[\n")
def test_lost_connection_during_xfail_details(self):
self.do_connection_lost("xfail", "[ multipart\n")
def test_lost_connection_during_uxsuccess(self):
self.do_connection_lost("uxsuccess", "[\n")
def test_lost_connection_during_uxsuccess_details(self):
self.do_connection_lost("uxsuccess", "[ multipart\n")
class TestInTestMultipart(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = subunit.RemotedTestCase("mcdonalds farm")
def test__outcome_sets_details_parser(self):
self.protocol._reading_success_details.details_parser = None
self.protocol._state._outcome(0, b"mcdonalds farm [ multipart\n", None, self.protocol._reading_success_details)
parser = self.protocol._reading_success_details.details_parser
self.assertNotEqual(None, parser)
self.assertIsInstance(parser, subunit.details.MultipartDetailsParser)
class TestTestProtocolServerAddError(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = subunit.RemotedTestCase("mcdonalds farm")
def simple_error_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
details = {}
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def test_simple_error(self):
self.simple_error_keyword("error")
def test_simple_error_colon(self):
self.simple_error_keyword("error:")
def test_error_empty_message(self):
self.protocol.lineReceived(b"error mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["traceback"] = Content(ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b""])
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def error_quoted_bracket(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm [\n" % keyword).encode())
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["traceback"] = Content(ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b"]\n"])
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def test_error_quoted_bracket(self):
self.error_quoted_bracket("error")
def test_error_colon_quoted_bracket(self):
self.error_quoted_bracket("error:")
class TestTestProtocolServerAddFailure(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = subunit.RemotedTestCase("mcdonalds farm")
def assertFailure(self, details):
self.assertEqual(
[
("startTest", self.test),
("addFailure", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def simple_failure_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
details = {}
self.assertFailure(details)
def test_simple_failure(self):
self.simple_failure_keyword("failure")
def test_simple_failure_colon(self):
self.simple_failure_keyword("failure:")
def test_failure_empty_message(self):
self.protocol.lineReceived(b"failure mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["traceback"] = Content(ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b""])
self.assertFailure(details)
def failure_quoted_bracket(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm [\n" % keyword).encode())
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["traceback"] = Content(ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b"]\n"])
self.assertFailure(details)
def test_failure_quoted_bracket(self):
self.failure_quoted_bracket("failure")
def test_failure_colon_quoted_bracket(self):
self.failure_quoted_bracket("failure:")
class TestTestProtocolServerAddxFail(unittest.TestCase):
"""Tests for the xfail keyword."""
def setUp(self):
"""Setup a test object ready to be xfailed with details."""
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = self.client._events[-1][-1]
def simple_xfail_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
self.check_xfail()
def check_xfail(self, error_message=None):
details = {}
if error_message is not None:
details["traceback"] = Content(
ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [error_message.encode()]
)
self.assertEqual(
[
("startTest", self.test),
("addExpectedFailure", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def test_simple_xfail(self):
self.simple_xfail_keyword("xfail")
def test_simple_xfail_colon(self):
self.simple_xfail_keyword("xfail:")
def test_xfail_empty_message(self):
self.protocol.lineReceived(b"xfail mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
self.check_xfail(error_message="")
def test_xfail_quoted_bracket(self):
self.protocol.lineReceived(b"xfail mcdonalds farm [\n")
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.check_xfail("]\n")
def test_xfail_colon_quoted_bracket(self):
self.protocol.lineReceived(b"xfail: mcdonalds farm [\n")
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.check_xfail("]\n")
class TestTestProtocolServerAddunexpectedSuccess(TestCase):
"""Tests for the uxsuccess keyword."""
def setUp(self):
"""Setup a test object ready for uxsuccess with details."""
super().setUp()
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = self.client._events[-1][-1]
def simple_uxsuccess_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
self.check_uxsuccess()
def check_uxsuccess(self, error_message=None):
if error_message is not None:
details = {}
details["traceback"] = Content(
ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [error_message.encode()]
)
expected_events = [
("startTest", self.test),
("addUnexpectedSuccess", self.test, details),
("stopTest", self.test),
]
else:
expected_events = [
("startTest", self.test),
("addUnexpectedSuccess", self.test),
("stopTest", self.test),
]
self.assertEqual(expected_events, self.client._events)
def test_simple_uxsuccess(self):
self.simple_uxsuccess_keyword("uxsuccess")
def test_simple_uxsuccess_colon(self):
self.simple_uxsuccess_keyword("uxsuccess:")
def test_uxsuccess_empty_message(self):
self.protocol.lineReceived(b"uxsuccess mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
self.check_uxsuccess(error_message="")
def test_uxsuccess_quoted_bracket(self):
self.protocol.lineReceived(b"uxsuccess mcdonalds farm [\n")
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.check_uxsuccess("]\n")
def test_uxsuccess_colon_quoted_bracket(self):
self.protocol.lineReceived(b"uxsuccess: mcdonalds farm [\n")
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.check_uxsuccess("]\n")
class TestTestProtocolServerAddSkip(unittest.TestCase):
"""Tests for the skip keyword.
In Python this meets the testtools extended TestResult contract.
(See https://launchpad.net/testtools).
"""
def setUp(self):
"""Setup a test object ready to be skipped."""
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = self.client._events[-1][-1]
def assertSkip(self, reason):
details = {}
if reason is not None:
details["reason"] = Content(ContentType("text", "plain"), lambda: [reason])
self.assertEqual(
[
("startTest", self.test),
("addSkip", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def simple_skip_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
self.assertSkip(None)
def test_simple_skip(self):
self.simple_skip_keyword("skip")
def test_simple_skip_colon(self):
self.simple_skip_keyword("skip:")
def test_skip_empty_message(self):
self.protocol.lineReceived(b"skip mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
self.assertSkip(b"")
def skip_quoted_bracket(self, keyword):
# This tests it is accepted, but cannot test it is used today, because
# of not having a way to expose it in Python so far.
self.protocol.lineReceived(("%s mcdonalds farm [\n" % keyword).encode())
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.assertSkip(b"]\n")
def test_skip_quoted_bracket(self):
self.skip_quoted_bracket("skip")
def test_skip_colon_quoted_bracket(self):
self.skip_quoted_bracket("skip:")
class TestTestProtocolServerAddSuccess(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = subunit.RemotedTestCase("mcdonalds farm")
def simple_success_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
self.assertEqual(
[
("startTest", self.test),
("addSuccess", self.test),
("stopTest", self.test),
],
self.client._events,
)
def test_simple_success(self):
self.simple_success_keyword("successful")
def test_simple_success_colon(self):
self.simple_success_keyword("successful:")
def assertSuccess(self, details):
self.assertEqual(
[
("startTest", self.test),
("addSuccess", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def test_success_empty_message(self):
self.protocol.lineReceived(b"success mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["message"] = Content(ContentType("text", "plain"), lambda: [b""])
self.assertSuccess(details)
def success_quoted_bracket(self, keyword):
# This tests it is accepted, but cannot test it is used today, because
# of not having a way to expose it in Python so far.
self.protocol.lineReceived(("%s mcdonalds farm [\n" % keyword).encode())
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["message"] = Content(ContentType("text", "plain"), lambda: [b"]\n"])
self.assertSuccess(details)
def test_success_quoted_bracket(self):
self.success_quoted_bracket("success")
def test_success_colon_quoted_bracket(self):
self.success_quoted_bracket("success:")
class TestTestProtocolServerProgress(unittest.TestCase):
"""Test receipt of progress: directives."""
def test_progress_accepted_stdlib(self):
self.result = ExtendedTestResult()
self.stream = BytesIO()
self.protocol = subunit.TestProtocolServer(self.result, stream=self.stream)
self.protocol.lineReceived(b"progress: 23")
self.protocol.lineReceived(b"progress: -2")
self.protocol.lineReceived(b"progress: +4")
self.assertEqual(b"", self.stream.getvalue())
def test_progress_accepted_extended(self):
# With a progress capable TestResult, progress events are emitted.
self.result = ExtendedTestResult()
self.stream = BytesIO()
self.protocol = subunit.TestProtocolServer(self.result, stream=self.stream)
self.protocol.lineReceived(b"progress: 23")
self.protocol.lineReceived(b"progress: push")
self.protocol.lineReceived(b"progress: -2")
self.protocol.lineReceived(b"progress: pop")
self.protocol.lineReceived(b"progress: +4")
self.assertEqual(b"", self.stream.getvalue())
self.assertEqual(
[
("progress", 23, subunit.PROGRESS_SET),
("progress", None, subunit.PROGRESS_PUSH),
("progress", -2, subunit.PROGRESS_CUR),
("progress", None, subunit.PROGRESS_POP),
("progress", 4, subunit.PROGRESS_CUR),
],
self.result._events,
)
class TestTestProtocolServerStreamTags(unittest.TestCase):
"""Test managing tags on the protocol level."""
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
def test_initial_tags(self):
self.protocol.lineReceived(b"tags: foo bar:baz quux\n")
self.assertEqual(
[
("tags", {"foo", "bar:baz", "quux"}, set()),
],
self.client._events,
)
def test_minus_removes_tags(self):
self.protocol.lineReceived(b"tags: -bar quux\n")
self.assertEqual(
[
("tags", {"quux"}, {"bar"}),
],
self.client._events,
)
def test_tags_do_not_get_set_on_test(self):
self.protocol.lineReceived(b"test mcdonalds farm\n")
test = self.client._events[0][-1]
self.assertEqual(None, getattr(test, "tags", None))
def test_tags_do_not_get_set_on_global_tags(self):
self.protocol.lineReceived(b"tags: foo bar\n")
self.protocol.lineReceived(b"test mcdonalds farm\n")
test = self.client._events[-1][-1]
self.assertEqual(None, getattr(test, "tags", None))
def test_tags_get_set_on_test_tags(self):
self.protocol.lineReceived(b"test mcdonalds farm\n")
test = self.client._events[-1][-1]
self.protocol.lineReceived(b"tags: foo bar\n")
self.protocol.lineReceived(b"success mcdonalds farm\n")
self.assertEqual(None, getattr(test, "tags", None))
class TestTestProtocolServerStreamTime(unittest.TestCase):
"""Test managing time information at the protocol level."""
def test_time_accepted_stdlib(self):
self.result = ExtendedTestResult()
self.stream = BytesIO()
self.protocol = subunit.TestProtocolServer(self.result, stream=self.stream)
self.protocol.lineReceived(b"time: 2001-12-12 12:59:59Z\n")
self.assertEqual(b"", self.stream.getvalue())
def test_time_accepted_extended(self):
self.result = ExtendedTestResult()
self.stream = BytesIO()
self.protocol = subunit.TestProtocolServer(self.result, stream=self.stream)
self.protocol.lineReceived(b"time: 2001-12-12 12:59:59Z\n")
self.assertEqual(b"", self.stream.getvalue())
self.assertEqual([("time", datetime.datetime(2001, 12, 12, 12, 59, 59, 0, iso8601.UTC))], self.result._events)
class TestRemotedTestCase(unittest.TestCase):
def test_simple(self):
test = subunit.RemotedTestCase("A test description")
self.assertRaises(NotImplementedError, test.setUp)
self.assertRaises(NotImplementedError, test.tearDown)
self.assertEqual("A test description", test.shortDescription())
self.assertEqual("A test description", test.id())
self.assertEqual("A test description (subunit.RemotedTestCase)", "%s" % test)
self.assertEqual("<subunit.RemotedTestCase description='A test description'>", "%r" % test)
result = unittest.TestResult()
test.run(result)
self.assertEqual([(test, _remote_exception_repr + ": " + "Cannot run RemotedTestCases.\n\n")], result.errors)
self.assertEqual(1, result.testsRun)
another_test = subunit.RemotedTestCase("A test description")
self.assertEqual(test, another_test)
different_test = subunit.RemotedTestCase("ofo")
self.assertNotEqual(test, different_test)
self.assertNotEqual(another_test, different_test)
class TestRemoteError(unittest.TestCase):
def test_eq(self):
error = subunit.RemoteError("Something went wrong")
another_error = subunit.RemoteError("Something went wrong")
different_error = subunit.RemoteError("boo!")
self.assertEqual(error, another_error)
self.assertNotEqual(error, different_error)
self.assertNotEqual(different_error, another_error)
def test_empty_constructor(self):
self.assertEqual(subunit.RemoteError(), subunit.RemoteError(""))
class TestExecTestCase(unittest.TestCase):
class SampleExecTestCase(subunit.ExecTestCase):
def test_sample_method(self):
"""sample-script.py"""
# the sample script runs three tests, one each
# that fails, errors and succeeds
def test_sample_method_args(self):
"""sample-script.py foo"""
# sample that will run just one test.
def test_construct(self):
test = self.SampleExecTestCase("test_sample_method")
self.assertEqual(test.script, subunit.join_dir(__file__, "sample-script.py"))
def test_args(self):
result = unittest.TestResult()
test = self.SampleExecTestCase("test_sample_method_args")
test.run(result)
self.assertEqual(1, result.testsRun)
def test_run(self):
result = ExtendedTestResult()
test = self.SampleExecTestCase("test_sample_method")
test.run(result)
mcdonald = subunit.RemotedTestCase("old mcdonald")
bing = subunit.RemotedTestCase("bing crosby")
bing_details = {}
bing_details["traceback"] = Content(
ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b"foo.c:53:ERROR invalid state\n"]
)
an_error = subunit.RemotedTestCase("an error")
error_details = {}
self.assertEqual(
[
("startTest", mcdonald),
("addSuccess", mcdonald),
("stopTest", mcdonald),
("startTest", bing),
("addFailure", bing, bing_details),
("stopTest", bing),
("startTest", an_error),
("addError", an_error, error_details),
("stopTest", an_error),
],
result._events,
)
def test_debug(self):
test = self.SampleExecTestCase("test_sample_method")
test.debug()
def test_count_test_cases(self):
"""Run the child process and count responses to determine the count."""
def test_join_dir(self):
sibling = subunit.join_dir(__file__, "foo")
filedir = os.path.abspath(os.path.dirname(__file__))
expected = os.path.join(filedir, "foo")
self.assertEqual(sibling, expected)
class DoExecTestCase(subunit.ExecTestCase):
def test_working_script(self):
"""sample-two-script.py"""
class TestIsolatedTestCase(TestCase):
class SampleIsolatedTestCase(subunit.IsolatedTestCase):
SETUP = False
TEARDOWN = False
TEST = False
def setUp(self):
TestIsolatedTestCase.SampleIsolatedTestCase.SETUP = True
def tearDown(self):
TestIsolatedTestCase.SampleIsolatedTestCase.TEARDOWN = True
def test_sets_global_state(self):
TestIsolatedTestCase.SampleIsolatedTestCase.TEST = True
def test_construct(self):
self.SampleIsolatedTestCase("test_sets_global_state")
@skipIf(os.name != "posix", "Need a posix system for forking tests")
def test_run(self):
result = unittest.TestResult()
test = self.SampleIsolatedTestCase("test_sets_global_state")
test.run(result)
self.assertEqual(result.testsRun, 1)
self.assertEqual(self.SampleIsolatedTestCase.SETUP, False)
self.assertEqual(self.SampleIsolatedTestCase.TEARDOWN, False)
self.assertEqual(self.SampleIsolatedTestCase.TEST, False)
def test_debug(self):
pass
# test = self.SampleExecTestCase("test_sample_method")
# test.debug()
class TestIsolatedTestSuite(TestCase):
class SampleTestToIsolate(unittest.TestCase):
SETUP = False
TEARDOWN = False
TEST = False
def setUp(self):
TestIsolatedTestSuite.SampleTestToIsolate.SETUP = True
def tearDown(self):
TestIsolatedTestSuite.SampleTestToIsolate.TEARDOWN = True
def test_sets_global_state(self):
TestIsolatedTestSuite.SampleTestToIsolate.TEST = True
def test_construct(self):
subunit.IsolatedTestSuite()
@skipIf(os.name != "posix", "Need a posix system for forking tests")
def test_run(self):
result = unittest.TestResult()
suite = subunit.IsolatedTestSuite()
sub_suite = unittest.TestSuite()
sub_suite.addTest(self.SampleTestToIsolate("test_sets_global_state"))
sub_suite.addTest(self.SampleTestToIsolate("test_sets_global_state"))
suite.addTest(sub_suite)
suite.addTest(self.SampleTestToIsolate("test_sets_global_state"))
suite.run(result)
self.assertEqual(result.testsRun, 3)
self.assertEqual(self.SampleTestToIsolate.SETUP, False)
self.assertEqual(self.SampleTestToIsolate.TEARDOWN, False)
self.assertEqual(self.SampleTestToIsolate.TEST, False)
# A number of these tests produce different output depending on the
# testtools version. testtools < 2.5.0 used traceback2, which incorrectly
# included the traceback header even for an exception with no traceback.
# testtools 2.5.0 switched to the Python 3 standard library's traceback
# module, which fixes this bug. See https://bugs.python.org/issue24695.
class TestTestProtocolClient(TestCase):
def setUp(self):
super().setUp()
self.io = BytesIO()
self.protocol = subunit.TestProtocolClient(self.io)
self.unicode_test = PlaceHolder("\u2603")
self.test = TestTestProtocolClient("test_start_test")
self.sample_details = {"something": Content(ContentType("text", "plain"), lambda: [b"serialised\nform"])}
self.sample_tb_details = dict(self.sample_details)
self.sample_tb_details["traceback"] = TracebackContent(subunit.RemoteError("boo qux"), self.test)
def test_start_test(self):
"""Test startTest on a TestProtocolClient."""
self.protocol.startTest(self.test)
self.assertEqual(self.io.getvalue(), ("test: %s\n" % self.test.id()).encode())
def test_start_test_unicode_id(self):
"""Test startTest on a TestProtocolClient."""
self.protocol.startTest(self.unicode_test)
expected = b"test: " + "\u2603".encode("utf8") + b"\n"
self.assertEqual(expected, self.io.getvalue())
def test_stop_test(self):
# stopTest doesn't output anything.
self.protocol.stopTest(self.test)
self.assertEqual(self.io.getvalue(), b"")
def test_add_success(self):
"""Test addSuccess on a TestProtocolClient."""
self.protocol.addSuccess(self.test)
self.assertEqual(self.io.getvalue(), ("successful: %s\n" % self.test.id()).encode())
def test_add_outcome_unicode_id(self):
"""Test addSuccess on a TestProtocolClient."""
self.protocol.addSuccess(self.unicode_test)
expected = b"successful: " + "\u2603".encode("utf8") + b"\n"
self.assertEqual(expected, self.io.getvalue())
def test_add_success_details(self):
"""Test addSuccess on a TestProtocolClient with details."""
self.protocol.addSuccess(self.test, details=self.sample_details)
self.assertEqual(
self.io.getvalue(),
(
"successful: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n]\n" % self.test.id()
).encode(),
)
def test_add_failure(self):
"""Test addFailure on a TestProtocolClient."""
self.protocol.addFailure(self.test, subunit.RemoteError("boo qux"))
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals((("failure: %s [\n" + _remote_exception_str + ": boo qux\n" + "]\n") % self.test.id()).encode()),
# testtools >= 2.5.0
Equals(
(("failure: %s [\n" + _remote_exception_repr + ": boo qux\n" + "]\n") % self.test.id()).encode()
),
),
)
def test_add_failure_details(self):
"""Test addFailure on a TestProtocolClient with details."""
self.protocol.addFailure(self.test, details=self.sample_tb_details)
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals(
(
(
"failure: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_str_chunked + "]\n"
)
% self.test.id()
).encode()
),
# testtools >= 2.5.0
Equals(
(
(
"failure: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_repr_chunked + "]\n"
)
% self.test.id()
).encode()
),
),
)
def test_add_error(self):
"""Test stopTest on a TestProtocolClient."""
self.protocol.addError(self.test, subunit.RemoteError("phwoar crikey"))
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals((("error: %s [\n" + _remote_exception_str + ": phwoar crikey\n]\n") % self.test.id()).encode()),
# testtools >= 2.5.0
Equals((("error: %s [\n" + _remote_exception_repr + ": phwoar crikey\n]\n") % self.test.id()).encode()),
),
)
def test_add_error_details(self):
"""Test stopTest on a TestProtocolClient with details."""
self.protocol.addError(self.test, details=self.sample_tb_details)
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals(
(
(
"error: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_str_chunked + "]\n"
)
% self.test.id()
).encode()
),
# testtools >= 2.5.0
Equals(
(
(
"error: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_repr_chunked + "]\n"
)
% self.test.id()
).encode()
),
),
)
def test_add_expected_failure(self):
"""Test addExpectedFailure on a TestProtocolClient."""
self.protocol.addExpectedFailure(self.test, subunit.RemoteError("phwoar crikey"))
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals((("xfail: %s [\n" + _remote_exception_str + ": phwoar crikey\n]\n") % self.test.id()).encode()),
# testtools >= 2.5.0
Equals((("xfail: %s [\n" + _remote_exception_repr + ": phwoar crikey\n]\n") % self.test.id()).encode()),
),
)
def test_add_expected_failure_details(self):
"""Test addExpectedFailure on a TestProtocolClient with details."""
self.protocol.addExpectedFailure(self.test, details=self.sample_tb_details)
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals(
(
(
"xfail: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_str_chunked + "]\n"
)
% self.test.id()
).encode()
),
# testtools >= 2.5.0
Equals(
(
(
"xfail: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_repr_chunked + "]\n"
)
% self.test.id()
).encode()
),
),
)
def test_add_skip(self):
"""Test addSkip on a TestProtocolClient."""
self.protocol.addSkip(self.test, "Has it really?")
self.assertEqual(self.io.getvalue(), ("skip: %s [\nHas it really?\n]\n" % self.test.id()).encode())
def test_add_skip_details(self):
"""Test addSkip on a TestProtocolClient with details."""
details = {"reason": Content(ContentType("text", "plain"), lambda: [b"Has it really?"])}
self.protocol.addSkip(self.test, details=details)
self.assertEqual(
self.io.getvalue(),
(
"skip: %s [ multipart\nContent-Type: text/plain\nreason\nE\r\nHas it really?0\r\n]\n" % self.test.id()
).encode(),
)
def test_progress_set(self):
self.protocol.progress(23, subunit.PROGRESS_SET)
self.assertEqual(self.io.getvalue(), b"progress: 23\n")
def test_progress_neg_cur(self):
self.protocol.progress(-23, subunit.PROGRESS_CUR)
self.assertEqual(self.io.getvalue(), b"progress: -23\n")
def test_progress_pos_cur(self):
self.protocol.progress(23, subunit.PROGRESS_CUR)
self.assertEqual(self.io.getvalue(), b"progress: +23\n")
def test_progress_pop(self):
self.protocol.progress(1234, subunit.PROGRESS_POP)
self.assertEqual(self.io.getvalue(), b"progress: pop\n")
def test_progress_push(self):
self.protocol.progress(1234, subunit.PROGRESS_PUSH)
self.assertEqual(self.io.getvalue(), b"progress: push\n")
def test_time(self):
# Calling time() outputs a time signal immediately.
self.protocol.time(datetime.datetime(2009, 10, 11, 12, 13, 14, 15, iso8601.UTC))
self.assertEqual(b"time: 2009-10-11 12:13:14.000015Z\n", self.io.getvalue())
def test_add_unexpected_success(self):
"""Test addUnexpectedSuccess on a TestProtocolClient."""
self.protocol.addUnexpectedSuccess(self.test)
self.assertEqual(self.io.getvalue(), ("uxsuccess: %s\n" % self.test.id()).encode())
def test_add_unexpected_success_details(self):
"""Test addUnexpectedSuccess on a TestProtocolClient with details."""
self.protocol.addUnexpectedSuccess(self.test, details=self.sample_details)
self.assertEqual(
self.io.getvalue(),
(
"uxsuccess: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n]\n" % self.test.id()
).encode(),
)
def test_tags_empty(self):
self.protocol.tags(set(), set())
self.assertEqual(b"", self.io.getvalue())
def test_tags_add(self):
self.protocol.tags({"foo"}, set())
self.assertEqual(b"tags: foo\n", self.io.getvalue())
def test_tags_both(self):
self.protocol.tags({"quux"}, {"bar"})
self.assertThat([b"tags: quux -bar\n", b"tags: -bar quux\n"], Contains(self.io.getvalue()))
def test_tags_gone(self):
self.protocol.tags(set(), {"bar"})
self.assertEqual(b"tags: -bar\n", self.io.getvalue())
+564
python/tests/test_test_protocol2.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2013 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import datetime
from io import BytesIO
from typing import Callable, Optional
from types import ModuleType
try:
from hypothesis import given
# To debug hypothesis
# from hypothesis import Settings, Verbosity
# Settings.default.verbosity = Verbosity.verbose
import hypothesis.strategies as st
except ImportError:
given: Optional[Callable[..., Callable[..., None]]] = None # type: ignore[assignment,no-redef]
st: Optional[ModuleType] = None # type: ignore[assignment,no-redef]
from testtools import TestCase
from testtools.matchers import Contains, HasLength
from testtools.testresult.doubles import StreamResult
try:
from testtools.tests.test_testresult import TestStreamResultContract # type: ignore[import-not-found]
except ImportError:
# testtools >= 2.8 no longer includes the tests submodule
TestStreamResultContract: Optional[type] = None # type: ignore[assignment,misc,no-redef]
import subunit
import iso8601
CONSTANT_ENUM = b"\xb3)\x01\x0c\x03foo\x08U_\x1b"
CONSTANT_INPROGRESS = b"\xb3)\x02\x0c\x03foo\x8e\xc1-\xb5"
CONSTANT_SUCCESS = b"\xb3)\x03\x0c\x03fooE\x9d\xfe\x10"
CONSTANT_UXSUCCESS = b"\xb3)\x04\x0c\x03fooX\x98\xce\xa8"
CONSTANT_SKIP = b"\xb3)\x05\x0c\x03foo\x93\xc4\x1d\r"
CONSTANT_FAIL = b"\xb3)\x06\x0c\x03foo\x15Po\xa3"
CONSTANT_XFAIL = b"\xb3)\x07\x0c\x03foo\xde\x0c\xbc\x06"
CONSTANT_EOF = b"\xb3!\x10\x08S\x15\x88\xdc"
CONSTANT_FILE_CONTENT = b"\xb3!@\x13\x06barney\x03wooA5\xe3\x8c"
CONSTANT_MIME = b"\xb3! #\x1aapplication/foo; charset=1x3Q\x15"
CONSTANT_TIMESTAMP = b"\xb3+\x03\x13<\x17T\xcf\x80\xaf\xc8\x03barI\x96>-"
CONSTANT_ROUTE_CODE = b"\xb3-\x03\x13\x03bar\x06source\x9cY9\x19"
CONSTANT_RUNNABLE = b"\xb3(\x03\x0c\x03foo\xe3\xea\xf5\xa4"
CONSTANT_TAGS = [
b"\xb3)\x80\x15\x03bar\x02\x03foo\x03barTHn\xb4",
b"\xb3)\x80\x15\x03bar\x02\x03bar\x03foo\xf8\xf1\x91o",
]
if TestStreamResultContract is not None:
class TestStreamResultToBytesContract(TestCase, TestStreamResultContract):
"""Check that StreamResult behaves as testtools expects."""
def _make_result(self):
return subunit.StreamResultToBytes(BytesIO())
class TestStreamResultToBytes(TestCase):
def _make_result(self):
output = BytesIO()
return subunit.StreamResultToBytes(output), output
def test_numbers(self):
result = subunit.StreamResultToBytes(BytesIO())
packet = []
self.assertRaises(Exception, result._write_number, -1, packet)
self.assertEqual([], packet)
result._write_number(0, packet)
self.assertEqual([b"\x00"], packet)
del packet[:]
result._write_number(63, packet)
self.assertEqual([b"\x3f"], packet)
del packet[:]
result._write_number(64, packet)
self.assertEqual([b"\x40\x40"], packet)
del packet[:]
result._write_number(16383, packet)
self.assertEqual([b"\x7f\xff"], packet)
del packet[:]
result._write_number(16384, packet)
self.assertEqual([b"\x80\x40", b"\x00"], packet)
del packet[:]
result._write_number(4194303, packet)
self.assertEqual([b"\xbf\xff", b"\xff"], packet)
del packet[:]
result._write_number(4194304, packet)
self.assertEqual([b"\xc0\x40\x00\x00"], packet)
del packet[:]
result._write_number(1073741823, packet)
self.assertEqual([b"\xff\xff\xff\xff"], packet)
del packet[:]
self.assertRaises(Exception, result._write_number, 1073741824, packet)
self.assertEqual([], packet)
def test_volatile_length(self):
# if the length of the packet data before the length itself is
# considered is right on the boundary for length's variable length
# encoding, it is easy to get the length wrong by not accounting for
# length itself.
# that is, the encoder has to ensure that length == sum (length_of_rest
# + length_of_length)
result, output = self._make_result()
# 1 byte short:
result.status(file_name="", file_bytes=b"\xff" * 0)
self.assertThat(output.getvalue(), HasLength(10))
self.assertEqual(b"\x0a", output.getvalue()[3:4])
output.seek(0)
output.truncate()
# 1 byte long:
result.status(file_name="", file_bytes=b"\xff" * 53)
self.assertThat(output.getvalue(), HasLength(63))
self.assertEqual(b"\x3f", output.getvalue()[3:4])
output.seek(0)
output.truncate()
# 2 bytes short
result.status(file_name="", file_bytes=b"\xff" * 54)
self.assertThat(output.getvalue(), HasLength(65))
self.assertEqual(b"\x40\x41", output.getvalue()[3:5])
output.seek(0)
output.truncate()
# 2 bytes long
result.status(file_name="", file_bytes=b"\xff" * 16371)
self.assertThat(output.getvalue(), HasLength(16383))
self.assertEqual(b"\x7f\xff", output.getvalue()[3:5])
output.seek(0)
output.truncate()
# 3 bytes short
result.status(file_name="", file_bytes=b"\xff" * 16372)
self.assertThat(output.getvalue(), HasLength(16385))
self.assertEqual(b"\x80\x40\x01", output.getvalue()[3:6])
output.seek(0)
output.truncate()
# 3 bytes long
result.status(file_name="", file_bytes=b"\xff" * 4194289)
self.assertThat(output.getvalue(), HasLength(4194303))
self.assertEqual(b"\xbf\xff\xff", output.getvalue()[3:6])
output.seek(0)
output.truncate()
self.assertRaises(Exception, result.status, file_name="", file_bytes=b"\xff" * 4194290)
def test_trivial_enumeration(self):
result, output = self._make_result()
result.status("foo", "exists")
self.assertEqual(CONSTANT_ENUM, output.getvalue())
def test_inprogress(self):
result, output = self._make_result()
result.status("foo", "inprogress")
self.assertEqual(CONSTANT_INPROGRESS, output.getvalue())
def test_success(self):
result, output = self._make_result()
result.status("foo", "success")
self.assertEqual(CONSTANT_SUCCESS, output.getvalue())
def test_uxsuccess(self):
result, output = self._make_result()
result.status("foo", "uxsuccess")
self.assertEqual(CONSTANT_UXSUCCESS, output.getvalue())
def test_skip(self):
result, output = self._make_result()
result.status("foo", "skip")
self.assertEqual(CONSTANT_SKIP, output.getvalue())
def test_fail(self):
result, output = self._make_result()
result.status("foo", "fail")
self.assertEqual(CONSTANT_FAIL, output.getvalue())
def test_xfail(self):
result, output = self._make_result()
result.status("foo", "xfail")
self.assertEqual(CONSTANT_XFAIL, output.getvalue())
def test_unknown_status(self):
result, output = self._make_result()
self.assertRaises(Exception, result.status, "foo", "boo")
self.assertEqual(b"", output.getvalue())
def test_eof(self):
result, output = self._make_result()
result.status(eof=True)
self.assertEqual(CONSTANT_EOF, output.getvalue())
def test_file_content(self):
result, output = self._make_result()
result.status(file_name="barney", file_bytes=b"woo")
self.assertEqual(CONSTANT_FILE_CONTENT, output.getvalue())
def test_mime(self):
result, output = self._make_result()
result.status(mime_type="application/foo; charset=1")
self.assertEqual(CONSTANT_MIME, output.getvalue())
def test_route_code(self):
result, output = self._make_result()
result.status(test_id="bar", test_status="success", route_code="source")
self.assertEqual(CONSTANT_ROUTE_CODE, output.getvalue())
def test_runnable(self):
result, output = self._make_result()
result.status("foo", "success", runnable=False)
self.assertEqual(CONSTANT_RUNNABLE, output.getvalue())
def test_tags(self):
result, output = self._make_result()
result.status(test_id="bar", test_tags={"foo", "bar"})
self.assertThat(CONSTANT_TAGS, Contains(output.getvalue()))
def test_timestamp(self):
timestamp = datetime.datetime(2001, 12, 12, 12, 59, 59, 45, iso8601.UTC)
result, output = self._make_result()
result.status(test_id="bar", test_status="success", timestamp=timestamp)
self.assertEqual(CONSTANT_TIMESTAMP, output.getvalue())
class TestByteStreamToStreamResult(TestCase):
def test_non_subunit_encapsulated(self):
source = BytesIO(b"foo\nbar\n")
result = StreamResult()
subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout").run(result)
self.assertEqual(
[
("status", None, None, None, True, "stdout", b"f", False, None, None, None),
("status", None, None, None, True, "stdout", b"o", False, None, None, None),
("status", None, None, None, True, "stdout", b"o", False, None, None, None),
("status", None, None, None, True, "stdout", b"\n", False, None, None, None),
("status", None, None, None, True, "stdout", b"b", False, None, None, None),
("status", None, None, None, True, "stdout", b"a", False, None, None, None),
("status", None, None, None, True, "stdout", b"r", False, None, None, None),
("status", None, None, None, True, "stdout", b"\n", False, None, None, None),
],
result._events,
)
self.assertEqual(b"", source.read())
def test_signature_middle_utf8_char(self):
utf8_bytes = b"\xe3\xb3\x8a"
source = BytesIO(utf8_bytes)
# Should be treated as one character (it is u'\u3cca') and wrapped
result = StreamResult()
subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout").run(result)
self.assertEqual(
[
("status", None, None, None, True, "stdout", b"\xe3", False, None, None, None),
("status", None, None, None, True, "stdout", b"\xb3", False, None, None, None),
("status", None, None, None, True, "stdout", b"\x8a", False, None, None, None),
],
result._events,
)
def test_non_subunit_disabled_raises(self):
source = BytesIO(b"foo\nbar\n")
result = StreamResult()
case = subunit.ByteStreamToStreamResult(source)
e = self.assertRaises(Exception, case.run, result)
self.assertEqual(b"f", e.args[1])
self.assertEqual(b"oo\nbar\n", source.read())
self.assertEqual([], result._events)
def test_trivial_enumeration(self):
source = BytesIO(CONSTANT_ENUM)
result = StreamResult()
subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout").run(result)
self.assertEqual(b"", source.read())
self.assertEqual(
[
("status", "foo", "exists", None, True, None, None, False, None, None, None),
],
result._events,
)
def test_multiple_events(self):
source = BytesIO(CONSTANT_ENUM + CONSTANT_ENUM)
result = StreamResult()
subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout").run(result)
self.assertEqual(b"", source.read())
self.assertEqual(
[
("status", "foo", "exists", None, True, None, None, False, None, None, None),
("status", "foo", "exists", None, True, None, None, False, None, None, None),
],
result._events,
)
def test_inprogress(self):
self.check_event(CONSTANT_INPROGRESS, "inprogress")
def test_success(self):
self.check_event(CONSTANT_SUCCESS, "success")
def test_uxsuccess(self):
self.check_event(CONSTANT_UXSUCCESS, "uxsuccess")
def test_skip(self):
self.check_event(CONSTANT_SKIP, "skip")
def test_fail(self):
self.check_event(CONSTANT_FAIL, "fail")
def test_xfail(self):
self.check_event(CONSTANT_XFAIL, "xfail")
def check_events(self, source_bytes, events):
source = BytesIO(source_bytes)
result = StreamResult()
subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout").run(result)
self.assertEqual(b"", source.read())
self.assertEqual(events, result._events)
# - any file attachments should be bytes equivalent [as users assume that].
for event in result._events:
if event[5] is not None:
bytes(event[6])
def check_event(
self,
source_bytes,
test_status=None,
test_id="foo",
route_code=None,
timestamp=None,
tags=None,
mime_type=None,
file_name=None,
file_bytes=None,
eof=False,
runnable=True,
):
event = self._event(
test_id=test_id,
test_status=test_status,
tags=tags,
runnable=runnable,
file_name=file_name,
file_bytes=file_bytes,
eof=eof,
mime_type=mime_type,
route_code=route_code,
timestamp=timestamp,
)
self.check_events(source_bytes, [event])
def _event(
self,
test_status=None,
test_id=None,
route_code=None,
timestamp=None,
tags=None,
mime_type=None,
file_name=None,
file_bytes=None,
eof=False,
runnable=True,
):
return (
"status",
test_id,
test_status,
tags,
runnable,
file_name,
file_bytes,
eof,
mime_type,
route_code,
timestamp,
)
def test_eof(self):
self.check_event(CONSTANT_EOF, test_id=None, eof=True)
def test_file_content(self):
self.check_event(CONSTANT_FILE_CONTENT, test_id=None, file_name="barney", file_bytes=b"woo")
def test_file_content_length_into_checksum(self):
# A bad file content length which creeps into the checksum.
bad_file_length_content = b"\xb3!@\x13\x06barney\x04woo\xdc\xe2\xdb\x35"
self.check_events(
bad_file_length_content,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=bad_file_length_content,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"File content extends past end of packet: claimed 4 bytes, 3 available",
mime_type="text/plain;charset=utf8",
),
],
)
def test_packet_length_4_word_varint(self):
packet_data = b"\xb3!@\xc0\x00\x11"
self.check_events(
packet_data,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=packet_data,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"3 byte maximum given but 4 byte value found.",
mime_type="text/plain;charset=utf8",
),
],
)
def test_mime(self):
self.check_event(CONSTANT_MIME, test_id=None, mime_type="application/foo; charset=1")
def test_route_code(self):
self.check_event(CONSTANT_ROUTE_CODE, "success", route_code="source", test_id="bar")
def test_runnable(self):
self.check_event(CONSTANT_RUNNABLE, test_status="success", runnable=False)
def test_tags(self):
self.check_event(CONSTANT_TAGS[0], None, tags={"foo", "bar"}, test_id="bar")
def test_timestamp(self):
timestamp = datetime.datetime(2001, 12, 12, 12, 59, 59, 45, iso8601.UTC)
self.check_event(CONSTANT_TIMESTAMP, "success", test_id="bar", timestamp=timestamp)
def test_bad_crc_errors_via_status(self):
file_bytes = CONSTANT_MIME[:-1] + b"\x00"
self.check_events(
file_bytes,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=file_bytes,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"Bad checksum - calculated (0x78335115), stored (0x78335100)",
mime_type="text/plain;charset=utf8",
),
],
)
def test_not_utf8_in_string(self):
file_bytes = CONSTANT_ROUTE_CODE[:5] + b"\xb4" + CONSTANT_ROUTE_CODE[6:-4] + b"\xce\x56\xc6\x17"
self.check_events(
file_bytes,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=file_bytes,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"UTF8 string at offset 2 is not UTF8",
mime_type="text/plain;charset=utf8",
),
],
)
def test_NULL_in_string(self):
file_bytes = CONSTANT_ROUTE_CODE[:6] + b"\x00" + CONSTANT_ROUTE_CODE[7:-4] + b"\xd7\x41\xac\xfe"
self.check_events(
file_bytes,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=file_bytes,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"UTF8 string at offset 2 contains NUL byte",
mime_type="text/plain;charset=utf8",
),
],
)
def test_bad_utf8_stringlength(self):
file_bytes = CONSTANT_ROUTE_CODE[:4] + b"\x3f" + CONSTANT_ROUTE_CODE[5:-4] + b"\xbe\x29\xe0\xc2"
self.check_events(
file_bytes,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=file_bytes,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"UTF8 string at offset 2 extends past end of packet: claimed 63 bytes, 10 available",
mime_type="text/plain;charset=utf8",
),
],
)
def test_route_code_and_file_content(self):
content = BytesIO()
subunit.StreamResultToBytes(content).status(
route_code="0", mime_type="text/plain", file_name="bar", file_bytes=b"foo"
)
self.check_event(
content.getvalue(), test_id=None, file_name="bar", route_code="0", mime_type="text/plain", file_bytes=b"foo"
)
if st is not None:
@given(st.binary())
def test_hypothesis_decoding(self, code_bytes):
source = BytesIO(code_bytes)
result = StreamResult()
stream = subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout")
stream.run(result)
self.assertEqual(b"", source.read())
+558
python/tests/test_test_results.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2009 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import csv
import datetime
import sys
import unittest
from io import StringIO
import testtools
from testtools import TestCase
from testtools.content import TracebackContent, text_content
from testtools.testresult.doubles import ExtendedTestResult
import subunit
import iso8601
import subunit.test_results
class LoggingDecorator(subunit.test_results.HookedTestResultDecorator):
def __init__(self, decorated):
self._calls = 0
super().__init__(decorated)
def _before_event(self):
self._calls += 1
class AssertBeforeTestResult(LoggingDecorator):
"""A TestResult for checking preconditions."""
def __init__(self, decorated, test):
self.test = test
super().__init__(decorated)
def _before_event(self):
self.test.assertEqual(1, self.earlier._calls)
super()._before_event()
class TimeCapturingResult(unittest.TestResult):
def __init__(self):
super().__init__()
self._calls = []
self.failfast = False
def time(self, a_datetime):
self._calls.append(a_datetime)
def progress(self, offset, whence):
pass
def addDuration(self, test, duration):
"""Called to add a test duration.
:param test: The test that completed.
:param duration: The duration of the test as a float in seconds.
"""
pass
class TestHookedTestResultDecorator(unittest.TestCase):
def setUp(self):
# An end to the chain
terminal = unittest.TestResult()
# Asserts that the call was made to self.result before asserter was
# called.
asserter = AssertBeforeTestResult(terminal, self)
# The result object we call, which much increase its call count.
self.result = LoggingDecorator(asserter)
asserter.earlier = self.result
self.decorated = asserter
def tearDown(self):
# The hook in self.result must have been called
self.assertEqual(1, self.result._calls)
# The hook in asserter must have been called too, otherwise the
# assertion about ordering won't have completed.
self.assertEqual(1, self.decorated._calls)
def test_startTest(self):
self.result.startTest(self)
def test_startTestRun(self):
self.result.startTestRun()
def test_stopTest(self):
self.result.stopTest(self)
def test_stopTestRun(self):
self.result.stopTestRun()
def test_addError(self):
self.result.addError(self, subunit.RemoteError())
def test_addError_details(self):
self.result.addError(self, details={})
def test_addFailure(self):
self.result.addFailure(self, subunit.RemoteError())
def test_addFailure_details(self):
self.result.addFailure(self, details={})
def test_addSuccess(self):
self.result.addSuccess(self)
def test_addSuccess_details(self):
self.result.addSuccess(self, details={})
def test_addSkip(self):
self.result.addSkip(self, "foo")
def test_addSkip_details(self):
self.result.addSkip(self, details={})
def test_addExpectedFailure(self):
self.result.addExpectedFailure(self, subunit.RemoteError())
def test_addExpectedFailure_details(self):
self.result.addExpectedFailure(self, details={})
def test_addUnexpectedSuccess(self):
self.result.addUnexpectedSuccess(self)
def test_addUnexpectedSuccess_details(self):
self.result.addUnexpectedSuccess(self, details={})
def test_progress(self):
self.result.progress(1, subunit.PROGRESS_SET)
def test_wasSuccessful(self):
self.result.wasSuccessful()
def test_shouldStop(self):
self.result.shouldStop
def test_stop(self):
self.result.stop()
def test_time(self):
self.result.time(None)
def test_addDuration(self):
self.result.addDuration(self, 1.5)
class TestAutoTimingTestResultDecorator(unittest.TestCase):
def setUp(self):
# And end to the chain which captures time events.
terminal = TimeCapturingResult()
# The result object under test.
self.result = subunit.test_results.AutoTimingTestResultDecorator(terminal)
self.decorated = terminal
def test_without_time_calls_time_is_called_and_not_None(self):
self.result.startTest(self)
self.assertEqual(1, len(self.decorated._calls))
self.assertNotEqual(None, self.decorated._calls[0])
def test_no_time_from_progress(self):
self.result.progress(1, subunit.PROGRESS_CUR)
self.assertEqual(0, len(self.decorated._calls))
def test_no_time_from_shouldStop(self):
self.decorated.stop()
self.result.shouldStop
self.assertEqual(0, len(self.decorated._calls))
def test_calling_time_inhibits_automatic_time(self):
# Calling time() outputs a time signal immediately and prevents
# automatically adding one when other methods are called.
time = datetime.datetime(2009, 10, 11, 12, 13, 14, 15, iso8601.UTC)
self.result.time(time)
self.result.startTest(self)
self.result.stopTest(self)
self.assertEqual(1, len(self.decorated._calls))
self.assertEqual(time, self.decorated._calls[0])
def test_calling_time_None_enables_automatic_time(self):
time = datetime.datetime(2009, 10, 11, 12, 13, 14, 15, iso8601.UTC)
self.result.time(time)
self.assertEqual(1, len(self.decorated._calls))
self.assertEqual(time, self.decorated._calls[0])
# Calling None passes the None through, in case other results care.
self.result.time(None)
self.assertEqual(2, len(self.decorated._calls))
self.assertEqual(None, self.decorated._calls[1])
# Calling other methods doesn't generate an automatic time event.
self.result.startTest(self)
self.assertEqual(3, len(self.decorated._calls))
self.assertNotEqual(None, self.decorated._calls[2])
def test_set_failfast_True(self):
self.assertFalse(self.decorated.failfast)
self.result.failfast = True
self.assertTrue(self.decorated.failfast)
class TestTagCollapsingDecorator(TestCase):
def test_tags_collapsed_outside_of_tests(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
tag_collapser.tags({"a"}, set())
tag_collapser.tags({"b"}, set())
tag_collapser.startTest(self)
self.assertEqual(
[
("tags", {"a", "b"}, set()),
("startTest", self),
],
result._events,
)
def test_tags_collapsed_outside_of_tests_are_flushed(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
tag_collapser.startTestRun()
tag_collapser.tags({"a"}, set())
tag_collapser.tags({"b"}, set())
tag_collapser.startTest(self)
tag_collapser.addSuccess(self)
tag_collapser.stopTest(self)
tag_collapser.stopTestRun()
self.assertEqual(
[
("startTestRun",),
("tags", {"a", "b"}, set()),
("startTest", self),
("addSuccess", self),
("stopTest", self),
("stopTestRun",),
],
result._events,
)
def test_tags_forwarded_after_tests(self):
test = subunit.RemotedTestCase("foo")
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
tag_collapser.startTestRun()
tag_collapser.startTest(test)
tag_collapser.addSuccess(test)
tag_collapser.stopTest(test)
tag_collapser.tags({"a"}, {"b"})
tag_collapser.stopTestRun()
self.assertEqual(
[
("startTestRun",),
("startTest", test),
("addSuccess", test),
("stopTest", test),
("tags", {"a"}, {"b"}),
("stopTestRun",),
],
result._events,
)
def test_tags_collapsed_inside_of_tests(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
test = subunit.RemotedTestCase("foo")
tag_collapser.startTest(test)
tag_collapser.tags({"a"}, set())
tag_collapser.tags({"b"}, {"a"})
tag_collapser.tags({"c"}, set())
tag_collapser.stopTest(test)
self.assertEqual([("startTest", test), ("tags", {"b", "c"}, {"a"}), ("stopTest", test)], result._events)
def test_tags_collapsed_inside_of_tests_different_ordering(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
test = subunit.RemotedTestCase("foo")
tag_collapser.startTest(test)
tag_collapser.tags(set(), {"a"})
tag_collapser.tags({"a", "b"}, set())
tag_collapser.tags({"c"}, set())
tag_collapser.stopTest(test)
self.assertEqual([("startTest", test), ("tags", {"a", "b", "c"}, set()), ("stopTest", test)], result._events)
def test_tags_sent_before_result(self):
# Because addSuccess and friends tend to send subunit output
# immediately, and because 'tags:' before a result line means
# something different to 'tags:' after a result line, we need to be
# sure that tags are emitted before 'addSuccess' (or whatever).
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
test = subunit.RemotedTestCase("foo")
tag_collapser.startTest(test)
tag_collapser.tags({"a"}, set())
tag_collapser.addSuccess(test)
tag_collapser.stopTest(test)
self.assertEqual(
[("startTest", test), ("tags", {"a"}, set()), ("addSuccess", test), ("stopTest", test)], result._events
)
class TestTimeCollapsingDecorator(TestCase):
def make_time(self):
# Heh heh.
return datetime.datetime(2000, 1, self.getUniqueInteger(), tzinfo=iso8601.UTC)
def test_initial_time_forwarded(self):
# We always forward the first time event we see.
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
a_time = self.make_time()
tag_collapser.time(a_time)
self.assertEqual([("time", a_time)], result._events)
def test_time_collapsed_to_first_and_last(self):
# If there are many consecutive time events, only the first and last
# are sent through.
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
times = [self.make_time() for i in range(5)]
for a_time in times:
tag_collapser.time(a_time)
tag_collapser.startTest(subunit.RemotedTestCase("foo"))
self.assertEqual([("time", times[0]), ("time", times[-1])], result._events[:-1])
def test_only_one_time_sent(self):
# If we receive a single time event followed by a non-time event, we
# send exactly one time event.
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
a_time = self.make_time()
tag_collapser.time(a_time)
tag_collapser.startTest(subunit.RemotedTestCase("foo"))
self.assertEqual([("time", a_time)], result._events[:-1])
def test_duplicate_times_not_sent(self):
# Many time events with the exact same time are collapsed into one
# time event.
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
a_time = self.make_time()
for i in range(5):
tag_collapser.time(a_time)
tag_collapser.startTest(subunit.RemotedTestCase("foo"))
self.assertEqual([("time", a_time)], result._events[:-1])
def test_no_times_inserted(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
a_time = self.make_time()
tag_collapser.time(a_time)
foo = subunit.RemotedTestCase("foo")
tag_collapser.startTest(foo)
tag_collapser.addSuccess(foo)
tag_collapser.stopTest(foo)
self.assertEqual([("time", a_time), ("startTest", foo), ("addSuccess", foo), ("stopTest", foo)], result._events)
class TestByTestResultTests(testtools.TestCase):
def setUp(self):
super().setUp()
self.log = []
self.result = subunit.test_results.TestByTestResult(self.on_test)
self.result._now = iter(range(5)).__next__
def assertCalled(self, **kwargs):
defaults = {
"test": self,
"tags": set(),
"details": None,
"start_time": 0,
"stop_time": 1,
}
defaults.update(kwargs)
self.assertEqual([defaults], self.log)
def on_test(self, **kwargs):
self.log.append(kwargs)
def test_no_tests_nothing_reported(self):
self.result.startTestRun()
self.result.stopTestRun()
self.assertEqual([], self.log)
def test_add_success(self):
self.result.startTest(self)
self.result.addSuccess(self)
self.result.stopTest(self)
self.assertCalled(status="success")
def test_add_success_details(self):
self.result.startTest(self)
details = {"foo": "bar"}
self.result.addSuccess(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="success", details=details)
def test_tags(self):
if not getattr(self.result, "tags", None):
self.skipTest("No tags in testtools")
self.result.tags(["foo"], [])
self.result.startTest(self)
self.result.addSuccess(self)
self.result.stopTest(self)
self.assertCalled(status="success", tags={"foo"})
def test_add_error(self):
self.result.startTest(self)
try:
1 / 0
except ZeroDivisionError:
error = sys.exc_info()
self.result.addError(self, error)
self.result.stopTest(self)
self.assertCalled(status="error", details={"traceback": TracebackContent(error, self)})
def test_add_error_details(self):
self.result.startTest(self)
details = {"foo": text_content("bar")}
self.result.addError(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="error", details=details)
def test_add_failure(self):
self.result.startTest(self)
try:
self.fail("intentional failure")
except self.failureException:
failure = sys.exc_info()
self.result.addFailure(self, failure)
self.result.stopTest(self)
self.assertCalled(status="failure", details={"traceback": TracebackContent(failure, self)})
def test_add_failure_details(self):
self.result.startTest(self)
details = {"foo": text_content("bar")}
self.result.addFailure(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="failure", details=details)
def test_add_xfail(self):
self.result.startTest(self)
try:
1 / 0
except ZeroDivisionError:
error = sys.exc_info()
self.result.addExpectedFailure(self, error)
self.result.stopTest(self)
self.assertCalled(status="xfail", details={"traceback": TracebackContent(error, self)})
def test_add_xfail_details(self):
self.result.startTest(self)
details = {"foo": text_content("bar")}
self.result.addExpectedFailure(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="xfail", details=details)
def test_add_unexpected_success(self):
self.result.startTest(self)
details = {"foo": "bar"}
self.result.addUnexpectedSuccess(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="success", details=details)
def test_add_skip_reason(self):
self.result.startTest(self)
reason = self.getUniqueString()
self.result.addSkip(self, reason)
self.result.stopTest(self)
self.assertCalled(status="skip", details={"reason": text_content(reason)})
def test_add_skip_details(self):
self.result.startTest(self)
details = {"foo": "bar"}
self.result.addSkip(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="skip", details=details)
def test_twice(self):
self.result.startTest(self)
self.result.addSuccess(self, details={"foo": "bar"})
self.result.stopTest(self)
self.result.startTest(self)
self.result.addSuccess(self)
self.result.stopTest(self)
self.assertEqual(
[
{
"test": self,
"status": "success",
"start_time": 0,
"stop_time": 1,
"tags": set(),
"details": {"foo": "bar"},
},
{"test": self, "status": "success", "start_time": 2, "stop_time": 3, "tags": set(), "details": None},
],
self.log,
)
class TestCsvResult(testtools.TestCase):
def parse_stream(self, stream):
stream.seek(0)
reader = csv.reader(stream)
return list(reader)
def test_csv_output(self):
stream = StringIO()
result = subunit.test_results.CsvResult(stream)
result._now = iter(range(5)).__next__
result.startTestRun()
result.startTest(self)
result.addSuccess(self)
result.stopTest(self)
result.stopTestRun()
self.assertEqual(
[
["test", "status", "start_time", "stop_time"],
[self.id(), "success", "0", "1"],
],
self.parse_stream(stream),
)
def test_just_header_when_no_tests(self):
stream = StringIO()
result = subunit.test_results.CsvResult(stream)
result.startTestRun()
result.stopTestRun()
self.assertEqual([["test", "status", "start_time", "stop_time"]], self.parse_stream(stream))
def test_no_output_before_events(self):
stream = StringIO()
subunit.test_results.CsvResult(stream)
self.assertEqual([], self.parse_stream(stream))
class TestTestIdPrintingResult(testtools.TestCase):
def setUp(self):
super().setUp()
self.stream = StringIO()
self.result = subunit.test_results.TestIdPrintingResult(self.stream)
def test_addDuration(self):
# addDuration should not raise an exception
test = subunit.RemotedTestCase("foo")
self.result.addDuration(test, 2.5)
# TestIdPrintingResult doesn't output anything for addDuration
self.assertEqual("", self.stream.getvalue())
+1
-1
COPYING

@@ -32,4 +32,4 @@ Subunit is licensed under two licenses, the Apache License, Version 2.0 or the

A known list of such code is maintained here:
* The runtests.py and python/subunit/tests/TestUtil.py module are GPL test
* The runtests.py and python/tests/TestUtil.py module are GPL test
support modules. They are not installed by Subunit - they are only ever
used on the build machine. Copyright 2004 Canonical Limited.
+1
-0
MANIFEST.in

@@ -21,1 +21,2 @@ exclude .gitignore

include NEWS
recursive-include python/tests *.py
+33
-0
NEWS

@@ -5,2 +5,35 @@ ---------------------

1.4.6 (2026-05-04)
---------------------
BUG FIXES
~~~~~~~~~
* Fix compatibility with testtools 2.8.2. (Jelmer Vernooij)
* Fix ``ExecTestCase`` to work without executable permissions on the
test script. (Jelmer Vernooij)
* Fix ``ExecTestCase`` to properly handle shell scripts. (Jelmer Vernooij)
IMPROVEMENTS
~~~~~~~~~~~~
* Stop installing tests. (Jelmer Vernooij)
* Add ``subunit-combine`` script that runs multiple subunit-producing
commands and merges their streams, optionally prefixing each command's
test ids. (Jelmer Vernooij, #2150097)
* ``subunit-combine`` supports testr-style ``$LISTOPT``, ``$IDOPTION``,
``$IDFILE`` and ``$IDLIST`` placeholders in command argv, along with
``--list``, ``--load-list`` and positional test ids for per-command
filtering. (Jelmer Vernooij)
* Add ``gojson2subunit`` script for converting ``go test -json`` streams
to subunit. (Jelmer Vernooij)
* Add ``junitxml2subunit`` script for converting JUnit XML test reports
to subunit. (Jelmer Vernooij)
1.4.5 (2025-11-10)

@@ -7,0 +40,0 @@ ---------------------

+13
-8
PKG-INFO
Metadata-Version: 2.4
Name: python-subunit
Version: 1.4.5
Version: 1.4.6
Summary: Python implementation of subunit test streaming protocol

@@ -24,2 +24,3 @@ Author-email: Robert Collins <subunit-dev@lists.launchpad.net>

Requires-Dist: iso8601
Requires-Dist: PyYAML
Requires-Dist: testtools>=2.7

@@ -153,2 +154,7 @@ Provides-Extra: docs

### perl
There is a perl module [Test::Subunit](https://metacpan.org/pod/Test::Subunit)
that can be used to emit and consume subunit v1 streams.
#### Filter recipes

@@ -510,8 +516,7 @@

* Update versions in configure.ac and python/subunit/__init__.py.
* Update NEWS.
* Do a make distcheck, which will update Makefile etc.
* Do a PyPI release: PYTHONPATH=python python setup.py sdist bdist_wheel; twine upload -s dist/*
* Upload the regular one to LP.
* Push a tagged commit.
git push -t origin master:master
* Update versions in `configure.ac` and `python/subunit/__init__.py`.
* Update `NEWS`.
* Do a ``make distcheck``, which will update `Makefile` etc.
* Push a tagged commit — this triggers the `publish.yml` GitHub Actions workflow,
which publishes to PyPI automatically via trusted publishing (OIDC), no API token needed:
``git push --tags origin master:master``
+13
-1
pyproject.toml

@@ -8,2 +8,11 @@ [build-system]

[[tool.mypy.overrides]]
module = [
"gi",
"gi.repository",
"junitxml",
"testscenarios",
]
ignore_missing_imports = true
[project]

@@ -24,3 +33,3 @@ authors = [

]
dependencies = ["iso8601", "testtools>=2.7"]
dependencies = ["iso8601", "PyYAML", "testtools>=2.7"]
description = "Python implementation of subunit test streaming protocol"

@@ -50,2 +59,3 @@ dynamic = ["version"]

"subunit-2to1" = "subunit.filter_scripts.subunit_2to1:main"
"subunit-combine" = "subunit.filter_scripts.subunit_combine:main"
"subunit-filter" = "subunit.filter_scripts.subunit_filter:main"

@@ -62,2 +72,4 @@ "subunit-ls" = "subunit.filter_scripts.subunit_ls:main"

"subunit2pyunit" = "subunit.filter_scripts.subunit2pyunit:main"
"gojson2subunit" = "subunit.filter_scripts.gojson2subunit:main"
"junitxml2subunit" = "subunit.filter_scripts.junitxml2subunit:main"
"tap2subunit" = "subunit.filter_scripts.tap2subunit:main"

@@ -64,0 +76,0 @@ 

+3
-0
python/python_subunit.egg-info/entry_points.txt
[console_scripts]
gojson2subunit = subunit.filter_scripts.gojson2subunit:main
junitxml2subunit = subunit.filter_scripts.junitxml2subunit:main
subunit-1to2 = subunit.filter_scripts.subunit_1to2:main
subunit-2to1 = subunit.filter_scripts.subunit_2to1:main
subunit-combine = subunit.filter_scripts.subunit_combine:main
subunit-filter = subunit.filter_scripts.subunit_filter:main

@@ -5,0 +8,0 @@ subunit-ls = subunit.filter_scripts.subunit_ls:main

+13
-8
python/python_subunit.egg-info/PKG-INFO
Metadata-Version: 2.4
Name: python-subunit
Version: 1.4.5
Version: 1.4.6
Summary: Python implementation of subunit test streaming protocol

@@ -24,2 +24,3 @@ Author-email: Robert Collins <subunit-dev@lists.launchpad.net>

Requires-Dist: iso8601
Requires-Dist: PyYAML
Requires-Dist: testtools>=2.7

@@ -153,2 +154,7 @@ Provides-Extra: docs

### perl
There is a perl module [Test::Subunit](https://metacpan.org/pod/Test::Subunit)
that can be used to emit and consume subunit v1 streams.
#### Filter recipes

@@ -510,8 +516,7 @@

* Update versions in configure.ac and python/subunit/__init__.py.
* Update NEWS.
* Do a make distcheck, which will update Makefile etc.
* Do a PyPI release: PYTHONPATH=python python setup.py sdist bdist_wheel; twine upload -s dist/*
* Upload the regular one to LP.
* Push a tagged commit.
git push -t origin master:master
* Update versions in `configure.ac` and `python/subunit/__init__.py`.
* Update `NEWS`.
* Do a ``make distcheck``, which will update `Makefile` etc.
* Push a tagged commit — this triggers the `publish.yml` GitHub Actions workflow,
which publishes to PyPI automatically via trusted publishing (OIDC), no API token needed:
``git push --tags origin master:master``
+1
-0
python/python_subunit.egg-info/requires.txt
iso8601
PyYAML
testtools>=2.7

@@ -3,0 +4,0 @@ 

+23
-28
python/python_subunit.egg-info/SOURCES.txt

@@ -1,5 +0,1 @@

.codespellrc
.git-blame-ignore-revs
Apache-2.0
BSD
COPYING

@@ -9,8 +5,4 @@ MANIFEST.in

README.md
all_tests.py
disperse.toml
pyproject.toml
setup.py
.github/dependabot.yaml
.github/workflows/main.yml
python/python_subunit.egg-info/PKG-INFO

@@ -33,2 +25,4 @@ python/python_subunit.egg-info/SOURCES.txt

python/subunit/filter_scripts/__init__.py
python/subunit/filter_scripts/gojson2subunit.py
python/subunit/filter_scripts/junitxml2subunit.py
python/subunit/filter_scripts/subunit2csv.py

@@ -41,2 +35,3 @@ python/subunit/filter_scripts/subunit2disk.py

python/subunit/filter_scripts/subunit_2to1.py
python/subunit/filter_scripts/subunit_combine.py
python/subunit/filter_scripts/subunit_filter.py

@@ -49,21 +44,21 @@ python/subunit/filter_scripts/subunit_ls.py

python/subunit/filter_scripts/tap2subunit.py
python/subunit/tests/__init__.py
python/subunit/tests/sample-script.py
python/subunit/tests/sample-two-script.py
python/subunit/tests/test_chunked.py
python/subunit/tests/test_details.py
python/subunit/tests/test_filter_to_disk.py
python/subunit/tests/test_filters.py
python/subunit/tests/test_output_filter.py
python/subunit/tests/test_progress_model.py
python/subunit/tests/test_run.py
python/subunit/tests/test_subunit_filter.py
python/subunit/tests/test_subunit_stats.py
python/subunit/tests/test_subunit_tags.py
python/subunit/tests/test_tap2subunit.py
python/subunit/tests/test_test_protocol.py
python/subunit/tests/test_test_protocol2.py
python/subunit/tests/test_test_results.py
vim/README.md
vim/ftdetect/subunit.vim
vim/syntax/subunit.vim
python/tests/__init__.py
python/tests/sample-script.py
python/tests/sample-two-script.py
python/tests/test_chunked.py
python/tests/test_details.py
python/tests/test_filter_to_disk.py
python/tests/test_filters.py
python/tests/test_gojson2subunit.py
python/tests/test_junitxml2subunit.py
python/tests/test_output_filter.py
python/tests/test_progress_model.py
python/tests/test_run.py
python/tests/test_subunit_combine.py
python/tests/test_subunit_filter.py
python/tests/test_subunit_stats.py
python/tests/test_subunit_tags.py
python/tests/test_tap2subunit.py
python/tests/test_test_protocol.py
python/tests/test_test_protocol2.py
python/tests/test_test_results.py
+302
-7
python/subunit/__init__.py

@@ -147,3 +147,3 @@ #

__version__ = (1, 4, 5, "final", 0)
__version__ = (1, 4, 6, "final", 0)

@@ -179,8 +179,2 @@ version_string = ".".join(map(str, __version__[:3]))

def test_suite():
import subunit.tests
return subunit.tests.test_suite()
def join_dir(base_path, path):

@@ -1124,2 +1118,303 @@ """

def GoJSON2SubUnit(gojson, output_stream):
"""Filter a `go test -json` stream into a subunit v2 byte stream.
`go test -json` (or `go tool test2json`) emits one JSON object per line.
Each object carries an `Action` (`run`, `output`, `pass`, `fail`, `skip`,
`pause`, `cont`, `bench`, `start`), a `Package`, an optional `Test`, and
on terminal events an `Elapsed` and `Time`. This function maps each
test's lifecycle to a pair of subunit packets — `inprogress` at the
`run` event and the final status at the terminal event — so the
consumer can derive a duration. Captured `output` lines for the test
are folded into a single `text/plain; charset=UTF8` attachment on the
terminal packet.
Test IDs are formed as ``<package>.<TestName>``. Subtests keep Go's
native ``Parent/Sub`` form, so the resulting ID is
``<package>.<Parent>/<Sub>``, which round-trips through
``go test -run '^Parent$/^Sub$'``.
Package-level failures (no `Test` field on a `fail` event — typically
a build error) are reported as a synthetic ``<package> [build]`` test
so the failure shows up alongside the test results instead of being
silently swallowed.
:param gojson: An iterable of text lines (e.g. ``sys.stdin``) carrying
the `go test -json` stream.
:param output_stream: A binary stream to write subunit v2 bytes to.
:return: 0 if no test failed, 1 otherwise — matching the convention
used by `TAP2SubUnit`.
"""
import json
output = StreamResultToBytes(output_stream)
UTF8_TEXT = "text/plain; charset=UTF8"
# Per-test buffered `output` chunks plus the start timestamp captured
# on the `run` event, keyed by full test_id.
buffers = {}
start_times = {}
# Per-package buffered output, used to attribute build / setup failures
# that don't carry a `Test` field.
pkg_buffers = {}
any_failed = False
def parse_time(value):
if not value:
return None
try:
return iso8601.parse_date(value)
except (TypeError, ValueError, iso8601.ParseError):
return None
def make_test_id(pkg, test):
# Both package and test are required for an unambiguous ID; the
# caller checks for `Test` before reaching here.
return "{}.{}".format(pkg, test)
for line in gojson:
line = line.strip()
if not line:
continue
try:
event = json.loads(line)
except (TypeError, ValueError):
# `go test -json` occasionally interleaves a non-JSON banner
# (e.g. on a panic during package init). Drop it rather than
# aborting the whole stream.
continue
if not isinstance(event, dict):
continue
action = event.get("Action")
pkg = event.get("Package") or ""
test = event.get("Test")
timestamp = parse_time(event.get("Time"))
if action == "output":
chunk = event.get("Output", "")
if test:
buffers.setdefault(make_test_id(pkg, test), []).append(chunk)
elif pkg:
pkg_buffers.setdefault(pkg, []).append(chunk)
continue
if action == "run" and test:
test_id = make_test_id(pkg, test)
start_times[test_id] = timestamp
output.status(
test_id=test_id,
test_status="inprogress",
timestamp=timestamp,
)
continue
if action in ("pass", "fail", "skip") and test:
status = {"pass": "success", "fail": "fail", "skip": "skip"}[action]
test_id = make_test_id(pkg, test)
chunks = buffers.pop(test_id, [])
start_times.pop(test_id, None)
file_bytes = ("".join(chunks)).encode("utf-8") if chunks else None
output.status(
test_id=test_id,
test_status=status,
eof=True,
file_name="go test output" if file_bytes else None,
file_bytes=file_bytes,
mime_type=UTF8_TEXT if file_bytes else None,
timestamp=timestamp,
)
if action == "fail":
any_failed = True
continue
if action == "fail" and not test and pkg:
# Package-level failure (build error, init panic, etc.). Emit
# a synthetic test so the failure is visible.
chunks = pkg_buffers.pop(pkg, [])
file_bytes = ("".join(chunks)).encode("utf-8") if chunks else None
output.status(
test_id="{} [build]".format(pkg),
test_status="fail",
eof=True,
file_name="go test output" if file_bytes else None,
file_bytes=file_bytes,
mime_type=UTF8_TEXT if file_bytes else None,
timestamp=timestamp,
)
any_failed = True
continue
# `pass`/`skip` without `Test` is a package-level summary; harmless
# to drop. `pause`/`cont`/`start`/`bench` aren't terminal — skip.
# Any tests still in-progress at EOF were aborted (the runner died
# mid-test). Surface them as failures so they're not silently lost.
for test_id, chunks in list(buffers.items()):
file_bytes = ("".join(chunks)).encode("utf-8") if chunks else None
output.status(
test_id=test_id,
test_status="fail",
eof=True,
file_name="go test output" if file_bytes else None,
file_bytes=file_bytes,
mime_type=UTF8_TEXT if file_bytes else None,
)
any_failed = True
return 1 if any_failed else 0
def JUnitXML2SubUnit(xml_files, output_stream):
"""Convert JUnit XML test reports to a subunit v2 byte stream.
Reads each path in ``xml_files`` (in the supplied order) and emits
one subunit packet pair per ``<testcase>`` element. The packet pair
is ``inprogress`` followed by the terminal status, with synthetic
timestamps spaced by the testcase's ``time`` attribute so consumers
can recover the recorded duration.
Test IDs are formed as ``<classname>::<name>`` from the testcase's
``classname`` and ``name`` attributes. Maven Surefire and Gradle
both populate ``classname`` with the fully-qualified Java class
(e.g. ``com.example.FooTest``), so the resulting ID is
``com.example.FooTest::testBar``.
Mapping rules:
* ``<failure>`` child → status ``fail`` (an assertion failure).
* ``<error>`` child → status ``fail`` (an unexpected exception);
subunit doesn't distinguish these and the JUnit author intent is
identical for our purposes (both mean "did not pass").
* ``<skipped>`` child → status ``skip``.
* Otherwise → status ``success``.
The body text and ``message``/``type`` attributes of the failure
element are folded into a single ``text/plain`` attachment on the
terminal packet, mirroring how ``GoJSON2SubUnit`` attaches captured
stdout to its results.
Per-class ``<system-out>`` / ``<system-err>`` blocks aren't attributed
to individual testcases by the JUnit schema (they cover the whole
suite). They're dropped — preserving them would require attaching
them to a synthetic suite-level packet, and most consumers don't
surface that.
:param xml_files: Iterable of file paths containing JUnit XML.
:param output_stream: A binary stream to write subunit v2 bytes to.
:return: 0 if no testcase failed or errored, 1 otherwise. Files that
fail to parse are reported on stderr and counted as a failure so
the broken XML doesn't get silently swallowed.
"""
import datetime
import xml.etree.ElementTree as ET
output = StreamResultToBytes(output_stream)
UTF8_TEXT = "text/plain; charset=UTF8"
any_failed = False
# Synthetic timestamps. We don't know when the JUnit run actually
# happened, but spacing the inprogress/terminal packets by each
# testcase's recorded `time` attribute lets consumers compute the
# right duration without making up wall-clock data.
clock = datetime.datetime(2000, 1, 1, tzinfo=datetime.timezone.utc)
def parse_time(value):
if value is None:
return 0.0
try:
return float(value)
except (TypeError, ValueError):
return 0.0
def iter_testsuites(root):
# JUnit XML files come in two shapes: a single ``<testsuite>``
# at the root, or a ``<testsuites>`` wrapper containing many.
if root.tag == "testsuite":
yield root
elif root.tag == "testsuites":
for ts in root.findall("testsuite"):
yield ts
# Anything else is silently ignored — a non-JUnit document.
for path in xml_files:
try:
tree = ET.parse(path)
except (OSError, ET.ParseError) as exc:
sys.stderr.write("JUnitXML2SubUnit: failed to parse {}: {}\n".format(path, exc))
any_failed = True
continue
root = tree.getroot()
for suite in iter_testsuites(root):
for case in suite.findall("testcase"):
classname = case.get("classname") or ""
name = case.get("name") or ""
if not name:
# Without a name there's no usable test_id; skip
# rather than emit a malformed ID.
continue
test_id = "{}::{}".format(classname, name) if classname else name
duration = parse_time(case.get("time"))
failure = case.find("failure")
error = case.find("error")
skipped = case.find("skipped")
if failure is not None or error is not None:
status = "fail"
detail = failure if failure is not None else error
file_bytes = _format_junit_detail(detail)
any_failed = True
elif skipped is not None:
status = "skip"
file_bytes = _format_junit_detail(skipped)
else:
status = "success"
file_bytes = None
start_ts = clock
end_ts = clock + datetime.timedelta(seconds=duration)
clock = end_ts
output.status(
test_id=test_id,
test_status="inprogress",
timestamp=start_ts,
)
output.status(
test_id=test_id,
test_status=status,
eof=True,
file_name="junit detail" if file_bytes else None,
file_bytes=file_bytes,
mime_type=UTF8_TEXT if file_bytes else None,
timestamp=end_ts,
)
return 1 if any_failed else 0
def _format_junit_detail(element):
"""Serialise a ``<failure>``/``<error>``/``<skipped>`` body to bytes.
JUnit elements carry the message and exception type as attributes and
the stack trace as element text. Combine both into a single
text/plain blob so the consumer sees everything on one packet. Returns
``None`` when there's nothing to attach (an empty ``<skipped/>``).
"""
parts = []
msg = element.get("message")
typ = element.get("type")
if typ and msg:
parts.append("{}: {}".format(typ, msg))
elif typ:
parts.append(typ)
elif msg:
parts.append(msg)
body = (element.text or "").strip()
if body:
parts.append(body)
if not parts:
return None
return ("\n".join(parts) + "\n").encode("utf-8")
def tag_stream(original, filtered, tags):

@@ -1126,0 +1421,0 @@ """Alter tags on a stream.

+1
-1
python/subunit/v2.py

@@ -51,3 +51,3 @@ #

# Contains True for types for which 'nul in thing' falsely returns false.
_nul_test_broken = {}
_nul_test_broken: dict[type, bool] = {}

@@ -54,0 +54,0 @@ 

+11
-7
README.md

@@ -120,2 +120,7 @@

### perl
There is a perl module [Test::Subunit](https://metacpan.org/pod/Test::Subunit)
that can be used to emit and consume subunit v1 streams.
#### Filter recipes

@@ -477,8 +482,7 @@

* Update versions in configure.ac and python/subunit/__init__.py.
* Update NEWS.
* Do a make distcheck, which will update Makefile etc.
* Do a PyPI release: PYTHONPATH=python python setup.py sdist bdist_wheel; twine upload -s dist/*
* Upload the regular one to LP.
* Push a tagged commit.
git push -t origin master:master
* Update versions in `configure.ac` and `python/subunit/__init__.py`.
* Update `NEWS`.
* Do a ``make distcheck``, which will update `Makefile` etc.
* Push a tagged commit — this triggers the `publish.yml` GitHub Actions workflow,
which publishes to PyPI automatically via trusted publishing (OIDC), no API token needed:
``git push --tags origin master:master``
.codespellrc

Sorry, the diff of this file is not supported yet

.git-blame-ignore-revs

Sorry, the diff of this file is not supported yet

-13
.github/dependabot.yaml
# Please see the documentation for all configuration options:
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: weekly
- package-ecosystem: "pip"
directory: "/"
schedule:
interval: weekly
-43
.github/workflows/main.yml
name: Tests
on:
push:
branches: [master]
pull_request:
branches: [master]
jobs:
tests:
name: tests-python${{ matrix.python-version }}-${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "pypy3.10"]
os: ["ubuntu-latest"]
steps:
- uses: actions/checkout@v5
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install Deps
run: sudo apt-get install check libcppunit-dev
- name: Install package
run: |
python -m pip install ruff
python -m pip install -U '.[test,docs]'
- name: Build
run: autoreconf -fi && ./configure && make
- name: Run ruff check
run: ruff check python
- name: Format
run: ruff format --check .
- name: Run make check
run: make check
- name: Run make distcheck
run: make distcheck
# Disabled; needs a tuit or two
# - name: Docs build
# run: md2html.py README.md README.html
# if: "matrix.python-version == '3.8'"
# - name: Docs build
# run: md2html README.md README.html
# if: "matrix.python-version != '3.8'"
-35
all_tests.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2013 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import unittest
import subunit
class ShellTests(subunit.ExecTestCase):
def test_sourcing(self):
"""./shell/tests/test_source_library.sh"""
def test_functions(self):
"""./shell/tests/test_function_output.sh"""
def test_suite():
result = unittest.TestSuite()
result.addTest(subunit.test_suite())
result.addTest(ShellTests("test_sourcing"))
result.addTest(ShellTests("test_functions"))
return result
Apache-2.0

Sorry, the diff of this file is not supported yet

BSD

Sorry, the diff of this file is not supported yet

-24
disperse.toml
tag-name = "$VERSION"
news-file = "NEWS"
verify-command = "autoreconf -i && ./configure && make distcheck"
tarball-location = []
release-timeout = 5
[[update_version]]
path = "configure.ac"
new-line = "m4_define([SUBUNIT_MAJOR_VERSION], [$MAJOR_VERSION])"
[[update_version]]
path = "configure.ac"
new-line = "m4_define([SUBUNIT_MINOR_VERSION], [$MINOR_VERSION])"
[[update_version]]
path = "configure.ac"
new-line = "m4_define([SUBUNIT_MICRO_VERSION], [$MICRO_VERSION])"
[[update_version]]
path = "python/subunit/__init__.py"
new-line = "__version__ = $STATUS_TUPLED_VERSION"
[launchpad]
project = "subunit"
-63
python/subunit/tests/__init__.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
from unittest import TestLoader
from testscenarios import generate_scenarios
# Before the test module imports to avoid circularity.
# For testing: different pythons have different str() implementations.
_remote_exception_repr = "testtools.testresult.real._StringException"
_remote_exception_repr_chunked = "34\r\n" + _remote_exception_repr + ": boo qux\n0\r\n"
_remote_exception_str = "Traceback (most recent call last):\ntesttools.testresult.real._StringException"
_remote_exception_str_chunked = "57\r\n" + _remote_exception_str + ": boo qux\n0\r\n"
from subunit.tests import ( # noqa: E402
test_chunked,
test_details,
test_filter_to_disk,
test_filters,
test_output_filter,
test_progress_model,
test_run,
test_subunit_filter,
test_subunit_stats,
test_subunit_tags,
test_tap2subunit,
test_test_protocol,
test_test_protocol2,
test_test_results,
)
def test_suite():
loader = TestLoader()
result = loader.loadTestsFromModule(test_chunked)
result.addTest(loader.loadTestsFromModule(test_details))
result.addTest(loader.loadTestsFromModule(test_filters))
result.addTest(loader.loadTestsFromModule(test_progress_model))
result.addTest(loader.loadTestsFromModule(test_test_results))
result.addTest(loader.loadTestsFromModule(test_test_protocol))
result.addTest(loader.loadTestsFromModule(test_test_protocol2))
result.addTest(loader.loadTestsFromModule(test_tap2subunit))
result.addTest(loader.loadTestsFromModule(test_filter_to_disk))
result.addTest(loader.loadTestsFromModule(test_subunit_filter))
result.addTest(loader.loadTestsFromModule(test_subunit_tags))
result.addTest(loader.loadTestsFromModule(test_subunit_stats))
result.addTest(loader.loadTestsFromModule(test_run))
result.addTests(generate_scenarios(loader.loadTestsFromModule(test_output_filter)))
return result
-24
python/subunit/tests/sample-script.py
#!/usr/bin/env python3
import sys
if sys.platform == "win32":
import msvcrt
import os
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
if len(sys.argv) == 2:
# subunit.tests.test_test_protocol.TestExecTestCase.test_sample_method_args
# uses this code path to be sure that the arguments were passed to
# sample-script.py
print("test fail")
print("error fail")
sys.exit(0)
print("test old mcdonald")
print("success old mcdonald")
print("test bing crosby")
print("failure bing crosby [")
print("foo.c:53:ERROR invalid state")
print("]")
print("test an error")
print("error an error")
sys.exit(0)
-8
python/subunit/tests/sample-two-script.py
#!/usr/bin/env python3
import sys
print("test old mcdonald")
print("success old mcdonald")
print("test bing crosby")
print("success bing crosby")
sys.exit(0)
-140
python/subunit/tests/test_chunked.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
# Copyright (C) 2011 Martin Pool <mbp@sourcefrog.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import unittest
from io import BytesIO
import subunit.chunked
class TestDecode(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.output = BytesIO()
self.decoder = subunit.chunked.Decoder(self.output)
def test_close_read_length_short_errors(self):
self.assertRaises(ValueError, self.decoder.close)
def test_close_body_short_errors(self):
self.assertEqual(None, self.decoder.write(b"2\r\na"))
self.assertRaises(ValueError, self.decoder.close)
def test_close_body_buffered_data_errors(self):
self.assertEqual(None, self.decoder.write(b"2\r"))
self.assertRaises(ValueError, self.decoder.close)
def test_close_after_finished_stream_safe(self):
self.assertEqual(None, self.decoder.write(b"2\r\nab"))
self.assertEqual(b"", self.decoder.write(b"0\r\n"))
self.decoder.close()
def test_decode_nothing(self):
self.assertEqual(b"", self.decoder.write(b"0\r\n"))
self.assertEqual(b"", self.output.getvalue())
def test_decode_serialised_form(self):
self.assertEqual(None, self.decoder.write(b"F\r\n"))
self.assertEqual(None, self.decoder.write(b"serialised\n"))
self.assertEqual(b"", self.decoder.write(b"form0\r\n"))
def test_decode_short(self):
self.assertEqual(b"", self.decoder.write(b"3\r\nabc0\r\n"))
self.assertEqual(b"abc", self.output.getvalue())
def test_decode_combines_short(self):
self.assertEqual(b"", self.decoder.write(b"6\r\nabcdef0\r\n"))
self.assertEqual(b"abcdef", self.output.getvalue())
def test_decode_excess_bytes_from_write(self):
self.assertEqual(b"1234", self.decoder.write(b"3\r\nabc0\r\n1234"))
self.assertEqual(b"abc", self.output.getvalue())
def test_decode_write_after_finished_errors(self):
self.assertEqual(b"1234", self.decoder.write(b"3\r\nabc0\r\n1234"))
self.assertRaises(ValueError, self.decoder.write, b"")
def test_decode_hex(self):
self.assertEqual(b"", self.decoder.write(b"A\r\n12345678900\r\n"))
self.assertEqual(b"1234567890", self.output.getvalue())
def test_decode_long_ranges(self):
self.assertEqual(None, self.decoder.write(b"10000\r\n"))
self.assertEqual(None, self.decoder.write(b"1" * 65536))
self.assertEqual(None, self.decoder.write(b"10000\r\n"))
self.assertEqual(None, self.decoder.write(b"2" * 65536))
self.assertEqual(b"", self.decoder.write(b"0\r\n"))
self.assertEqual(b"1" * 65536 + b"2" * 65536, self.output.getvalue())
def test_decode_newline_nonstrict(self):
"""Tolerate chunk markers with no CR character."""
# From <http://pad.lv/505078>
self.decoder = subunit.chunked.Decoder(self.output, strict=False)
self.assertEqual(None, self.decoder.write(b"a\n"))
self.assertEqual(None, self.decoder.write(b"abcdeabcde"))
self.assertEqual(b"", self.decoder.write(b"0\n"))
self.assertEqual(b"abcdeabcde", self.output.getvalue())
def test_decode_strict_newline_only(self):
"""Reject chunk markers with no CR character in strict mode."""
# From <http://pad.lv/505078>
self.assertRaises(ValueError, self.decoder.write, b"a\n")
def test_decode_strict_multiple_crs(self):
self.assertRaises(ValueError, self.decoder.write, b"a\r\r\n")
def test_decode_short_header(self):
self.assertRaises(ValueError, self.decoder.write, b"\n")
class TestEncode(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.output = BytesIO()
self.encoder = subunit.chunked.Encoder(self.output)
def test_encode_nothing(self):
self.encoder.close()
self.assertEqual(b"0\r\n", self.output.getvalue())
def test_encode_empty(self):
self.encoder.write(b"")
self.encoder.close()
self.assertEqual(b"0\r\n", self.output.getvalue())
def test_encode_short(self):
self.encoder.write(b"abc")
self.encoder.close()
self.assertEqual(b"3\r\nabc0\r\n", self.output.getvalue())
def test_encode_combines_short(self):
self.encoder.write(b"abc")
self.encoder.write(b"def")
self.encoder.close()
self.assertEqual(b"6\r\nabcdef0\r\n", self.output.getvalue())
def test_encode_over_9_is_in_hex(self):
self.encoder.write(b"1234567890")
self.encoder.close()
self.assertEqual(b"A\r\n12345678900\r\n", self.output.getvalue())
def test_encode_long_ranges_not_combined(self):
self.encoder.write(b"1" * 65536)
self.encoder.write(b"2" * 65536)
self.encoder.close()
self.assertEqual(b"10000\r\n" + b"1" * 65536 + b"10000\r\n" + b"2" * 65536 + b"0\r\n", self.output.getvalue())
-90
python/subunit/tests/test_details.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import unittest
from subunit import content, content_type, details
class TestSimpleDetails(unittest.TestCase):
def test_lineReceived(self):
parser = details.SimpleDetailsParser(None)
parser.lineReceived(b"foo\n")
parser.lineReceived(b"bar\n")
self.assertEqual(b"foo\nbar\n", parser._message)
def test_lineReceived_escaped_bracket(self):
parser = details.SimpleDetailsParser(None)
parser.lineReceived(b"foo\n")
parser.lineReceived(b" ]are\n")
parser.lineReceived(b"bar\n")
self.assertEqual(b"foo\n]are\nbar\n", parser._message)
def test_get_message(self):
parser = details.SimpleDetailsParser(None)
self.assertEqual(b"", parser.get_message())
def test_get_details(self):
parser = details.SimpleDetailsParser(None)
expected = {}
expected["traceback"] = content.Content(
content_type.ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b""]
)
found = parser.get_details()
self.assertEqual(expected.keys(), found.keys())
self.assertEqual(expected["traceback"].content_type, found["traceback"].content_type)
self.assertEqual(b"".join(expected["traceback"].iter_bytes()), b"".join(found["traceback"].iter_bytes()))
def test_get_details_skip(self):
parser = details.SimpleDetailsParser(None)
expected = {}
expected["reason"] = content.Content(content_type.ContentType("text", "plain"), lambda: [b""])
found = parser.get_details("skip")
self.assertEqual(expected, found)
def test_get_details_success(self):
parser = details.SimpleDetailsParser(None)
expected = {}
expected["message"] = content.Content(content_type.ContentType("text", "plain"), lambda: [b""])
found = parser.get_details("success")
self.assertEqual(expected, found)
class TestMultipartDetails(unittest.TestCase):
def test_get_message_is_None(self):
parser = details.MultipartDetailsParser(None)
self.assertEqual(None, parser.get_message())
def test_get_details(self):
parser = details.MultipartDetailsParser(None)
self.assertEqual({}, parser.get_details())
def test_parts(self):
parser = details.MultipartDetailsParser(None)
parser.lineReceived(b"Content-Type: text/plain\n")
parser.lineReceived(b"something\n")
parser.lineReceived(b"F\r\n")
parser.lineReceived(b"serialised\n")
parser.lineReceived(b"form0\r\n")
expected = {}
expected["something"] = content.Content(
content_type.ContentType("text", "plain"), lambda: [b"serialised\nform"]
)
found = parser.get_details()
self.assertEqual(expected.keys(), found.keys())
self.assertEqual(expected["something"].content_type, found["something"].content_type)
self.assertEqual(b"".join(expected["something"].iter_bytes()), b"".join(found["something"].iter_bytes()))
-46
python/subunit/tests/test_filter_to_disk.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2013 Subunit Contributors
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
import io
import os.path
from fixtures import TempDir
from testtools import TestCase
from testtools.matchers import FileContains
from subunit import _to_disk
from subunit.v2 import StreamResultToBytes
class SmokeTest(TestCase):
def test_smoke(self):
output = os.path.join(self.useFixture(TempDir()).path, "output")
stdin = io.BytesIO()
stdout = io.StringIO()
writer = StreamResultToBytes(stdin)
writer.startTestRun()
writer.status(
"foo", "success", {"tag"}, file_name="fred", file_bytes=b"abcdefg", eof=True, mime_type="text/plain"
)
writer.stopTestRun()
stdin.seek(0)
_to_disk.to_disk(["-d", output], stdin=stdin, stdout=stdout)
self.expectThat(
os.path.join(output, "foo/test.json"),
FileContains(
'{"details": ["fred"], "id": "foo", "start": null, "status": "success", "stop": null, "tags": ["tag"]}'
),
)
self.expectThat(os.path.join(output, "foo/fred"), FileContains("abcdefg"))
-42
python/subunit/tests/test_filters.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2013 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
from tempfile import NamedTemporaryFile
from testtools import TestCase
from subunit import read_test_list
from subunit.filters import find_stream
class TestReadTestList(TestCase):
def test_read_list(self):
with NamedTemporaryFile() as f:
f.write(b"foo\nbar\n# comment\nother # comment\n")
f.flush()
self.assertEqual(read_test_list(f.name), ["foo", "bar", "other"])
class TestFindStream(TestCase):
def test_no_argv(self):
self.assertEqual("foo", find_stream("foo", []))
def test_opens_file(self):
f = NamedTemporaryFile()
f.write(b"foo")
f.flush()
stream = find_stream("bar", [f.name])
self.assertEqual(b"foo", stream.read())
-580
python/subunit/tests/test_output_filter.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2013 Subunit Contributors
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import datetime
import optparse
from contextlib import contextmanager
from functools import partial
from io import BytesIO, TextIOWrapper
from tempfile import NamedTemporaryFile
from iso8601 import UTC
from testtools import TestCase
from testtools.matchers import Equals, Matcher, MatchesAny, MatchesListwise, Mismatch, raises
from testtools.testresult.doubles import StreamResult
import subunit._output as _o
from subunit._output import _ALL_ACTIONS, _FINAL_ACTIONS, generate_stream_results, parse_arguments
class SafeOptionParser(optparse.OptionParser):
"""An ArgumentParser class that doesn't call sys.exit."""
def exit(self, status=0, message=""):
raise RuntimeError(message)
def error(self, message):
raise RuntimeError(message)
safe_parse_arguments = partial(parse_arguments, ParserClass=SafeOptionParser)
class TestStatusArgParserTests(TestCase):
scenarios = [(cmd, dict(command=cmd, option="--" + cmd)) for cmd in _ALL_ACTIONS]
def test_can_parse_all_commands_with_test_id(self):
test_id = self.getUniqueString()
args = safe_parse_arguments(args=[self.option, test_id])
self.assertThat(args.action, Equals(self.command))
self.assertThat(args.test_id, Equals(test_id))
def test_all_commands_parse_file_attachment(self):
with NamedTemporaryFile() as tmp_file:
args = safe_parse_arguments(args=[self.option, "foo", "--attach-file", tmp_file.name])
self.assertThat(args.attach_file.name, Equals(tmp_file.name))
def test_all_commands_accept_mimetype_argument(self):
with NamedTemporaryFile() as tmp_file:
args = safe_parse_arguments(
args=[self.option, "foo", "--attach-file", tmp_file.name, "--mimetype", "text/plain"]
)
self.assertThat(args.mimetype, Equals("text/plain"))
def test_all_commands_accept_file_name_argument(self):
with NamedTemporaryFile() as tmp_file:
args = safe_parse_arguments(args=[self.option, "foo", "--attach-file", tmp_file.name, "--file-name", "foo"])
self.assertThat(args.file_name, Equals("foo"))
def test_all_commands_accept_tags_argument(self):
args = safe_parse_arguments(args=[self.option, "foo", "--tag", "foo", "--tag", "bar", "--tag", "baz"])
self.assertThat(args.tags, Equals(["foo", "bar", "baz"]))
def test_attach_file_with_hyphen_opens_stdin(self):
self.patch(_o.sys, "stdin", TextIOWrapper(BytesIO(b"Hello")))
args = safe_parse_arguments(args=[self.option, "foo", "--attach-file", "-"])
self.assertThat(args.attach_file.read(), Equals(b"Hello"))
def test_attach_file_with_hyphen_sets_filename_to_stdin(self):
args = safe_parse_arguments(args=[self.option, "foo", "--attach-file", "-"])
self.assertThat(args.file_name, Equals("stdin"))
def test_can_override_stdin_filename(self):
args = safe_parse_arguments(args=[self.option, "foo", "--attach-file", "-", "--file-name", "foo"])
self.assertThat(args.file_name, Equals("foo"))
def test_requires_test_id(self):
def fn():
return safe_parse_arguments(args=[self.option])
self.assertThat(fn, raises(RuntimeError("argument %s: must specify a single TEST_ID." % self.option)))
class ArgParserTests(TestCase):
def test_can_parse_attach_file_without_test_id(self):
with NamedTemporaryFile() as tmp_file:
args = safe_parse_arguments(args=["--attach-file", tmp_file.name])
self.assertThat(args.attach_file.name, Equals(tmp_file.name))
def test_can_run_without_args(self):
safe_parse_arguments([])
def test_cannot_specify_more_than_one_status_command(self):
def fn():
return safe_parse_arguments(["--fail", "foo", "--skip", "bar"])
self.assertThat(fn, raises(RuntimeError("argument --skip: Only one status may be specified at once.")))
def test_cannot_specify_mimetype_without_attach_file(self):
def fn():
return safe_parse_arguments(["--mimetype", "foo"])
self.assertThat(fn, raises(RuntimeError("Cannot specify --mimetype without --attach-file")))
def test_cannot_specify_filename_without_attach_file(self):
def fn():
return safe_parse_arguments(["--file-name", "foo"])
self.assertThat(fn, raises(RuntimeError("Cannot specify --file-name without --attach-file")))
def test_can_specify_tags_without_status_command(self):
args = safe_parse_arguments(["--tag", "foo"])
self.assertEqual(["foo"], args.tags)
def test_must_specify_tags_with_tags_options(self):
def fn():
return safe_parse_arguments(["--fail", "foo", "--tag"])
self.assertThat(
fn,
MatchesAny(
raises(RuntimeError("--tag option requires 1 argument")),
raises(RuntimeError("--tag option requires an argument")),
),
)
def get_result_for(commands):
"""Get a result object from *commands.
Runs the 'generate_stream_results' function from subunit._output after
parsing *commands as if they were specified on the command line. The
resulting bytestream is then converted back into a result object and
returned.
"""
result = StreamResult()
args = safe_parse_arguments(commands)
generate_stream_results(args, result)
return result
@contextmanager
def temp_file_contents(data):
"""Create a temporary file on disk containing 'data'."""
with NamedTemporaryFile() as f:
f.write(data)
f.seek(0)
yield f
class StatusStreamResultTests(TestCase):
scenarios = [(s, dict(status=s, option="--" + s)) for s in _ALL_ACTIONS]
_dummy_timestamp = datetime.datetime(2013, 1, 1, 0, 0, 0, 0, UTC)
def setUp(self):
super().setUp()
self.patch(_o, "create_timestamp", lambda: self._dummy_timestamp)
self.test_id = self.getUniqueString()
def test_only_one_packet_is_generated(self):
result = get_result_for([self.option, self.test_id])
self.assertThat(
len(result._events),
Equals(3), # startTestRun and stopTestRun are also called, making 3 total.
)
def test_correct_status_is_generated(self):
result = get_result_for([self.option, self.test_id])
self.assertThat(result._events[1], MatchesStatusCall(test_status=self.status))
def test_all_commands_generate_tags(self):
result = get_result_for([self.option, self.test_id, "--tag", "hello", "--tag", "world"])
self.assertThat(result._events[1], MatchesStatusCall(test_tags={"hello", "world"}))
def test_all_commands_generate_timestamp(self):
result = get_result_for([self.option, self.test_id])
self.assertThat(result._events[1], MatchesStatusCall(timestamp=self._dummy_timestamp))
def test_all_commands_generate_correct_test_id(self):
result = get_result_for([self.option, self.test_id])
self.assertThat(result._events[1], MatchesStatusCall(test_id=self.test_id))
def test_file_is_sent_in_single_packet(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_can_read_binary_files(self):
with temp_file_contents(b"\xde\xad\xbe\xef") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_bytes=b"\xde\xad\xbe\xef", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_can_read_empty_files(self):
with temp_file_contents(b"") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_bytes=b"", file_name=f.name, eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_can_read_stdin(self):
self.patch(_o.sys, "stdin", TextIOWrapper(BytesIO(b"\xfe\xed\xfa\xce")))
result = get_result_for([self.option, self.test_id, "--attach-file", "-"])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_bytes=b"\xfe\xed\xfa\xce", file_name="stdin", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_is_sent_with_test_id(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_is_sent_with_test_status(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_status=self.status, file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_chunk_size_is_honored(self):
with temp_file_contents(b"Hello") as f:
self.patch(_o, "_CHUNK_SIZE", 1)
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"H", eof=False),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"e", eof=False),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"l", eof=False),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"l", eof=False),
MatchesStatusCall(test_id=self.test_id, file_bytes=b"o", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_mimetype_specified_once_only(self):
with temp_file_contents(b"Hi") as f:
self.patch(_o, "_CHUNK_SIZE", 1)
result = get_result_for(
[
self.option,
self.test_id,
"--attach-file",
f.name,
"--mimetype",
"text/plain",
]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=self.test_id, mime_type="text/plain", file_bytes=b"H", eof=False),
MatchesStatusCall(test_id=self.test_id, mime_type=None, file_bytes=b"i", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_tags_specified_once_only(self):
with temp_file_contents(b"Hi") as f:
self.patch(_o, "_CHUNK_SIZE", 1)
result = get_result_for(
[
self.option,
self.test_id,
"--attach-file",
f.name,
"--tag",
"foo",
"--tag",
"bar",
]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=self.test_id, test_tags={"foo", "bar"}),
MatchesStatusCall(test_id=self.test_id, test_tags=None),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_timestamp_specified_once_only(self):
with temp_file_contents(b"Hi") as f:
self.patch(_o, "_CHUNK_SIZE", 1)
result = get_result_for(
[
self.option,
self.test_id,
"--attach-file",
f.name,
]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=self.test_id, timestamp=self._dummy_timestamp),
MatchesStatusCall(test_id=self.test_id, timestamp=None),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_test_status_specified_once_only(self):
with temp_file_contents(b"Hi") as f:
self.patch(_o, "_CHUNK_SIZE", 1)
result = get_result_for(
[
self.option,
self.test_id,
"--attach-file",
f.name,
]
)
# 'inprogress' status should be on the first packet only, all other
# statuses should be on the last packet.
if self.status in _FINAL_ACTIONS:
first_call = MatchesStatusCall(test_id=self.test_id, test_status=None)
last_call = MatchesStatusCall(test_id=self.test_id, test_status=self.status)
else:
first_call = MatchesStatusCall(test_id=self.test_id, test_status=self.status)
last_call = MatchesStatusCall(test_id=self.test_id, test_status=None)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
first_call,
last_call,
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_filename_can_be_overridden(self):
with temp_file_contents(b"Hello") as f:
specified_file_name = self.getUniqueString()
result = get_result_for(
[self.option, self.test_id, "--attach-file", f.name, "--file-name", specified_file_name]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_name=specified_file_name, file_bytes=b"Hello"),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_name_is_used_by_default(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for([self.option, self.test_id, "--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_name=f.name, file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
class FileDataTests(TestCase):
def test_can_attach_file_without_test_id(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for(["--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_id=None, file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_file_name_is_used_by_default(self):
with temp_file_contents(b"Hello") as f:
result = get_result_for(["--attach-file", f.name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_name=f.name, file_bytes=b"Hello", eof=True),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_filename_can_be_overridden(self):
with temp_file_contents(b"Hello") as f:
specified_file_name = self.getUniqueString()
result = get_result_for(["--attach-file", f.name, "--file-name", specified_file_name])
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_name=specified_file_name, file_bytes=b"Hello"),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_files_have_timestamp(self):
_dummy_timestamp = datetime.datetime(2013, 1, 1, 0, 0, 0, 0, UTC)
self.patch(_o, "create_timestamp", lambda: _dummy_timestamp)
with temp_file_contents(b"Hello") as f:
self.getUniqueString()
result = get_result_for(
[
"--attach-file",
f.name,
]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(file_bytes=b"Hello", timestamp=_dummy_timestamp),
MatchesStatusCall(call="stopTestRun"),
]
),
)
def test_can_specify_tags_without_test_status(self):
result = get_result_for(
[
"--tag",
"foo",
]
)
self.assertThat(
result._events,
MatchesListwise(
[
MatchesStatusCall(call="startTestRun"),
MatchesStatusCall(test_tags={"foo"}),
MatchesStatusCall(call="stopTestRun"),
]
),
)
class MatchesStatusCall(Matcher):
_position_lookup = {
"call": 0,
"test_id": 1,
"test_status": 2,
"test_tags": 3,
"runnable": 4,
"file_name": 5,
"file_bytes": 6,
"eof": 7,
"mime_type": 8,
"route_code": 9,
"timestamp": 10,
}
def __init__(self, **kwargs):
unknown_kwargs = list(filter(lambda k: k not in self._position_lookup, kwargs))
if unknown_kwargs:
raise ValueError("Unknown keywords: %s" % ",".join(unknown_kwargs))
self._filters = kwargs
def match(self, call_tuple):
for k, v in self._filters.items():
try:
pos = self._position_lookup[k]
if call_tuple[pos] != v:
return Mismatch("Value for key is {!r}, not {!r}".format(call_tuple[pos], v))
except IndexError:
return Mismatch("Key %s is not present." % k)
def __str__(self):
return "<MatchesStatusCall %r>" % self._filters
-110
python/subunit/tests/test_progress_model.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2009 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import unittest
from subunit.progress_model import ProgressModel
class TestProgressModel(unittest.TestCase):
def assertProgressSummary(self, pos, total, progress):
"""Assert that a progress model has reached a particular point."""
self.assertEqual(pos, progress.pos())
self.assertEqual(total, progress.width())
def test_new_progress_0_0(self):
progress = ProgressModel()
self.assertProgressSummary(0, 0, progress)
def test_advance_0_0(self):
progress = ProgressModel()
progress.advance()
self.assertProgressSummary(1, 0, progress)
def test_advance_1_0(self):
progress = ProgressModel()
progress.advance()
self.assertProgressSummary(1, 0, progress)
def test_set_width_absolute(self):
progress = ProgressModel()
progress.set_width(10)
self.assertProgressSummary(0, 10, progress)
def test_set_width_absolute_preserves_pos(self):
progress = ProgressModel()
progress.advance()
progress.set_width(2)
self.assertProgressSummary(1, 2, progress)
def test_adjust_width(self):
progress = ProgressModel()
progress.adjust_width(10)
self.assertProgressSummary(0, 10, progress)
progress.adjust_width(-10)
self.assertProgressSummary(0, 0, progress)
def test_adjust_width_preserves_pos(self):
progress = ProgressModel()
progress.advance()
progress.adjust_width(10)
self.assertProgressSummary(1, 10, progress)
progress.adjust_width(-10)
self.assertProgressSummary(1, 0, progress)
def test_push_preserves_progress(self):
progress = ProgressModel()
progress.adjust_width(3)
progress.advance()
progress.push()
self.assertProgressSummary(1, 3, progress)
def test_advance_advances_substack(self):
progress = ProgressModel()
progress.adjust_width(3)
progress.advance()
progress.push()
progress.adjust_width(1)
progress.advance()
self.assertProgressSummary(2, 3, progress)
def test_adjust_width_adjusts_substack(self):
progress = ProgressModel()
progress.adjust_width(3)
progress.advance()
progress.push()
progress.adjust_width(2)
progress.advance()
self.assertProgressSummary(3, 6, progress)
def test_set_width_adjusts_substack(self):
progress = ProgressModel()
progress.adjust_width(3)
progress.advance()
progress.push()
progress.set_width(2)
progress.advance()
self.assertProgressSummary(3, 6, progress)
def test_pop_restores_progress(self):
progress = ProgressModel()
progress.adjust_width(3)
progress.advance()
progress.push()
progress.adjust_width(1)
progress.advance()
progress.pop()
self.assertProgressSummary(1, 3, progress)
-115
python/subunit/tests/test_run.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2011 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import io
import unittest
from testtools import PlaceHolder, TestCase
from testtools.matchers import StartsWith
from testtools.testresult.doubles import StreamResult
import subunit
from subunit import run
from subunit.run import SubunitTestRunner
class TestSubunitTestRunner(TestCase):
def test_includes_timing_output(self):
bytestream = io.BytesIO()
runner = SubunitTestRunner(stream=bytestream)
test = PlaceHolder("name")
runner.run(test)
bytestream.seek(0)
eventstream = StreamResult()
subunit.ByteStreamToStreamResult(bytestream).run(eventstream)
timestamps = [event[-1] for event in eventstream._events if event is not None]
self.assertNotEqual([], timestamps)
def test_enumerates_tests_before_run(self):
bytestream = io.BytesIO()
runner = SubunitTestRunner(stream=bytestream)
test1 = PlaceHolder("name1")
test2 = PlaceHolder("name2")
case = unittest.TestSuite([test1, test2])
runner.run(case)
bytestream.seek(0)
eventstream = StreamResult()
subunit.ByteStreamToStreamResult(bytestream).run(eventstream)
self.assertEqual(
[
("status", "name1", "exists"),
("status", "name2", "exists"),
],
[event[:3] for event in eventstream._events[:2]],
)
def test_list_errors_if_errors_from_list_test(self):
bytestream = io.BytesIO()
runner = SubunitTestRunner(stream=bytestream)
def list_test(test):
return [], ["failed import"]
self.patch(run, "list_test", list_test)
exc = self.assertRaises(SystemExit, runner.list, None)
self.assertEqual((2,), exc.args)
def test_list_includes_loader_errors(self):
bytestream = io.BytesIO()
runner = SubunitTestRunner(stream=bytestream)
def list_test(test):
return [], []
class Loader:
errors = ["failed import"]
loader = Loader()
self.patch(run, "list_test", list_test)
exc = self.assertRaises(SystemExit, runner.list, None, loader=loader)
self.assertEqual((2,), exc.args)
class FailingTest(TestCase):
def test_fail(self):
1 / 0
def test_exits_zero_when_tests_fail(self):
bytestream = io.BytesIO()
stream = io.TextIOWrapper(bytestream, encoding="utf8")
try:
self.assertEqual(
None,
run.main(argv=["progName", "subunit.tests.test_run.TestSubunitTestRunner.FailingTest"], stdout=stream),
)
except SystemExit:
self.fail("SystemExit raised")
self.assertThat(bytestream.getvalue(), StartsWith(b"\xb3"))
class ExitingTest(TestCase):
def test_exit(self):
raise SystemExit(0)
def test_exits_nonzero_when_execution_errors(self):
bytestream = io.BytesIO()
stream = io.TextIOWrapper(bytestream, encoding="utf8")
exc = self.assertRaises(
SystemExit,
run.main,
argv=["progName", "subunit.tests.test_run.TestSubunitTestRunner.ExitingTest"],
stdout=stream,
)
self.assertEqual(0, exc.args[0])
-322
python/subunit/tests/test_subunit_filter.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Tests for subunit.TestResultFilter."""
import subprocess
import sys
import unittest
from datetime import datetime
from io import BytesIO
from testtools import TestCase
from testtools.testresult.doubles import ExtendedTestResult, StreamResult
import iso8601
import subunit
from subunit.test_results import make_tag_filter, TestResultFilter
from subunit import ByteStreamToStreamResult, StreamResultToBytes
class TestTestResultFilter(TestCase):
"""Test for TestResultFilter, a TestResult object which filters tests."""
# While TestResultFilter works on python objects, using a subunit stream
# is an easy pithy way of getting a series of test objects to call into
# the TestResult, and as TestResultFilter is intended for use with subunit
# also has the benefit of detecting any interface skew issues.
example_subunit_stream = b"""\
tags: global
test passed
success passed
test failed
tags: local
failure failed
test error
error error [
error details
]
test skipped
skip skipped
test todo
xfail todo
"""
def run_tests(self, result_filter, input_stream=None):
"""Run tests through the given filter.
:param result_filter: A filtering TestResult object.
:param input_stream: Bytes of subunit stream data. If not provided,
uses TestTestResultFilter.example_subunit_stream.
"""
if input_stream is None:
input_stream = self.example_subunit_stream
test = subunit.ProtocolTestCase(BytesIO(input_stream))
test.run(result_filter)
def test_default(self):
"""The default is to exclude success and include everything else."""
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result)
self.run_tests(result_filter)
# skips are seen as success by default python TestResult.
self.assertEqual(["error"], [error[0].id() for error in filtered_result.errors])
self.assertEqual(["failed"], [failure[0].id() for failure in filtered_result.failures])
self.assertEqual(4, filtered_result.testsRun)
def test_tag_filter(self):
tag_filter = make_tag_filter(["global"], ["local"])
result = ExtendedTestResult()
result_filter = TestResultFilter(result, filter_success=False, filter_predicate=tag_filter)
self.run_tests(result_filter)
tests_included = [event[1] for event in result._events if event[0] == "startTest"]
tests_expected = list(map(subunit.RemotedTestCase, ["passed", "error", "skipped", "todo"]))
self.assertEqual(tests_expected, tests_included)
def test_tags_tracked_correctly(self):
tag_filter = make_tag_filter(["a"], [])
result = ExtendedTestResult()
result_filter = TestResultFilter(result, filter_success=False, filter_predicate=tag_filter)
input_stream = b"test: foo\ntags: a\nsuccessful: foo\ntest: bar\nsuccessful: bar\n"
self.run_tests(result_filter, input_stream)
foo = subunit.RemotedTestCase("foo")
self.assertEqual(
[
("startTest", foo),
("tags", {"a"}, set()),
("addSuccess", foo),
("stopTest", foo),
],
result._events,
)
def test_exclude_errors(self):
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, filter_error=True)
self.run_tests(result_filter)
# skips are seen as errors by default python TestResult.
self.assertEqual([], filtered_result.errors)
self.assertEqual(["failed"], [failure[0].id() for failure in filtered_result.failures])
self.assertEqual(3, filtered_result.testsRun)
def test_fixup_expected_failures(self):
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, fixup_expected_failures={"failed"})
self.run_tests(result_filter)
self.assertEqual(["failed", "todo"], [failure[0].id() for failure in filtered_result.expectedFailures])
self.assertEqual([], filtered_result.failures)
self.assertEqual(4, filtered_result.testsRun)
def test_fixup_expected_errors(self):
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, fixup_expected_failures={"error"})
self.run_tests(result_filter)
self.assertEqual(["error", "todo"], [failure[0].id() for failure in filtered_result.expectedFailures])
self.assertEqual([], filtered_result.errors)
self.assertEqual(4, filtered_result.testsRun)
def test_fixup_unexpected_success(self):
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, filter_success=False, fixup_expected_failures={"passed"})
self.run_tests(result_filter)
self.assertEqual(["passed"], [passed.id() for passed in filtered_result.unexpectedSuccesses])
self.assertEqual(5, filtered_result.testsRun)
def test_exclude_failure(self):
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, filter_failure=True)
self.run_tests(result_filter)
self.assertEqual(["error"], [error[0].id() for error in filtered_result.errors])
self.assertEqual([], [failure[0].id() for failure in filtered_result.failures])
self.assertEqual(3, filtered_result.testsRun)
def test_exclude_skips(self):
filtered_result = subunit.TestResultStats(None)
result_filter = TestResultFilter(filtered_result, filter_skip=True)
self.run_tests(result_filter)
self.assertEqual(0, filtered_result.skipped_tests)
self.assertEqual(2, filtered_result.failed_tests)
self.assertEqual(3, filtered_result.testsRun)
def test_include_success(self):
"""Successes can be included if requested."""
filtered_result = unittest.TestResult()
result_filter = TestResultFilter(filtered_result, filter_success=False)
self.run_tests(result_filter)
self.assertEqual(["error"], [error[0].id() for error in filtered_result.errors])
self.assertEqual(["failed"], [failure[0].id() for failure in filtered_result.failures])
self.assertEqual(5, filtered_result.testsRun)
def test_filter_predicate(self):
"""You can filter by predicate callbacks"""
# 0.0.7 and earlier did not support the 'tags' parameter, so we need
# to test that we still support behaviour without it.
filtered_result = unittest.TestResult()
def filter_cb(test, outcome, err, details):
return outcome == "success"
result_filter = TestResultFilter(filtered_result, filter_predicate=filter_cb, filter_success=False)
self.run_tests(result_filter)
# Only success should pass
self.assertEqual(1, filtered_result.testsRun)
def test_filter_predicate_with_tags(self):
"""You can filter by predicate callbacks that accept tags"""
filtered_result = unittest.TestResult()
def filter_cb(test, outcome, err, details, tags):
return outcome == "success"
result_filter = TestResultFilter(filtered_result, filter_predicate=filter_cb, filter_success=False)
self.run_tests(result_filter)
# Only success should pass
self.assertEqual(1, filtered_result.testsRun)
def test_time_ordering_preserved(self):
# Passing a subunit stream through TestResultFilter preserves the
# relative ordering of 'time' directives and any other subunit
# directives that are still included.
date_a = datetime(year=2000, month=1, day=1, tzinfo=iso8601.UTC)
date_b = datetime(year=2000, month=1, day=2, tzinfo=iso8601.UTC)
date_c = datetime(year=2000, month=1, day=3, tzinfo=iso8601.UTC)
subunit_stream = (
"\n".join(["time: %s", "test: foo", "time: %s", "error: foo", "time: %s", ""]) % (date_a, date_b, date_c)
).encode()
result = ExtendedTestResult()
result_filter = TestResultFilter(result)
self.run_tests(result_filter, subunit_stream)
foo = subunit.RemotedTestCase("foo")
self.maxDiff = None
self.assertEqual(
[
("time", date_a),
("time", date_b),
("startTest", foo),
("addError", foo, {}),
("stopTest", foo),
("time", date_c),
],
result._events,
)
def test_time_passes_through_filtered_tests(self):
# Passing a subunit stream through TestResultFilter preserves 'time'
# directives even if a specific test is filtered out.
date_a = datetime(year=2000, month=1, day=1, tzinfo=iso8601.UTC)
date_b = datetime(year=2000, month=1, day=2, tzinfo=iso8601.UTC)
date_c = datetime(year=2000, month=1, day=3, tzinfo=iso8601.UTC)
subunit_stream = (
"\n".join(["time: %s", "test: foo", "time: %s", "success: foo", "time: %s", ""]) % (date_a, date_b, date_c)
).encode()
result = ExtendedTestResult()
result_filter = TestResultFilter(result)
result_filter.startTestRun()
self.run_tests(result_filter, subunit_stream)
result_filter.stopTestRun()
subunit.RemotedTestCase("foo")
self.maxDiff = None
self.assertEqual(
[
("startTestRun",),
("time", date_a),
("time", date_c),
("stopTestRun",),
],
result._events,
)
def test_skip_preserved(self):
subunit_stream = "\n".join(["test: foo", "skip: foo", ""]).encode()
result = ExtendedTestResult()
result_filter = TestResultFilter(result)
self.run_tests(result_filter, subunit_stream)
foo = subunit.RemotedTestCase("foo")
self.assertEqual(
[
("startTest", foo),
("addSkip", foo, {}),
("stopTest", foo),
],
result._events,
)
def test_renames(self):
def rename(name):
return name + " - renamed"
result = ExtendedTestResult()
result_filter = TestResultFilter(result, filter_success=False, rename=rename)
input_stream = b"test: foo\nsuccessful: foo\n"
self.run_tests(result_filter, input_stream)
self.assertEqual(
[("startTest", "foo - renamed"), ("addSuccess", "foo - renamed"), ("stopTest", "foo - renamed")],
[(ev[0], ev[1].id()) for ev in result._events],
)
class TestFilterCommand(TestCase):
def run_command(self, args, stream):
command = [sys.executable, "-m", "subunit.filter_scripts.subunit_filter"] + list(args)
ps = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = ps.communicate(stream)
if ps.returncode != 0:
raise RuntimeError("{} failed: {}".format(command, err))
return out
def test_default(self):
byte_stream = BytesIO()
stream = StreamResultToBytes(byte_stream)
stream.status(test_id="foo", test_status="inprogress")
stream.status(test_id="foo", test_status="skip")
output = self.run_command([], byte_stream.getvalue())
events = StreamResult()
ByteStreamToStreamResult(BytesIO(output)).run(events)
{event[1] for event in events._events}
self.assertEqual(
[
("status", "foo", "inprogress"),
("status", "foo", "skip"),
],
[event[:3] for event in events._events],
)
def test_tags(self):
byte_stream = BytesIO()
stream = StreamResultToBytes(byte_stream)
stream.status(test_id="foo", test_status="inprogress", test_tags={"a"})
stream.status(test_id="foo", test_status="success", test_tags={"a"})
stream.status(test_id="bar", test_status="inprogress")
stream.status(test_id="bar", test_status="inprogress")
stream.status(test_id="baz", test_status="inprogress", test_tags={"a"})
stream.status(test_id="baz", test_status="success", test_tags={"a"})
output = self.run_command(["-s", "--with-tag", "a"], byte_stream.getvalue())
events = StreamResult()
ByteStreamToStreamResult(BytesIO(output)).run(events)
ids = {event[1] for event in events._events}
self.assertEqual({"foo", "baz"}, ids)
def test_no_passthrough(self):
output = self.run_command(["--no-passthrough"], b"hi thar")
self.assertEqual(b"", output)
def test_passthrough(self):
output = self.run_command([], b"hi thar")
byte_stream = BytesIO()
stream = StreamResultToBytes(byte_stream)
stream.status(file_name="stdout", file_bytes=b"hi thar")
self.assertEqual(byte_stream.getvalue(), output)
-82
python/subunit/tests/test_subunit_stats.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Tests for subunit.TestResultStats."""
import unittest
from io import BytesIO, StringIO
import subunit
class TestTestResultStats(unittest.TestCase):
"""Test for TestResultStats, a TestResult object that generates stats."""
def setUp(self):
self.output = StringIO()
self.result = subunit.TestResultStats(self.output)
self.input_stream = BytesIO()
self.test = subunit.ProtocolTestCase(self.input_stream)
def test_stats_empty(self):
self.test.run(self.result)
self.assertEqual(0, self.result.total_tests)
self.assertEqual(0, self.result.passed_tests)
self.assertEqual(0, self.result.failed_tests)
self.assertEqual(set(), self.result.seen_tags)
def setUpUsedStream(self):
self.input_stream.write(
b"""tags: global
test passed
success passed
test failed
tags: local
failure failed
test error
error error
test skipped
skip skipped
test todo
xfail todo
"""
)
self.input_stream.seek(0)
self.test.run(self.result)
def test_stats_smoke_everything(self):
# Statistics are calculated usefully.
self.setUpUsedStream()
self.assertEqual(5, self.result.total_tests)
self.assertEqual(2, self.result.passed_tests)
self.assertEqual(2, self.result.failed_tests)
self.assertEqual(1, self.result.skipped_tests)
self.assertEqual({"global", "local"}, self.result.seen_tags)
def test_stat_formatting(self):
expected = (
"""
Total tests: 5
Passed tests: 2
Failed tests: 2
Skipped tests: 1
Seen tags: global, local
"""
)[1:]
self.setUpUsedStream()
self.result.formatStats()
self.assertEqual(expected, self.output.getvalue())
-80
python/subunit/tests/test_subunit_tags.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Tests for subunit.tag_stream."""
from io import BytesIO
import testtools
from testtools.matchers import Contains
import subunit
import subunit.test_results
class TestSubUnitTags(testtools.TestCase):
def setUp(self):
super().setUp()
self.original = BytesIO()
self.filtered = BytesIO()
def test_add_tag(self):
# Literal values to avoid set sort-order dependencies. Python code show
# derivation.
# reference = BytesIO()
# stream = subunit.StreamResultToBytes(reference)
# stream.status(
# test_id='test', test_status='inprogress', test_tags=set(['quux', 'foo']))
# stream.status(
# test_id='test', test_status='success', test_tags=set(['bar', 'quux', 'foo']))
reference = [
b"\xb3)\x82\x17\x04test\x02\x04quux\x03foo\x05\x97n\x86"
b"\xb3)\x83\x1b\x04test\x03\x03bar\x04quux\x03fooqn\xab)",
b"\xb3)\x82\x17\x04test\x02\x04quux\x03foo\x05\x97n\x86"
b"\xb3)\x83\x1b\x04test\x03\x04quux\x03foo\x03bar\xaf\xbd\x9d\xd6",
b"\xb3)\x82\x17\x04test\x02\x04quux\x03foo\x05\x97n\x86"
b"\xb3)\x83\x1b\x04test\x03\x04quux\x03bar\x03foo\x03\x04b\r",
b"\xb3)\x82\x17\x04test\x02\x04quux\x03foo\x05\x97n\x86"
b"\xb3)\x83\x1b\x04test\x03\x03bar\x03foo\x04quux\xd2\x18\x1bC",
b"\xb3)\x82\x17\x04test\x02\x04quux\x03foo\x05\x97n\x86"
b"\xb3)\x83\x1b\x04test\x03\x03foo\x04quux\x03bar\x08\xc2X\x83",
b"\xb3)\x82\x17\x04test\x02\x03foo\x04quux\xa6\xe1\xde\xec"
b"\xb3)\x83\x1b\x04test\x03\x03foo\x04quux\x03bar\x08\xc2X\x83",
b"\xb3)\x82\x17\x04test\x02\x03foo\x04quux\xa6\xe1\xde\xec"
b"\xb3)\x83\x1b\x04test\x03\x03bar\x03foo\x04quux\xd2\x18\x1bC",
b"\xb3)\x82\x17\x04test\x02\x03foo\x04quux\xa6\xe1\xde\xec"
b"\xb3)\x83\x1b\x04test\x03\x03foo\x03bar\x04quux:\x05e\x80",
b"\xb3)\x82\x17\x04test\x02\x03foo\x04quux\xa6\xe1\xde\xec"
b"\xb3)\x83\x1b\x04test\x03\x04quux\x03foo\x03bar\xaf\xbd\x9d\xd6",
]
stream = subunit.StreamResultToBytes(self.original)
stream.status(test_id="test", test_status="inprogress", test_tags={"foo"})
stream.status(test_id="test", test_status="success", test_tags={"foo", "bar"})
self.original.seek(0)
self.assertEqual(0, subunit.tag_stream(self.original, self.filtered, ["quux"]))
self.assertThat(reference, Contains(bytes(self.filtered.getvalue())))
def test_remove_tag(self):
reference = BytesIO()
stream = subunit.StreamResultToBytes(reference)
stream.status(test_id="test", test_status="inprogress", test_tags={"foo"})
stream.status(test_id="test", test_status="success", test_tags={"foo"})
stream = subunit.StreamResultToBytes(self.original)
stream.status(test_id="test", test_status="inprogress", test_tags={"foo"})
stream.status(test_id="test", test_status="success", test_tags={"foo", "bar"})
self.original.seek(0)
self.assertEqual(0, subunit.tag_stream(self.original, self.filtered, ["-bar"]))
self.assertEqual(reference.getvalue(), self.filtered.getvalue())
-570
python/subunit/tests/test_tap2subunit.py
#
# subunit: extensions to python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
"""Tests for TAP2SubUnit."""
from io import BytesIO, StringIO
from testtools import TestCase
from testtools.testresult.doubles import StreamResult
import subunit
UTF8_TEXT = "text/plain; charset=UTF8"
class TestTAP2SubUnit(TestCase):
"""Tests for TAP2SubUnit.
These tests test TAP string data in, and subunit string data out.
This is ok because the subunit protocol is intended to be stable,
but it might be easier/pithier to write tests against TAP string in,
parsed subunit objects out (by hooking the subunit stream to a subunit
protocol server.
"""
def setUp(self):
super().setUp()
self.tap = StringIO()
self.subunit = BytesIO()
def test_skip_entire_file(self):
# A file
# 1..- # Skipped: comment
# results in a single skipped test.
self.tap.write("1..0 # Skipped: entire file skipped\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"file skip",
"skip",
None,
True,
"tap comment",
b"Skipped: entire file skipped",
True,
None,
None,
None,
)
]
)
def test_ok_test_pass(self):
# A file
# ok
# results in a passed test with name 'test 1' (a synthetic name as tap
# does not require named fixtures - it is the first test in the tap
# stream).
self.tap.write("ok\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events([("status", "test 1", "success", None, False, None, None, True, None, None, None)])
def test_ok_test_number_pass(self):
# A file
# ok 1
# results in a passed test with name 'test 1'
self.tap.write("ok 1\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events([("status", "test 1", "success", None, False, None, None, True, None, None, None)])
def test_ok_test_number_description_pass(self):
# A file
# ok 1 - There is a description
# results in a passed test with name 'test 1 - There is a description'
self.tap.write("ok 1 - There is a description\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[("status", "test 1 - There is a description", "success", None, False, None, None, True, None, None, None)]
)
def test_ok_test_description_pass(self):
# A file
# ok There is a description
# results in a passed test with name 'test 1 There is a description'
self.tap.write("ok There is a description\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[("status", "test 1 There is a description", "success", None, False, None, None, True, None, None, None)]
)
def test_ok_SKIP_skip(self):
# A file
# ok # SKIP
# results in a skkip test with name 'test 1'
self.tap.write("ok # SKIP\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events([("status", "test 1", "skip", None, False, None, None, True, None, None, None)])
def test_ok_skip_number_comment_lowercase(self):
self.tap.write("ok 1 # skip no samba environment available, skipping compilation\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1",
"skip",
None,
False,
"tap comment",
b"no samba environment available, skipping compilation",
True,
"text/plain; charset=UTF8",
None,
None,
)
]
)
def test_ok_number_description_SKIP_skip_comment(self):
# A file
# ok 1 foo # SKIP Not done yet
# results in a skip test with name 'test 1 foo' and a log of
# Not done yet
self.tap.write("ok 1 foo # SKIP Not done yet\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1 foo",
"skip",
None,
False,
"tap comment",
b"Not done yet",
True,
"text/plain; charset=UTF8",
None,
None,
)
]
)
def test_ok_SKIP_skip_comment(self):
# A file
# ok # SKIP Not done yet
# results in a skip test with name 'test 1' and a log of Not done yet
self.tap.write("ok # SKIP Not done yet\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1",
"skip",
None,
False,
"tap comment",
b"Not done yet",
True,
"text/plain; charset=UTF8",
None,
None,
)
]
)
def test_ok_TODO_xfail(self):
# A file
# ok # TODO
# results in a xfail test with name 'test 1'
self.tap.write("ok # TODO\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events([("status", "test 1", "xfail", None, False, None, None, True, None, None, None)])
def test_ok_TODO_xfail_comment(self):
# A file
# ok # TODO Not done yet
# results in a xfail test with name 'test 1' and a log of Not done yet
self.tap.write("ok # TODO Not done yet\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1",
"xfail",
None,
False,
"tap comment",
b"Not done yet",
True,
"text/plain; charset=UTF8",
None,
None,
)
]
)
def test_bail_out_errors(self):
# A file with line in it
# Bail out! COMMENT
# is treated as an error
self.tap.write("ok 1 foo\n")
self.tap.write("Bail out! Lifejacket engaged\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
("status", "test 1 foo", "success", None, False, None, None, True, None, None, None),
("status", "Bail out! Lifejacket engaged", "fail", None, False, None, None, True, None, None, None),
]
)
def test_missing_test_at_end_with_plan_adds_error(self):
# A file
# 1..3
# ok first test
# not ok third test
# results in three tests, with the third being created
self.tap.write("1..3\n")
self.tap.write("ok first test\n")
self.tap.write("not ok second test\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
("status", "test 1 first test", "success", None, False, None, None, True, None, None, None),
("status", "test 2 second test", "fail", None, False, None, None, True, None, None, None),
(
"status",
"test 3",
"fail",
None,
False,
"tap meta",
b"test missing from TAP output",
True,
"text/plain; charset=UTF8",
None,
None,
),
]
)
def test_missing_test_with_plan_adds_error(self):
# A file
# 1..3
# ok first test
# not ok 3 third test
# results in three tests, with the second being created
self.tap.write("1..3\n")
self.tap.write("ok first test\n")
self.tap.write("not ok 3 third test\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
("status", "test 1 first test", "success", None, False, None, None, True, None, None, None),
(
"status",
"test 2",
"fail",
None,
False,
"tap meta",
b"test missing from TAP output",
True,
"text/plain; charset=UTF8",
None,
None,
),
("status", "test 3 third test", "fail", None, False, None, None, True, None, None, None),
]
)
def test_missing_test_no_plan_adds_error(self):
# A file
# ok first test
# not ok 3 third test
# results in three tests, with the second being created
self.tap.write("ok first test\n")
self.tap.write("not ok 3 third test\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
("status", "test 1 first test", "success", None, False, None, None, True, None, None, None),
(
"status",
"test 2",
"fail",
None,
False,
"tap meta",
b"test missing from TAP output",
True,
"text/plain; charset=UTF8",
None,
None,
),
("status", "test 3 third test", "fail", None, False, None, None, True, None, None, None),
]
)
def test_four_tests_in_a_row_trailing_plan(self):
# A file
# ok 1 - first test in a script with no plan at all
# not ok 2 - second
# ok 3 - third
# not ok 4 - fourth
# 1..4
# results in four tests numbered and named
self.tap.write("ok 1 - first test in a script with trailing plan\n")
self.tap.write("not ok 2 - second\n")
self.tap.write("ok 3 - third\n")
self.tap.write("not ok 4 - fourth\n")
self.tap.write("1..4\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1 - first test in a script with trailing plan",
"success",
None,
False,
None,
None,
True,
None,
None,
None,
),
("status", "test 2 - second", "fail", None, False, None, None, True, None, None, None),
("status", "test 3 - third", "success", None, False, None, None, True, None, None, None),
("status", "test 4 - fourth", "fail", None, False, None, None, True, None, None, None),
]
)
def test_four_tests_in_a_row_with_plan(self):
# A file
# 1..4
# ok 1 - first test in a script with no plan at all
# not ok 2 - second
# ok 3 - third
# not ok 4 - fourth
# results in four tests numbered and named
self.tap.write("1..4\n")
self.tap.write("ok 1 - first test in a script with a plan\n")
self.tap.write("not ok 2 - second\n")
self.tap.write("ok 3 - third\n")
self.tap.write("not ok 4 - fourth\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1 - first test in a script with a plan",
"success",
None,
False,
None,
None,
True,
None,
None,
None,
),
("status", "test 2 - second", "fail", None, False, None, None, True, None, None, None),
("status", "test 3 - third", "success", None, False, None, None, True, None, None, None),
("status", "test 4 - fourth", "fail", None, False, None, None, True, None, None, None),
]
)
def test_four_tests_in_a_row_no_plan(self):
# A file
# ok 1 - first test in a script with no plan at all
# not ok 2 - second
# ok 3 - third
# not ok 4 - fourth
# results in four tests numbered and named
self.tap.write("ok 1 - first test in a script with no plan at all\n")
self.tap.write("not ok 2 - second\n")
self.tap.write("ok 3 - third\n")
self.tap.write("not ok 4 - fourth\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1 - first test in a script with no plan at all",
"success",
None,
False,
None,
None,
True,
None,
None,
None,
),
("status", "test 2 - second", "fail", None, False, None, None, True, None, None, None),
("status", "test 3 - third", "success", None, False, None, None, True, None, None, None),
("status", "test 4 - fourth", "fail", None, False, None, None, True, None, None, None),
]
)
def test_todo_and_skip(self):
# A file
# not ok 1 - a fail but # TODO but is TODO
# not ok 2 - another fail # SKIP instead
# results in two tests, numbered and commented.
self.tap.write("not ok 1 - a fail but # TODO but is TODO\n")
self.tap.write("not ok 2 - another fail # SKIP instead\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.subunit.seek(0)
events = StreamResult()
subunit.ByteStreamToStreamResult(self.subunit).run(events)
self.check_events(
[
(
"status",
"test 1 - a fail but",
"xfail",
None,
False,
"tap comment",
b"but is TODO",
True,
"text/plain; charset=UTF8",
None,
None,
),
(
"status",
"test 2 - another fail",
"skip",
None,
False,
"tap comment",
b"instead",
True,
"text/plain; charset=UTF8",
None,
None,
),
]
)
def test_leading_comments_add_to_next_test_log(self):
# A file
# # comment
# ok
# ok
# results in a single test with the comment included
# in the first test and not the second.
self.tap.write("# comment\n")
self.tap.write("ok\n")
self.tap.write("ok\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
(
"status",
"test 1",
"success",
None,
False,
"tap comment",
b"# comment",
True,
"text/plain; charset=UTF8",
None,
None,
),
("status", "test 2", "success", None, False, None, None, True, None, None, None),
]
)
def test_trailing_comments_are_included_in_last_test_log(self):
# A file
# ok foo
# ok foo
# # comment
# results in a two tests, with the second having the comment
# attached to its log.
self.tap.write("ok\n")
self.tap.write("ok\n")
self.tap.write("# comment\n")
self.tap.seek(0)
result = subunit.TAP2SubUnit(self.tap, self.subunit)
self.assertEqual(0, result)
self.check_events(
[
("status", "test 1", "success", None, False, None, None, True, None, None, None),
(
"status",
"test 2",
"success",
None,
False,
"tap comment",
b"# comment",
True,
"text/plain; charset=UTF8",
None,
None,
),
]
)
def check_events(self, events):
self.subunit.seek(0)
eventstream = StreamResult()
subunit.ByteStreamToStreamResult(self.subunit).run(eventstream)
self.assertEqual(events, eventstream._events)
-1340
python/subunit/tests/test_test_protocol.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import datetime
import io
import os
import tempfile
import unittest
from io import BytesIO
from testtools import PlaceHolder, TestCase, TestResult, skipIf
from testtools.content import Content, TracebackContent, text_content
from testtools.content_type import ContentType
from testtools.testresult.doubles import ExtendedTestResult
from testtools.matchers import Contains, Equals, MatchesAny
import iso8601
import subunit
from subunit.tests import (
_remote_exception_repr,
_remote_exception_repr_chunked,
_remote_exception_str,
_remote_exception_str_chunked,
)
tb_prelude = "Traceback (most recent call last):\n"
def details_to_str(details):
return TestResult()._err_details_to_string(None, details=details)
class TestHelpers(TestCase):
def test__unwrap_text_file_read_mode(self):
fd, file_path = tempfile.mkstemp()
self.addCleanup(os.remove, file_path)
fake_file = os.fdopen(fd, "r")
self.assertEqual(fake_file.buffer, subunit._unwrap_text(fake_file))
def test__unwrap_text_file_write_mode(self):
fd, file_path = tempfile.mkstemp()
self.addCleanup(os.remove, file_path)
fake_file = os.fdopen(fd, "w")
self.assertEqual(fake_file.buffer, subunit._unwrap_text(fake_file))
def test__unwrap_text_fileIO_read_mode(self):
fd, file_path = tempfile.mkstemp()
self.addCleanup(os.remove, file_path)
fake_file = io.FileIO(file_path, "r")
self.assertEqual(fake_file, subunit._unwrap_text(fake_file))
def test__unwrap_text_fileIO_write_mode(self):
fd, file_path = tempfile.mkstemp()
self.addCleanup(os.remove, file_path)
fake_file = io.FileIO(file_path, "w")
self.assertEqual(fake_file, subunit._unwrap_text(fake_file))
def test__unwrap_text_BytesIO(self):
fake_stream = io.BytesIO()
self.assertEqual(fake_stream, subunit._unwrap_text(fake_stream))
class TestTestImports(unittest.TestCase):
def test_imports(self):
from subunit import ( # noqa: F401
DiscardStream,
ExecTestCase,
IsolatedTestCase,
ProtocolTestCase,
RemotedTestCase,
RemoteError,
TestProtocolClient,
TestProtocolServer,
) # noqa: F401
class TestDiscardStream(unittest.TestCase):
def test_write(self):
subunit.DiscardStream().write("content")
class TestProtocolServerForward(unittest.TestCase):
def test_story(self):
client = unittest.TestResult()
out = BytesIO()
protocol = subunit.TestProtocolServer(client, forward_stream=out)
pipe = BytesIO(b"test old mcdonald\nsuccess old mcdonald\n")
protocol.readFrom(pipe)
self.assertEqual(client.testsRun, 1)
self.assertEqual(pipe.getvalue(), out.getvalue())
def test_not_command(self):
client = unittest.TestResult()
out = BytesIO()
protocol = subunit.TestProtocolServer(client, stream=subunit.DiscardStream(), forward_stream=out)
pipe = BytesIO(b"success old mcdonald\n")
protocol.readFrom(pipe)
self.assertEqual(client.testsRun, 0)
self.assertEqual(b"", out.getvalue())
class TestTestProtocolServerPipe(unittest.TestCase):
def test_story(self):
client = unittest.TestResult()
protocol = subunit.TestProtocolServer(client)
traceback = "foo.c:53:ERROR invalid state\n"
pipe = BytesIO(
b"test old mcdonald\n"
b"success old mcdonald\n"
b"test bing crosby\n"
b"failure bing crosby [\n" + traceback.encode() + b"]\n"
b"test an error\n"
b"error an error\n"
)
protocol.readFrom(pipe)
bing = subunit.RemotedTestCase("bing crosby")
an_error = subunit.RemotedTestCase("an error")
self.assertEqual(
client.errors,
[(an_error, _remote_exception_repr + "\n")],
)
self.assertEqual(
client.failures,
[(bing, _remote_exception_repr + ": " + details_to_str({"traceback": text_content(traceback)}) + "\n")],
)
self.assertEqual(client.testsRun, 3)
def test_non_test_characters_forwarded_immediately(self):
pass
class TestTestProtocolServerStartTest(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.stream = BytesIO()
self.protocol = subunit.TestProtocolServer(self.client, self.stream)
def test_start_test(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.assertEqual(self.client._events, [("startTest", subunit.RemotedTestCase("old mcdonald"))])
def test_start_testing(self):
self.protocol.lineReceived(b"testing old mcdonald\n")
self.assertEqual(self.client._events, [("startTest", subunit.RemotedTestCase("old mcdonald"))])
def test_start_test_colon(self):
self.protocol.lineReceived(b"test: old mcdonald\n")
self.assertEqual(self.client._events, [("startTest", subunit.RemotedTestCase("old mcdonald"))])
def test_indented_test_colon_ignored(self):
ignored_line = b" test: old mcdonald\n"
self.protocol.lineReceived(ignored_line)
self.assertEqual([], self.client._events)
self.assertEqual(self.stream.getvalue(), ignored_line)
def test_start_testing_colon(self):
self.protocol.lineReceived(b"testing: old mcdonald\n")
self.assertEqual(self.client._events, [("startTest", subunit.RemotedTestCase("old mcdonald"))])
class TestTestProtocolServerPassThrough(unittest.TestCase):
def setUp(self):
self.stdout = BytesIO()
self.test = subunit.RemotedTestCase("old mcdonald")
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client, self.stdout)
def keywords_before_test(self):
self.protocol.lineReceived(b"failure a\n")
self.protocol.lineReceived(b"failure: a\n")
self.protocol.lineReceived(b"error a\n")
self.protocol.lineReceived(b"error: a\n")
self.protocol.lineReceived(b"success a\n")
self.protocol.lineReceived(b"success: a\n")
self.protocol.lineReceived(b"successful a\n")
self.protocol.lineReceived(b"successful: a\n")
self.protocol.lineReceived(b"]\n")
self.assertEqual(
self.stdout.getvalue(),
b"failure a\nfailure: a\nerror a\nerror: a\nsuccess a\nsuccess: a\nsuccessful a\nsuccessful: a\n]\n",
)
def test_keywords_before_test(self):
self.keywords_before_test()
self.assertEqual(self.client._events, [])
def test_keywords_after_error(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"error old mcdonald\n")
self.keywords_before_test()
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, {}),
("stopTest", self.test),
],
self.client._events,
)
def test_keywords_after_failure(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"failure old mcdonald\n")
self.keywords_before_test()
self.assertEqual(
self.client._events,
[
("startTest", self.test),
("addFailure", self.test, {}),
("stopTest", self.test),
],
)
def test_keywords_after_success(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"success old mcdonald\n")
self.keywords_before_test()
self.assertEqual(
[
("startTest", self.test),
("addSuccess", self.test),
("stopTest", self.test),
],
self.client._events,
)
def test_keywords_after_test(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"failure a\n")
self.protocol.lineReceived(b"failure: a\n")
self.protocol.lineReceived(b"error a\n")
self.protocol.lineReceived(b"error: a\n")
self.protocol.lineReceived(b"success a\n")
self.protocol.lineReceived(b"success: a\n")
self.protocol.lineReceived(b"successful a\n")
self.protocol.lineReceived(b"successful: a\n")
self.protocol.lineReceived(b"]\n")
self.protocol.lineReceived(b"failure old mcdonald\n")
self.assertEqual(
self.stdout.getvalue(),
b"test old mcdonald\n"
b"failure a\n"
b"failure: a\n"
b"error a\n"
b"error: a\n"
b"success a\n"
b"success: a\n"
b"successful a\n"
b"successful: a\n"
b"]\n",
)
self.assertEqual(
self.client._events,
[
("startTest", self.test),
("addFailure", self.test, {}),
("stopTest", self.test),
],
)
def test_keywords_during_failure(self):
# A smoke test to make sure that the details parsers have control
# appropriately.
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"failure: old mcdonald [\n")
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"failure a\n")
self.protocol.lineReceived(b"failure: a\n")
self.protocol.lineReceived(b"error a\n")
self.protocol.lineReceived(b"error: a\n")
self.protocol.lineReceived(b"success a\n")
self.protocol.lineReceived(b"success: a\n")
self.protocol.lineReceived(b"successful a\n")
self.protocol.lineReceived(b"successful: a\n")
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.assertEqual(self.stdout.getvalue(), b"")
details = {}
details["traceback"] = Content(
ContentType("text", "x-traceback", {"charset": "utf8"}),
lambda: [
b"test old mcdonald\n"
b"failure a\n"
b"failure: a\n"
b"error a\n"
b"error: a\n"
b"success a\n"
b"success: a\n"
b"successful a\n"
b"successful: a\n"
b"]\n"
],
)
self.assertEqual(
self.client._events,
[
("startTest", self.test),
("addFailure", self.test, details),
("stopTest", self.test),
],
)
def test_stdout_passthrough(self):
"""Lines received which cannot be interpreted as any protocol action
should be passed through to sys.stdout.
"""
bytes = b"randombytes\n"
self.protocol.lineReceived(bytes)
self.assertEqual(self.stdout.getvalue(), bytes)
class TestTestProtocolServerLostConnection(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.test = subunit.RemotedTestCase("old mcdonald")
def test_lost_connection_no_input(self):
self.protocol.lostConnection()
self.assertEqual([], self.client._events)
def test_lost_connection_after_start(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lostConnection()
failure = subunit.RemoteError("lost connection during test 'old mcdonald'")
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, failure),
("stopTest", self.test),
],
self.client._events,
)
def test_lost_connected_after_error(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"error old mcdonald\n")
self.protocol.lostConnection()
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, {}),
("stopTest", self.test),
],
self.client._events,
)
def do_connection_lost(self, outcome, opening):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived("{} old mcdonald {}".format(outcome, opening).encode())
self.protocol.lostConnection()
failure = subunit.RemoteError("lost connection during %s report of test 'old mcdonald'" % outcome)
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, failure),
("stopTest", self.test),
],
self.client._events,
)
def test_lost_connection_during_error(self):
self.do_connection_lost("error", "[\n")
def test_lost_connection_during_error_details(self):
self.do_connection_lost("error", "[ multipart\n")
def test_lost_connected_after_failure(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"failure old mcdonald\n")
self.protocol.lostConnection()
self.assertEqual(
[
("startTest", self.test),
("addFailure", self.test, {}),
("stopTest", self.test),
],
self.client._events,
)
def test_lost_connection_during_failure(self):
self.do_connection_lost("failure", "[\n")
def test_lost_connection_during_failure_details(self):
self.do_connection_lost("failure", "[ multipart\n")
def test_lost_connection_after_success(self):
self.protocol.lineReceived(b"test old mcdonald\n")
self.protocol.lineReceived(b"success old mcdonald\n")
self.protocol.lostConnection()
self.assertEqual(
[
("startTest", self.test),
("addSuccess", self.test),
("stopTest", self.test),
],
self.client._events,
)
def test_lost_connection_during_success(self):
self.do_connection_lost("success", "[\n")
def test_lost_connection_during_success_details(self):
self.do_connection_lost("success", "[ multipart\n")
def test_lost_connection_during_skip(self):
self.do_connection_lost("skip", "[\n")
def test_lost_connection_during_skip_details(self):
self.do_connection_lost("skip", "[ multipart\n")
def test_lost_connection_during_xfail(self):
self.do_connection_lost("xfail", "[\n")
def test_lost_connection_during_xfail_details(self):
self.do_connection_lost("xfail", "[ multipart\n")
def test_lost_connection_during_uxsuccess(self):
self.do_connection_lost("uxsuccess", "[\n")
def test_lost_connection_during_uxsuccess_details(self):
self.do_connection_lost("uxsuccess", "[ multipart\n")
class TestInTestMultipart(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = subunit.RemotedTestCase("mcdonalds farm")
def test__outcome_sets_details_parser(self):
self.protocol._reading_success_details.details_parser = None
self.protocol._state._outcome(0, b"mcdonalds farm [ multipart\n", None, self.protocol._reading_success_details)
parser = self.protocol._reading_success_details.details_parser
self.assertNotEqual(None, parser)
self.assertIsInstance(parser, subunit.details.MultipartDetailsParser)
class TestTestProtocolServerAddError(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = subunit.RemotedTestCase("mcdonalds farm")
def simple_error_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
details = {}
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def test_simple_error(self):
self.simple_error_keyword("error")
def test_simple_error_colon(self):
self.simple_error_keyword("error:")
def test_error_empty_message(self):
self.protocol.lineReceived(b"error mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["traceback"] = Content(ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b""])
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def error_quoted_bracket(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm [\n" % keyword).encode())
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["traceback"] = Content(ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b"]\n"])
self.assertEqual(
[
("startTest", self.test),
("addError", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def test_error_quoted_bracket(self):
self.error_quoted_bracket("error")
def test_error_colon_quoted_bracket(self):
self.error_quoted_bracket("error:")
class TestTestProtocolServerAddFailure(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = subunit.RemotedTestCase("mcdonalds farm")
def assertFailure(self, details):
self.assertEqual(
[
("startTest", self.test),
("addFailure", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def simple_failure_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
details = {}
self.assertFailure(details)
def test_simple_failure(self):
self.simple_failure_keyword("failure")
def test_simple_failure_colon(self):
self.simple_failure_keyword("failure:")
def test_failure_empty_message(self):
self.protocol.lineReceived(b"failure mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["traceback"] = Content(ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b""])
self.assertFailure(details)
def failure_quoted_bracket(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm [\n" % keyword).encode())
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["traceback"] = Content(ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b"]\n"])
self.assertFailure(details)
def test_failure_quoted_bracket(self):
self.failure_quoted_bracket("failure")
def test_failure_colon_quoted_bracket(self):
self.failure_quoted_bracket("failure:")
class TestTestProtocolServerAddxFail(unittest.TestCase):
"""Tests for the xfail keyword."""
def setUp(self):
"""Setup a test object ready to be xfailed with details."""
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = self.client._events[-1][-1]
def simple_xfail_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
self.check_xfail()
def check_xfail(self, error_message=None):
details = {}
if error_message is not None:
details["traceback"] = Content(
ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [error_message.encode()]
)
self.assertEqual(
[
("startTest", self.test),
("addExpectedFailure", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def test_simple_xfail(self):
self.simple_xfail_keyword("xfail")
def test_simple_xfail_colon(self):
self.simple_xfail_keyword("xfail:")
def test_xfail_empty_message(self):
self.protocol.lineReceived(b"xfail mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
self.check_xfail(error_message="")
def test_xfail_quoted_bracket(self):
self.protocol.lineReceived(b"xfail mcdonalds farm [\n")
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.check_xfail("]\n")
def test_xfail_colon_quoted_bracket(self):
self.protocol.lineReceived(b"xfail: mcdonalds farm [\n")
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.check_xfail("]\n")
class TestTestProtocolServerAddunexpectedSuccess(TestCase):
"""Tests for the uxsuccess keyword."""
def setUp(self):
"""Setup a test object ready for uxsuccess with details."""
super().setUp()
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = self.client._events[-1][-1]
def simple_uxsuccess_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
self.check_uxsuccess()
def check_uxsuccess(self, error_message=None):
if error_message is not None:
details = {}
details["traceback"] = Content(
ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [error_message.encode()]
)
expected_events = [
("startTest", self.test),
("addUnexpectedSuccess", self.test, details),
("stopTest", self.test),
]
else:
expected_events = [
("startTest", self.test),
("addUnexpectedSuccess", self.test),
("stopTest", self.test),
]
self.assertEqual(expected_events, self.client._events)
def test_simple_uxsuccess(self):
self.simple_uxsuccess_keyword("uxsuccess")
def test_simple_uxsuccess_colon(self):
self.simple_uxsuccess_keyword("uxsuccess:")
def test_uxsuccess_empty_message(self):
self.protocol.lineReceived(b"uxsuccess mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
self.check_uxsuccess(error_message="")
def test_uxsuccess_quoted_bracket(self):
self.protocol.lineReceived(b"uxsuccess mcdonalds farm [\n")
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.check_uxsuccess("]\n")
def test_uxsuccess_colon_quoted_bracket(self):
self.protocol.lineReceived(b"uxsuccess: mcdonalds farm [\n")
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.check_uxsuccess("]\n")
class TestTestProtocolServerAddSkip(unittest.TestCase):
"""Tests for the skip keyword.
In Python this meets the testtools extended TestResult contract.
(See https://launchpad.net/testtools).
"""
def setUp(self):
"""Setup a test object ready to be skipped."""
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = self.client._events[-1][-1]
def assertSkip(self, reason):
details = {}
if reason is not None:
details["reason"] = Content(ContentType("text", "plain"), lambda: [reason])
self.assertEqual(
[
("startTest", self.test),
("addSkip", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def simple_skip_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
self.assertSkip(None)
def test_simple_skip(self):
self.simple_skip_keyword("skip")
def test_simple_skip_colon(self):
self.simple_skip_keyword("skip:")
def test_skip_empty_message(self):
self.protocol.lineReceived(b"skip mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
self.assertSkip(b"")
def skip_quoted_bracket(self, keyword):
# This tests it is accepted, but cannot test it is used today, because
# of not having a way to expose it in Python so far.
self.protocol.lineReceived(("%s mcdonalds farm [\n" % keyword).encode())
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
self.assertSkip(b"]\n")
def test_skip_quoted_bracket(self):
self.skip_quoted_bracket("skip")
def test_skip_colon_quoted_bracket(self):
self.skip_quoted_bracket("skip:")
class TestTestProtocolServerAddSuccess(unittest.TestCase):
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
self.protocol.lineReceived(b"test mcdonalds farm\n")
self.test = subunit.RemotedTestCase("mcdonalds farm")
def simple_success_keyword(self, keyword):
self.protocol.lineReceived(("%s mcdonalds farm\n" % keyword).encode())
self.assertEqual(
[
("startTest", self.test),
("addSuccess", self.test),
("stopTest", self.test),
],
self.client._events,
)
def test_simple_success(self):
self.simple_success_keyword("successful")
def test_simple_success_colon(self):
self.simple_success_keyword("successful:")
def assertSuccess(self, details):
self.assertEqual(
[
("startTest", self.test),
("addSuccess", self.test, details),
("stopTest", self.test),
],
self.client._events,
)
def test_success_empty_message(self):
self.protocol.lineReceived(b"success mcdonalds farm [\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["message"] = Content(ContentType("text", "plain"), lambda: [b""])
self.assertSuccess(details)
def success_quoted_bracket(self, keyword):
# This tests it is accepted, but cannot test it is used today, because
# of not having a way to expose it in Python so far.
self.protocol.lineReceived(("%s mcdonalds farm [\n" % keyword).encode())
self.protocol.lineReceived(b" ]\n")
self.protocol.lineReceived(b"]\n")
details = {}
details["message"] = Content(ContentType("text", "plain"), lambda: [b"]\n"])
self.assertSuccess(details)
def test_success_quoted_bracket(self):
self.success_quoted_bracket("success")
def test_success_colon_quoted_bracket(self):
self.success_quoted_bracket("success:")
class TestTestProtocolServerProgress(unittest.TestCase):
"""Test receipt of progress: directives."""
def test_progress_accepted_stdlib(self):
self.result = ExtendedTestResult()
self.stream = BytesIO()
self.protocol = subunit.TestProtocolServer(self.result, stream=self.stream)
self.protocol.lineReceived(b"progress: 23")
self.protocol.lineReceived(b"progress: -2")
self.protocol.lineReceived(b"progress: +4")
self.assertEqual(b"", self.stream.getvalue())
def test_progress_accepted_extended(self):
# With a progress capable TestResult, progress events are emitted.
self.result = ExtendedTestResult()
self.stream = BytesIO()
self.protocol = subunit.TestProtocolServer(self.result, stream=self.stream)
self.protocol.lineReceived(b"progress: 23")
self.protocol.lineReceived(b"progress: push")
self.protocol.lineReceived(b"progress: -2")
self.protocol.lineReceived(b"progress: pop")
self.protocol.lineReceived(b"progress: +4")
self.assertEqual(b"", self.stream.getvalue())
self.assertEqual(
[
("progress", 23, subunit.PROGRESS_SET),
("progress", None, subunit.PROGRESS_PUSH),
("progress", -2, subunit.PROGRESS_CUR),
("progress", None, subunit.PROGRESS_POP),
("progress", 4, subunit.PROGRESS_CUR),
],
self.result._events,
)
class TestTestProtocolServerStreamTags(unittest.TestCase):
"""Test managing tags on the protocol level."""
def setUp(self):
self.client = ExtendedTestResult()
self.protocol = subunit.TestProtocolServer(self.client)
def test_initial_tags(self):
self.protocol.lineReceived(b"tags: foo bar:baz quux\n")
self.assertEqual(
[
("tags", {"foo", "bar:baz", "quux"}, set()),
],
self.client._events,
)
def test_minus_removes_tags(self):
self.protocol.lineReceived(b"tags: -bar quux\n")
self.assertEqual(
[
("tags", {"quux"}, {"bar"}),
],
self.client._events,
)
def test_tags_do_not_get_set_on_test(self):
self.protocol.lineReceived(b"test mcdonalds farm\n")
test = self.client._events[0][-1]
self.assertEqual(None, getattr(test, "tags", None))
def test_tags_do_not_get_set_on_global_tags(self):
self.protocol.lineReceived(b"tags: foo bar\n")
self.protocol.lineReceived(b"test mcdonalds farm\n")
test = self.client._events[-1][-1]
self.assertEqual(None, getattr(test, "tags", None))
def test_tags_get_set_on_test_tags(self):
self.protocol.lineReceived(b"test mcdonalds farm\n")
test = self.client._events[-1][-1]
self.protocol.lineReceived(b"tags: foo bar\n")
self.protocol.lineReceived(b"success mcdonalds farm\n")
self.assertEqual(None, getattr(test, "tags", None))
class TestTestProtocolServerStreamTime(unittest.TestCase):
"""Test managing time information at the protocol level."""
def test_time_accepted_stdlib(self):
self.result = ExtendedTestResult()
self.stream = BytesIO()
self.protocol = subunit.TestProtocolServer(self.result, stream=self.stream)
self.protocol.lineReceived(b"time: 2001-12-12 12:59:59Z\n")
self.assertEqual(b"", self.stream.getvalue())
def test_time_accepted_extended(self):
self.result = ExtendedTestResult()
self.stream = BytesIO()
self.protocol = subunit.TestProtocolServer(self.result, stream=self.stream)
self.protocol.lineReceived(b"time: 2001-12-12 12:59:59Z\n")
self.assertEqual(b"", self.stream.getvalue())
self.assertEqual([("time", datetime.datetime(2001, 12, 12, 12, 59, 59, 0, iso8601.UTC))], self.result._events)
class TestRemotedTestCase(unittest.TestCase):
def test_simple(self):
test = subunit.RemotedTestCase("A test description")
self.assertRaises(NotImplementedError, test.setUp)
self.assertRaises(NotImplementedError, test.tearDown)
self.assertEqual("A test description", test.shortDescription())
self.assertEqual("A test description", test.id())
self.assertEqual("A test description (subunit.RemotedTestCase)", "%s" % test)
self.assertEqual("<subunit.RemotedTestCase description='A test description'>", "%r" % test)
result = unittest.TestResult()
test.run(result)
self.assertEqual([(test, _remote_exception_repr + ": " + "Cannot run RemotedTestCases.\n\n")], result.errors)
self.assertEqual(1, result.testsRun)
another_test = subunit.RemotedTestCase("A test description")
self.assertEqual(test, another_test)
different_test = subunit.RemotedTestCase("ofo")
self.assertNotEqual(test, different_test)
self.assertNotEqual(another_test, different_test)
class TestRemoteError(unittest.TestCase):
def test_eq(self):
error = subunit.RemoteError("Something went wrong")
another_error = subunit.RemoteError("Something went wrong")
different_error = subunit.RemoteError("boo!")
self.assertEqual(error, another_error)
self.assertNotEqual(error, different_error)
self.assertNotEqual(different_error, another_error)
def test_empty_constructor(self):
self.assertEqual(subunit.RemoteError(), subunit.RemoteError(""))
class TestExecTestCase(unittest.TestCase):
class SampleExecTestCase(subunit.ExecTestCase):
def test_sample_method(self):
"""sample-script.py"""
# the sample script runs three tests, one each
# that fails, errors and succeeds
def test_sample_method_args(self):
"""sample-script.py foo"""
# sample that will run just one test.
def test_construct(self):
test = self.SampleExecTestCase("test_sample_method")
self.assertEqual(test.script, subunit.join_dir(__file__, "sample-script.py"))
def test_args(self):
result = unittest.TestResult()
test = self.SampleExecTestCase("test_sample_method_args")
test.run(result)
self.assertEqual(1, result.testsRun)
def test_run(self):
result = ExtendedTestResult()
test = self.SampleExecTestCase("test_sample_method")
test.run(result)
mcdonald = subunit.RemotedTestCase("old mcdonald")
bing = subunit.RemotedTestCase("bing crosby")
bing_details = {}
bing_details["traceback"] = Content(
ContentType("text", "x-traceback", {"charset": "utf8"}), lambda: [b"foo.c:53:ERROR invalid state\n"]
)
an_error = subunit.RemotedTestCase("an error")
error_details = {}
self.assertEqual(
[
("startTest", mcdonald),
("addSuccess", mcdonald),
("stopTest", mcdonald),
("startTest", bing),
("addFailure", bing, bing_details),
("stopTest", bing),
("startTest", an_error),
("addError", an_error, error_details),
("stopTest", an_error),
],
result._events,
)
def test_debug(self):
test = self.SampleExecTestCase("test_sample_method")
test.debug()
def test_count_test_cases(self):
"""Run the child process and count responses to determine the count."""
def test_join_dir(self):
sibling = subunit.join_dir(__file__, "foo")
filedir = os.path.abspath(os.path.dirname(__file__))
expected = os.path.join(filedir, "foo")
self.assertEqual(sibling, expected)
class DoExecTestCase(subunit.ExecTestCase):
def test_working_script(self):
"""sample-two-script.py"""
class TestIsolatedTestCase(TestCase):
class SampleIsolatedTestCase(subunit.IsolatedTestCase):
SETUP = False
TEARDOWN = False
TEST = False
def setUp(self):
TestIsolatedTestCase.SampleIsolatedTestCase.SETUP = True
def tearDown(self):
TestIsolatedTestCase.SampleIsolatedTestCase.TEARDOWN = True
def test_sets_global_state(self):
TestIsolatedTestCase.SampleIsolatedTestCase.TEST = True
def test_construct(self):
self.SampleIsolatedTestCase("test_sets_global_state")
@skipIf(os.name != "posix", "Need a posix system for forking tests")
def test_run(self):
result = unittest.TestResult()
test = self.SampleIsolatedTestCase("test_sets_global_state")
test.run(result)
self.assertEqual(result.testsRun, 1)
self.assertEqual(self.SampleIsolatedTestCase.SETUP, False)
self.assertEqual(self.SampleIsolatedTestCase.TEARDOWN, False)
self.assertEqual(self.SampleIsolatedTestCase.TEST, False)
def test_debug(self):
pass
# test = self.SampleExecTestCase("test_sample_method")
# test.debug()
class TestIsolatedTestSuite(TestCase):
class SampleTestToIsolate(unittest.TestCase):
SETUP = False
TEARDOWN = False
TEST = False
def setUp(self):
TestIsolatedTestSuite.SampleTestToIsolate.SETUP = True
def tearDown(self):
TestIsolatedTestSuite.SampleTestToIsolate.TEARDOWN = True
def test_sets_global_state(self):
TestIsolatedTestSuite.SampleTestToIsolate.TEST = True
def test_construct(self):
subunit.IsolatedTestSuite()
@skipIf(os.name != "posix", "Need a posix system for forking tests")
def test_run(self):
result = unittest.TestResult()
suite = subunit.IsolatedTestSuite()
sub_suite = unittest.TestSuite()
sub_suite.addTest(self.SampleTestToIsolate("test_sets_global_state"))
sub_suite.addTest(self.SampleTestToIsolate("test_sets_global_state"))
suite.addTest(sub_suite)
suite.addTest(self.SampleTestToIsolate("test_sets_global_state"))
suite.run(result)
self.assertEqual(result.testsRun, 3)
self.assertEqual(self.SampleTestToIsolate.SETUP, False)
self.assertEqual(self.SampleTestToIsolate.TEARDOWN, False)
self.assertEqual(self.SampleTestToIsolate.TEST, False)
# A number of these tests produce different output depending on the
# testtools version. testtools < 2.5.0 used traceback2, which incorrectly
# included the traceback header even for an exception with no traceback.
# testtools 2.5.0 switched to the Python 3 standard library's traceback
# module, which fixes this bug. See https://bugs.python.org/issue24695.
class TestTestProtocolClient(TestCase):
def setUp(self):
super().setUp()
self.io = BytesIO()
self.protocol = subunit.TestProtocolClient(self.io)
self.unicode_test = PlaceHolder("\u2603")
self.test = TestTestProtocolClient("test_start_test")
self.sample_details = {"something": Content(ContentType("text", "plain"), lambda: [b"serialised\nform"])}
self.sample_tb_details = dict(self.sample_details)
self.sample_tb_details["traceback"] = TracebackContent(subunit.RemoteError("boo qux"), self.test)
def test_start_test(self):
"""Test startTest on a TestProtocolClient."""
self.protocol.startTest(self.test)
self.assertEqual(self.io.getvalue(), ("test: %s\n" % self.test.id()).encode())
def test_start_test_unicode_id(self):
"""Test startTest on a TestProtocolClient."""
self.protocol.startTest(self.unicode_test)
expected = b"test: " + "\u2603".encode("utf8") + b"\n"
self.assertEqual(expected, self.io.getvalue())
def test_stop_test(self):
# stopTest doesn't output anything.
self.protocol.stopTest(self.test)
self.assertEqual(self.io.getvalue(), b"")
def test_add_success(self):
"""Test addSuccess on a TestProtocolClient."""
self.protocol.addSuccess(self.test)
self.assertEqual(self.io.getvalue(), ("successful: %s\n" % self.test.id()).encode())
def test_add_outcome_unicode_id(self):
"""Test addSuccess on a TestProtocolClient."""
self.protocol.addSuccess(self.unicode_test)
expected = b"successful: " + "\u2603".encode("utf8") + b"\n"
self.assertEqual(expected, self.io.getvalue())
def test_add_success_details(self):
"""Test addSuccess on a TestProtocolClient with details."""
self.protocol.addSuccess(self.test, details=self.sample_details)
self.assertEqual(
self.io.getvalue(),
(
"successful: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n]\n" % self.test.id()
).encode(),
)
def test_add_failure(self):
"""Test addFailure on a TestProtocolClient."""
self.protocol.addFailure(self.test, subunit.RemoteError("boo qux"))
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals((("failure: %s [\n" + _remote_exception_str + ": boo qux\n" + "]\n") % self.test.id()).encode()),
# testtools >= 2.5.0
Equals(
(("failure: %s [\n" + _remote_exception_repr + ": boo qux\n" + "]\n") % self.test.id()).encode()
),
),
)
def test_add_failure_details(self):
"""Test addFailure on a TestProtocolClient with details."""
self.protocol.addFailure(self.test, details=self.sample_tb_details)
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals(
(
(
"failure: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_str_chunked + "]\n"
)
% self.test.id()
).encode()
),
# testtools >= 2.5.0
Equals(
(
(
"failure: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_repr_chunked + "]\n"
)
% self.test.id()
).encode()
),
),
)
def test_add_error(self):
"""Test stopTest on a TestProtocolClient."""
self.protocol.addError(self.test, subunit.RemoteError("phwoar crikey"))
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals((("error: %s [\n" + _remote_exception_str + ": phwoar crikey\n]\n") % self.test.id()).encode()),
# testtools >= 2.5.0
Equals((("error: %s [\n" + _remote_exception_repr + ": phwoar crikey\n]\n") % self.test.id()).encode()),
),
)
def test_add_error_details(self):
"""Test stopTest on a TestProtocolClient with details."""
self.protocol.addError(self.test, details=self.sample_tb_details)
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals(
(
(
"error: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_str_chunked + "]\n"
)
% self.test.id()
).encode()
),
# testtools >= 2.5.0
Equals(
(
(
"error: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_repr_chunked + "]\n"
)
% self.test.id()
).encode()
),
),
)
def test_add_expected_failure(self):
"""Test addExpectedFailure on a TestProtocolClient."""
self.protocol.addExpectedFailure(self.test, subunit.RemoteError("phwoar crikey"))
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals((("xfail: %s [\n" + _remote_exception_str + ": phwoar crikey\n]\n") % self.test.id()).encode()),
# testtools >= 2.5.0
Equals((("xfail: %s [\n" + _remote_exception_repr + ": phwoar crikey\n]\n") % self.test.id()).encode()),
),
)
def test_add_expected_failure_details(self):
"""Test addExpectedFailure on a TestProtocolClient with details."""
self.protocol.addExpectedFailure(self.test, details=self.sample_tb_details)
self.assertThat(
self.io.getvalue(),
MatchesAny(
# testtools < 2.5.0
Equals(
(
(
"xfail: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_str_chunked + "]\n"
)
% self.test.id()
).encode()
),
# testtools >= 2.5.0
Equals(
(
(
"xfail: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n"
"Content-Type: text/x-traceback;charset=utf8,language=python\n"
"traceback\n" + _remote_exception_repr_chunked + "]\n"
)
% self.test.id()
).encode()
),
),
)
def test_add_skip(self):
"""Test addSkip on a TestProtocolClient."""
self.protocol.addSkip(self.test, "Has it really?")
self.assertEqual(self.io.getvalue(), ("skip: %s [\nHas it really?\n]\n" % self.test.id()).encode())
def test_add_skip_details(self):
"""Test addSkip on a TestProtocolClient with details."""
details = {"reason": Content(ContentType("text", "plain"), lambda: [b"Has it really?"])}
self.protocol.addSkip(self.test, details=details)
self.assertEqual(
self.io.getvalue(),
(
"skip: %s [ multipart\nContent-Type: text/plain\nreason\nE\r\nHas it really?0\r\n]\n" % self.test.id()
).encode(),
)
def test_progress_set(self):
self.protocol.progress(23, subunit.PROGRESS_SET)
self.assertEqual(self.io.getvalue(), b"progress: 23\n")
def test_progress_neg_cur(self):
self.protocol.progress(-23, subunit.PROGRESS_CUR)
self.assertEqual(self.io.getvalue(), b"progress: -23\n")
def test_progress_pos_cur(self):
self.protocol.progress(23, subunit.PROGRESS_CUR)
self.assertEqual(self.io.getvalue(), b"progress: +23\n")
def test_progress_pop(self):
self.protocol.progress(1234, subunit.PROGRESS_POP)
self.assertEqual(self.io.getvalue(), b"progress: pop\n")
def test_progress_push(self):
self.protocol.progress(1234, subunit.PROGRESS_PUSH)
self.assertEqual(self.io.getvalue(), b"progress: push\n")
def test_time(self):
# Calling time() outputs a time signal immediately.
self.protocol.time(datetime.datetime(2009, 10, 11, 12, 13, 14, 15, iso8601.UTC))
self.assertEqual(b"time: 2009-10-11 12:13:14.000015Z\n", self.io.getvalue())
def test_add_unexpected_success(self):
"""Test addUnexpectedSuccess on a TestProtocolClient."""
self.protocol.addUnexpectedSuccess(self.test)
self.assertEqual(self.io.getvalue(), ("uxsuccess: %s\n" % self.test.id()).encode())
def test_add_unexpected_success_details(self):
"""Test addUnexpectedSuccess on a TestProtocolClient with details."""
self.protocol.addUnexpectedSuccess(self.test, details=self.sample_details)
self.assertEqual(
self.io.getvalue(),
(
"uxsuccess: %s [ multipart\n"
"Content-Type: text/plain\n"
"something\n"
"F\r\nserialised\nform0\r\n]\n" % self.test.id()
).encode(),
)
def test_tags_empty(self):
self.protocol.tags(set(), set())
self.assertEqual(b"", self.io.getvalue())
def test_tags_add(self):
self.protocol.tags({"foo"}, set())
self.assertEqual(b"tags: foo\n", self.io.getvalue())
def test_tags_both(self):
self.protocol.tags({"quux"}, {"bar"})
self.assertThat([b"tags: quux -bar\n", b"tags: -bar quux\n"], Contains(self.io.getvalue()))
def test_tags_gone(self):
self.protocol.tags(set(), {"bar"})
self.assertEqual(b"tags: -bar\n", self.io.getvalue())
-555
python/subunit/tests/test_test_protocol2.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2013 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import datetime
from io import BytesIO
try:
from hypothesis import given
# To debug hypothesis
# from hypothesis import Settings, Verbosity
# Settings.default.verbosity = Verbosity.verbose
import hypothesis.strategies as st
except ImportError:
given = None
st = None
from testtools import TestCase
from testtools.matchers import Contains, HasLength
from testtools.testresult.doubles import StreamResult
from testtools.tests.test_testresult import TestStreamResultContract
import subunit
import iso8601
CONSTANT_ENUM = b"\xb3)\x01\x0c\x03foo\x08U_\x1b"
CONSTANT_INPROGRESS = b"\xb3)\x02\x0c\x03foo\x8e\xc1-\xb5"
CONSTANT_SUCCESS = b"\xb3)\x03\x0c\x03fooE\x9d\xfe\x10"
CONSTANT_UXSUCCESS = b"\xb3)\x04\x0c\x03fooX\x98\xce\xa8"
CONSTANT_SKIP = b"\xb3)\x05\x0c\x03foo\x93\xc4\x1d\r"
CONSTANT_FAIL = b"\xb3)\x06\x0c\x03foo\x15Po\xa3"
CONSTANT_XFAIL = b"\xb3)\x07\x0c\x03foo\xde\x0c\xbc\x06"
CONSTANT_EOF = b"\xb3!\x10\x08S\x15\x88\xdc"
CONSTANT_FILE_CONTENT = b"\xb3!@\x13\x06barney\x03wooA5\xe3\x8c"
CONSTANT_MIME = b"\xb3! #\x1aapplication/foo; charset=1x3Q\x15"
CONSTANT_TIMESTAMP = b"\xb3+\x03\x13<\x17T\xcf\x80\xaf\xc8\x03barI\x96>-"
CONSTANT_ROUTE_CODE = b"\xb3-\x03\x13\x03bar\x06source\x9cY9\x19"
CONSTANT_RUNNABLE = b"\xb3(\x03\x0c\x03foo\xe3\xea\xf5\xa4"
CONSTANT_TAGS = [
b"\xb3)\x80\x15\x03bar\x02\x03foo\x03barTHn\xb4",
b"\xb3)\x80\x15\x03bar\x02\x03bar\x03foo\xf8\xf1\x91o",
]
class TestStreamResultToBytesContract(TestCase, TestStreamResultContract):
"""Check that StreamResult behaves as testtools expects."""
def _make_result(self):
return subunit.StreamResultToBytes(BytesIO())
class TestStreamResultToBytes(TestCase):
def _make_result(self):
output = BytesIO()
return subunit.StreamResultToBytes(output), output
def test_numbers(self):
result = subunit.StreamResultToBytes(BytesIO())
packet = []
self.assertRaises(Exception, result._write_number, -1, packet)
self.assertEqual([], packet)
result._write_number(0, packet)
self.assertEqual([b"\x00"], packet)
del packet[:]
result._write_number(63, packet)
self.assertEqual([b"\x3f"], packet)
del packet[:]
result._write_number(64, packet)
self.assertEqual([b"\x40\x40"], packet)
del packet[:]
result._write_number(16383, packet)
self.assertEqual([b"\x7f\xff"], packet)
del packet[:]
result._write_number(16384, packet)
self.assertEqual([b"\x80\x40", b"\x00"], packet)
del packet[:]
result._write_number(4194303, packet)
self.assertEqual([b"\xbf\xff", b"\xff"], packet)
del packet[:]
result._write_number(4194304, packet)
self.assertEqual([b"\xc0\x40\x00\x00"], packet)
del packet[:]
result._write_number(1073741823, packet)
self.assertEqual([b"\xff\xff\xff\xff"], packet)
del packet[:]
self.assertRaises(Exception, result._write_number, 1073741824, packet)
self.assertEqual([], packet)
def test_volatile_length(self):
# if the length of the packet data before the length itself is
# considered is right on the boundary for length's variable length
# encoding, it is easy to get the length wrong by not accounting for
# length itself.
# that is, the encoder has to ensure that length == sum (length_of_rest
# + length_of_length)
result, output = self._make_result()
# 1 byte short:
result.status(file_name="", file_bytes=b"\xff" * 0)
self.assertThat(output.getvalue(), HasLength(10))
self.assertEqual(b"\x0a", output.getvalue()[3:4])
output.seek(0)
output.truncate()
# 1 byte long:
result.status(file_name="", file_bytes=b"\xff" * 53)
self.assertThat(output.getvalue(), HasLength(63))
self.assertEqual(b"\x3f", output.getvalue()[3:4])
output.seek(0)
output.truncate()
# 2 bytes short
result.status(file_name="", file_bytes=b"\xff" * 54)
self.assertThat(output.getvalue(), HasLength(65))
self.assertEqual(b"\x40\x41", output.getvalue()[3:5])
output.seek(0)
output.truncate()
# 2 bytes long
result.status(file_name="", file_bytes=b"\xff" * 16371)
self.assertThat(output.getvalue(), HasLength(16383))
self.assertEqual(b"\x7f\xff", output.getvalue()[3:5])
output.seek(0)
output.truncate()
# 3 bytes short
result.status(file_name="", file_bytes=b"\xff" * 16372)
self.assertThat(output.getvalue(), HasLength(16385))
self.assertEqual(b"\x80\x40\x01", output.getvalue()[3:6])
output.seek(0)
output.truncate()
# 3 bytes long
result.status(file_name="", file_bytes=b"\xff" * 4194289)
self.assertThat(output.getvalue(), HasLength(4194303))
self.assertEqual(b"\xbf\xff\xff", output.getvalue()[3:6])
output.seek(0)
output.truncate()
self.assertRaises(Exception, result.status, file_name="", file_bytes=b"\xff" * 4194290)
def test_trivial_enumeration(self):
result, output = self._make_result()
result.status("foo", "exists")
self.assertEqual(CONSTANT_ENUM, output.getvalue())
def test_inprogress(self):
result, output = self._make_result()
result.status("foo", "inprogress")
self.assertEqual(CONSTANT_INPROGRESS, output.getvalue())
def test_success(self):
result, output = self._make_result()
result.status("foo", "success")
self.assertEqual(CONSTANT_SUCCESS, output.getvalue())
def test_uxsuccess(self):
result, output = self._make_result()
result.status("foo", "uxsuccess")
self.assertEqual(CONSTANT_UXSUCCESS, output.getvalue())
def test_skip(self):
result, output = self._make_result()
result.status("foo", "skip")
self.assertEqual(CONSTANT_SKIP, output.getvalue())
def test_fail(self):
result, output = self._make_result()
result.status("foo", "fail")
self.assertEqual(CONSTANT_FAIL, output.getvalue())
def test_xfail(self):
result, output = self._make_result()
result.status("foo", "xfail")
self.assertEqual(CONSTANT_XFAIL, output.getvalue())
def test_unknown_status(self):
result, output = self._make_result()
self.assertRaises(Exception, result.status, "foo", "boo")
self.assertEqual(b"", output.getvalue())
def test_eof(self):
result, output = self._make_result()
result.status(eof=True)
self.assertEqual(CONSTANT_EOF, output.getvalue())
def test_file_content(self):
result, output = self._make_result()
result.status(file_name="barney", file_bytes=b"woo")
self.assertEqual(CONSTANT_FILE_CONTENT, output.getvalue())
def test_mime(self):
result, output = self._make_result()
result.status(mime_type="application/foo; charset=1")
self.assertEqual(CONSTANT_MIME, output.getvalue())
def test_route_code(self):
result, output = self._make_result()
result.status(test_id="bar", test_status="success", route_code="source")
self.assertEqual(CONSTANT_ROUTE_CODE, output.getvalue())
def test_runnable(self):
result, output = self._make_result()
result.status("foo", "success", runnable=False)
self.assertEqual(CONSTANT_RUNNABLE, output.getvalue())
def test_tags(self):
result, output = self._make_result()
result.status(test_id="bar", test_tags={"foo", "bar"})
self.assertThat(CONSTANT_TAGS, Contains(output.getvalue()))
def test_timestamp(self):
timestamp = datetime.datetime(2001, 12, 12, 12, 59, 59, 45, iso8601.UTC)
result, output = self._make_result()
result.status(test_id="bar", test_status="success", timestamp=timestamp)
self.assertEqual(CONSTANT_TIMESTAMP, output.getvalue())
class TestByteStreamToStreamResult(TestCase):
def test_non_subunit_encapsulated(self):
source = BytesIO(b"foo\nbar\n")
result = StreamResult()
subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout").run(result)
self.assertEqual(
[
("status", None, None, None, True, "stdout", b"f", False, None, None, None),
("status", None, None, None, True, "stdout", b"o", False, None, None, None),
("status", None, None, None, True, "stdout", b"o", False, None, None, None),
("status", None, None, None, True, "stdout", b"\n", False, None, None, None),
("status", None, None, None, True, "stdout", b"b", False, None, None, None),
("status", None, None, None, True, "stdout", b"a", False, None, None, None),
("status", None, None, None, True, "stdout", b"r", False, None, None, None),
("status", None, None, None, True, "stdout", b"\n", False, None, None, None),
],
result._events,
)
self.assertEqual(b"", source.read())
def test_signature_middle_utf8_char(self):
utf8_bytes = b"\xe3\xb3\x8a"
source = BytesIO(utf8_bytes)
# Should be treated as one character (it is u'\u3cca') and wrapped
result = StreamResult()
subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout").run(result)
self.assertEqual(
[
("status", None, None, None, True, "stdout", b"\xe3", False, None, None, None),
("status", None, None, None, True, "stdout", b"\xb3", False, None, None, None),
("status", None, None, None, True, "stdout", b"\x8a", False, None, None, None),
],
result._events,
)
def test_non_subunit_disabled_raises(self):
source = BytesIO(b"foo\nbar\n")
result = StreamResult()
case = subunit.ByteStreamToStreamResult(source)
e = self.assertRaises(Exception, case.run, result)
self.assertEqual(b"f", e.args[1])
self.assertEqual(b"oo\nbar\n", source.read())
self.assertEqual([], result._events)
def test_trivial_enumeration(self):
source = BytesIO(CONSTANT_ENUM)
result = StreamResult()
subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout").run(result)
self.assertEqual(b"", source.read())
self.assertEqual(
[
("status", "foo", "exists", None, True, None, None, False, None, None, None),
],
result._events,
)
def test_multiple_events(self):
source = BytesIO(CONSTANT_ENUM + CONSTANT_ENUM)
result = StreamResult()
subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout").run(result)
self.assertEqual(b"", source.read())
self.assertEqual(
[
("status", "foo", "exists", None, True, None, None, False, None, None, None),
("status", "foo", "exists", None, True, None, None, False, None, None, None),
],
result._events,
)
def test_inprogress(self):
self.check_event(CONSTANT_INPROGRESS, "inprogress")
def test_success(self):
self.check_event(CONSTANT_SUCCESS, "success")
def test_uxsuccess(self):
self.check_event(CONSTANT_UXSUCCESS, "uxsuccess")
def test_skip(self):
self.check_event(CONSTANT_SKIP, "skip")
def test_fail(self):
self.check_event(CONSTANT_FAIL, "fail")
def test_xfail(self):
self.check_event(CONSTANT_XFAIL, "xfail")
def check_events(self, source_bytes, events):
source = BytesIO(source_bytes)
result = StreamResult()
subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout").run(result)
self.assertEqual(b"", source.read())
self.assertEqual(events, result._events)
# - any file attachments should be bytes equivalent [as users assume that].
for event in result._events:
if event[5] is not None:
bytes(event[6])
def check_event(
self,
source_bytes,
test_status=None,
test_id="foo",
route_code=None,
timestamp=None,
tags=None,
mime_type=None,
file_name=None,
file_bytes=None,
eof=False,
runnable=True,
):
event = self._event(
test_id=test_id,
test_status=test_status,
tags=tags,
runnable=runnable,
file_name=file_name,
file_bytes=file_bytes,
eof=eof,
mime_type=mime_type,
route_code=route_code,
timestamp=timestamp,
)
self.check_events(source_bytes, [event])
def _event(
self,
test_status=None,
test_id=None,
route_code=None,
timestamp=None,
tags=None,
mime_type=None,
file_name=None,
file_bytes=None,
eof=False,
runnable=True,
):
return (
"status",
test_id,
test_status,
tags,
runnable,
file_name,
file_bytes,
eof,
mime_type,
route_code,
timestamp,
)
def test_eof(self):
self.check_event(CONSTANT_EOF, test_id=None, eof=True)
def test_file_content(self):
self.check_event(CONSTANT_FILE_CONTENT, test_id=None, file_name="barney", file_bytes=b"woo")
def test_file_content_length_into_checksum(self):
# A bad file content length which creeps into the checksum.
bad_file_length_content = b"\xb3!@\x13\x06barney\x04woo\xdc\xe2\xdb\x35"
self.check_events(
bad_file_length_content,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=bad_file_length_content,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"File content extends past end of packet: claimed 4 bytes, 3 available",
mime_type="text/plain;charset=utf8",
),
],
)
def test_packet_length_4_word_varint(self):
packet_data = b"\xb3!@\xc0\x00\x11"
self.check_events(
packet_data,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=packet_data,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"3 byte maximum given but 4 byte value found.",
mime_type="text/plain;charset=utf8",
),
],
)
def test_mime(self):
self.check_event(CONSTANT_MIME, test_id=None, mime_type="application/foo; charset=1")
def test_route_code(self):
self.check_event(CONSTANT_ROUTE_CODE, "success", route_code="source", test_id="bar")
def test_runnable(self):
self.check_event(CONSTANT_RUNNABLE, test_status="success", runnable=False)
def test_tags(self):
self.check_event(CONSTANT_TAGS[0], None, tags={"foo", "bar"}, test_id="bar")
def test_timestamp(self):
timestamp = datetime.datetime(2001, 12, 12, 12, 59, 59, 45, iso8601.UTC)
self.check_event(CONSTANT_TIMESTAMP, "success", test_id="bar", timestamp=timestamp)
def test_bad_crc_errors_via_status(self):
file_bytes = CONSTANT_MIME[:-1] + b"\x00"
self.check_events(
file_bytes,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=file_bytes,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"Bad checksum - calculated (0x78335115), stored (0x78335100)",
mime_type="text/plain;charset=utf8",
),
],
)
def test_not_utf8_in_string(self):
file_bytes = CONSTANT_ROUTE_CODE[:5] + b"\xb4" + CONSTANT_ROUTE_CODE[6:-4] + b"\xce\x56\xc6\x17"
self.check_events(
file_bytes,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=file_bytes,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"UTF8 string at offset 2 is not UTF8",
mime_type="text/plain;charset=utf8",
),
],
)
def test_NULL_in_string(self):
file_bytes = CONSTANT_ROUTE_CODE[:6] + b"\x00" + CONSTANT_ROUTE_CODE[7:-4] + b"\xd7\x41\xac\xfe"
self.check_events(
file_bytes,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=file_bytes,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"UTF8 string at offset 2 contains NUL byte",
mime_type="text/plain;charset=utf8",
),
],
)
def test_bad_utf8_stringlength(self):
file_bytes = CONSTANT_ROUTE_CODE[:4] + b"\x3f" + CONSTANT_ROUTE_CODE[5:-4] + b"\xbe\x29\xe0\xc2"
self.check_events(
file_bytes,
[
self._event(
test_id="subunit.parser",
eof=True,
file_name="Packet data",
file_bytes=file_bytes,
mime_type="application/octet-stream",
),
self._event(
test_id="subunit.parser",
test_status="fail",
eof=True,
file_name="Parser Error",
file_bytes=b"UTF8 string at offset 2 extends past end of packet: claimed 63 bytes, 10 available",
mime_type="text/plain;charset=utf8",
),
],
)
def test_route_code_and_file_content(self):
content = BytesIO()
subunit.StreamResultToBytes(content).status(
route_code="0", mime_type="text/plain", file_name="bar", file_bytes=b"foo"
)
self.check_event(
content.getvalue(), test_id=None, file_name="bar", route_code="0", mime_type="text/plain", file_bytes=b"foo"
)
if st is not None:
@given(st.binary())
def test_hypothesis_decoding(self, code_bytes):
source = BytesIO(code_bytes)
result = StreamResult()
stream = subunit.ByteStreamToStreamResult(source, non_subunit_name="stdout")
stream.run(result)
self.assertEqual(b"", source.read())
-558
python/subunit/tests/test_test_results.py
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2009 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import csv
import datetime
import sys
import unittest
from io import StringIO
import testtools
from testtools import TestCase
from testtools.content import TracebackContent, text_content
from testtools.testresult.doubles import ExtendedTestResult
import subunit
import iso8601
import subunit.test_results
class LoggingDecorator(subunit.test_results.HookedTestResultDecorator):
def __init__(self, decorated):
self._calls = 0
super().__init__(decorated)
def _before_event(self):
self._calls += 1
class AssertBeforeTestResult(LoggingDecorator):
"""A TestResult for checking preconditions."""
def __init__(self, decorated, test):
self.test = test
super().__init__(decorated)
def _before_event(self):
self.test.assertEqual(1, self.earlier._calls)
super()._before_event()
class TimeCapturingResult(unittest.TestResult):
def __init__(self):
super().__init__()
self._calls = []
self.failfast = False
def time(self, a_datetime):
self._calls.append(a_datetime)
def progress(self, offset, whence):
pass
def addDuration(self, test, duration):
"""Called to add a test duration.
:param test: The test that completed.
:param duration: The duration of the test as a float in seconds.
"""
pass
class TestHookedTestResultDecorator(unittest.TestCase):
def setUp(self):
# An end to the chain
terminal = unittest.TestResult()
# Asserts that the call was made to self.result before asserter was
# called.
asserter = AssertBeforeTestResult(terminal, self)
# The result object we call, which much increase its call count.
self.result = LoggingDecorator(asserter)
asserter.earlier = self.result
self.decorated = asserter
def tearDown(self):
# The hook in self.result must have been called
self.assertEqual(1, self.result._calls)
# The hook in asserter must have been called too, otherwise the
# assertion about ordering won't have completed.
self.assertEqual(1, self.decorated._calls)
def test_startTest(self):
self.result.startTest(self)
def test_startTestRun(self):
self.result.startTestRun()
def test_stopTest(self):
self.result.stopTest(self)
def test_stopTestRun(self):
self.result.stopTestRun()
def test_addError(self):
self.result.addError(self, subunit.RemoteError())
def test_addError_details(self):
self.result.addError(self, details={})
def test_addFailure(self):
self.result.addFailure(self, subunit.RemoteError())
def test_addFailure_details(self):
self.result.addFailure(self, details={})
def test_addSuccess(self):
self.result.addSuccess(self)
def test_addSuccess_details(self):
self.result.addSuccess(self, details={})
def test_addSkip(self):
self.result.addSkip(self, "foo")
def test_addSkip_details(self):
self.result.addSkip(self, details={})
def test_addExpectedFailure(self):
self.result.addExpectedFailure(self, subunit.RemoteError())
def test_addExpectedFailure_details(self):
self.result.addExpectedFailure(self, details={})
def test_addUnexpectedSuccess(self):
self.result.addUnexpectedSuccess(self)
def test_addUnexpectedSuccess_details(self):
self.result.addUnexpectedSuccess(self, details={})
def test_progress(self):
self.result.progress(1, subunit.PROGRESS_SET)
def test_wasSuccessful(self):
self.result.wasSuccessful()
def test_shouldStop(self):
self.result.shouldStop
def test_stop(self):
self.result.stop()
def test_time(self):
self.result.time(None)
def test_addDuration(self):
self.result.addDuration(self, 1.5)
class TestAutoTimingTestResultDecorator(unittest.TestCase):
def setUp(self):
# And end to the chain which captures time events.
terminal = TimeCapturingResult()
# The result object under test.
self.result = subunit.test_results.AutoTimingTestResultDecorator(terminal)
self.decorated = terminal
def test_without_time_calls_time_is_called_and_not_None(self):
self.result.startTest(self)
self.assertEqual(1, len(self.decorated._calls))
self.assertNotEqual(None, self.decorated._calls[0])
def test_no_time_from_progress(self):
self.result.progress(1, subunit.PROGRESS_CUR)
self.assertEqual(0, len(self.decorated._calls))
def test_no_time_from_shouldStop(self):
self.decorated.stop()
self.result.shouldStop
self.assertEqual(0, len(self.decorated._calls))
def test_calling_time_inhibits_automatic_time(self):
# Calling time() outputs a time signal immediately and prevents
# automatically adding one when other methods are called.
time = datetime.datetime(2009, 10, 11, 12, 13, 14, 15, iso8601.UTC)
self.result.time(time)
self.result.startTest(self)
self.result.stopTest(self)
self.assertEqual(1, len(self.decorated._calls))
self.assertEqual(time, self.decorated._calls[0])
def test_calling_time_None_enables_automatic_time(self):
time = datetime.datetime(2009, 10, 11, 12, 13, 14, 15, iso8601.UTC)
self.result.time(time)
self.assertEqual(1, len(self.decorated._calls))
self.assertEqual(time, self.decorated._calls[0])
# Calling None passes the None through, in case other results care.
self.result.time(None)
self.assertEqual(2, len(self.decorated._calls))
self.assertEqual(None, self.decorated._calls[1])
# Calling other methods doesn't generate an automatic time event.
self.result.startTest(self)
self.assertEqual(3, len(self.decorated._calls))
self.assertNotEqual(None, self.decorated._calls[2])
def test_set_failfast_True(self):
self.assertFalse(self.decorated.failfast)
self.result.failfast = True
self.assertTrue(self.decorated.failfast)
class TestTagCollapsingDecorator(TestCase):
def test_tags_collapsed_outside_of_tests(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
tag_collapser.tags({"a"}, set())
tag_collapser.tags({"b"}, set())
tag_collapser.startTest(self)
self.assertEqual(
[
("tags", {"a", "b"}, set()),
("startTest", self),
],
result._events,
)
def test_tags_collapsed_outside_of_tests_are_flushed(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
tag_collapser.startTestRun()
tag_collapser.tags({"a"}, set())
tag_collapser.tags({"b"}, set())
tag_collapser.startTest(self)
tag_collapser.addSuccess(self)
tag_collapser.stopTest(self)
tag_collapser.stopTestRun()
self.assertEqual(
[
("startTestRun",),
("tags", {"a", "b"}, set()),
("startTest", self),
("addSuccess", self),
("stopTest", self),
("stopTestRun",),
],
result._events,
)
def test_tags_forwarded_after_tests(self):
test = subunit.RemotedTestCase("foo")
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
tag_collapser.startTestRun()
tag_collapser.startTest(test)
tag_collapser.addSuccess(test)
tag_collapser.stopTest(test)
tag_collapser.tags({"a"}, {"b"})
tag_collapser.stopTestRun()
self.assertEqual(
[
("startTestRun",),
("startTest", test),
("addSuccess", test),
("stopTest", test),
("tags", {"a"}, {"b"}),
("stopTestRun",),
],
result._events,
)
def test_tags_collapsed_inside_of_tests(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
test = subunit.RemotedTestCase("foo")
tag_collapser.startTest(test)
tag_collapser.tags({"a"}, set())
tag_collapser.tags({"b"}, {"a"})
tag_collapser.tags({"c"}, set())
tag_collapser.stopTest(test)
self.assertEqual([("startTest", test), ("tags", {"b", "c"}, {"a"}), ("stopTest", test)], result._events)
def test_tags_collapsed_inside_of_tests_different_ordering(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
test = subunit.RemotedTestCase("foo")
tag_collapser.startTest(test)
tag_collapser.tags(set(), {"a"})
tag_collapser.tags({"a", "b"}, set())
tag_collapser.tags({"c"}, set())
tag_collapser.stopTest(test)
self.assertEqual([("startTest", test), ("tags", {"a", "b", "c"}, set()), ("stopTest", test)], result._events)
def test_tags_sent_before_result(self):
# Because addSuccess and friends tend to send subunit output
# immediately, and because 'tags:' before a result line means
# something different to 'tags:' after a result line, we need to be
# sure that tags are emitted before 'addSuccess' (or whatever).
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
test = subunit.RemotedTestCase("foo")
tag_collapser.startTest(test)
tag_collapser.tags({"a"}, set())
tag_collapser.addSuccess(test)
tag_collapser.stopTest(test)
self.assertEqual(
[("startTest", test), ("tags", {"a"}, set()), ("addSuccess", test), ("stopTest", test)], result._events
)
class TestTimeCollapsingDecorator(TestCase):
def make_time(self):
# Heh heh.
return datetime.datetime(2000, 1, self.getUniqueInteger(), tzinfo=iso8601.UTC)
def test_initial_time_forwarded(self):
# We always forward the first time event we see.
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
a_time = self.make_time()
tag_collapser.time(a_time)
self.assertEqual([("time", a_time)], result._events)
def test_time_collapsed_to_first_and_last(self):
# If there are many consecutive time events, only the first and last
# are sent through.
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
times = [self.make_time() for i in range(5)]
for a_time in times:
tag_collapser.time(a_time)
tag_collapser.startTest(subunit.RemotedTestCase("foo"))
self.assertEqual([("time", times[0]), ("time", times[-1])], result._events[:-1])
def test_only_one_time_sent(self):
# If we receive a single time event followed by a non-time event, we
# send exactly one time event.
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
a_time = self.make_time()
tag_collapser.time(a_time)
tag_collapser.startTest(subunit.RemotedTestCase("foo"))
self.assertEqual([("time", a_time)], result._events[:-1])
def test_duplicate_times_not_sent(self):
# Many time events with the exact same time are collapsed into one
# time event.
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
a_time = self.make_time()
for i in range(5):
tag_collapser.time(a_time)
tag_collapser.startTest(subunit.RemotedTestCase("foo"))
self.assertEqual([("time", a_time)], result._events[:-1])
def test_no_times_inserted(self):
result = ExtendedTestResult()
tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
a_time = self.make_time()
tag_collapser.time(a_time)
foo = subunit.RemotedTestCase("foo")
tag_collapser.startTest(foo)
tag_collapser.addSuccess(foo)
tag_collapser.stopTest(foo)
self.assertEqual([("time", a_time), ("startTest", foo), ("addSuccess", foo), ("stopTest", foo)], result._events)
class TestByTestResultTests(testtools.TestCase):
def setUp(self):
super().setUp()
self.log = []
self.result = subunit.test_results.TestByTestResult(self.on_test)
self.result._now = iter(range(5)).__next__
def assertCalled(self, **kwargs):
defaults = {
"test": self,
"tags": set(),
"details": None,
"start_time": 0,
"stop_time": 1,
}
defaults.update(kwargs)
self.assertEqual([defaults], self.log)
def on_test(self, **kwargs):
self.log.append(kwargs)
def test_no_tests_nothing_reported(self):
self.result.startTestRun()
self.result.stopTestRun()
self.assertEqual([], self.log)
def test_add_success(self):
self.result.startTest(self)
self.result.addSuccess(self)
self.result.stopTest(self)
self.assertCalled(status="success")
def test_add_success_details(self):
self.result.startTest(self)
details = {"foo": "bar"}
self.result.addSuccess(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="success", details=details)
def test_tags(self):
if not getattr(self.result, "tags", None):
self.skipTest("No tags in testtools")
self.result.tags(["foo"], [])
self.result.startTest(self)
self.result.addSuccess(self)
self.result.stopTest(self)
self.assertCalled(status="success", tags={"foo"})
def test_add_error(self):
self.result.startTest(self)
try:
1 / 0
except ZeroDivisionError:
error = sys.exc_info()
self.result.addError(self, error)
self.result.stopTest(self)
self.assertCalled(status="error", details={"traceback": TracebackContent(error, self)})
def test_add_error_details(self):
self.result.startTest(self)
details = {"foo": text_content("bar")}
self.result.addError(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="error", details=details)
def test_add_failure(self):
self.result.startTest(self)
try:
self.fail("intentional failure")
except self.failureException:
failure = sys.exc_info()
self.result.addFailure(self, failure)
self.result.stopTest(self)
self.assertCalled(status="failure", details={"traceback": TracebackContent(failure, self)})
def test_add_failure_details(self):
self.result.startTest(self)
details = {"foo": text_content("bar")}
self.result.addFailure(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="failure", details=details)
def test_add_xfail(self):
self.result.startTest(self)
try:
1 / 0
except ZeroDivisionError:
error = sys.exc_info()
self.result.addExpectedFailure(self, error)
self.result.stopTest(self)
self.assertCalled(status="xfail", details={"traceback": TracebackContent(error, self)})
def test_add_xfail_details(self):
self.result.startTest(self)
details = {"foo": text_content("bar")}
self.result.addExpectedFailure(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="xfail", details=details)
def test_add_unexpected_success(self):
self.result.startTest(self)
details = {"foo": "bar"}
self.result.addUnexpectedSuccess(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="success", details=details)
def test_add_skip_reason(self):
self.result.startTest(self)
reason = self.getUniqueString()
self.result.addSkip(self, reason)
self.result.stopTest(self)
self.assertCalled(status="skip", details={"reason": text_content(reason)})
def test_add_skip_details(self):
self.result.startTest(self)
details = {"foo": "bar"}
self.result.addSkip(self, details=details)
self.result.stopTest(self)
self.assertCalled(status="skip", details=details)
def test_twice(self):
self.result.startTest(self)
self.result.addSuccess(self, details={"foo": "bar"})
self.result.stopTest(self)
self.result.startTest(self)
self.result.addSuccess(self)
self.result.stopTest(self)
self.assertEqual(
[
{
"test": self,
"status": "success",
"start_time": 0,
"stop_time": 1,
"tags": set(),
"details": {"foo": "bar"},
},
{"test": self, "status": "success", "start_time": 2, "stop_time": 3, "tags": set(), "details": None},
],
self.log,
)
class TestCsvResult(testtools.TestCase):
def parse_stream(self, stream):
stream.seek(0)
reader = csv.reader(stream)
return list(reader)
def test_csv_output(self):
stream = StringIO()
result = subunit.test_results.CsvResult(stream)
result._now = iter(range(5)).__next__
result.startTestRun()
result.startTest(self)
result.addSuccess(self)
result.stopTest(self)
result.stopTestRun()
self.assertEqual(
[
["test", "status", "start_time", "stop_time"],
[self.id(), "success", "0", "1"],
],
self.parse_stream(stream),
)
def test_just_header_when_no_tests(self):
stream = StringIO()
result = subunit.test_results.CsvResult(stream)
result.startTestRun()
result.stopTestRun()
self.assertEqual([["test", "status", "start_time", "stop_time"]], self.parse_stream(stream))
def test_no_output_before_events(self):
stream = StringIO()
subunit.test_results.CsvResult(stream)
self.assertEqual([], self.parse_stream(stream))
class TestTestIdPrintingResult(testtools.TestCase):
def setUp(self):
super().setUp()
self.stream = StringIO()
self.result = subunit.test_results.TestIdPrintingResult(self.stream)
def test_addDuration(self):
# addDuration should not raise an exception
test = subunit.RemotedTestCase("foo")
self.result.addDuration(test, 2.5)
# TestIdPrintingResult doesn't output anything for addDuration
self.assertEqual("", self.stream.getvalue())
vim/ftdetect/subunit.vim

Sorry, the diff of this file is not supported yet

-31
vim/README.md
# Subunit Vim Syntax Highlighting
This directory contains Vim syntax highlighting for Subunit v1 test protocol streams.
## Installation
### Option 1: Manual Installation
Copy the files to your Vim configuration directory:
```bash
cp -r syntax ~/.vim/
cp -r ftdetect ~/.vim/
```
### Option 2: Using a Plugin Manager
If you use a Vim plugin manager like vim-plug, add to your `.vimrc`:
```vim
Plug 'path/to/subunit/vim'
```
### Option 3: Pathogen
If you use Pathogen:
```bash
cd ~/.vim/bundle
ln -s /path/to/subunit/vim subunit
```
vim/syntax/subunit.vim

Sorry, the diff of this file is not supported yet