🎩 You're Invited:Meet the Socket team at Black Hat in Las Vegas, August 3-6.RSVP →
python-subunit
Artifacts are distributable versions of a package. It typically includes the source code and metadata
Advanced tools
python-subunit - pypi Package Compare versions
| name: Tests | ||
| on: | ||
| push: | ||
| branches: [ master ] | ||
| pull_request: | ||
| branches: [ master ] | ||
| jobs: | ||
| tests: | ||
| name: tests-python${{ matrix.python-version }}-${{ matrix.os }} | ||
| runs-on: ${{ matrix.os }} | ||
| strategy: | ||
| matrix: | ||
| python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', 'pypy3'] | ||
| os: ['ubuntu-latest'] | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - name: Set up Python ${{ matrix.python-version }} | ||
| uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: ${{ matrix.python-version }} | ||
| - name: Install Deps | ||
| run: sudo apt-get install check libcppunit-dev | ||
| - name: Install package | ||
| run: python -m pip install -U '.[test,docs]' | ||
| - name: Build | ||
| run: autoreconf -fi && ./configure && make | ||
| - name: Run make check | ||
| run: make check | ||
| - name: Run make distcheck | ||
| run: make distcheck | ||
| - name: Docs build | ||
| run: rst2html.py README.rst README.html |
| [build-system] | ||
| # These are the assumed default build requirements from pip: | ||
| # https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support | ||
| requires = ["setuptools>=43.0.0"] | ||
| build-backend = "setuptools.build_meta" |
| [console_scripts] | ||
| subunit-1to2 = subunit.filter_scripts.subunit_1to2:main | ||
| subunit-2to1 = subunit.filter_scripts.subunit_2to1:main | ||
| subunit-filter = subunit.filter_scripts.subunit_filter:main | ||
| subunit-ls = subunit.filter_scripts.subunit_ls:main | ||
| subunit-notify = subunit.filter_scripts.subunit_notify:main | ||
| subunit-output = subunit.filter_scripts.subunit_output:main | ||
| subunit-stats = subunit.filter_scripts.subunit_stats:main | ||
| subunit-tags = subunit.filter_scripts.subunit_tags:main | ||
| subunit2csv = subunit.filter_scripts.subunit2csv:main | ||
| subunit2disk = subunit.filter_scripts.subunit2disk:main | ||
| subunit2gtk = subunit.filter_scripts.subunit2gtk:main | ||
| subunit2junitxml = subunit.filter_scripts.subunit2junitxml:main | ||
| subunit2pyunit = subunit.filter_scripts.subunit2pyunit:main | ||
| tap2subunit = subunit.filter_scripts.tap2subunit:main |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2013 Robert Collins <robertc@robertcollins.net> | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| """Convert a version 1 subunit stream to version 2 stream.""" | ||
| from optparse import OptionParser | ||
| import sys | ||
| from testtools import ExtendedToStreamDecorator | ||
| from subunit import StreamResultToBytes | ||
| from subunit.filters import find_stream, run_tests_from_stream | ||
| def make_options(description): | ||
| parser = OptionParser(description=__doc__) | ||
| return parser | ||
| def main(): | ||
| parser = make_options(__doc__) | ||
| (options, args) = parser.parse_args() | ||
| run_tests_from_stream(find_stream(sys.stdin, args), | ||
| ExtendedToStreamDecorator(StreamResultToBytes(sys.stdout))) | ||
| sys.exit(0) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2013 Robert Collins <robertc@robertcollins.net> | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| """Convert a version 2 subunit stream to a version 1 stream.""" | ||
| from optparse import OptionParser | ||
| import sys | ||
| from testtools import ( | ||
| StreamToExtendedDecorator, | ||
| StreamResultRouter, | ||
| ) | ||
| from subunit import ByteStreamToStreamResult, TestProtocolClient | ||
| from subunit.filters import find_stream | ||
| from subunit.test_results import CatFiles | ||
| def make_options(description): | ||
| parser = OptionParser(description=__doc__) | ||
| return parser | ||
| def main(): | ||
| parser = make_options(__doc__) | ||
| (options, args) = parser.parse_args() | ||
| case = ByteStreamToStreamResult( | ||
| find_stream(sys.stdin, args), non_subunit_name='stdout') | ||
| result = StreamToExtendedDecorator(TestProtocolClient(sys.stdout)) | ||
| result = StreamResultRouter(result) | ||
| cat = CatFiles(sys.stdout) | ||
| result.add_rule(cat, 'test_id', test_id=None) | ||
| result.startTestRun() | ||
| case.run(result) | ||
| result.stopTestRun() | ||
| sys.exit(0) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 200-2013 Robert Collins <robertc@robertcollins.net> | ||
| # (C) 2009 Martin Pool | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| """Filter a subunit stream to include/exclude tests. | ||
| The default is to strip successful tests. | ||
| Tests can be filtered by Python regular expressions with --with and --without, | ||
| which match both the test name and the error text (if any). The result | ||
| contains tests which match any of the --with expressions and none of the | ||
| --without expressions. For case-insensitive matching prepend '(?i)'. | ||
| Remember to quote shell metacharacters. | ||
| """ | ||
| from optparse import OptionParser | ||
| import sys | ||
| import re | ||
| from testtools import ExtendedToStreamDecorator, StreamToExtendedDecorator | ||
| from subunit import ( | ||
| StreamResultToBytes, | ||
| read_test_list, | ||
| ) | ||
| from subunit.filters import filter_by_result, find_stream | ||
| from subunit.test_results import ( | ||
| and_predicates, | ||
| make_tag_filter, | ||
| TestResultFilter, | ||
| ) | ||
| def make_options(description): | ||
| parser = OptionParser(description=__doc__) | ||
| parser.add_option("--error", action="store_false", | ||
| help="include errors", default=False, dest="error") | ||
| parser.add_option("-e", "--no-error", action="store_true", | ||
| help="exclude errors", dest="error") | ||
| parser.add_option("--failure", action="store_false", | ||
| help="include failures", default=False, dest="failure") | ||
| parser.add_option("-f", "--no-failure", action="store_true", | ||
| help="exclude failures", dest="failure") | ||
| parser.add_option("--passthrough", action="store_false", | ||
| help="Forward non-subunit input as 'stdout'.", default=False, | ||
| dest="no_passthrough") | ||
| parser.add_option("--no-passthrough", action="store_true", | ||
| help="Discard all non subunit input.", default=False, | ||
| dest="no_passthrough") | ||
| parser.add_option("-s", "--success", action="store_false", | ||
| help="include successes", dest="success") | ||
| parser.add_option("--no-success", action="store_true", | ||
| help="exclude successes", default=True, dest="success") | ||
| parser.add_option("--no-skip", action="store_true", | ||
| help="exclude skips", dest="skip") | ||
| parser.add_option("--xfail", action="store_false", | ||
| help="include expected failures", default=True, dest="xfail") | ||
| parser.add_option("--no-xfail", action="store_true", | ||
| help="exclude expected failures", default=True, dest="xfail") | ||
| parser.add_option( | ||
| "--with-tag", type=str, | ||
| help="include tests with these tags", action="append", dest="with_tags") | ||
| parser.add_option( | ||
| "--without-tag", type=str, | ||
| help="exclude tests with these tags", action="append", dest="without_tags") | ||
| parser.add_option("-m", "--with", type=str, | ||
| help="regexp to include (case-sensitive by default)", | ||
| action="append", dest="with_regexps") | ||
| parser.add_option("--fixup-expected-failures", type=str, | ||
| help="File with list of test ids that are expected to fail; on failure " | ||
| "their result will be changed to xfail; on success they will be " | ||
| "changed to error.", dest="fixup_expected_failures", action="append") | ||
| parser.add_option("--without", type=str, | ||
| help="regexp to exclude (case-sensitive by default)", | ||
| action="append", dest="without_regexps") | ||
| parser.add_option("-F", "--only-genuine-failures", action="callback", | ||
| callback=only_genuine_failures_callback, | ||
| help="Only pass through failures and exceptions.") | ||
| parser.add_option("--rename", action="append", nargs=2, | ||
| help="Apply specified regex subsitutions to test names.", | ||
| dest="renames", default=[]) | ||
| return parser | ||
| def only_genuine_failures_callback(option, opt, value, parser): | ||
| parser.rargs.insert(0, '--no-passthrough') | ||
| parser.rargs.insert(0, '--no-xfail') | ||
| parser.rargs.insert(0, '--no-skip') | ||
| parser.rargs.insert(0, '--no-success') | ||
| def _compile_re_from_list(l): | ||
| return re.compile("|".join(l), re.MULTILINE) | ||
| def _make_regexp_filter(with_regexps, without_regexps): | ||
| """Make a callback that checks tests against regexps. | ||
| with_regexps and without_regexps are each either a list of regexp strings, | ||
| or None. | ||
| """ | ||
| with_re = with_regexps and _compile_re_from_list(with_regexps) | ||
| without_re = without_regexps and _compile_re_from_list(without_regexps) | ||
| def check_regexps(test, outcome, err, details, tags): | ||
| """Check if this test and error match the regexp filters.""" | ||
| test_str = str(test) + outcome + str(err) + str(details) | ||
| if with_re and not with_re.search(test_str): | ||
| return False | ||
| if without_re and without_re.search(test_str): | ||
| return False | ||
| return True | ||
| return check_regexps | ||
| def _compile_rename(patterns): | ||
| def rename(name): | ||
| for (from_pattern, to_pattern) in patterns: | ||
| name = re.sub(from_pattern, to_pattern, name) | ||
| return name | ||
| return rename | ||
| def _make_result(output, options, predicate): | ||
| """Make the result that we'll send the test outcomes to.""" | ||
| fixup_expected_failures = set() | ||
| for path in options.fixup_expected_failures or (): | ||
| fixup_expected_failures.update(read_test_list(path)) | ||
| return StreamToExtendedDecorator(TestResultFilter( | ||
| ExtendedToStreamDecorator( | ||
| StreamResultToBytes(output)), | ||
| filter_error=options.error, | ||
| filter_failure=options.failure, | ||
| filter_success=options.success, | ||
| filter_skip=options.skip, | ||
| filter_xfail=options.xfail, | ||
| filter_predicate=predicate, | ||
| fixup_expected_failures=fixup_expected_failures, | ||
| rename=_compile_rename(options.renames))) | ||
| def main(): | ||
| parser = make_options(__doc__) | ||
| (options, args) = parser.parse_args() | ||
| regexp_filter = _make_regexp_filter( | ||
| options.with_regexps, options.without_regexps) | ||
| tag_filter = make_tag_filter(options.with_tags, options.without_tags) | ||
| filter_predicate = and_predicates([regexp_filter, tag_filter]) | ||
| filter_by_result( | ||
| lambda output_to: _make_result(sys.stdout, options, filter_predicate), | ||
| output_path=None, | ||
| passthrough=(not options.no_passthrough), | ||
| forward=False, | ||
| protocol_version=2, | ||
| input_stream=find_stream(sys.stdin, args)) | ||
| sys.exit(0) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2008 Robert Collins <robertc@robertcollins.net> | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| """List tests in a subunit stream.""" | ||
| from optparse import OptionParser | ||
| import sys | ||
| from testtools import ( | ||
| CopyStreamResult, StreamResultRouter, | ||
| StreamSummary) | ||
| from subunit import ByteStreamToStreamResult | ||
| from subunit.filters import find_stream | ||
| from subunit.test_results import ( | ||
| CatFiles, | ||
| TestIdPrintingResult, | ||
| ) | ||
| def main(): | ||
| parser = OptionParser(description=__doc__) | ||
| parser.add_option("--times", action="store_true", | ||
| help="list the time each test took (requires a timestamped stream)", | ||
| default=False) | ||
| parser.add_option("--exists", action="store_true", | ||
| help="list tests that are reported as existing (as well as ran)", | ||
| default=False) | ||
| parser.add_option("--no-passthrough", action="store_true", | ||
| help="Hide all non subunit input.", default=False, dest="no_passthrough") | ||
| (options, args) = parser.parse_args() | ||
| test = ByteStreamToStreamResult( | ||
| find_stream(sys.stdin, args), non_subunit_name="stdout") | ||
| result = TestIdPrintingResult(sys.stdout, options.times, options.exists) | ||
| if not options.no_passthrough: | ||
| result = StreamResultRouter(result) | ||
| cat = CatFiles(sys.stdout) | ||
| result.add_rule(cat, 'test_id', test_id=None) | ||
| summary = StreamSummary() | ||
| result = CopyStreamResult([result, summary]) | ||
| result.startTestRun() | ||
| test.run(result) | ||
| result.stopTestRun() | ||
| if summary.wasSuccessful(): | ||
| exit_code = 0 | ||
| else: | ||
| exit_code = 1 | ||
| sys.exit(exit_code) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org> | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| """Notify the user of a finished test run.""" | ||
| import gi | ||
| gi.require_version('Gtk', '3.0') | ||
| from gi.repository import Notify | ||
| from testtools import StreamToExtendedDecorator | ||
| from subunit import TestResultStats | ||
| from subunit.filters import run_filter_script | ||
| if not Notify.init("Subunit-notify"): | ||
| sys.exit(1) | ||
| def notify_of_result(result): | ||
| result = result.decorated | ||
| if result.failed_tests > 0: | ||
| summary = "Test run failed" | ||
| else: | ||
| summary = "Test run successful" | ||
| body = "Total tests: %d; Passed: %d; Failed: %d" % ( | ||
| result.total_tests, | ||
| result.passed_tests, | ||
| result.failed_tests, | ||
| ) | ||
| nw = Notify.Notification(summary, body) | ||
| nw.show() | ||
| def main(): | ||
| run_filter_script( | ||
| lambda output:StreamToExtendedDecorator(TestResultStats(output)), | ||
| __doc__, notify_of_result, protocol_version=2) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2013 Subunit Contributors | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| """A command-line tool to generate a subunit result byte-stream.""" | ||
| import sys | ||
| from subunit._output import output_main | ||
| def main(): | ||
| sys.exit(output_main()) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2009 Robert Collins <robertc@robertcollins.net> | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| """Filter a subunit stream to get aggregate statistics.""" | ||
| import sys | ||
| from testtools import StreamToExtendedDecorator | ||
| from subunit import TestResultStats | ||
| from subunit.filters import run_filter_script | ||
| def main(): | ||
| result = TestResultStats(sys.stdout) | ||
| def show_stats(r): | ||
| r.decorated.formatStats() | ||
| run_filter_script( | ||
| lambda output:StreamToExtendedDecorator(result), | ||
| __doc__, show_stats, protocol_version=2, passthrough_subunit=False) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2009 Robert Collins <robertc@robertcollins.net> | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| """A filter to change tags on a subunit stream. | ||
| subunit-tags foo -> adds foo | ||
| subunit-tags foo -bar -> adds foo and removes bar | ||
| """ | ||
| import sys | ||
| from subunit import tag_stream | ||
| def main(): | ||
| sys.exit(tag_stream(sys.stdin, sys.stdout, sys.argv[1:])) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2009 Robert Collins <robertc@robertcollins.net> | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is d on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| """Turn a subunit stream into a CSV""" | ||
| from testtools import StreamToExtendedDecorator | ||
| from subunit.filters import run_filter_script | ||
| from subunit.test_results import CsvResult | ||
| def main(): | ||
| run_filter_script(lambda output: StreamToExtendedDecorator( | ||
| CsvResult(output)), __doc__, protocol_version=2) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2013 Subunit Contributors | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| """Export a stream to files and directories on disk.""" | ||
| import sys | ||
| from subunit._to_disk import to_disk | ||
| def main(): | ||
| sys.exit(to_disk()) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2009 Robert Collins <robertc@robertcollins.net> | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| ### The GTK progress bar __init__ function is derived from the pygtk tutorial: | ||
| # The PyGTK Tutorial is Copyright (C) 2001-2005 John Finlay. | ||
| # | ||
| # The GTK Tutorial is Copyright (C) 1997 Ian Main. | ||
| # | ||
| # Copyright (C) 1998-1999 Tony Gale. | ||
| # | ||
| # Permission is granted to make and distribute verbatim copies of this manual | ||
| # provided the copyright notice and this permission notice are preserved on all | ||
| # copies. | ||
| # | ||
| # Permission is granted to copy and distribute modified versions of this | ||
| # document under the conditions for verbatim copying, provided that this | ||
| # copyright notice is included exactly as in the original, and that the entire | ||
| # resulting derived work is distributed under the terms of a permission notice | ||
| # identical to this one. | ||
| # | ||
| # Permission is granted to copy and distribute translations of this document | ||
| # into another language, under the above conditions for modified versions. | ||
| # | ||
| # If you are intending to incorporate this document into a published work, | ||
| # please contact the maintainer, and we will make an effort to ensure that you | ||
| # have the most up to date information available. | ||
| # | ||
| # There is no guarantee that this document lives up to its intended purpose. | ||
| # This is simply provided as a free resource. As such, the authors and | ||
| # maintainers of the information provided within can not make any guarantee | ||
| # that the information is even accurate. | ||
| """Display a subunit stream in a gtk progress window.""" | ||
| import sys | ||
| import threading | ||
| import unittest | ||
| import gi | ||
| gi.require_version('Gtk', '3.0') | ||
| from gi.repository import Gtk, GObject | ||
| from testtools import StreamToExtendedDecorator | ||
| from subunit import ( | ||
| PROGRESS_POP, | ||
| PROGRESS_PUSH, | ||
| PROGRESS_SET, | ||
| ByteStreamToStreamResult, | ||
| ) | ||
| from subunit.progress_model import ProgressModel | ||
| class GTKTestResult(unittest.TestResult): | ||
| def __init__(self): | ||
| super(GTKTestResult, self).__init__() | ||
| # Instance variables (in addition to TestResult) | ||
| self.window = None | ||
| self.run_label = None | ||
| self.ok_label = None | ||
| self.not_ok_label = None | ||
| self.total_tests = None | ||
| self.window = Gtk.Window(Gtk.WindowType.TOPLEVEL) | ||
| self.window.set_resizable(True) | ||
| self.window.connect("destroy", Gtk.main_quit) | ||
| self.window.set_title("Tests...") | ||
| self.window.set_border_width(0) | ||
| vbox = Gtk.VBox(False, 5) | ||
| vbox.set_border_width(10) | ||
| self.window.add(vbox) | ||
| vbox.show() | ||
| # Create a centering alignment object | ||
| align = Gtk.Alignment.new(0.5, 0.5, 0, 0) | ||
| vbox.pack_start(align, False, False, 5) | ||
| align.show() | ||
| # Create the ProgressBar | ||
| self.pbar = Gtk.ProgressBar() | ||
| align.add(self.pbar) | ||
| self.pbar.set_text("Running") | ||
| self.pbar.show() | ||
| self.progress_model = ProgressModel() | ||
| separator = Gtk.HSeparator() | ||
| vbox.pack_start(separator, False, False, 0) | ||
| separator.show() | ||
| # rows, columns, homogeneous | ||
| table = Gtk.Table(2, 3, False) | ||
| vbox.pack_start(table, False, True, 0) | ||
| table.show() | ||
| # Show summary details about the run. Could use an expander. | ||
| label = Gtk.Label(label="Run:") | ||
| table.attach(label, 0, 1, 1, 2, Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, | ||
| Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, 5, 5) | ||
| label.show() | ||
| self.run_label = Gtk.Label(label="N/A") | ||
| table.attach(self.run_label, 1, 2, 1, 2, Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, | ||
| Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, 5, 5) | ||
| self.run_label.show() | ||
| label = Gtk.Label(label="OK:") | ||
| table.attach(label, 0, 1, 2, 3, Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, | ||
| Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, 5, 5) | ||
| label.show() | ||
| self.ok_label = Gtk.Label(label="N/A") | ||
| table.attach(self.ok_label, 1, 2, 2, 3, Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, | ||
| Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, 5, 5) | ||
| self.ok_label.show() | ||
| label = Gtk.Label(label="Not OK:") | ||
| table.attach(label, 0, 1, 3, 4, Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, | ||
| Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, 5, 5) | ||
| label.show() | ||
| self.not_ok_label = Gtk.Label(label="N/A") | ||
| table.attach(self.not_ok_label, 1, 2, 3, 4, Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, | ||
| Gtk.AttachOptions.EXPAND | Gtk.AttachOptions.FILL, 5, 5) | ||
| self.not_ok_label.show() | ||
| self.window.show() | ||
| # For the demo. | ||
| self.window.set_keep_above(True) | ||
| self.window.present() | ||
| def stopTest(self, test): | ||
| super(GTKTestResult, self).stopTest(test) | ||
| GObject.idle_add(self._stopTest) | ||
| def _stopTest(self): | ||
| self.progress_model.advance() | ||
| if self.progress_model.width() == 0: | ||
| self.pbar.pulse() | ||
| else: | ||
| pos = self.progress_model.pos() | ||
| width = self.progress_model.width() | ||
| percentage = (pos / float(width)) | ||
| self.pbar.set_fraction(percentage) | ||
| def stopTestRun(self): | ||
| try: | ||
| super(GTKTestResult, self).stopTestRun() | ||
| except AttributeError: | ||
| pass | ||
| GObject.idle_add(self.pbar.set_text, 'Finished') | ||
| def addError(self, test, err): | ||
| super(GTKTestResult, self).addError(test, err) | ||
| GObject.idle_add(self.update_counts) | ||
| def addFailure(self, test, err): | ||
| super(GTKTestResult, self).addFailure(test, err) | ||
| GObject.idle_add(self.update_counts) | ||
| def addSuccess(self, test): | ||
| super(GTKTestResult, self).addSuccess(test) | ||
| GObject.idle_add(self.update_counts) | ||
| def addSkip(self, test, reason): | ||
| super(GTKTestResult, self).addSkip(test, reason) | ||
| GObject.idle_add(self.update_counts) | ||
| def addExpectedFailure(self, test, err): | ||
| super(GTKTestResult, self).addExpectedFailure(test, err) | ||
| GObject.idle_add(self.update_counts) | ||
| def addUnexpectedSuccess(self, test): | ||
| super(GTKTestResult, self).addUnexpectedSuccess(test) | ||
| GObject.idle_add(self.update_counts) | ||
| def progress(self, offset, whence): | ||
| if whence == PROGRESS_PUSH: | ||
| self.progress_model.push() | ||
| elif whence == PROGRESS_POP: | ||
| self.progress_model.pop() | ||
| elif whence == PROGRESS_SET: | ||
| self.total_tests = offset | ||
| self.progress_model.set_width(offset) | ||
| else: | ||
| self.total_tests += offset | ||
| self.progress_model.adjust_width(offset) | ||
| def time(self, a_datetime): | ||
| # We don't try to estimate completion yet. | ||
| pass | ||
| def update_counts(self): | ||
| self.run_label.set_text(str(self.testsRun)) | ||
| bad = len(self.failures + self.errors) | ||
| self.ok_label.set_text(str(self.testsRun - bad)) | ||
| self.not_ok_label.set_text(str(bad)) | ||
| def main(): | ||
| GObject.threads_init() | ||
| result = StreamToExtendedDecorator(GTKTestResult()) | ||
| test = ByteStreamToStreamResult(sys.stdin, non_subunit_name='stdout') | ||
| # Get setup | ||
| while Gtk.events_pending(): | ||
| Gtk.main_iteration() | ||
| # Start IO | ||
| def run_and_finish(): | ||
| test.run(result) | ||
| result.stopTestRun() | ||
| t = threading.Thread(target=run_and_finish) | ||
| t.daemon = True | ||
| result.startTestRun() | ||
| t.start() | ||
| Gtk.main() | ||
| if result.decorated.wasSuccessful(): | ||
| exit_code = 0 | ||
| else: | ||
| exit_code = 1 | ||
| sys.exit(exit_code) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2009 Robert Collins <robertc@robertcollins.net> | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| """Filter a subunit stream to get aggregate statistics.""" | ||
| import sys | ||
| from testtools import StreamToExtendedDecorator | ||
| from subunit.filters import run_filter_script | ||
| try: | ||
| from junitxml import JUnitXmlResult | ||
| except ImportError: | ||
| sys.stderr.write("python-junitxml (https://launchpad.net/pyjunitxml or " | ||
| "http://pypi.python.org/pypi/junitxml) is required for this filter.") | ||
| raise | ||
| def main(): | ||
| run_filter_script( | ||
| lambda output: StreamToExtendedDecorator( | ||
| JUnitXmlResult(output)), __doc__, protocol_version=2) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2009 Robert Collins <robertc@robertcollins.net> | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| """Display a subunit stream through python's unittest test runner.""" | ||
| from operator import methodcaller | ||
| from optparse import OptionParser | ||
| import sys | ||
| import unittest | ||
| from testtools import StreamToExtendedDecorator, DecorateTestCaseResult, StreamResultRouter | ||
| from subunit import ByteStreamToStreamResult | ||
| from subunit.filters import find_stream | ||
| from subunit.test_results import CatFiles | ||
| def main(): | ||
| parser = OptionParser(description=__doc__) | ||
| parser.add_option("--no-passthrough", action="store_true", | ||
| help="Hide all non subunit input.", | ||
| default=False, dest="no_passthrough") | ||
| parser.add_option("--progress", action="store_true", | ||
| help="Use bzrlib's test reporter (requires bzrlib)", | ||
| default=False) | ||
| (options, args) = parser.parse_args() | ||
| test = ByteStreamToStreamResult( | ||
| find_stream(sys.stdin, args), non_subunit_name='stdout') | ||
| def wrap_result(result): | ||
| result = StreamToExtendedDecorator(result) | ||
| if not options.no_passthrough: | ||
| result = StreamResultRouter(result) | ||
| result.add_rule(CatFiles(sys.stdout), 'test_id', test_id=None) | ||
| return result | ||
| test = DecorateTestCaseResult(test, wrap_result, | ||
| before_run=methodcaller('startTestRun'), | ||
| after_run=methodcaller('stopTestRun')) | ||
| if options.progress: | ||
| from bzrlib.tests import TextTestRunner | ||
| from bzrlib import ui | ||
| ui.ui_factory = ui.make_ui_for_terminal(None, sys.stdout, sys.stderr) | ||
| runner = TextTestRunner() | ||
| else: | ||
| runner = unittest.TextTestRunner(verbosity=2) | ||
| if runner.run(test).wasSuccessful(): | ||
| exit_code = 0 | ||
| else: | ||
| exit_code = 1 | ||
| sys.exit(exit_code) | ||
| if __name__ == '__main__': | ||
| main() |
| #!/usr/bin/env python3 | ||
| # subunit: extensions to python unittest to get test results from subprocesses. | ||
| # Copyright (C) 2009 Robert Collins <robertc@robertcollins.net> | ||
| # | ||
| # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| # license at the users choice. A copy of both licenses are available in the | ||
| # project source as Apache-2.0 and BSD. You may not use this file except in | ||
| # compliance with one of these two licences. | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| # license you chose for the specific language governing permissions and | ||
| # limitations under that license. | ||
| # | ||
| """A filter that reads a TAP stream and outputs a subunit stream. | ||
| More information on TAP is available at | ||
| http://testanything.org/wiki/index.php/Main_Page. | ||
| """ | ||
| import sys | ||
| from subunit import TAP2SubUnit | ||
| def main(): | ||
| sys.exit(TAP2SubUnit(sys.stdin, sys.stdout)) | ||
| if __name__ == '__main__': | ||
| main() |
+42
-0
@@ -8,2 +8,44 @@ --------------------- | ||
| IMPROVEMENTS | ||
| ~~~~~~~~~~~~ | ||
| BUG FIXES | ||
| ~~~~~~~~~ | ||
| 1.4.1 | ||
| IMPROVEMENTS | ||
| ~~~~~~~~~~~~ | ||
| * Add support for Python 3.9 | ||
| (Thomas Grainger) | ||
| * Add support for Python 3.10 | ||
| (Stephen Finucane) | ||
| * Drop support for Python 2.7, 3.4, and 3.5 | ||
| (Stephen Finucane) | ||
| * Convert python scripts to entry_points. | ||
| (Matthew Treinish) | ||
| * Migrate CI from travis to GitHub actions. | ||
| (Matthew Treinish) | ||
| * Add options to output filter to set timestamps. | ||
| (Matthew Treinish) | ||
| * Remove dependency on unittest2. | ||
| (Matěj Cepl) | ||
| BUGFIXES | ||
| ~~~~~~~~ | ||
| * Fix tests with testtools >= 2.5.0. | ||
| (Colin Watson) | ||
| * Mark rawstrings as such, fixing warnings. | ||
| (Stephen Finucane) | ||
| 1.4.0 | ||
@@ -10,0 +52,0 @@ ----- |
+499
-498
| Metadata-Version: 2.1 | ||
| Name: python-subunit | ||
| Version: 1.4.0 | ||
| Version: 1.4.1 | ||
| Summary: Python implementation of subunit test streaming protocol | ||
@@ -11,509 +11,510 @@ Home-page: http://launchpad.net/subunit | ||
| Project-URL: Source Code, https://github.com/testing-cabal/subunit/ | ||
| Description: | ||
| subunit: A streaming protocol for test results | ||
| Copyright (C) 2005-2013 Robert Collins <robertc@robertcollins.net> | ||
| Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| license at the users choice. A copy of both licenses are available in the | ||
| project source as Apache-2.0 and BSD. You may not use this file except in | ||
| compliance with one of these two licences. | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| license you chose for the specific language governing permissions and | ||
| limitations under that license. | ||
| See the COPYING file for full details on the licensing of Subunit. | ||
| subunit reuses iso8601 by Michael Twomey, distributed under an MIT style | ||
| licence - see python/iso8601/LICENSE for details. | ||
| Subunit | ||
| ------- | ||
| Subunit is a streaming protocol for test results. | ||
| There are two major revisions of the protocol. Version 1 was trivially human | ||
| readable but had significant defects as far as highly parallel testing was | ||
| concerned - it had no room for doing discovery and execution in parallel, | ||
| required substantial buffering when multiplexing and was fragile - a corrupt | ||
| byte could cause an entire stream to be misparsed. Version 1.1 added | ||
| encapsulation of binary streams which mitigated some of the issues but the | ||
| core remained. | ||
| Version 2 shares many of the good characteristics of Version 1 - it can be | ||
| embedded into a regular text stream (e.g. from a build system) and it still | ||
| models xUnit style test execution. It also fixes many of the issues with | ||
| Version 1 - Version 2 can be multiplexed without excessive buffering (in | ||
| time or space), it has a well defined recovery mechanism for dealing with | ||
| corrupted streams (e.g. where two processes write to the same stream | ||
| concurrently, or where the stream generator suffers a bug). | ||
| More details on both protocol version s can be found in the 'Protocol' section | ||
| of this document. | ||
| Subunit comes with command line filters to process a subunit stream and | ||
| language bindings for python, C, C++ and shell. Bindings are easy to write | ||
| for other languages. | ||
| A number of useful things can be done easily with subunit: | ||
| * Test aggregation: Tests run separately can be combined and then | ||
| reported/displayed together. For instance, tests from different languages | ||
| can be shown as a seamless whole, and tests running on multiple machines | ||
| can be aggregated into a single stream through a multiplexer. | ||
| * Test archiving: A test run may be recorded and replayed later. | ||
| * Test isolation: Tests that may crash or otherwise interact badly with each | ||
| other can be run seperately and then aggregated, rather than interfering | ||
| with each other or requiring an adhoc test->runner reporting protocol. | ||
| * Grid testing: subunit can act as the necessary serialisation and | ||
| deserialiation to get test runs on distributed machines to be reported in | ||
| real time. | ||
| Subunit supplies the following filters: | ||
| * tap2subunit - convert perl's TestAnythingProtocol to subunit. | ||
| * subunit2csv - convert a subunit stream to csv. | ||
| * subunit2disk - export a subunit stream to files on disk. | ||
| * subunit2pyunit - convert a subunit stream to pyunit test results. | ||
| * subunit2gtk - show a subunit stream in GTK. | ||
| * subunit2junitxml - convert a subunit stream to JUnit's XML format. | ||
| * subunit-diff - compare two subunit streams. | ||
| * subunit-filter - filter out tests from a subunit stream. | ||
| * subunit-ls - list info about tests present in a subunit stream. | ||
| * subunit-stats - generate a summary of a subunit stream. | ||
| * subunit-tags - add or remove tags from a stream. | ||
| Integration with other tools | ||
| ---------------------------- | ||
| Subunit's language bindings act as integration with various test runners like | ||
| 'check', 'cppunit', Python's 'unittest'. Beyond that a small amount of glue | ||
| (typically a few lines) will allow Subunit to be used in more sophisticated | ||
| ways. | ||
| Python | ||
| ====== | ||
| Subunit has excellent Python support: most of the filters and tools are written | ||
| in python and there are facilities for using Subunit to increase test isolation | ||
| seamlessly within a test suite. | ||
| The most common way is to run an existing python test suite and have it output | ||
| subunit via the ``subunit.run`` module:: | ||
| $ python -m subunit.run mypackage.tests.test_suite | ||
| For more information on the Python support Subunit offers , please see | ||
| ``pydoc subunit``, or the source in ``python/subunit/`` | ||
| C | ||
| = | ||
| Subunit has C bindings to emit the protocol. The 'check' C unit testing project | ||
| has included subunit support in their project for some years now. See | ||
| 'c/README' for more details. | ||
| C++ | ||
| === | ||
| The C library is includable and usable directly from C++. A TestListener for | ||
| CPPUnit is included in the Subunit distribution. See 'c++/README' for details. | ||
| shell | ||
| ===== | ||
| There are two sets of shell tools. There are filters, which accept a subunit | ||
| stream on stdin and output processed data (or a transformed stream) on stdout. | ||
| Then there are unittest facilities similar to those for C : shell bindings | ||
| consisting of simple functions to output protocol elements, and a patch for | ||
| adding subunit output to the 'ShUnit' shell test runner. See 'shell/README' for | ||
| details. | ||
| Filter recipes | ||
| -------------- | ||
| To ignore some failing tests whose root cause is already known:: | ||
| subunit-filter --without 'AttributeError.*flavor' | ||
| The xUnit test model | ||
| -------------------- | ||
| Subunit implements a slightly modified xUnit test model. The stock standard | ||
| model is that there are tests, which have an id(), can be run, and when run | ||
| start, emit an outcome (like success or failure) and then finish. | ||
| Subunit extends this with the idea of test enumeration (find out about tests | ||
| a runner has without running them), tags (allow users to describe tests in | ||
| ways the test framework doesn't apply any semantic value to), file attachments | ||
| (allow arbitrary data to make analysing a failure easy) and timestamps. | ||
| The protocol | ||
| ------------ | ||
| Version 2, or v2 is new and still under development, but is intended to | ||
| supercede version 1 in the very near future. Subunit's bundled tools accept | ||
| only version 2 and only emit version 2, but the new filters subunit-1to2 and | ||
| subunit-2to1 can be used to interoperate with older third party libraries. | ||
| Version 2 | ||
| ========= | ||
| Version 2 is a binary protocol consisting of independent packets that can be | ||
| embedded in the output from tools like make - as long as each packet has no | ||
| other bytes mixed in with it (which 'make -j N>1' has a tendency of doing). | ||
| Version 2 is currently in draft form, and early adopters should be willing | ||
| to either discard stored results (if protocol changes are made), or bulk | ||
| convert them back to v1 and then to a newer edition of v2. | ||
| The protocol synchronises at the start of the stream, after a packet, or | ||
| after any 0x0A byte. That is, a subunit v2 packet starts after a newline or | ||
| directly after the end of the prior packet. | ||
| Subunit is intended to be transported over a reliable streaming protocol such | ||
| as TCP. As such it does not concern itself with out of order delivery of | ||
| packets. However, because of the possibility of corruption due to either | ||
| bugs in the sender, or due to mixed up data from concurrent writes to the same | ||
| fd when being embedded, subunit strives to recover reasonably gracefully from | ||
| damaged data. | ||
| A key design goal for Subunit version 2 is to allow processing and multiplexing | ||
| without forcing buffering for semantic correctness, as buffering tends to hide | ||
| hung or otherwise misbehaving tests. That said, limited time based buffering | ||
| for network efficiency is a good idea - this is ultimately implementator | ||
| choice. Line buffering is also discouraged for subunit streams, as dropping | ||
| into a debugger or other tool may require interactive traffic even if line | ||
| buffering would not otherwise be a problem. | ||
| In version two there are two conceptual events - a test status event and a file | ||
| attachment event. Events may have timestamps, and the path of multiplexers that | ||
| an event is routed through is recorded to permit sending actions back to the | ||
| source (such as new tests to run or stdin for driving debuggers and other | ||
| interactive input). Test status events are used to enumerate tests, to report | ||
| tests and test helpers as they run. Tests may have tags, used to allow | ||
| tunnelling extra meanings through subunit without requiring parsing of | ||
| arbitrary file attachments. Things that are not standalone tests get marked | ||
| as such by setting the 'Runnable' flag to false. (For instance, individual | ||
| assertions in TAP are not runnable tests, only the top level TAP test script | ||
| is runnable). | ||
| File attachments are used to provide rich detail about the nature of a failure. | ||
| File attachments can also be used to encapsulate stdout and stderr both during | ||
| and outside tests. | ||
| Most numbers are stored in network byte order - Most Significant Byte first | ||
| encoded using a variation of http://www.dlugosz.com/ZIP2/VLI.html. The first | ||
| byte's top 2 high order bits encode the total number of octets in the number. | ||
| This encoding can encode values from 0 to 2**30-1, enough to encode a | ||
| nanosecond. Numbers that are not variable length encoded are still stored in | ||
| MSB order. | ||
| +--------+--------+---------+------------+ | ||
| | prefix | octets | max | max | | ||
| +========+========+=========+============+ | ||
| | 00 | 1 | 2**6-1 | 63 | | ||
| +--------+--------+---------+------------+ | ||
| | 01 | 2 | 2**14-1 | 16383 | | ||
| +--------+--------+---------+------------+ | ||
| | 10 | 3 | 2**22-1 | 4194303 | | ||
| +--------+--------+---------+------------+ | ||
| | 11 | 4 | 2**30-1 | 1073741823 | | ||
| +--------+--------+---------+------------+ | ||
| All variable length elements of the packet are stored with a length prefix | ||
| number allowing them to be skipped over for consumers that don't need to | ||
| interpret them. | ||
| UTF-8 strings are with no terminating NUL and should not have any embedded NULs | ||
| (implementations SHOULD validate any such strings that they process and take | ||
| some remedial action (such as discarding the packet as corrupt). | ||
| In short the structure of a packet is: | ||
| PACKET := SIGNATURE FLAGS PACKET_LENGTH TIMESTAMP? TESTID? TAGS? MIME? | ||
| FILECONTENT? ROUTING_CODE? CRC32 | ||
| In more detail... | ||
| Packets are identified by a single byte signature - 0xB3, which is never legal | ||
| in a UTF-8 stream as the first byte of a character. 0xB3 starts with the first | ||
| bit set and the second not, which is the UTF-8 signature for a continuation | ||
| byte. 0xB3 was chosen as 0x73 ('s' in ASCII') with the top two bits replaced by | ||
| the 1 and 0 for a continuation byte. | ||
| If subunit packets are being embedded in a non-UTF-8 text stream, where 0x73 is | ||
| a legal character, consider either recoding the text to UTF-8, or using | ||
| subunit's 'file' packets to embed the text stream in subunit, rather than the | ||
| other way around. | ||
| Following the signature byte comes a 16-bit flags field, which includes a | ||
| 4-bit version field - if the version is not 0x2 then the packet cannot be | ||
| read. It is recommended to signal an error at this point (e.g. by emitting | ||
| a synthetic error packet and returning to the top level loop to look for | ||
| new packets, or exiting with an error). If recovery is desired, treat the | ||
| packet signature as an opaque byte and scan for a new synchronisation point. | ||
| NB: Subunit V1 and V2 packets may legitimately included 0xB3 internally, | ||
| as they are an 8-bit safe container format, so recovery from this situation | ||
| may involve an arbitrary number of false positives until an actual packet | ||
| is encountered : and even then it may still be false, failing after passing | ||
| the version check due to coincidence. | ||
| Flags are stored in network byte order too. | ||
| +------------+------------+------------------------+ | ||
| | High byte | Low byte | | ||
| +------------+------------+------------------------+ | ||
| | 15 14 13 12 11 10 9 8 | 7 6 5 4 3 2 1 0 | | ||
| +------------+------------+------------------------+ | ||
| | VERSION | feature bits | | ||
| +------------+-------------------------------------+ | ||
| Valid version values are: | ||
| 0x2 - version 2 | ||
| Feature bits: | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 11 | mask 0x0800 | Test id present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 10 | mask 0x0400 | Routing code present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 9 | mask 0x0200 | Timestamp present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 8 | mask 0x0100 | Test is 'runnable'. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 7 | mask 0x0080 | Tags are present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 6 | mask 0x0040 | File content is present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 5 | mask 0x0020 | File MIME type is present.| | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 4 | mask 0x0010 | EOF marker. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 3 | mask 0x0008 | Must be zero in version 2.| | ||
| +---------+-------------+---------------------------+ | ||
| Test status gets three bits: | ||
| Bit 2 | Bit 1 | Bit 0 - mask 0x0007 - A test status enum lookup: | ||
| * 000 - undefined / no test | ||
| * 001 - Enumeration / existence | ||
| * 002 - In progress | ||
| * 003 - Success | ||
| * 004 - Unexpected Success | ||
| * 005 - Skipped | ||
| * 006 - Failed | ||
| * 007 - Expected failure | ||
| After the flags field is a number field giving the length in bytes for the | ||
| entire packet including the signature and the checksum. This length must | ||
| be less than 4MiB - 4194303 bytes. The encoding can obviously record a larger | ||
| number but one of the goals is to avoid requiring large buffers, or causing | ||
| large latency in the packet forward/processing pipeline. Larger file | ||
| attachments can be communicated in multiple packets, and the overhead in such a | ||
| 4MiB packet is approximately 0.2%. | ||
| The rest of the packet is a series of optional features as specified by the set | ||
| feature bits in the flags field. When absent they are entirely absent. | ||
| Forwarding and multiplexing of packets can be done without interpreting the | ||
| remainder of the packet until the routing code and checksum (which are both at | ||
| the end of the packet). Additionally, routers can often avoid copying or moving | ||
| the bulk of the packet, as long as the routing code size increase doesn't force | ||
| the length encoding to take up a new byte (which will only happen to packets | ||
| less than or equal to 16KiB in length) - large packets are very efficient to | ||
| route. | ||
| Timestamp when present is a 32 bit unsigned integer for seconds, and a variable | ||
| length number for nanoseconds, representing UTC time since Unix Epoch in | ||
| seconds and nanoseconds. | ||
| Test id when present is a UTF-8 string. The test id should uniquely identify | ||
| runnable tests such that they can be selected individually. For tests and other | ||
| actions which cannot be individually run (such as test | ||
| fixtures/layers/subtests) uniqueness is not required (though being human | ||
| meaningful is highly recommended). | ||
| Tags when present is a length prefixed vector of UTF-8 strings, one per tag. | ||
| There are no restrictions on tag content (other than the restrictions on UTF-8 | ||
| strings in subunit in general). Tags have no ordering. | ||
| When a MIME type is present, it defines the MIME type for the file across all | ||
| packets same file (routing code + testid + name uniquely identifies a file, | ||
| reset when EOF is flagged). If a file never has a MIME type set, it should be | ||
| treated as application/octet-stream. | ||
| File content when present is a UTF-8 string for the name followed by the length | ||
| in bytes of the content, and then the content octets. | ||
| If present routing code is a UTF-8 string. The routing code is used to | ||
| determine which test backend a test was running on when doing data analysis, | ||
| and to route stdin to the test process if interaction is required. | ||
| Multiplexers SHOULD add a routing code if none is present, and prefix any | ||
| existing routing code with a routing code ('/' separated) if one is already | ||
| present. For example, a multiplexer might label each stream it is multiplexing | ||
| with a simple ordinal ('0', '1' etc), and given an incoming packet with route | ||
| code '3' from stream '0' would adjust the route code when forwarding the packet | ||
| to be '0/3'. | ||
| Following the end of the packet is a CRC-32 checksum of the contents of the | ||
| packet including the signature. | ||
| Example packets | ||
| ~~~~~~~~~~~~~~~ | ||
| Trivial test "foo" enumeration packet, with test id, runnable set, | ||
| status=enumeration. Spaces below are to visually break up signature / flags / | ||
| length / testid / crc32 | ||
| b3 2901 0c 03666f6f 08555f1b | ||
| Version 1 (and 1.1) | ||
| =================== | ||
| Version 1 (and 1.1) are mostly human readable protocols. | ||
| Sample subunit wire contents | ||
| ---------------------------- | ||
| The following:: | ||
| test: test foo works | ||
| success: test foo works | ||
| test: tar a file. | ||
| failure: tar a file. [ | ||
| .. | ||
| ].. space is eaten. | ||
| foo.c:34 WARNING foo is not defined. | ||
| ] | ||
| a writeln to stdout | ||
| When run through subunit2pyunit:: | ||
| .F | ||
| a writeln to stdout | ||
| ======================== | ||
| FAILURE: tar a file. | ||
| ------------------- | ||
| .. | ||
| ].. space is eaten. | ||
| foo.c:34 WARNING foo is not defined. | ||
| Subunit v1 protocol description | ||
| =============================== | ||
| This description is being ported to an EBNF style. Currently its only partly in | ||
| that style, but should be fairly clear all the same. When in doubt, refer the | ||
| source (and ideally help fix up the description!). Generally the protocol is | ||
| line orientated and consists of either directives and their parameters, or | ||
| when outside a DETAILS region unexpected lines which are not interpreted by | ||
| the parser - they should be forwarded unaltered:: | ||
| test|testing|test:|testing: test LABEL | ||
| success|success:|successful|successful: test LABEL | ||
| success|success:|successful|successful: test LABEL DETAILS | ||
| failure: test LABEL | ||
| failure: test LABEL DETAILS | ||
| error: test LABEL | ||
| error: test LABEL DETAILS | ||
| skip[:] test LABEL | ||
| skip[:] test LABEL DETAILS | ||
| xfail[:] test LABEL | ||
| xfail[:] test LABEL DETAILS | ||
| uxsuccess[:] test LABEL | ||
| uxsuccess[:] test LABEL DETAILS | ||
| progress: [+|-]X | ||
| progress: push | ||
| progress: pop | ||
| tags: [-]TAG ... | ||
| time: YYYY-MM-DD HH:MM:SSZ | ||
| LABEL: UTF8* | ||
| NAME: UTF8* | ||
| DETAILS ::= BRACKETED | MULTIPART | ||
| BRACKETED ::= '[' CR UTF8-lines ']' CR | ||
| MULTIPART ::= '[ multipart' CR PART* ']' CR | ||
| PART ::= PART_TYPE CR NAME CR PART_BYTES CR | ||
| PART_TYPE ::= Content-Type: type/sub-type(;parameter=value,parameter=value) | ||
| PART_BYTES ::= (DIGITS CR LF BYTE{DIGITS})* '0' CR LF | ||
| unexpected output on stdout -> stdout. | ||
| exit w/0 or last test completing -> error | ||
| Tags given outside a test are applied to all following tests | ||
| Tags given after a test: line and before the result line for the same test | ||
| apply only to that test, and inherit the current global tags. | ||
| A '-' before a tag is used to remove tags - e.g. to prevent a global tag | ||
| applying to a single test, or to cancel a global tag. | ||
| The progress directive is used to provide progress information about a stream | ||
| so that stream consumer can provide completion estimates, progress bars and so | ||
| on. Stream generators that know how many tests will be present in the stream | ||
| should output "progress: COUNT". Stream filters that add tests should output | ||
| "progress: +COUNT", and those that remove tests should output | ||
| "progress: -COUNT". An absolute count should reset the progress indicators in | ||
| use - it indicates that two separate streams from different generators have | ||
| been trivially concatenated together, and there is no knowledge of how many | ||
| more complete streams are incoming. Smart concatenation could scan each stream | ||
| for their count and sum them, or alternatively translate absolute counts into | ||
| relative counts inline. It is recommended that outputters avoid absolute counts | ||
| unless necessary. The push and pop directives are used to provide local regions | ||
| for progress reporting. This fits with hierarchically operating test | ||
| environments - such as those that organise tests into suites - the top-most | ||
| runner can report on the number of suites, and each suite surround its output | ||
| with a (push, pop) pair. Interpreters should interpret a pop as also advancing | ||
| the progress of the restored level by one step. Encountering progress | ||
| directives between the start and end of a test pair indicates that a previous | ||
| test was interrupted and did not cleanly terminate: it should be implicitly | ||
| closed with an error (the same as when a stream ends with no closing test | ||
| directive for the most recently started test). | ||
| The time directive acts as a clock event - it sets the time for all future | ||
| events. The value should be a valid ISO8601 time. | ||
| The skip, xfail and uxsuccess outcomes are not supported by all testing | ||
| environments. In Python the testttools (https://launchpad.net/testtools) | ||
| library is used to translate these automatically if an older Python version | ||
| that does not support them is in use. See the testtools documentation for the | ||
| translation policy. | ||
| skip is used to indicate a test was discovered but not executed. xfail is used | ||
| to indicate a test that errored in some expected fashion (also know as "TODO" | ||
| tests in some frameworks). uxsuccess is used to indicate and unexpected success | ||
| where a test though to be failing actually passes. It is complementary to | ||
| xfail. | ||
| Hacking on subunit | ||
| ------------------ | ||
| Releases | ||
| ======== | ||
| * Update versions in configure.ac and python/subunit/__init__.py. | ||
| * Update NEWS. | ||
| * Do a make distcheck, which will update Makefile etc. | ||
| * Do a PyPI release: PYTHONPATH=../../python python ../../setup.py sdist bdist_wheel upload -s | ||
| * Upload the regular one to LP. | ||
| * Push a tagged commit. | ||
| git push -t origin master:master | ||
| Keywords: python test streaming | ||
| Platform: UNKNOWN | ||
| Classifier: Intended Audience :: Developers | ||
| Classifier: Operating System :: OS Independent | ||
| Classifier: Programming Language :: Python | ||
| Classifier: Programming Language :: Python :: 2.7 | ||
| Classifier: Programming Language :: Python :: 3 | ||
| Classifier: Programming Language :: Python :: 3.5 | ||
| Classifier: Programming Language :: Python :: 3.6 | ||
| Classifier: Programming Language :: Python :: 3.7 | ||
| Classifier: Programming Language :: Python :: 3.8 | ||
| Classifier: Programming Language :: Python :: 3.9 | ||
| Classifier: Programming Language :: Python :: 3.10 | ||
| Classifier: Topic :: Software Development :: Testing | ||
| Requires-Python: >=3.6 | ||
| Provides-Extra: docs | ||
| Provides-Extra: test | ||
| License-File: COPYING | ||
| subunit: A streaming protocol for test results | ||
| Copyright (C) 2005-2013 Robert Collins <robertc@robertcollins.net> | ||
| Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| license at the users choice. A copy of both licenses are available in the | ||
| project source as Apache-2.0 and BSD. You may not use this file except in | ||
| compliance with one of these two licences. | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| license you chose for the specific language governing permissions and | ||
| limitations under that license. | ||
| See the COPYING file for full details on the licensing of Subunit. | ||
| subunit reuses iso8601 by Michael Twomey, distributed under an MIT style | ||
| licence - see python/iso8601/LICENSE for details. | ||
| Subunit | ||
| ------- | ||
| Subunit is a streaming protocol for test results. | ||
| There are two major revisions of the protocol. Version 1 was trivially human | ||
| readable but had significant defects as far as highly parallel testing was | ||
| concerned - it had no room for doing discovery and execution in parallel, | ||
| required substantial buffering when multiplexing and was fragile - a corrupt | ||
| byte could cause an entire stream to be misparsed. Version 1.1 added | ||
| encapsulation of binary streams which mitigated some of the issues but the | ||
| core remained. | ||
| Version 2 shares many of the good characteristics of Version 1 - it can be | ||
| embedded into a regular text stream (e.g. from a build system) and it still | ||
| models xUnit style test execution. It also fixes many of the issues with | ||
| Version 1 - Version 2 can be multiplexed without excessive buffering (in | ||
| time or space), it has a well defined recovery mechanism for dealing with | ||
| corrupted streams (e.g. where two processes write to the same stream | ||
| concurrently, or where the stream generator suffers a bug). | ||
| More details on both protocol version s can be found in the 'Protocol' section | ||
| of this document. | ||
| Subunit comes with command line filters to process a subunit stream and | ||
| language bindings for python, C, C++ and shell. Bindings are easy to write | ||
| for other languages. | ||
| A number of useful things can be done easily with subunit: | ||
| * Test aggregation: Tests run separately can be combined and then | ||
| reported/displayed together. For instance, tests from different languages | ||
| can be shown as a seamless whole, and tests running on multiple machines | ||
| can be aggregated into a single stream through a multiplexer. | ||
| * Test archiving: A test run may be recorded and replayed later. | ||
| * Test isolation: Tests that may crash or otherwise interact badly with each | ||
| other can be run seperately and then aggregated, rather than interfering | ||
| with each other or requiring an adhoc test->runner reporting protocol. | ||
| * Grid testing: subunit can act as the necessary serialisation and | ||
| deserialiation to get test runs on distributed machines to be reported in | ||
| real time. | ||
| Subunit supplies the following filters: | ||
| * tap2subunit - convert perl's TestAnythingProtocol to subunit. | ||
| * subunit2csv - convert a subunit stream to csv. | ||
| * subunit2disk - export a subunit stream to files on disk. | ||
| * subunit2pyunit - convert a subunit stream to pyunit test results. | ||
| * subunit2gtk - show a subunit stream in GTK. | ||
| * subunit2junitxml - convert a subunit stream to JUnit's XML format. | ||
| * subunit-diff - compare two subunit streams. | ||
| * subunit-filter - filter out tests from a subunit stream. | ||
| * subunit-ls - list info about tests present in a subunit stream. | ||
| * subunit-stats - generate a summary of a subunit stream. | ||
| * subunit-tags - add or remove tags from a stream. | ||
| Integration with other tools | ||
| ---------------------------- | ||
| Subunit's language bindings act as integration with various test runners like | ||
| 'check', 'cppunit', Python's 'unittest'. Beyond that a small amount of glue | ||
| (typically a few lines) will allow Subunit to be used in more sophisticated | ||
| ways. | ||
| Python | ||
| ====== | ||
| Subunit has excellent Python support: most of the filters and tools are written | ||
| in python and there are facilities for using Subunit to increase test isolation | ||
| seamlessly within a test suite. | ||
| The most common way is to run an existing python test suite and have it output | ||
| subunit via the ``subunit.run`` module:: | ||
| $ python -m subunit.run mypackage.tests.test_suite | ||
| For more information on the Python support Subunit offers , please see | ||
| ``pydoc subunit``, or the source in ``python/subunit/`` | ||
| C | ||
| = | ||
| Subunit has C bindings to emit the protocol. The 'check' C unit testing project | ||
| has included subunit support in their project for some years now. See | ||
| 'c/README' for more details. | ||
| C++ | ||
| === | ||
| The C library is includable and usable directly from C++. A TestListener for | ||
| CPPUnit is included in the Subunit distribution. See 'c++/README' for details. | ||
| shell | ||
| ===== | ||
| There are two sets of shell tools. There are filters, which accept a subunit | ||
| stream on stdin and output processed data (or a transformed stream) on stdout. | ||
| Then there are unittest facilities similar to those for C : shell bindings | ||
| consisting of simple functions to output protocol elements, and a patch for | ||
| adding subunit output to the 'ShUnit' shell test runner. See 'shell/README' for | ||
| details. | ||
| Filter recipes | ||
| -------------- | ||
| To ignore some failing tests whose root cause is already known:: | ||
| subunit-filter --without 'AttributeError.*flavor' | ||
| The xUnit test model | ||
| -------------------- | ||
| Subunit implements a slightly modified xUnit test model. The stock standard | ||
| model is that there are tests, which have an id(), can be run, and when run | ||
| start, emit an outcome (like success or failure) and then finish. | ||
| Subunit extends this with the idea of test enumeration (find out about tests | ||
| a runner has without running them), tags (allow users to describe tests in | ||
| ways the test framework doesn't apply any semantic value to), file attachments | ||
| (allow arbitrary data to make analysing a failure easy) and timestamps. | ||
| The protocol | ||
| ------------ | ||
| Version 2, or v2 is new and still under development, but is intended to | ||
| supercede version 1 in the very near future. Subunit's bundled tools accept | ||
| only version 2 and only emit version 2, but the new filters subunit-1to2 and | ||
| subunit-2to1 can be used to interoperate with older third party libraries. | ||
| Version 2 | ||
| ========= | ||
| Version 2 is a binary protocol consisting of independent packets that can be | ||
| embedded in the output from tools like make - as long as each packet has no | ||
| other bytes mixed in with it (which 'make -j N>1' has a tendency of doing). | ||
| Version 2 is currently in draft form, and early adopters should be willing | ||
| to either discard stored results (if protocol changes are made), or bulk | ||
| convert them back to v1 and then to a newer edition of v2. | ||
| The protocol synchronises at the start of the stream, after a packet, or | ||
| after any 0x0A byte. That is, a subunit v2 packet starts after a newline or | ||
| directly after the end of the prior packet. | ||
| Subunit is intended to be transported over a reliable streaming protocol such | ||
| as TCP. As such it does not concern itself with out of order delivery of | ||
| packets. However, because of the possibility of corruption due to either | ||
| bugs in the sender, or due to mixed up data from concurrent writes to the same | ||
| fd when being embedded, subunit strives to recover reasonably gracefully from | ||
| damaged data. | ||
| A key design goal for Subunit version 2 is to allow processing and multiplexing | ||
| without forcing buffering for semantic correctness, as buffering tends to hide | ||
| hung or otherwise misbehaving tests. That said, limited time based buffering | ||
| for network efficiency is a good idea - this is ultimately implementator | ||
| choice. Line buffering is also discouraged for subunit streams, as dropping | ||
| into a debugger or other tool may require interactive traffic even if line | ||
| buffering would not otherwise be a problem. | ||
| In version two there are two conceptual events - a test status event and a file | ||
| attachment event. Events may have timestamps, and the path of multiplexers that | ||
| an event is routed through is recorded to permit sending actions back to the | ||
| source (such as new tests to run or stdin for driving debuggers and other | ||
| interactive input). Test status events are used to enumerate tests, to report | ||
| tests and test helpers as they run. Tests may have tags, used to allow | ||
| tunnelling extra meanings through subunit without requiring parsing of | ||
| arbitrary file attachments. Things that are not standalone tests get marked | ||
| as such by setting the 'Runnable' flag to false. (For instance, individual | ||
| assertions in TAP are not runnable tests, only the top level TAP test script | ||
| is runnable). | ||
| File attachments are used to provide rich detail about the nature of a failure. | ||
| File attachments can also be used to encapsulate stdout and stderr both during | ||
| and outside tests. | ||
| Most numbers are stored in network byte order - Most Significant Byte first | ||
| encoded using a variation of http://www.dlugosz.com/ZIP2/VLI.html. The first | ||
| byte's top 2 high order bits encode the total number of octets in the number. | ||
| This encoding can encode values from 0 to 2**30-1, enough to encode a | ||
| nanosecond. Numbers that are not variable length encoded are still stored in | ||
| MSB order. | ||
| +--------+--------+---------+------------+ | ||
| | prefix | octets | max | max | | ||
| +========+========+=========+============+ | ||
| | 00 | 1 | 2**6-1 | 63 | | ||
| +--------+--------+---------+------------+ | ||
| | 01 | 2 | 2**14-1 | 16383 | | ||
| +--------+--------+---------+------------+ | ||
| | 10 | 3 | 2**22-1 | 4194303 | | ||
| +--------+--------+---------+------------+ | ||
| | 11 | 4 | 2**30-1 | 1073741823 | | ||
| +--------+--------+---------+------------+ | ||
| All variable length elements of the packet are stored with a length prefix | ||
| number allowing them to be skipped over for consumers that don't need to | ||
| interpret them. | ||
| UTF-8 strings are with no terminating NUL and should not have any embedded NULs | ||
| (implementations SHOULD validate any such strings that they process and take | ||
| some remedial action (such as discarding the packet as corrupt). | ||
| In short the structure of a packet is: | ||
| PACKET := SIGNATURE FLAGS PACKET_LENGTH TIMESTAMP? TESTID? TAGS? MIME? | ||
| FILECONTENT? ROUTING_CODE? CRC32 | ||
| In more detail... | ||
| Packets are identified by a single byte signature - 0xB3, which is never legal | ||
| in a UTF-8 stream as the first byte of a character. 0xB3 starts with the first | ||
| bit set and the second not, which is the UTF-8 signature for a continuation | ||
| byte. 0xB3 was chosen as 0x73 ('s' in ASCII') with the top two bits replaced by | ||
| the 1 and 0 for a continuation byte. | ||
| If subunit packets are being embedded in a non-UTF-8 text stream, where 0x73 is | ||
| a legal character, consider either recoding the text to UTF-8, or using | ||
| subunit's 'file' packets to embed the text stream in subunit, rather than the | ||
| other way around. | ||
| Following the signature byte comes a 16-bit flags field, which includes a | ||
| 4-bit version field - if the version is not 0x2 then the packet cannot be | ||
| read. It is recommended to signal an error at this point (e.g. by emitting | ||
| a synthetic error packet and returning to the top level loop to look for | ||
| new packets, or exiting with an error). If recovery is desired, treat the | ||
| packet signature as an opaque byte and scan for a new synchronisation point. | ||
| NB: Subunit V1 and V2 packets may legitimately included 0xB3 internally, | ||
| as they are an 8-bit safe container format, so recovery from this situation | ||
| may involve an arbitrary number of false positives until an actual packet | ||
| is encountered : and even then it may still be false, failing after passing | ||
| the version check due to coincidence. | ||
| Flags are stored in network byte order too. | ||
| +------------+------------+------------------------+ | ||
| | High byte | Low byte | | ||
| +------------+------------+------------------------+ | ||
| | 15 14 13 12 11 10 9 8 | 7 6 5 4 3 2 1 0 | | ||
| +------------+------------+------------------------+ | ||
| | VERSION | feature bits | | ||
| +------------+-------------------------------------+ | ||
| Valid version values are: | ||
| 0x2 - version 2 | ||
| Feature bits: | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 11 | mask 0x0800 | Test id present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 10 | mask 0x0400 | Routing code present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 9 | mask 0x0200 | Timestamp present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 8 | mask 0x0100 | Test is 'runnable'. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 7 | mask 0x0080 | Tags are present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 6 | mask 0x0040 | File content is present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 5 | mask 0x0020 | File MIME type is present.| | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 4 | mask 0x0010 | EOF marker. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 3 | mask 0x0008 | Must be zero in version 2.| | ||
| +---------+-------------+---------------------------+ | ||
| Test status gets three bits: | ||
| Bit 2 | Bit 1 | Bit 0 - mask 0x0007 - A test status enum lookup: | ||
| * 000 - undefined / no test | ||
| * 001 - Enumeration / existence | ||
| * 002 - In progress | ||
| * 003 - Success | ||
| * 004 - Unexpected Success | ||
| * 005 - Skipped | ||
| * 006 - Failed | ||
| * 007 - Expected failure | ||
| After the flags field is a number field giving the length in bytes for the | ||
| entire packet including the signature and the checksum. This length must | ||
| be less than 4MiB - 4194303 bytes. The encoding can obviously record a larger | ||
| number but one of the goals is to avoid requiring large buffers, or causing | ||
| large latency in the packet forward/processing pipeline. Larger file | ||
| attachments can be communicated in multiple packets, and the overhead in such a | ||
| 4MiB packet is approximately 0.2%. | ||
| The rest of the packet is a series of optional features as specified by the set | ||
| feature bits in the flags field. When absent they are entirely absent. | ||
| Forwarding and multiplexing of packets can be done without interpreting the | ||
| remainder of the packet until the routing code and checksum (which are both at | ||
| the end of the packet). Additionally, routers can often avoid copying or moving | ||
| the bulk of the packet, as long as the routing code size increase doesn't force | ||
| the length encoding to take up a new byte (which will only happen to packets | ||
| less than or equal to 16KiB in length) - large packets are very efficient to | ||
| route. | ||
| Timestamp when present is a 32 bit unsigned integer for seconds, and a variable | ||
| length number for nanoseconds, representing UTC time since Unix Epoch in | ||
| seconds and nanoseconds. | ||
| Test id when present is a UTF-8 string. The test id should uniquely identify | ||
| runnable tests such that they can be selected individually. For tests and other | ||
| actions which cannot be individually run (such as test | ||
| fixtures/layers/subtests) uniqueness is not required (though being human | ||
| meaningful is highly recommended). | ||
| Tags when present is a length prefixed vector of UTF-8 strings, one per tag. | ||
| There are no restrictions on tag content (other than the restrictions on UTF-8 | ||
| strings in subunit in general). Tags have no ordering. | ||
| When a MIME type is present, it defines the MIME type for the file across all | ||
| packets same file (routing code + testid + name uniquely identifies a file, | ||
| reset when EOF is flagged). If a file never has a MIME type set, it should be | ||
| treated as application/octet-stream. | ||
| File content when present is a UTF-8 string for the name followed by the length | ||
| in bytes of the content, and then the content octets. | ||
| If present routing code is a UTF-8 string. The routing code is used to | ||
| determine which test backend a test was running on when doing data analysis, | ||
| and to route stdin to the test process if interaction is required. | ||
| Multiplexers SHOULD add a routing code if none is present, and prefix any | ||
| existing routing code with a routing code ('/' separated) if one is already | ||
| present. For example, a multiplexer might label each stream it is multiplexing | ||
| with a simple ordinal ('0', '1' etc), and given an incoming packet with route | ||
| code '3' from stream '0' would adjust the route code when forwarding the packet | ||
| to be '0/3'. | ||
| Following the end of the packet is a CRC-32 checksum of the contents of the | ||
| packet including the signature. | ||
| Example packets | ||
| ~~~~~~~~~~~~~~~ | ||
| Trivial test "foo" enumeration packet, with test id, runnable set, | ||
| status=enumeration. Spaces below are to visually break up signature / flags / | ||
| length / testid / crc32 | ||
| b3 2901 0c 03666f6f 08555f1b | ||
| Version 1 (and 1.1) | ||
| =================== | ||
| Version 1 (and 1.1) are mostly human readable protocols. | ||
| Sample subunit wire contents | ||
| ---------------------------- | ||
| The following:: | ||
| test: test foo works | ||
| success: test foo works | ||
| test: tar a file. | ||
| failure: tar a file. [ | ||
| .. | ||
| ].. space is eaten. | ||
| foo.c:34 WARNING foo is not defined. | ||
| ] | ||
| a writeln to stdout | ||
| When run through subunit2pyunit:: | ||
| .F | ||
| a writeln to stdout | ||
| ======================== | ||
| FAILURE: tar a file. | ||
| ------------------- | ||
| .. | ||
| ].. space is eaten. | ||
| foo.c:34 WARNING foo is not defined. | ||
| Subunit v1 protocol description | ||
| =============================== | ||
| This description is being ported to an EBNF style. Currently its only partly in | ||
| that style, but should be fairly clear all the same. When in doubt, refer the | ||
| source (and ideally help fix up the description!). Generally the protocol is | ||
| line orientated and consists of either directives and their parameters, or | ||
| when outside a DETAILS region unexpected lines which are not interpreted by | ||
| the parser - they should be forwarded unaltered:: | ||
| test|testing|test:|testing: test LABEL | ||
| success|success:|successful|successful: test LABEL | ||
| success|success:|successful|successful: test LABEL DETAILS | ||
| failure: test LABEL | ||
| failure: test LABEL DETAILS | ||
| error: test LABEL | ||
| error: test LABEL DETAILS | ||
| skip[:] test LABEL | ||
| skip[:] test LABEL DETAILS | ||
| xfail[:] test LABEL | ||
| xfail[:] test LABEL DETAILS | ||
| uxsuccess[:] test LABEL | ||
| uxsuccess[:] test LABEL DETAILS | ||
| progress: [+|-]X | ||
| progress: push | ||
| progress: pop | ||
| tags: [-]TAG ... | ||
| time: YYYY-MM-DD HH:MM:SSZ | ||
| LABEL: UTF8* | ||
| NAME: UTF8* | ||
| DETAILS ::= BRACKETED | MULTIPART | ||
| BRACKETED ::= '[' CR UTF8-lines ']' CR | ||
| MULTIPART ::= '[ multipart' CR PART* ']' CR | ||
| PART ::= PART_TYPE CR NAME CR PART_BYTES CR | ||
| PART_TYPE ::= Content-Type: type/sub-type(;parameter=value,parameter=value) | ||
| PART_BYTES ::= (DIGITS CR LF BYTE{DIGITS})* '0' CR LF | ||
| unexpected output on stdout -> stdout. | ||
| exit w/0 or last test completing -> error | ||
| Tags given outside a test are applied to all following tests | ||
| Tags given after a test: line and before the result line for the same test | ||
| apply only to that test, and inherit the current global tags. | ||
| A '-' before a tag is used to remove tags - e.g. to prevent a global tag | ||
| applying to a single test, or to cancel a global tag. | ||
| The progress directive is used to provide progress information about a stream | ||
| so that stream consumer can provide completion estimates, progress bars and so | ||
| on. Stream generators that know how many tests will be present in the stream | ||
| should output "progress: COUNT". Stream filters that add tests should output | ||
| "progress: +COUNT", and those that remove tests should output | ||
| "progress: -COUNT". An absolute count should reset the progress indicators in | ||
| use - it indicates that two separate streams from different generators have | ||
| been trivially concatenated together, and there is no knowledge of how many | ||
| more complete streams are incoming. Smart concatenation could scan each stream | ||
| for their count and sum them, or alternatively translate absolute counts into | ||
| relative counts inline. It is recommended that outputters avoid absolute counts | ||
| unless necessary. The push and pop directives are used to provide local regions | ||
| for progress reporting. This fits with hierarchically operating test | ||
| environments - such as those that organise tests into suites - the top-most | ||
| runner can report on the number of suites, and each suite surround its output | ||
| with a (push, pop) pair. Interpreters should interpret a pop as also advancing | ||
| the progress of the restored level by one step. Encountering progress | ||
| directives between the start and end of a test pair indicates that a previous | ||
| test was interrupted and did not cleanly terminate: it should be implicitly | ||
| closed with an error (the same as when a stream ends with no closing test | ||
| directive for the most recently started test). | ||
| The time directive acts as a clock event - it sets the time for all future | ||
| events. The value should be a valid ISO8601 time. | ||
| The skip, xfail and uxsuccess outcomes are not supported by all testing | ||
| environments. In Python the testttools (https://launchpad.net/testtools) | ||
| library is used to translate these automatically if an older Python version | ||
| that does not support them is in use. See the testtools documentation for the | ||
| translation policy. | ||
| skip is used to indicate a test was discovered but not executed. xfail is used | ||
| to indicate a test that errored in some expected fashion (also know as "TODO" | ||
| tests in some frameworks). uxsuccess is used to indicate and unexpected success | ||
| where a test though to be failing actually passes. It is complementary to | ||
| xfail. | ||
| Hacking on subunit | ||
| ------------------ | ||
| Releases | ||
| ======== | ||
| * Update versions in configure.ac and python/subunit/__init__.py. | ||
| * Update NEWS. | ||
| * Do a make distcheck, which will update Makefile etc. | ||
| * Do a PyPI release: PYTHONPATH=../../python python ../../setup.py sdist bdist_wheel upload -s | ||
| * Upload the regular one to LP. | ||
| * Push a tagged commit. | ||
| git push -t origin master:master |
+499
-498
| Metadata-Version: 2.1 | ||
| Name: python-subunit | ||
| Version: 1.4.0 | ||
| Version: 1.4.1 | ||
| Summary: Python implementation of subunit test streaming protocol | ||
@@ -11,509 +11,510 @@ Home-page: http://launchpad.net/subunit | ||
| Project-URL: Source Code, https://github.com/testing-cabal/subunit/ | ||
| Description: | ||
| subunit: A streaming protocol for test results | ||
| Copyright (C) 2005-2013 Robert Collins <robertc@robertcollins.net> | ||
| Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| license at the users choice. A copy of both licenses are available in the | ||
| project source as Apache-2.0 and BSD. You may not use this file except in | ||
| compliance with one of these two licences. | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| license you chose for the specific language governing permissions and | ||
| limitations under that license. | ||
| See the COPYING file for full details on the licensing of Subunit. | ||
| subunit reuses iso8601 by Michael Twomey, distributed under an MIT style | ||
| licence - see python/iso8601/LICENSE for details. | ||
| Subunit | ||
| ------- | ||
| Subunit is a streaming protocol for test results. | ||
| There are two major revisions of the protocol. Version 1 was trivially human | ||
| readable but had significant defects as far as highly parallel testing was | ||
| concerned - it had no room for doing discovery and execution in parallel, | ||
| required substantial buffering when multiplexing and was fragile - a corrupt | ||
| byte could cause an entire stream to be misparsed. Version 1.1 added | ||
| encapsulation of binary streams which mitigated some of the issues but the | ||
| core remained. | ||
| Version 2 shares many of the good characteristics of Version 1 - it can be | ||
| embedded into a regular text stream (e.g. from a build system) and it still | ||
| models xUnit style test execution. It also fixes many of the issues with | ||
| Version 1 - Version 2 can be multiplexed without excessive buffering (in | ||
| time or space), it has a well defined recovery mechanism for dealing with | ||
| corrupted streams (e.g. where two processes write to the same stream | ||
| concurrently, or where the stream generator suffers a bug). | ||
| More details on both protocol version s can be found in the 'Protocol' section | ||
| of this document. | ||
| Subunit comes with command line filters to process a subunit stream and | ||
| language bindings for python, C, C++ and shell. Bindings are easy to write | ||
| for other languages. | ||
| A number of useful things can be done easily with subunit: | ||
| * Test aggregation: Tests run separately can be combined and then | ||
| reported/displayed together. For instance, tests from different languages | ||
| can be shown as a seamless whole, and tests running on multiple machines | ||
| can be aggregated into a single stream through a multiplexer. | ||
| * Test archiving: A test run may be recorded and replayed later. | ||
| * Test isolation: Tests that may crash or otherwise interact badly with each | ||
| other can be run seperately and then aggregated, rather than interfering | ||
| with each other or requiring an adhoc test->runner reporting protocol. | ||
| * Grid testing: subunit can act as the necessary serialisation and | ||
| deserialiation to get test runs on distributed machines to be reported in | ||
| real time. | ||
| Subunit supplies the following filters: | ||
| * tap2subunit - convert perl's TestAnythingProtocol to subunit. | ||
| * subunit2csv - convert a subunit stream to csv. | ||
| * subunit2disk - export a subunit stream to files on disk. | ||
| * subunit2pyunit - convert a subunit stream to pyunit test results. | ||
| * subunit2gtk - show a subunit stream in GTK. | ||
| * subunit2junitxml - convert a subunit stream to JUnit's XML format. | ||
| * subunit-diff - compare two subunit streams. | ||
| * subunit-filter - filter out tests from a subunit stream. | ||
| * subunit-ls - list info about tests present in a subunit stream. | ||
| * subunit-stats - generate a summary of a subunit stream. | ||
| * subunit-tags - add or remove tags from a stream. | ||
| Integration with other tools | ||
| ---------------------------- | ||
| Subunit's language bindings act as integration with various test runners like | ||
| 'check', 'cppunit', Python's 'unittest'. Beyond that a small amount of glue | ||
| (typically a few lines) will allow Subunit to be used in more sophisticated | ||
| ways. | ||
| Python | ||
| ====== | ||
| Subunit has excellent Python support: most of the filters and tools are written | ||
| in python and there are facilities for using Subunit to increase test isolation | ||
| seamlessly within a test suite. | ||
| The most common way is to run an existing python test suite and have it output | ||
| subunit via the ``subunit.run`` module:: | ||
| $ python -m subunit.run mypackage.tests.test_suite | ||
| For more information on the Python support Subunit offers , please see | ||
| ``pydoc subunit``, or the source in ``python/subunit/`` | ||
| C | ||
| = | ||
| Subunit has C bindings to emit the protocol. The 'check' C unit testing project | ||
| has included subunit support in their project for some years now. See | ||
| 'c/README' for more details. | ||
| C++ | ||
| === | ||
| The C library is includable and usable directly from C++. A TestListener for | ||
| CPPUnit is included in the Subunit distribution. See 'c++/README' for details. | ||
| shell | ||
| ===== | ||
| There are two sets of shell tools. There are filters, which accept a subunit | ||
| stream on stdin and output processed data (or a transformed stream) on stdout. | ||
| Then there are unittest facilities similar to those for C : shell bindings | ||
| consisting of simple functions to output protocol elements, and a patch for | ||
| adding subunit output to the 'ShUnit' shell test runner. See 'shell/README' for | ||
| details. | ||
| Filter recipes | ||
| -------------- | ||
| To ignore some failing tests whose root cause is already known:: | ||
| subunit-filter --without 'AttributeError.*flavor' | ||
| The xUnit test model | ||
| -------------------- | ||
| Subunit implements a slightly modified xUnit test model. The stock standard | ||
| model is that there are tests, which have an id(), can be run, and when run | ||
| start, emit an outcome (like success or failure) and then finish. | ||
| Subunit extends this with the idea of test enumeration (find out about tests | ||
| a runner has without running them), tags (allow users to describe tests in | ||
| ways the test framework doesn't apply any semantic value to), file attachments | ||
| (allow arbitrary data to make analysing a failure easy) and timestamps. | ||
| The protocol | ||
| ------------ | ||
| Version 2, or v2 is new and still under development, but is intended to | ||
| supercede version 1 in the very near future. Subunit's bundled tools accept | ||
| only version 2 and only emit version 2, but the new filters subunit-1to2 and | ||
| subunit-2to1 can be used to interoperate with older third party libraries. | ||
| Version 2 | ||
| ========= | ||
| Version 2 is a binary protocol consisting of independent packets that can be | ||
| embedded in the output from tools like make - as long as each packet has no | ||
| other bytes mixed in with it (which 'make -j N>1' has a tendency of doing). | ||
| Version 2 is currently in draft form, and early adopters should be willing | ||
| to either discard stored results (if protocol changes are made), or bulk | ||
| convert them back to v1 and then to a newer edition of v2. | ||
| The protocol synchronises at the start of the stream, after a packet, or | ||
| after any 0x0A byte. That is, a subunit v2 packet starts after a newline or | ||
| directly after the end of the prior packet. | ||
| Subunit is intended to be transported over a reliable streaming protocol such | ||
| as TCP. As such it does not concern itself with out of order delivery of | ||
| packets. However, because of the possibility of corruption due to either | ||
| bugs in the sender, or due to mixed up data from concurrent writes to the same | ||
| fd when being embedded, subunit strives to recover reasonably gracefully from | ||
| damaged data. | ||
| A key design goal for Subunit version 2 is to allow processing and multiplexing | ||
| without forcing buffering for semantic correctness, as buffering tends to hide | ||
| hung or otherwise misbehaving tests. That said, limited time based buffering | ||
| for network efficiency is a good idea - this is ultimately implementator | ||
| choice. Line buffering is also discouraged for subunit streams, as dropping | ||
| into a debugger or other tool may require interactive traffic even if line | ||
| buffering would not otherwise be a problem. | ||
| In version two there are two conceptual events - a test status event and a file | ||
| attachment event. Events may have timestamps, and the path of multiplexers that | ||
| an event is routed through is recorded to permit sending actions back to the | ||
| source (such as new tests to run or stdin for driving debuggers and other | ||
| interactive input). Test status events are used to enumerate tests, to report | ||
| tests and test helpers as they run. Tests may have tags, used to allow | ||
| tunnelling extra meanings through subunit without requiring parsing of | ||
| arbitrary file attachments. Things that are not standalone tests get marked | ||
| as such by setting the 'Runnable' flag to false. (For instance, individual | ||
| assertions in TAP are not runnable tests, only the top level TAP test script | ||
| is runnable). | ||
| File attachments are used to provide rich detail about the nature of a failure. | ||
| File attachments can also be used to encapsulate stdout and stderr both during | ||
| and outside tests. | ||
| Most numbers are stored in network byte order - Most Significant Byte first | ||
| encoded using a variation of http://www.dlugosz.com/ZIP2/VLI.html. The first | ||
| byte's top 2 high order bits encode the total number of octets in the number. | ||
| This encoding can encode values from 0 to 2**30-1, enough to encode a | ||
| nanosecond. Numbers that are not variable length encoded are still stored in | ||
| MSB order. | ||
| +--------+--------+---------+------------+ | ||
| | prefix | octets | max | max | | ||
| +========+========+=========+============+ | ||
| | 00 | 1 | 2**6-1 | 63 | | ||
| +--------+--------+---------+------------+ | ||
| | 01 | 2 | 2**14-1 | 16383 | | ||
| +--------+--------+---------+------------+ | ||
| | 10 | 3 | 2**22-1 | 4194303 | | ||
| +--------+--------+---------+------------+ | ||
| | 11 | 4 | 2**30-1 | 1073741823 | | ||
| +--------+--------+---------+------------+ | ||
| All variable length elements of the packet are stored with a length prefix | ||
| number allowing them to be skipped over for consumers that don't need to | ||
| interpret them. | ||
| UTF-8 strings are with no terminating NUL and should not have any embedded NULs | ||
| (implementations SHOULD validate any such strings that they process and take | ||
| some remedial action (such as discarding the packet as corrupt). | ||
| In short the structure of a packet is: | ||
| PACKET := SIGNATURE FLAGS PACKET_LENGTH TIMESTAMP? TESTID? TAGS? MIME? | ||
| FILECONTENT? ROUTING_CODE? CRC32 | ||
| In more detail... | ||
| Packets are identified by a single byte signature - 0xB3, which is never legal | ||
| in a UTF-8 stream as the first byte of a character. 0xB3 starts with the first | ||
| bit set and the second not, which is the UTF-8 signature for a continuation | ||
| byte. 0xB3 was chosen as 0x73 ('s' in ASCII') with the top two bits replaced by | ||
| the 1 and 0 for a continuation byte. | ||
| If subunit packets are being embedded in a non-UTF-8 text stream, where 0x73 is | ||
| a legal character, consider either recoding the text to UTF-8, or using | ||
| subunit's 'file' packets to embed the text stream in subunit, rather than the | ||
| other way around. | ||
| Following the signature byte comes a 16-bit flags field, which includes a | ||
| 4-bit version field - if the version is not 0x2 then the packet cannot be | ||
| read. It is recommended to signal an error at this point (e.g. by emitting | ||
| a synthetic error packet and returning to the top level loop to look for | ||
| new packets, or exiting with an error). If recovery is desired, treat the | ||
| packet signature as an opaque byte and scan for a new synchronisation point. | ||
| NB: Subunit V1 and V2 packets may legitimately included 0xB3 internally, | ||
| as they are an 8-bit safe container format, so recovery from this situation | ||
| may involve an arbitrary number of false positives until an actual packet | ||
| is encountered : and even then it may still be false, failing after passing | ||
| the version check due to coincidence. | ||
| Flags are stored in network byte order too. | ||
| +------------+------------+------------------------+ | ||
| | High byte | Low byte | | ||
| +------------+------------+------------------------+ | ||
| | 15 14 13 12 11 10 9 8 | 7 6 5 4 3 2 1 0 | | ||
| +------------+------------+------------------------+ | ||
| | VERSION | feature bits | | ||
| +------------+-------------------------------------+ | ||
| Valid version values are: | ||
| 0x2 - version 2 | ||
| Feature bits: | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 11 | mask 0x0800 | Test id present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 10 | mask 0x0400 | Routing code present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 9 | mask 0x0200 | Timestamp present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 8 | mask 0x0100 | Test is 'runnable'. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 7 | mask 0x0080 | Tags are present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 6 | mask 0x0040 | File content is present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 5 | mask 0x0020 | File MIME type is present.| | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 4 | mask 0x0010 | EOF marker. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 3 | mask 0x0008 | Must be zero in version 2.| | ||
| +---------+-------------+---------------------------+ | ||
| Test status gets three bits: | ||
| Bit 2 | Bit 1 | Bit 0 - mask 0x0007 - A test status enum lookup: | ||
| * 000 - undefined / no test | ||
| * 001 - Enumeration / existence | ||
| * 002 - In progress | ||
| * 003 - Success | ||
| * 004 - Unexpected Success | ||
| * 005 - Skipped | ||
| * 006 - Failed | ||
| * 007 - Expected failure | ||
| After the flags field is a number field giving the length in bytes for the | ||
| entire packet including the signature and the checksum. This length must | ||
| be less than 4MiB - 4194303 bytes. The encoding can obviously record a larger | ||
| number but one of the goals is to avoid requiring large buffers, or causing | ||
| large latency in the packet forward/processing pipeline. Larger file | ||
| attachments can be communicated in multiple packets, and the overhead in such a | ||
| 4MiB packet is approximately 0.2%. | ||
| The rest of the packet is a series of optional features as specified by the set | ||
| feature bits in the flags field. When absent they are entirely absent. | ||
| Forwarding and multiplexing of packets can be done without interpreting the | ||
| remainder of the packet until the routing code and checksum (which are both at | ||
| the end of the packet). Additionally, routers can often avoid copying or moving | ||
| the bulk of the packet, as long as the routing code size increase doesn't force | ||
| the length encoding to take up a new byte (which will only happen to packets | ||
| less than or equal to 16KiB in length) - large packets are very efficient to | ||
| route. | ||
| Timestamp when present is a 32 bit unsigned integer for seconds, and a variable | ||
| length number for nanoseconds, representing UTC time since Unix Epoch in | ||
| seconds and nanoseconds. | ||
| Test id when present is a UTF-8 string. The test id should uniquely identify | ||
| runnable tests such that they can be selected individually. For tests and other | ||
| actions which cannot be individually run (such as test | ||
| fixtures/layers/subtests) uniqueness is not required (though being human | ||
| meaningful is highly recommended). | ||
| Tags when present is a length prefixed vector of UTF-8 strings, one per tag. | ||
| There are no restrictions on tag content (other than the restrictions on UTF-8 | ||
| strings in subunit in general). Tags have no ordering. | ||
| When a MIME type is present, it defines the MIME type for the file across all | ||
| packets same file (routing code + testid + name uniquely identifies a file, | ||
| reset when EOF is flagged). If a file never has a MIME type set, it should be | ||
| treated as application/octet-stream. | ||
| File content when present is a UTF-8 string for the name followed by the length | ||
| in bytes of the content, and then the content octets. | ||
| If present routing code is a UTF-8 string. The routing code is used to | ||
| determine which test backend a test was running on when doing data analysis, | ||
| and to route stdin to the test process if interaction is required. | ||
| Multiplexers SHOULD add a routing code if none is present, and prefix any | ||
| existing routing code with a routing code ('/' separated) if one is already | ||
| present. For example, a multiplexer might label each stream it is multiplexing | ||
| with a simple ordinal ('0', '1' etc), and given an incoming packet with route | ||
| code '3' from stream '0' would adjust the route code when forwarding the packet | ||
| to be '0/3'. | ||
| Following the end of the packet is a CRC-32 checksum of the contents of the | ||
| packet including the signature. | ||
| Example packets | ||
| ~~~~~~~~~~~~~~~ | ||
| Trivial test "foo" enumeration packet, with test id, runnable set, | ||
| status=enumeration. Spaces below are to visually break up signature / flags / | ||
| length / testid / crc32 | ||
| b3 2901 0c 03666f6f 08555f1b | ||
| Version 1 (and 1.1) | ||
| =================== | ||
| Version 1 (and 1.1) are mostly human readable protocols. | ||
| Sample subunit wire contents | ||
| ---------------------------- | ||
| The following:: | ||
| test: test foo works | ||
| success: test foo works | ||
| test: tar a file. | ||
| failure: tar a file. [ | ||
| .. | ||
| ].. space is eaten. | ||
| foo.c:34 WARNING foo is not defined. | ||
| ] | ||
| a writeln to stdout | ||
| When run through subunit2pyunit:: | ||
| .F | ||
| a writeln to stdout | ||
| ======================== | ||
| FAILURE: tar a file. | ||
| ------------------- | ||
| .. | ||
| ].. space is eaten. | ||
| foo.c:34 WARNING foo is not defined. | ||
| Subunit v1 protocol description | ||
| =============================== | ||
| This description is being ported to an EBNF style. Currently its only partly in | ||
| that style, but should be fairly clear all the same. When in doubt, refer the | ||
| source (and ideally help fix up the description!). Generally the protocol is | ||
| line orientated and consists of either directives and their parameters, or | ||
| when outside a DETAILS region unexpected lines which are not interpreted by | ||
| the parser - they should be forwarded unaltered:: | ||
| test|testing|test:|testing: test LABEL | ||
| success|success:|successful|successful: test LABEL | ||
| success|success:|successful|successful: test LABEL DETAILS | ||
| failure: test LABEL | ||
| failure: test LABEL DETAILS | ||
| error: test LABEL | ||
| error: test LABEL DETAILS | ||
| skip[:] test LABEL | ||
| skip[:] test LABEL DETAILS | ||
| xfail[:] test LABEL | ||
| xfail[:] test LABEL DETAILS | ||
| uxsuccess[:] test LABEL | ||
| uxsuccess[:] test LABEL DETAILS | ||
| progress: [+|-]X | ||
| progress: push | ||
| progress: pop | ||
| tags: [-]TAG ... | ||
| time: YYYY-MM-DD HH:MM:SSZ | ||
| LABEL: UTF8* | ||
| NAME: UTF8* | ||
| DETAILS ::= BRACKETED | MULTIPART | ||
| BRACKETED ::= '[' CR UTF8-lines ']' CR | ||
| MULTIPART ::= '[ multipart' CR PART* ']' CR | ||
| PART ::= PART_TYPE CR NAME CR PART_BYTES CR | ||
| PART_TYPE ::= Content-Type: type/sub-type(;parameter=value,parameter=value) | ||
| PART_BYTES ::= (DIGITS CR LF BYTE{DIGITS})* '0' CR LF | ||
| unexpected output on stdout -> stdout. | ||
| exit w/0 or last test completing -> error | ||
| Tags given outside a test are applied to all following tests | ||
| Tags given after a test: line and before the result line for the same test | ||
| apply only to that test, and inherit the current global tags. | ||
| A '-' before a tag is used to remove tags - e.g. to prevent a global tag | ||
| applying to a single test, or to cancel a global tag. | ||
| The progress directive is used to provide progress information about a stream | ||
| so that stream consumer can provide completion estimates, progress bars and so | ||
| on. Stream generators that know how many tests will be present in the stream | ||
| should output "progress: COUNT". Stream filters that add tests should output | ||
| "progress: +COUNT", and those that remove tests should output | ||
| "progress: -COUNT". An absolute count should reset the progress indicators in | ||
| use - it indicates that two separate streams from different generators have | ||
| been trivially concatenated together, and there is no knowledge of how many | ||
| more complete streams are incoming. Smart concatenation could scan each stream | ||
| for their count and sum them, or alternatively translate absolute counts into | ||
| relative counts inline. It is recommended that outputters avoid absolute counts | ||
| unless necessary. The push and pop directives are used to provide local regions | ||
| for progress reporting. This fits with hierarchically operating test | ||
| environments - such as those that organise tests into suites - the top-most | ||
| runner can report on the number of suites, and each suite surround its output | ||
| with a (push, pop) pair. Interpreters should interpret a pop as also advancing | ||
| the progress of the restored level by one step. Encountering progress | ||
| directives between the start and end of a test pair indicates that a previous | ||
| test was interrupted and did not cleanly terminate: it should be implicitly | ||
| closed with an error (the same as when a stream ends with no closing test | ||
| directive for the most recently started test). | ||
| The time directive acts as a clock event - it sets the time for all future | ||
| events. The value should be a valid ISO8601 time. | ||
| The skip, xfail and uxsuccess outcomes are not supported by all testing | ||
| environments. In Python the testttools (https://launchpad.net/testtools) | ||
| library is used to translate these automatically if an older Python version | ||
| that does not support them is in use. See the testtools documentation for the | ||
| translation policy. | ||
| skip is used to indicate a test was discovered but not executed. xfail is used | ||
| to indicate a test that errored in some expected fashion (also know as "TODO" | ||
| tests in some frameworks). uxsuccess is used to indicate and unexpected success | ||
| where a test though to be failing actually passes. It is complementary to | ||
| xfail. | ||
| Hacking on subunit | ||
| ------------------ | ||
| Releases | ||
| ======== | ||
| * Update versions in configure.ac and python/subunit/__init__.py. | ||
| * Update NEWS. | ||
| * Do a make distcheck, which will update Makefile etc. | ||
| * Do a PyPI release: PYTHONPATH=../../python python ../../setup.py sdist bdist_wheel upload -s | ||
| * Upload the regular one to LP. | ||
| * Push a tagged commit. | ||
| git push -t origin master:master | ||
| Keywords: python test streaming | ||
| Platform: UNKNOWN | ||
| Classifier: Intended Audience :: Developers | ||
| Classifier: Operating System :: OS Independent | ||
| Classifier: Programming Language :: Python | ||
| Classifier: Programming Language :: Python :: 2.7 | ||
| Classifier: Programming Language :: Python :: 3 | ||
| Classifier: Programming Language :: Python :: 3.5 | ||
| Classifier: Programming Language :: Python :: 3.6 | ||
| Classifier: Programming Language :: Python :: 3.7 | ||
| Classifier: Programming Language :: Python :: 3.8 | ||
| Classifier: Programming Language :: Python :: 3.9 | ||
| Classifier: Programming Language :: Python :: 3.10 | ||
| Classifier: Topic :: Software Development :: Testing | ||
| Requires-Python: >=3.6 | ||
| Provides-Extra: docs | ||
| Provides-Extra: test | ||
| License-File: COPYING | ||
| subunit: A streaming protocol for test results | ||
| Copyright (C) 2005-2013 Robert Collins <robertc@robertcollins.net> | ||
| Licensed under either the Apache License, Version 2.0 or the BSD 3-clause | ||
| license at the users choice. A copy of both licenses are available in the | ||
| project source as Apache-2.0 and BSD. You may not use this file except in | ||
| compliance with one of these two licences. | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| license you chose for the specific language governing permissions and | ||
| limitations under that license. | ||
| See the COPYING file for full details on the licensing of Subunit. | ||
| subunit reuses iso8601 by Michael Twomey, distributed under an MIT style | ||
| licence - see python/iso8601/LICENSE for details. | ||
| Subunit | ||
| ------- | ||
| Subunit is a streaming protocol for test results. | ||
| There are two major revisions of the protocol. Version 1 was trivially human | ||
| readable but had significant defects as far as highly parallel testing was | ||
| concerned - it had no room for doing discovery and execution in parallel, | ||
| required substantial buffering when multiplexing and was fragile - a corrupt | ||
| byte could cause an entire stream to be misparsed. Version 1.1 added | ||
| encapsulation of binary streams which mitigated some of the issues but the | ||
| core remained. | ||
| Version 2 shares many of the good characteristics of Version 1 - it can be | ||
| embedded into a regular text stream (e.g. from a build system) and it still | ||
| models xUnit style test execution. It also fixes many of the issues with | ||
| Version 1 - Version 2 can be multiplexed without excessive buffering (in | ||
| time or space), it has a well defined recovery mechanism for dealing with | ||
| corrupted streams (e.g. where two processes write to the same stream | ||
| concurrently, or where the stream generator suffers a bug). | ||
| More details on both protocol version s can be found in the 'Protocol' section | ||
| of this document. | ||
| Subunit comes with command line filters to process a subunit stream and | ||
| language bindings for python, C, C++ and shell. Bindings are easy to write | ||
| for other languages. | ||
| A number of useful things can be done easily with subunit: | ||
| * Test aggregation: Tests run separately can be combined and then | ||
| reported/displayed together. For instance, tests from different languages | ||
| can be shown as a seamless whole, and tests running on multiple machines | ||
| can be aggregated into a single stream through a multiplexer. | ||
| * Test archiving: A test run may be recorded and replayed later. | ||
| * Test isolation: Tests that may crash or otherwise interact badly with each | ||
| other can be run seperately and then aggregated, rather than interfering | ||
| with each other or requiring an adhoc test->runner reporting protocol. | ||
| * Grid testing: subunit can act as the necessary serialisation and | ||
| deserialiation to get test runs on distributed machines to be reported in | ||
| real time. | ||
| Subunit supplies the following filters: | ||
| * tap2subunit - convert perl's TestAnythingProtocol to subunit. | ||
| * subunit2csv - convert a subunit stream to csv. | ||
| * subunit2disk - export a subunit stream to files on disk. | ||
| * subunit2pyunit - convert a subunit stream to pyunit test results. | ||
| * subunit2gtk - show a subunit stream in GTK. | ||
| * subunit2junitxml - convert a subunit stream to JUnit's XML format. | ||
| * subunit-diff - compare two subunit streams. | ||
| * subunit-filter - filter out tests from a subunit stream. | ||
| * subunit-ls - list info about tests present in a subunit stream. | ||
| * subunit-stats - generate a summary of a subunit stream. | ||
| * subunit-tags - add or remove tags from a stream. | ||
| Integration with other tools | ||
| ---------------------------- | ||
| Subunit's language bindings act as integration with various test runners like | ||
| 'check', 'cppunit', Python's 'unittest'. Beyond that a small amount of glue | ||
| (typically a few lines) will allow Subunit to be used in more sophisticated | ||
| ways. | ||
| Python | ||
| ====== | ||
| Subunit has excellent Python support: most of the filters and tools are written | ||
| in python and there are facilities for using Subunit to increase test isolation | ||
| seamlessly within a test suite. | ||
| The most common way is to run an existing python test suite and have it output | ||
| subunit via the ``subunit.run`` module:: | ||
| $ python -m subunit.run mypackage.tests.test_suite | ||
| For more information on the Python support Subunit offers , please see | ||
| ``pydoc subunit``, or the source in ``python/subunit/`` | ||
| C | ||
| = | ||
| Subunit has C bindings to emit the protocol. The 'check' C unit testing project | ||
| has included subunit support in their project for some years now. See | ||
| 'c/README' for more details. | ||
| C++ | ||
| === | ||
| The C library is includable and usable directly from C++. A TestListener for | ||
| CPPUnit is included in the Subunit distribution. See 'c++/README' for details. | ||
| shell | ||
| ===== | ||
| There are two sets of shell tools. There are filters, which accept a subunit | ||
| stream on stdin and output processed data (or a transformed stream) on stdout. | ||
| Then there are unittest facilities similar to those for C : shell bindings | ||
| consisting of simple functions to output protocol elements, and a patch for | ||
| adding subunit output to the 'ShUnit' shell test runner. See 'shell/README' for | ||
| details. | ||
| Filter recipes | ||
| -------------- | ||
| To ignore some failing tests whose root cause is already known:: | ||
| subunit-filter --without 'AttributeError.*flavor' | ||
| The xUnit test model | ||
| -------------------- | ||
| Subunit implements a slightly modified xUnit test model. The stock standard | ||
| model is that there are tests, which have an id(), can be run, and when run | ||
| start, emit an outcome (like success or failure) and then finish. | ||
| Subunit extends this with the idea of test enumeration (find out about tests | ||
| a runner has without running them), tags (allow users to describe tests in | ||
| ways the test framework doesn't apply any semantic value to), file attachments | ||
| (allow arbitrary data to make analysing a failure easy) and timestamps. | ||
| The protocol | ||
| ------------ | ||
| Version 2, or v2 is new and still under development, but is intended to | ||
| supercede version 1 in the very near future. Subunit's bundled tools accept | ||
| only version 2 and only emit version 2, but the new filters subunit-1to2 and | ||
| subunit-2to1 can be used to interoperate with older third party libraries. | ||
| Version 2 | ||
| ========= | ||
| Version 2 is a binary protocol consisting of independent packets that can be | ||
| embedded in the output from tools like make - as long as each packet has no | ||
| other bytes mixed in with it (which 'make -j N>1' has a tendency of doing). | ||
| Version 2 is currently in draft form, and early adopters should be willing | ||
| to either discard stored results (if protocol changes are made), or bulk | ||
| convert them back to v1 and then to a newer edition of v2. | ||
| The protocol synchronises at the start of the stream, after a packet, or | ||
| after any 0x0A byte. That is, a subunit v2 packet starts after a newline or | ||
| directly after the end of the prior packet. | ||
| Subunit is intended to be transported over a reliable streaming protocol such | ||
| as TCP. As such it does not concern itself with out of order delivery of | ||
| packets. However, because of the possibility of corruption due to either | ||
| bugs in the sender, or due to mixed up data from concurrent writes to the same | ||
| fd when being embedded, subunit strives to recover reasonably gracefully from | ||
| damaged data. | ||
| A key design goal for Subunit version 2 is to allow processing and multiplexing | ||
| without forcing buffering for semantic correctness, as buffering tends to hide | ||
| hung or otherwise misbehaving tests. That said, limited time based buffering | ||
| for network efficiency is a good idea - this is ultimately implementator | ||
| choice. Line buffering is also discouraged for subunit streams, as dropping | ||
| into a debugger or other tool may require interactive traffic even if line | ||
| buffering would not otherwise be a problem. | ||
| In version two there are two conceptual events - a test status event and a file | ||
| attachment event. Events may have timestamps, and the path of multiplexers that | ||
| an event is routed through is recorded to permit sending actions back to the | ||
| source (such as new tests to run or stdin for driving debuggers and other | ||
| interactive input). Test status events are used to enumerate tests, to report | ||
| tests and test helpers as they run. Tests may have tags, used to allow | ||
| tunnelling extra meanings through subunit without requiring parsing of | ||
| arbitrary file attachments. Things that are not standalone tests get marked | ||
| as such by setting the 'Runnable' flag to false. (For instance, individual | ||
| assertions in TAP are not runnable tests, only the top level TAP test script | ||
| is runnable). | ||
| File attachments are used to provide rich detail about the nature of a failure. | ||
| File attachments can also be used to encapsulate stdout and stderr both during | ||
| and outside tests. | ||
| Most numbers are stored in network byte order - Most Significant Byte first | ||
| encoded using a variation of http://www.dlugosz.com/ZIP2/VLI.html. The first | ||
| byte's top 2 high order bits encode the total number of octets in the number. | ||
| This encoding can encode values from 0 to 2**30-1, enough to encode a | ||
| nanosecond. Numbers that are not variable length encoded are still stored in | ||
| MSB order. | ||
| +--------+--------+---------+------------+ | ||
| | prefix | octets | max | max | | ||
| +========+========+=========+============+ | ||
| | 00 | 1 | 2**6-1 | 63 | | ||
| +--------+--------+---------+------------+ | ||
| | 01 | 2 | 2**14-1 | 16383 | | ||
| +--------+--------+---------+------------+ | ||
| | 10 | 3 | 2**22-1 | 4194303 | | ||
| +--------+--------+---------+------------+ | ||
| | 11 | 4 | 2**30-1 | 1073741823 | | ||
| +--------+--------+---------+------------+ | ||
| All variable length elements of the packet are stored with a length prefix | ||
| number allowing them to be skipped over for consumers that don't need to | ||
| interpret them. | ||
| UTF-8 strings are with no terminating NUL and should not have any embedded NULs | ||
| (implementations SHOULD validate any such strings that they process and take | ||
| some remedial action (such as discarding the packet as corrupt). | ||
| In short the structure of a packet is: | ||
| PACKET := SIGNATURE FLAGS PACKET_LENGTH TIMESTAMP? TESTID? TAGS? MIME? | ||
| FILECONTENT? ROUTING_CODE? CRC32 | ||
| In more detail... | ||
| Packets are identified by a single byte signature - 0xB3, which is never legal | ||
| in a UTF-8 stream as the first byte of a character. 0xB3 starts with the first | ||
| bit set and the second not, which is the UTF-8 signature for a continuation | ||
| byte. 0xB3 was chosen as 0x73 ('s' in ASCII') with the top two bits replaced by | ||
| the 1 and 0 for a continuation byte. | ||
| If subunit packets are being embedded in a non-UTF-8 text stream, where 0x73 is | ||
| a legal character, consider either recoding the text to UTF-8, or using | ||
| subunit's 'file' packets to embed the text stream in subunit, rather than the | ||
| other way around. | ||
| Following the signature byte comes a 16-bit flags field, which includes a | ||
| 4-bit version field - if the version is not 0x2 then the packet cannot be | ||
| read. It is recommended to signal an error at this point (e.g. by emitting | ||
| a synthetic error packet and returning to the top level loop to look for | ||
| new packets, or exiting with an error). If recovery is desired, treat the | ||
| packet signature as an opaque byte and scan for a new synchronisation point. | ||
| NB: Subunit V1 and V2 packets may legitimately included 0xB3 internally, | ||
| as they are an 8-bit safe container format, so recovery from this situation | ||
| may involve an arbitrary number of false positives until an actual packet | ||
| is encountered : and even then it may still be false, failing after passing | ||
| the version check due to coincidence. | ||
| Flags are stored in network byte order too. | ||
| +------------+------------+------------------------+ | ||
| | High byte | Low byte | | ||
| +------------+------------+------------------------+ | ||
| | 15 14 13 12 11 10 9 8 | 7 6 5 4 3 2 1 0 | | ||
| +------------+------------+------------------------+ | ||
| | VERSION | feature bits | | ||
| +------------+-------------------------------------+ | ||
| Valid version values are: | ||
| 0x2 - version 2 | ||
| Feature bits: | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 11 | mask 0x0800 | Test id present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 10 | mask 0x0400 | Routing code present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 9 | mask 0x0200 | Timestamp present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 8 | mask 0x0100 | Test is 'runnable'. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 7 | mask 0x0080 | Tags are present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 6 | mask 0x0040 | File content is present. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 5 | mask 0x0020 | File MIME type is present.| | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 4 | mask 0x0010 | EOF marker. | | ||
| +---------+-------------+---------------------------+ | ||
| | Bit 3 | mask 0x0008 | Must be zero in version 2.| | ||
| +---------+-------------+---------------------------+ | ||
| Test status gets three bits: | ||
| Bit 2 | Bit 1 | Bit 0 - mask 0x0007 - A test status enum lookup: | ||
| * 000 - undefined / no test | ||
| * 001 - Enumeration / existence | ||
| * 002 - In progress | ||
| * 003 - Success | ||
| * 004 - Unexpected Success | ||
| * 005 - Skipped | ||
| * 006 - Failed | ||
| * 007 - Expected failure | ||
| After the flags field is a number field giving the length in bytes for the | ||
| entire packet including the signature and the checksum. This length must | ||
| be less than 4MiB - 4194303 bytes. The encoding can obviously record a larger | ||
| number but one of the goals is to avoid requiring large buffers, or causing | ||
| large latency in the packet forward/processing pipeline. Larger file | ||
| attachments can be communicated in multiple packets, and the overhead in such a | ||
| 4MiB packet is approximately 0.2%. | ||
| The rest of the packet is a series of optional features as specified by the set | ||
| feature bits in the flags field. When absent they are entirely absent. | ||
| Forwarding and multiplexing of packets can be done without interpreting the | ||
| remainder of the packet until the routing code and checksum (which are both at | ||
| the end of the packet). Additionally, routers can often avoid copying or moving | ||
| the bulk of the packet, as long as the routing code size increase doesn't force | ||
| the length encoding to take up a new byte (which will only happen to packets | ||
| less than or equal to 16KiB in length) - large packets are very efficient to | ||
| route. | ||
| Timestamp when present is a 32 bit unsigned integer for seconds, and a variable | ||
| length number for nanoseconds, representing UTC time since Unix Epoch in | ||
| seconds and nanoseconds. | ||
| Test id when present is a UTF-8 string. The test id should uniquely identify | ||
| runnable tests such that they can be selected individually. For tests and other | ||
| actions which cannot be individually run (such as test | ||
| fixtures/layers/subtests) uniqueness is not required (though being human | ||
| meaningful is highly recommended). | ||
| Tags when present is a length prefixed vector of UTF-8 strings, one per tag. | ||
| There are no restrictions on tag content (other than the restrictions on UTF-8 | ||
| strings in subunit in general). Tags have no ordering. | ||
| When a MIME type is present, it defines the MIME type for the file across all | ||
| packets same file (routing code + testid + name uniquely identifies a file, | ||
| reset when EOF is flagged). If a file never has a MIME type set, it should be | ||
| treated as application/octet-stream. | ||
| File content when present is a UTF-8 string for the name followed by the length | ||
| in bytes of the content, and then the content octets. | ||
| If present routing code is a UTF-8 string. The routing code is used to | ||
| determine which test backend a test was running on when doing data analysis, | ||
| and to route stdin to the test process if interaction is required. | ||
| Multiplexers SHOULD add a routing code if none is present, and prefix any | ||
| existing routing code with a routing code ('/' separated) if one is already | ||
| present. For example, a multiplexer might label each stream it is multiplexing | ||
| with a simple ordinal ('0', '1' etc), and given an incoming packet with route | ||
| code '3' from stream '0' would adjust the route code when forwarding the packet | ||
| to be '0/3'. | ||
| Following the end of the packet is a CRC-32 checksum of the contents of the | ||
| packet including the signature. | ||
| Example packets | ||
| ~~~~~~~~~~~~~~~ | ||
| Trivial test "foo" enumeration packet, with test id, runnable set, | ||
| status=enumeration. Spaces below are to visually break up signature / flags / | ||
| length / testid / crc32 | ||
| b3 2901 0c 03666f6f 08555f1b | ||
| Version 1 (and 1.1) | ||
| =================== | ||
| Version 1 (and 1.1) are mostly human readable protocols. | ||
| Sample subunit wire contents | ||
| ---------------------------- | ||
| The following:: | ||
| test: test foo works | ||
| success: test foo works | ||
| test: tar a file. | ||
| failure: tar a file. [ | ||
| .. | ||
| ].. space is eaten. | ||
| foo.c:34 WARNING foo is not defined. | ||
| ] | ||
| a writeln to stdout | ||
| When run through subunit2pyunit:: | ||
| .F | ||
| a writeln to stdout | ||
| ======================== | ||
| FAILURE: tar a file. | ||
| ------------------- | ||
| .. | ||
| ].. space is eaten. | ||
| foo.c:34 WARNING foo is not defined. | ||
| Subunit v1 protocol description | ||
| =============================== | ||
| This description is being ported to an EBNF style. Currently its only partly in | ||
| that style, but should be fairly clear all the same. When in doubt, refer the | ||
| source (and ideally help fix up the description!). Generally the protocol is | ||
| line orientated and consists of either directives and their parameters, or | ||
| when outside a DETAILS region unexpected lines which are not interpreted by | ||
| the parser - they should be forwarded unaltered:: | ||
| test|testing|test:|testing: test LABEL | ||
| success|success:|successful|successful: test LABEL | ||
| success|success:|successful|successful: test LABEL DETAILS | ||
| failure: test LABEL | ||
| failure: test LABEL DETAILS | ||
| error: test LABEL | ||
| error: test LABEL DETAILS | ||
| skip[:] test LABEL | ||
| skip[:] test LABEL DETAILS | ||
| xfail[:] test LABEL | ||
| xfail[:] test LABEL DETAILS | ||
| uxsuccess[:] test LABEL | ||
| uxsuccess[:] test LABEL DETAILS | ||
| progress: [+|-]X | ||
| progress: push | ||
| progress: pop | ||
| tags: [-]TAG ... | ||
| time: YYYY-MM-DD HH:MM:SSZ | ||
| LABEL: UTF8* | ||
| NAME: UTF8* | ||
| DETAILS ::= BRACKETED | MULTIPART | ||
| BRACKETED ::= '[' CR UTF8-lines ']' CR | ||
| MULTIPART ::= '[ multipart' CR PART* ']' CR | ||
| PART ::= PART_TYPE CR NAME CR PART_BYTES CR | ||
| PART_TYPE ::= Content-Type: type/sub-type(;parameter=value,parameter=value) | ||
| PART_BYTES ::= (DIGITS CR LF BYTE{DIGITS})* '0' CR LF | ||
| unexpected output on stdout -> stdout. | ||
| exit w/0 or last test completing -> error | ||
| Tags given outside a test are applied to all following tests | ||
| Tags given after a test: line and before the result line for the same test | ||
| apply only to that test, and inherit the current global tags. | ||
| A '-' before a tag is used to remove tags - e.g. to prevent a global tag | ||
| applying to a single test, or to cancel a global tag. | ||
| The progress directive is used to provide progress information about a stream | ||
| so that stream consumer can provide completion estimates, progress bars and so | ||
| on. Stream generators that know how many tests will be present in the stream | ||
| should output "progress: COUNT". Stream filters that add tests should output | ||
| "progress: +COUNT", and those that remove tests should output | ||
| "progress: -COUNT". An absolute count should reset the progress indicators in | ||
| use - it indicates that two separate streams from different generators have | ||
| been trivially concatenated together, and there is no knowledge of how many | ||
| more complete streams are incoming. Smart concatenation could scan each stream | ||
| for their count and sum them, or alternatively translate absolute counts into | ||
| relative counts inline. It is recommended that outputters avoid absolute counts | ||
| unless necessary. The push and pop directives are used to provide local regions | ||
| for progress reporting. This fits with hierarchically operating test | ||
| environments - such as those that organise tests into suites - the top-most | ||
| runner can report on the number of suites, and each suite surround its output | ||
| with a (push, pop) pair. Interpreters should interpret a pop as also advancing | ||
| the progress of the restored level by one step. Encountering progress | ||
| directives between the start and end of a test pair indicates that a previous | ||
| test was interrupted and did not cleanly terminate: it should be implicitly | ||
| closed with an error (the same as when a stream ends with no closing test | ||
| directive for the most recently started test). | ||
| The time directive acts as a clock event - it sets the time for all future | ||
| events. The value should be a valid ISO8601 time. | ||
| The skip, xfail and uxsuccess outcomes are not supported by all testing | ||
| environments. In Python the testttools (https://launchpad.net/testtools) | ||
| library is used to translate these automatically if an older Python version | ||
| that does not support them is in use. See the testtools documentation for the | ||
| translation policy. | ||
| skip is used to indicate a test was discovered but not executed. xfail is used | ||
| to indicate a test that errored in some expected fashion (also know as "TODO" | ||
| tests in some frameworks). uxsuccess is used to indicate and unexpected success | ||
| where a test though to be failing actually passes. It is complementary to | ||
| xfail. | ||
| Hacking on subunit | ||
| ------------------ | ||
| Releases | ||
| ======== | ||
| * Update versions in configure.ac and python/subunit/__init__.py. | ||
| * Update NEWS. | ||
| * Do a make distcheck, which will update Makefile etc. | ||
| * Do a PyPI release: PYTHONPATH=../../python python ../../setup.py sdist bdist_wheel upload -s | ||
| * Upload the regular one to LP. | ||
| * Push a tagged commit. | ||
| git push -t origin master:master |
@@ -9,5 +9,3 @@ extras | ||
| fixtures | ||
| hypothesis | ||
| testscenarios | ||
| [test:python_version!="3.2"] | ||
| hypothesis |
@@ -1,2 +0,1 @@ | ||
| .travis.yml | ||
| Apache-2.0 | ||
@@ -9,18 +8,5 @@ BSD | ||
| all_tests.py | ||
| setup.cfg | ||
| pyproject.toml | ||
| setup.py | ||
| filters/subunit-1to2 | ||
| filters/subunit-2to1 | ||
| filters/subunit-filter | ||
| filters/subunit-ls | ||
| filters/subunit-notify | ||
| filters/subunit-output | ||
| filters/subunit-stats | ||
| filters/subunit-tags | ||
| filters/subunit2csv | ||
| filters/subunit2disk | ||
| filters/subunit2gtk | ||
| filters/subunit2junitxml | ||
| filters/subunit2pyunit | ||
| filters/tap2subunit | ||
| .github/workflows/main.yml | ||
| python/iso8601/LICENSE | ||
@@ -42,2 +28,17 @@ python/iso8601/README | ||
| python/subunit/v2.py | ||
| python/subunit/filter_scripts/__init__.py | ||
| python/subunit/filter_scripts/subunit2csv.py | ||
| python/subunit/filter_scripts/subunit2disk.py | ||
| python/subunit/filter_scripts/subunit2gtk.py | ||
| python/subunit/filter_scripts/subunit2junitxml.py | ||
| python/subunit/filter_scripts/subunit2pyunit.py | ||
| python/subunit/filter_scripts/subunit_1to2.py | ||
| python/subunit/filter_scripts/subunit_2to1.py | ||
| python/subunit/filter_scripts/subunit_filter.py | ||
| python/subunit/filter_scripts/subunit_ls.py | ||
| python/subunit/filter_scripts/subunit_notify.py | ||
| python/subunit/filter_scripts/subunit_output.py | ||
| python/subunit/filter_scripts/subunit_stats.py | ||
| python/subunit/filter_scripts/subunit_tags.py | ||
| python/subunit/filter_scripts/tap2subunit.py | ||
| python/subunit/tests/__init__.py | ||
@@ -63,3 +64,4 @@ python/subunit/tests/sample-script.py | ||
| python_subunit.egg-info/dependency_links.txt | ||
| python_subunit.egg-info/entry_points.txt | ||
| python_subunit.egg-info/requires.txt | ||
| python_subunit.egg-info/top_level.txt |
@@ -45,4 +45,3 @@ # | ||
| the extension methods will not cause errors to be raised, instead the extension | ||
| will either lose fidelity (for instance, folding expected failures to success | ||
| in Python versions < 2.7 or 3.1), or discard the extended data (for extra | ||
| will either lose fidelity, or discard the extended data (for extra | ||
| details, tags, timestamping and progress markers). | ||
@@ -120,2 +119,5 @@ | ||
| from io import BytesIO | ||
| from io import StringIO | ||
| from io import UnsupportedOperation as _UnsupportedOperation | ||
| import os | ||
@@ -126,6 +128,2 @@ import re | ||
| import unittest | ||
| try: | ||
| from io import UnsupportedOperation as _UnsupportedOperation | ||
| except ImportError: | ||
| _UnsupportedOperation = AttributeError | ||
@@ -135,3 +133,3 @@ from extras import safe_hasattr | ||
| from testtools.content import TracebackContent | ||
| from testtools.compat import _b, _u, BytesIO, StringIO | ||
| from testtools.compat import _b, _u | ||
| try: | ||
@@ -160,3 +158,3 @@ from testtools.testresult.real import _StringException | ||
| __version__ = (1, 4, 0, 'final', 0) | ||
| __version__ = (1, 4, 1, 'final', 0) | ||
@@ -517,5 +515,3 @@ PROGRESS_SET = 0 | ||
| if stream is None: | ||
| stream = sys.stdout | ||
| if sys.version_info > (3, 0): | ||
| stream = stream.buffer | ||
| stream = sys.stdout.buffer | ||
| self._stream = stream | ||
@@ -826,3 +822,3 @@ self._forward_stream = forward_stream or DiscardStream() | ||
| param_strs = [] | ||
| for param, value in parameters.items(): | ||
| for param, value in sorted(parameters.items()): | ||
| param_strs.append("%s=%s" % (param, value)) | ||
@@ -1043,3 +1039,3 @@ self._stream.write(_b(",".join(param_strs))) | ||
| if state == BEFORE_PLAN: | ||
| match = re.match("(\d+)\.\.(\d+)\s*(?:\#\s+(.*))?\n", line) | ||
| match = re.match(r"(\d+)\.\.(\d+)\s*(?:\#\s+(.*))?\n", line) | ||
| if match: | ||
@@ -1057,3 +1053,3 @@ state = AFTER_PLAN | ||
| # not a plan line, or have seen one before | ||
| match = re.match("(ok|not ok)(?:\s+(\d+)?)?(?:\s+([^#]*[^#\s]+)\s*)?(?:\s+#\s+(TODO|SKIP|skip|todo)(?:\s+(.*))?)?\n", line) | ||
| match = re.match(r"(ok|not ok)(?:\s+(\d+)?)?(?:\s+([^#]*[^#\s]+)\s*)?(?:\s+#\s+(TODO|SKIP|skip|todo)(?:\s+(.*))?)?\n", line) | ||
| if match: | ||
@@ -1086,3 +1082,3 @@ # new test, emit current one. | ||
| continue | ||
| match = re.match("Bail out\!(?:\s*(.*))?\n", line) | ||
| match = re.match(r"Bail out\!(?:\s*(.*))?\n", line) | ||
| if match: | ||
@@ -1099,3 +1095,3 @@ reason, = match.groups() | ||
| continue | ||
| match = re.match("\#.*\n", line) | ||
| match = re.match(r"\#.*\n", line) | ||
| if match: | ||
@@ -1302,7 +1298,3 @@ log.append(line[:-1]) | ||
| exceptions = (_UnsupportedOperation, IOError) | ||
| if sys.version_info > (3, 0): | ||
| unicode_type = str | ||
| else: | ||
| unicode_type = unicode | ||
| exceptions += (ValueError,) | ||
| unicode_type = str | ||
| try: | ||
@@ -1309,0 +1301,0 @@ # Read streams |
@@ -9,3 +9,3 @@ # | ||
| # compliance with one of these two licences. | ||
| # | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
@@ -20,4 +20,6 @@ # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| from io import BytesIO, StringIO | ||
| from testtools import content, content_type | ||
| from testtools.compat import _b, BytesIO | ||
| from testtools.compat import _b | ||
@@ -24,0 +26,0 @@ from subunit import chunked |
| # Copyright (c) 2007 Michael Twomey | ||
| # | ||
| # | ||
| # Permission is hereby granted, free of charge, to any person obtaining a | ||
@@ -10,6 +10,6 @@ # copy of this software and associated documentation files (the | ||
| # the following conditions: | ||
| # | ||
| # | ||
| # The above copyright notice and this permission notice shall be included | ||
| # in all copies or substantial portions of the Software. | ||
| # | ||
| # | ||
| # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS | ||
@@ -35,3 +35,2 @@ # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | ||
| import re | ||
| import sys | ||
@@ -52,6 +51,2 @@ __all__ = ["parse_date", "ParseError"] | ||
| if sys.version_info < (3, 0): | ||
| bytes = str | ||
| class ParseError(Exception): | ||
@@ -64,3 +59,3 @@ """Raised when there is a problem parsing a date string""" | ||
| """UTC | ||
| """ | ||
@@ -79,3 +74,3 @@ def utcoffset(self, dt): | ||
| """Fixed offset in hours and minutes from UTC | ||
| """ | ||
@@ -94,3 +89,3 @@ def __init__(self, offset_hours, offset_minutes, name): | ||
| return ZERO | ||
| def __repr__(self): | ||
@@ -101,3 +96,3 @@ return "<FixedOffset %r>" % self.__name | ||
| """Parses ISO 8601 time zone specs into tzinfo offsets | ||
| """ | ||
@@ -121,3 +116,3 @@ if tzstring == zulu: | ||
| """Parses ISO 8601 dates into datetime objects | ||
| The timezone is parsed from the date string. However it is quite common to | ||
@@ -124,0 +119,0 @@ have dates without a timezone (not strictly correct). In this case the |
@@ -1,2 +0,2 @@ | ||
| #!/usr/bin/python | ||
| #!/usr/bin/python3 | ||
| # | ||
@@ -139,6 +139,3 @@ # Simple subunit testrunner for python | ||
| binstdout = io.open(stdout.fileno(), 'wb', 0) | ||
| if sys.version_info[0] > 2: | ||
| sys.stdout = io.TextIOWrapper(binstdout, encoding=sys.stdout.encoding) | ||
| else: | ||
| sys.stdout = binstdout | ||
| sys.stdout = io.TextIOWrapper(binstdout, encoding=sys.stdout.encoding) | ||
| stdout = sys.stdout | ||
@@ -145,0 +142,0 @@ SubunitTestProgram(module=None, argv=argv, testRunner=runner, |
@@ -26,2 +26,3 @@ # | ||
| _remote_exception_repr = "testtools.testresult.real._StringException" | ||
| _remote_exception_repr_chunked = "34\r\n" + _remote_exception_repr + ": boo qux\n0\r\n" | ||
| _remote_exception_str = "Traceback (most recent call last):\ntesttools.testresult.real._StringException" | ||
@@ -28,0 +29,0 @@ _remote_exception_str_chunked = "57\r\n" + _remote_exception_str + ": boo qux\n0\r\n" |
@@ -1,2 +0,2 @@ | ||
| #!/usr/bin/env python | ||
| #!/usr/bin/env python3 | ||
| import sys | ||
@@ -3,0 +3,0 @@ if sys.platform == "win32": |
@@ -1,2 +0,2 @@ | ||
| #!/usr/bin/env python | ||
| #!/usr/bin/env python3 | ||
| import sys | ||
@@ -3,0 +3,0 @@ print("test old mcdonald") |
@@ -10,3 +10,3 @@ # | ||
| # compliance with one of these two licences. | ||
| # | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
@@ -19,5 +19,6 @@ # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| from io import BytesIO | ||
| import unittest | ||
| from testtools.compat import _b, BytesIO | ||
| from testtools.compat import _b | ||
@@ -24,0 +25,0 @@ import subunit.chunked |
@@ -19,3 +19,3 @@ # | ||
| from testtools.compat import _b, StringIO | ||
| from testtools.compat import _b | ||
@@ -22,0 +22,0 @@ import subunit.tests |
@@ -20,13 +20,14 @@ # | ||
| from datetime import datetime | ||
| from io import BytesIO | ||
| import os | ||
| import subprocess | ||
| import sys | ||
| from subunit import iso8601 | ||
| import unittest | ||
| from testtools.compat import _b | ||
| from testtools import TestCase | ||
| from testtools.compat import _b, BytesIO | ||
| from testtools.testresult.doubles import ExtendedTestResult, StreamResult | ||
| import subunit | ||
| from subunit import iso8601 | ||
| from subunit.test_results import make_tag_filter, TestResultFilter | ||
@@ -300,14 +301,7 @@ from subunit import ByteStreamToStreamResult, StreamResultToBytes | ||
| if sys.version_info < (2, 7): | ||
| # These tests require Python >=2.7. | ||
| del test_fixup_expected_failures, test_fixup_expected_errors, test_fixup_unexpected_success | ||
| class TestFilterCommand(TestCase): | ||
| def run_command(self, args, stream): | ||
| root = os.path.dirname( | ||
| os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) | ||
| script_path = os.path.join(root, 'filters', 'subunit-filter') | ||
| command = [sys.executable, script_path] + list(args) | ||
| command = [sys.executable, '-m', 'subunit.filter_scripts.subunit_filter'] + list(args) | ||
| ps = subprocess.Popen( | ||
@@ -314,0 +308,0 @@ command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, |
@@ -9,3 +9,3 @@ # | ||
| # compliance with one of these two licences. | ||
| # | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
@@ -20,5 +20,7 @@ # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT | ||
| from io import BytesIO | ||
| from io import StringIO | ||
| import unittest | ||
| from testtools.compat import _b, BytesIO, StringIO | ||
| from testtools.compat import _b | ||
@@ -25,0 +27,0 @@ import subunit |
@@ -59,2 +59,4 @@ # | ||
| b'\x83\x1b\x04test\x03\x03foo\x03bar\x04quux:\x05e\x80', | ||
| b'\xb3)\x82\x17\x04test\x02\x03foo\x04quux\xa6\xe1\xde\xec\xb3)' | ||
| b'\x83\x1b\x04test\x03\x04quux\x03foo\x03bar\xaf\xbd\x9d\xd6', | ||
| ] | ||
@@ -61,0 +63,0 @@ stream = subunit.StreamResultToBytes(self.original) |
@@ -19,9 +19,10 @@ # | ||
| import io | ||
| import unittest2 as unittest | ||
| from io import BytesIO | ||
| from io import StringIO | ||
| import os | ||
| import sys | ||
| import tempfile | ||
| import unittest | ||
| from testtools import PlaceHolder, skipIf, TestCase, TestResult | ||
| from testtools.compat import _b, _u, BytesIO | ||
| from testtools.compat import _b, _u | ||
| from testtools.content import Content, TracebackContent, text_content | ||
@@ -41,3 +42,3 @@ from testtools.content_type import ContentType | ||
| ) | ||
| from testtools.matchers import Contains | ||
| from testtools.matchers import Contains, Equals, MatchesAny | ||
@@ -47,2 +48,3 @@ import subunit | ||
| _remote_exception_repr, | ||
| _remote_exception_repr_chunked, | ||
| _remote_exception_str, | ||
@@ -54,3 +56,3 @@ _remote_exception_str_chunked, | ||
| tb_prelude = "Traceback (most recent call last):\n" | ||
| tb_prelude = "Traceback (most recent call last):\n" | ||
@@ -67,7 +69,3 @@ | ||
| fake_file = os.fdopen(fd, 'r') | ||
| if sys.version_info > (3, 0): | ||
| self.assertEqual(fake_file.buffer, | ||
| subunit._unwrap_text(fake_file)) | ||
| else: | ||
| self.assertEqual(fake_file, subunit._unwrap_text(fake_file)) | ||
| self.assertEqual(fake_file.buffer, subunit._unwrap_text(fake_file)) | ||
@@ -78,7 +76,3 @@ def test__unwrap_text_file_write_mode(self): | ||
| fake_file = os.fdopen(fd, 'w') | ||
| if sys.version_info > (3, 0): | ||
| self.assertEqual(fake_file.buffer, | ||
| subunit._unwrap_text(fake_file)) | ||
| else: | ||
| self.assertEqual(fake_file, subunit._unwrap_text(fake_file)) | ||
| self.assertEqual(fake_file.buffer, subunit._unwrap_text(fake_file)) | ||
@@ -162,8 +156,9 @@ def test__unwrap_text_fileIO_read_mode(self): | ||
| self.assertEqual( | ||
| client.errors, | ||
| [(an_error, tb_prelude + _remote_exception_repr + '\n')]) | ||
| client.errors, [(an_error, _remote_exception_repr + '\n')], | ||
| ) | ||
| self.assertEqual( | ||
| client.failures, | ||
| [(bing, tb_prelude + _remote_exception_repr + ": " | ||
| + details_to_str({'traceback': text_content(traceback)}) + "\n")]) | ||
| [(bing, _remote_exception_repr + ": " | ||
| + details_to_str({'traceback': text_content(traceback)}) + "\n")], | ||
| ) | ||
| self.assertEqual(client.testsRun, 3) | ||
@@ -1022,4 +1017,4 @@ | ||
| test.run(result) | ||
| self.assertEqual([(test, tb_prelude + _remote_exception_repr + ": " | ||
| "Cannot run RemotedTestCases.\n\n")], | ||
| self.assertEqual([(test, _remote_exception_repr + ': ' + | ||
| "Cannot run RemotedTestCases.\n\n")], | ||
| result.errors) | ||
@@ -1189,2 +1184,7 @@ self.assertEqual(1, result.testsRun) | ||
| # A number of these tests produce different output depending on the | ||
| # testtools version. testtools < 2.5.0 used traceback2, which incorrectly | ||
| # included the traceback header even for an exception with no traceback. | ||
| # testtools 2.5.0 switched to the Python 3 standard library's traceback | ||
| # module, which fixes this bug. See https://bugs.python.org/issue24695. | ||
| class TestTestProtocolClient(TestCase): | ||
@@ -1245,6 +1245,13 @@ | ||
| self.test, subunit.RemoteError(_u("boo qux"))) | ||
| self.assertEqual( | ||
| self.io.getvalue(), | ||
| _b(('failure: %s [\n' + _remote_exception_str + ': boo qux\n]\n') | ||
| % self.test.id())) | ||
| self.assertThat(self.io.getvalue(), MatchesAny( | ||
| # testtools < 2.5.0 | ||
| Equals(_b(( | ||
| 'failure: %s [\n' + | ||
| _remote_exception_str + ': boo qux\n' + | ||
| ']\n') % self.test.id())), | ||
| # testtools >= 2.5.0 | ||
| Equals(_b(( | ||
| 'failure: %s [\n' + | ||
| _remote_exception_repr + ': boo qux\n' + | ||
| ']\n') % self.test.id())))) | ||
@@ -1255,19 +1262,21 @@ def test_add_failure_details(self): | ||
| self.test, details=self.sample_tb_details) | ||
| self.assertThat([ | ||
| _b(("failure: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;charset=utf8,language=python\n" | ||
| "traceback\n" + _remote_exception_str_chunked + | ||
| "]\n") % self.test.id()), | ||
| _b(("failure: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;language=python,charset=utf8\n" | ||
| "traceback\n" + _remote_exception_str_chunked + | ||
| "]\n") % self.test.id()), | ||
| ], | ||
| Contains(self.io.getvalue())), | ||
| self.assertThat(self.io.getvalue(), MatchesAny( | ||
| # testtools < 2.5.0 | ||
| Equals(_b(( | ||
| "failure: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;charset=utf8,language=python\n" | ||
| "traceback\n" + _remote_exception_str_chunked + | ||
| "]\n") % self.test.id())), | ||
| # testtools >= 2.5.0 | ||
| Equals(_b(( | ||
| "failure: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;charset=utf8,language=python\n" | ||
| "traceback\n" + _remote_exception_repr_chunked + | ||
| "]\n") % self.test.id())))) | ||
@@ -1278,7 +1287,13 @@ def test_add_error(self): | ||
| self.test, subunit.RemoteError(_u("phwoar crikey"))) | ||
| self.assertEqual( | ||
| self.io.getvalue(), | ||
| _b(('error: %s [\n' + | ||
| _remote_exception_str + ": phwoar crikey\n" | ||
| "]\n") % self.test.id())) | ||
| self.assertThat(self.io.getvalue(), MatchesAny( | ||
| # testtools < 2.5.0 | ||
| Equals(_b(( | ||
| 'error: %s [\n' + | ||
| _remote_exception_str + ": phwoar crikey\n" | ||
| "]\n") % self.test.id())), | ||
| # testtools >= 2.5.0 | ||
| Equals(_b(( | ||
| 'error: %s [\n' + | ||
| _remote_exception_repr + ": phwoar crikey\n" | ||
| "]\n") % self.test.id())))) | ||
@@ -1289,19 +1304,21 @@ def test_add_error_details(self): | ||
| self.test, details=self.sample_tb_details) | ||
| self.assertThat([ | ||
| _b(("error: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;charset=utf8,language=python\n" | ||
| "traceback\n" + _remote_exception_str_chunked + | ||
| "]\n") % self.test.id()), | ||
| _b(("error: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;language=python,charset=utf8\n" | ||
| "traceback\n" + _remote_exception_str_chunked + | ||
| "]\n") % self.test.id()), | ||
| ], | ||
| Contains(self.io.getvalue())), | ||
| self.assertThat(self.io.getvalue(), MatchesAny( | ||
| # testtools < 2.5.0 | ||
| Equals(_b(( | ||
| "error: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;charset=utf8,language=python\n" | ||
| "traceback\n" + _remote_exception_str_chunked + | ||
| "]\n") % self.test.id())), | ||
| # testtools >= 2.5.0 | ||
| Equals(_b(( | ||
| "error: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;charset=utf8,language=python\n" | ||
| "traceback\n" + _remote_exception_repr_chunked + | ||
| "]\n") % self.test.id())))) | ||
@@ -1312,7 +1329,13 @@ def test_add_expected_failure(self): | ||
| self.test, subunit.RemoteError(_u("phwoar crikey"))) | ||
| self.assertEqual( | ||
| self.io.getvalue(), | ||
| _b(('xfail: %s [\n' + | ||
| _remote_exception_str + ": phwoar crikey\n" | ||
| "]\n") % self.test.id())) | ||
| self.assertThat(self.io.getvalue(), MatchesAny( | ||
| # testtools < 2.5.0 | ||
| Equals(_b(( | ||
| 'xfail: %s [\n' + | ||
| _remote_exception_str + ": phwoar crikey\n" | ||
| "]\n") % self.test.id())), | ||
| # testtools >= 2.5.0 | ||
| Equals(_b(( | ||
| 'xfail: %s [\n' + | ||
| _remote_exception_repr + ": phwoar crikey\n" | ||
| "]\n") % self.test.id())))) | ||
@@ -1323,19 +1346,21 @@ def test_add_expected_failure_details(self): | ||
| self.test, details=self.sample_tb_details) | ||
| self.assertThat([ | ||
| _b(("xfail: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;charset=utf8,language=python\n" | ||
| "traceback\n" + _remote_exception_str_chunked + | ||
| "]\n") % self.test.id()), | ||
| _b(("xfail: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;language=python,charset=utf8\n" | ||
| "traceback\n" + _remote_exception_str_chunked + | ||
| "]\n") % self.test.id()), | ||
| ], | ||
| Contains(self.io.getvalue())), | ||
| self.assertThat(self.io.getvalue(), MatchesAny( | ||
| # testtools < 2.5.0 | ||
| Equals(_b(( | ||
| "xfail: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;charset=utf8,language=python\n" | ||
| "traceback\n" + _remote_exception_str_chunked + | ||
| "]\n") % self.test.id())), | ||
| # testtools >= 2.5.0 | ||
| Equals(_b(( | ||
| "xfail: %s [ multipart\n" | ||
| "Content-Type: text/plain\n" | ||
| "something\n" | ||
| "F\r\nserialised\nform0\r\n" | ||
| "Content-Type: text/x-traceback;charset=utf8,language=python\n" | ||
| "traceback\n" + _remote_exception_repr_chunked + | ||
| "]\n") % self.test.id())))) | ||
@@ -1342,0 +1367,0 @@ def test_add_skip(self): |
@@ -19,2 +19,3 @@ # | ||
| import datetime | ||
| from io import StringIO | ||
| import sys | ||
@@ -24,7 +25,6 @@ import unittest | ||
| from testtools import TestCase | ||
| from testtools.compat import StringIO | ||
| from testtools.content import ( | ||
| text_content, | ||
| TracebackContent, | ||
| ) | ||
| ) | ||
| from testtools.testresult.doubles import ExtendedTestResult | ||
@@ -382,6 +382,3 @@ | ||
| self.result = subunit.test_results.TestByTestResult(self.on_test) | ||
| if sys.version_info >= (3, 0): | ||
| self.result._now = iter(range(5)).__next__ | ||
| else: | ||
| self.result._now = iter(range(5)).next | ||
| self.result._now = iter(range(5)).__next__ | ||
@@ -542,6 +539,3 @@ def assertCalled(self, **kwargs): | ||
| result = subunit.test_results.CsvResult(stream) | ||
| if sys.version_info >= (3, 0): | ||
| result._now = iter(range(5)).__next__ | ||
| else: | ||
| result._now = iter(range(5)).next | ||
| result._now = iter(range(5)).__next__ | ||
| result.startTestRun() | ||
@@ -548,0 +542,0 @@ result.startTest(self) |
+14
-19
@@ -18,6 +18,3 @@ # | ||
| import codecs | ||
| utf_8_decode = codecs.utf_8_decode | ||
| import datetime | ||
| from io import UnsupportedOperation | ||
| import os | ||
| import select | ||
@@ -34,2 +31,4 @@ import struct | ||
| utf_8_decode = codecs.utf_8_decode | ||
| __all__ = [ | ||
@@ -58,3 +57,2 @@ 'ByteStreamToStreamResult', | ||
| _nul_test_broken = {} | ||
| _PY3 = (sys.version_info >= (3,)) | ||
@@ -238,17 +236,14 @@ | ||
| data = content + struct.pack(FMT_32, zlib.crc32(content) & 0xffffffff) | ||
| if _PY3: | ||
| # On eventlet 0.17.3, GreenIO.write() can make partial write. | ||
| # Use a loop to ensure that all bytes are written. | ||
| # See also the eventlet issue: | ||
| # https://github.com/eventlet/eventlet/issues/248 | ||
| view = memoryview(data) | ||
| datalen = len(data) | ||
| offset = 0 | ||
| while offset < datalen: | ||
| written = self.output_stream.write(view[offset:]) | ||
| if written is None: | ||
| break | ||
| offset += written | ||
| else: | ||
| self.output_stream.write(data) | ||
| # On eventlet 0.17.3, GreenIO.write() can make partial write. | ||
| # Use a loop to ensure that all bytes are written. | ||
| # See also the eventlet issue: | ||
| # https://github.com/eventlet/eventlet/issues/248 | ||
| view = memoryview(data) | ||
| datalen = len(data) | ||
| offset = 0 | ||
| while offset < datalen: | ||
| written = self.output_stream.write(view[offset:]) | ||
| if written is None: | ||
| break | ||
| offset += written | ||
| self.output_stream.flush() | ||
@@ -255,0 +250,0 @@ |
+0
-3
@@ -1,4 +0,1 @@ | ||
| [bdist_wheel] | ||
| universal = 1 | ||
| [egg_info] | ||
@@ -5,0 +2,0 @@ tag_build = |
+44
-47
@@ -1,27 +0,5 @@ | ||
| #!/usr/bin/env python | ||
| #!/usr/bin/env python3 | ||
| import os.path | ||
| try: | ||
| # If the user has setuptools / distribute installed, use it | ||
| from setuptools import setup | ||
| except ImportError: | ||
| # Otherwise, fall back to distutils. | ||
| from distutils.core import setup | ||
| extra = {} | ||
| else: | ||
| extra = { | ||
| 'install_requires': [ | ||
| 'extras', | ||
| 'testtools>=0.9.34', | ||
| ], | ||
| 'tests_require': [ | ||
| 'fixtures', | ||
| 'hypothesis', | ||
| 'testscenarios', | ||
| ], | ||
| 'extras_require': { | ||
| 'docs': ['docutils'], | ||
| 'test': ['fixtures', 'testscenarios'], | ||
| 'test:python_version!="3.2"': ['hypothesis'], | ||
| }, | ||
| } | ||
| from setuptools import setup | ||
@@ -32,4 +10,5 @@ | ||
| try: | ||
| return [x for x in open(filename) | ||
| if x.startswith(start_of_line)][-1].split(split_marker)[1].strip() | ||
| return [ | ||
| x for x in open(filename) if x.startswith(start_of_line) | ||
| ][-1].split(split_marker)[1].strip() | ||
| except (IOError, IndexError): | ||
@@ -44,3 +23,4 @@ return None | ||
| or _get_version_from_file('Makefile', 'VERSION', '=') | ||
| or "0.0") | ||
| or "0.0" | ||
| ) | ||
@@ -51,2 +31,3 @@ | ||
| os.chdir(relpath) | ||
| setup( | ||
@@ -61,8 +42,8 @@ name='python-subunit', | ||
| 'Programming Language :: Python', | ||
| 'Programming Language :: Python :: 2.7', | ||
| 'Programming Language :: Python :: 3', | ||
| 'Programming Language :: Python :: 3.5', | ||
| 'Programming Language :: Python :: 3.6', | ||
| 'Programming Language :: Python :: 3.7', | ||
| 'Programming Language :: Python :: 3.8', | ||
| 'Programming Language :: Python :: 3.9', | ||
| 'Programming Language :: Python :: 3.10', | ||
| 'Topic :: Software Development :: Testing', | ||
@@ -79,21 +60,37 @@ ], | ||
| }, | ||
| packages=['subunit', 'subunit.tests'], | ||
| packages=['subunit', 'subunit.tests', 'subunit.filter_scripts'], | ||
| package_dir={'subunit': 'python/subunit'}, | ||
| scripts = [ | ||
| 'filters/subunit-1to2', | ||
| 'filters/subunit-2to1', | ||
| 'filters/subunit-filter', | ||
| 'filters/subunit-ls', | ||
| 'filters/subunit-notify', | ||
| 'filters/subunit-output', | ||
| 'filters/subunit-stats', | ||
| 'filters/subunit-tags', | ||
| 'filters/subunit2csv', | ||
| 'filters/subunit2disk', | ||
| 'filters/subunit2gtk', | ||
| 'filters/subunit2junitxml', | ||
| 'filters/subunit2pyunit', | ||
| 'filters/tap2subunit', | ||
| python_requires=">=3.6", | ||
| install_requires=[ | ||
| 'extras', | ||
| 'testtools>=0.9.34', | ||
| ], | ||
| **extra | ||
| entry_points={ | ||
| 'console_scripts': [ | ||
| 'subunit-1to2=subunit.filter_scripts.subunit_1to2:main', | ||
| 'subunit-2to1=subunit.filter_scripts.subunit_2to1:main', | ||
| 'subunit-filter=subunit.filter_scripts.subunit_filter:main', | ||
| 'subunit-ls=subunit.filter_scripts.subunit_ls:main', | ||
| 'subunit-notify=subunit.filter_scripts.subunit_notify:main', | ||
| 'subunit-output=subunit.filter_scripts.subunit_output:main', | ||
| 'subunit-stats=subunit.filter_scripts.subunit_stats:main', | ||
| 'subunit-tags=subunit.filter_scripts.subunit_tags:main', | ||
| 'subunit2csv=subunit.filter_scripts.subunit2csv:main', | ||
| 'subunit2disk=subunit.filter_scripts.subunit2disk:main', | ||
| 'subunit2gtk=subunit.filter_scripts.subunit2gtk:main', | ||
| 'subunit2junitxml=subunit.filter_scripts.subunit2junitxml:main', | ||
| 'subunit2pyunit=subunit.filter_scripts.subunit2pyunit:main', | ||
| 'tap2subunit=subunit.filter_scripts.tap2subunit:main', | ||
| ] | ||
| }, | ||
| tests_require=[ | ||
| 'fixtures', | ||
| 'hypothesis', | ||
| 'testscenarios', | ||
| ], | ||
| extras_require={ | ||
| 'docs': ['docutils'], | ||
| 'test': ['fixtures', 'testscenarios', 'hypothesis'], | ||
| }, | ||
| ) |
-32
| sudo: false | ||
| addons: | ||
| apt: | ||
| packages: | ||
| - check | ||
| - libcppunit-dev | ||
| language: python | ||
| python: | ||
| - "2.7" | ||
| - "3.4" | ||
| - "3.5" | ||
| - "3.6" | ||
| - "3.7" | ||
| - "3.8" | ||
| - pypy | ||
| - pypy3.5 | ||
| matrix: | ||
| include: | ||
| # Travis nightly look to be 3.5.0a4, b3 is out and the error we see | ||
| # doesn't happen in trunk. | ||
| # - python: "nightly" | ||
| install: | ||
| - pip install -U pip | ||
| - pip install -U wheel setuptools | ||
| - pip install -U .[test,docs] | ||
| - pip list | ||
| - python --version | ||
| - autoreconf -fi && ./configure && make | ||
| script: | ||
| - make check | ||
| - make distcheck | ||
| - rst2html.py README.rst README.html |
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Alert delta unavailable
Currently unable to show alert delta for PyPI packages.
Improved metrics
- Number of package files
66
4.76%- Lines of code
8134
10.2%Worsened metrics
- Total package byte prevSize
467648
-0.91%