pytest-memray - pypi Package Compare versions

Comparing version 1.5.0 to 1.6.0

docs/configuration.rst

@@ -32,2 +32,5 @@ Configuration
     Record allocations made by the Pymalloc allocator (will be slower)
+  
+  ``--fail-on-increase``
+    Fail a test with the limit_memory marker if it uses more memory than its last successful run
 
@@ -53,1 +56,4 @@ .. tab:: Config file options
     Record allocations made by the Pymalloc allocator (will be slower)
+
+  ``--fail-on-increase(bool)``
+    Fail a test with the limit_memory marker if it uses more memory than its last successful run

docs/news.rst

@@ -8,2 +8,34 @@ Release History
 
+v1.6.0 (2024-04-18)
+-------------------
+
+Features - 1.6.0
+~~~~~~~~~~~~~~~~
+- Add a new --fail-on-increase option that fails a test with the ``limit_memory`` marker if it uses more memory than its last successful run. (:issue:`91`)
+- Use aggregated capture files, reducing the amount of temporary disk space required in order to run tests. (:issue:`107`)
+- Add a new ``current_thread_only`` keyword argument to the ``limit_memory`` and
+  ``limit_leaks`` markers to ignore all allocations made in threads other than
+  the one running the test. (:issue:`117`)
+
+Bug Fixes - 1.6.0
+~~~~~~~~~~~~~~~~~
+- Fix the generation of histograms when the tests performed zero-byte allocations. (:issue:`113`)
+
+v1.5.0 (2023-08-23)
+-------------------
+
+Features - 1.5.0
+~~~~~~~~~~~~~~~~
+
+- Add a new ``limit_leaks`` marker to check for memory leaks in tests. (:issue:`45`)
+- Support passing ``--trace-python-allocators`` to memray to track all Python allocations. (:issue:`78` and :issue:`64`)
+
+v1.4.1 (2023-06-06)
+-------------------
+
+Bug Fixes - 1.4.1
+~~~~~~~~~~~~~~~~~
+
+- Fix long test names with xdis (:issue:`68`)
+
 v1.4.0 (2022-12-02)
@@ -10,0 +42,0 @@ -------------------

docs/usage.rst

@@ -38,9 +38,10 @@ Usage
 
-.. py:function:: pytest.mark.limit_memory(memory_limit: str)
+.. py:function:: pytest.mark.limit_memory(memory_limit: str, current_thread_only: bool = False)
 
-    Fail the execution of the test if the test allocates more memory than allowed.
+    Fail the execution of the test if the test allocates more peak memory than allowed.
 
     When this marker is applied to a test, it will cause the test to fail if the
-    execution of the test allocates more memory than allowed. It takes a single argument
-    with a string indicating the maximum memory that the test can allocate.
+    execution of the test allocates more memory (at the peak/high watermark) than allowed.
+    It takes a single argument with a string indicating the maximum memory that the test
+    can allocate.
 
@@ -50,2 +51,6 @@ The format for the string is ``<NUMBER> ([KMGTP]B|B)``. The marker will raise
 
+    If the optional keyword-only argument ``current_thread_only`` is set to *True*, the
+    plugin will only track memory allocations made by the current thread and all other
+    allocations will be ignored.
+
     .. warning::
@@ -67,3 +72,3 @@
 
-.. py:function:: pytest.mark.limit_leaks(location_limit: str, filter_fn: LeaksFilterFunction | None = None)
+.. py:function:: pytest.mark.limit_leaks(location_limit: str, filter_fn: LeaksFilterFunction | None = None, current_thread_only: bool = False)
 
@@ -101,2 +106,6 @@ Fail the execution of the test if any call stack in the test leaks more memory than
 
+    If the optional keyword-only argument ``current_thread_only`` is set to *True*, the
+    plugin will only track memory allocations made by the current thread and all other
+    allocations will be ignored.
+
     .. tip::
@@ -103,0 +112,0 @@

PKG-INFO

@@ -1,4 +0,4 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: pytest-memray
-Version: 1.5.0
+Version: 1.6.0
 Summary: A simple plugin to use with pytest
@@ -22,3 +22,3 @@ Project-URL: Bug Tracker, https://github.com/bloomberg/pytest-memray/issues
 Requires-Python: >=3.8
-Requires-Dist: memray>=1.5
+Requires-Dist: memray>=1.12
 Requires-Dist: pytest>=7.2
@@ -144,2 +144,4 @@ Provides-Extra: docs
 - `--trace-python-allocators` - Record allocations made by the Pymalloc allocator (will be slower)
+- `--fail-on-increase` - Fail a test with the `limit_memory`` marker if it uses
+  more memory than its last successful run
 
@@ -153,3 +155,5 @@ ## Configuration - INI
 - `native(bool)`- Show native frames when showing tracebacks of memory allocations (will be slower)
-- `trace_python_allocators` - Record allocations made by the Pymalloc allocator (will be slower)
+- `trace_python_allocators(bool)` - Record allocations made by the Pymalloc allocator (will be slower)
+- `fail-on-increase(bool)` - Fail a test with the `limit_memory` marker if it
+  uses more memory than its last successful run
 
@@ -156,0 +160,0 @@ ## License

pyproject.toml

@@ -23,3 +23,3 @@ [build-system]
   "pytest>=7.2",
-  "memray>=1.5",
+  "memray>=1.12",
 ]
@@ -26,0 +26,0 @@ optional-dependencies.docs = [

README.md

@@ -100,2 +100,4 @@ <img src="https://raw.githubusercontent.com/bloomberg/pytest-memray/main/docs/_static/images/logo.png" width="70%" style="display: block; margin: 0 auto"  alt="logo"/>
 - `--trace-python-allocators` - Record allocations made by the Pymalloc allocator (will be slower)
+- `--fail-on-increase` - Fail a test with the `limit_memory`` marker if it uses
+  more memory than its last successful run
 
@@ -109,3 +111,5 @@ ## Configuration - INI
 - `native(bool)`- Show native frames when showing tracebacks of memory allocations (will be slower)
-- `trace_python_allocators` - Record allocations made by the Pymalloc allocator (will be slower)
+- `trace_python_allocators(bool)` - Record allocations made by the Pymalloc allocator (will be slower)
+- `fail-on-increase(bool)` - Fail a test with the `limit_memory` marker if it
+  uses more memory than its last successful run
 
@@ -112,0 +116,0 @@ ## License

src/pytest_memray/_version.py

@@ -1,1 +0,1 @@
-__version__ = "1.5.0"
+__version__ = "1.6.0"

src/pytest_memray/marks.py

@@ -130,2 +130,24 @@ from __future__ import annotations
 
+@dataclass
+class _MoreMemoryInfo:
+    previous_memory: float
+    new_memory: float
+
+    @property
+    def section(self) -> PytestSection:
+        """Return a tuple in the format expected by section reporters."""
+        return (
+            "memray-max-memory",
+            "Test uses more memory than previous run",
+        )
+
+    @property
+    def long_repr(self) -> str:
+        """Generate a longrepr user-facing error message."""
+        return (
+            f"Test previously used {sizeof_fmt(self.previous_memory)} "
+            f"but now uses {sizeof_fmt(self.new_memory)}"
+        )
+
+
 def _generate_section_text(
@@ -166,11 +188,31 @@ allocations: list[AllocationRecord], native_stacks: bool, num_stacks: int
 def limit_memory(
-    limit: str, *, _result_file: Path, _config: Config
-) -> _MemoryInfo | None:
+    limit: str,
+    *,
+    current_thread_only: bool = False,
+    _result_file: Path,
+    _config: Config,
+    _test_id: str,
+) -> _MemoryInfo | _MoreMemoryInfo | None:
     """Limit memory used by the test."""
     reader = FileReader(_result_file)
-    allocations: list[AllocationRecord] = list(
-        reader.get_high_watermark_allocation_records(merge_threads=True)
-    )
+    allocations: list[AllocationRecord] = [
+        record
+        for record in reader.get_high_watermark_allocation_records(
+            merge_threads=not current_thread_only
+        )
+        if not current_thread_only or record.tid == reader.metadata.main_thread_id
+    ]
     max_memory = parse_memory_string(limit)
     total_allocated_memory = sum(record.size for record in allocations)
+
+    if _config.cache is not None:
+        cache = _config.cache.get(f"memray/{_test_id}", {})
+        previous = cache.get("total_allocated_memory", float("inf"))
+        fail_on_increase = cast(bool, value_or_ini(_config, "fail_on_increase"))
+        if fail_on_increase and total_allocated_memory > previous:
+            return _MoreMemoryInfo(previous, total_allocated_memory)
+
+        cache["total_allocated_memory"] = total_allocated_memory
+        _config.cache.set(f"memray/{_test_id}", cache)
+
     if total_allocated_memory < max_memory:
@@ -193,9 +235,15 @@ return None
     filter_fn: Optional[LeaksFilterFunction] = None,
+    current_thread_only: bool = False,
     _result_file: Path,
     _config: Config,
+    _test_id: str,
 ) -> _LeakedInfo | None:
     reader = FileReader(_result_file)
-    allocations: list[AllocationRecord] = list(
-        reader.get_leaked_allocation_records(merge_threads=True)
-    )
+    allocations: list[AllocationRecord] = [
+        record
+        for record in reader.get_leaked_allocation_records(
+            merge_threads=not current_thread_only
+        )
+        if not current_thread_only or record.tid == reader.metadata.main_thread_id
+    ]
 
@@ -202,0 +250,0 @@ memory_limit = parse_memory_string(location_limit)

src/pytest_memray/plugin.py

@@ -24,2 +24,3 @@ from __future__ import annotations
 from memray import AllocationRecord
+from memray import FileFormat
 from memray import FileReader
@@ -56,2 +57,3 @@ from memray import Metadata
         _config: Config,
+        _test_id: str,
         **kwargs: Any,
@@ -81,3 +83,3 @@ ) -> SectionMetadata | None:
     """
-    step = ((high - low) / bins) or low
+    step = ((high - low) / bins) or low or 1
     dist = collections.Counter((x - low) // step for x in iterable)
@@ -89,11 +91,9 @@ return [dist[b] for b in range(bins)]
     bars = " ▁▂▃▄▅▆▇█"
+    if log_scale:
+        data = [math.log(number if number else 1) for number in data]
     low = min(data)
     high = max(data)
-    if log_scale:
-        data = map(math.log, filter(lambda number: number != 0, data))
-        low = math.log(low)
-        high = math.log(high)
     data_bins = histogram(data, low=low, high=high, bins=bins)
     bar_indexes = (int(elem * (len(bars) - 1) / max(data_bins)) for elem in data_bins)
-    result = " ".join(bars[bar_index] for bar_index in bar_indexes)
+    result = "".join(bars[bar_index] for bar_index in bar_indexes)
     return result
@@ -140,3 +140,3 @@
 
-    @hookimpl(hookwrapper=True)  # type: ignore[misc] # Untyped decorator
+    @hookimpl(hookwrapper=True)
     def pytest_unconfigure(self, config: Config) -> Generator[None, None, None]:
@@ -149,3 +149,3 @@ yield
 
-    @hookimpl(hookwrapper=True)  # type: ignore[misc] # Untyped decorator
+    @hookimpl(hookwrapper=True)
     def pytest_pyfunc_call(self, pyfuncitem: Function) -> object | None:
@@ -196,2 +196,3 @@ func = pyfuncitem.obj
                     trace_python_allocators=trace_python_allocators,
+                    file_format=FileFormat.AGGREGATED_ALLOCATIONS,
                 ):
@@ -222,3 +223,3 @@ test_result = func(*args, **kwargs)
 
-    @hookimpl(hookwrapper=True)  # type: ignore[misc] # Untyped decorator
+    @hookimpl(hookwrapper=True)
     def pytest_runtest_makereport(
@@ -248,2 +249,3 @@ self, item: Item, call: CallInfo[None]
                 _config=self.config,
+                _test_id=item.nodeid,
             )
@@ -257,3 +259,3 @@ if res:
 
-    @hookimpl(hookwrapper=True, trylast=True)  # type: ignore[misc] # Untyped decorator
+    @hookimpl(hookwrapper=True, trylast=True)
     def pytest_report_teststatus(
@@ -270,3 +272,3 @@ self, report: CollectReport | TestReport
 
-    @hookimpl  # type: ignore[misc] # Untyped decorator
+    @hookimpl
     def pytest_terminal_summary(
@@ -301,2 +303,4 @@ self, terminalreporter: TerminalReporter, exitstatus: ExitCode
         max_results = cast(int, value_or_ini(self.config, "most_allocations"))
+        if max_results == 0:
+            max_results = len(total_sizes)
 
@@ -397,2 +401,8 @@ for test_id, total_size in total_sizes.most_common(max_results):
     )
+    group.addoption(
+        "--fail-on-increase",
+        action="store_true",
+        default=False,
+        help="Fail a test with the limit_memory marker if it uses more memory than its last successful run",
+    )
 
@@ -421,2 +431,7 @@ parser.addini("memray", "Activate pytest.ini setting", type="bool")
     )
+    parser.addini(
+        "fail-on-increase",
+        help="Fail a test with the limit_memory marker if it uses more memory than its last successful run",
+        type="bool",
+    )
     help_msg = "Show the N tests that allocate most memory (N=0 for all)"
@@ -423,0 +438,0 @@ parser.addini("most_allocations", help_msg)

src/pytest_memray/utils.py

@@ -51,3 +51,3 @@ from __future__ import annotations
     value = config.getvalue(key)
-    if value:
+    if value is not None:
         return value
@@ -54,0 +54,0 @@ try:

tests/test_pytest_memray.py

 from __future__ import annotations
 
+import re
 import xml.etree.ElementTree as ET
@@ -9,2 +10,3 @@ from types import SimpleNamespace
 import pytest
+from memray import FileFormat
 from memray import Tracker
@@ -14,3 +16,22 @@ from pytest import ExitCode
 
+from pytest_memray.marks import StackFrame
 
+
+def extract_stacks(test_output: str) -> list[list[StackFrame]]:
+    ret: list[list[StackFrame]] = []
+    before_start = True
+    for line in test_output.splitlines():
+        if before_start:
+            if "List of allocations:" in line:
+                before_start = False
+        elif "allocated here" in line:
+            ret.append([])
+        elif (match := re.match(r"^ {8}([^:]+):(.*):(\d+)$", line)) is not None:
+            ret[-1].append(
+                StackFrame(function=match[1], filename=match[2], lineno=int(match[3]))
+            )
+
+    return ret
+
+
 def test_help_message(pytester: Pytester) -> None:
@@ -181,8 +202,9 @@ result = pytester.runpytest("--help")
 
-    output = result.stdout.str()
+    stacks = extract_stacks(result.stdout.str())
+    valloc_stacks = [stack for stack in stacks if stack[0].function == "valloc"]
+    (valloc_stack,) = valloc_stacks
+    num_rec_frames = sum(1 for frame in valloc_stack if frame.function == "rec")
+    assert num_rec_frames == min(num_stacks - 1, 10)
 
-    assert "valloc:" in output
-    assert output.count("rec:") == min(num_stacks - 1, 10)
 
-
 @pytest.mark.parametrize("native", [True, False])
@@ -210,3 +232,6 @@ def test_memray_report_native(native: bool, pytester: Pytester) -> None:
     mock.assert_called_once_with(
-        ANY, native_traces=native, trace_python_allocators=False
+        ANY,
+        native_traces=native,
+        trace_python_allocators=False,
+        file_format=FileFormat.AGGREGATED_ALLOCATIONS,
     )
@@ -254,3 +279,6 @@
     mock.assert_called_once_with(
-        ANY, native_traces=False, trace_python_allocators=trace_python_allocators
+        ANY,
+        native_traces=False,
+        trace_python_allocators=trace_python_allocators,
+        file_format=FileFormat.AGGREGATED_ALLOCATIONS,
     )
@@ -356,7 +384,7 @@
         def allocating_func1():
-            allocator.valloc(1024)
+            allocator.valloc(1024*1024)
             allocator.free()
 
         def allocating_func2():
-            allocator.valloc(1024*2)
+            allocator.valloc(1024*1024*2)
             allocator.free()
@@ -382,2 +410,35 @@
 
+def test_memray_report_limit_without_limit(pytester: Pytester) -> None:
+    pytester.makepyfile(
+        """
+        import pytest
+        from memray._test import MemoryAllocator
+        allocator = MemoryAllocator()
+
+        def allocating_func1():
+            allocator.valloc(1024)
+            allocator.free()
+
+        def allocating_func2():
+            allocator.valloc(1024*2)
+            allocator.free()
+
+        def test_foo():
+            allocating_func1()
+
+        def test_bar():
+            allocating_func2()
+    """
+    )
+
+    result = pytester.runpytest("--memray", "--most-allocations=0")
+
+    assert result.ret == ExitCode.OK
+
+    output = result.stdout.str()
+
+    assert "results for test_memray_report_limit_without_limit.py::test_foo" in output
+    assert "results for test_memray_report_limit_without_limit.py::test_bar" in output
+
+
 def test_failing_tests_are_not_reported(pytester: Pytester) -> None:
@@ -609,2 +670,3 @@ pytester.makepyfile(
     [
+        (0, ExitCode.OK),
         (1, ExitCode.OK),
@@ -638,6 +700,5 @@ (1024 * 1 / 10, ExitCode.OK),
     [
-        (1, ExitCode.OK),
-        (1024 * 1 / 10, ExitCode.OK),
-        (1024 * 1, ExitCode.TESTS_FAILED),
-        (1024 * 10, ExitCode.TESTS_FAILED),
+        (4 * 1024, ExitCode.OK),
+        (0.4 * 1024 * 1024, ExitCode.OK),
+        (4 * 1024 * 1024, ExitCode.TESTS_FAILED),
     ],
@@ -658,3 +719,3 @@ )
                 # No free call here
-        @pytest.mark.limit_leaks("5KB")
+        @pytest.mark.limit_leaks("20MB")
         def test_memory_alloc_fails():
@@ -755,1 +816,116 @@ t = threading.Thread(target=allocating_func)
     assert "Only one Memray marker can be applied to each test" in output
+
+
+def test_fail_on_increase(pytester: Pytester):
+    pytester.makepyfile(
+        """
+        import pytest
+        from memray._test import MemoryAllocator
+        allocator = MemoryAllocator()
+
+        @pytest.mark.limit_memory("100MB")
+        def test_memory_alloc_fails():
+            allocator.valloc(1024)
+            allocator.free()
+        """
+    )
+    result = pytester.runpytest("--memray")
+    assert result.ret == ExitCode.OK
+    pytester.makepyfile(
+        """
+        import pytest
+        from memray._test import MemoryAllocator
+        allocator = MemoryAllocator()
+
+        @pytest.mark.limit_memory("100MB")
+        def test_memory_alloc_fails():
+            allocator.valloc(1024 * 10)
+            allocator.free()
+        """
+    )
+    result = pytester.runpytest("--memray", "--fail-on-increase")
+    assert result.ret == ExitCode.TESTS_FAILED
+    output = result.stdout.str()
+    assert "Test uses more memory than previous run" in output
+    assert "Test previously used 1.0KiB but now uses 10.0KiB" in output
+
+
+def test_fail_on_increase_unset(pytester: Pytester):
+    pytester.makepyfile(
+        """
+        import pytest
+        from memray._test import MemoryAllocator
+        allocator = MemoryAllocator()
+
+        @pytest.mark.limit_memory("100MB")
+        def test_memory_alloc_fails():
+            allocator.valloc(1024)
+            allocator.free()
+        """
+    )
+    result = pytester.runpytest("--memray")
+    assert result.ret == ExitCode.OK
+    pytester.makepyfile(
+        """
+        import pytest
+        from memray._test import MemoryAllocator
+        allocator = MemoryAllocator()
+
+        @pytest.mark.limit_memory("100MB")
+        def test_memory_alloc_fails():
+            allocator.valloc(1024 * 10)
+            allocator.free()
+        """
+    )
+    result = pytester.runpytest("--memray")
+    assert result.ret == ExitCode.OK
+
+
+def test_limit_memory_in_current_thread(pytester: Pytester) -> None:
+    pytester.makepyfile(
+        """
+        import pytest
+        from memray._test import MemoryAllocator
+        allocator = MemoryAllocator()
+        import threading
+        def allocating_func():
+            for _ in range(10):
+                allocator.valloc(1024*5)
+                # No free call here
+
+        @pytest.mark.limit_memory("5KB", current_thread_only=True)
+        def test_memory_alloc_fails():
+            t = threading.Thread(target=allocating_func)
+            t.start()
+            t.join()
+        """
+    )
+
+    result = pytester.runpytest("--memray")
+
+    assert result.ret == ExitCode.OK
+
+
+def test_leaks_in_current_thread(pytester: Pytester) -> None:
+    pytester.makepyfile(
+        """
+        import pytest
+        from memray._test import MemoryAllocator
+        allocator = MemoryAllocator()
+        import threading
+        def allocating_func():
+            for _ in range(10):
+                allocator.valloc(1024*5)
+                # No free call here
+
+        @pytest.mark.limit_leaks("5KB", current_thread_only=True)
+        def test_memory_alloc_fails():
+            t = threading.Thread(target=allocating_func)
+            t.start()
+            t.join()
+        """
+    )
+
+    result = pytester.runpytest("--memray")
+
+    assert result.ret == ExitCode.OK

tests/test_utils.py

@@ -18,2 +18,3 @@ from __future__ import annotations
 from pytest_memray.utils import parse_memory_string
+from pytest_memray.plugin import cli_hist
 
@@ -127,1 +128,23 @@
         tmp_path.chmod(tmp_path.stat().st_mode | write)
+
+
+def test_histogram_with_zero_byte_allocations():
+    # GIVEN
+    allocations = [0, 100, 990, 1000, 50000]
+
+    # WHEN
+    histogram = cli_hist(allocations, bins=5)
+
+    # THEN
+    assert histogram == "▄ ▄█▄"
+
+
+def test_histogram_with_only_zero_byte_allocations():
+    # GIVEN
+    allocations = [0, 0, 0, 0]
+
+    # WHEN
+    histogram = cli_hist(allocations, bins=5)
+
+    # THEN
+    assert histogram == "█    "

tox.ini

 [tox]
 envlist =
-    py310-cov
+    py312-cov
+    py312
+    py311
     py310
@@ -17,4 +19,4 @@ py39
     CI
-    PYTEST_
-    VIRTUALENV_
+    PYTEST_*
+    VIRTUALENV_*
 setenv =
@@ -33,3 +35,3 @@ COVERAGE_FILE = {toxworkdir}/.coverage.{envname}
 
-[testenv:py310-cov]
+[testenv:py312-cov]
 commands =
@@ -36,0 +38,0 @@ make coverage

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

364518

2.79%

Lines of code

1611

15.48%