D-MemFS
Advanced tools
| """Physical memory guard strategies for D-MemFS.""" | ||
| from __future__ import annotations | ||
| import threading | ||
| import time | ||
| import warnings | ||
| from abc import ABC, abstractmethod | ||
| from ._memory_info import get_available_memory_bytes | ||
| class MemoryGuard(ABC): | ||
| def __init__(self, action: str = "warn") -> None: | ||
| if action not in ("warn", "raise"): | ||
| raise ValueError( | ||
| f"Invalid memory_guard_action: {action!r}. Expected 'warn' or 'raise'." | ||
| ) | ||
| self._action = action | ||
| @abstractmethod | ||
| def check_init(self, max_quota: int) -> None: ... | ||
| @abstractmethod | ||
| def check_before_write(self, size: int) -> None: ... | ||
| def _handle_violation(self, message: str) -> None: | ||
| if self._action == "raise": | ||
| raise MemoryError(message) | ||
| warnings.warn(message, ResourceWarning, stacklevel=4) | ||
| class NullGuard(MemoryGuard): | ||
| def __init__(self) -> None: | ||
| super().__init__(action="warn") | ||
| def check_init(self, max_quota: int) -> None: | ||
| return None | ||
| def check_before_write(self, size: int) -> None: | ||
| return None | ||
| class InitGuard(MemoryGuard): | ||
| def check_init(self, max_quota: int) -> None: | ||
| avail = get_available_memory_bytes() | ||
| if avail is not None and max_quota > avail: | ||
| self._handle_violation( | ||
| f"max_quota ({max_quota:,} bytes) exceeds available physical RAM " | ||
| f"({avail:,} bytes). MemoryError may occur before quota limit is reached." | ||
| ) | ||
| def check_before_write(self, size: int) -> None: | ||
| return None | ||
| class PerWriteGuard(MemoryGuard): | ||
| def __init__(self, action: str = "warn", interval: float = 1.0) -> None: | ||
| super().__init__(action=action) | ||
| self._interval = interval | ||
| self._last_check = 0.0 | ||
| self._last_avail: int | None = None | ||
| self._lock = threading.Lock() | ||
| def check_init(self, max_quota: int) -> None: | ||
| avail = get_available_memory_bytes() | ||
| if avail is not None and max_quota > avail: | ||
| self._handle_violation( | ||
| f"max_quota ({max_quota:,} bytes) exceeds available physical RAM " | ||
| f"({avail:,} bytes). MemoryError may occur before quota limit is reached." | ||
| ) | ||
| with self._lock: | ||
| self._last_avail = avail | ||
| self._last_check = time.monotonic() | ||
| def check_before_write(self, size: int) -> None: | ||
| now = time.monotonic() | ||
| with self._lock: | ||
| if now - self._last_check >= self._interval: | ||
| self._last_avail = get_available_memory_bytes() | ||
| self._last_check = now | ||
| avail = self._last_avail | ||
| age = now - self._last_check | ||
| if avail is not None and size > avail: | ||
| self._handle_violation( | ||
| f"Write of {size:,} bytes requested but only {avail:,} bytes of " | ||
| f"physical RAM available (checked {age:.1f}s ago)." | ||
| ) | ||
| def create_memory_guard( | ||
| mode: str = "none", | ||
| action: str = "warn", | ||
| interval: float = 1.0, | ||
| ) -> MemoryGuard: | ||
| if action not in ("warn", "raise"): | ||
| raise ValueError(f"Invalid memory_guard_action: {action!r}. Expected 'warn' or 'raise'.") | ||
| if mode == "none": | ||
| return NullGuard() | ||
| if mode == "init": | ||
| return InitGuard(action=action) | ||
| if mode == "per_write": | ||
| return PerWriteGuard(action=action, interval=interval) | ||
| raise ValueError(f"Invalid memory_guard: {mode!r}. Expected 'none', 'init', or 'per_write'.") |
| """OS-specific available physical memory detection. | ||
| Internal module - not part of the public API. | ||
| Uses only Python standard library. | ||
| """ | ||
| from __future__ import annotations | ||
| import ctypes | ||
| import ctypes.util | ||
| import platform | ||
| from collections.abc import Callable | ||
| from typing import cast | ||
| _SYSTEM = platform.system() | ||
| _UNPROBED = object() | ||
| _linux_reader: Callable[[], int | None] | None | object = _UNPROBED | ||
| def get_available_memory_bytes() -> int | None: | ||
| """Return available physical memory in bytes, or None if unavailable.""" | ||
| try: | ||
| if _SYSTEM == "Windows": | ||
| return _windows_avail() | ||
| if _SYSTEM == "Linux": | ||
| return _linux_avail() | ||
| if _SYSTEM == "Darwin": | ||
| return _macos_avail() | ||
| except Exception: | ||
| return None | ||
| return None | ||
| def _windows_avail() -> int: | ||
| class MEMORYSTATUSEX(ctypes.Structure): | ||
| _fields_ = [ | ||
| ("dwLength", ctypes.c_ulong), | ||
| ("dwMemoryLoad", ctypes.c_ulong), | ||
| ("ullTotalPhys", ctypes.c_ulonglong), | ||
| ("ullAvailPhys", ctypes.c_ulonglong), | ||
| ("ullTotalPageFile", ctypes.c_ulonglong), | ||
| ("ullAvailPageFile", ctypes.c_ulonglong), | ||
| ("ullTotalVirtual", ctypes.c_ulonglong), | ||
| ("ullAvailVirtual", ctypes.c_ulonglong), | ||
| ("ullAvailExtendedVirtual", ctypes.c_ulonglong), | ||
| ] | ||
| stat = MEMORYSTATUSEX() | ||
| stat.dwLength = ctypes.sizeof(stat) | ||
| result = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) | ||
| if not result: | ||
| raise OSError("GlobalMemoryStatusEx failed") | ||
| return int(stat.ullAvailPhys) | ||
| def _linux_avail() -> int | None: | ||
| global _linux_reader | ||
| if _linux_reader is _UNPROBED: | ||
| _linux_reader = _probe_linux_source() | ||
| if _linux_reader is None: | ||
| return None | ||
| return cast(Callable[[], int | None], _linux_reader)() | ||
| def _probe_linux_source() -> Callable[[], int | None] | None: | ||
| if _is_cgroup_v2_limited(): | ||
| return _read_cgroup_v2 | ||
| if _is_cgroup_v1_limited(): | ||
| return _read_cgroup_v1 | ||
| if _is_procmeminfo_available(): | ||
| return _read_procmeminfo | ||
| return None | ||
| def _is_cgroup_v2_limited() -> bool: | ||
| try: | ||
| with open("/sys/fs/cgroup/memory.max", encoding="utf-8") as handle: | ||
| return handle.read().strip() != "max" | ||
| except (FileNotFoundError, PermissionError): | ||
| return False | ||
| def _is_cgroup_v1_limited() -> bool: | ||
| try: | ||
| with open("/sys/fs/cgroup/memory/memory.limit_in_bytes", encoding="utf-8") as handle: | ||
| limit = int(handle.read().strip()) | ||
| host_total = _read_procmeminfo_total() | ||
| if host_total is not None and limit >= host_total: | ||
| return False | ||
| return True | ||
| except (FileNotFoundError, PermissionError, ValueError): | ||
| return False | ||
| def _is_procmeminfo_available() -> bool: | ||
| try: | ||
| with open("/proc/meminfo", encoding="utf-8") as handle: | ||
| handle.readline() | ||
| return True | ||
| except (FileNotFoundError, PermissionError): | ||
| return False | ||
| def _read_cgroup_v2() -> int | None: | ||
| with open("/sys/fs/cgroup/memory.max", encoding="utf-8") as handle: | ||
| value = handle.read().strip() | ||
| if value == "max": | ||
| return None | ||
| max_mem = int(value) | ||
| with open("/sys/fs/cgroup/memory.current", encoding="utf-8") as handle: | ||
| current = int(handle.read().strip()) | ||
| return max(0, max_mem - current) | ||
| def _read_cgroup_v1() -> int: | ||
| with open("/sys/fs/cgroup/memory/memory.limit_in_bytes", encoding="utf-8") as handle: | ||
| limit = int(handle.read().strip()) | ||
| with open("/sys/fs/cgroup/memory/memory.usage_in_bytes", encoding="utf-8") as handle: | ||
| usage = int(handle.read().strip()) | ||
| return max(0, limit - usage) | ||
| def _read_procmeminfo_total() -> int | None: | ||
| try: | ||
| with open("/proc/meminfo", encoding="utf-8") as handle: | ||
| for line in handle: | ||
| if line.startswith("MemTotal:"): | ||
| return int(line.split()[1]) * 1024 | ||
| except Exception: | ||
| return None | ||
| return None | ||
| def _read_procmeminfo() -> int: | ||
| with open("/proc/meminfo", encoding="utf-8") as handle: | ||
| content = handle.read() | ||
| mem_data: dict[str, int] = {} | ||
| for line in content.splitlines(): | ||
| parts = line.split() | ||
| if len(parts) >= 2: | ||
| mem_data[parts[0].rstrip(":")] = int(parts[1]) | ||
| free_kb = mem_data.get("MemAvailable", mem_data.get("MemFree", 0)) | ||
| return free_kb * 1024 | ||
| def _read_proc_meminfo() -> int: | ||
| return _read_procmeminfo() | ||
| def _macos_avail() -> int: | ||
| libc_path = ctypes.util.find_library("c") or "/usr/lib/libSystem.B.dylib" | ||
| libc = ctypes.CDLL(libc_path) | ||
| host_vm_info64 = 4 | ||
| host_vm_info64_count = 48 | ||
| class VMStatistics64(ctypes.Structure): | ||
| _fields_ = [ | ||
| ("free_count", ctypes.c_uint64), | ||
| ("active_count", ctypes.c_uint64), | ||
| ("inactive_count", ctypes.c_uint64), | ||
| ("wire_count", ctypes.c_uint64), | ||
| ("zero_fill_count", ctypes.c_uint64), | ||
| ("reactivations", ctypes.c_uint64), | ||
| ("pageins", ctypes.c_uint64), | ||
| ("pageouts", ctypes.c_uint64), | ||
| ("faults", ctypes.c_uint64), | ||
| ("cow_faults", ctypes.c_uint64), | ||
| ("lookups", ctypes.c_uint64), | ||
| ("hits", ctypes.c_uint64), | ||
| ("purges", ctypes.c_uint64), | ||
| ("purgeable_count", ctypes.c_uint64), | ||
| ("speculative_count", ctypes.c_uint64), | ||
| ("decompressions", ctypes.c_uint64), | ||
| ("compressions", ctypes.c_uint64), | ||
| ("swapins", ctypes.c_uint64), | ||
| ("swapouts", ctypes.c_uint64), | ||
| ("compressor_page_count", ctypes.c_uint64), | ||
| ("throttled_count", ctypes.c_uint64), | ||
| ("external_page_count", ctypes.c_uint64), | ||
| ("internal_page_count", ctypes.c_uint64), | ||
| ("total_uncompressed_pages_in_compressor", ctypes.c_uint64), | ||
| ] | ||
| libc.mach_host_self.restype = ctypes.c_uint | ||
| libc.host_page_size.argtypes = [ctypes.c_uint, ctypes.POINTER(ctypes.c_uint)] | ||
| libc.host_page_size.restype = ctypes.c_int | ||
| libc.host_statistics64.argtypes = [ | ||
| ctypes.c_uint, | ||
| ctypes.c_int, | ||
| ctypes.c_void_p, | ||
| ctypes.POINTER(ctypes.c_uint), | ||
| ] | ||
| libc.host_statistics64.restype = ctypes.c_int | ||
| host = libc.mach_host_self() | ||
| page_size = ctypes.c_uint() | ||
| if libc.host_page_size(host, ctypes.byref(page_size)) != 0: | ||
| raise OSError("host_page_size failed") | ||
| vm_stat = VMStatistics64() | ||
| count = ctypes.c_uint(host_vm_info64_count) | ||
| ret = libc.host_statistics64( | ||
| host, | ||
| host_vm_info64, | ||
| ctypes.byref(vm_stat), | ||
| ctypes.byref(count), | ||
| ) | ||
| if ret != 0: | ||
| raise OSError(f"host_statistics64 failed with code {ret}") | ||
| free_pages = vm_stat.free_count + vm_stat.speculative_count | ||
| return int(free_pages * page_size.value) |
| Metadata-Version: 2.4 | ||
| Name: D-MemFS | ||
| Version: 0.2.1 | ||
| Version: 0.3.0 | ||
| Summary: In-process virtual filesystem with hard quota for Python | ||
@@ -104,2 +104,6 @@ Author: D | ||
| - `chunk_overhead_override` (default `None`): override the per-chunk overhead estimate used for quota accounting | ||
| - `default_lock_timeout` (default `30.0`): default timeout in seconds for file-lock acquisition during `open()`. Use `None` to wait indefinitely. | ||
| - `memory_guard` (default `"none"`): physical memory protection mode — `"none"` / `"init"` / `"per_write"` | ||
| - `memory_guard_action` (default `"warn"`): action when the guard triggers — `"warn"` (`ResourceWarning`) / `"raise"` (`MemoryError`) | ||
| - `memory_guard_interval` (default `1.0`): minimum seconds between OS memory queries (`"per_write"` only) | ||
@@ -109,7 +113,49 @@ > **Note:** The `BytesIO` returned by `export_as_bytesio()` is outside quota management. | ||
| > **Note — Quota and free-threaded Python:** | ||
| > The per-chunk overhead estimate used for quota accounting is calibrated at import time | ||
| > via `sys.getsizeof()`. Free-threaded Python (3.13t, `PYTHON_GIL=0`) has larger object | ||
| > headers than the standard build, so `CHUNK_OVERHEAD_ESTIMATE` is higher (~117 bytes vs | ||
| > ~93 bytes on CPython 3.13). This means the same `max_quota` yields slightly less | ||
| > effective storage capacity on free-threaded builds, especially for workloads with many | ||
| > small files or small appends. This is not a bug — it reflects real memory consumption. | ||
| > To ensure consistent behaviour across builds, use `chunk_overhead_override` to pin the | ||
| > value, or inspect `stats()["overhead_per_chunk_estimate"]` at runtime. | ||
| Supported binary modes: `rb`, `wb`, `ab`, `r+b`, `xb` | ||
| ## Memory Guard | ||
| MFS enforces a logical quota, but that quota can still be configured larger than the | ||
| currently available physical RAM. `memory_guard` provides an optional safety net. | ||
| ```python | ||
| from dmemfs import MemoryFileSystem | ||
| # Warn if max_quota exceeds available RAM | ||
| mfs = MemoryFileSystem(max_quota=8 * 1024**3, memory_guard="init") | ||
| # Raise MemoryError before writes when RAM is insufficient | ||
| mfs = MemoryFileSystem( | ||
| max_quota=8 * 1024**3, | ||
| memory_guard="per_write", | ||
| memory_guard_action="raise", | ||
| ) | ||
| ``` | ||
| | Mode | Initialization | Each Write | Overhead | | ||
| |---|---|---|---| | ||
| | `"none"` | — | — | Zero | | ||
| | `"init"` | Check once | — | Negligible | | ||
| | `"per_write"` | Check once | Cached check | About 1 OS call/sec | | ||
| When `memory_guard_action="warn"`, the guard emits `ResourceWarning` and allows the operation to continue. | ||
| When `memory_guard_action="raise"`, the guard rejects the operation with `MemoryError` before the actual allocation path. | ||
| `AsyncMemoryFileSystem` accepts the same constructor parameters and forwards them to the synchronous implementation. | ||
| ### `MemoryFileHandle` | ||
| - `io.RawIOBase`-compatible binary handle | ||
| - `read`, `write`, `seek`, `tell`, `truncate`, `flush`, `close` | ||
| - `readinto` | ||
| - file-like capability checks: `readable`, `writable`, `seekable` | ||
@@ -151,3 +197,3 @@ | ||
| `MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`. | ||
| `MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. `read(size)` counts characters, not bytes, so multibyte text can be read safely without splitting code points. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`. | ||
@@ -389,4 +435,5 @@ --- | ||
| - [Architecture Spec v13](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v13.md) — API design, internal structure, CI matrix | ||
| - [Detailed Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec.md) — component-level design and rationale | ||
| - [Test Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test.md) — test case table and pseudocode | ||
| - [Architecture Spec v14](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v14.md) — MemoryGuard-integrated architecture spec | ||
| - [Detailed Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_v2.md) — component-level design and rationale | ||
| - [Test Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test_v2.md) — test case table and pseudocode | ||
@@ -393,0 +440,0 @@ > These documents are written in Japanese and serve as internal design references. |
@@ -15,2 +15,4 @@ LICENSE | ||
| dmemfs/_lock.py | ||
| dmemfs/_memory_guard.py | ||
| dmemfs/_memory_info.py | ||
| dmemfs/_path.py | ||
@@ -17,0 +19,0 @@ dmemfs/_pytest_plugin.py |
@@ -34,2 +34,2 @@ from typing import TYPE_CHECKING | ||
| ] | ||
| __version__ = "0.2.0" | ||
| __version__ = "0.3.0" |
+11
-9
@@ -73,2 +73,6 @@ """Async wrapper around MemoryFileSystem. | ||
| default_storage: str = "auto", | ||
| default_lock_timeout: float | None = 30.0, | ||
| memory_guard: str = "none", | ||
| memory_guard_action: str = "warn", | ||
| memory_guard_interval: float = 1.0, | ||
| ) -> None: | ||
@@ -81,2 +85,6 @@ self._sync = MemoryFileSystem( | ||
| default_storage=default_storage, | ||
| default_lock_timeout=default_lock_timeout, | ||
| memory_guard=memory_guard, | ||
| memory_guard_action=memory_guard_action, | ||
| memory_guard_interval=memory_guard_interval, | ||
| ) | ||
@@ -91,5 +99,3 @@ | ||
| ) -> AsyncMemoryFileHandle: | ||
| h = await asyncio.to_thread( | ||
| self._sync.open, path, mode, preallocate, lock_timeout | ||
| ) | ||
| h = await asyncio.to_thread(self._sync.open, path, mode, preallocate, lock_timeout) | ||
| return AsyncMemoryFileHandle(h) | ||
@@ -133,10 +139,6 @@ | ||
| async def export_as_bytesio( | ||
| self, path: str, max_size: int | None = None | ||
| ) -> io.BytesIO: | ||
| async def export_as_bytesio(self, path: str, max_size: int | None = None) -> io.BytesIO: | ||
| return await asyncio.to_thread(self._sync.export_as_bytesio, path, max_size) | ||
| async def export_tree( | ||
| self, prefix: str = "/", only_dirty: bool = False | ||
| ) -> dict[str, bytes]: | ||
| async def export_tree(self, prefix: str = "/", only_dirty: bool = False) -> dict[str, bytes]: | ||
| return await asyncio.to_thread(self._sync.export_tree, prefix, only_dirty) | ||
@@ -143,0 +145,0 @@ |
+119
-36
| import bisect | ||
| import io | ||
| import sys | ||
| from abc import ABC, abstractmethod | ||
@@ -8,13 +7,13 @@ from typing import TYPE_CHECKING | ||
| if TYPE_CHECKING: | ||
| from ._memory_guard import MemoryGuard | ||
| from ._quota import QuotaManager | ||
| def _calibrate_chunk_overhead(safety_mul: float = 1.5, safety_add: int = 32) -> int: | ||
| empty_bytes_size = sys.getsizeof(b"") | ||
| list_ptr_size = sys.getsizeof([None]) - sys.getsizeof([]) | ||
| raw = empty_bytes_size + list_ptr_size | ||
| return int(raw * safety_mul) + safety_add | ||
| def _wrap_memory_error(message: str) -> MemoryError: | ||
| return MemoryError(message) | ||
| CHUNK_OVERHEAD_ESTIMATE: int = _calibrate_chunk_overhead() | ||
| # Keep quota behavior deterministic across standard and free-threaded builds. | ||
| # A fixed conservative value avoids runtime-dependent quota boundaries. | ||
| CHUNK_OVERHEAD_ESTIMATE: int = 128 | ||
@@ -34,8 +33,16 @@ | ||
| def write_at( | ||
| self, offset: int, data: bytes, quota_mgr: "QuotaManager" | ||
| ) -> "tuple[int, RandomAccessMemoryFile | None, int]": | ||
| ... | ||
| self, | ||
| offset: int, | ||
| data: bytes, | ||
| quota_mgr: "QuotaManager", | ||
| memory_guard: "MemoryGuard | None" = None, | ||
| ) -> "tuple[int, RandomAccessMemoryFile | None, int]": ... | ||
| @abstractmethod | ||
| def truncate(self, size: int, quota_mgr: "QuotaManager") -> None: ... | ||
| def truncate( | ||
| self, | ||
| size: int, | ||
| quota_mgr: "QuotaManager", | ||
| memory_guard: "MemoryGuard | None" = None, | ||
| ) -> None: ... | ||
@@ -56,3 +63,8 @@ @abstractmethod | ||
| def __init__(self, chunk_overhead: int = CHUNK_OVERHEAD_ESTIMATE, promotion_hard_limit: int | None = None, allow_promotion: bool = True) -> None: | ||
| def __init__( | ||
| self, | ||
| chunk_overhead: int = CHUNK_OVERHEAD_ESTIMATE, | ||
| promotion_hard_limit: int | None = None, | ||
| allow_promotion: bool = True, | ||
| ) -> None: | ||
| super().__init__() | ||
@@ -92,3 +104,9 @@ self._chunks: list[bytes] = [] | ||
| def write_at(self, offset: int, data: bytes, quota_mgr: "QuotaManager") -> "tuple[int, RandomAccessMemoryFile | None, int]": | ||
| def write_at( | ||
| self, | ||
| offset: int, | ||
| data: bytes, | ||
| quota_mgr: "QuotaManager", | ||
| memory_guard: "MemoryGuard | None" = None, | ||
| ) -> "tuple[int, RandomAccessMemoryFile | None, int]": | ||
| if offset != self._size: | ||
@@ -100,3 +118,3 @@ if not self._allow_promotion: | ||
| ) | ||
| return self._promote_and_write(offset, data, quota_mgr) | ||
| return self._promote_and_write(offset, data, quota_mgr, memory_guard) | ||
| n = len(data) | ||
@@ -106,9 +124,24 @@ if n == 0: | ||
| overhead = self._chunk_overhead | ||
| if memory_guard is not None: | ||
| memory_guard.check_before_write(n + overhead) | ||
| with quota_mgr.reserve(n + overhead): | ||
| self._chunks.append(data) | ||
| self._size += n | ||
| self._cumulative.append(self._size) | ||
| try: | ||
| self._chunks.append(data) | ||
| self._size += n | ||
| self._cumulative.append(self._size) | ||
| except MemoryError: | ||
| raise _wrap_memory_error( | ||
| f"OS memory allocation failed while writing {n:,} bytes. " | ||
| f"MFS quota had {quota_mgr.free:,} bytes remaining. " | ||
| "The max_quota may exceed available system RAM. " | ||
| "Consider reducing max_quota or using memory_guard='init'." | ||
| ) from None | ||
| return n, None, 0 | ||
| def truncate(self, size: int, quota_mgr: "QuotaManager") -> None: | ||
| def truncate( | ||
| self, | ||
| size: int, | ||
| quota_mgr: "QuotaManager", | ||
| memory_guard: "MemoryGuard | None" = None, | ||
| ) -> None: | ||
| if size == self._size: | ||
@@ -120,6 +153,14 @@ return | ||
| overhead = self._chunk_overhead | ||
| if memory_guard is not None: | ||
| memory_guard.check_before_write(len(pad) + overhead) | ||
| with quota_mgr.reserve(len(pad) + overhead): | ||
| self._chunks.append(pad) | ||
| self._size = size | ||
| self._cumulative.append(size) | ||
| try: | ||
| self._chunks.append(pad) | ||
| self._size = size | ||
| self._cumulative.append(size) | ||
| except MemoryError: | ||
| raise _wrap_memory_error( | ||
| f"OS memory allocation failed while extending file to {size:,} bytes. " | ||
| "Consider reducing max_quota or using memory_guard='init'." | ||
| ) from None | ||
| return | ||
@@ -150,3 +191,9 @@ data = b"".join(self._chunks)[:size] | ||
| def _promote_and_write(self, offset: int, data: bytes, quota_mgr: "QuotaManager") -> "tuple[int, RandomAccessMemoryFile, int]": | ||
| def _promote_and_write( | ||
| self, | ||
| offset: int, | ||
| data: bytes, | ||
| quota_mgr: "QuotaManager", | ||
| memory_guard: "MemoryGuard | None" = None, | ||
| ) -> "tuple[int, RandomAccessMemoryFile, int]": | ||
| # NOTE: During promotion, both the original chunk list and the new | ||
@@ -161,8 +208,16 @@ # bytearray coexist temporarily, consuming ~2x the file size in memory. | ||
| ) | ||
| if memory_guard is not None: | ||
| memory_guard.check_before_write(current_size) | ||
| with quota_mgr.reserve(current_size): | ||
| new_buf = bytearray(b"".join(self._chunks)) | ||
| try: | ||
| new_buf = bytearray(b"".join(self._chunks)) | ||
| except MemoryError: | ||
| raise _wrap_memory_error( | ||
| f"OS memory allocation failed during storage promotion (file size: {current_size:,} bytes). " | ||
| "Consider reducing max_quota or using memory_guard='init'." | ||
| ) from None | ||
| old_overhead = len(self._chunks) * self._chunk_overhead | ||
| quota_mgr.release(old_overhead) | ||
| promoted = RandomAccessMemoryFile.from_bytearray(new_buf) | ||
| written, _, _ = promoted.write_at(offset, data, quota_mgr) | ||
| written, _, _ = promoted.write_at(offset, data, quota_mgr, memory_guard) | ||
| return written, promoted, current_size | ||
@@ -194,5 +249,11 @@ | ||
| return bytes(self._buf[offset:]) | ||
| return bytes(self._buf[offset: offset + size]) | ||
| return bytes(self._buf[offset : offset + size]) | ||
| def write_at(self, offset: int, data: bytes, quota_mgr: "QuotaManager") -> "tuple[int, None, int]": | ||
| def write_at( | ||
| self, | ||
| offset: int, | ||
| data: bytes, | ||
| quota_mgr: "QuotaManager", | ||
| memory_guard: "MemoryGuard | None" = None, | ||
| ) -> "tuple[int, None, int]": | ||
| n = len(data) | ||
@@ -205,15 +266,29 @@ if n == 0: | ||
| if extend > 0: | ||
| if memory_guard is not None: | ||
| memory_guard.check_before_write(extend) | ||
| with quota_mgr.reserve(extend): | ||
| if offset > current_len: | ||
| self._buf.extend(bytes(offset - current_len)) | ||
| self._buf.extend(data) | ||
| else: | ||
| overlap = current_len - offset | ||
| self._buf[offset:current_len] = data[:overlap] | ||
| self._buf.extend(data[overlap:]) | ||
| try: | ||
| if offset > current_len: | ||
| self._buf.extend(bytes(offset - current_len)) | ||
| self._buf.extend(data) | ||
| else: | ||
| overlap = current_len - offset | ||
| self._buf[offset:current_len] = data[:overlap] | ||
| self._buf.extend(data[overlap:]) | ||
| except MemoryError: | ||
| raise _wrap_memory_error( | ||
| f"OS memory allocation failed while writing {n:,} bytes. " | ||
| f"MFS quota had {quota_mgr.free:,} bytes remaining. " | ||
| "Consider reducing max_quota or using memory_guard='init'." | ||
| ) from None | ||
| else: | ||
| self._buf[offset: offset + n] = data | ||
| self._buf[offset : offset + n] = data | ||
| return n, None, 0 | ||
| def truncate(self, size: int, quota_mgr: "QuotaManager") -> None: | ||
| def truncate( | ||
| self, | ||
| size: int, | ||
| quota_mgr: "QuotaManager", | ||
| memory_guard: "MemoryGuard | None" = None, | ||
| ) -> None: | ||
| old_size = len(self._buf) | ||
@@ -225,4 +300,12 @@ if size == old_size: | ||
| extend = size - old_size | ||
| if memory_guard is not None: | ||
| memory_guard.check_before_write(extend) | ||
| with quota_mgr.reserve(extend): | ||
| self._buf.extend(bytes(extend)) | ||
| try: | ||
| self._buf.extend(bytes(extend)) | ||
| except MemoryError: | ||
| raise _wrap_memory_error( | ||
| f"OS memory allocation failed while extending file to {size:,} bytes. " | ||
| "Consider reducing max_quota or using memory_guard='init'." | ||
| ) from None | ||
| return | ||
@@ -229,0 +312,0 @@ release = old_size - size |
+47
-33
@@ -19,2 +19,3 @@ from __future__ import annotations | ||
| from ._lock import ReadWriteLock | ||
| from ._memory_guard import create_memory_guard | ||
| from ._path import normalize_path | ||
@@ -76,2 +77,6 @@ from ._quota import QuotaManager | ||
| default_storage: str = "auto", | ||
| default_lock_timeout: float | None = 30.0, | ||
| memory_guard: str = "none", | ||
| memory_guard_action: str = "warn", | ||
| memory_guard_interval: float = 1.0, | ||
| ) -> None: | ||
@@ -85,2 +90,7 @@ if default_storage not in ("auto", "sequential", "random_access"): | ||
| self._global_lock = threading.RLock() | ||
| self._memory_guard = create_memory_guard( | ||
| mode=memory_guard, | ||
| action=memory_guard_action, | ||
| interval=memory_guard_interval, | ||
| ) | ||
| self._chunk_overhead: int = ( | ||
@@ -94,2 +104,3 @@ chunk_overhead_override | ||
| self._default_storage: str = default_storage | ||
| self._default_lock_timeout: float | None = default_lock_timeout | ||
| self._nodes: dict[int, Node] = {} | ||
@@ -99,2 +110,3 @@ self._next_node_id: int = 0 | ||
| self._root = self._alloc_dir() | ||
| self._memory_guard.check_init(max_quota) | ||
@@ -107,4 +119,6 @@ # -- node allocation helpers -- | ||
| return RandomAccessMemoryFile() | ||
| allow_promotion = (self._default_storage != "sequential") | ||
| return SequentialMemoryFile(self._chunk_overhead, self._promotion_hard_limit, allow_promotion) | ||
| allow_promotion = self._default_storage != "sequential" | ||
| return SequentialMemoryFile( | ||
| self._chunk_overhead, self._promotion_hard_limit, allow_promotion | ||
| ) | ||
@@ -173,2 +187,3 @@ def _alloc_dir(self) -> DirNode: | ||
| fnode: FileNode | None = None | ||
| effective_timeout = lock_timeout if lock_timeout is not None else self._default_lock_timeout | ||
| with self._global_lock: | ||
@@ -183,3 +198,3 @@ node = self._resolve_path(npath) | ||
| raise FileNotFoundError(f"No such file: '{path}'") | ||
| fnode._rw_lock.acquire_read(timeout=lock_timeout) | ||
| fnode._rw_lock.acquire_read(timeout=effective_timeout) | ||
| handle = MemoryFileHandle(self, fnode, npath, mode) | ||
@@ -191,8 +206,8 @@ | ||
| fnode = self._create_file(npath) | ||
| fnode._rw_lock.acquire_write(timeout=lock_timeout) | ||
| fnode._rw_lock.acquire_write(timeout=effective_timeout) | ||
| handle = MemoryFileHandle(self, fnode, npath, mode) | ||
| else: | ||
| # Existing file: truncate and update metadata | ||
| fnode._rw_lock.acquire_write(timeout=lock_timeout) | ||
| fnode.storage.truncate(0, self._quota) | ||
| fnode._rw_lock.acquire_write(timeout=effective_timeout) | ||
| fnode.storage.truncate(0, self._quota, self._memory_guard) | ||
| fnode.generation += 1 | ||
@@ -205,3 +220,3 @@ fnode.modified_at = time.time() | ||
| fnode = self._create_file(npath) | ||
| fnode._rw_lock.acquire_write(timeout=lock_timeout) | ||
| fnode._rw_lock.acquire_write(timeout=effective_timeout) | ||
| handle = MemoryFileHandle(self, fnode, npath, mode, is_append=True) | ||
@@ -212,3 +227,3 @@ | ||
| raise FileNotFoundError(f"No such file: '{path}'") | ||
| fnode._rw_lock.acquire_write(timeout=lock_timeout) | ||
| fnode._rw_lock.acquire_write(timeout=effective_timeout) | ||
| handle = MemoryFileHandle(self, fnode, npath, mode) | ||
@@ -220,3 +235,3 @@ | ||
| fnode = self._create_file(npath) | ||
| fnode._rw_lock.acquire_write(timeout=lock_timeout) | ||
| fnode._rw_lock.acquire_write(timeout=effective_timeout) | ||
| handle = MemoryFileHandle(self, fnode, npath, mode) | ||
@@ -229,3 +244,6 @@ | ||
| n, promoted, old_quota = fnode.storage.write_at( | ||
| current, bytes(preallocate - current), self._quota | ||
| current, | ||
| bytes(preallocate - current), | ||
| self._quota, | ||
| self._memory_guard, | ||
| ) | ||
@@ -517,5 +535,3 @@ if promoted is not None: | ||
| def export_tree( | ||
| self, prefix: str = "/", only_dirty: bool = False | ||
| ) -> dict[str, bytes]: | ||
| def export_tree(self, prefix: str = "/", only_dirty: bool = False) -> dict[str, bytes]: | ||
| return dict(self.iter_export_tree(prefix=prefix, only_dirty=only_dirty)) | ||
@@ -567,7 +583,3 @@ | ||
| node = self._resolve_path(npath) | ||
| if ( | ||
| node is not None | ||
| and isinstance(node, FileNode) | ||
| and node._rw_lock.is_locked | ||
| ): | ||
| if node is not None and isinstance(node, FileNode) and node._rw_lock.is_locked: | ||
| raise BlockingIOError(f"Cannot import: file is open: '{npath}'") | ||
@@ -596,2 +608,3 @@ | ||
| raise MFSQuotaExceededError(requested=net, available=avail) | ||
| self._memory_guard.check_before_write(net) | ||
@@ -606,3 +619,10 @@ written_npaths: list[str] = [] | ||
| storage = self._create_storage() | ||
| storage._bulk_load(data) | ||
| try: | ||
| storage._bulk_load(data) | ||
| except MemoryError: | ||
| raise MemoryError( | ||
| f"OS memory allocation failed during import_tree " | ||
| f"(file: '{npath}', size: {len(data):,} bytes). " | ||
| "Consider reducing max_quota or using memory_guard='init'." | ||
| ) from None | ||
| fnode = self._alloc_file(storage) | ||
@@ -662,5 +682,3 @@ fnode.generation = 0 | ||
| def _ensure_parents( | ||
| self, npath: str, created_dirs: list[str] | None = None | ||
| ) -> None: | ||
| def _ensure_parents(self, npath: str, created_dirs: list[str] | None = None) -> None: | ||
| parent_path = posixpath.dirname(npath) or "/" | ||
@@ -688,3 +706,5 @@ if self._resolve_path(parent_path) is None: | ||
| if data: | ||
| n, promoted, old_quota = fnode.storage.write_at(0, data, self._quota) | ||
| n, promoted, old_quota = fnode.storage.write_at( | ||
| 0, data, self._quota, self._memory_guard | ||
| ) | ||
| if promoted is not None: | ||
@@ -715,2 +735,3 @@ fnode.storage = promoted | ||
| raise MFSQuotaExceededError(requested=total_data, available=avail) | ||
| self._memory_guard.check_before_write(total_data) | ||
| # Deep copy the subtree with rollback on failure | ||
@@ -729,5 +750,3 @@ dst_parent, dst_name = dst_pinfo | ||
| def _deep_copy_subtree( | ||
| self, node: Node, created_node_ids: list[int] | ||
| ) -> Node: | ||
| def _deep_copy_subtree(self, node: Node, created_node_ids: list[int]) -> Node: | ||
| if isinstance(node, FileNode): | ||
@@ -848,6 +867,3 @@ # Read data under read lock | ||
| next_part = parts[idx + 1] | ||
| if ( | ||
| fnmatch.fnmatch(name, next_part) | ||
| and idx + 1 == len(parts) - 1 | ||
| ): | ||
| if fnmatch.fnmatch(name, next_part) and idx + 1 == len(parts) - 1: | ||
| results.append(child_path) | ||
@@ -870,5 +886,3 @@ else: | ||
| def _collect_all_paths( | ||
| self, node: DirNode, current_path: str, results: list[str] | ||
| ) -> None: | ||
| def _collect_all_paths(self, node: DirNode, current_path: str, results: list[str]) -> None: | ||
| with self._global_lock: | ||
@@ -875,0 +889,0 @@ snapshot = list(node.children.items()) |
+24
-10
@@ -6,3 +6,3 @@ from __future__ import annotations | ||
| import warnings | ||
| from typing import TYPE_CHECKING | ||
| from typing import TYPE_CHECKING, Any | ||
@@ -13,3 +13,3 @@ if TYPE_CHECKING: | ||
| class MemoryFileHandle: | ||
| class MemoryFileHandle(io.RawIOBase): | ||
| def __init__( | ||
@@ -23,2 +23,3 @@ self, | ||
| ) -> None: | ||
| super().__init__() | ||
| self._mfs = mfs | ||
@@ -41,3 +42,3 @@ self._fnode = fnode | ||
| def _assert_open(self) -> None: | ||
| if self._is_closed: | ||
| if self.closed or self._is_closed: | ||
| raise ValueError("I/O operation on closed file.") | ||
@@ -61,9 +62,12 @@ | ||
| def write(self, data: bytes) -> int: | ||
| def write(self, data: Any) -> int: | ||
| self._assert_open() | ||
| self._assert_writable() | ||
| if not isinstance(data, (bytes, bytearray, memoryview)): | ||
| raise TypeError("a bytes-like object is required") | ||
| payload = bytes(data) | ||
| if self._is_append: | ||
| self._cursor = self._fnode.storage.get_size() | ||
| n, promoted, old_quota = self._fnode.storage.write_at( | ||
| self._cursor, data, self._mfs._quota | ||
| self._cursor, payload, self._mfs._quota, self._mfs._memory_guard | ||
| ) | ||
@@ -79,2 +83,11 @@ if promoted is not None: | ||
| def readinto(self, buffer: Any) -> int: | ||
| self._assert_open() | ||
| self._assert_readable() | ||
| view = memoryview(buffer).cast("B") | ||
| data = self.read(len(view)) | ||
| n = len(data) | ||
| view[:n] = data | ||
| return n | ||
| def seek(self, offset: int, whence: int = 0) -> int: | ||
@@ -113,3 +126,3 @@ self._assert_open() | ||
| before = self._fnode.storage.get_size() | ||
| self._fnode.storage.truncate(target, self._mfs._quota) | ||
| self._fnode.storage.truncate(target, self._mfs._quota, self._mfs._memory_guard) | ||
| if self._cursor > target: | ||
@@ -139,5 +152,4 @@ self._cursor = target | ||
| def close(self) -> None: | ||
| if self._is_closed: | ||
| if self.closed or self._is_closed: | ||
| return | ||
| self._is_closed = True | ||
| mode = self._mode | ||
@@ -148,2 +160,4 @@ if mode in ("wb", "ab", "r+b", "xb"): | ||
| self._fnode._rw_lock.release_read() | ||
| super().close() | ||
| self._is_closed = True | ||
@@ -153,7 +167,7 @@ def __enter__(self) -> MemoryFileHandle: | ||
| def __exit__(self, *args) -> None: | ||
| def __exit__(self, *args: object) -> None: | ||
| self.close() | ||
| def __del__(self) -> None: | ||
| if not self._is_closed: | ||
| if not getattr(self, "_is_closed", True) and not getattr(self, "closed", True): | ||
| warnings.warn( | ||
@@ -160,0 +174,0 @@ "MFS MemoryFileHandle was not closed properly. " |
+2
-1
| import threading | ||
| from collections.abc import Iterator | ||
| from contextlib import contextmanager | ||
@@ -14,3 +15,3 @@ | ||
| @contextmanager | ||
| def reserve(self, size: int): | ||
| def reserve(self, size: int) -> Iterator[None]: | ||
| if size <= 0: | ||
@@ -17,0 +18,0 @@ yield |
+56
-20
@@ -9,2 +9,3 @@ """MFSTextHandle: bufferless text I/O helper. | ||
| import codecs | ||
| from collections.abc import Iterator | ||
@@ -47,2 +48,3 @@ from typing import TYPE_CHECKING | ||
| self._errors = errors | ||
| self._decoded_buffer = "" | ||
@@ -83,10 +85,46 @@ @property | ||
| Maximum number of characters to read. ``-1`` reads everything. | ||
| Note that this is an approximation in characters, not bytes. | ||
| """ | ||
| if size < 0: | ||
| raw = self._handle.read() | ||
| else: | ||
| raw = self._handle.read(size) | ||
| return raw.decode(self._encoding, self._errors) | ||
| if self._decoded_buffer: | ||
| prefix = self._decoded_buffer | ||
| self._decoded_buffer = "" | ||
| return prefix + raw.decode(self._encoding, self._errors) | ||
| return raw.decode(self._encoding, self._errors) | ||
| if size == 0: | ||
| return "" | ||
| parts: list[str] = [] | ||
| remaining = size | ||
| if self._decoded_buffer: | ||
| take = self._decoded_buffer[:remaining] | ||
| parts.append(take) | ||
| self._decoded_buffer = self._decoded_buffer[len(take) :] | ||
| remaining -= len(take) | ||
| if remaining == 0: | ||
| return "".join(parts) | ||
| decoder = codecs.getincrementaldecoder(self._encoding)(errors=self._errors) | ||
| while remaining > 0: | ||
| chunk = self._handle.read(1) | ||
| if not chunk: | ||
| tail = decoder.decode(b"", final=True) | ||
| if tail: | ||
| take = tail[:remaining] | ||
| parts.append(take) | ||
| self._decoded_buffer = tail[len(take) :] + self._decoded_buffer | ||
| break | ||
| decoded = decoder.decode(chunk, final=False) | ||
| if not decoded: | ||
| continue | ||
| take = decoded[:remaining] | ||
| parts.append(take) | ||
| remaining -= len(take) | ||
| if len(decoded) > len(take): | ||
| self._decoded_buffer = decoded[len(take) :] + self._decoded_buffer | ||
| break | ||
| return "".join(parts) | ||
| def readline(self, limit: int = -1) -> str: | ||
@@ -100,24 +138,22 @@ """Read one line. | ||
| limit: | ||
| Maximum number of bytes to read (``-1`` means unlimited). | ||
| Maximum number of characters to read (``-1`` means unlimited). | ||
| """ | ||
| buf = bytearray() | ||
| chars: list[str] = [] | ||
| while True: | ||
| if limit >= 0 and len(buf) >= limit: | ||
| if limit >= 0 and len(chars) >= limit: | ||
| break | ||
| b = self._handle.read(1) | ||
| if not b: | ||
| ch = self.read(1) | ||
| if not ch: | ||
| break | ||
| buf.extend(b) | ||
| if b == b"\n": | ||
| chars.append(ch) | ||
| if ch == "\n": | ||
| break | ||
| if b == b"\r": | ||
| # Peek at the next byte to determine \r\n vs bare \r | ||
| next_b = self._handle.read(1) | ||
| if next_b == b"\n": | ||
| buf.extend(next_b) | ||
| elif next_b: | ||
| # Bare \r: seek back one byte | ||
| self._handle.seek(self._handle.tell() - 1) | ||
| if ch == "\r": | ||
| next_ch = self.read(1) | ||
| if next_ch == "\n": | ||
| chars.append(next_ch) | ||
| elif next_ch: | ||
| self._decoded_buffer = next_ch + self._decoded_buffer | ||
| break | ||
| return buf.decode(self._encoding, self._errors) | ||
| return "".join(chars) | ||
@@ -124,0 +160,0 @@ def __iter__(self) -> Iterator[str]: |
+51
-4
| Metadata-Version: 2.4 | ||
| Name: D-MemFS | ||
| Version: 0.2.1 | ||
| Version: 0.3.0 | ||
| Summary: In-process virtual filesystem with hard quota for Python | ||
@@ -104,2 +104,6 @@ Author: D | ||
| - `chunk_overhead_override` (default `None`): override the per-chunk overhead estimate used for quota accounting | ||
| - `default_lock_timeout` (default `30.0`): default timeout in seconds for file-lock acquisition during `open()`. Use `None` to wait indefinitely. | ||
| - `memory_guard` (default `"none"`): physical memory protection mode — `"none"` / `"init"` / `"per_write"` | ||
| - `memory_guard_action` (default `"warn"`): action when the guard triggers — `"warn"` (`ResourceWarning`) / `"raise"` (`MemoryError`) | ||
| - `memory_guard_interval` (default `1.0`): minimum seconds between OS memory queries (`"per_write"` only) | ||
@@ -109,7 +113,49 @@ > **Note:** The `BytesIO` returned by `export_as_bytesio()` is outside quota management. | ||
| > **Note — Quota and free-threaded Python:** | ||
| > The per-chunk overhead estimate used for quota accounting is calibrated at import time | ||
| > via `sys.getsizeof()`. Free-threaded Python (3.13t, `PYTHON_GIL=0`) has larger object | ||
| > headers than the standard build, so `CHUNK_OVERHEAD_ESTIMATE` is higher (~117 bytes vs | ||
| > ~93 bytes on CPython 3.13). This means the same `max_quota` yields slightly less | ||
| > effective storage capacity on free-threaded builds, especially for workloads with many | ||
| > small files or small appends. This is not a bug — it reflects real memory consumption. | ||
| > To ensure consistent behaviour across builds, use `chunk_overhead_override` to pin the | ||
| > value, or inspect `stats()["overhead_per_chunk_estimate"]` at runtime. | ||
| Supported binary modes: `rb`, `wb`, `ab`, `r+b`, `xb` | ||
| ## Memory Guard | ||
| MFS enforces a logical quota, but that quota can still be configured larger than the | ||
| currently available physical RAM. `memory_guard` provides an optional safety net. | ||
| ```python | ||
| from dmemfs import MemoryFileSystem | ||
| # Warn if max_quota exceeds available RAM | ||
| mfs = MemoryFileSystem(max_quota=8 * 1024**3, memory_guard="init") | ||
| # Raise MemoryError before writes when RAM is insufficient | ||
| mfs = MemoryFileSystem( | ||
| max_quota=8 * 1024**3, | ||
| memory_guard="per_write", | ||
| memory_guard_action="raise", | ||
| ) | ||
| ``` | ||
| | Mode | Initialization | Each Write | Overhead | | ||
| |---|---|---|---| | ||
| | `"none"` | — | — | Zero | | ||
| | `"init"` | Check once | — | Negligible | | ||
| | `"per_write"` | Check once | Cached check | About 1 OS call/sec | | ||
| When `memory_guard_action="warn"`, the guard emits `ResourceWarning` and allows the operation to continue. | ||
| When `memory_guard_action="raise"`, the guard rejects the operation with `MemoryError` before the actual allocation path. | ||
| `AsyncMemoryFileSystem` accepts the same constructor parameters and forwards them to the synchronous implementation. | ||
| ### `MemoryFileHandle` | ||
| - `io.RawIOBase`-compatible binary handle | ||
| - `read`, `write`, `seek`, `tell`, `truncate`, `flush`, `close` | ||
| - `readinto` | ||
| - file-like capability checks: `readable`, `writable`, `seekable` | ||
@@ -151,3 +197,3 @@ | ||
| `MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`. | ||
| `MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. `read(size)` counts characters, not bytes, so multibyte text can be read safely without splitting code points. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`. | ||
@@ -389,4 +435,5 @@ --- | ||
| - [Architecture Spec v13](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v13.md) — API design, internal structure, CI matrix | ||
| - [Detailed Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec.md) — component-level design and rationale | ||
| - [Test Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test.md) — test case table and pseudocode | ||
| - [Architecture Spec v14](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v14.md) — MemoryGuard-integrated architecture spec | ||
| - [Detailed Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_v2.md) — component-level design and rationale | ||
| - [Test Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test_v2.md) — test case table and pseudocode | ||
@@ -393,0 +440,0 @@ > These documents are written in Japanese and serve as internal design references. |
+1
-1
@@ -7,3 +7,3 @@ [build-system] | ||
| name = "D-MemFS" | ||
| version = "0.2.1" | ||
| version = "0.3.0" | ||
| description = "In-process virtual filesystem with hard quota for Python" | ||
@@ -10,0 +10,0 @@ readme = "README.md" |
+50
-3
@@ -81,2 +81,6 @@ # D-MemFS | ||
| - `chunk_overhead_override` (default `None`): override the per-chunk overhead estimate used for quota accounting | ||
| - `default_lock_timeout` (default `30.0`): default timeout in seconds for file-lock acquisition during `open()`. Use `None` to wait indefinitely. | ||
| - `memory_guard` (default `"none"`): physical memory protection mode — `"none"` / `"init"` / `"per_write"` | ||
| - `memory_guard_action` (default `"warn"`): action when the guard triggers — `"warn"` (`ResourceWarning`) / `"raise"` (`MemoryError`) | ||
| - `memory_guard_interval` (default `1.0`): minimum seconds between OS memory queries (`"per_write"` only) | ||
@@ -86,7 +90,49 @@ > **Note:** The `BytesIO` returned by `export_as_bytesio()` is outside quota management. | ||
| > **Note — Quota and free-threaded Python:** | ||
| > The per-chunk overhead estimate used for quota accounting is calibrated at import time | ||
| > via `sys.getsizeof()`. Free-threaded Python (3.13t, `PYTHON_GIL=0`) has larger object | ||
| > headers than the standard build, so `CHUNK_OVERHEAD_ESTIMATE` is higher (~117 bytes vs | ||
| > ~93 bytes on CPython 3.13). This means the same `max_quota` yields slightly less | ||
| > effective storage capacity on free-threaded builds, especially for workloads with many | ||
| > small files or small appends. This is not a bug — it reflects real memory consumption. | ||
| > To ensure consistent behaviour across builds, use `chunk_overhead_override` to pin the | ||
| > value, or inspect `stats()["overhead_per_chunk_estimate"]` at runtime. | ||
| Supported binary modes: `rb`, `wb`, `ab`, `r+b`, `xb` | ||
| ## Memory Guard | ||
| MFS enforces a logical quota, but that quota can still be configured larger than the | ||
| currently available physical RAM. `memory_guard` provides an optional safety net. | ||
| ```python | ||
| from dmemfs import MemoryFileSystem | ||
| # Warn if max_quota exceeds available RAM | ||
| mfs = MemoryFileSystem(max_quota=8 * 1024**3, memory_guard="init") | ||
| # Raise MemoryError before writes when RAM is insufficient | ||
| mfs = MemoryFileSystem( | ||
| max_quota=8 * 1024**3, | ||
| memory_guard="per_write", | ||
| memory_guard_action="raise", | ||
| ) | ||
| ``` | ||
| | Mode | Initialization | Each Write | Overhead | | ||
| |---|---|---|---| | ||
| | `"none"` | — | — | Zero | | ||
| | `"init"` | Check once | — | Negligible | | ||
| | `"per_write"` | Check once | Cached check | About 1 OS call/sec | | ||
| When `memory_guard_action="warn"`, the guard emits `ResourceWarning` and allows the operation to continue. | ||
| When `memory_guard_action="raise"`, the guard rejects the operation with `MemoryError` before the actual allocation path. | ||
| `AsyncMemoryFileSystem` accepts the same constructor parameters and forwards them to the synchronous implementation. | ||
| ### `MemoryFileHandle` | ||
| - `io.RawIOBase`-compatible binary handle | ||
| - `read`, `write`, `seek`, `tell`, `truncate`, `flush`, `close` | ||
| - `readinto` | ||
| - file-like capability checks: `readable`, `writable`, `seekable` | ||
@@ -128,3 +174,3 @@ | ||
| `MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`. | ||
| `MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. `read(size)` counts characters, not bytes, so multibyte text can be read safely without splitting code points. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`. | ||
@@ -366,4 +412,5 @@ --- | ||
| - [Architecture Spec v13](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v13.md) — API design, internal structure, CI matrix | ||
| - [Detailed Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec.md) — component-level design and rationale | ||
| - [Test Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test.md) — test case table and pseudocode | ||
| - [Architecture Spec v14](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v14.md) — MemoryGuard-integrated architecture spec | ||
| - [Detailed Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_v2.md) — component-level design and rationale | ||
| - [Test Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test_v2.md) — test case table and pseudocode | ||
@@ -370,0 +417,0 @@ > These documents are written in Japanese and serve as internal design references. |
Alert delta unavailable
Currently unable to show alert delta for PyPI packages.
134890
22.46%24
9.09%1952
26.1%