You're Invited:Meet the Socket Team at RSAC and BSidesSF 2026, March 23–26.RSVP
Socket
Book a DemoSign in
Socket

D-MemFS

Package Overview
Dependencies
Maintainers
1
Versions
5
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

D-MemFS - pypi Package Compare versions

Comparing version
0.2.1
to
0.3.0
+104
dmemfs/_memory_guard.py
"""Physical memory guard strategies for D-MemFS."""
from __future__ import annotations
import threading
import time
import warnings
from abc import ABC, abstractmethod
from ._memory_info import get_available_memory_bytes
class MemoryGuard(ABC):
def __init__(self, action: str = "warn") -> None:
if action not in ("warn", "raise"):
raise ValueError(
f"Invalid memory_guard_action: {action!r}. Expected 'warn' or 'raise'."
)
self._action = action
@abstractmethod
def check_init(self, max_quota: int) -> None: ...
@abstractmethod
def check_before_write(self, size: int) -> None: ...
def _handle_violation(self, message: str) -> None:
if self._action == "raise":
raise MemoryError(message)
warnings.warn(message, ResourceWarning, stacklevel=4)
class NullGuard(MemoryGuard):
def __init__(self) -> None:
super().__init__(action="warn")
def check_init(self, max_quota: int) -> None:
return None
def check_before_write(self, size: int) -> None:
return None
class InitGuard(MemoryGuard):
def check_init(self, max_quota: int) -> None:
avail = get_available_memory_bytes()
if avail is not None and max_quota > avail:
self._handle_violation(
f"max_quota ({max_quota:,} bytes) exceeds available physical RAM "
f"({avail:,} bytes). MemoryError may occur before quota limit is reached."
)
def check_before_write(self, size: int) -> None:
return None
class PerWriteGuard(MemoryGuard):
def __init__(self, action: str = "warn", interval: float = 1.0) -> None:
super().__init__(action=action)
self._interval = interval
self._last_check = 0.0
self._last_avail: int | None = None
self._lock = threading.Lock()
def check_init(self, max_quota: int) -> None:
avail = get_available_memory_bytes()
if avail is not None and max_quota > avail:
self._handle_violation(
f"max_quota ({max_quota:,} bytes) exceeds available physical RAM "
f"({avail:,} bytes). MemoryError may occur before quota limit is reached."
)
with self._lock:
self._last_avail = avail
self._last_check = time.monotonic()
def check_before_write(self, size: int) -> None:
now = time.monotonic()
with self._lock:
if now - self._last_check >= self._interval:
self._last_avail = get_available_memory_bytes()
self._last_check = now
avail = self._last_avail
age = now - self._last_check
if avail is not None and size > avail:
self._handle_violation(
f"Write of {size:,} bytes requested but only {avail:,} bytes of "
f"physical RAM available (checked {age:.1f}s ago)."
)
def create_memory_guard(
mode: str = "none",
action: str = "warn",
interval: float = 1.0,
) -> MemoryGuard:
if action not in ("warn", "raise"):
raise ValueError(f"Invalid memory_guard_action: {action!r}. Expected 'warn' or 'raise'.")
if mode == "none":
return NullGuard()
if mode == "init":
return InitGuard(action=action)
if mode == "per_write":
return PerWriteGuard(action=action, interval=interval)
raise ValueError(f"Invalid memory_guard: {mode!r}. Expected 'none', 'init', or 'per_write'.")
"""OS-specific available physical memory detection.
Internal module - not part of the public API.
Uses only Python standard library.
"""
from __future__ import annotations
import ctypes
import ctypes.util
import platform
from collections.abc import Callable
from typing import cast
_SYSTEM = platform.system()
_UNPROBED = object()
_linux_reader: Callable[[], int | None] | None | object = _UNPROBED
def get_available_memory_bytes() -> int | None:
"""Return available physical memory in bytes, or None if unavailable."""
try:
if _SYSTEM == "Windows":
return _windows_avail()
if _SYSTEM == "Linux":
return _linux_avail()
if _SYSTEM == "Darwin":
return _macos_avail()
except Exception:
return None
return None
def _windows_avail() -> int:
class MEMORYSTATUSEX(ctypes.Structure):
_fields_ = [
("dwLength", ctypes.c_ulong),
("dwMemoryLoad", ctypes.c_ulong),
("ullTotalPhys", ctypes.c_ulonglong),
("ullAvailPhys", ctypes.c_ulonglong),
("ullTotalPageFile", ctypes.c_ulonglong),
("ullAvailPageFile", ctypes.c_ulonglong),
("ullTotalVirtual", ctypes.c_ulonglong),
("ullAvailVirtual", ctypes.c_ulonglong),
("ullAvailExtendedVirtual", ctypes.c_ulonglong),
]
stat = MEMORYSTATUSEX()
stat.dwLength = ctypes.sizeof(stat)
result = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
if not result:
raise OSError("GlobalMemoryStatusEx failed")
return int(stat.ullAvailPhys)
def _linux_avail() -> int | None:
global _linux_reader
if _linux_reader is _UNPROBED:
_linux_reader = _probe_linux_source()
if _linux_reader is None:
return None
return cast(Callable[[], int | None], _linux_reader)()
def _probe_linux_source() -> Callable[[], int | None] | None:
if _is_cgroup_v2_limited():
return _read_cgroup_v2
if _is_cgroup_v1_limited():
return _read_cgroup_v1
if _is_procmeminfo_available():
return _read_procmeminfo
return None
def _is_cgroup_v2_limited() -> bool:
try:
with open("/sys/fs/cgroup/memory.max", encoding="utf-8") as handle:
return handle.read().strip() != "max"
except (FileNotFoundError, PermissionError):
return False
def _is_cgroup_v1_limited() -> bool:
try:
with open("/sys/fs/cgroup/memory/memory.limit_in_bytes", encoding="utf-8") as handle:
limit = int(handle.read().strip())
host_total = _read_procmeminfo_total()
if host_total is not None and limit >= host_total:
return False
return True
except (FileNotFoundError, PermissionError, ValueError):
return False
def _is_procmeminfo_available() -> bool:
try:
with open("/proc/meminfo", encoding="utf-8") as handle:
handle.readline()
return True
except (FileNotFoundError, PermissionError):
return False
def _read_cgroup_v2() -> int | None:
with open("/sys/fs/cgroup/memory.max", encoding="utf-8") as handle:
value = handle.read().strip()
if value == "max":
return None
max_mem = int(value)
with open("/sys/fs/cgroup/memory.current", encoding="utf-8") as handle:
current = int(handle.read().strip())
return max(0, max_mem - current)
def _read_cgroup_v1() -> int:
with open("/sys/fs/cgroup/memory/memory.limit_in_bytes", encoding="utf-8") as handle:
limit = int(handle.read().strip())
with open("/sys/fs/cgroup/memory/memory.usage_in_bytes", encoding="utf-8") as handle:
usage = int(handle.read().strip())
return max(0, limit - usage)
def _read_procmeminfo_total() -> int | None:
try:
with open("/proc/meminfo", encoding="utf-8") as handle:
for line in handle:
if line.startswith("MemTotal:"):
return int(line.split()[1]) * 1024
except Exception:
return None
return None
def _read_procmeminfo() -> int:
with open("/proc/meminfo", encoding="utf-8") as handle:
content = handle.read()
mem_data: dict[str, int] = {}
for line in content.splitlines():
parts = line.split()
if len(parts) >= 2:
mem_data[parts[0].rstrip(":")] = int(parts[1])
free_kb = mem_data.get("MemAvailable", mem_data.get("MemFree", 0))
return free_kb * 1024
def _read_proc_meminfo() -> int:
return _read_procmeminfo()
def _macos_avail() -> int:
libc_path = ctypes.util.find_library("c") or "/usr/lib/libSystem.B.dylib"
libc = ctypes.CDLL(libc_path)
host_vm_info64 = 4
host_vm_info64_count = 48
class VMStatistics64(ctypes.Structure):
_fields_ = [
("free_count", ctypes.c_uint64),
("active_count", ctypes.c_uint64),
("inactive_count", ctypes.c_uint64),
("wire_count", ctypes.c_uint64),
("zero_fill_count", ctypes.c_uint64),
("reactivations", ctypes.c_uint64),
("pageins", ctypes.c_uint64),
("pageouts", ctypes.c_uint64),
("faults", ctypes.c_uint64),
("cow_faults", ctypes.c_uint64),
("lookups", ctypes.c_uint64),
("hits", ctypes.c_uint64),
("purges", ctypes.c_uint64),
("purgeable_count", ctypes.c_uint64),
("speculative_count", ctypes.c_uint64),
("decompressions", ctypes.c_uint64),
("compressions", ctypes.c_uint64),
("swapins", ctypes.c_uint64),
("swapouts", ctypes.c_uint64),
("compressor_page_count", ctypes.c_uint64),
("throttled_count", ctypes.c_uint64),
("external_page_count", ctypes.c_uint64),
("internal_page_count", ctypes.c_uint64),
("total_uncompressed_pages_in_compressor", ctypes.c_uint64),
]
libc.mach_host_self.restype = ctypes.c_uint
libc.host_page_size.argtypes = [ctypes.c_uint, ctypes.POINTER(ctypes.c_uint)]
libc.host_page_size.restype = ctypes.c_int
libc.host_statistics64.argtypes = [
ctypes.c_uint,
ctypes.c_int,
ctypes.c_void_p,
ctypes.POINTER(ctypes.c_uint),
]
libc.host_statistics64.restype = ctypes.c_int
host = libc.mach_host_self()
page_size = ctypes.c_uint()
if libc.host_page_size(host, ctypes.byref(page_size)) != 0:
raise OSError("host_page_size failed")
vm_stat = VMStatistics64()
count = ctypes.c_uint(host_vm_info64_count)
ret = libc.host_statistics64(
host,
host_vm_info64,
ctypes.byref(vm_stat),
ctypes.byref(count),
)
if ret != 0:
raise OSError(f"host_statistics64 failed with code {ret}")
free_pages = vm_stat.free_count + vm_stat.speculative_count
return int(free_pages * page_size.value)
+51
-4
Metadata-Version: 2.4
Name: D-MemFS
Version: 0.2.1
Version: 0.3.0
Summary: In-process virtual filesystem with hard quota for Python

@@ -104,2 +104,6 @@ Author: D

- `chunk_overhead_override` (default `None`): override the per-chunk overhead estimate used for quota accounting
- `default_lock_timeout` (default `30.0`): default timeout in seconds for file-lock acquisition during `open()`. Use `None` to wait indefinitely.
- `memory_guard` (default `"none"`): physical memory protection mode — `"none"` / `"init"` / `"per_write"`
- `memory_guard_action` (default `"warn"`): action when the guard triggers — `"warn"` (`ResourceWarning`) / `"raise"` (`MemoryError`)
- `memory_guard_interval` (default `1.0`): minimum seconds between OS memory queries (`"per_write"` only)

@@ -109,7 +113,49 @@ > **Note:** The `BytesIO` returned by `export_as_bytesio()` is outside quota management.

> **Note — Quota and free-threaded Python:**
> The per-chunk overhead estimate used for quota accounting is calibrated at import time
> via `sys.getsizeof()`. Free-threaded Python (3.13t, `PYTHON_GIL=0`) has larger object
> headers than the standard build, so `CHUNK_OVERHEAD_ESTIMATE` is higher (~117 bytes vs
> ~93 bytes on CPython 3.13). This means the same `max_quota` yields slightly less
> effective storage capacity on free-threaded builds, especially for workloads with many
> small files or small appends. This is not a bug — it reflects real memory consumption.
> To ensure consistent behaviour across builds, use `chunk_overhead_override` to pin the
> value, or inspect `stats()["overhead_per_chunk_estimate"]` at runtime.
Supported binary modes: `rb`, `wb`, `ab`, `r+b`, `xb`
## Memory Guard
MFS enforces a logical quota, but that quota can still be configured larger than the
currently available physical RAM. `memory_guard` provides an optional safety net.
```python
from dmemfs import MemoryFileSystem
# Warn if max_quota exceeds available RAM
mfs = MemoryFileSystem(max_quota=8 * 1024**3, memory_guard="init")
# Raise MemoryError before writes when RAM is insufficient
mfs = MemoryFileSystem(
max_quota=8 * 1024**3,
memory_guard="per_write",
memory_guard_action="raise",
)
```
| Mode | Initialization | Each Write | Overhead |
|---|---|---|---|
| `"none"` | — | — | Zero |
| `"init"` | Check once | — | Negligible |
| `"per_write"` | Check once | Cached check | About 1 OS call/sec |
When `memory_guard_action="warn"`, the guard emits `ResourceWarning` and allows the operation to continue.
When `memory_guard_action="raise"`, the guard rejects the operation with `MemoryError` before the actual allocation path.
`AsyncMemoryFileSystem` accepts the same constructor parameters and forwards them to the synchronous implementation.
### `MemoryFileHandle`
- `io.RawIOBase`-compatible binary handle
- `read`, `write`, `seek`, `tell`, `truncate`, `flush`, `close`
- `readinto`
- file-like capability checks: `readable`, `writable`, `seekable`

@@ -151,3 +197,3 @@

`MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`.
`MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. `read(size)` counts characters, not bytes, so multibyte text can be read safely without splitting code points. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`.

@@ -389,4 +435,5 @@ ---

- [Architecture Spec v13](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v13.md) — API design, internal structure, CI matrix
- [Detailed Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec.md) — component-level design and rationale
- [Test Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test.md) — test case table and pseudocode
- [Architecture Spec v14](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v14.md) — MemoryGuard-integrated architecture spec
- [Detailed Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_v2.md) — component-level design and rationale
- [Test Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test_v2.md) — test case table and pseudocode

@@ -393,0 +440,0 @@ > These documents are written in Japanese and serve as internal design references.

@@ -15,2 +15,4 @@ LICENSE

dmemfs/_lock.py
dmemfs/_memory_guard.py
dmemfs/_memory_info.py
dmemfs/_path.py

@@ -17,0 +19,0 @@ dmemfs/_pytest_plugin.py

+1
-1

@@ -34,2 +34,2 @@ from typing import TYPE_CHECKING

]
__version__ = "0.2.0"
__version__ = "0.3.0"

@@ -73,2 +73,6 @@ """Async wrapper around MemoryFileSystem.

default_storage: str = "auto",
default_lock_timeout: float | None = 30.0,
memory_guard: str = "none",
memory_guard_action: str = "warn",
memory_guard_interval: float = 1.0,
) -> None:

@@ -81,2 +85,6 @@ self._sync = MemoryFileSystem(

default_storage=default_storage,
default_lock_timeout=default_lock_timeout,
memory_guard=memory_guard,
memory_guard_action=memory_guard_action,
memory_guard_interval=memory_guard_interval,
)

@@ -91,5 +99,3 @@

) -> AsyncMemoryFileHandle:
h = await asyncio.to_thread(
self._sync.open, path, mode, preallocate, lock_timeout
)
h = await asyncio.to_thread(self._sync.open, path, mode, preallocate, lock_timeout)
return AsyncMemoryFileHandle(h)

@@ -133,10 +139,6 @@

async def export_as_bytesio(
self, path: str, max_size: int | None = None
) -> io.BytesIO:
async def export_as_bytesio(self, path: str, max_size: int | None = None) -> io.BytesIO:
return await asyncio.to_thread(self._sync.export_as_bytesio, path, max_size)
async def export_tree(
self, prefix: str = "/", only_dirty: bool = False
) -> dict[str, bytes]:
async def export_tree(self, prefix: str = "/", only_dirty: bool = False) -> dict[str, bytes]:
return await asyncio.to_thread(self._sync.export_tree, prefix, only_dirty)

@@ -143,0 +145,0 @@

import bisect
import io
import sys
from abc import ABC, abstractmethod

@@ -8,13 +7,13 @@ from typing import TYPE_CHECKING

if TYPE_CHECKING:
from ._memory_guard import MemoryGuard
from ._quota import QuotaManager
def _calibrate_chunk_overhead(safety_mul: float = 1.5, safety_add: int = 32) -> int:
empty_bytes_size = sys.getsizeof(b"")
list_ptr_size = sys.getsizeof([None]) - sys.getsizeof([])
raw = empty_bytes_size + list_ptr_size
return int(raw * safety_mul) + safety_add
def _wrap_memory_error(message: str) -> MemoryError:
return MemoryError(message)
CHUNK_OVERHEAD_ESTIMATE: int = _calibrate_chunk_overhead()
# Keep quota behavior deterministic across standard and free-threaded builds.
# A fixed conservative value avoids runtime-dependent quota boundaries.
CHUNK_OVERHEAD_ESTIMATE: int = 128

@@ -34,8 +33,16 @@

def write_at(
self, offset: int, data: bytes, quota_mgr: "QuotaManager"
) -> "tuple[int, RandomAccessMemoryFile | None, int]":
...
self,
offset: int,
data: bytes,
quota_mgr: "QuotaManager",
memory_guard: "MemoryGuard | None" = None,
) -> "tuple[int, RandomAccessMemoryFile | None, int]": ...
@abstractmethod
def truncate(self, size: int, quota_mgr: "QuotaManager") -> None: ...
def truncate(
self,
size: int,
quota_mgr: "QuotaManager",
memory_guard: "MemoryGuard | None" = None,
) -> None: ...

@@ -56,3 +63,8 @@ @abstractmethod

def __init__(self, chunk_overhead: int = CHUNK_OVERHEAD_ESTIMATE, promotion_hard_limit: int | None = None, allow_promotion: bool = True) -> None:
def __init__(
self,
chunk_overhead: int = CHUNK_OVERHEAD_ESTIMATE,
promotion_hard_limit: int | None = None,
allow_promotion: bool = True,
) -> None:
super().__init__()

@@ -92,3 +104,9 @@ self._chunks: list[bytes] = []

def write_at(self, offset: int, data: bytes, quota_mgr: "QuotaManager") -> "tuple[int, RandomAccessMemoryFile | None, int]":
def write_at(
self,
offset: int,
data: bytes,
quota_mgr: "QuotaManager",
memory_guard: "MemoryGuard | None" = None,
) -> "tuple[int, RandomAccessMemoryFile | None, int]":
if offset != self._size:

@@ -100,3 +118,3 @@ if not self._allow_promotion:

)
return self._promote_and_write(offset, data, quota_mgr)
return self._promote_and_write(offset, data, quota_mgr, memory_guard)
n = len(data)

@@ -106,9 +124,24 @@ if n == 0:

overhead = self._chunk_overhead
if memory_guard is not None:
memory_guard.check_before_write(n + overhead)
with quota_mgr.reserve(n + overhead):
self._chunks.append(data)
self._size += n
self._cumulative.append(self._size)
try:
self._chunks.append(data)
self._size += n
self._cumulative.append(self._size)
except MemoryError:
raise _wrap_memory_error(
f"OS memory allocation failed while writing {n:,} bytes. "
f"MFS quota had {quota_mgr.free:,} bytes remaining. "
"The max_quota may exceed available system RAM. "
"Consider reducing max_quota or using memory_guard='init'."
) from None
return n, None, 0
def truncate(self, size: int, quota_mgr: "QuotaManager") -> None:
def truncate(
self,
size: int,
quota_mgr: "QuotaManager",
memory_guard: "MemoryGuard | None" = None,
) -> None:
if size == self._size:

@@ -120,6 +153,14 @@ return

overhead = self._chunk_overhead
if memory_guard is not None:
memory_guard.check_before_write(len(pad) + overhead)
with quota_mgr.reserve(len(pad) + overhead):
self._chunks.append(pad)
self._size = size
self._cumulative.append(size)
try:
self._chunks.append(pad)
self._size = size
self._cumulative.append(size)
except MemoryError:
raise _wrap_memory_error(
f"OS memory allocation failed while extending file to {size:,} bytes. "
"Consider reducing max_quota or using memory_guard='init'."
) from None
return

@@ -150,3 +191,9 @@ data = b"".join(self._chunks)[:size]

def _promote_and_write(self, offset: int, data: bytes, quota_mgr: "QuotaManager") -> "tuple[int, RandomAccessMemoryFile, int]":
def _promote_and_write(
self,
offset: int,
data: bytes,
quota_mgr: "QuotaManager",
memory_guard: "MemoryGuard | None" = None,
) -> "tuple[int, RandomAccessMemoryFile, int]":
# NOTE: During promotion, both the original chunk list and the new

@@ -161,8 +208,16 @@ # bytearray coexist temporarily, consuming ~2x the file size in memory.

)
if memory_guard is not None:
memory_guard.check_before_write(current_size)
with quota_mgr.reserve(current_size):
new_buf = bytearray(b"".join(self._chunks))
try:
new_buf = bytearray(b"".join(self._chunks))
except MemoryError:
raise _wrap_memory_error(
f"OS memory allocation failed during storage promotion (file size: {current_size:,} bytes). "
"Consider reducing max_quota or using memory_guard='init'."
) from None
old_overhead = len(self._chunks) * self._chunk_overhead
quota_mgr.release(old_overhead)
promoted = RandomAccessMemoryFile.from_bytearray(new_buf)
written, _, _ = promoted.write_at(offset, data, quota_mgr)
written, _, _ = promoted.write_at(offset, data, quota_mgr, memory_guard)
return written, promoted, current_size

@@ -194,5 +249,11 @@

return bytes(self._buf[offset:])
return bytes(self._buf[offset: offset + size])
return bytes(self._buf[offset : offset + size])
def write_at(self, offset: int, data: bytes, quota_mgr: "QuotaManager") -> "tuple[int, None, int]":
def write_at(
self,
offset: int,
data: bytes,
quota_mgr: "QuotaManager",
memory_guard: "MemoryGuard | None" = None,
) -> "tuple[int, None, int]":
n = len(data)

@@ -205,15 +266,29 @@ if n == 0:

if extend > 0:
if memory_guard is not None:
memory_guard.check_before_write(extend)
with quota_mgr.reserve(extend):
if offset > current_len:
self._buf.extend(bytes(offset - current_len))
self._buf.extend(data)
else:
overlap = current_len - offset
self._buf[offset:current_len] = data[:overlap]
self._buf.extend(data[overlap:])
try:
if offset > current_len:
self._buf.extend(bytes(offset - current_len))
self._buf.extend(data)
else:
overlap = current_len - offset
self._buf[offset:current_len] = data[:overlap]
self._buf.extend(data[overlap:])
except MemoryError:
raise _wrap_memory_error(
f"OS memory allocation failed while writing {n:,} bytes. "
f"MFS quota had {quota_mgr.free:,} bytes remaining. "
"Consider reducing max_quota or using memory_guard='init'."
) from None
else:
self._buf[offset: offset + n] = data
self._buf[offset : offset + n] = data
return n, None, 0
def truncate(self, size: int, quota_mgr: "QuotaManager") -> None:
def truncate(
self,
size: int,
quota_mgr: "QuotaManager",
memory_guard: "MemoryGuard | None" = None,
) -> None:
old_size = len(self._buf)

@@ -225,4 +300,12 @@ if size == old_size:

extend = size - old_size
if memory_guard is not None:
memory_guard.check_before_write(extend)
with quota_mgr.reserve(extend):
self._buf.extend(bytes(extend))
try:
self._buf.extend(bytes(extend))
except MemoryError:
raise _wrap_memory_error(
f"OS memory allocation failed while extending file to {size:,} bytes. "
"Consider reducing max_quota or using memory_guard='init'."
) from None
return

@@ -229,0 +312,0 @@ release = old_size - size

@@ -19,2 +19,3 @@ from __future__ import annotations

from ._lock import ReadWriteLock
from ._memory_guard import create_memory_guard
from ._path import normalize_path

@@ -76,2 +77,6 @@ from ._quota import QuotaManager

default_storage: str = "auto",
default_lock_timeout: float | None = 30.0,
memory_guard: str = "none",
memory_guard_action: str = "warn",
memory_guard_interval: float = 1.0,
) -> None:

@@ -85,2 +90,7 @@ if default_storage not in ("auto", "sequential", "random_access"):

self._global_lock = threading.RLock()
self._memory_guard = create_memory_guard(
mode=memory_guard,
action=memory_guard_action,
interval=memory_guard_interval,
)
self._chunk_overhead: int = (

@@ -94,2 +104,3 @@ chunk_overhead_override

self._default_storage: str = default_storage
self._default_lock_timeout: float | None = default_lock_timeout
self._nodes: dict[int, Node] = {}

@@ -99,2 +110,3 @@ self._next_node_id: int = 0

self._root = self._alloc_dir()
self._memory_guard.check_init(max_quota)

@@ -107,4 +119,6 @@ # -- node allocation helpers --

return RandomAccessMemoryFile()
allow_promotion = (self._default_storage != "sequential")
return SequentialMemoryFile(self._chunk_overhead, self._promotion_hard_limit, allow_promotion)
allow_promotion = self._default_storage != "sequential"
return SequentialMemoryFile(
self._chunk_overhead, self._promotion_hard_limit, allow_promotion
)

@@ -173,2 +187,3 @@ def _alloc_dir(self) -> DirNode:

fnode: FileNode | None = None
effective_timeout = lock_timeout if lock_timeout is not None else self._default_lock_timeout
with self._global_lock:

@@ -183,3 +198,3 @@ node = self._resolve_path(npath)

raise FileNotFoundError(f"No such file: '{path}'")
fnode._rw_lock.acquire_read(timeout=lock_timeout)
fnode._rw_lock.acquire_read(timeout=effective_timeout)
handle = MemoryFileHandle(self, fnode, npath, mode)

@@ -191,8 +206,8 @@

fnode = self._create_file(npath)
fnode._rw_lock.acquire_write(timeout=lock_timeout)
fnode._rw_lock.acquire_write(timeout=effective_timeout)
handle = MemoryFileHandle(self, fnode, npath, mode)
else:
# Existing file: truncate and update metadata
fnode._rw_lock.acquire_write(timeout=lock_timeout)
fnode.storage.truncate(0, self._quota)
fnode._rw_lock.acquire_write(timeout=effective_timeout)
fnode.storage.truncate(0, self._quota, self._memory_guard)
fnode.generation += 1

@@ -205,3 +220,3 @@ fnode.modified_at = time.time()

fnode = self._create_file(npath)
fnode._rw_lock.acquire_write(timeout=lock_timeout)
fnode._rw_lock.acquire_write(timeout=effective_timeout)
handle = MemoryFileHandle(self, fnode, npath, mode, is_append=True)

@@ -212,3 +227,3 @@

raise FileNotFoundError(f"No such file: '{path}'")
fnode._rw_lock.acquire_write(timeout=lock_timeout)
fnode._rw_lock.acquire_write(timeout=effective_timeout)
handle = MemoryFileHandle(self, fnode, npath, mode)

@@ -220,3 +235,3 @@

fnode = self._create_file(npath)
fnode._rw_lock.acquire_write(timeout=lock_timeout)
fnode._rw_lock.acquire_write(timeout=effective_timeout)
handle = MemoryFileHandle(self, fnode, npath, mode)

@@ -229,3 +244,6 @@

n, promoted, old_quota = fnode.storage.write_at(
current, bytes(preallocate - current), self._quota
current,
bytes(preallocate - current),
self._quota,
self._memory_guard,
)

@@ -517,5 +535,3 @@ if promoted is not None:

def export_tree(
self, prefix: str = "/", only_dirty: bool = False
) -> dict[str, bytes]:
def export_tree(self, prefix: str = "/", only_dirty: bool = False) -> dict[str, bytes]:
return dict(self.iter_export_tree(prefix=prefix, only_dirty=only_dirty))

@@ -567,7 +583,3 @@

node = self._resolve_path(npath)
if (
node is not None
and isinstance(node, FileNode)
and node._rw_lock.is_locked
):
if node is not None and isinstance(node, FileNode) and node._rw_lock.is_locked:
raise BlockingIOError(f"Cannot import: file is open: '{npath}'")

@@ -596,2 +608,3 @@

raise MFSQuotaExceededError(requested=net, available=avail)
self._memory_guard.check_before_write(net)

@@ -606,3 +619,10 @@ written_npaths: list[str] = []

storage = self._create_storage()
storage._bulk_load(data)
try:
storage._bulk_load(data)
except MemoryError:
raise MemoryError(
f"OS memory allocation failed during import_tree "
f"(file: '{npath}', size: {len(data):,} bytes). "
"Consider reducing max_quota or using memory_guard='init'."
) from None
fnode = self._alloc_file(storage)

@@ -662,5 +682,3 @@ fnode.generation = 0

def _ensure_parents(
self, npath: str, created_dirs: list[str] | None = None
) -> None:
def _ensure_parents(self, npath: str, created_dirs: list[str] | None = None) -> None:
parent_path = posixpath.dirname(npath) or "/"

@@ -688,3 +706,5 @@ if self._resolve_path(parent_path) is None:

if data:
n, promoted, old_quota = fnode.storage.write_at(0, data, self._quota)
n, promoted, old_quota = fnode.storage.write_at(
0, data, self._quota, self._memory_guard
)
if promoted is not None:

@@ -715,2 +735,3 @@ fnode.storage = promoted

raise MFSQuotaExceededError(requested=total_data, available=avail)
self._memory_guard.check_before_write(total_data)
# Deep copy the subtree with rollback on failure

@@ -729,5 +750,3 @@ dst_parent, dst_name = dst_pinfo

def _deep_copy_subtree(
self, node: Node, created_node_ids: list[int]
) -> Node:
def _deep_copy_subtree(self, node: Node, created_node_ids: list[int]) -> Node:
if isinstance(node, FileNode):

@@ -848,6 +867,3 @@ # Read data under read lock

next_part = parts[idx + 1]
if (
fnmatch.fnmatch(name, next_part)
and idx + 1 == len(parts) - 1
):
if fnmatch.fnmatch(name, next_part) and idx + 1 == len(parts) - 1:
results.append(child_path)

@@ -870,5 +886,3 @@ else:

def _collect_all_paths(
self, node: DirNode, current_path: str, results: list[str]
) -> None:
def _collect_all_paths(self, node: DirNode, current_path: str, results: list[str]) -> None:
with self._global_lock:

@@ -875,0 +889,0 @@ snapshot = list(node.children.items())

@@ -6,3 +6,3 @@ from __future__ import annotations

import warnings
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any

@@ -13,3 +13,3 @@ if TYPE_CHECKING:

class MemoryFileHandle:
class MemoryFileHandle(io.RawIOBase):
def __init__(

@@ -23,2 +23,3 @@ self,

) -> None:
super().__init__()
self._mfs = mfs

@@ -41,3 +42,3 @@ self._fnode = fnode

def _assert_open(self) -> None:
if self._is_closed:
if self.closed or self._is_closed:
raise ValueError("I/O operation on closed file.")

@@ -61,9 +62,12 @@

def write(self, data: bytes) -> int:
def write(self, data: Any) -> int:
self._assert_open()
self._assert_writable()
if not isinstance(data, (bytes, bytearray, memoryview)):
raise TypeError("a bytes-like object is required")
payload = bytes(data)
if self._is_append:
self._cursor = self._fnode.storage.get_size()
n, promoted, old_quota = self._fnode.storage.write_at(
self._cursor, data, self._mfs._quota
self._cursor, payload, self._mfs._quota, self._mfs._memory_guard
)

@@ -79,2 +83,11 @@ if promoted is not None:

def readinto(self, buffer: Any) -> int:
self._assert_open()
self._assert_readable()
view = memoryview(buffer).cast("B")
data = self.read(len(view))
n = len(data)
view[:n] = data
return n
def seek(self, offset: int, whence: int = 0) -> int:

@@ -113,3 +126,3 @@ self._assert_open()

before = self._fnode.storage.get_size()
self._fnode.storage.truncate(target, self._mfs._quota)
self._fnode.storage.truncate(target, self._mfs._quota, self._mfs._memory_guard)
if self._cursor > target:

@@ -139,5 +152,4 @@ self._cursor = target

def close(self) -> None:
if self._is_closed:
if self.closed or self._is_closed:
return
self._is_closed = True
mode = self._mode

@@ -148,2 +160,4 @@ if mode in ("wb", "ab", "r+b", "xb"):

self._fnode._rw_lock.release_read()
super().close()
self._is_closed = True

@@ -153,7 +167,7 @@ def __enter__(self) -> MemoryFileHandle:

def __exit__(self, *args) -> None:
def __exit__(self, *args: object) -> None:
self.close()
def __del__(self) -> None:
if not self._is_closed:
if not getattr(self, "_is_closed", True) and not getattr(self, "closed", True):
warnings.warn(

@@ -160,0 +174,0 @@ "MFS MemoryFileHandle was not closed properly. "

import threading
from collections.abc import Iterator
from contextlib import contextmanager

@@ -14,3 +15,3 @@

@contextmanager
def reserve(self, size: int):
def reserve(self, size: int) -> Iterator[None]:
if size <= 0:

@@ -17,0 +18,0 @@ yield

@@ -9,2 +9,3 @@ """MFSTextHandle: bufferless text I/O helper.

import codecs
from collections.abc import Iterator

@@ -47,2 +48,3 @@ from typing import TYPE_CHECKING

self._errors = errors
self._decoded_buffer = ""

@@ -83,10 +85,46 @@ @property

Maximum number of characters to read. ``-1`` reads everything.
Note that this is an approximation in characters, not bytes.
"""
if size < 0:
raw = self._handle.read()
else:
raw = self._handle.read(size)
return raw.decode(self._encoding, self._errors)
if self._decoded_buffer:
prefix = self._decoded_buffer
self._decoded_buffer = ""
return prefix + raw.decode(self._encoding, self._errors)
return raw.decode(self._encoding, self._errors)
if size == 0:
return ""
parts: list[str] = []
remaining = size
if self._decoded_buffer:
take = self._decoded_buffer[:remaining]
parts.append(take)
self._decoded_buffer = self._decoded_buffer[len(take) :]
remaining -= len(take)
if remaining == 0:
return "".join(parts)
decoder = codecs.getincrementaldecoder(self._encoding)(errors=self._errors)
while remaining > 0:
chunk = self._handle.read(1)
if not chunk:
tail = decoder.decode(b"", final=True)
if tail:
take = tail[:remaining]
parts.append(take)
self._decoded_buffer = tail[len(take) :] + self._decoded_buffer
break
decoded = decoder.decode(chunk, final=False)
if not decoded:
continue
take = decoded[:remaining]
parts.append(take)
remaining -= len(take)
if len(decoded) > len(take):
self._decoded_buffer = decoded[len(take) :] + self._decoded_buffer
break
return "".join(parts)
def readline(self, limit: int = -1) -> str:

@@ -100,24 +138,22 @@ """Read one line.

limit:
Maximum number of bytes to read (``-1`` means unlimited).
Maximum number of characters to read (``-1`` means unlimited).
"""
buf = bytearray()
chars: list[str] = []
while True:
if limit >= 0 and len(buf) >= limit:
if limit >= 0 and len(chars) >= limit:
break
b = self._handle.read(1)
if not b:
ch = self.read(1)
if not ch:
break
buf.extend(b)
if b == b"\n":
chars.append(ch)
if ch == "\n":
break
if b == b"\r":
# Peek at the next byte to determine \r\n vs bare \r
next_b = self._handle.read(1)
if next_b == b"\n":
buf.extend(next_b)
elif next_b:
# Bare \r: seek back one byte
self._handle.seek(self._handle.tell() - 1)
if ch == "\r":
next_ch = self.read(1)
if next_ch == "\n":
chars.append(next_ch)
elif next_ch:
self._decoded_buffer = next_ch + self._decoded_buffer
break
return buf.decode(self._encoding, self._errors)
return "".join(chars)

@@ -124,0 +160,0 @@ def __iter__(self) -> Iterator[str]:

Metadata-Version: 2.4
Name: D-MemFS
Version: 0.2.1
Version: 0.3.0
Summary: In-process virtual filesystem with hard quota for Python

@@ -104,2 +104,6 @@ Author: D

- `chunk_overhead_override` (default `None`): override the per-chunk overhead estimate used for quota accounting
- `default_lock_timeout` (default `30.0`): default timeout in seconds for file-lock acquisition during `open()`. Use `None` to wait indefinitely.
- `memory_guard` (default `"none"`): physical memory protection mode — `"none"` / `"init"` / `"per_write"`
- `memory_guard_action` (default `"warn"`): action when the guard triggers — `"warn"` (`ResourceWarning`) / `"raise"` (`MemoryError`)
- `memory_guard_interval` (default `1.0`): minimum seconds between OS memory queries (`"per_write"` only)

@@ -109,7 +113,49 @@ > **Note:** The `BytesIO` returned by `export_as_bytesio()` is outside quota management.

> **Note — Quota and free-threaded Python:**
> The per-chunk overhead estimate used for quota accounting is calibrated at import time
> via `sys.getsizeof()`. Free-threaded Python (3.13t, `PYTHON_GIL=0`) has larger object
> headers than the standard build, so `CHUNK_OVERHEAD_ESTIMATE` is higher (~117 bytes vs
> ~93 bytes on CPython 3.13). This means the same `max_quota` yields slightly less
> effective storage capacity on free-threaded builds, especially for workloads with many
> small files or small appends. This is not a bug — it reflects real memory consumption.
> To ensure consistent behaviour across builds, use `chunk_overhead_override` to pin the
> value, or inspect `stats()["overhead_per_chunk_estimate"]` at runtime.
Supported binary modes: `rb`, `wb`, `ab`, `r+b`, `xb`
## Memory Guard
MFS enforces a logical quota, but that quota can still be configured larger than the
currently available physical RAM. `memory_guard` provides an optional safety net.
```python
from dmemfs import MemoryFileSystem
# Warn if max_quota exceeds available RAM
mfs = MemoryFileSystem(max_quota=8 * 1024**3, memory_guard="init")
# Raise MemoryError before writes when RAM is insufficient
mfs = MemoryFileSystem(
max_quota=8 * 1024**3,
memory_guard="per_write",
memory_guard_action="raise",
)
```
| Mode | Initialization | Each Write | Overhead |
|---|---|---|---|
| `"none"` | — | — | Zero |
| `"init"` | Check once | — | Negligible |
| `"per_write"` | Check once | Cached check | About 1 OS call/sec |
When `memory_guard_action="warn"`, the guard emits `ResourceWarning` and allows the operation to continue.
When `memory_guard_action="raise"`, the guard rejects the operation with `MemoryError` before the actual allocation path.
`AsyncMemoryFileSystem` accepts the same constructor parameters and forwards them to the synchronous implementation.
### `MemoryFileHandle`
- `io.RawIOBase`-compatible binary handle
- `read`, `write`, `seek`, `tell`, `truncate`, `flush`, `close`
- `readinto`
- file-like capability checks: `readable`, `writable`, `seekable`

@@ -151,3 +197,3 @@

`MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`.
`MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. `read(size)` counts characters, not bytes, so multibyte text can be read safely without splitting code points. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`.

@@ -389,4 +435,5 @@ ---

- [Architecture Spec v13](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v13.md) — API design, internal structure, CI matrix
- [Detailed Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec.md) — component-level design and rationale
- [Test Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test.md) — test case table and pseudocode
- [Architecture Spec v14](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v14.md) — MemoryGuard-integrated architecture spec
- [Detailed Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_v2.md) — component-level design and rationale
- [Test Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test_v2.md) — test case table and pseudocode

@@ -393,0 +440,0 @@ > These documents are written in Japanese and serve as internal design references.

@@ -7,3 +7,3 @@ [build-system]

name = "D-MemFS"
version = "0.2.1"
version = "0.3.0"
description = "In-process virtual filesystem with hard quota for Python"

@@ -10,0 +10,0 @@ readme = "README.md"

@@ -81,2 +81,6 @@ # D-MemFS

- `chunk_overhead_override` (default `None`): override the per-chunk overhead estimate used for quota accounting
- `default_lock_timeout` (default `30.0`): default timeout in seconds for file-lock acquisition during `open()`. Use `None` to wait indefinitely.
- `memory_guard` (default `"none"`): physical memory protection mode — `"none"` / `"init"` / `"per_write"`
- `memory_guard_action` (default `"warn"`): action when the guard triggers — `"warn"` (`ResourceWarning`) / `"raise"` (`MemoryError`)
- `memory_guard_interval` (default `1.0`): minimum seconds between OS memory queries (`"per_write"` only)

@@ -86,7 +90,49 @@ > **Note:** The `BytesIO` returned by `export_as_bytesio()` is outside quota management.

> **Note — Quota and free-threaded Python:**
> The per-chunk overhead estimate used for quota accounting is calibrated at import time
> via `sys.getsizeof()`. Free-threaded Python (3.13t, `PYTHON_GIL=0`) has larger object
> headers than the standard build, so `CHUNK_OVERHEAD_ESTIMATE` is higher (~117 bytes vs
> ~93 bytes on CPython 3.13). This means the same `max_quota` yields slightly less
> effective storage capacity on free-threaded builds, especially for workloads with many
> small files or small appends. This is not a bug — it reflects real memory consumption.
> To ensure consistent behaviour across builds, use `chunk_overhead_override` to pin the
> value, or inspect `stats()["overhead_per_chunk_estimate"]` at runtime.
Supported binary modes: `rb`, `wb`, `ab`, `r+b`, `xb`
## Memory Guard
MFS enforces a logical quota, but that quota can still be configured larger than the
currently available physical RAM. `memory_guard` provides an optional safety net.
```python
from dmemfs import MemoryFileSystem
# Warn if max_quota exceeds available RAM
mfs = MemoryFileSystem(max_quota=8 * 1024**3, memory_guard="init")
# Raise MemoryError before writes when RAM is insufficient
mfs = MemoryFileSystem(
max_quota=8 * 1024**3,
memory_guard="per_write",
memory_guard_action="raise",
)
```
| Mode | Initialization | Each Write | Overhead |
|---|---|---|---|
| `"none"` | — | — | Zero |
| `"init"` | Check once | — | Negligible |
| `"per_write"` | Check once | Cached check | About 1 OS call/sec |
When `memory_guard_action="warn"`, the guard emits `ResourceWarning` and allows the operation to continue.
When `memory_guard_action="raise"`, the guard rejects the operation with `MemoryError` before the actual allocation path.
`AsyncMemoryFileSystem` accepts the same constructor parameters and forwards them to the synchronous implementation.
### `MemoryFileHandle`
- `io.RawIOBase`-compatible binary handle
- `read`, `write`, `seek`, `tell`, `truncate`, `flush`, `close`
- `readinto`
- file-like capability checks: `readable`, `writable`, `seekable`

@@ -128,3 +174,3 @@

`MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`.
`MFSTextHandle` is a thin, bufferless wrapper. It encodes on `write()` and decodes on `read()` / `readline()`. `read(size)` counts characters, not bytes, so multibyte text can be read safely without splitting code points. Unlike `io.TextIOWrapper`, it introduces no buffering issues when used with `MemoryFileHandle`.

@@ -366,4 +412,5 @@ ---

- [Architecture Spec v13](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v13.md) — API design, internal structure, CI matrix
- [Detailed Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec.md) — component-level design and rationale
- [Test Design Spec](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test.md) — test case table and pseudocode
- [Architecture Spec v14](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/spec_v14.md) — MemoryGuard-integrated architecture spec
- [Detailed Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_v2.md) — component-level design and rationale
- [Test Design Spec v2](https://github.com/nightmarewalker/D-MemFS/blob/main/docs/design/DetailedDesignSpec_test_v2.md) — test case table and pseudocode

@@ -370,0 +417,0 @@ > These documents are written in Japanese and serve as internal design references.