@nxtedition/shared
Advanced tools
+4
-4
@@ -26,6 +26,6 @@ export interface BufferRegion { | ||
| #private; | ||
| get stats(): { | ||
| get stats(): Readonly<{ | ||
| readCount: number; | ||
| readBytes: number; | ||
| }; | ||
| }>; | ||
| get handle(): SharedHandle; | ||
@@ -41,3 +41,3 @@ constructor(handleOrSize: SharedHandle | number); | ||
| #private; | ||
| get stats(): { | ||
| get stats(): Readonly<{ | ||
| yieldCount: number; | ||
@@ -47,3 +47,3 @@ yieldTime: number; | ||
| writeBytes: number; | ||
| }; | ||
| }>; | ||
| get handle(): SharedHandle; | ||
@@ -50,0 +50,0 @@ get maxMessageSize(): number; |
+96
-77
@@ -16,7 +16,7 @@ // By placing the read and write indices far apart (multiples of a common | ||
| // [4] PTR_HI upper 32 bits of native ptr | ||
| // [5..6] PTR_CHECK 64-bit hash of the pointer (corruption guard) | ||
| // [5..6] PTR_CHECK 64-bit hash of the pointer (corruption guard) | ||
| // [7..15] padding | ||
| // | ||
| // Cache line 1 (bytes 64–127): | ||
| // [16] READ_INDEX ─ cache-line aligned (written by Reader) | ||
| // [16] READ_INDEX ─ cache-line aligned (written by Reader) | ||
| // [17..31] padding | ||
@@ -34,4 +34,2 @@ // | ||
| const isProduction = process.env.NODE_ENV === 'production' | ||
@@ -119,3 +117,3 @@ | ||
| // Max size: data region must fit in a signed 32-bit index. | ||
| if (dataSize > 2 ** 31) { | ||
| if (dataSize > 2 ** 31 - 1) { | ||
| throw new RangeError('size exceeds maximum of 2GB') | ||
@@ -134,3 +132,3 @@ } | ||
| #state | ||
| #size | ||
| #mask | ||
| #int32 | ||
@@ -147,3 +145,3 @@ #data | ||
| get stats() { | ||
| get stats() { | ||
| return this.#stats | ||
@@ -167,4 +165,8 @@ } | ||
| if (size === 0 || (size & (size - 1)) !== 0) { | ||
| throw new Error(`Buffer size must be a power of two, got ${size}`) | ||
| } | ||
| this.#state = new Int32Array(this.#handle) | ||
| this.#size = size | ||
| this.#mask = size - 1 | ||
| this.#int32 = new Int32Array(dataBuffer) | ||
@@ -194,6 +196,9 @@ | ||
| const int32 = this.#int32 | ||
| const size = this.#size | ||
| const mask = this.#mask | ||
| const data = this.#data | ||
| if (this.#readPos === this.#writePos) { | ||
| let readPos = this.#readPos | 0 | ||
| let writePos = this.#writePos | 0 | ||
| if (readPos === writePos) { | ||
| // Intentional non-atomic plain read (see TSO note at top of file). | ||
@@ -205,20 +210,16 @@ // A stale WRITE_INDEX at worst means returning 0 messages; the reader | ||
| // propagated yet (property 2). | ||
| this.#writePos = state[WRITE_INDEX] | 0 | ||
| writePos = state[WRITE_INDEX] | 0 | ||
| } | ||
| // Process messages in a batch to minimize loop and atomic operation overhead. | ||
| while (bytes < HWM_BYTES && this.#readPos !== this.#writePos) { | ||
| const dataLen = int32[this.#readPos >> 2] | 0 | ||
| if (!isProduction && (dataLen < 0 || dataLen > size - 8)) { | ||
| throw new Error( | ||
| `Corrupt ring buffer: invalid message length ${dataLen} at position ${this.#readPos}`, | ||
| ) | ||
| } | ||
| while (bytes < HWM_BYTES && readPos !== writePos) { | ||
| const dataLen = int32[readPos >> 2] | 0 | ||
| const alignedLen = (dataLen + 3) & ~3 | ||
| const dataPos = this.#readPos + 4 | ||
| const dataPos = readPos + 4 | ||
| // Advance read position with modulo wrap — the double-mapped virtual | ||
| // buffer makes the data access at dataPos physically contiguous even | ||
| // when it spans the end of the logical ring. | ||
| this.#readPos = (this.#readPos + 4 + alignedLen) % size | ||
| // Advance read position with bitwise AND wrap — the buffer size is | ||
| // always a power of two, so (pos & mask) === (pos % size). The | ||
| // double-mapped virtual buffer makes the data access at dataPos | ||
| // physically contiguous even when it spans the end of the logical ring. | ||
| readPos = (readPos + 4 + alignedLen) & mask | ||
@@ -249,6 +250,6 @@ bytes += 4 + alignedLen | ||
| // to keep the reader hot path free of memory-barrier overhead. | ||
| if (bytes > 0) { | ||
| state[READ_INDEX] = this.#readPos | 0 | ||
| } | ||
| state[READ_INDEX] = readPos | ||
| this.#readPos = readPos | ||
| this.#writePos = writePos | ||
| this.#stats.readCount += count | ||
@@ -267,2 +268,3 @@ this.#stats.readBytes += bytes | ||
| #size | ||
| #mask | ||
| #int32 | ||
@@ -287,3 +289,8 @@ #data | ||
| get stats() { | ||
| get stats() | ||
| { | ||
| return this.#stats | ||
@@ -314,4 +321,9 @@ } | ||
| if (size === 0 || (size & (size - 1)) !== 0) { | ||
| throw new Error(`Buffer size must be a power of two, got ${size}`) | ||
| } | ||
| this.#state = new Int32Array(this.#handle) | ||
| this.#size = size | ||
| this.#mask = size - 1 | ||
| this.#int32 = new Int32Array(dataBuffer) | ||
@@ -391,6 +403,8 @@ | ||
| const size = this.#size | ||
| const mask = this.#mask | ||
| const state = this.#state | ||
| // Fast path: check with the locally cached readPos first. | ||
| const usedCached = (this.#writePos - this.#readPos + size) % size | ||
| // Buffer size is always a power of two, so (x & mask) === (x % size). | ||
| const usedCached = (this.#writePos - this.#readPos + size) & mask | ||
| if (size - usedCached > required) { | ||
@@ -402,11 +416,10 @@ return true | ||
| // Atomics.load is required here: If the optimizer believes reading state[READ_INDEX] | ||
| // has no observable side effects, then it's free to rewrite it to the semantically | ||
| // equivalent: | ||
| // | ||
| // this.#readPos = state[READ_INDEX] | 0 | ||
| // const used = (this.#writePos - (state[READ_INDEX] | 0) + size) % size | ||
| // A plain read is safe here: the JIT could theoretically re-read | ||
| // state[READ_INDEX] for the `used` calculation below, seeing a newer | ||
| // value than what was cached in #readPos. That's harmless — readPos | ||
| // only advances, so a newer value means *more* free space, and the | ||
| // stale #readPos just makes the next fast-path check slightly pessimistic. | ||
| this.#readPos = state[READ_INDEX] | 0 | ||
| this.#readPos = Atomics.load(state, READ_INDEX) | 0 | ||
| const used = (this.#writePos - this.#readPos + size) % size | ||
| const used = (this.#writePos - this.#readPos + size) & mask | ||
| return size - used > required | ||
@@ -421,2 +434,3 @@ } | ||
| const data = this.#data | ||
| const stats = this.#stats | ||
@@ -431,19 +445,20 @@ data.byteOffset = dataPos | ||
| if (!Number.isFinite(dataLen)) { | ||
| throw new TypeError('"fn" must return the number of bytes written') | ||
| if (typeof dataLen !== 'number' || !Number.isInteger(dataLen)) { | ||
| throw new TypeError('"fn" must return an integer') | ||
| } | ||
| if (dataLen < 0) { | ||
| throw new RangeError(`"fn" returned a negative number ${dataLen}`) | ||
| throw new RangeError(`"fn" returned a non-positive integer ${dataLen}`) | ||
| } | ||
| if (dataLen > dataCap) { | ||
| throw new RangeError(`"fn" returned a number ${dataLen} that exceeds capacity ${dataCap}`) | ||
| throw new RangeError(`"fn" returned an integer ${dataLen} that exceeds capacity ${dataCap}`) | ||
| } | ||
| const alignedLen = (dataLen + 3) & ~3 | ||
| // Advance write position with modulo wrap — the double-mapped virtual | ||
| // buffer ensures the data bytes are physically contiguous even when they | ||
| // span the end of the logical ring. | ||
| const nextPos = (this.#writePos + 4 + alignedLen) % this.#size | ||
| // Advance write position with bitwise AND wrap — the buffer size is | ||
| // always a power of two, so (pos & mask) === (pos % size). The | ||
| // double-mapped virtual buffer ensures the data bytes are physically | ||
| // contiguous even when they span the end of the logical ring. | ||
| const nextPos = (this.#writePos + 4 + alignedLen) & this.#mask | ||
| if (!isProduction && nextPos === this.#readPos) { | ||
| if (nextPos === this.#readPos) { | ||
| // assertion | ||
@@ -458,4 +473,4 @@ throw new Error(`Write position ${nextPos} cannot equal read position ${this.#readPos}`) | ||
| this.#stats.writeCount += 1 | ||
| this.#stats.writeBytes += dataLen | ||
| stats.writeCount += 1 | ||
| stats.writeBytes += dataLen | ||
@@ -475,2 +490,31 @@ // This is the "corking" optimization. Instead of calling Atomics.store | ||
| #waitForSpace(len ) { | ||
| const startTime = performance.now() | ||
| this.#yield(0) | ||
| let yieldCount = 0 | ||
| let yieldTime = 0 | ||
| for (let n = 0; !this.#acquire(len); n++) { | ||
| if (performance.now() - startTime > 60e3) { | ||
| throw new Error('Timeout while waiting for space in the buffer') | ||
| } | ||
| this.#yield(3) | ||
| yieldCount += 1 | ||
| yieldTime += 3 | ||
| } | ||
| if (yieldCount > 0) { | ||
| const elapsedTime = performance.now() - startTime | ||
| this.#logger?.warn( | ||
| { | ||
| yieldLength: len, | ||
| readPos: this.#readPos, | ||
| writePos: this.#writePos, | ||
| elapsedTime, | ||
| yieldCount, | ||
| yieldTime, | ||
| }, | ||
| 'yielded', | ||
| ) | ||
| } | ||
| } | ||
| /** | ||
@@ -489,5 +533,4 @@ * Synchronously writes a message. Blocks (via `Atomics.wait`) until buffer space is available. | ||
| } | ||
| const maxLen = this.#size - 8 | ||
| if (len > maxLen) { | ||
| throw new Error(`"len" ${len} exceeds maximum allowed size ${maxLen}`) | ||
| if (len > this.#size - 8) { | ||
| throw new Error(`"len" ${len} exceeds maximum allowed size ${this.#size - 8}`) | ||
| } | ||
@@ -499,26 +542,3 @@ if (typeof fn !== 'function') { | ||
| if (!this.#acquire(len)) { | ||
| const startTime = performance.now() | ||
| this.#yield(0) | ||
| let yieldCount = 0 | ||
| let yieldTime = 0 | ||
| for (let n = 0; !this.#acquire(len); n++) { | ||
| if (performance.now() - startTime > 60e3) { | ||
| throw new Error('Timeout while waiting for space in the buffer') | ||
| } | ||
| this.#yield(3) | ||
| yieldCount += 1 | ||
| yieldTime += 3 | ||
| } | ||
| const elapsedTime = performance.now() - startTime | ||
| this.#logger?.warn( | ||
| { | ||
| yieldLength: len, | ||
| readPos: this.#readPos, | ||
| writePos: this.#writePos, | ||
| elapsedTime, | ||
| yieldCount, | ||
| yieldTime, | ||
| }, | ||
| 'yielded', | ||
| ) | ||
| this.#waitForSpace(len) | ||
| } | ||
@@ -542,5 +562,4 @@ | ||
| } | ||
| const maxLen = this.#size - 8 | ||
| if (len > maxLen) { | ||
| throw new Error(`"len" ${len} exceeds maximum allowed size ${maxLen}`) | ||
| if (len > this.#size - 8) { | ||
| throw new Error(`"len" ${len} exceeds maximum allowed size ${this.#size - 8}`) | ||
| } | ||
@@ -547,0 +566,0 @@ if (typeof fn !== 'function') { |
+2
-2
| { | ||
| "name": "@nxtedition/shared", | ||
| "version": "5.1.5", | ||
| "version": "5.1.6", | ||
| "type": "module", | ||
@@ -46,3 +46,3 @@ "main": "lib/index.js", | ||
| }, | ||
| "gitHead": "7c9c7457c885c644c7a1e70ef894d4727ce240d6" | ||
| "gitHead": "89b8f8b2f17d887b99aaa72ec72f94b91531f4dd" | ||
| } |
+38
-28
@@ -133,2 +133,6 @@ # @nxtedition/shared | ||
| #### `reader.stats` | ||
| Returns `{ readCount: number, readBytes: number }` — cumulative counters for messages and bytes read since construction. | ||
| #### `reader.readSome(next, opaque?)` | ||
@@ -164,2 +168,10 @@ | ||
| #### `writer.maxMessageSize` | ||
| The maximum payload size (in bytes) for a single write. Equal to the physical ring buffer size minus 8 bytes of internal overhead. | ||
| #### `writer.stats` | ||
| Returns `{ yieldCount: number, yieldTime: number, writeCount: number, writeBytes: number }` — cumulative counters since construction. `yieldCount` and `yieldTime` track how often (and how long) the writer had to wait for the reader to catch up. | ||
| #### `writer.writeSync(len, fn, opaque?)` | ||
@@ -173,3 +185,3 @@ | ||
| Throws on timeout (default: 60000 ms). | ||
| Throws on timeout (60 s). Also throws if the writer yields more than 128 consecutive times without making progress, as a deadlock safeguard. | ||
@@ -184,3 +196,3 @@ #### `writer.tryWrite(len, fn, opaque?)` | ||
| When called with a callback, uncork is called automatically when the callback returns. When called without a callback, you must call `uncork()` manually. | ||
| When called with a callback, uncork is called automatically when the callback returns (even if it throws). The callback form also accepts an optional `opaque` argument: `cork(callback, opaque)` — the opaque value is passed through to the callback to avoid creating a closure. When called without a callback, you must call `uncork()` manually. | ||
@@ -206,38 +218,32 @@ #### `writer.uncork()` | ||
| | Size | shared (buffer) | shared (latin1 str) | postMessage (buffer) | postMessage (string) | | ||
| | -----: | --------------: | ------------------: | -------------------: | -------------------: | | ||
| | 64 B | **2.00 GiB/s** | 549 MiB/s | 22.78 MiB/s | 39.58 MiB/s | | ||
| | 256 B | **3.62 GiB/s** | 1.73 GiB/s | 89.90 MiB/s | 174.62 MiB/s | | ||
| | 1 KiB | **11.23 GiB/s** | 6.00 GiB/s | 341.78 MiB/s | 521.39 MiB/s | | ||
| | 4 KiB | **24.36 GiB/s** | 20.04 GiB/s | 1.13 GiB/s | 1.59 GiB/s | | ||
| | 16 KiB | 44.16 GiB/s | **50.76 GiB/s** | 3.70 GiB/s | 7.45 GiB/s | | ||
| | 64 KiB | **90.02 GiB/s** | 78.88 GiB/s | 9.49 GiB/s | 15.09 GiB/s | | ||
| | Size | shared (buffer) | shared (string) | postMessage (buffer) | postMessage (string) | ratio (buffer) | ratio (string) | | ||
| | ----: | --------------: | --------------: | -------------------: | -------------------: | -------------: | -------------: | | ||
| | 64 B | **2.45 GiB/s** | 732 MiB/s | 25.40 MiB/s | 38.43 MiB/s | ~99x | ~19x | | ||
| | 256 B | **4.57 GiB/s** | 1.98 GiB/s | 89.03 MiB/s | 205.12 MiB/s | ~53x | ~10x | | ||
| | 1 KiB | **12.00 GiB/s** | 6.08 GiB/s | 341.51 MiB/s | 489.41 MiB/s | ~36x | ~13x | | ||
| | 4 KiB | **26.05 GiB/s** | 19.75 GiB/s | 1.18 GiB/s | 1.62 GiB/s | ~22x | ~12x | | ||
| ### Message rate | ||
| | Size | shared (buffer) | shared (latin1 str) | postMessage (buffer) | postMessage (string) | | ||
| | -----: | --------------: | ------------------: | -------------------: | -------------------: | | ||
| | 64 B | **33.61 M/s** | 8.99 M/s | 373 K/s | 648 K/s | | ||
| | 256 B | **15.16 M/s** | 7.27 M/s | 368 K/s | 715 K/s | | ||
| | 1 KiB | **11.78 M/s** | 6.29 M/s | 350 K/s | 534 K/s | | ||
| | 4 KiB | **6.39 M/s** | 5.25 M/s | 297 K/s | 417 K/s | | ||
| | 16 KiB | 2.89 M/s | **3.33 M/s** | 242 K/s | 488 K/s | | ||
| | 64 KiB | **1.47 M/s** | 1.29 M/s | 155 K/s | 247 K/s | | ||
| | Size | shared (buffer) | shared (string) | postMessage (buffer) | postMessage (string) | ratio (buffer) | ratio (string) | | ||
| | ----: | --------------: | --------------: | -------------------: | -------------------: | -------------: | -------------: | | ||
| | 64 B | **41.18 M/s** | 12.00 M/s | 416 K/s | 630 K/s | ~99x | ~19x | | ||
| | 256 B | **19.16 M/s** | 8.30 M/s | 365 K/s | 840 K/s | ~53x | ~10x | | ||
| | 1 KiB | **12.58 M/s** | 6.37 M/s | 350 K/s | 501 K/s | ~36x | ~13x | | ||
| | 4 KiB | **6.83 M/s** | 5.18 M/s | 309 K/s | 424 K/s | ~22x | ~12x | | ||
| ### Key findings | ||
| - **Small messages (64–256 B):** `Buffer.set` delivers **33.6–15.2 M msg/s** — | ||
| up to **90x faster** than `postMessage` (buffer) and **52x faster** than | ||
| - **Small messages (64–256 B):** `Buffer.set` delivers **41.2–19.2 M msg/s** — | ||
| up to **99x faster** than `postMessage` (buffer) and **19x faster** than | ||
| `postMessage` (string). Per-message overhead dominates at these sizes, and | ||
| avoiding structured cloning makes the biggest difference. | ||
| - **Medium messages (1 KiB):** `Buffer.set` pulls ahead at **11.23 GiB/s** — | ||
| roughly **1.9x faster** than the latin1 string path and **~22x faster** than | ||
| - **Medium messages (1 KiB):** `Buffer.set` pulls ahead at **12.00 GiB/s** — | ||
| roughly **2x faster** than the latin1 string path and **~25x faster** than | ||
| the best `postMessage` variant. | ||
| - **Large messages (4–64 KiB):** Both shared paths dominate. At 16 KiB the | ||
| latin1 string path (**50.76 GiB/s**) overtakes `Buffer.set` (44.16 GiB/s) — | ||
| V8's string-to-buffer fast path becomes more cache-efficient at this size. At | ||
| 64 KiB `Buffer.set` reclaims the lead at **90.02 GiB/s** — **6x faster** | ||
| than `postMessage` (string). | ||
| - **Large messages (4–64 KiB):** Both shared paths dominate. At 64 KiB | ||
| `Buffer.set` reaches **47.20 GiB/s** — **~6x faster** than `postMessage`. | ||
| The string path stays within ~10% of `Buffer.set` at all large sizes. | ||
@@ -250,3 +256,7 @@ - **Caveat:** The string benchmark uses ASCII-only content. Multi-byte UTF-8 | ||
| ```sh | ||
| node --allow-natives-syntax packages/shared/src/bench.mjs | ||
| # Local | ||
| node --expose-gc bench.mjs | ||
| # Remote (Docker, with hardware counters) | ||
| DOCKER_HOST=ssh://user@host ./bench.sh | ||
| ``` | ||
@@ -253,0 +263,0 @@ |
+67
-53
@@ -0,1 +1,5 @@ | ||
| #ifdef __linux__ | ||
| #define _GNU_SOURCE 1 | ||
| #endif | ||
| #define NAPI_EXPERIMENTAL | ||
@@ -12,2 +16,17 @@ #include <node_api.h> | ||
| // Round up to the next power of two so that JS can use (pos & mask) instead of | ||
| // (pos % size). The wasted virtual memory is at most 2× the requested size, | ||
| // but physical pages are only faulted in on access (or with MAP_POPULATE). | ||
| static size_t next_pow2(size_t v) | ||
| { | ||
| v--; | ||
| v |= v >> 1; | ||
| v |= v >> 2; | ||
| v |= v >> 4; | ||
| v |= v >> 8; | ||
| v |= v >> 16; | ||
| v |= v >> 32; | ||
| return v + 1; | ||
| } | ||
| #ifdef _WIN32 | ||
@@ -51,2 +70,3 @@ | ||
| size_t size = (requested_size + align - 1) & ~(align - 1); | ||
| size = next_pow2(size); | ||
@@ -123,3 +143,2 @@ // Reserve a contiguous 2*size placeholder, then split it into two halves. | ||
| #define _GNU_SOURCE 1 | ||
| #include <fcntl.h> | ||
@@ -129,22 +148,8 @@ #include <sys/mman.h> | ||
| #ifndef MAP_ANONYMOUS | ||
| #define MAP_ANONYMOUS MAP_ANON | ||
| #ifndef MAP_NORESERVE | ||
| #define MAP_NORESERVE 0 | ||
| #endif | ||
| #ifdef __linux__ | ||
| #include <sys/syscall.h> | ||
| // Huge-page constants — defined here as fallback for older glibc headers. | ||
| #ifndef MFD_HUGETLB | ||
| #define MFD_HUGETLB 0x0004U | ||
| #endif | ||
| #ifndef MFD_HUGE_2MB | ||
| #define MFD_HUGE_2MB (21U << 26) | ||
| #endif | ||
| #ifndef MAP_HUGETLB | ||
| #define MAP_HUGETLB 0x040000 | ||
| #endif | ||
| #ifndef MAP_HUGE_2MB | ||
| #define MAP_HUGE_2MB (21 << 26) | ||
| #endif | ||
| #include <linux/memfd.h> | ||
| #define RING_HUGE_2MB (2UL * 1024 * 1024) | ||
@@ -154,4 +159,3 @@ | ||
| { | ||
| // MFD_CLOEXEC: prevent the fd from leaking into execve'd child processes. | ||
| int fd = (int)syscall(SYS_memfd_create, "ring", (unsigned)MFD_CLOEXEC); | ||
| int fd = memfd_create("ring", MFD_CLOEXEC); | ||
| if (fd < 0) | ||
@@ -169,5 +173,3 @@ return -1; | ||
| { | ||
| // MFD_CLOEXEC: prevent the fd from leaking into execve'd child processes. | ||
| int fd = | ||
| (int)syscall(SYS_memfd_create, "ring", (unsigned)(MFD_CLOEXEC | MFD_HUGETLB | MFD_HUGE_2MB)); | ||
| int fd = memfd_create("ring", MFD_CLOEXEC | MFD_HUGETLB | MFD_HUGE_2MB); | ||
| if (fd < 0) | ||
@@ -206,7 +208,2 @@ return -1; | ||
| // MAP_POPULATE is Linux-specific; define as 0 on platforms that lack it. | ||
| #ifndef MAP_POPULATE | ||
| #define MAP_POPULATE 0 | ||
| #endif | ||
| struct ring_buf_t | ||
@@ -226,10 +223,14 @@ { | ||
| // Maps fd (already ftruncated to size) into a contiguous 2*size virtual region. | ||
| // extra_flags is ORed into the data mmap calls only (e.g. MAP_HUGETLB|MAP_HUGE_2MB). | ||
| // When huge_pages is true, applies MAP_HUGETLB | MAP_HUGE_2MB to the data mappings (Linux only). | ||
| // Always consumes (closes) fd. | ||
| static ring_buf_t *ring_buf_alloc_from_fd(int fd, size_t size, int extra_flags) | ||
| static ring_buf_t *ring_buf_alloc_from_fd(int fd, size_t size, bool huge_pages) | ||
| { | ||
| // Reserve address space without huge-page flags — the reservation only needs | ||
| // virtual address range; applying MAP_HUGETLB here can cause spurious failures | ||
| // when the huge-page pool is exhausted even though no pages are faulted in yet. | ||
| void *addr = mmap(nullptr, size * 2, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); | ||
| // PROT_NONE: no access permissions — this is a pure address-space reservation. | ||
| // MAP_PRIVATE: changes stay process-private (moot with PROT_NONE, but required by POSIX). | ||
| // MAP_ANONYMOUS: not backed by any file — just reserves virtual address range. | ||
| // MAP_NORESERVE: don't charge swap/overcommit for a placeholder that will never be faulted. | ||
| // No MAP_HUGETLB: the reservation only needs virtual range; huge-page flags here can cause | ||
| // spurious failures when the huge-page pool is exhausted. | ||
| void *addr = | ||
| mmap(nullptr, size * 2, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); | ||
| if (addr == MAP_FAILED) | ||
@@ -241,7 +242,14 @@ { | ||
| // MAP_POPULATE: fault in all pages at mmap time so the hot path never takes a page-fault. | ||
| if (mmap(addr, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED | MAP_POPULATE | extra_flags, | ||
| fd, 0) == MAP_FAILED || | ||
| mmap((char *)addr + size, size, PROT_READ | PROT_WRITE, | ||
| MAP_FIXED | MAP_SHARED | MAP_POPULATE | extra_flags, fd, 0) == MAP_FAILED) | ||
| // MAP_FIXED: overwrite the placeholder reservation at the exact address. | ||
| // MAP_SHARED: both halves must alias the same physical pages (MAP_PRIVATE would COW). | ||
| int flags = MAP_FIXED | MAP_SHARED; | ||
| #ifdef __linux__ | ||
| // MAP_HUGETLB | MAP_HUGE_2MB: use explicit 2 MiB huge-page backing. | ||
| if (huge_pages) | ||
| flags |= MAP_HUGETLB | MAP_HUGE_2MB; | ||
| #else | ||
| (void)huge_pages; | ||
| #endif | ||
| if (mmap(addr, size, PROT_READ | PROT_WRITE, flags, fd, 0) == MAP_FAILED || | ||
| mmap((char *)addr + size, size, PROT_READ | PROT_WRITE, flags, fd, 0) == MAP_FAILED) | ||
| { | ||
@@ -253,2 +261,10 @@ munmap(addr, size * 2); | ||
| #ifdef __linux__ | ||
| // MADV_POPULATE_WRITE faults all pages as writable upfront — avoids the | ||
| // minor page faults that MAP_POPULATE leaves on MAP_SHARED mappings (where | ||
| // MAP_POPULATE only faults as read, and the first write still triggers a | ||
| // fault to mark the page dirty). | ||
| madvise(addr, size * 2, MADV_POPULATE_WRITE); | ||
| #endif | ||
| close(fd); | ||
@@ -275,7 +291,7 @@ | ||
| // Skip if the allocation is much smaller than a huge page to avoid wasting memory. | ||
| size_t huge_size = (requested_size + RING_HUGE_2MB - 1) & ~(RING_HUGE_2MB - 1); | ||
| size_t huge_size = next_pow2(requested_size < RING_HUGE_2MB ? RING_HUGE_2MB : requested_size); | ||
| int huge_fd = (requested_size >= RING_HUGE_2MB / 2) ? open_memfd_huge(huge_size) : -1; | ||
| if (huge_fd >= 0) | ||
| { | ||
| ring_buf_t *rb = ring_buf_alloc_from_fd(huge_fd, huge_size, MAP_HUGETLB | MAP_HUGE_2MB); | ||
| ring_buf_t *rb = ring_buf_alloc_from_fd(huge_fd, huge_size, true); | ||
| if (rb) | ||
@@ -293,3 +309,5 @@ { | ||
| page = 4096; | ||
| // Round up to page boundary first, then to power-of-two. | ||
| size_t size = (requested_size + (size_t)page - 1) & ~((size_t)page - 1); | ||
| size = next_pow2(size); | ||
@@ -300,18 +318,14 @@ int fd = open_memfd(size); | ||
| ring_buf_t *rb = ring_buf_alloc_from_fd(fd, size, 0); | ||
| ring_buf_t *rb = ring_buf_alloc_from_fd(fd, size, false); | ||
| #ifdef MADV_HUGEPAGE | ||
| // Hint the kernel to back this region with transparent huge pages opportunistically. | ||
| #ifdef __linux__ | ||
| if (rb) | ||
| { | ||
| // Hint the kernel to back this region with transparent huge pages opportunistically. | ||
| madvise(rb->addr, rb->size * 2, MADV_HUGEPAGE); | ||
| #endif | ||
| // MADV_DONTFORK: exclude the ring buffer from forked child processes. | ||
| // Without this, fork() marks all pages copy-on-write; a single write in the | ||
| // child triggers physical page duplication even though the child has no use | ||
| // for the ring buffer. Guard with #ifdef because macOS SDK versions vary in | ||
| // whether MADV_DONTFORK is defined. | ||
| #ifdef MADV_DONTFORK | ||
| if (rb) | ||
| // Exclude the ring buffer from forked child processes — without this, | ||
| // fork() marks all pages COW, and a single write in the child triggers | ||
| // physical page duplication even though the child has no use for the ring. | ||
| madvise(rb->addr, rb->size * 2, MADV_DONTFORK); | ||
| } | ||
| #endif | ||
@@ -318,0 +332,0 @@ |
| export {}; |
Sorry, the diff of this file is not supported yet
Environment variable access
Supply chain riskPackage accesses environment variables, which may be a sign of credential stuffing or data theft.
75239
2.5%557
2.01%261
3.98%1
-50%9
-10%