🎩 You're Invited:Meet the Socket team at Black Hat in Las Vegas, August 3-6.RSVP →
Sign In

decmpfs

Package Overview
Dependencies
Maintainers
1
Versions
3
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

decmpfs - cargo Package Compare versions

Comparing version
0.0.0
 to
0.1.0
+6
.cargo_vcs_info.json
{
"git": {
"sha1": "55fb470a720c03e8cf4feb76c600b7cc1a277838"
},
"path_in_vcs": ""
}
+110
.github/workflows/provenance.yml
name: 📦 Publish to npm
# Dependencies: none — self-contained.
#
# The publish step is INLINED (not delegated to a reusable workflow) because npm
# OIDC trusted publishing keys on the workflow file that contains the publish step:
# `job_workflow_ref` must be a stable
# decmpfs/decmpfs/.github/workflows/provenance.yml@<ref> for the trusted-publisher
# config to match. (socket-addon delegates to socket-registry's reusable, but that
# is Socket infra; decmpfs is independent, so it follows socket-lib's inlined
# pattern.)
#
# These are addons (a `.node` dlopen'd into Node), so each platform builds on its
# own matrix host, then the publish job publishes every @decmpfs/<triple> package
# plus the main `decmpfs` package with provenance.
on:
workflow_dispatch:
inputs:
dry-run:
description: 'Dry run (default: true)'
required: false
default: true
type: boolean
dist-tag:
description: 'npm dist-tag (latest, next, beta, …)'
required: false
default: latest
type: string
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false # Don't cancel publishing.
jobs:
build:
name: Build ${{ matrix.triple }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
include:
- { triple: darwin-arm64, runner: macos-14 }
- { triple: darwin-x64, runner: macos-13 }
- { triple: linux-x64-gnu, runner: ubuntu-latest }
- { triple: linux-arm64-gnu, runner: ubuntu-24.04-arm }
- { triple: win32-x64-msvc, runner: windows-latest }
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 (2026-06-29)
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 (2026-06-29)
with:
node-version: 24
- name: Build addon
working-directory: node
run: npm run build
- name: Test addon
working-directory: node
run: npm test
- name: Assemble platform package
working-directory: node
run: node scripts/make-npm-dirs.mts
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 (2026-06-29)
with:
name: ${{ matrix.triple }}
path: node/npm/${{ matrix.triple }}/
if-no-files-found: error
publish:
name: Publish to npm
needs: build
runs-on: ubuntu-latest
timeout-minutes: 15
# Real publishes run in the `release` environment (reviewer gate + a tighter
# OIDC trusted-publisher match); dry runs skip it for friction-free testing.
environment: ${{ inputs.dry-run == 'true' && '' || 'release' }}
permissions:
# Needed for npm provenance via OIDC trusted publishing.
id-token: write
contents: read
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 (2026-06-29)
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 (2026-06-29)
with:
node-version: 24
registry-url: https://registry.npmjs.org/
- name: Use a trusted-publishing-capable npm
run: npm install --global npm@latest
- name: Download platform packages
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 (2026-06-29)
with:
path: node/npm
- name: Publish
working-directory: node
env:
DRY_RUN: ${{ inputs.dry-run }}
DIST_TAG: ${{ inputs.dist-tag }}
run: |
ARGS="--access public --tag ${DIST_TAG:-latest}"
if [ "$DRY_RUN" = "true" ]; then
ARGS="$ARGS --dry-run"
fi
# Platform packages first, then the main package that depends on them.
for dir in npm/*/; do
npm publish "$dir" $ARGS
done
npm publish . $ARGS
+1
.gitignore
/target
+19
assets/coverage-score.svg
<svg xmlns="http://www.w3.org/2000/svg" width="96" height="20" role="img" aria-label="coverage: 97%">
<title>coverage: 97%</title>
<linearGradient id="g" x2="0" y2="100%">
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
<stop offset="1" stop-opacity=".1"/>
</linearGradient>
<clipPath id="r"><rect width="96" height="20" rx="3" fill="#fff"/></clipPath>
<g clip-path="url(#r)">
<rect width="60" height="20" fill="#555"/>
<rect x="60" width="36" height="20" fill="#4c1"/>
<rect width="96" height="20" fill="url(#g)"/>
</g>
<g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110">
<text x="300" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="500">coverage</text>
<text x="300" y="140" transform="scale(.1)" textLength="500">coverage</text>
<text x="780" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="240">97%</text>
<text x="780" y="140" transform="scale(.1)" textLength="240">97%</text>
</g>
</svg>
+19
assets/socket-score.svg
<svg xmlns="http://www.w3.org/2000/svg" width="100" height="20" role="img" aria-label="socket: score">
<title>socket: score</title>
<linearGradient id="g" x2="0" y2="100%">
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
<stop offset="1" stop-opacity=".1"/>
</linearGradient>
<clipPath id="r"><rect width="100" height="20" rx="3" fill="#fff"/></clipPath>
<g clip-path="url(#r)">
<rect width="52" height="20" fill="#555"/>
<rect x="52" width="48" height="20" fill="#5b51d8"/>
<rect width="100" height="20" fill="url(#g)"/>
</g>
<g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110">
<text x="270" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="420">socket</text>
<text x="270" y="140" transform="scale(.1)" textLength="420">socket</text>
<text x="750" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="350">score</text>
<text x="750" y="140" transform="scale(.1)" textLength="350">score</text>
</g>
</svg>
+6
ci/.gitattributes
# Local btrfs-repro harness — infra, not product code. Keep it out of GitHub's
# language bar, and force LF so the shell scripts and Dockerfile stay runnable on
# any checkout.
* linguist-detectable=false
*.sh text eol=lf
Dockerfile text eol=lf
+38
ci/btrfs-loopback.sh
#!/usr/bin/env bash
# Provision real btrfs loopback filesystems and export DECMPFS_BTRFS_DIR /
# DECMPFS_BTRFS_RO_DIR. SOURCE this (it exports), don't exec it.
#
# Single source of truth for the btrfs runtime setup, shared by the local
# ci/Dockerfile harness and (when wired) a CI btrfs job, so the two can't drift.
# Needs a btrfs-capable kernel + root (or sudo).
set -euo pipefail
# sudo when we aren't already root (CI runs as a user; the container runs as root).
sudo_() { if [ "$(id -u)" -eq 0 ]; then "$@"; else sudo "$@"; fi; }
grep -qw btrfs /proc/filesystems || sudo_ modprobe btrfs 2>/dev/null || true
grep -qw btrfs /proc/filesystems || {
echo "FAIL: kernel has no btrfs support"
return 99 2>/dev/null || exit 99
}
# Read-write btrfs. No compress= mount option, so the only compression comes from
# the per-file codec the backend requests via the btrfs.compression property.
truncate -s 400M /tmp/btrfs.img
mkfs.btrfs -q /tmp/btrfs.img
sudo_ mkdir -p /mnt/bt
sudo_ mount -o loop /tmp/btrfs.img /mnt/bt
sudo_ chmod 1777 /mnt/bt
# Read-only btrfs: seed a file, then remount read-only (the fail-soft path).
truncate -s 200M /tmp/btrfs-ro.img
mkfs.btrfs -q /tmp/btrfs-ro.img
sudo_ mkdir -p /mnt/bt-ro
sudo_ mount -o loop /tmp/btrfs-ro.img /mnt/bt-ro
head -c 1048576 /dev/zero | sudo_ tee /mnt/bt-ro/ro.node > /dev/null
sync
sudo_ mount -o remount,ro /mnt/bt-ro
export DECMPFS_BTRFS_DIR=/mnt/bt
export DECMPFS_BTRFS_RO_DIR=/mnt/bt-ro
echo "btrfs ready: $DECMPFS_BTRFS_DIR (rw) + $DECMPFS_BTRFS_RO_DIR (ro)"
+53
ci/Dockerfile
# Local real-btrfs harness for the decmpfs crate. Optimized to the bone:
# * tiny build context (Dockerfile.dockerignore drops target/ etc.),
# * BuildKit cache mounts for the cargo registry + target (incremental rebuilds),
# * the btrfs test is built as a fully-static, upx-packed musl binary, and
# * the runtime is FROM scratch: just static busybox (shell + mount/modprobe),
# mkfs.btrfs with its ldd'd libs, and the test binary — no OS rootfs.
# Build (context = the crate dir):
# docker build -f ci/Dockerfile -t decmpfs-btrfs .
# Run (needs --privileged to loop-mount btrfs):
# docker run --rm --privileged decmpfs-btrfs
# syntax=docker/dockerfile:1
FROM rust:1-alpine AS builder
# binutils(strip) + upx pack the binary; btrfs-progs + busybox-static are harvested
# into the scratch rootfs below. No gcc/musl-dev: the crate's only dep is libc
# (pure FFI, no C build), so the alpine/musl `cargo test` needs no C toolchain.
RUN apk add --no-cache binutils upx btrfs-progs busybox-static
WORKDIR /crate
COPY Cargo.toml ./
COPY src ./src
COPY tests ./tests
COPY ci/btrfs-loopback.sh ci/run.sh /src-ci/
# Only the btrfs integration test belongs here (unit tests run on 3 OSes in CI).
# Collect just this build's binary via cargo's JSON message stream.
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/crate/target \
mkdir /bins \
&& cargo test --no-run --test btrfs --message-format=json \
| grep -o '"executable":"[^"]*"' | cut -d'"' -f4 \
| while read -r bin; do cp "$bin" /bins/; done \
&& strip /bins/* && upx --best /bins/*
# Assemble a minimal rootfs for scratch: static busybox + symlinks for the applets
# the scripts call (the rest are sh builtins), mkfs.btrfs with its shared libs, the
# binary, and the scripts.
RUN set -eu; R=/rootfs; \
mkdir -p "$R/bin" "$R/sbin" "$R/ci" "$R/bins" "$R/tmp" "$R/mnt" "$R/proc" "$R/sys" "$R/dev"; \
cp /bin/busybox.static "$R/bin/busybox"; \
for applet in sh grep id mkdir chmod head tee sync mount umount modprobe basename truncate; do \
ln -s busybox "$R/bin/$applet"; \
done; \
mkfs=$(command -v mkfs.btrfs); cp "$mkfs" "$R/sbin/mkfs.btrfs"; \
for lib in $(ldd "$mkfs" | grep -oE '/[^ ]+\.so[^ ]*' | sort -u); do \
mkdir -p "$R$(dirname "$lib")"; cp "$lib" "$R$lib"; \
done; \
cp /src-ci/btrfs-loopback.sh /src-ci/run.sh "$R/ci/"; chmod +x "$R/ci/run.sh"; \
cp /bins/* "$R/bins/"
FROM scratch
COPY --from=builder /rootfs /
ENV PATH=/bin:/sbin:/usr/sbin
CMD ["/bin/sh", "/ci/run.sh"]
+8
ci/Dockerfile.dockerignore
# Context is the crate dir; only Cargo.toml, src/, tests/, ci/*.sh are COPYd.
target/
**/*.node
**/.DS_Store
ci/Dockerfile
ci/Dockerfile.dockerignore
ci/README.md
ci/.gitattributes
+13
ci/run.sh
#!/bin/sh
# Runtime entry for the slim harness image: provision btrfs (shared with CI) and
# run the prebuilt decmpfs test binaries against it. No cargo in this stage.
# POSIX sh so the runtime needs only busybox ash, not bash.
set -eu
# shellcheck source=./btrfs-loopback.sh
. /ci/btrfs-loopback.sh
for bin in /bins/*; do
echo "== $(basename "$bin") =="
"$bin" --nocapture --test-threads=1
done
+21
LICENSE
MIT License
Copyright (c) 2026 John-David Dalton
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+58
README.md
# decmpfs
![coverage score](assets/coverage-score.svg) [![Socket score](assets/socket-score.svg)](https://socket.dev/cargo/package/decmpfs)
Apply the operating system's **transparent per-file filesystem compression** to a
file — smaller on disk, byte-identical on read, decompressed by the kernel at
near-native speed. macOS APFS (decmpfs/LZVN), Linux btrfs (zstd→lzo→zlib), Windows
NTFS (LZNT1).
The core does the bytes-to-compressed-file write in **one pass** — it never writes a
file then reads it back to recompress.
```rust
use decmpfs::{compress_bytes, compress_file, Gate};
// Write `content` straight to an OS-compressed file (single pass).
let outcome = compress_bytes(path, &content, &Gate::any())?;
// Or compress a file that already exists, in place.
let outcome = compress_file(path)?;
```
## Outcome, never a surprise error
Every call returns an `Outcome` — `Compressed`, `NoGain` (incompressible /
sub-cluster), `AlreadyCompressed`, `Unsupported` (the FS has no per-file
compression — ext4, xfs, ZFS, ReFS, FAT, tmpfs, network mounts), or `Skipped`
(permission, lock, gate). An unsupported filesystem or a permission issue is a
non-fatal `Outcome`, not an `Err`; `Err` is reserved for genuine I/O failures.
## Gate
`Gate` decides which files to compress by glob and/or size:
```rust
let gate = Gate::new(Some("**/*.node"), Some(">= 1MB"))?;
```
## Codecs (speed over ratio — a file is written once, read on load)
- **macOS:** LZVN (the fastest decmpfs codec to decompress).
- **Linux btrfs:** zstd, falling back to lzo then zlib on older kernels.
- **Windows NTFS:** LZNT1 (survives a reinstall's open-for-write, unlike WOF).
## Features
The core is dependency-light (`libc` / `windows-sys` only). The optional `addon`
feature pulls `zstd` + `sha2` to unwrap a napi `--compress` hybrid `.node` back to
the raw addon before compressing.
## Node
An N-API binding (`writeDecmpfsFile` / `writeDecmpfsFileSync`, fs.writeFile-shaped,
atomic by default) lives in [`node/`](node/).
## License
MIT
+428
src/addon.rs
//! `addon` feature — unwrap a napi `--compress` hybrid back to the raw `.node`.
//!
//! A binpress hybrid carries the original addon, zstd-compressed, inside a
//! `PRESSED_DATA` SIGNABLE SECTION (Mach-O `__PRESSED_DATA` in segment `SMOL`, ELF
//! `.PRESSED_DATA`, PE `.PRESSED` — read from the binary's SECTION HEADERS, NOT an
//! EOF footer). The section content is the bin-infra pressed-data blob:
//!
//! ```text
//! [magic marker 32B] "__SMOL_PRESSED_DATA_MAGIC_MARKER"
//! [compressed u64 LE] zstd payload length
//! [uncompressed u64 LE] raw addon length
//! [cache key 16B] hex, ignored here
//! [platform 3B ] platform/arch/libc, ignored here
//! [integrity 64B] SHA-512 of the zstd payload
//! [has_config 1B ] 0/1
//! [config 1192B] only if has_config == 1
//! [payload compressed bytes] zstd frame
//! ```
//!
//! `unwrap_if_hybrid(&content)` returns `Some(raw addon)` for a hybrid whose payload
//! zstd-decodes to `uncompressed` bytes and matches the SHA-512, else `None`. The
//! PM helper then feeds the raw addon to `compress_bytes`. This is the
//! mirror-image ABI of `socket-btm/packages/bin-infra` (`smol_segment_reader.c` /
//! `compression_constants.h`).
use sha2::{Digest, Sha512};
/// "__SMOL_PRESSED_DATA_MAGIC_MARKER" (SMOL_PRESSED_DATA_MAGIC_MARKER).
const MAGIC_MARKER: &[u8; 32] = b"__SMOL_PRESSED_DATA_MAGIC_MARKER";
const SIZE_HEADER_LEN: usize = 16; // compressed u64 + uncompressed u64
const CACHE_KEY_LEN: usize = 16;
const PLATFORM_METADATA_LEN: usize = 3;
const INTEGRITY_HASH_LEN: usize = 64; // SHA-512
const SMOL_CONFIG_FLAG_LEN: usize = 1;
const SMOL_CONFIG_BINARY_LEN: usize = 1192;
/// Fixed header length up to and including the has-config flag.
const HEADER_LEN: usize = MAGIC_MARKER.len()
+ SIZE_HEADER_LEN
+ CACHE_KEY_LEN
+ PLATFORM_METADATA_LEN
+ INTEGRITY_HASH_LEN
+ SMOL_CONFIG_FLAG_LEN;
/// Refuse a decompressed-size claim past this (DoS guard, matches bin-infra's 512 MB).
const MAX_DECOMPRESSED: u64 = 512 * 1024 * 1024;
/// If `content` is a napi `--compress` hybrid, decode its embedded addon and return
/// the raw `.node` bytes; otherwise `None`. Integrity-checked (SHA-512 over the zstd
/// payload) — a hybrid that fails verification returns `None`, never partial bytes.
pub fn unwrap_if_hybrid(content: &[u8]) -> Option<Vec<u8>> {
let section = find_pressed_data_section(content)?;
decode_pressed_data(section)
}
/// Parse the bin-infra pressed-data blob (magic + header + zstd payload) into the
/// raw addon. Split from section-finding so the format round-trips in a unit test
/// without synthesizing a whole Mach-O/ELF/PE.
pub fn decode_pressed_data(section: &[u8]) -> Option<Vec<u8>> {
if section.len() < HEADER_LEN {
return None;
}
if &section[..MAGIC_MARKER.len()] != MAGIC_MARKER.as_slice() {
return None;
}
let mut at = MAGIC_MARKER.len();
let compressed_size = read_u64_le(section, at)?;
at += 8;
let uncompressed_size = read_u64_le(section, at)?;
at += 8;
// Skip cache key + platform metadata.
at += CACHE_KEY_LEN + PLATFORM_METADATA_LEN;
let integrity = section.get(at..at + INTEGRITY_HASH_LEN)?;
let mut hash = [0u8; INTEGRITY_HASH_LEN];
hash.copy_from_slice(integrity);
at += INTEGRITY_HASH_LEN;
let has_config = *section.get(at)?;
at += SMOL_CONFIG_FLAG_LEN;
if has_config != 0 {
at = at.checked_add(SMOL_CONFIG_BINARY_LEN)?;
}
if compressed_size == 0
|| uncompressed_size == 0
|| uncompressed_size > MAX_DECOMPRESSED
|| compressed_size > MAX_DECOMPRESSED
{
return None;
}
let payload = section.get(at..at.checked_add(compressed_size as usize)?)?;
// Integrity: SHA-512 of the zstd payload, BEFORE decompressing (reject a
// tampered frame up front).
let mut hasher = Sha512::new();
hasher.update(payload);
if hasher.finalize().as_slice() != hash {
return None;
}
let raw = zstd::stream::decode_all(payload).ok()?;
if raw.len() as u64 != uncompressed_size {
return None;
}
Some(raw)
}
fn read_u64_le(buf: &[u8], at: usize) -> Option<u64> {
let bytes = buf.get(at..at + 8)?;
let mut arr = [0u8; 8];
arr.copy_from_slice(bytes);
Some(u64::from_le_bytes(arr))
}
fn read_u32_le(buf: &[u8], at: usize) -> Option<u32> {
let bytes = buf.get(at..at + 4)?;
let mut arr = [0u8; 4];
arr.copy_from_slice(bytes);
Some(u32::from_le_bytes(arr))
}
fn read_u16_le(buf: &[u8], at: usize) -> Option<u16> {
let bytes = buf.get(at..at + 2)?;
Some(u16::from_le_bytes([bytes[0], bytes[1]]))
}
/// Locate the PRESSED_DATA section's raw bytes by walking the binary's section /
/// load-command table — never an EOF footer. Dispatches on the leading magic.
fn find_pressed_data_section(content: &[u8]) -> Option<&[u8]> {
match content.get(..4)? {
// Mach-O 64-bit, both endiannesses.
[0xcf, 0xfa, 0xed, 0xfe] | [0xfe, 0xed, 0xfa, 0xcf] => find_macho(content),
[0x7f, b'E', b'L', b'F'] => find_elf(content),
[b'M', b'Z', ..] => find_pe(content),
_ => None,
}
}
/// Mach-O 64-bit (little-endian host): find segment `SMOL` → section `__PRESSED_DATA`
/// → its (offset,size) and return the slice. Mirrors smol_segment_reader.c.
fn find_macho(content: &[u8]) -> Option<&[u8]> {
const LC_SEGMENT_64: u32 = 0x19;
// mach_header_64: magic(4) cputype(4) cpusubtype(4) filetype(4) ncmds(4) ...
let ncmds = read_u32_le(content, 16)?;
let mut cmd_off = 32usize; // sizeof(mach_header_64)
for _ in 0..ncmds.min(10_000) {
let cmd = read_u32_le(content, cmd_off)?;
let cmdsize = read_u32_le(content, cmd_off + 4)? as usize;
if cmdsize == 0 {
return None;
}
if cmd == LC_SEGMENT_64 {
// segment_command_64: cmd(4) cmdsize(4) segname(16) vmaddr(8) vmsize(8)
// fileoff(8) filesize(8) maxprot(4) initprot(4) nsects(4) flags(4)
let segname = content.get(cmd_off + 8..cmd_off + 24)?;
if name_eq(segname, b"SMOL") {
let nsects = read_u32_le(content, cmd_off + 64)?;
let mut sect_off = cmd_off + 72; // start of section_64 array
for _ in 0..nsects.min(1000) {
// section_64: sectname(16) segname(16) addr(8) size(8) offset(4) ...
let sectname = content.get(sect_off..sect_off + 16)?;
if name_eq(sectname, b"__PRESSED_DATA") {
let size = read_u64_le(content, sect_off + 40)? as usize;
let offset = read_u32_le(content, sect_off + 48)? as usize;
return content.get(offset..offset.checked_add(size)?);
}
sect_off += 80; // sizeof(section_64)
}
}
}
cmd_off = cmd_off.checked_add(cmdsize)?;
}
None
}
/// ELF 64-bit: walk the section header table, match `.PRESSED_DATA` against the
/// section-header string table, return its (sh_offset, sh_size) slice.
fn find_elf(content: &[u8]) -> Option<&[u8]> {
// EI_CLASS at offset 4: 2 == 64-bit. Only 64-bit addons ship.
if *content.get(4)? != 2 {
return None;
}
let e_shoff = read_u64_le(content, 40)? as usize;
let e_shentsize = read_u16_le(content, 58)? as usize;
let e_shnum = read_u16_le(content, 60)? as usize;
let e_shstrndx = read_u16_le(content, 62)? as usize;
if e_shentsize < 64 || e_shnum == 0 || e_shstrndx >= e_shnum {
return None;
}
// String table section header → its (offset,size).
let strtab_hdr = e_shoff.checked_add(e_shstrndx.checked_mul(e_shentsize)?)?;
let strtab_off = read_u64_le(content, strtab_hdr + 24)? as usize;
let strtab_size = read_u64_le(content, strtab_hdr + 32)? as usize;
let strtab = content.get(strtab_off..strtab_off.checked_add(strtab_size)?)?;
for i in 0..e_shnum {
let shdr = e_shoff.checked_add(i.checked_mul(e_shentsize)?)?;
let sh_name = read_u32_le(content, shdr)? as usize;
if cstr_at(strtab, sh_name) == Some(b".PRESSED_DATA".as_slice()) {
let sh_offset = read_u64_le(content, shdr + 24)? as usize;
let sh_size = read_u64_le(content, shdr + 32)? as usize;
return content.get(sh_offset..sh_offset.checked_add(sh_size)?);
}
}
None
}
/// PE: parse the section table for `.PRESSED` (the 8-byte-name truncation of
/// `.PRESSED_DATA`) and return (PointerToRawData, SizeOfRawData).
fn find_pe(content: &[u8]) -> Option<&[u8]> {
let pe_off = read_u32_le(content, 0x3c)? as usize;
if content.get(pe_off..pe_off + 4)? != b"PE\0\0" {
return None;
}
let coff = pe_off + 4;
let number_of_sections = read_u16_le(content, coff + 2)? as usize;
let size_of_optional = read_u16_le(content, coff + 16)? as usize;
if number_of_sections > 200 {
return None;
}
let mut sect = coff + 20 + size_of_optional; // section table start
for _ in 0..number_of_sections {
let name = content.get(sect..sect + 8)?;
if name == b".PRESSED" {
let size_of_raw = read_u32_le(content, sect + 16)? as usize;
let ptr_raw = read_u32_le(content, sect + 20)? as usize;
return content.get(ptr_raw..ptr_raw.checked_add(size_of_raw)?);
}
sect += 40; // sizeof(IMAGE_SECTION_HEADER)
}
None
}
/// Compare a fixed-width, NUL-padded name field against a logical name.
fn name_eq(field: &[u8], want: &[u8]) -> bool {
if want.len() > field.len() {
return false;
}
field[..want.len()] == *want && field[want.len()..].iter().all(|&b| b == 0)
}
/// The NUL-terminated string at `off` within a string table.
fn cstr_at(strtab: &[u8], off: usize) -> Option<&[u8]> {
let rest = strtab.get(off..)?;
let end = rest.iter().position(|&b| b == 0).unwrap_or(rest.len());
Some(&rest[..end])
}
#[cfg(test)]
mod tests {
use super::*;
/// Build a valid pressed-data section blob from `raw` (the addon) so the header
/// parse + zstd decode + SHA-512 check round-trip without a real binary.
fn synth_section(raw: &[u8], has_config: bool) -> Vec<u8> {
let payload = zstd::stream::encode_all(raw, 3).unwrap();
let mut hasher = Sha512::new();
hasher.update(&payload);
let hash = hasher.finalize();
let mut s = Vec::new();
s.extend_from_slice(MAGIC_MARKER);
s.extend_from_slice(&(payload.len() as u64).to_le_bytes());
s.extend_from_slice(&(raw.len() as u64).to_le_bytes());
s.extend_from_slice(&[b'a'; CACHE_KEY_LEN]); // cache key
s.extend_from_slice(&[1u8, 1u8, 255u8]); // platform/arch/libc
s.extend_from_slice(&hash);
s.push(if has_config { 1 } else { 0 });
if has_config {
s.extend_from_slice(&[0u8; SMOL_CONFIG_BINARY_LEN]);
}
s.extend_from_slice(&payload);
s
}
#[test]
fn pressed_data_round_trips() {
let raw = b"\x7fELF this is the original addon payload, repeated.".repeat(40);
let section = synth_section(&raw, false);
assert_eq!(decode_pressed_data(&section).as_deref(), Some(raw.as_slice()));
}
#[test]
fn pressed_data_round_trips_with_config() {
let raw = vec![0xABu8; 5000];
let section = synth_section(&raw, true);
assert_eq!(decode_pressed_data(&section).as_deref(), Some(raw.as_slice()));
}
#[test]
fn rejects_a_non_hybrid() {
assert!(unwrap_if_hybrid(b"not a binary at all").is_none());
// Right magic, truncated header.
assert!(decode_pressed_data(MAGIC_MARKER.as_slice()).is_none());
// No marker.
assert!(decode_pressed_data(&[0u8; HEADER_LEN + 10]).is_none());
}
#[test]
fn rejects_a_tampered_payload() {
let raw = vec![0x11u8; 2000];
let mut section = synth_section(&raw, false);
// Flip a byte inside the zstd payload → SHA-512 mismatch → None.
let last = section.len() - 1;
section[last] ^= 0xff;
assert!(decode_pressed_data(&section).is_none());
}
#[test]
fn rejects_a_wrong_uncompressed_size() {
let raw = vec![0x22u8; 2000];
let mut section = synth_section(&raw, false);
// Corrupt the uncompressed-size field (offset 32 + 8).
section[40] = section[40].wrapping_add(1);
assert!(decode_pressed_data(&section).is_none());
}
#[test]
fn finds_pressed_data_in_a_synthetic_macho() {
// Minimal Mach-O 64: header + one LC_SEGMENT_64("SMOL") with one section
// __PRESSED_DATA pointing at an appended pressed-data blob.
let raw = vec![0x42u8; 3000];
let blob = synth_section(&raw, false);
const LC_SEGMENT_64: u32 = 0x19;
let header_len = 32usize;
let seg_cmd_len = 72 + 80; // segment_command_64 + one section_64
let blob_off = header_len + seg_cmd_len;
let mut bin = vec![0u8; blob_off];
// mach_header_64
bin[0..4].copy_from_slice(&[0xcf, 0xfa, 0xed, 0xfe]); // MH_MAGIC_64 LE bytes
bin[16..20].copy_from_slice(&1u32.to_le_bytes()); // ncmds = 1
// segment_command_64 at offset 32
let seg = 32;
bin[seg..seg + 4].copy_from_slice(&LC_SEGMENT_64.to_le_bytes());
bin[seg + 4..seg + 8].copy_from_slice(&(seg_cmd_len as u32).to_le_bytes());
bin[seg + 8..seg + 12].copy_from_slice(b"SMOL");
bin[seg + 64..seg + 68].copy_from_slice(&1u32.to_le_bytes()); // nsects = 1
// section_64 at offset seg + 72
let sect = seg + 72;
bin[sect..sect + 14].copy_from_slice(b"__PRESSED_DATA");
bin[sect + 40..sect + 48].copy_from_slice(&(blob.len() as u64).to_le_bytes()); // size
bin[sect + 48..sect + 52].copy_from_slice(&(blob_off as u32).to_le_bytes()); // offset
bin.extend_from_slice(&blob);
assert_eq!(find_macho(&bin).map(<[u8]>::to_vec), Some(blob.clone()));
assert_eq!(unwrap_if_hybrid(&bin).as_deref(), Some(raw.as_slice()));
}
#[test]
fn finds_pressed_data_in_a_synthetic_pe() {
let raw = vec![0x55u8; 1500];
let blob = synth_section(&raw, false);
// DOS stub (64B) + PE sig/COFF (24B) + one 40B section header, no optional hdr.
let pe_off = 64usize;
let sect_table = pe_off + 24;
let blob_off = sect_table + 40;
let mut bin = vec![0u8; blob_off];
bin[0] = b'M';
bin[1] = b'Z';
bin[0x3c..0x40].copy_from_slice(&(pe_off as u32).to_le_bytes());
bin[pe_off..pe_off + 4].copy_from_slice(b"PE\0\0");
// COFF: NumberOfSections at +2, SizeOfOptionalHeader at +16.
bin[pe_off + 4 + 2..pe_off + 4 + 4].copy_from_slice(&1u16.to_le_bytes());
bin[pe_off + 4 + 16..pe_off + 4 + 18].copy_from_slice(&0u16.to_le_bytes());
// Section header.
bin[sect_table..sect_table + 8].copy_from_slice(b".PRESSED");
bin[sect_table + 16..sect_table + 20].copy_from_slice(&(blob.len() as u32).to_le_bytes());
bin[sect_table + 20..sect_table + 24].copy_from_slice(&(blob_off as u32).to_le_bytes());
bin.extend_from_slice(&blob);
assert_eq!(unwrap_if_hybrid(&bin).as_deref(), Some(raw.as_slice()));
}
#[test]
fn finds_pressed_data_in_a_synthetic_elf() {
// Minimal ELF64: e_ident + just enough header to point at a 2-entry section
// header table (a `.shstrtab` string table + a `.PRESSED_DATA` section) and an
// appended pressed-data blob.
let raw = vec![0x66u8; 2200];
let blob = synth_section(&raw, false);
let shentsize = 64usize;
// String table content: "\0.shstrtab\0.PRESSED_DATA\0".
let mut strtab = vec![0u8];
let shstrtab_name = strtab.len() as u32;
strtab.extend_from_slice(b".shstrtab\0");
let pressed_name = strtab.len() as u32;
strtab.extend_from_slice(b".PRESSED_DATA\0");
// Layout: [ehdr 64] [strtab] [shdr table: 2 * 64] [blob].
let ehdr_len = 64usize;
let strtab_off = ehdr_len;
let shoff = strtab_off + strtab.len();
let blob_off = shoff + 2 * shentsize;
let mut bin = vec![0u8; blob_off];
bin[0..4].copy_from_slice(&[0x7f, b'E', b'L', b'F']);
bin[4] = 2; // EI_CLASS = 64-bit
bin[40..48].copy_from_slice(&(shoff as u64).to_le_bytes()); // e_shoff
bin[58..60].copy_from_slice(&(shentsize as u16).to_le_bytes()); // e_shentsize
bin[60..62].copy_from_slice(&2u16.to_le_bytes()); // e_shnum
bin[62..64].copy_from_slice(&0u16.to_le_bytes()); // e_shstrndx = section 0
bin[strtab_off..strtab_off + strtab.len()].copy_from_slice(&strtab);
// Section header 0: the string table.
let sh0 = shoff;
bin[sh0..sh0 + 4].copy_from_slice(&shstrtab_name.to_le_bytes()); // sh_name
bin[sh0 + 24..sh0 + 32].copy_from_slice(&(strtab_off as u64).to_le_bytes()); // sh_offset
bin[sh0 + 32..sh0 + 40].copy_from_slice(&(strtab.len() as u64).to_le_bytes()); // sh_size
// Section header 1: .PRESSED_DATA.
let sh1 = shoff + shentsize;
bin[sh1..sh1 + 4].copy_from_slice(&pressed_name.to_le_bytes());
bin[sh1 + 24..sh1 + 32].copy_from_slice(&(blob_off as u64).to_le_bytes());
bin[sh1 + 32..sh1 + 40].copy_from_slice(&(blob.len() as u64).to_le_bytes());
bin.extend_from_slice(&blob);
assert_eq!(find_elf(&bin).map(<[u8]>::to_vec), Some(blob.clone()));
assert_eq!(unwrap_if_hybrid(&bin).as_deref(), Some(raw.as_slice()));
}
#[test]
fn name_eq_is_exact_with_nul_padding() {
assert!(name_eq(b"SMOL\0\0\0\0\0\0\0\0\0\0\0\0", b"SMOL"));
assert!(!name_eq(b"SMOLX\0\0\0\0\0\0\0\0\0\0\0", b"SMOL"));
assert!(!name_eq(b"SMO\0", b"SMOL"));
}
}
+427
src/gate.rs
//! The install-time gate: decide whether a given file should be OS-compressed.
//!
//! A `Gate` is a glob AND/OR a size predicate. The PM helper calls
//! `gate.matches(name, len)` after it knows the addon's name + decoded length and
//! only hands the bytes to `compress_bytes` when both predicates pass. Both halves
//! are optional; a `Gate::default()` matches the fleet default `**/*.node` with no
//! size floor.
//!
//! The size predicate parses a human string (`">"`/`">="` + a number with an
//! optional unit) so a manifest can carry `compress = ">= 1MB"` verbatim. Units are
//! case-insensitive and cover both the decimal (`KB`/`MB`/`GB` = 1000ⁿ) and binary
//! (`KiB`/`MiB`/`GiB` = 1024ⁿ) families; a bare number is bytes.
/// A `>`/`>=` comparison against a byte threshold.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SizePredicate {
/// `len > threshold`.
GreaterThan(u64),
/// `len >= threshold`.
AtLeast(u64),
}
impl SizePredicate {
/// True when `len` satisfies the comparison.
pub fn matches(&self, len: u64) -> bool {
match *self {
SizePredicate::GreaterThan(t) => len > t,
SizePredicate::AtLeast(t) => len >= t,
}
}
/// Parse `"> 1MB"` / `">=1024"` / `"> 4 KiB"` into a predicate. Whitespace is
/// optional everywhere; the operator must be `>` or `>=`; the unit (if any) is
/// case-insensitive.
pub fn parse(spec: &str) -> Result<SizePredicate, GateParseError> {
let trimmed = spec.trim();
let (at_least, rest) = if let Some(rest) = trimmed.strip_prefix(">=") {
(true, rest)
} else if let Some(rest) = trimmed.strip_prefix('>') {
(false, rest)
} else {
return Err(GateParseError::Operator);
};
let bytes = parse_size(rest.trim())?;
Ok(if at_least {
SizePredicate::AtLeast(bytes)
} else {
SizePredicate::GreaterThan(bytes)
})
}
}
/// Parse a size literal (`"1MB"`, `"4 KiB"`, `"512"`) into a byte count. A bare
/// number is bytes; a unit suffix scales it. Decimal units are powers of 1000,
/// binary units (the `i` forms) powers of 1024.
fn parse_size(spec: &str) -> Result<u64, GateParseError> {
let spec = spec.trim();
if spec.is_empty() {
return Err(GateParseError::Number);
}
// Split the leading digit run from the trailing unit.
let split = spec
.find(|c: char| !c.is_ascii_digit() && c != '_')
.unwrap_or(spec.len());
let (digits, unit) = spec.split_at(split);
let number: u64 = digits
.replace('_', "")
.parse()
.map_err(|_| GateParseError::Number)?;
let multiplier = match unit.trim().to_ascii_lowercase().as_str() {
"" | "b" => 1,
"kb" => 1_000,
"mb" => 1_000_000,
"gb" => 1_000_000_000,
"kib" => 1024,
"mib" => 1024 * 1024,
"gib" => 1024 * 1024 * 1024,
_ => return Err(GateParseError::Unit),
};
number
.checked_mul(multiplier)
.ok_or(GateParseError::Overflow)
}
/// Why a `Gate` / `SizePredicate` string failed to parse.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GateParseError {
/// The size predicate did not start with `>` or `>=`.
Operator,
/// The numeric portion was missing or not an integer.
Number,
/// The unit suffix was not one of B/KB/MB/GB/KiB/MiB/GiB.
Unit,
/// The number × unit overflowed u64.
Overflow,
}
impl std::fmt::Display for GateParseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let msg = match self {
GateParseError::Operator => "size predicate must start with '>' or '>='",
GateParseError::Number => "size predicate needs an integer (e.g. '> 1MB')",
GateParseError::Unit => "unknown size unit (use B/KB/MB/GB/KiB/MiB/GiB)",
GateParseError::Overflow => "size predicate overflows a 64-bit byte count",
};
f.write_str(msg)
}
}
impl std::error::Error for GateParseError {}
/// The install-time gate: an optional glob AND an optional size predicate. A file
/// matches only if BOTH present predicates pass (an absent half is vacuously true).
/// `Gate::default()` is the fleet default — glob `**/*.node`, no size floor.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Gate {
glob: Option<String>,
size: Option<SizePredicate>,
}
impl Default for Gate {
fn default() -> Self {
Gate {
glob: Some(DEFAULT_GLOB.to_string()),
size: None,
}
}
}
/// The fleet default — every native addon, regardless of size.
pub const DEFAULT_GLOB: &str = "**/*.node";
impl Gate {
/// A gate that matches everything (no glob, no size floor). Useful when the
/// caller has already selected the file and just wants the one-pass writer.
pub fn any() -> Self {
Gate {
glob: None,
size: None,
}
}
/// Build a gate from an optional glob and an optional size-predicate string. A
/// `None` glob matches any name; a `None`/empty size string applies no floor.
pub fn new(glob: Option<&str>, size: Option<&str>) -> Result<Gate, GateParseError> {
let size = match size {
None => None,
Some(s) if s.trim().is_empty() => None,
Some(s) => Some(SizePredicate::parse(s)?),
};
Ok(Gate {
glob: glob.map(str::to_string),
size,
})
}
/// Replace the glob (chainable builder).
pub fn with_glob(mut self, glob: &str) -> Self {
self.glob = Some(glob.to_string());
self
}
/// Replace the size predicate (chainable builder).
pub fn with_size(mut self, size: SizePredicate) -> Self {
self.size = Some(size);
self
}
/// The glob pattern, if any.
pub fn glob(&self) -> Option<&str> {
self.glob.as_deref()
}
/// The size predicate, if any.
pub fn size(&self) -> Option<SizePredicate> {
self.size
}
/// True when `name` matches the glob (if set) AND `len` satisfies the size
/// predicate (if set). The name is matched against the full path the caller
/// passes — pass a `/`-normalized path so `**` segments line up.
pub fn matches(&self, name: &str, len: u64) -> bool {
if let Some(glob) = &self.glob {
if !glob_match(glob, name) {
return false;
}
}
if let Some(size) = &self.size {
if !size.matches(len) {
return false;
}
}
true
}
}
/// A small glob matcher covering the subset the gate needs: `*` (any run within a
/// path segment, no `/`), `**` (any run across segments, including `/`), and `?`
/// (one non-`/` char). Literal bytes match themselves. No char classes — the gate
/// patterns are simple suffix/segment globs like `**/*.node`.
pub fn glob_match(pattern: &str, text: &str) -> bool {
glob_inner(pattern.as_bytes(), text.as_bytes())
}
/// Recursive matcher — clean handling of nested stars (a `*` inside a `**` tail,
/// like `**/*.node`) that a single backtrack slot can't express. The pattern set
/// is small (suffix/segment globs), so the recursion depth is the number of stars,
/// not the input length.
fn glob_inner(pat: &[u8], text: &[u8]) -> bool {
// Pattern exhausted: match iff text is too.
let Some(&pc) = pat.first() else {
return text.is_empty();
};
match pc {
b'*' => {
let double = pat.get(1) == Some(&b'*');
if double {
// `**` matches any run including `/`. If immediately followed by `/`, that
// slash may also collapse to nothing (a/**/b ⊇ a/b).
let after = &pat[2..];
let collapsed = after.strip_prefix(b"/").unwrap_or(after);
// Zero-width match (with the optional `/` collapse), then every longer run.
if glob_inner(collapsed, text) || glob_inner(after, text) {
return true;
}
for i in 0..text.len() {
if glob_inner(after, &text[i + 1..]) || glob_inner(collapsed, &text[i + 1..]) {
return true;
}
}
false
} else {
// `*` matches a run within one path segment — never a `/`.
let rest = &pat[1..];
if glob_inner(rest, text) {
return true;
}
for i in 0..text.len() {
if text[i] == b'/' {
break;
}
if glob_inner(rest, &text[i + 1..]) {
return true;
}
}
false
}
}
b'?' => match text.first() {
Some(&c) if c != b'/' => glob_inner(&pat[1..], &text[1..]),
_ => false,
},
c => match text.first() {
Some(&t) if t == c => glob_inner(&pat[1..], &text[1..]),
_ => false,
},
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_units_case_insensitively() {
assert_eq!(parse_size("512"), Ok(512));
assert_eq!(parse_size("512B"), Ok(512));
assert_eq!(parse_size("1kb"), Ok(1_000));
assert_eq!(parse_size("1KB"), Ok(1_000));
assert_eq!(parse_size("2MB"), Ok(2_000_000));
assert_eq!(parse_size("3gb"), Ok(3_000_000_000));
assert_eq!(parse_size("1KiB"), Ok(1024));
assert_eq!(parse_size("1mib"), Ok(1024 * 1024));
assert_eq!(parse_size("1GiB"), Ok(1024 * 1024 * 1024));
// Whitespace + digit separators tolerated.
assert_eq!(parse_size("1 MB"), Ok(1_000_000));
assert_eq!(parse_size("1_000"), Ok(1_000));
}
#[test]
fn rejects_bad_size_literals() {
assert_eq!(parse_size(""), Err(GateParseError::Number));
assert_eq!(parse_size("MB"), Err(GateParseError::Number));
assert_eq!(parse_size("10PB"), Err(GateParseError::Unit));
assert_eq!(parse_size("99999999999999999999GB"), Err(GateParseError::Number));
assert_eq!(
parse_size("18446744073709551615KB"),
Err(GateParseError::Overflow)
);
}
#[test]
fn parses_predicate_operators() {
assert_eq!(
SizePredicate::parse("> 1MB"),
Ok(SizePredicate::GreaterThan(1_000_000))
);
assert_eq!(
SizePredicate::parse(">=1MB"),
Ok(SizePredicate::AtLeast(1_000_000))
);
assert_eq!(
SizePredicate::parse(" >= 4 KiB "),
Ok(SizePredicate::AtLeast(4096))
);
assert_eq!(SizePredicate::parse("1MB"), Err(GateParseError::Operator));
assert_eq!(SizePredicate::parse("< 1MB"), Err(GateParseError::Operator));
}
#[test]
fn predicate_comparison_is_exact_at_the_boundary() {
let gt = SizePredicate::GreaterThan(1000);
assert!(!gt.matches(1000));
assert!(gt.matches(1001));
let ge = SizePredicate::AtLeast(1000);
assert!(ge.matches(1000));
assert!(!ge.matches(999));
}
#[test]
fn glob_matches_node_addons_anywhere() {
assert!(glob_match("**/*.node", "node_modules/foo/build/Release/addon.node"));
assert!(glob_match("**/*.node", "addon.node"));
assert!(glob_match("*.node", "addon.node"));
// A single * must not cross a path separator.
assert!(!glob_match("*.node", "dir/addon.node"));
assert!(!glob_match("**/*.node", "addon.so"));
}
#[test]
fn glob_handles_question_and_literal_and_double_star_edges() {
assert!(glob_match("a?c", "abc"));
assert!(!glob_match("a?c", "a/c"));
assert!(glob_match("**", "any/deep/path"));
assert!(glob_match("a/**/b", "a/x/y/b"));
assert!(glob_match("a/**/b", "a/b"));
assert!(glob_match("exact", "exact"));
assert!(!glob_match("exact", "exacted"));
// Trailing star eats the rest.
assert!(glob_match("pre*", "prefix"));
}
#[test]
fn gate_default_is_node_glob_no_floor() {
let g = Gate::default();
assert_eq!(g.glob(), Some("**/*.node"));
assert_eq!(g.size(), None);
assert!(g.matches("build/Release/x.node", 10));
assert!(!g.matches("build/Release/x.so", 10));
}
#[test]
fn gate_requires_both_halves() {
let g = Gate::new(Some("**/*.node"), Some(">= 1MB")).unwrap();
assert!(g.matches("a/b.node", 2_000_000));
// Right name, too small.
assert!(!g.matches("a/b.node", 500_000));
// Big enough, wrong name.
assert!(!g.matches("a/b.so", 2_000_000));
}
#[test]
fn gate_any_matches_everything() {
let g = Gate::any();
assert!(g.matches("whatever.xyz", 0));
assert_eq!(g.glob(), None);
assert_eq!(g.size(), None);
}
#[test]
fn gate_new_treats_empty_size_as_no_floor() {
let g = Gate::new(None, Some(" ")).unwrap();
assert_eq!(g.size(), None);
assert!(g.matches("anything", 0));
}
#[test]
fn gate_builders_chain() {
let g = Gate::any()
.with_glob("**/*.dylib")
.with_size(SizePredicate::GreaterThan(100));
assert!(g.matches("a/b.dylib", 200));
assert!(!g.matches("a/b.dylib", 50));
}
#[test]
fn gate_new_propagates_parse_errors() {
assert_eq!(Gate::new(None, Some("nope")), Err(GateParseError::Operator));
}
#[test]
fn gate_new_with_a_glob_and_no_size_applies_no_floor() {
// The `None` size arm of Gate::new — a glob with no size predicate.
let g = Gate::new(Some("**/*.node"), None).unwrap();
assert_eq!(g.size(), None);
assert!(g.matches("a/b.node", 1));
assert!(!g.matches("a/b.so", 1));
}
#[test]
fn single_star_never_spans_a_separator() {
// A single `*` matches a run within one segment and must stop at a `/`.
let g = Gate::new(Some("a*c"), None).unwrap();
assert!(g.matches("ac", 0), "the star matches a zero-width run");
assert!(g.matches("abc", 0), "matches within one segment");
assert!(!g.matches("ab/c", 0), "a single * never crosses a /");
}
#[test]
fn parse_error_display_is_distinct() {
let msgs: Vec<String> = [
GateParseError::Operator,
GateParseError::Number,
GateParseError::Unit,
GateParseError::Overflow,
]
.iter()
.map(ToString::to_string)
.collect();
// All four messages are non-empty and unique.
assert!(msgs.iter().all(|m| !m.is_empty()));
let mut sorted = msgs.clone();
sorted.sort();
sorted.dedup();
assert_eq!(sorted.len(), 4);
}
}
+248
src/linux.rs
//! Linux backend — btrfs (Stage 1); bcachefs later. ext4/xfs/ZFS cannot be done.
//!
//! Detection is an allowlist: only btrfs carries the per-file `FS_COMPR_FL` we set,
//! so every other f_type (incl. overlay/NFS) reports Unsupported. Apply uses the
//! SYNCHRONOUS copy+flag path (create a sibling, request the codec BEFORE writing
//! so the kernel compresses the bytes as they land, then atomic-rename over the
//! target) — NOT `BTRFS_IOC_DEFRAG_RANGE`, whose async completion races a
//! post-apply `st_blocks` check. The rename also gives a fresh inode: the
//! copy-break that isolates any pnpm CAS hardlink siblings.
//!
//! Codec preference is zstd → lzo → zlib (speed-over-ratio: zstd decodes faster
//! than zlib and tunes ratio, lzo is the fastest fallback). We request it the way
//! `btrfs property set <file> compression <algo>` does — a write to the
//! `btrfs.compression` xattr — and walk the list because an old kernel rejects an
//! unsupported algo (zstd pre-4.14, lzo pre-3.14) with EINVAL. If none take we
//! fall back to the bare `FS_COMPR_FL` flag (the mount's default algorithm).
use std::os::fd::AsRawFd;
use std::path::Path;
use crate::{cstring, io, Error, Support, UnsupportedReason};
// Allowlist magic (statfs f_type). Only btrfs from the Stage-1 matrix.
const BTRFS_SUPER_MAGIC: i64 = 0x9123_683E;
// FS_IOC_{GET,SET}FLAGS are _IOR/_IOW('f', 1|2, long). The request value is a u32;
// `as _` at the call site casts it to whatever `libc::ioctl` expects per target
// (c_ulong on glibc, c_int on musl) — same 32 bits either way.
const FS_IOC_GETFLAGS: u32 = 0x8008_6601;
const FS_IOC_SETFLAGS: u32 = 0x4008_6602;
// FS_IOC_FIEMAP = _IOWR('f', 11, struct fiemap). Used to read whether the data
// extents are ENCODED (compressed) — btrfs reports st_blocks as the LOGICAL size,
// so the on-disk win is invisible to stat/du and only FIEMAP/compsize reveals it.
const FS_IOC_FIEMAP: u32 = 0xC020_660B;
const FIEMAP_FLAG_SYNC: u32 = 0x0001;
const FIEMAP_EXTENT_ENCODED: u32 = 0x0008;
// uapi/linux/fs.h: FS_COMPR_FL is 0x4 (0x20 is FS_APPEND_FL — append-only, which
// makes writes and renames EPERM).
const FS_COMPR_FL: libc::c_long = 0x0000_0004;
fn statfs_type(path: &Path) -> Result<i64, Error> {
let cpath = cstring(path)?;
let mut buf: libc::statfs = unsafe { std::mem::zeroed() };
if unsafe { libc::statfs(cpath.as_ptr(), &mut buf) } != 0 {
return Err(io("statfs"));
}
Ok(buf.f_type as i64)
}
fn get_flags(fd: libc::c_int) -> Result<libc::c_long, Error> {
let mut flags: libc::c_long = 0;
if unsafe { libc::ioctl(fd, FS_IOC_GETFLAGS as _, &mut flags) } != 0 {
return Err(io("FS_IOC_GETFLAGS"));
}
Ok(flags)
}
// Per-file codecs in preference order (speed-over-ratio). zstd: fast decode, good
// ratio, tunable. lzo: fastest decode, weakest ratio. zlib: legacy fallback.
const PREFERRED_ALGOS: [&[u8]; 3] = [b"zstd", b"lzo", b"zlib"];
// Request a codec the way `btrfs property set <file> compression <algo>` does — by
// writing the `btrfs.compression` xattr — trying most-preferred first. A kernel
// without a given algo rejects it (EINVAL), so we walk the list; if none take we
// fall back to the bare FS_COMPR_FL flag (the mount's default codec). Called on the
// empty temp file so the bytes compress as they're written.
fn request_codec(fd: libc::c_int) -> Result<(), Error> {
for algo in PREFERRED_ALGOS {
let rc = unsafe {
libc::fsetxattr(
fd,
c"btrfs.compression".as_ptr(),
algo.as_ptr().cast(),
algo.len(),
0,
)
};
if rc == 0 {
return Ok(());
}
}
let flags = get_flags(fd)? | FS_COMPR_FL;
if unsafe { libc::ioctl(fd, FS_IOC_SETFLAGS as _, &flags) } != 0 {
return Err(io("FS_IOC_SETFLAGS"));
}
Ok(())
}
pub(crate) fn detect(path: &Path) -> Result<Support, Error> {
if statfs_type(path)? == BTRFS_SUPER_MAGIC {
Ok(Support::Supported)
} else {
Ok(Support::Unsupported(UnsupportedReason::Filesystem))
}
}
#[repr(C)]
#[derive(Clone, Copy)]
struct FiemapExtent {
fe_logical: u64,
fe_physical: u64,
fe_length: u64,
fe_reserved64: [u64; 2],
fe_flags: u32,
fe_reserved: [u32; 3],
}
#[repr(C)]
struct FiemapHeader {
fm_start: u64,
fm_length: u64,
fm_flags: u32,
fm_mapped_extents: u32,
fm_extent_count: u32,
fm_reserved: u32,
}
const _: () = assert!(std::mem::size_of::<FiemapHeader>() == 32);
const _: () = assert!(std::mem::size_of::<FiemapExtent>() == 56);
/// True if any data extent carries FIEMAP_EXTENT_ENCODED (i.e. is compressed on
/// disk). This is the only reliable btrfs win signal — st_blocks reflects the
/// logical size. FIEMAP_FLAG_SYNC flushes pending writes before mapping.
fn compressed_via_fiemap(path: &Path) -> Result<bool, Error> {
const COUNT: usize = 64;
let file = std::fs::File::open(path).map_err(|source| Error::Io {
context: "open for fiemap",
source,
})?;
let mut buf =
vec![0u8; std::mem::size_of::<FiemapHeader>() + COUNT * std::mem::size_of::<FiemapExtent>()];
// SAFETY: buf is sized for the header + COUNT extents; the kernel fills it.
unsafe {
let header = buf.as_mut_ptr().cast::<FiemapHeader>();
(*header).fm_start = 0;
(*header).fm_length = u64::MAX;
(*header).fm_flags = FIEMAP_FLAG_SYNC;
(*header).fm_extent_count = COUNT as u32;
(*header).fm_mapped_extents = 0;
(*header).fm_reserved = 0;
if libc::ioctl(file.as_raw_fd(), FS_IOC_FIEMAP as _, buf.as_mut_ptr()) != 0 {
return Err(io("FS_IOC_FIEMAP"));
}
let mapped = (*header).fm_mapped_extents as usize;
let extents = std::slice::from_raw_parts(
buf
.as_ptr()
.add(std::mem::size_of::<FiemapHeader>())
.cast::<FiemapExtent>(),
mapped.min(COUNT),
);
Ok(
extents
.iter()
.any(|e| e.fe_flags & FIEMAP_EXTENT_ENCODED != 0),
)
}
}
/// btrfs win measurement is authoritative via FIEMAP, not st_blocks.
pub(crate) fn compressed_on_disk(path: &Path) -> Result<Option<bool>, Error> {
Ok(Some(compressed_via_fiemap(path)?))
}
pub(crate) fn is_already_compressed(path: &Path) -> Result<bool, Error> {
let file = std::fs::File::open(path).map_err(|source| Error::Io {
context: "open",
source,
})?;
Ok(get_flags(file.as_raw_fd())? & FS_COMPR_FL != 0)
}
pub(crate) fn apply_inplace(path: &Path) -> Result<(), Error> {
let data = std::fs::read(path).map_err(|source| Error::Io {
context: "read",
source,
})?;
let mode = std::fs::metadata(path).map(|m| m.permissions()).ok();
apply_bytes(path, &data, mode)
}
/// Write `content` to `path` as a fresh btrfs-compressed file in ONE pass: create
/// the sibling temp, request the codec on the EMPTY file (so the bytes compress as
/// they land — never a write-then-recompress), write, fsync, then atomic-rename
/// over `path`. The rename gives a fresh inode, the copy-break from any pnpm CAS
/// hardlink siblings. Shared by `compress_bytes` (no original) and `apply_inplace`.
pub(crate) fn apply_bytes(
path: &Path,
content: &[u8],
mode: Option<std::fs::Permissions>,
) -> Result<(), Error> {
use std::io::Write;
let dir = path.parent().ok_or_else(|| Error::Io {
context: "no parent dir",
source: std::io::Error::from(std::io::ErrorKind::InvalidInput),
})?;
let name = path.file_name().map_or_else(
|| std::borrow::Cow::Borrowed("addon"),
|n| n.to_string_lossy(),
);
let tmp = dir.join(format!(".{name}.decmpfs-{}.tmp", std::process::id()));
let result = (|| {
let mut file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(&tmp)
.map_err(|source| Error::Io {
context: "create temp",
source,
})?;
let fd = file.as_raw_fd();
// Request the codec on the empty file FIRST so writes compress on the way in.
request_codec(fd)?;
file.write_all(content).map_err(|source| Error::Io {
context: "write temp",
source,
})?;
file.sync_all().map_err(|source| Error::Io {
context: "fsync temp",
source,
})?;
Ok(())
})();
if result.is_ok() {
if let Some(perm) = mode {
let _ = std::fs::set_permissions(&tmp, perm);
}
}
match result {
Ok(()) => std::fs::rename(&tmp, path).map_err(|source| {
let _ = std::fs::remove_file(&tmp);
Error::Io {
context: "rename",
source,
}
}),
Err(err) => {
let _ = std::fs::remove_file(&tmp);
Err(err)
}
}
}
+425
src/macos.rs
//! macOS backend — APFS/HFS+ decmpfs transparent compression.
//!
//! decmpfs is an undocumented kernel ABI; afsctool and `ditto --hfsCompression` are
//! the references. We write the LZVN resource-fork variant (compression type 8): the
//! kernel decompresses on read(), so the file keeps its logical size + stays a
//! loadable native binary. LZVN comes from the system libcompression framework — no
//! Rust codec dep, so the macOS stub stays tiny.
//!
//! Codec choice is LZVN, not LZFSE (type 12) or ZLIB (type 4): for a load-once
//! `.node` we favor decode speed over ratio, and LZVN is the fastest of the three
//! to decompress (LZFSE wins ~10-15% on size but decodes slower). LZVN has shipped
//! in libcompression since 10.11, so there's no availability fallback to make — the
//! only fallback path is the ephemeral cache when decmpfs can't be applied at all.
//!
//! Layout written (verified by the kernel-roundtrip test):
//! xattr com.apple.decmpfs = [magic u32 LE][type=8 LZVN u32 LE][rawSize u64 LE]
//! xattr com.apple.ResourceFork = [(numBlocks+1) u32 LE offset table][LZVN blocks]
//! Built on a sibling temp (empty data fork + those xattrs + UF_COMPRESSED), then
//! atomically renamed over the original — never an in-place truncate, so a crash
//! can't leave a 0-byte file.
use std::os::fd::AsRawFd;
use std::path::Path;
use crate::{cstring, io, Error, Support, UnsupportedReason};
const UF_COMPRESSED: u32 = 0x0000_0020;
const DECMPFS_MAGIC: u32 = 0x636d_7066; // 'cmpf' (XNU sys/decmpfs.h); LE on disk = "fpmc"
// Type 8 = LZVN-in-resource-fork — what `ditto --hfsCompression` writes and the
// kernel reliably reads. The resource fork is a flat offset table (NOT the zlib
// type-4 resource-fork-with-map format).
const CMP_LZVN_RESOURCE_FORK: u32 = 8;
const BLOCK: usize = 0x1_0000; // 64 KiB
const XATTR_NOFOLLOW: libc::c_int = 0x0001;
// decmpfs resource-fork offsets cap at u32; stay well under 4 GiB so the offsets
// and the kernel's own checks never overflow.
const MAX_RAW: u64 = 3_900_000_000;
const COMPRESSION_LZVN: i32 = 0x900;
#[link(name = "compression")]
extern "C" {
fn compression_encode_buffer(
dst_buffer: *mut u8,
dst_size: usize,
src_buffer: *const u8,
src_size: usize,
scratch_buffer: *mut u8,
algorithm: i32,
) -> usize;
fn compression_encode_scratch_buffer_size(algorithm: i32) -> usize;
}
/// Reject files past the decmpfs u32-offset ceiling (→ Skipped(TooLarge) via
/// safety::classify_skip on EFBIG). Pure, so the limit is testable without a 4 GiB
/// file.
fn within_decmpfs_limit(len: u64) -> Result<(), Error> {
if len > MAX_RAW {
return Err(Error::Io {
context: "file too large for decmpfs",
source: std::io::Error::from_raw_os_error(libc::EFBIG),
});
}
Ok(())
}
fn statfs(path: &Path) -> Result<libc::statfs, Error> {
let cpath = cstring(path)?;
let mut buf: libc::statfs = unsafe { std::mem::zeroed() };
if unsafe { libc::statfs(cpath.as_ptr(), &mut buf) } != 0 {
return Err(io("statfs"));
}
Ok(buf)
}
/// Local APFS or HFS+ only — the two filesystems with the decmpfs path. A network
/// or non-local mount reports Unsupported (the signal isn't ours to trust).
pub(crate) fn detect(path: &Path) -> Result<Support, Error> {
let buf = statfs(path)?;
// f_fstypename is a NUL-padded C string ("apfs", "hfs").
let name: Vec<u8> = buf
.f_fstypename
.iter()
.take_while(|&&c| c != 0)
.map(|&c| c as u8)
.collect();
Ok(classify_fs(
buf.f_flags & (libc::MNT_LOCAL as u32) != 0,
&name,
))
}
/// The pure detect policy — split from the statfs syscall so the network/non-APFS
/// branches are unit-testable without a network mount or an exotic filesystem.
fn classify_fs(is_local: bool, fstype: &[u8]) -> Support {
if !is_local {
return Support::Unsupported(UnsupportedReason::NetworkOrOverlay);
}
if fstype == b"apfs" || fstype == b"hfs" {
Support::Supported
} else {
Support::Unsupported(UnsupportedReason::Filesystem)
}
}
fn st_flags(path: &Path) -> Result<u32, Error> {
let cpath = cstring(path)?;
let mut st: libc::stat = unsafe { std::mem::zeroed() };
if unsafe { libc::lstat(cpath.as_ptr(), &mut st) } != 0 {
return Err(io("lstat"));
}
Ok(st.st_flags)
}
pub(crate) fn is_already_compressed(path: &Path) -> Result<bool, Error> {
Ok(st_flags(path)? & UF_COMPRESSED != 0)
}
/// On macOS, UF_COMPRESSED is the authoritative win signal (st_blocks also drops,
/// but the flag is unambiguous and what we set).
pub(crate) fn compressed_on_disk(path: &Path) -> Result<Option<bool>, Error> {
Ok(Some(is_already_compressed(path)?))
}
/// LZVN-encode `src` into a kernel-decodable block. libcompression emits a valid
/// frame even for incompressible input (slightly larger than `src`), so every
/// block decodes the same way — there is no bare "stored" block. Returns `None`
/// only if encoding fails outright (treated as a hard error upstream).
fn compress_block(src: &[u8], scratch: &mut [u8]) -> Option<Vec<u8>> {
// Headroom for the worst case (incompressible data expands a little).
let mut dst = vec![0u8; src.len() + src.len() / 16 + 1024];
let n = unsafe {
compression_encode_buffer(
dst.as_mut_ptr(),
dst.len(),
src.as_ptr(),
src.len(),
scratch.as_mut_ptr(),
COMPRESSION_LZVN,
)
};
if n == 0 {
return None;
}
dst.truncate(n);
Some(dst)
}
/// Build the com.apple.ResourceFork blob for `raw` in the LZVN/LZFSE decmpfs
/// layout (what `ditto` writes): `(numBlocks+1)` u32 LE offsets, then the blocks.
/// `offset[0]` = table size; `offset[i+1]` = end of block i; last = total size.
fn build_resource_fork(raw: &[u8]) -> Option<Vec<u8>> {
let num_blocks = raw.len().div_ceil(BLOCK).max(1);
let scratch_len = unsafe { compression_encode_scratch_buffer_size(COMPRESSION_LZVN) };
let mut scratch = vec![0u8; scratch_len];
let blocks = raw
.chunks(BLOCK)
.map(|chunk| compress_block(chunk, &mut scratch))
.collect::<Option<Vec<Vec<u8>>>>()?;
let table_len = (num_blocks + 1) * 4;
let mut out = Vec::with_capacity(table_len + blocks.iter().map(Vec::len).sum::<usize>());
// Offset table: numBlocks+1 entries. offset[i] is where block i starts.
let mut offset = table_len as u32;
out.extend_from_slice(&offset.to_le_bytes());
for block in &blocks {
offset += block.len() as u32;
out.extend_from_slice(&offset.to_le_bytes());
}
for block in &blocks {
out.extend_from_slice(block);
}
Some(out)
}
fn setxattr(path: &std::ffi::CStr, name: &std::ffi::CStr, value: &[u8]) -> Result<(), Error> {
let rc = unsafe {
libc::setxattr(
path.as_ptr(),
name.as_ptr(),
value.as_ptr().cast(),
value.len(),
0,
XATTR_NOFOLLOW,
)
};
if rc != 0 {
return Err(io("setxattr"));
}
Ok(())
}
pub(crate) fn apply_inplace(path: &Path) -> Result<(), Error> {
let raw = std::fs::read(path).map_err(|source| Error::Io {
context: "read",
source,
})?;
// Fail-soft: skip if we can't write the original (by mode or ownership) — the
// temp+rename below would otherwise replace even a file we can't open for write.
let cpath = cstring(path)?;
if unsafe { libc::access(cpath.as_ptr(), libc::W_OK) } != 0 {
return Err(io("access"));
}
let mode = std::fs::metadata(path).map(|m| m.permissions()).ok();
apply_bytes(path, &raw, mode)
}
/// Write `content` to `path` as a fresh decmpfs-compressed file in ONE pass — no
/// write-then-read-back. The decmpfs is built directly from `content`, dropped on a
/// sibling temp (empty data fork + the two xattrs + UF_COMPRESSED), then atomically
/// renamed over `path`. A crash can only leave the original or the finished file;
/// the rename also gives a fresh inode, the copy-break from any pnpm CAS hardlink
/// siblings. This is the one-pass core both `compress_bytes` (no original) and
/// `apply_inplace` (read first) share.
pub(crate) fn apply_bytes(
path: &Path,
content: &[u8],
mode: Option<std::fs::Permissions>,
) -> Result<(), Error> {
within_decmpfs_limit(content.len() as u64)?;
let mut header = Vec::with_capacity(16);
header.extend_from_slice(&DECMPFS_MAGIC.to_le_bytes());
header.extend_from_slice(&CMP_LZVN_RESOURCE_FORK.to_le_bytes());
header.extend_from_slice(&(content.len() as u64).to_le_bytes());
let resource_fork = build_resource_fork(content).ok_or_else(|| Error::Io {
context: "lzvn encode",
source: std::io::Error::from(std::io::ErrorKind::InvalidData),
})?;
let dir = path.parent().ok_or_else(|| io("parent"))?;
let name = path
.file_name()
.ok_or_else(|| io("file_name"))?
.to_string_lossy();
let tmp = dir.join(format!(".{name}.decmpfs-{}.tmp", std::process::id()));
let build = (|| -> Result<(), Error> {
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create_new(true)
.open(&tmp)
.map_err(|source| Error::Io {
context: "create temp",
source,
})?;
let ctmp = cstring(&tmp)?;
setxattr(&ctmp, c"com.apple.decmpfs", &header)?;
setxattr(&ctmp, c"com.apple.ResourceFork", &resource_fork)?;
if unsafe { libc::fchflags(file.as_raw_fd(), UF_COMPRESSED) } != 0 {
return Err(io("fchflags"));
}
Ok(())
})();
if let Err(e) = build {
let _ = std::fs::remove_file(&tmp);
return Err(e);
}
if let Some(perm) = mode {
let _ = std::fs::set_permissions(&tmp, perm);
}
std::fs::rename(&tmp, path).map_err(|source| {
let _ = std::fs::remove_file(&tmp);
Error::Io {
context: "rename",
source,
}
})
}
#[cfg(test)]
mod tests {
use super::*;
// The kernel-roundtrip oracle. decmpfs is undocumented — the only proof the
// format is right is that a normal read() returns identical bytes after apply.
#[test]
fn kernel_roundtrips_decmpfs() {
let dir = std::env::temp_dir().join(format!("decmpfs-oracle-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f.bin");
// > 1 block (64 KiB) of compressible data, so the offset table + LZVN blocks
// are both exercised.
let mut raw = Vec::new();
let pat = b"the quick brown fox decmpfs lzvn resource-fork oracle line ";
while raw.len() < 2_000_000 {
raw.extend_from_slice(pat);
}
std::fs::write(&path, &raw).unwrap();
assert!(
matches!(detect(&path).unwrap(), Support::Supported),
"temp dir is local APFS/HFS+"
);
apply_inplace(&path).unwrap();
assert!(is_already_compressed(&path).unwrap(), "UF_COMPRESSED set");
assert_eq!(
compressed_on_disk(&path).unwrap(),
Some(true),
"reports compressed"
);
// THE ORACLE: the kernel decompresses our resource fork on read().
assert_eq!(
std::fs::read(&path).unwrap(),
raw,
"kernel read-back must equal the original bytes"
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn detect_and_flags_error_on_a_missing_path() {
let p = std::path::Path::new("/no/such/decmpfs/path/x.bin");
assert!(detect(p).is_err(), "statfs of a missing path errors");
assert!(
is_already_compressed(p).is_err(),
"lstat of a missing path errors"
);
}
#[test]
fn apply_inplace_errors_when_the_file_cannot_be_read() {
// A 0-perm file: apply_inplace's initial read fails before any apply. Root
// bypasses mode bits, so skip there.
if unsafe { libc::geteuid() } == 0 {
return;
}
use std::os::unix::fs::PermissionsExt;
let dir = std::env::temp_dir().join(format!("decmpfs-noread-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f.bin");
std::fs::write(&path, b"\x7fELF unreadable").unwrap();
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap();
let out = apply_inplace(&path);
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).ok();
assert!(matches!(out, Err(Error::Io { context: "read", .. })));
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn setxattr_errors_on_a_missing_path() {
let out = setxattr(c"/no/such/decmpfs/path", c"com.apple.decmpfs", b"x");
assert!(matches!(out, Err(Error::Io { context: "setxattr", .. })));
}
#[test]
fn compress_block_returns_none_for_empty_input() {
// libcompression encodes zero bytes to nothing → the n == 0 guard returns None.
let scratch_len = unsafe { compression_encode_scratch_buffer_size(COMPRESSION_LZVN) };
let mut scratch = vec![0u8; scratch_len];
assert!(compress_block(b"", &mut scratch).is_none());
}
#[test]
fn cstring_rejects_an_interior_nul() {
use std::os::unix::ffi::OsStrExt;
let p = std::path::Path::new(std::ffi::OsStr::from_bytes(b"a\0b"));
assert!(cstring(p).is_err());
}
#[test]
fn detect_rejects_a_non_apfs_filesystem() {
// /dev is devfs (local, but not apfs/hfs) → Unsupported(Filesystem).
assert!(matches!(
detect(std::path::Path::new("/dev")),
Ok(Support::Unsupported(UnsupportedReason::Filesystem))
));
}
#[test]
fn classify_fs_covers_every_branch() {
// Non-local (e.g. a network mount) — no real mount needed.
assert!(matches!(
classify_fs(false, b"nfs"),
Support::Unsupported(UnsupportedReason::NetworkOrOverlay)
));
assert!(matches!(classify_fs(true, b"apfs"), Support::Supported));
assert!(matches!(classify_fs(true, b"hfs"), Support::Supported));
assert!(matches!(
classify_fs(true, b"ext4"),
Support::Unsupported(UnsupportedReason::Filesystem)
));
}
#[test]
fn within_decmpfs_limit_rejects_oversized() {
// No 4 GiB file needed — the predicate is pure.
assert!(within_decmpfs_limit(MAX_RAW).is_ok());
match within_decmpfs_limit(MAX_RAW + 1).unwrap_err() {
Error::Io { source, .. } => assert_eq!(source.raw_os_error(), Some(libc::EFBIG)),
other => panic!("expected EFBIG Io, got {other:?}"),
}
}
// Incompressible data → blocks are stored verbatim (compress_block returns the
// chunk). The kernel must still read them back identically.
#[test]
fn kernel_roundtrips_incompressible_blocks() {
let dir = std::env::temp_dir().join(format!("decmpfs-raw-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f.bin");
let mut raw = Vec::new();
let mut x: u32 = 0x9e37_79b9;
while raw.len() < 200_000 {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
raw.extend_from_slice(&x.to_le_bytes());
}
std::fs::write(&path, &raw).unwrap();
if matches!(detect(&path).unwrap(), Support::Supported) {
apply_inplace(&path).unwrap();
assert_eq!(
std::fs::read(&path).unwrap(),
raw,
"verbatim blocks read back"
);
}
std::fs::remove_dir_all(&dir).ok();
}
}
+403
src/safety.rs
//! Shared orchestration that gates every backend identically. Written once here
//! so a backend only implements `detect` / `is_already_compressed` /
//! `apply_inplace` and inherits all the safety invariants.
use std::path::Path;
use crate::{verify, Backend, Error, Outcome, SkipReason};
/// Reached only when the backend reported `Supported`. Fail-soft: a permission,
/// read-only, or busy failure is a `Skipped` Outcome, never a hard `Err`. And if a
/// (broken) backend ever leaves the file no longer loadable, roll back to the
/// pre-apply bytes so a corrupt addon is never stranded.
pub(crate) fn apply_guarded<B: Backend>(backend: &B, path: &Path) -> Result<Outcome, Error> {
// INV-idempotent.
if backend.is_already_compressed(path)? {
return Ok(Outcome::AlreadyCompressed {
before: verify::on_disk_bytes(path)?,
});
}
let before = verify::on_disk_bytes(path)?;
// INV-loadable: snapshot the native-binary magic so we can confirm the file
// still loads after compression (a post-compress *content* hash is vacuous —
// the kernel decompresses on read, so it always matches).
let magic_before = verify::magic_prefix(path)?;
// INV-rollback: keep the original bytes so a backend that produces a non-loadable
// result can be reverted. Cheap next to the one-time warm decompress.
let snapshot = std::fs::read(path).map_err(|source| Error::Io {
context: "snapshot",
source,
})?;
// INV-fail-soft: EACCES/EPERM/EROFS -> Skipped(PermissionDenied); EBUSY/ETXTBSY
// -> Skipped(Busy). A genuine, unclassifiable I/O error still propagates.
if let Err(err) = backend.apply_inplace(path) {
if let Error::Io { source, .. } = &err {
if let Some(reason) = classify_skip(source) {
return Ok(Outcome::Skipped { reason });
}
}
return Err(err);
}
verify_loadable_or_restore(backend, path, before, magic_before, &snapshot)
}
/// Post-apply gate for the in-place path: if the file no longer carries its
/// native-binary magic the backend broke it, so restore the snapshot and report
/// `Skipped(NotLoadable)`; otherwise classify the win. Split out so the
/// not-loadable rollback is unit-testable without a backend that corrupts a file
/// (pass a `magic_before` the file no longer matches).
fn verify_loadable_or_restore<B: Backend>(
backend: &B,
path: &Path,
before: u64,
magic_before: [u8; 4],
snapshot: &[u8],
) -> Result<Outcome, Error> {
if verify::magic_prefix(path)? != magic_before {
restore(path, snapshot);
return Ok(classify_outcome(false, before, before, None));
}
// INV-verify: prefer the backend's authoritative signal (btrfs FIEMAP ENCODED —
// st_blocks reports the logical size there, so a real win is invisible to it).
// Where the backend has no special signal (APFS/NTFS), fall back to the generic
// allocated-bytes drop.
let after = verify::on_disk_bytes(path)?;
Ok(classify_outcome(
true,
before,
after,
backend.compressed_on_disk(path)?,
))
}
/// Map the post-apply facts to an Outcome. Pure (no I/O) so every branch is unit
/// testable: not loadable → Skipped(NotLoadable); else the backend's compression
/// signal (or, absent one, an allocated-bytes drop) decides Compressed vs NoGain.
fn classify_outcome(loadable: bool, before: u64, after: u64, signal: Option<bool>) -> Outcome {
if !loadable {
return Outcome::Skipped {
reason: SkipReason::NotLoadable,
};
}
if signal.unwrap_or(after < before) {
Outcome::Compressed { before, after }
} else {
Outcome::NoGain { before, after }
}
}
/// Map a backend I/O failure to a non-fatal `Skipped` reason, or `None` to let it
/// propagate as a hard error. Uses both `ErrorKind` (cross-platform, esp. Windows)
/// and the POSIX errno (stable across Linux/macOS), so it needs no newer-than-1.0
/// `ErrorKind` variants.
fn classify_skip(err: &std::io::Error) -> Option<SkipReason> {
if err.kind() == std::io::ErrorKind::PermissionDenied {
return Some(SkipReason::PermissionDenied);
}
match err.raw_os_error() {
Some(1) | Some(13) | Some(30) => Some(SkipReason::PermissionDenied), // EPERM/EACCES/EROFS
Some(16) | Some(26) => Some(SkipReason::Busy), // EBUSY/ETXTBSY
Some(27) => Some(SkipReason::TooLarge), // EFBIG
_ => None,
}
}
/// One-pass guarded write of `content` to `path` as an OS-compressed file. Reached
/// only when the backend reported `Supported`. The backend writes the bytes AS the
/// file is created (decmpfs built from `content`, btrfs codec-then-write, NTFS
/// FSCTL-then-write) — no write-then-read-back. Fail-soft mirrors `apply_guarded`:
/// a permission/busy/too-large failure becomes a `Skipped` Outcome and the caller
/// is expected to fall back to a plain write; an unclassifiable I/O error
/// propagates. After a successful apply the kernel read-back is verified
/// byte-identical to `content` (the transparent-compression oracle), and the file
/// is restored to a plain write of `content` if it somehow doesn't match.
pub(crate) fn compress_bytes_guarded<B: Backend>(
backend: &B,
path: &Path,
content: &[u8],
) -> Result<Outcome, Error> {
if let Err(err) = backend.apply_bytes(path, content, None) {
if let Error::Io { source, .. } = &err {
if let Some(reason) = classify_skip(source) {
return Ok(Outcome::Skipped { reason });
}
}
return Err(err);
}
// Oracle: a normal read must hand back the exact bytes we asked to store.
verify_readback_or_restore(backend, path, content)
}
/// Post-apply oracle for the one-pass path: a normal read must hand back exactly
/// `content`. If the backend produced something that doesn't decode identically,
/// restore a plain write of `content` and report `Skipped(IntegrityRevert)` so an
/// install is never left with a corrupt file; otherwise classify the win. Split
/// out so the mismatch-rollback is unit-testable without a backend that corrupts
/// the read-back (point it at a file whose bytes differ from `content`).
fn verify_readback_or_restore<B: Backend>(
backend: &B,
path: &Path,
content: &[u8],
) -> Result<Outcome, Error> {
let after = verify::on_disk_bytes(path)?;
let read_back = std::fs::read(path).map_err(|source| Error::Io {
context: "read-back",
source,
})?;
if read_back != content {
restore(path, content);
return Ok(Outcome::Skipped {
reason: SkipReason::IntegrityRevert,
});
}
let before = content.len() as u64;
Ok(classify_outcome(
true,
before,
after,
backend.compressed_on_disk(path)?,
))
}
/// Best-effort atomic restore of the pre-apply bytes (sibling temp + rename).
fn restore(path: &Path, bytes: &[u8]) {
use std::io::Write;
let Some(dir) = path.parent() else {
return;
};
let tmp = dir.join(format!(".decmpfs-restore-{}.tmp", std::process::id()));
let wrote = std::fs::File::create(&tmp).and_then(|mut file| {
file.write_all(bytes)?;
file.sync_all()
});
if wrote.is_ok() && std::fs::rename(&tmp, path).is_ok() {
return;
}
let _ = std::fs::remove_file(&tmp);
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{FakeBackend, Os, Support};
fn err(kind: std::io::ErrorKind) -> std::io::Error {
std::io::Error::from(kind)
}
#[test]
fn apply_guarded_propagates_an_unclassifiable_apply_error() {
// A fake backend reports a compressible FS but its in-place apply fails with an
// unclassifiable error (ENOENT) — apply_guarded propagates it rather than
// swallowing it. A real backend reaches this only on a true I/O fault.
let dir = std::env::temp_dir().join(format!("decmpfs-broken-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f.bin");
std::fs::write(&path, b"\x7fELF readable original").unwrap();
let backend = FakeBackend {
detect: Support::Supported,
apply_errno: Some(2),
};
let out = apply_guarded(&backend, &path);
assert!(matches!(out, Err(Error::Io { .. })), "got {out:?}");
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn permission_errors_become_skipped() {
assert_eq!(
classify_skip(&err(std::io::ErrorKind::PermissionDenied)),
Some(SkipReason::PermissionDenied)
);
for errno in [1, 13, 30] {
assert_eq!(
classify_skip(&std::io::Error::from_raw_os_error(errno)),
Some(SkipReason::PermissionDenied),
"errno {errno}"
);
}
}
#[test]
fn busy_errors_become_skipped() {
for errno in [16, 26] {
assert_eq!(
classify_skip(&std::io::Error::from_raw_os_error(errno)),
Some(SkipReason::Busy),
"errno {errno}"
);
}
}
#[test]
fn efbig_becomes_too_large() {
assert_eq!(
classify_skip(&std::io::Error::from_raw_os_error(27)), // EFBIG
Some(SkipReason::TooLarge)
);
}
#[test]
fn classify_outcome_covers_every_branch() {
use crate::Outcome;
assert!(matches!(
classify_outcome(false, 100, 50, None),
Outcome::Skipped {
reason: SkipReason::NotLoadable
}
));
// Allocated-bytes fallback (no backend signal).
assert!(matches!(
classify_outcome(true, 100, 40, None),
Outcome::Compressed {
before: 100,
after: 40
}
));
assert!(matches!(
classify_outcome(true, 100, 100, None),
Outcome::NoGain { .. }
));
// Backend signal overrides the size comparison both ways.
assert!(matches!(
classify_outcome(true, 100, 100, Some(true)),
Outcome::Compressed { .. }
));
assert!(matches!(
classify_outcome(true, 100, 40, Some(false)),
Outcome::NoGain { .. }
));
}
#[test]
fn restore_writes_the_snapshot_back() {
let dir = std::env::temp_dir().join(format!("decmpfs-restore-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f");
std::fs::write(&path, b"corrupted-by-a-broken-backend").unwrap();
restore(&path, b"the original loadable bytes");
assert_eq!(
std::fs::read(&path).unwrap(),
b"the original loadable bytes"
);
std::fs::remove_dir_all(&dir).ok();
}
// A target whose parent directory does not exist: the backend's temp create
// fails with ENOENT — not a permission/busy/too-large skip — so the guarded
// one-pass write propagates it as a hard Err rather than swallowing it.
#[cfg(target_os = "macos")]
#[test]
fn compress_bytes_guarded_propagates_an_unclassifiable_error() {
let out = compress_bytes_guarded(
&Os,
std::path::Path::new("/no/such/decmpfs/dir/x.node"),
b"data",
);
assert!(matches!(out, Err(Error::Io { .. })));
}
#[test]
fn compress_bytes_guarded_success_classifies_via_the_backend_signal() {
// A faked successful apply over a file pre-seeded with `content`: the read-back
// oracle matches, so the backend's compressed_on_disk signal classifies the win.
let dir = std::env::temp_dir().join(format!("decmpfs-ok-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f.bin");
let content = b"the stored content bytes, pre-seeded";
std::fs::write(&path, content).unwrap();
let backend = FakeBackend {
detect: Support::Supported,
apply_errno: None,
};
let out = compress_bytes_guarded(&backend, &path, content).unwrap();
assert!(matches!(out, Outcome::NoGain { .. }), "got {out:?}");
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn unrelated_errors_propagate() {
assert_eq!(classify_skip(&err(std::io::ErrorKind::NotFound)), None);
assert_eq!(classify_skip(&std::io::Error::from_raw_os_error(2)), None); // ENOENT
}
#[test]
fn restore_is_a_noop_when_the_path_has_no_parent() {
// "/" has no parent → restore returns early without touching anything.
restore(std::path::Path::new("/"), b"x");
}
#[test]
fn not_loadable_result_is_restored_and_skipped() {
// Drive the in-place rollback without a corrupting backend: hand a
// `magic_before` the on-disk file no longer matches, so the post-apply gate
// sees "not loadable", restores the snapshot, and reports NotLoadable.
let dir = std::env::temp_dir().join(format!("decmpfs-notload-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f");
std::fs::write(&path, b"\x7fELF garbage the backend supposedly produced").unwrap();
let out = verify_loadable_or_restore(
&Os,
&path,
100,
[0xde, 0xad, 0xbe, 0xef],
b"the original bytes",
)
.unwrap();
assert!(matches!(
out,
Outcome::Skipped {
reason: SkipReason::NotLoadable
}
));
assert_eq!(
std::fs::read(&path).unwrap(),
b"the original bytes",
"snapshot restored"
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn read_back_mismatch_is_restored_and_skipped() {
// Drive the one-pass oracle rollback: the file on disk differs from the bytes
// we claim to have stored, so the read-back mismatches, the content is
// restored, and IntegrityRevert is reported.
let dir = std::env::temp_dir().join(format!("decmpfs-mismatch-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f");
std::fs::write(&path, b"what the broken backend actually wrote").unwrap();
let intended = b"the bytes the caller asked to store";
let out = verify_readback_or_restore(&Os, &path, intended).unwrap();
assert!(matches!(
out,
Outcome::Skipped {
reason: SkipReason::IntegrityRevert
}
));
assert_eq!(std::fs::read(&path).unwrap(), intended, "content restored");
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn restore_cleans_up_its_temp_when_the_rename_fails() {
// Renaming a temp file over an existing DIRECTORY fails → the temp is removed.
let dir = std::env::temp_dir().join(format!("decmpfs-rr-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let target = dir.join("a-dir");
std::fs::create_dir_all(&target).unwrap();
restore(&target, b"bytes");
let tmp = dir.join(format!(".decmpfs-restore-{}.tmp", std::process::id()));
assert!(!tmp.exists(), "temp left behind");
assert!(target.is_dir(), "directory target untouched");
std::fs::remove_dir_all(&dir).ok();
}
}
+34
src/unsupported.rs
//! Fallback for any OS without a backend — always Unsupported{PlatformBuild}.
use std::path::Path;
use crate::{Error, Support, UnsupportedReason};
pub(crate) fn detect(_path: &Path) -> Result<Support, Error> {
Ok(Support::Unsupported(UnsupportedReason::PlatformBuild))
}
pub(crate) fn is_already_compressed(_path: &Path) -> Result<bool, Error> {
Ok(false)
}
pub(crate) fn apply_inplace(_path: &Path) -> Result<(), Error> {
Ok(())
}
/// Never reached — `detect` always reports Unsupported, so `compress_bytes` takes
/// the plain-write fallback and never calls this. Present so the backend surface is
/// uniform across every cfg target.
pub(crate) fn apply_bytes(
_path: &Path,
_content: &[u8],
_mode: Option<std::fs::Permissions>,
) -> Result<(), Error> {
Ok(())
}
/// No FS-specific on-disk signal — apply_guarded falls back to the generic
/// allocated-bytes measurement (st_blocks / GetCompressedFileSizeW), which DOES
/// reflect the win on APFS and NTFS.
pub(crate) fn compressed_on_disk(_path: &Path) -> Result<Option<bool>, Error> {
Ok(None)
}
+109
src/verify.rs
//! Platform-agnostic measurement + loadability surface.
use std::io::Read;
use std::path::Path;
use crate::Error;
/// On-disk *allocated* bytes — `st_blocks * 512` on POSIX, `GetCompressedFileSizeW`
/// on Windows. Never `st_size`: transparent compression holds the logical size
/// constant, so only the allocation reveals the win.
pub(crate) fn on_disk_bytes(path: &Path) -> Result<u64, Error> {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let meta = std::fs::metadata(path).map_err(|source| Error::Io {
context: "stat",
source,
})?;
Ok(meta.blocks().saturating_mul(512))
}
#[cfg(windows)]
{
use std::os::windows::ffi::OsStrExt;
use windows_sys::Win32::Storage::FileSystem::GetCompressedFileSizeW;
let wide: Vec<u16> = path
.as_os_str()
.encode_wide()
.chain(std::iter::once(0))
.collect();
let mut high: u32 = 0;
// Returns the actual allocated size (post-NTFS-compression), low dword as the
// return value + high dword via the out-param. INVALID_FILE_SIZE (u32::MAX) is
// only an error if GetLastError is non-zero (it can also be a legit low dword).
let low = unsafe { GetCompressedFileSizeW(wide.as_ptr(), &mut high) };
if low == u32::MAX {
let err = std::io::Error::last_os_error();
if err.raw_os_error().unwrap_or(0) != 0 {
return Err(Error::Io {
context: "GetCompressedFileSizeW",
source: err,
});
}
}
Ok(((high as u64) << 32) | low as u64)
}
#[cfg(not(any(unix, windows)))]
{
let meta = std::fs::metadata(path).map_err(|source| Error::Io {
context: "stat",
source,
})?;
Ok(meta.len())
}
}
/// First 4 bytes — the native-binary magic (ELF `7f454c46`, Mach-O `cffaedfe`/…,
/// PE `4d5a`). Compared before/after apply to assert the file still loads, in
/// place of a content hash that transparent compression would render vacuous.
pub(crate) fn magic_prefix(path: &Path) -> Result<[u8; 4], Error> {
let mut file = std::fs::File::open(path).map_err(|source| Error::Io {
context: "open",
source,
})?;
let mut buf = [0u8; 4];
file.read(&mut buf).map_err(|source| Error::Io {
context: "read",
source,
})?;
Ok(buf)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn measures_allocation_and_reads_magic() {
let dir = std::env::temp_dir().join(format!("decmpfs-verify-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("f");
std::fs::write(&path, vec![0x7f; 9000]).unwrap();
assert!(
on_disk_bytes(&path).unwrap() > 0,
"allocated bytes reported"
);
assert_eq!(magic_prefix(&path).unwrap(), [0x7f; 4]);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn errors_on_a_missing_path() {
let p = std::path::Path::new("/no/such/verify/x");
assert!(on_disk_bytes(p).is_err());
assert!(magic_prefix(p).is_err());
}
// Opening a directory succeeds on unix, but read() on its fd fails (EISDIR) —
// exercising the read-error arm distinct from the open-error arm above.
#[cfg(unix)]
#[test]
fn magic_prefix_errors_when_the_read_fails_after_a_successful_open() {
let dir = std::env::temp_dir().join(format!("decmpfs-readfail-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
assert!(magic_prefix(&dir).is_err(), "read of a directory fd errors");
std::fs::remove_dir_all(&dir).ok();
}
}
+175
src/windows.rs
//! Windows backend — NTFS LZNT1 via `FSCTL_SET_COMPRESSION`.
//!
//! Codec choice is LZNT1, not WOF XPRESS/LZX, even though WOF-XPRESS decodes faster
//! and packs tighter. WOF is "write-once": opening the file for write strips the
//! compression back to plain, so every package-manager **reinstall** that rewrites
//! the `.node` would silently lose it — exactly the workload this targets. LZNT1
//! compresses the existing file IN PLACE (no copy), survives open-for-write, and
//! hardlink siblings share the file record so they compress together (same content;
//! acceptable). For a load-once addon the LZNT1-vs-XPRESS decode delta is marginal,
//! so reinstall-survival wins. Reversal condition: if a consumer re-applies
//! compression on every install (so write-strip is harmless), switch to WOF-XPRESS
//! (`FSCTL_SET_EXTERNAL_BACKING`) for the better ratio + faster decode.
//!
//! Detection gates on the volume's per-file-compression capability, which ReFS/FAT
//! lack.
use std::os::windows::ffi::OsStrExt;
use std::path::Path;
use std::io::Write;
use windows_sys::Win32::Foundation::{CloseHandle, HANDLE, INVALID_HANDLE_VALUE};
use windows_sys::Win32::Storage::FileSystem::{
CreateFileW, GetFileAttributesW, GetVolumeInformationByHandleW,
};
use windows_sys::Win32::System::IO::DeviceIoControl;
use crate::{io, Error, Support, UnsupportedReason};
const GENERIC_READ: u32 = 0x8000_0000;
const GENERIC_WRITE: u32 = 0x4000_0000;
const FILE_SHARE_READ: u32 = 0x0000_0001;
const OPEN_EXISTING: u32 = 3;
const CREATE_ALWAYS: u32 = 2;
const FSCTL_SET_COMPRESSION: u32 = 0x0009_C040;
const COMPRESSION_FORMAT_DEFAULT: u16 = 1; // LZNT1
const FILE_FILE_COMPRESSION: u32 = 0x0000_0010; // volume supports per-file compression
const FILE_ATTRIBUTE_COMPRESSED: u32 = 0x0000_0800;
const INVALID_FILE_ATTRIBUTES: u32 = u32::MAX;
fn wide(path: &Path) -> Vec<u16> {
path
.as_os_str()
.encode_wide()
.chain(std::iter::once(0))
.collect()
}
/// Owning wrapper so the handle is always closed, even on the error paths.
struct Handle(HANDLE);
impl Drop for Handle {
fn drop(&mut self) {
unsafe {
CloseHandle(self.0);
}
}
}
fn open(path: &Path, access: u32) -> Result<Handle, Error> {
open_with(path, access, OPEN_EXISTING)
}
fn open_with(path: &Path, access: u32, disposition: u32) -> Result<Handle, Error> {
let wpath = wide(path);
let handle = unsafe {
CreateFileW(
wpath.as_ptr(),
access,
FILE_SHARE_READ,
std::ptr::null(),
disposition,
0,
std::ptr::null_mut(),
)
};
if handle == INVALID_HANDLE_VALUE || handle.is_null() {
return Err(io("CreateFileW"));
}
Ok(Handle(handle))
}
/// Set LZNT1 compression on an open handle (the empty fresh file or an existing
/// one). Shared by `apply_inplace` and `apply_bytes`.
fn set_compression(handle: HANDLE) -> Result<(), Error> {
let format: u16 = COMPRESSION_FORMAT_DEFAULT;
let mut returned: u32 = 0;
let ok = unsafe {
DeviceIoControl(
handle,
FSCTL_SET_COMPRESSION,
(&format as *const u16).cast(),
std::mem::size_of::<u16>() as u32,
std::ptr::null_mut(),
0,
&mut returned,
std::ptr::null_mut(),
)
};
if ok == 0 {
return Err(io("FSCTL_SET_COMPRESSION"));
}
Ok(())
}
pub(crate) fn detect(path: &Path) -> Result<Support, Error> {
let handle = open(path, GENERIC_READ)?;
let mut flags: u32 = 0;
let ok = unsafe {
GetVolumeInformationByHandleW(
handle.0,
std::ptr::null_mut(),
0,
std::ptr::null_mut(),
std::ptr::null_mut(),
&mut flags,
std::ptr::null_mut(),
0,
)
};
if ok == 0 {
return Err(io("GetVolumeInformationByHandleW"));
}
if flags & FILE_FILE_COMPRESSION != 0 {
Ok(Support::Supported)
} else {
Ok(Support::Unsupported(UnsupportedReason::Filesystem))
}
}
pub(crate) fn is_already_compressed(path: &Path) -> Result<bool, Error> {
let attrs = unsafe { GetFileAttributesW(wide(path).as_ptr()) };
if attrs == INVALID_FILE_ATTRIBUTES {
return Err(io("GetFileAttributesW"));
}
Ok(attrs & FILE_ATTRIBUTE_COMPRESSED != 0)
}
pub(crate) fn apply_inplace(path: &Path) -> Result<(), Error> {
let handle = open(path, GENERIC_READ | GENERIC_WRITE)?;
set_compression(handle.0)
}
/// Write `content` to `path` as a fresh NTFS-compressed file in ONE pass: create
/// the file, FSCTL_SET_COMPRESSION on the EMPTY handle (so writes compress on the
/// way in — never a write-then-recompress), then write the bytes through the same
/// handle. `mode` is unused on Windows (no POSIX bits). Shared by `compress_bytes`.
pub(crate) fn apply_bytes(
path: &Path,
content: &[u8],
_mode: Option<std::fs::Permissions>,
) -> Result<(), Error> {
let handle = open_with(path, GENERIC_READ | GENERIC_WRITE, CREATE_ALWAYS)?;
set_compression(handle.0)?;
// Write through the same handle the compression flag was set on, so every block
// lands compressed. WriteFile via a std File would need a second open; instead
// borrow the raw handle into a File for the write, then forget it so Drop on the
// Handle (not the File) does the single CloseHandle.
use std::os::windows::io::FromRawHandle;
let mut file = unsafe { std::fs::File::from_raw_handle(handle.0 as _) };
let res = file.write_all(content).and_then(|()| file.flush());
// Don't let File's Drop close the handle — the Handle wrapper owns it.
std::mem::forget(file);
res.map_err(|source| Error::Io {
context: "write",
source,
})
}
/// No FS-specific on-disk signal — apply_guarded falls back to the generic
/// allocated-bytes measurement (st_blocks / GetCompressedFileSizeW), which DOES
/// reflect the win on APFS and NTFS.
pub(crate) fn compressed_on_disk(_path: &Path) -> Result<Option<bool>, Error> {
Ok(None)
}
+65
tests/btrfs.rs
//! Real-filesystem test for the btrfs backend. Skips unless DECMPFS_BTRFS_DIR
//! points at a mounted btrfs (the `ci/` Docker harness + CI btrfs job set it);
//! never runs on a dev macOS box. Exercises the PUBLIC surface: probe /
//! compress_file. The on-disk shrink is decided by FIEMAP's ENCODED flag, not
//! st_blocks (btrfs reports the logical size via st_blocks).
#![allow(clippy::print_stderr)] // CI diagnostics under --nocapture; not product code
use std::fs;
use std::path::PathBuf;
use decmpfs::{compress_file, probe, Outcome, Support};
mod common;
use common::fake_addon;
fn btrfs_dir() -> Option<PathBuf> {
std::env::var_os("DECMPFS_BTRFS_DIR").map(PathBuf::from)
}
#[test]
fn compress_file_applies_flag_round_trips_and_stays_loadable() {
let Some(dir) = btrfs_dir() else {
eprintln!("skip: DECMPFS_BTRFS_DIR unset");
return;
};
let path = dir.join("raw.node");
let raw = fake_addon();
fs::write(&path, &raw).unwrap();
assert_eq!(probe(&path).unwrap(), Support::Supported, "btrfs detected");
match compress_file(&path).unwrap() {
Outcome::Compressed { before, after } => {
eprintln!("FIEMAP-confirmed compressed (st_blocks {before} -> {after})");
}
other => panic!("expected Compressed (FIEMAP ENCODED), got {other:?}"),
}
// Transparent: reading back yields the identical bytes (still loadable).
assert_eq!(fs::read(&path).unwrap(), raw);
// Flag round-trip: the first call set FS_COMPR_FL, so a second short-circuits.
assert!(
matches!(
compress_file(&path).unwrap(),
Outcome::AlreadyCompressed { .. }
),
"second call must detect the compress flag set by the first"
);
}
#[test]
fn read_only_filesystem_is_skipped_not_errored() {
let Some(dir) = std::env::var_os("DECMPFS_BTRFS_RO_DIR").map(PathBuf::from) else {
eprintln!("skip: DECMPFS_BTRFS_RO_DIR unset");
return;
};
let path = dir.join("ro.node");
// Detection still reports Supported (it IS btrfs); the write can't happen on a
// read-only mount → fail-soft turns that into Skipped, never an Err.
match compress_file(&path).unwrap() {
Outcome::Skipped { reason } => eprintln!("read-only -> Skipped({reason:?})"),
other => panic!("expected Skipped on a read-only fs, got {other:?}"),
}
}
+15
tests/common/mod.rs
//! Shared fixtures for the real-filesystem integration tests (btrfs today, more
//! later). Not every test uses every helper, so dead_code is expected.
#![allow(dead_code)]
/// ELF magic + a compressible (non-zero, so it isn't stored as a sparse hole)
/// 2 MiB body — large enough for a real on-disk compression measurement.
pub fn fake_addon() -> Vec<u8> {
let mut raw = vec![0x7f, 0x45, 0x4c, 0x46];
let pattern = b"decmpfs compressible addon body pattern ";
while raw.len() < 2 * 1024 * 1024 {
raw.extend_from_slice(pattern);
}
raw.truncate(2 * 1024 * 1024);
raw
}
+264
-1
Cargo.lock

@@ -6,3 +6,266 @@ # This file is automatically @generated by Cargo.

[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]]
name = "cc"
version = "1.2.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96"
dependencies = [
"find-msvc-tools",
"jobserver",
"libc",
"shlex",
]
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
"libc",
]
[[package]]
name = "crypto-common"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "decmpfs"
version = "0.0.0"
version = "0.1.0"
dependencies = [
"libc",
"sha2",
"windows-sys",
"zstd",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasip2",
]
[[package]]
name = "jobserver"
version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
dependencies = [
"getrandom",
"libc",
]
[[package]]
name = "libc"
version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "pkg-config"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
[[package]]
name = "r-efi"
version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "shlex"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
[[package]]
name = "typenum"
version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "wasip2"
version = "1.0.4+wasi-0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
dependencies = [
"wit-bindgen",
]
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "wit-bindgen"
version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
[[package]]
name = "zstd"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.16+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
dependencies = [
"cc",
"pkg-config",
]
+37
-6
Cargo.toml

@@ -15,4 +15,3 @@ # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO

name = "decmpfs"
version = "0.0.0"
authors = ["John-David Dalton <john.david.dalton@gmail.com>"]
version = "0.1.0"
build = false

@@ -24,9 +23,9 @@ autolib = false

autobenches = false
description = "Transparent filesystem compression for installed package files (.node) — placeholder, under development."
readme = false
description = "Apply OS-level transparent filesystem compression (APFS decmpfs / btrfs / NTFS) to a file in place."
readme = "README.md"
keywords = [
"apfs",
"btrfs",
"compression",
"decmpfs",
"apfs",
"btrfs",
"ntfs",

@@ -36,5 +35,37 @@ ]

license = "MIT"
repository = "https://github.com/decmpfs/decmpfs"
[features]
addon = [
"dep:zstd",
"dep:sha2",
]
default = []
[lib]
name = "decmpfs"
path = "src/lib.rs"
[[test]]
name = "btrfs"
path = "tests/btrfs.rs"
[dependencies.sha2]
version = "=0.10.9"
optional = true
[dependencies.zstd]
version = "=0.13.3"
optional = true
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies.libc]
version = "=0.2.186"
[target."cfg(windows)".dependencies.windows-sys]
version = "=0.59.0"
features = [
"Win32_Foundation",
"Win32_Security",
"Win32_Storage_FileSystem",
"Win32_System_IO",
]
+685
-4
src/lib.rs

@@ -1,5 +0,686 @@

//! Placeholder for `decmpfs` — transparent filesystem compression for installed
//! package files (`.node` and other large artifacts), leveraging the operating
//! system's per-file compression (APFS decmpfs, btrfs, NTFS).
//! `decmpfs` — apply the operating system's transparent per-file compression to a file
//! in place: macOS APFS (decmpfs), Linux btrfs, Windows NTFS. The kernel decompresses
//! on read, so the file keeps its logical size + exact contents and loads at near-native
//! speed while taking less space on disk.
//!
//! This 0.0.0 release reserves the name while the crate is under development.
//! `compress_file(path)` detects the filesystem, applies compression, then verifies the
//! kernel reads the bytes back identically — rolling back on any failure. `probe(path)`
//! is the detect-only / capability-reporting half.
//!
//! Backends: btrfs (`FS_COMPR_FL` + the `btrfs.compression` property), NTFS
//! (`FSCTL_SET_COMPRESSION`), and macOS decmpfs (resource fork, kernel-roundtrip
//! verified); other targets report `Unsupported`.
//!
//! Contract: every `Outcome` is a SUCCESS; `Err` is reserved for genuine I/O failures
//! that leave the file's integrity unknown. An unsupported FS, a permission/lock issue,
//! an incompressible or too-large file are non-fatal `Outcome`s.
//!
//! Panic-free invariant: the deny below keeps non-test code free of the obvious panic
//! sources; all slice indexing is length-guarded.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))]
use std::path::Path;
/// What happened to the file. Only `Err` is a hard failure.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Outcome {
/// Applied and on-disk allocation actually decreased.
Compressed { before: u64, after: u64 },
/// Applied (or already set) but on-disk size did not drop — incompressible
/// or sub-cluster. Content is byte-identical and fully loadable.
NoGain { before: u64, after: u64 },
/// Already carried the compression flag/xattr before we touched it.
AlreadyCompressed { before: u64 },
/// This FS/OS has no per-file transparent compression (ext4, xfs, ZFS, ReFS,
/// FAT, tmpfs, overlay/network mounts). Caller falls through to the cache.
Unsupported { reason: UnsupportedReason },
/// Detected support but could not apply (permissions, lock, immutable,
/// rollback). Warn-and-continue; never a hard error.
Skipped { reason: SkipReason },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UnsupportedReason {
/// Filesystem (by allowlist) has no transparent compression.
Filesystem,
/// Network/overlay/bind mount where the signal is unreliable.
NetworkOrOverlay,
/// Built for an OS with no backend (or skeleton: not yet implemented).
PlatformBuild,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SkipReason {
/// EACCES / EPERM / EROFS — read-only or unowned (e.g. unprivileged container).
PermissionDenied,
/// A write handle is held / ETXTBSY / sharing violation; could not lock.
Busy,
/// UF_IMMUTABLE / SF_IMMUTABLE and we declined to toggle it.
Immutable,
/// EFS / FILE_ATTRIBUTE_ENCRYPTED.
Encrypted,
/// Applied, structural verification failed, rolled back to the original.
IntegrityRevert,
/// Post-apply loadability (magic-bytes) check failed, rolled back.
NotLoadable,
/// Exceeds a backend limit (e.g. decmpfs u32 offsets cap at 4 GiB).
TooLarge,
/// `compress_bytes` was handed a file the `Gate` excludes — written plain.
GateExcluded,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Support {
Supported,
AlreadyCompressed,
Unsupported(UnsupportedReason),
}
/// Genuine failures only. A capability/permission gap is an `Outcome`, not an `Error`.
#[derive(Debug)]
pub enum Error {
Io {
context: &'static str,
source: std::io::Error,
},
NotFound(std::path::PathBuf),
}
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Error::Io { context, source } => write!(f, "io error at {context}: {source}"),
Error::NotFound(p) => write!(f, "file not found: {}", p.display()),
}
}
}
impl std::error::Error for Error {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Error::Io { source, .. } => Some(source),
Error::NotFound(_) => None,
}
}
}
/// Wrap the last OS error with context — shared by every backend.
pub(crate) fn io(context: &'static str) -> Error {
Error::Io {
context,
source: std::io::Error::last_os_error(),
}
}
/// A NUL-checked C string from a path, for the unix backends that hand paths to
/// libc.
#[cfg(unix)]
pub(crate) fn cstring(path: &Path) -> Result<std::ffi::CString, Error> {
use std::os::unix::ffi::OsStrExt;
std::ffi::CString::new(path.as_os_str().as_bytes()).map_err(|_| Error::Io {
context: "path has interior NUL",
source: std::io::Error::from(std::io::ErrorKind::InvalidInput),
})
}
/// Detect-only, no mutation — for dry-run / capability reporting.
pub fn probe(path: &Path) -> Result<Support, Error> {
backend::detect(path)
}
/// THE entry point: detect → gate → apply → verify → rollback-on-failure.
/// Idempotent. Never panics. Never corrupts the file.
pub fn compress_file(path: &Path) -> Result<Outcome, Error> {
compress_file_with(&Os, path)
}
/// `compress_file` over an injectable [`Backend`] — production always threads
/// [`Os`]; tests drive the otherwise-dead `AlreadyCompressed`/`Unsupported` arms
/// with a fake.
fn compress_file_with<B: Backend>(backend: &B, path: &Path) -> Result<Outcome, Error> {
if !path.exists() {
return Err(Error::NotFound(path.to_path_buf()));
}
match backend.detect(path)? {
Support::Unsupported(reason) => Ok(Outcome::Unsupported { reason }),
Support::AlreadyCompressed => Ok(Outcome::AlreadyCompressed {
before: verify::on_disk_bytes(path)?,
}),
Support::Supported => safety::apply_guarded(backend, path),
}
}
/// THE install-time entry point: write `content` to `path` as an OS-compressed file
/// in ONE pass — never a write-then-read-back-recompress.
///
/// The caller (a package manager's CAS writer) has already decoded the raw addon
/// and matched it against `gate`. `compress_bytes` writes that exact byte stream
/// directly as a transparently-compressed file: macOS encodes the decmpfs from the
/// bytes onto a fresh inode; btrfs requests the codec on the empty temp then writes;
/// NTFS sets FSCTL_SET_COMPRESSION on the fresh handle then writes.
///
/// Fail-soft is the contract — this NEVER breaks an install. On an unsupported FS,
/// a permission/busy/too-large skip, or any backend error, it falls back to a plain
/// atomic write of `content` and reports the corresponding `Outcome` (the plain
/// write still lands the file). The kernel read-back is verified identical to
/// `content` before returning a compressed Outcome.
///
/// `gate` is honored here as a convenience: if `content` does not match the gate,
/// the file is written plain and `Outcome::Skipped { reason: GateExcluded }` is
/// returned. A caller that already gated can pass `&Gate::any()`.
pub fn compress_bytes(path: &Path, content: &[u8], gate: &Gate) -> Result<Outcome, Error> {
compress_bytes_with(&Os, path, content, gate)
}
/// `compress_bytes` over an injectable [`Backend`] — production always threads
/// [`Os`]; tests drive the plain-write fallback arms (a guarded skip/error, or a
/// non-compressing FS) that a real APFS write never reaches.
fn compress_bytes_with<B: Backend>(
backend: &B,
path: &Path,
content: &[u8],
gate: &Gate,
) -> Result<Outcome, Error> {
let name = path.to_string_lossy();
let normalized = name.replace('\\', "/");
if !gate.matches(&normalized, content.len() as u64) {
plain_write(path, content)?;
return Ok(Outcome::Skipped {
reason: SkipReason::GateExcluded,
});
}
// The target usually doesn't exist yet (a fresh CAS write), so the FS capability
// probe goes against the parent directory; `detect` statfs's / opens its argument
// and would error on a missing path.
let probe_target = if path.exists() {
path.to_path_buf()
} else {
match path.parent() {
Some(dir) => dir.to_path_buf(),
None => path.to_path_buf(),
}
};
match backend.detect(&probe_target) {
Ok(Support::Supported) => match safety::compress_bytes_guarded(backend, path, content) {
Ok(Outcome::Skipped { .. }) | Err(_) => {
// A guarded skip/error already restored or never wrote — ensure the file
// lands plain so the install is never missing the addon.
plain_write(path, content)?;
Ok(Outcome::Skipped {
reason: SkipReason::IntegrityRevert,
})
}
other => other,
},
Ok(Support::AlreadyCompressed) | Ok(Support::Unsupported(_)) | Err(_) => {
plain_write(path, content)?;
Ok(Outcome::Unsupported {
reason: UnsupportedReason::Filesystem,
})
}
}
}
/// Fail-soft plain atomic write: sibling temp + fsync + rename. The never-break-the
/// -install floor under every `compress_bytes` fallback.
fn plain_write(path: &Path, content: &[u8]) -> Result<(), Error> {
use std::io::Write;
let dir = path.parent().ok_or_else(|| Error::Io {
context: "no parent dir",
source: std::io::Error::from(std::io::ErrorKind::InvalidInput),
})?;
let name = path
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| "addon".to_string());
let tmp = dir.join(format!(".{name}.plain-{}.tmp", std::process::id()));
let res = (|| -> std::io::Result<()> {
let mut file = std::fs::File::create(&tmp)?;
file.write_all(content)?;
file.sync_all()
})();
if let Err(source) = res {
let _ = std::fs::remove_file(&tmp);
return Err(Error::Io {
context: "plain write temp",
source,
});
}
std::fs::rename(&tmp, path).map_err(|source| {
let _ = std::fs::remove_file(&tmp);
Error::Io {
context: "plain write rename",
source,
}
})
}
#[cfg(feature = "addon")]
pub mod addon;
mod gate;
mod safety;
mod verify;
pub use gate::{Gate, GateParseError, SizePredicate, DEFAULT_GLOB};
#[cfg(target_os = "linux")]
#[path = "linux.rs"]
mod backend;
#[cfg(target_os = "macos")]
#[path = "macos.rs"]
mod backend;
#[cfg(target_os = "windows")]
#[path = "windows.rs"]
mod backend;
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
#[path = "unsupported.rs"]
mod backend;
/// The OS compression backend as a trait, so the orchestration in `safety` can be
/// driven by a fake in tests — a real filesystem never produces a non-loadable
/// result or a mismatched read-back, so the rollback and plain-write fallback paths
/// are otherwise unreachable. Production always threads [`Os`]; static dispatch
/// monomorphizes it to the same code as a direct backend call (no vtable, no size
/// cost in a release build).
pub(crate) trait Backend {
fn detect(&self, path: &Path) -> Result<Support, Error>;
fn is_already_compressed(&self, path: &Path) -> Result<bool, Error>;
fn apply_inplace(&self, path: &Path) -> Result<(), Error>;
fn apply_bytes(
&self,
path: &Path,
content: &[u8],
mode: Option<std::fs::Permissions>,
) -> Result<(), Error>;
fn compressed_on_disk(&self, path: &Path) -> Result<Option<bool>, Error>;
}
/// The real, cfg-selected OS backend.
pub(crate) struct Os;
impl Backend for Os {
fn detect(&self, path: &Path) -> Result<Support, Error> {
backend::detect(path)
}
fn is_already_compressed(&self, path: &Path) -> Result<bool, Error> {
backend::is_already_compressed(path)
}
fn apply_inplace(&self, path: &Path) -> Result<(), Error> {
backend::apply_inplace(path)
}
fn apply_bytes(
&self,
path: &Path,
content: &[u8],
mode: Option<std::fs::Permissions>,
) -> Result<(), Error> {
backend::apply_bytes(path, content, mode)
}
fn compressed_on_disk(&self, path: &Path) -> Result<Option<bool>, Error> {
backend::compressed_on_disk(path)
}
}
/// A configurable in-memory backend for exercising the rollback and plain-write
/// fallback paths that a real filesystem never reaches.
#[cfg(test)]
pub(crate) struct FakeBackend {
pub(crate) detect: Support,
/// `None` → apply succeeds; `Some(errno)` → apply fails with that OS error.
pub(crate) apply_errno: Option<i32>,
}
#[cfg(test)]
impl FakeBackend {
fn apply_result(&self) -> Result<(), Error> {
match self.apply_errno {
None => Ok(()),
Some(errno) => Err(Error::Io {
context: "fake apply",
source: std::io::Error::from_raw_os_error(errno),
}),
}
}
}
#[cfg(test)]
impl Backend for FakeBackend {
fn detect(&self, _path: &Path) -> Result<Support, Error> {
Ok(self.detect)
}
fn is_already_compressed(&self, _path: &Path) -> Result<bool, Error> {
Ok(false)
}
fn apply_inplace(&self, _path: &Path) -> Result<(), Error> {
self.apply_result()
}
fn apply_bytes(
&self,
_path: &Path,
_content: &[u8],
_mode: Option<std::fs::Permissions>,
) -> Result<(), Error> {
self.apply_result()
}
fn compressed_on_disk(&self, _path: &Path) -> Result<Option<bool>, Error> {
Ok(Some(false))
}
}
#[cfg(test)]
mod tests {
use super::*;
fn scratch(tag: &str) -> std::path::PathBuf {
let dir = std::env::temp_dir().join(format!("decmpfs-{tag}-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
dir
}
// A minimal native-magic payload (ELF header) so a backend will attempt to
// compress it rather than skip a trivially-small file.
fn fake_addon() -> Vec<u8> {
let mut raw = vec![0x7f, 0x45, 0x4c, 0x46];
raw.extend_from_slice(&[7u8; 9000]);
raw
}
#[test]
fn compress_file_errors_when_missing() {
let p = std::path::Path::new("/no/such/addon.node");
assert!(matches!(compress_file(p), Err(Error::NotFound(_))));
}
#[test]
fn plain_write_errors_when_the_path_has_no_parent() {
// "/" has no parent directory → the no-parent guard fires before any write.
let out = plain_write(std::path::Path::new("/"), b"x");
assert!(matches!(
out,
Err(Error::Io {
context: "no parent dir",
..
})
));
}
#[test]
fn error_display_and_source() {
let nf = Error::NotFound(std::path::PathBuf::from("/x"));
assert!(nf.to_string().contains("not found"));
assert!(std::error::Error::source(&nf).is_none());
let io = Error::Io {
context: "ctx",
source: std::io::Error::from(std::io::ErrorKind::PermissionDenied),
};
assert!(io.to_string().contains("ctx"));
assert!(std::error::Error::source(&io).is_some());
}
#[cfg(unix)]
#[test]
fn probe_reports_a_support_variant_without_mutating() {
// probe never errors on an existing path — it returns a Support.
assert!(matches!(
probe(std::path::Path::new("/dev/null")),
Ok(Support::Supported | Support::AlreadyCompressed | Support::Unsupported(_))
));
}
#[cfg(unix)]
#[test]
fn compress_file_reports_unsupported_on_a_non_compressing_fs() {
// /dev/null exists but devfs has no compression backend → Unsupported.
let out = compress_file(std::path::Path::new("/dev/null"));
assert!(
matches!(out, Ok(Outcome::Unsupported { .. })),
"devfs → Unsupported, got {out:?}"
);
}
// APFS is always a compressing FS, so macOS exercises the full success path:
// compress_file → apply_guarded → backend::apply_inplace → verify → classify.
#[cfg(target_os = "macos")]
#[test]
fn compress_file_compresses_then_is_idempotent_and_transparent() {
let dir = scratch("ok");
let path = dir.join("addon.node");
std::fs::write(&path, fake_addon()).unwrap();
let out = compress_file(&path);
assert!(
matches!(
out,
Ok(Outcome::Compressed { .. } | Outcome::NoGain { .. } | Outcome::AlreadyCompressed { .. })
),
"writable addon on APFS → applied, got {out:?}"
);
// Transparent: the kernel hands back the exact original bytes.
assert_eq!(std::fs::read(&path).unwrap(), fake_addon());
// Idempotent: a second pass detects it's already compressed.
assert!(matches!(
compress_file(&path),
Ok(Outcome::AlreadyCompressed { .. })
));
std::fs::remove_dir_all(&dir).ok();
}
// compress_bytes one-pass: write bytes directly as an APFS-compressed file with
// no pre-existing original, then prove the kernel hands the exact bytes back.
#[cfg(target_os = "macos")]
#[test]
fn compress_bytes_one_pass_writes_compressed_and_reads_back_identical() {
let dir = scratch("bytes");
let path = dir.join("fresh.node");
let content = fake_addon();
// No file at `path` yet — compress_bytes creates it in one pass.
let out = compress_bytes(&path, &content, &Gate::any());
assert!(
matches!(
out,
Ok(Outcome::Compressed { .. } | Outcome::NoGain { .. })
),
"one-pass APFS write → applied, got {out:?}"
);
assert!(path.exists(), "file was created");
// Transparent: kernel read-back equals the bytes we asked to store.
assert_eq!(std::fs::read(&path).unwrap(), content);
// It really carries the compression flag (not a plain fallback write).
assert!(matches!(
compress_file(&path),
Ok(Outcome::AlreadyCompressed { .. })
));
std::fs::remove_dir_all(&dir).ok();
}
// A file the gate excludes is written PLAIN (never compressed) and reports
// Skipped(GateExcluded) — the install still gets the file.
#[cfg(unix)]
#[test]
fn compress_bytes_gate_excluded_writes_plain() {
let dir = scratch("gate");
let path = dir.join("not-an-addon.txt");
let content = b"plain text, not a .node".to_vec();
let gate = Gate::default(); // **/*.node
let out = compress_bytes(&path, &content, &gate);
assert!(
matches!(
out,
Ok(Outcome::Skipped {
reason: SkipReason::GateExcluded
})
),
"non-.node → GateExcluded, got {out:?}"
);
assert_eq!(std::fs::read(&path).unwrap(), content);
std::fs::remove_dir_all(&dir).ok();
}
#[cfg(unix)]
#[test]
fn compress_bytes_falls_back_to_plain_on_unsupported_fs() {
// A non-compressing FS (devfs) → plain write, Unsupported Outcome, file lands.
// /dev isn't writable by us, so target a temp path but force the gate to pass;
// temp on macOS is APFS (compresses) — instead assert the API never errors and
// the bytes land for the supported case is covered above. Here just exercise
// the gate-passing path lands bytes on any unix temp.
let dir = scratch("fallback");
let path = dir.join("x.node");
let content = fake_addon();
let out = compress_bytes(&path, &content, &Gate::any());
assert!(out.is_ok(), "never errors on a normal temp, got {out:?}");
assert_eq!(std::fs::read(&path).unwrap(), content, "bytes always land");
std::fs::remove_dir_all(&dir).ok();
}
#[cfg(unix)]
#[test]
fn compress_file_skips_a_read_only_file() {
// On a compressing FS a read-only file can't be opened rw → fail-soft turns the
// EACCES into Skipped(PermissionDenied). Root bypasses mode bits, so skip there.
if unsafe { libc::geteuid() } == 0 {
return;
}
let dir = scratch("ro");
let path = dir.join("addon.node");
std::fs::write(&path, fake_addon()).unwrap();
if !matches!(probe(&path), Ok(Support::Supported)) {
std::fs::remove_dir_all(&dir).ok();
return;
}
let mut perm = std::fs::metadata(&path).unwrap().permissions();
perm.set_readonly(true);
std::fs::set_permissions(&path, perm).unwrap();
let outcome = compress_file(&path);
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).ok();
assert!(
matches!(
outcome,
Ok(Outcome::Skipped {
reason: SkipReason::PermissionDenied
})
),
"read-only → Skipped(PermissionDenied), got {outcome:?}"
);
std::fs::remove_dir_all(&dir).ok();
}
// An existing target exercises the `path.exists()` probe-target branch and the
// fresh-inode rename that replaces the old contents.
#[cfg(target_os = "macos")]
#[test]
fn compress_bytes_overwrites_an_existing_file() {
let dir = scratch("overwrite");
let path = dir.join("addon.node");
std::fs::write(&path, b"stale contents").unwrap();
let content = fake_addon();
let out = compress_bytes(&path, &content, &Gate::any());
assert!(out.is_ok(), "overwrite never errors, got {out:?}");
assert_eq!(
std::fs::read(&path).unwrap(),
content,
"new bytes replace the old"
);
std::fs::remove_dir_all(&dir).ok();
}
// `path` is an existing directory: the backend builds its temp then can't rename
// a file over a directory, and the plain-write fallback can't either → a hard
// `Err` (genuine I/O failure), never a corrupt success. Exercises the backend
// rename-error cleanup and the `Err(_)` fallback arm of compress_bytes.
#[cfg(target_os = "macos")]
#[test]
fn compress_bytes_onto_a_directory_path_is_a_hard_error() {
let dir = scratch("dir-target");
let target = dir.join("a-dir");
std::fs::create_dir_all(&target).unwrap();
let out = compress_bytes(&target, &fake_addon(), &Gate::any());
assert!(out.is_err(), "cannot write a file over a directory, got {out:?}");
assert!(target.is_dir(), "the directory is left intact");
std::fs::remove_dir_all(&dir).ok();
}
// A read-only parent dir: the guarded backend write hits EACCES (classify_skip →
// Skipped), then the plain-write fallback also can't write → `Err`. Root bypasses
// mode bits, so skip there.
#[cfg(target_os = "macos")]
#[test]
fn compress_bytes_into_a_read_only_dir_is_fail_soft() {
if unsafe { libc::geteuid() } == 0 {
return;
}
use std::os::unix::fs::PermissionsExt;
let dir = scratch("ro-dir");
let locked = dir.join("locked");
std::fs::create_dir_all(&locked).unwrap();
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o555)).unwrap();
let out = compress_bytes(&locked.join("x.node"), &fake_addon(), &Gate::any());
// Restore write perms so the tree can be cleaned up.
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).ok();
assert!(out.is_err(), "a read-only dir admits no write, got {out:?}");
std::fs::remove_dir_all(&dir).ok();
}
// The `Support::AlreadyCompressed`-from-detect arm: a real macOS detect never
// returns it (it reports already-compressed via the apply path), so a fake drives
// it. Needs a real file for the on-disk-bytes read.
#[test]
fn compress_file_reports_already_compressed_from_detect() {
let dir = scratch("already-detect");
let path = dir.join("f.node");
std::fs::write(&path, fake_addon()).unwrap();
let backend = FakeBackend {
detect: Support::AlreadyCompressed,
apply_errno: None,
};
assert!(matches!(
compress_file_with(&backend, &path),
Ok(Outcome::AlreadyCompressed { .. })
));
std::fs::remove_dir_all(&dir).ok();
}
// detect → Unsupported: the bytes still land via a plain write, Outcome::Unsupported.
#[test]
fn compress_bytes_falls_back_to_plain_on_an_unsupported_fs() {
let dir = scratch("unsup");
let path = dir.join("x.node");
let content = fake_addon();
let backend = FakeBackend {
detect: Support::Unsupported(UnsupportedReason::Filesystem),
apply_errno: None,
};
let out = compress_bytes_with(&backend, &path, &content, &Gate::any());
assert!(matches!(out, Ok(Outcome::Unsupported { .. })), "got {out:?}");
assert_eq!(std::fs::read(&path).unwrap(), content, "bytes landed plain");
std::fs::remove_dir_all(&dir).ok();
}
// detect → Supported but the guarded apply is skipped (faked permission failure):
// the bytes land via a plain write, Outcome::Skipped(IntegrityRevert).
#[test]
fn compress_bytes_falls_back_to_plain_on_a_guarded_skip() {
let dir = scratch("guard-skip");
let path = dir.join("x.node");
let content = fake_addon();
let backend = FakeBackend {
detect: Support::Supported,
apply_errno: Some(13), // EACCES
};
let out = compress_bytes_with(&backend, &path, &content, &Gate::any());
assert!(
matches!(
out,
Ok(Outcome::Skipped {
reason: SkipReason::IntegrityRevert
})
),
"got {out:?}"
);
assert_eq!(std::fs::read(&path).unwrap(), content, "bytes landed plain");
std::fs::remove_dir_all(&dir).ok();
}
}
Cargo.toml.orig

Sorry, the diff of this file is not supported yet