@huggingface/xetchunk-wasm - npm Package Compare versions

Comparing version 0.0.1 to 0.0.3

dist/commonjs/hash-utils.d.ts

+import type { Chunk } from "./xet-chunker.js";
+/**
+ * file_hash = hmac(xorb_hash(chunks), zero_key)
+ *
+ * Matches Rust's `merklehash::file_hash` which calls
+ * `file_hash_with_salt(chunks, &[0; 32])`.
+ */
+export declare function fileHash(chunks: Chunk[]): Uint8Array;
+/**
+ * HMAC: blake3_keyed_hash(key_bytes, hash_bytes)
+ *
+ * Both inputs are 32-byte Uint8Arrays.
+ * Matches Rust's `DataHash::hmac`.
+ *
+ * Uses a fresh hasher per call since the key varies.
+ */
+export declare function hmac(hash: Uint8Array, key: Uint8Array): Uint8Array;
+/**
+ * Verification hash for a range of chunk hashes.
+ * Concatenates all 32-byte hashes and applies blake3_keyed_hash
+ * with VERIFICATION_KEY.
+ *
+ * Matches Rust's `chunk_verification::range_hash_from_chunks`.
+ */
+export declare function verificationHash(chunkHashes: Uint8Array[]): Uint8Array;
+//# sourceMappingURL=hash-utils.d.ts.map

dist/commonjs/hash-utils.d.ts.map

+{"version":3,"file":"hash-utils.d.ts","sourceRoot":"","sources":["../../src/hash-utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAa9C;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,UAAU,CAGpD;AAED;;;;;;;GAOG;AACH,wBAAgB,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,GAAG,UAAU,CAElE;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,UAAU,CAMtE"}

dist/commonjs/hash-utils.js

+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fileHash = fileHash;
+exports.hmac = hmac;
+exports.verificationHash = verificationHash;
+const blake3_jit_1 = require("@huggingface/blake3-jit");
+const xorb_hash_js_1 = require("./xorb-hash.js");
+const ZERO_KEY = new Uint8Array(32);
+const VERIFICATION_KEY = new Uint8Array([
+    127, 24, 87, 214, 206, 86, 237, 102, 18, 127, 249, 19, 231, 165, 195, 243, 164, 205, 38, 213, 181, 219, 73, 230,
+    65, 36, 152, 127, 40, 251, 148, 195,
+]);
+const fileHasher = blake3_jit_1.Hasher.newKeyed(ZERO_KEY);
+const verificationHasher = blake3_jit_1.Hasher.newKeyed(VERIFICATION_KEY);
+/**
+ * file_hash = hmac(xorb_hash(chunks), zero_key)
+ *
+ * Matches Rust's `merklehash::file_hash` which calls
+ * `file_hash_with_salt(chunks, &[0; 32])`.
+ */
+function fileHash(chunks) {
+    const xorb = (0, xorb_hash_js_1.xorbHash)(chunks);
+    return fileHasher.reset().update(xorb).finalize(32);
+}
+/**
+ * HMAC: blake3_keyed_hash(key_bytes, hash_bytes)
+ *
+ * Both inputs are 32-byte Uint8Arrays.
+ * Matches Rust's `DataHash::hmac`.
+ *
+ * Uses a fresh hasher per call since the key varies.
+ */
+function hmac(hash, key) {
+    return blake3_jit_1.Hasher.newKeyed(key).update(hash).finalize(32);
+}
+/**
+ * Verification hash for a range of chunk hashes.
+ * Concatenates all 32-byte hashes and applies blake3_keyed_hash
+ * with VERIFICATION_KEY.
+ *
+ * Matches Rust's `chunk_verification::range_hash_from_chunks`.
+ */
+function verificationHash(chunkHashes) {
+    const combined = new Uint8Array(chunkHashes.length * 32);
+    for (let i = 0; i < chunkHashes.length; i++) {
+        combined.set(chunkHashes[i], i * 32);
+    }
+    return verificationHasher.reset().update(combined).finalize(32);
+}

dist/commonjs/index.d.ts

+export { createChunker, finalize, nextBlock, getChunks, hashToHex, hexToBytes, type Chunk } from "./xet-chunker.js";
+export { xorbHash } from "./xorb-hash.js";
+export { fileHash, hmac, verificationHash } from "./hash-utils.js";
+//# sourceMappingURL=index.d.ts.map

dist/commonjs/index.d.ts.map

+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACpH,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC"}

dist/commonjs/index.js

+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.verificationHash = exports.hmac = exports.fileHash = exports.xorbHash = exports.hexToBytes = exports.hashToHex = exports.getChunks = exports.nextBlock = exports.finalize = exports.createChunker = void 0;
+var xet_chunker_js_1 = require("./xet-chunker.js");
+Object.defineProperty(exports, "createChunker", { enumerable: true, get: function () { return xet_chunker_js_1.createChunker; } });
+Object.defineProperty(exports, "finalize", { enumerable: true, get: function () { return xet_chunker_js_1.finalize; } });
+Object.defineProperty(exports, "nextBlock", { enumerable: true, get: function () { return xet_chunker_js_1.nextBlock; } });
+Object.defineProperty(exports, "getChunks", { enumerable: true, get: function () { return xet_chunker_js_1.getChunks; } });
+Object.defineProperty(exports, "hashToHex", { enumerable: true, get: function () { return xet_chunker_js_1.hashToHex; } });
+Object.defineProperty(exports, "hexToBytes", { enumerable: true, get: function () { return xet_chunker_js_1.hexToBytes; } });
+var xorb_hash_js_1 = require("./xorb-hash.js");
+Object.defineProperty(exports, "xorbHash", { enumerable: true, get: function () { return xorb_hash_js_1.xorbHash; } });
+var hash_utils_js_1 = require("./hash-utils.js");
+Object.defineProperty(exports, "fileHash", { enumerable: true, get: function () { return hash_utils_js_1.fileHash; } });
+Object.defineProperty(exports, "hmac", { enumerable: true, get: function () { return hash_utils_js_1.hmac; } });
+Object.defineProperty(exports, "verificationHash", { enumerable: true, get: function () { return hash_utils_js_1.verificationHash; } });

dist/commonjs/package.json

+{
+  "type": "commonjs"
+}

dist/commonjs/xet-chunker.d.ts

+export interface Chunk {
+    hash: Uint8Array;
+    length: number;
+}
+interface NextResult {
+    chunk: Chunk | null;
+    bytesConsumed: number;
+}
+declare class XetChunker {
+    private minimumChunk;
+    private maximumChunk;
+    private chunkBuf;
+    private curChunkLen;
+    private gear;
+    private blake3;
+    constructor(targetChunkSize?: number);
+    /**
+     * Streaming entry point: accepts an arbitrary slice of data, accumulates
+     * it, and emits a chunk when a boundary (or max size) is reached.
+     * Data is copied into an internal buffer because it may span calls.
+     */
+    next(data: Uint8Array, isFinal: boolean): NextResult;
+    /**
+     * Batch entry point: processes a large contiguous buffer and returns all
+     * complete chunks. Hashes directly from `data` — no intermediate copy
+     * to chunkBuf — for every chunk whose bytes are fully within `data`.
+     */
+    nextBlock(data: Uint8Array, isFinal: boolean): Chunk[];
+    finish(): Chunk | null;
+}
+export declare function createChunker(targetChunkSize?: number): XetChunker;
+export declare function nextBlock(chunker: XetChunker, data: Uint8Array): Chunk[];
+export declare function finalize(chunker: XetChunker): Chunk | null;
+export declare function getChunks(data: Uint8Array, targetChunkSize?: number): Chunk[];
+export declare function hashToHex(hash: Uint8Array): string;
+export declare function hexToBytes(hex: string): Uint8Array;
+export {};
+//# sourceMappingURL=xet-chunker.d.ts.map

dist/commonjs/xet-chunker.d.ts.map

+{"version":3,"file":"xet-chunker.d.ts","sourceRoot":"","sources":["../../src/xet-chunker.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,KAAK;IACrB,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CACf;AAED,UAAU,UAAU;IACnB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;CACtB;AAED,cAAM,UAAU;IACf,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,MAAM,CAAe;gBAEjB,eAAe,GAAE,MAA0B;IAkCvD;;;;OAIG;IACH,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,UAAU;IAwDpD;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,KAAK,EAAE;IA2DtD,MAAM,IAAI,KAAK,GAAG,IAAI;CAWtB;AAED,wBAAgB,aAAa,CAAC,eAAe,GAAE,MAA0B,GAAG,UAAU,CAErF;AAED,wBAAgB,SAAS,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,GAAG,KAAK,EAAE,CAExE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,KAAK,GAAG,IAAI,CAE1D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,eAAe,GAAE,MAA0B,GAAG,KAAK,EAAE,CAGhG;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAalD;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,CAQlD"}

dist/commonjs/xet-chunker.js

+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createChunker = createChunker;
+exports.nextBlock = nextBlock;
+exports.finalize = finalize;
+exports.getChunks = getChunks;
+exports.hashToHex = hashToHex;
+exports.hexToBytes = hexToBytes;
+const gearhash_jit_1 = require("gearhash-jit");
+const blake3_jit_1 = require("@huggingface/blake3-jit");
+const TARGET_CHUNK_SIZE = 64 * 1024; // 64KB
+const MINIMUM_CHUNK_DIVISOR = 8;
+const MAXIMUM_CHUNK_MULTIPLIER = 2;
+const HASH_WINDOW_SIZE = 64;
+const BLAKE3_DATA_KEY = new Uint8Array([
+    102, 151, 245, 119, 91, 149, 80, 222, 49, 53, 203, 172, 165, 151, 24, 28, 157, 228, 33, 16, 155, 235, 43, 88, 180,
+    208, 176, 75, 147, 173, 242, 41,
+]);
+class XetChunker {
+    minimumChunk;
+    maximumChunk;
+    chunkBuf;
+    curChunkLen;
+    gear;
+    blake3;
+    constructor(targetChunkSize = TARGET_CHUNK_SIZE) {
+        if (targetChunkSize <= 0) {
+            throw new Error("Target chunk size must be greater than 0");
+        }
+        if ((targetChunkSize & (targetChunkSize - 1)) !== 0) {
+            throw new Error("Target chunk size must be a power of 2");
+        }
+        if (targetChunkSize <= HASH_WINDOW_SIZE) {
+            throw new Error("Target chunk size must be greater than hash window size");
+        }
+        if (targetChunkSize >= Number.MAX_SAFE_INTEGER) {
+            throw new Error("Target chunk size must be less than Number.MAX_SAFE_INTEGER");
+        }
+        let mask = BigInt(targetChunkSize - 1);
+        let leadingZeros = 0;
+        for (let i = 63; i >= 0; i--) {
+            if ((mask & (1n << BigInt(i))) !== 0n) {
+                break;
+            }
+            leadingZeros++;
+        }
+        mask = mask << BigInt(leadingZeros);
+        const maximumChunk = targetChunkSize * MAXIMUM_CHUNK_MULTIPLIER;
+        this.minimumChunk = targetChunkSize / MINIMUM_CHUNK_DIVISOR;
+        this.maximumChunk = maximumChunk;
+        this.chunkBuf = new Uint8Array(maximumChunk);
+        this.curChunkLen = 0;
+        this.gear = new gearhash_jit_1.Hasher(mask);
+        this.blake3 = blake3_jit_1.Hasher.newKeyed(BLAKE3_DATA_KEY);
+    }
+    /**
+     * Streaming entry point: accepts an arbitrary slice of data, accumulates
+     * it, and emits a chunk when a boundary (or max size) is reached.
+     * Data is copied into an internal buffer because it may span calls.
+     */
+    next(data, isFinal) {
+        const nBytes = data.length;
+        let createChunk = false;
+        let consumeLen = 0;
+        if (nBytes !== 0) {
+            if (this.curChunkLen + HASH_WINDOW_SIZE < this.minimumChunk) {
+                const maxAdvance = Math.min(this.minimumChunk - this.curChunkLen - HASH_WINDOW_SIZE - 1, nBytes - consumeLen);
+                consumeLen += maxAdvance;
+                this.curChunkLen += maxAdvance;
+            }
+            const readEnd = Math.min(nBytes, consumeLen + this.maximumChunk - this.curChunkLen);
+            let bytesToNextBoundary;
+            const position = this.gear.nextMatch(data.subarray(consumeLen, readEnd));
+            if (position !== -1) {
+                bytesToNextBoundary = position;
+                createChunk = true;
+            }
+            else {
+                bytesToNextBoundary = readEnd - consumeLen;
+            }
+            if (bytesToNextBoundary + this.curChunkLen >= this.maximumChunk) {
+                bytesToNextBoundary = this.maximumChunk - this.curChunkLen;
+                createChunk = true;
+            }
+            this.curChunkLen += bytesToNextBoundary;
+            consumeLen += bytesToNextBoundary;
+            this.chunkBuf.set(data.subarray(0, consumeLen), this.curChunkLen - consumeLen);
+        }
+        if (createChunk || (isFinal && this.curChunkLen > 0)) {
+            const chunkData = this.chunkBuf.subarray(0, this.curChunkLen);
+            const hash = this.blake3.reset().update(chunkData).finalize(32);
+            const chunk = {
+                length: chunkData.length,
+                hash: hash,
+            };
+            this.curChunkLen = 0;
+            this.gear.resetHash();
+            return {
+                chunk,
+                bytesConsumed: consumeLen,
+            };
+        }
+        return {
+            chunk: null,
+            bytesConsumed: consumeLen,
+        };
+    }
+    /**
+     * Batch entry point: processes a large contiguous buffer and returns all
+     * complete chunks. Hashes directly from `data` — no intermediate copy
+     * to chunkBuf — for every chunk whose bytes are fully within `data`.
+     */
+    nextBlock(data, isFinal) {
+        const chunks = [];
+        let pos = 0;
+        // Drain any leftover from a previous nextBlock / next call.
+        while (pos < data.length && this.curChunkLen > 0) {
+            const result = this.next(data.subarray(pos), false);
+            if (result.chunk)
+                chunks.push(result.chunk);
+            pos += result.bytesConsumed;
+        }
+        const minSkip = this.minimumChunk > HASH_WINDOW_SIZE
+            ? this.minimumChunk - HASH_WINDOW_SIZE - 1
+            : 0;
+        while (pos < data.length) {
+            const chunkStart = pos;
+            const scanStart = Math.min(pos + minSkip, data.length);
+            const scanEnd = Math.min(data.length, pos + this.maximumChunk);
+            const position = this.gear.nextMatch(data.subarray(scanStart, scanEnd));
+            let chunkEnd;
+            let foundBoundary;
+            if (position !== -1 && scanStart + position - chunkStart <= this.maximumChunk) {
+                chunkEnd = scanStart + position;
+                foundBoundary = true;
+            }
+            else if (scanEnd - chunkStart >= this.maximumChunk) {
+                chunkEnd = chunkStart + this.maximumChunk;
+                foundBoundary = true;
+            }
+            else {
+                foundBoundary = false;
+                chunkEnd = scanEnd;
+            }
+            if (foundBoundary) {
+                const hash = this.blake3.reset()
+                    .update(data.subarray(chunkStart, chunkEnd))
+                    .finalize(32);
+                chunks.push({ length: chunkEnd - chunkStart, hash });
+                pos = chunkEnd;
+                this.gear.resetHash();
+            }
+            else if (isFinal) {
+                const hash = this.blake3.reset()
+                    .update(data.subarray(chunkStart))
+                    .finalize(32);
+                chunks.push({ length: data.length - chunkStart, hash });
+                pos = data.length;
+            }
+            else {
+                this.chunkBuf.set(data.subarray(chunkStart), 0);
+                this.curChunkLen = data.length - chunkStart;
+                pos = data.length;
+            }
+        }
+        return chunks;
+    }
+    finish() {
+        if (this.curChunkLen > 0) {
+            const chunkData = this.chunkBuf.subarray(0, this.curChunkLen);
+            const hash = this.blake3.reset().update(chunkData).finalize(32);
+            const chunk = { length: this.curChunkLen, hash };
+            this.curChunkLen = 0;
+            this.gear.resetHash();
+            return chunk;
+        }
+        return null;
+    }
+}
+function createChunker(targetChunkSize = TARGET_CHUNK_SIZE) {
+    return new XetChunker(targetChunkSize);
+}
+function nextBlock(chunker, data) {
+    return chunker.nextBlock(data, false);
+}
+function finalize(chunker) {
+    return chunker.finish();
+}
+function getChunks(data, targetChunkSize = TARGET_CHUNK_SIZE) {
+    const chunker = createChunker(targetChunkSize);
+    return chunker.nextBlock(data, true);
+}
+function hashToHex(hash) {
+    const view = new DataView(hash.buffer, hash.byteOffset, hash.byteLength);
+    const u64 = view.getBigUint64(0, true);
+    const u64_2 = view.getBigUint64(8, true);
+    const u64_3 = view.getBigUint64(16, true);
+    const u64_4 = view.getBigUint64(24, true);
+    return (u64.toString(16).padStart(16, "0") +
+        u64_2.toString(16).padStart(16, "0") +
+        u64_3.toString(16).padStart(16, "0") +
+        u64_4.toString(16).padStart(16, "0"));
+}
+function hexToBytes(hex) {
+    const bytes = new Uint8Array(32);
+    const view = new DataView(bytes.buffer);
+    view.setBigUint64(0, BigInt("0x" + hex.slice(0, 16)), true);
+    view.setBigUint64(8, BigInt("0x" + hex.slice(16, 32)), true);
+    view.setBigUint64(16, BigInt("0x" + hex.slice(32, 48)), true);
+    view.setBigUint64(24, BigInt("0x" + hex.slice(48, 64)), true);
+    return bytes;
+}

dist/commonjs/xorb-hash.d.ts

+import type { Chunk } from "./xet-chunker.js";
+export declare function xorbHash(chunks: Chunk[]): Uint8Array;
+//# sourceMappingURL=xorb-hash.d.ts.map

dist/commonjs/xorb-hash.d.ts.map

+{"version":3,"file":"xorb-hash.d.ts","sourceRoot":"","sources":["../../src/xorb-hash.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAc9C,wBAAgB,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,UAAU,CA8BpD"}

dist/commonjs/xorb-hash.js

+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.xorbHash = xorbHash;
+const blake3_jit_1 = require("@huggingface/blake3-jit");
+const xet_chunker_js_1 = require("./xet-chunker.js");
+const MEAN_CHUNK_PER_NODE = 4;
+const BLAKE3_NODE_KEY = new Uint8Array([
+    1, 126, 197, 199, 165, 71, 41, 150, 253, 148, 102, 102, 180, 138, 2, 230, 93, 221, 83, 111, 55, 199, 109, 210, 248,
+    99, 82, 230, 74, 83, 113, 63,
+]);
+const INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH = 3 * 8;
+const nodeHasher = blake3_jit_1.Hasher.newKeyed(BLAKE3_NODE_KEY);
+function xorbHash(chunks) {
+    if (chunks.length === 0) {
+        return new Uint8Array(32);
+    }
+    let currentChunks = chunks;
+    while (currentChunks.length > 1) {
+        const nodes = [];
+        let currentIndex = 0;
+        let numOfChildrenSoFar = 0;
+        for (let i = 0; i < currentChunks.length; i++) {
+            if (i === currentChunks.length - 1 ||
+                numOfChildrenSoFar === 2 * MEAN_CHUNK_PER_NODE ||
+                (numOfChildrenSoFar >= 2 &&
+                    currentChunks[i].hash[INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH] % MEAN_CHUNK_PER_NODE === 0)) {
+                nodes.push(mergedHashOfSequence(currentChunks.slice(currentIndex, i + 1)));
+                currentIndex = i + 1;
+                numOfChildrenSoFar = 0;
+            }
+            else {
+                numOfChildrenSoFar++;
+            }
+        }
+        currentChunks = nodes;
+    }
+    return currentChunks[0].hash;
+}
+/**
+ * Matches Rust's `merged_hash_of_sequence`: serializes each entry as
+ * "{hash_hex} : {length_decimal}\n" then hashes with BLAKE3_NODE_KEY.
+ */
+function mergedHashOfSequence(chunks) {
+    let text = "";
+    let totalLength = 0;
+    for (const chunk of chunks) {
+        text += (0, xet_chunker_js_1.hashToHex)(chunk.hash) + " : " + chunk.length + "\n";
+        totalLength += chunk.length;
+    }
+    const bytes = new Uint8Array(text.length);
+    for (let i = 0; i < text.length; i++) {
+        bytes[i] = text.charCodeAt(i);
+    }
+    const hash = nodeHasher.reset().update(bytes).finalize(32);
+    return { hash, length: totalLength };
+}

dist/esm/hash-utils.d.ts

+import type { Chunk } from "./xet-chunker.js";
+/**
+ * file_hash = hmac(xorb_hash(chunks), zero_key)
+ *
+ * Matches Rust's `merklehash::file_hash` which calls
+ * `file_hash_with_salt(chunks, &[0; 32])`.
+ */
+export declare function fileHash(chunks: Chunk[]): Uint8Array;
+/**
+ * HMAC: blake3_keyed_hash(key_bytes, hash_bytes)
+ *
+ * Both inputs are 32-byte Uint8Arrays.
+ * Matches Rust's `DataHash::hmac`.
+ *
+ * Uses a fresh hasher per call since the key varies.
+ */
+export declare function hmac(hash: Uint8Array, key: Uint8Array): Uint8Array;
+/**
+ * Verification hash for a range of chunk hashes.
+ * Concatenates all 32-byte hashes and applies blake3_keyed_hash
+ * with VERIFICATION_KEY.
+ *
+ * Matches Rust's `chunk_verification::range_hash_from_chunks`.
+ */
+export declare function verificationHash(chunkHashes: Uint8Array[]): Uint8Array;
+//# sourceMappingURL=hash-utils.d.ts.map

dist/esm/hash-utils.d.ts.map

+{"version":3,"file":"hash-utils.d.ts","sourceRoot":"","sources":["../../src/hash-utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAa9C;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,UAAU,CAGpD;AAED;;;;;;;GAOG;AACH,wBAAgB,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,GAAG,UAAU,CAElE;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,UAAU,CAMtE"}

dist/esm/hash-utils.js

+import { Hasher } from "@huggingface/blake3-jit";
+import { xorbHash } from "./xorb-hash.js";
+const ZERO_KEY = new Uint8Array(32);
+const VERIFICATION_KEY = new Uint8Array([
+    127, 24, 87, 214, 206, 86, 237, 102, 18, 127, 249, 19, 231, 165, 195, 243, 164, 205, 38, 213, 181, 219, 73, 230,
+    65, 36, 152, 127, 40, 251, 148, 195,
+]);
+const fileHasher = Hasher.newKeyed(ZERO_KEY);
+const verificationHasher = Hasher.newKeyed(VERIFICATION_KEY);
+/**
+ * file_hash = hmac(xorb_hash(chunks), zero_key)
+ *
+ * Matches Rust's `merklehash::file_hash` which calls
+ * `file_hash_with_salt(chunks, &[0; 32])`.
+ */
+export function fileHash(chunks) {
+    const xorb = xorbHash(chunks);
+    return fileHasher.reset().update(xorb).finalize(32);
+}
+/**
+ * HMAC: blake3_keyed_hash(key_bytes, hash_bytes)
+ *
+ * Both inputs are 32-byte Uint8Arrays.
+ * Matches Rust's `DataHash::hmac`.
+ *
+ * Uses a fresh hasher per call since the key varies.
+ */
+export function hmac(hash, key) {
+    return Hasher.newKeyed(key).update(hash).finalize(32);
+}
+/**
+ * Verification hash for a range of chunk hashes.
+ * Concatenates all 32-byte hashes and applies blake3_keyed_hash
+ * with VERIFICATION_KEY.
+ *
+ * Matches Rust's `chunk_verification::range_hash_from_chunks`.
+ */
+export function verificationHash(chunkHashes) {
+    const combined = new Uint8Array(chunkHashes.length * 32);
+    for (let i = 0; i < chunkHashes.length; i++) {
+        combined.set(chunkHashes[i], i * 32);
+    }
+    return verificationHasher.reset().update(combined).finalize(32);
+}

dist/esm/index.d.ts

+export { createChunker, finalize, nextBlock, getChunks, hashToHex, hexToBytes, type Chunk } from "./xet-chunker.js";
+export { xorbHash } from "./xorb-hash.js";
+export { fileHash, hmac, verificationHash } from "./hash-utils.js";
+//# sourceMappingURL=index.d.ts.map

dist/esm/index.d.ts.map

+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACpH,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC"}

dist/esm/index.js

+export { createChunker, finalize, nextBlock, getChunks, hashToHex, hexToBytes } from "./xet-chunker.js";
+export { xorbHash } from "./xorb-hash.js";
+export { fileHash, hmac, verificationHash } from "./hash-utils.js";

dist/esm/package.json

+{
+  "type": "module"
+}

dist/esm/xet-chunker.d.ts

+export interface Chunk {
+    hash: Uint8Array;
+    length: number;
+}
+interface NextResult {
+    chunk: Chunk | null;
+    bytesConsumed: number;
+}
+declare class XetChunker {
+    private minimumChunk;
+    private maximumChunk;
+    private chunkBuf;
+    private curChunkLen;
+    private gear;
+    private blake3;
+    constructor(targetChunkSize?: number);
+    /**
+     * Streaming entry point: accepts an arbitrary slice of data, accumulates
+     * it, and emits a chunk when a boundary (or max size) is reached.
+     * Data is copied into an internal buffer because it may span calls.
+     */
+    next(data: Uint8Array, isFinal: boolean): NextResult;
+    /**
+     * Batch entry point: processes a large contiguous buffer and returns all
+     * complete chunks. Hashes directly from `data` — no intermediate copy
+     * to chunkBuf — for every chunk whose bytes are fully within `data`.
+     */
+    nextBlock(data: Uint8Array, isFinal: boolean): Chunk[];
+    finish(): Chunk | null;
+}
+export declare function createChunker(targetChunkSize?: number): XetChunker;
+export declare function nextBlock(chunker: XetChunker, data: Uint8Array): Chunk[];
+export declare function finalize(chunker: XetChunker): Chunk | null;
+export declare function getChunks(data: Uint8Array, targetChunkSize?: number): Chunk[];
+export declare function hashToHex(hash: Uint8Array): string;
+export declare function hexToBytes(hex: string): Uint8Array;
+export {};
+//# sourceMappingURL=xet-chunker.d.ts.map

dist/esm/xet-chunker.d.ts.map

+{"version":3,"file":"xet-chunker.d.ts","sourceRoot":"","sources":["../../src/xet-chunker.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,KAAK;IACrB,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CACf;AAED,UAAU,UAAU;IACnB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;CACtB;AAED,cAAM,UAAU;IACf,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,MAAM,CAAe;gBAEjB,eAAe,GAAE,MAA0B;IAkCvD;;;;OAIG;IACH,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,UAAU;IAwDpD;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,KAAK,EAAE;IA2DtD,MAAM,IAAI,KAAK,GAAG,IAAI;CAWtB;AAED,wBAAgB,aAAa,CAAC,eAAe,GAAE,MAA0B,GAAG,UAAU,CAErF;AAED,wBAAgB,SAAS,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,GAAG,KAAK,EAAE,CAExE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,KAAK,GAAG,IAAI,CAE1D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,eAAe,GAAE,MAA0B,GAAG,KAAK,EAAE,CAGhG;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAalD;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,CAQlD"}

dist/esm/xet-chunker.js

+import { Hasher } from "gearhash-jit";
+import { Hasher as Blake3Hasher } from "@huggingface/blake3-jit";
+const TARGET_CHUNK_SIZE = 64 * 1024; // 64KB
+const MINIMUM_CHUNK_DIVISOR = 8;
+const MAXIMUM_CHUNK_MULTIPLIER = 2;
+const HASH_WINDOW_SIZE = 64;
+const BLAKE3_DATA_KEY = new Uint8Array([
+    102, 151, 245, 119, 91, 149, 80, 222, 49, 53, 203, 172, 165, 151, 24, 28, 157, 228, 33, 16, 155, 235, 43, 88, 180,
+    208, 176, 75, 147, 173, 242, 41,
+]);
+class XetChunker {
+    minimumChunk;
+    maximumChunk;
+    chunkBuf;
+    curChunkLen;
+    gear;
+    blake3;
+    constructor(targetChunkSize = TARGET_CHUNK_SIZE) {
+        if (targetChunkSize <= 0) {
+            throw new Error("Target chunk size must be greater than 0");
+        }
+        if ((targetChunkSize & (targetChunkSize - 1)) !== 0) {
+            throw new Error("Target chunk size must be a power of 2");
+        }
+        if (targetChunkSize <= HASH_WINDOW_SIZE) {
+            throw new Error("Target chunk size must be greater than hash window size");
+        }
+        if (targetChunkSize >= Number.MAX_SAFE_INTEGER) {
+            throw new Error("Target chunk size must be less than Number.MAX_SAFE_INTEGER");
+        }
+        let mask = BigInt(targetChunkSize - 1);
+        let leadingZeros = 0;
+        for (let i = 63; i >= 0; i--) {
+            if ((mask & (1n << BigInt(i))) !== 0n) {
+                break;
+            }
+            leadingZeros++;
+        }
+        mask = mask << BigInt(leadingZeros);
+        const maximumChunk = targetChunkSize * MAXIMUM_CHUNK_MULTIPLIER;
+        this.minimumChunk = targetChunkSize / MINIMUM_CHUNK_DIVISOR;
+        this.maximumChunk = maximumChunk;
+        this.chunkBuf = new Uint8Array(maximumChunk);
+        this.curChunkLen = 0;
+        this.gear = new Hasher(mask);
+        this.blake3 = Blake3Hasher.newKeyed(BLAKE3_DATA_KEY);
+    }
+    /**
+     * Streaming entry point: accepts an arbitrary slice of data, accumulates
+     * it, and emits a chunk when a boundary (or max size) is reached.
+     * Data is copied into an internal buffer because it may span calls.
+     */
+    next(data, isFinal) {
+        const nBytes = data.length;
+        let createChunk = false;
+        let consumeLen = 0;
+        if (nBytes !== 0) {
+            if (this.curChunkLen + HASH_WINDOW_SIZE < this.minimumChunk) {
+                const maxAdvance = Math.min(this.minimumChunk - this.curChunkLen - HASH_WINDOW_SIZE - 1, nBytes - consumeLen);
+                consumeLen += maxAdvance;
+                this.curChunkLen += maxAdvance;
+            }
+            const readEnd = Math.min(nBytes, consumeLen + this.maximumChunk - this.curChunkLen);
+            let bytesToNextBoundary;
+            const position = this.gear.nextMatch(data.subarray(consumeLen, readEnd));
+            if (position !== -1) {
+                bytesToNextBoundary = position;
+                createChunk = true;
+            }
+            else {
+                bytesToNextBoundary = readEnd - consumeLen;
+            }
+            if (bytesToNextBoundary + this.curChunkLen >= this.maximumChunk) {
+                bytesToNextBoundary = this.maximumChunk - this.curChunkLen;
+                createChunk = true;
+            }
+            this.curChunkLen += bytesToNextBoundary;
+            consumeLen += bytesToNextBoundary;
+            this.chunkBuf.set(data.subarray(0, consumeLen), this.curChunkLen - consumeLen);
+        }
+        if (createChunk || (isFinal && this.curChunkLen > 0)) {
+            const chunkData = this.chunkBuf.subarray(0, this.curChunkLen);
+            const hash = this.blake3.reset().update(chunkData).finalize(32);
+            const chunk = {
+                length: chunkData.length,
+                hash: hash,
+            };
+            this.curChunkLen = 0;
+            this.gear.resetHash();
+            return {
+                chunk,
+                bytesConsumed: consumeLen,
+            };
+        }
+        return {
+            chunk: null,
+            bytesConsumed: consumeLen,
+        };
+    }
+    /**
+     * Batch entry point: processes a large contiguous buffer and returns all
+     * complete chunks. Hashes directly from `data` — no intermediate copy
+     * to chunkBuf — for every chunk whose bytes are fully within `data`.
+     */
+    nextBlock(data, isFinal) {
+        const chunks = [];
+        let pos = 0;
+        // Drain any leftover from a previous nextBlock / next call.
+        while (pos < data.length && this.curChunkLen > 0) {
+            const result = this.next(data.subarray(pos), false);
+            if (result.chunk)
+                chunks.push(result.chunk);
+            pos += result.bytesConsumed;
+        }
+        const minSkip = this.minimumChunk > HASH_WINDOW_SIZE
+            ? this.minimumChunk - HASH_WINDOW_SIZE - 1
+            : 0;
+        while (pos < data.length) {
+            const chunkStart = pos;
+            const scanStart = Math.min(pos + minSkip, data.length);
+            const scanEnd = Math.min(data.length, pos + this.maximumChunk);
+            const position = this.gear.nextMatch(data.subarray(scanStart, scanEnd));
+            let chunkEnd;
+            let foundBoundary;
+            if (position !== -1 && scanStart + position - chunkStart <= this.maximumChunk) {
+                chunkEnd = scanStart + position;
+                foundBoundary = true;
+            }
+            else if (scanEnd - chunkStart >= this.maximumChunk) {
+                chunkEnd = chunkStart + this.maximumChunk;
+                foundBoundary = true;
+            }
+            else {
+                foundBoundary = false;
+                chunkEnd = scanEnd;
+            }
+            if (foundBoundary) {
+                const hash = this.blake3.reset()
+                    .update(data.subarray(chunkStart, chunkEnd))
+                    .finalize(32);
+                chunks.push({ length: chunkEnd - chunkStart, hash });
+                pos = chunkEnd;
+                this.gear.resetHash();
+            }
+            else if (isFinal) {
+                const hash = this.blake3.reset()
+                    .update(data.subarray(chunkStart))
+                    .finalize(32);
+                chunks.push({ length: data.length - chunkStart, hash });
+                pos = data.length;
+            }
+            else {
+                this.chunkBuf.set(data.subarray(chunkStart), 0);
+                this.curChunkLen = data.length - chunkStart;
+                pos = data.length;
+            }
+        }
+        return chunks;
+    }
+    finish() {
+        if (this.curChunkLen > 0) {
+            const chunkData = this.chunkBuf.subarray(0, this.curChunkLen);
+            const hash = this.blake3.reset().update(chunkData).finalize(32);
+            const chunk = { length: this.curChunkLen, hash };
+            this.curChunkLen = 0;
+            this.gear.resetHash();
+            return chunk;
+        }
+        return null;
+    }
+}
+export function createChunker(targetChunkSize = TARGET_CHUNK_SIZE) {
+    return new XetChunker(targetChunkSize);
+}
+export function nextBlock(chunker, data) {
+    return chunker.nextBlock(data, false);
+}
+export function finalize(chunker) {
+    return chunker.finish();
+}
+export function getChunks(data, targetChunkSize = TARGET_CHUNK_SIZE) {
+    const chunker = createChunker(targetChunkSize);
+    return chunker.nextBlock(data, true);
+}
+export function hashToHex(hash) {
+    const view = new DataView(hash.buffer, hash.byteOffset, hash.byteLength);
+    const u64 = view.getBigUint64(0, true);
+    const u64_2 = view.getBigUint64(8, true);
+    const u64_3 = view.getBigUint64(16, true);
+    const u64_4 = view.getBigUint64(24, true);
+    return (u64.toString(16).padStart(16, "0") +
+        u64_2.toString(16).padStart(16, "0") +
+        u64_3.toString(16).padStart(16, "0") +
+        u64_4.toString(16).padStart(16, "0"));
+}
+export function hexToBytes(hex) {
+    const bytes = new Uint8Array(32);
+    const view = new DataView(bytes.buffer);
+    view.setBigUint64(0, BigInt("0x" + hex.slice(0, 16)), true);
+    view.setBigUint64(8, BigInt("0x" + hex.slice(16, 32)), true);
+    view.setBigUint64(16, BigInt("0x" + hex.slice(32, 48)), true);
+    view.setBigUint64(24, BigInt("0x" + hex.slice(48, 64)), true);
+    return bytes;
+}

dist/esm/xorb-hash.d.ts

+import type { Chunk } from "./xet-chunker.js";
+export declare function xorbHash(chunks: Chunk[]): Uint8Array;
+//# sourceMappingURL=xorb-hash.d.ts.map

dist/esm/xorb-hash.d.ts.map

+{"version":3,"file":"xorb-hash.d.ts","sourceRoot":"","sources":["../../src/xorb-hash.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAc9C,wBAAgB,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,UAAU,CA8BpD"}

dist/esm/xorb-hash.js

+import { Hasher } from "@huggingface/blake3-jit";
+import { hashToHex } from "./xet-chunker.js";
+const MEAN_CHUNK_PER_NODE = 4;
+const BLAKE3_NODE_KEY = new Uint8Array([
+    1, 126, 197, 199, 165, 71, 41, 150, 253, 148, 102, 102, 180, 138, 2, 230, 93, 221, 83, 111, 55, 199, 109, 210, 248,
+    99, 82, 230, 74, 83, 113, 63,
+]);
+const INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH = 3 * 8;
+const nodeHasher = Hasher.newKeyed(BLAKE3_NODE_KEY);
+export function xorbHash(chunks) {
+    if (chunks.length === 0) {
+        return new Uint8Array(32);
+    }
+    let currentChunks = chunks;
+    while (currentChunks.length > 1) {
+        const nodes = [];
+        let currentIndex = 0;
+        let numOfChildrenSoFar = 0;
+        for (let i = 0; i < currentChunks.length; i++) {
+            if (i === currentChunks.length - 1 ||
+                numOfChildrenSoFar === 2 * MEAN_CHUNK_PER_NODE ||
+                (numOfChildrenSoFar >= 2 &&
+                    currentChunks[i].hash[INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH] % MEAN_CHUNK_PER_NODE === 0)) {
+                nodes.push(mergedHashOfSequence(currentChunks.slice(currentIndex, i + 1)));
+                currentIndex = i + 1;
+                numOfChildrenSoFar = 0;
+            }
+            else {
+                numOfChildrenSoFar++;
+            }
+        }
+        currentChunks = nodes;
+    }
+    return currentChunks[0].hash;
+}
+/**
+ * Matches Rust's `merged_hash_of_sequence`: serializes each entry as
+ * "{hash_hex} : {length_decimal}\n" then hashes with BLAKE3_NODE_KEY.
+ */
+function mergedHashOfSequence(chunks) {
+    let text = "";
+    let totalLength = 0;
+    for (const chunk of chunks) {
+        text += hashToHex(chunk.hash) + " : " + chunk.length + "\n";
+        totalLength += chunk.length;
+    }
+    const bytes = new Uint8Array(text.length);
+    for (let i = 0; i < text.length; i++) {
+        bytes[i] = text.charCodeAt(i);
+    }
+    const hash = nodeHasher.reset().update(bytes).finalize(32);
+    return { hash, length: totalLength };
+}

package.json

 {
-  "name": "@huggingface/xetchunk-wasm",
-  "version": "0.0.1",
-  "keywords": [
-    "xet",
-    "chunk",
-    "chunking",
-    "assemblyscript",
-    "assembly",
-    "wasm"
-  ],
-  "dependencies": {
-    "@huggingface/blake3-wasm": "0.0.2",
-    "@huggingface/gearhash-wasm": "0.0.1"
-  },
-  "type": "module",
-  "exports": {
-    ".": {
-      "import": "./build/release.js",
-      "types": "./build/release.d.ts"
-    },
-    "./assembly": {
-      "import": "./assembly/index.ts"
-    },
-    "./wasm": {
-      "import": "./build/release.wasm"
-    }
-  },
-  "main": "./build/release.js",
-  "types": "./build/release.d.ts",
-  "devDependencies": {
-    "assemblyscript": "0.27.36",
-    "@huggingface/splitmix64-wasm": "0.0.1"
-  },
-  "files": [
-    "build/release.js",
-    "build/release.d.ts",
-    "build/release.wasm",
-    "build/release.wat",
-    "build/release.wasm.map",
-    "README.md",
-    "asconfig.json",
-    "assembly"
-  ],
-  "scripts": {
-    "build:debug": "asc assembly/index.ts --target debug",
-    "build:release": "asc assembly/index.ts --target release",
-    "build": "pnpm run build:debug && npm run build:release",
-    "test": "vitest run"
-  }
-}
+	"name": "@huggingface/xetchunk-wasm",
+	"version": "0.0.3",
+	"description": "Content-defined chunking and hashing for Hugging Face Xet storage",
+	"keywords": [
+		"xet",
+		"chunk",
+		"chunking"
+	],
+	"license": "MIT",
+	"author": "Hugging Face",
+	"publishConfig": {
+		"access": "public"
+	},
+	"files": [
+		"dist",
+		"README.md"
+	],
+	"scripts": {
+		"prepare": "tshy",
+		"test": "vitest run",
+		"bench": "node tests/bench.js"
+	},
+	"tshy": {
+		"exports": {
+			".": "./src/index.ts",
+			"./package.json": "./package.json"
+		}
+	},
+	"dependencies": {
+		"gearhash-jit": "workspace:*",
+		"@huggingface/blake3-jit": "^0.0.1"
+	},
+	"devDependencies": {
+		"@huggingface/splitmix64-wasm": "workspace:*"
+	},
+	"type": "module",
+	"exports": {
+		".": {
+			"import": {
+				"types": "./dist/esm/index.d.ts",
+				"default": "./dist/esm/index.js"
+			},
+			"require": {
+				"types": "./dist/commonjs/index.d.ts",
+				"default": "./dist/commonjs/index.js"
+			}
+		},
+		"./package.json": "./package.json"
+	},
+	"main": "./dist/commonjs/index.js",
+	"types": "./dist/commonjs/index.d.ts",
+	"module": "./dist/esm/index.js"
+}

README.md

@@ -1,27 +0,62 @@
-JS and WASM implementations of https://github.com/huggingface/xet-core/blob/main/deduplication/src/chunking.rs
+# @huggingface/xetchunk-wasm
 
-Using [AssemblyScript](https://www.assemblyscript.org/) to generate a lean WASM.
+Content-defined chunking and hashing for Hugging Face [Xet storage](https://huggingface.co/docs/hub/storage-regions), matching the [Rust reference implementation](https://github.com/huggingface/xet-core/blob/main/deduplication/src/chunking.rs).
 
+Uses [`gearhash-jit`](https://www.npmjs.com/package/gearhash-jit) for fast GEAR rolling hash boundary detection and [`@huggingface/blake3-jit`](https://www.npmjs.com/package/@huggingface/blake3-jit) for BLAKE3 chunk hashing.
+
 ## Usage
 
-```javascript
-import { createChunker, getChunks, nextBlock, finalize } from '@huggingface/xetchunk-wasm';
+```typescript
+import { createChunker, nextBlock, finalize, getChunks, hashToHex, xorbHash, fileHash } from '@huggingface/xetchunk-wasm';
 
-const TARGET_CHUNK_SIZE = Math.pow(2, 12);
+// One-shot: chunk all data at once
+const data = new Uint8Array(1_000_000);
+const chunks = getChunks(data);
 
-// Create a Uint8Array of data to search through
-const data = new Uint8Array(1000000); // Example: 1MB of data
-// ... fill data with your content ...
+console.log(`${chunks.length} chunks`);
+console.log('xorb hash:', hashToHex(xorbHash(chunks)));
+console.log('file hash:', hashToHex(fileHash(chunks)));
 
-const chunks = getChunks(data, TARGET_CHUNK_SIZE);
+// Streaming: process data incrementally
+const chunker = createChunker();
 
-// Alternative, in case your data is streaming
-const chunker = createChunker(TARGET_CHUNK_SIZE);
-
-for await (const data of source) {
-  const chunks = nextBlock(chunker, data);
-  console.log(chunks);
+for await (const buf of source) {
+  const chunks = nextBlock(chunker, buf);
+  for (const chunk of chunks) {
+    console.log(hashToHex(chunk.hash), chunk.length);
+  }
 }
 
-console.log("last chunk", finalize(chunker));
+const lastChunk = finalize(chunker);
 ```
+
+## API
+
+### Chunking
+
+- **`createChunker(targetChunkSize?: number)`** — Create a chunker (default 64KB target).
+- **`nextBlock(chunker, data: Uint8Array): Chunk[]`** — Feed data, get complete chunks.
+- **`finalize(chunker): Chunk | null`** — Flush remaining data as a final chunk.
+- **`getChunks(data: Uint8Array, targetChunkSize?: number): Chunk[]`** — One-shot convenience.
+
+### Hash functions
+
+All hash functions return `Uint8Array` (32 bytes). Use `hashToHex()` to convert to hex strings.
+
+- **`xorbHash(chunks: Chunk[]): Uint8Array`** — Merkle tree hash over chunks (matches Rust `xorb_hash`).
+- **`fileHash(chunks: Chunk[]): Uint8Array`** — File-level hash (matches Rust `file_hash`).
+- **`hmac(hash: Uint8Array, key: Uint8Array): Uint8Array`** — BLAKE3 keyed hash (matches Rust `DataHash::hmac`).
+- **`verificationHash(chunkHashes: Uint8Array[]): Uint8Array`** — Range verification hash (matches Rust `range_hash_from_chunks`).
+
+### Utilities
+
+- **`hashToHex(hash: Uint8Array): string`** — Convert 32-byte hash to hex string.
+- **`hexToBytes(hex: string): Uint8Array`** — Convert 64-char hex string to 32 bytes.
+
+## Benchmarking
+
+```shell
+pnpm --filter @huggingface/xetchunk-wasm bench
+# or with a specific file:
+pnpm --filter @huggingface/xetchunk-wasm bench path/to/large-file
+```

asconfig.json

-{
-	"targets": {
-		"debug": {
-			"outFile": "build/debug.wasm",
-			"textFile": "build/debug.wat",
-			"sourceMap": true,
-			"debug": true
-		},
-		"release": {
-			"outFile": "build/release.wasm",
-			"textFile": "build/release.wat",
-			"sourceMap": true,
-			"optimizeLevel": 3,
-			"shrinkLevel": 0,
-			"converge": false,
-			"noAssert": false
-		}
-	},
-	"options": {
-		"bindings": "esm"
-	}
-}

assembly/index.ts

-export { createChunker, finalize, nextBlock, getChunks } from "./xet-chunker";

assembly/tsconfig.json

-{
-	"extends": "../node_modules/.pnpm/assemblyscript@0.27.36/node_modules/assemblyscript/std/assembly.json",
-	"include": ["./**/*.ts"]
-}

assembly/xet-chunker.ts

-import { nextMatch } from "@huggingface/gearhash-wasm/assembly";
-import { blake3 } from "@huggingface/blake3-wasm/assembly";
-
-// Constants
-const TARGET_CHUNK_SIZE: i32 = 64 * 1024; // 64KB
-const MINIMUM_CHUNK_DIVISOR: i32 = 8;
-const MAXIMUM_CHUNK_MULTIPLIER: i32 = 2;
-const HASH_WINDOW_SIZE: i32 = 64;
-
-export class Chunk {
-	hash: Uint8Array;
-	length: i32;
-}
-
-// Type for the next() method return value
-class NextResult {
-	chunk: Chunk | null;
-	bytesConsumed: i32;
-
-	constructor(chunk: Chunk | null, bytesConsumed: i32) {
-		this.chunk = chunk;
-		this.bytesConsumed = bytesConsumed;
-	}
-}
-
-class XetChunker {
-	private minimumChunk: i32;
-	private maximumChunk: i32;
-	private mask: u64;
-	private chunkBuf: Uint8Array;
-	private curChunkLen: i32;
-	private hash: u64;
-
-	constructor(targetChunkSize: i32 = TARGET_CHUNK_SIZE) {
-		// Validate target chunk size is a power of 2
-		assert(targetChunkSize > 0, "Target chunk size must be greater than 0");
-		assert((targetChunkSize & (targetChunkSize - 1)) == 0, "Target chunk size must be a power of 2");
-		assert(targetChunkSize > HASH_WINDOW_SIZE, "Target chunk size must be greater than hash window size");
-		assert(targetChunkSize < i32.MAX_VALUE, "Target chunk size must be less than i32.MAX_VALUE");
-
-		let mask = (targetChunkSize - 1) as u64;
-		// Shift mask left by leading zeros count
-		mask = mask << clz(mask);
-
-		const maximumChunk = targetChunkSize * MAXIMUM_CHUNK_MULTIPLIER;
-
-		this.minimumChunk = targetChunkSize / MINIMUM_CHUNK_DIVISOR;
-		this.maximumChunk = maximumChunk;
-		this.mask = mask;
-		this.chunkBuf = new Uint8Array(maximumChunk);
-		this.curChunkLen = 0;
-		this.hash = 0;
-	}
-
-	next(data: Uint8Array, isFinal: boolean): NextResult {
-		const nBytes = data.length;
-		let createChunk = false;
-		let consumeLen: i32 = 0;
-
-		if (nBytes != 0) {
-			// Skip minimum chunk size
-			if (this.curChunkLen + HASH_WINDOW_SIZE < this.minimumChunk) {
-				const maxAdvance = min(this.minimumChunk - this.curChunkLen - HASH_WINDOW_SIZE - 1, nBytes - consumeLen);
-				consumeLen += maxAdvance;
-				this.curChunkLen += maxAdvance;
-			}
-
-			// Calculate read end
-			const readEnd = min(nBytes, consumeLen + this.maximumChunk - this.curChunkLen);
-
-			let bytesToNextBoundary: i32;
-			const matchResult = nextMatch(data.subarray(consumeLen, readEnd), this.mask, this.hash);
-
-			if (matchResult.position != -1) {
-				bytesToNextBoundary = matchResult.position;
-				createChunk = true;
-				this.hash = matchResult.hash;
-			} else {
-				bytesToNextBoundary = readEnd - consumeLen;
-				this.hash = matchResult.hash;
-			}
-
-			// Check if we hit maximum chunk
-			if (bytesToNextBoundary + this.curChunkLen >= this.maximumChunk) {
-				bytesToNextBoundary = this.maximumChunk - this.curChunkLen;
-				createChunk = true;
-			}
-
-			this.curChunkLen += bytesToNextBoundary;
-			consumeLen += bytesToNextBoundary;
-
-			// Copy data to chunk buffer
-			this.chunkBuf.set(data.subarray(0, consumeLen), this.curChunkLen - consumeLen);
-		}
-
-		if (createChunk || (isFinal && this.curChunkLen > 0)) {
-			const chunkData = this.chunkBuf.subarray(0, this.curChunkLen);
-			const chunk: Chunk = {
-				length: chunkData.length,
-				hash: blake3(chunkData),
-			};
-			this.curChunkLen = 0;
-			this.hash = 0;
-			return new NextResult(chunk, consumeLen);
-		}
-
-		return new NextResult(null, consumeLen);
-	}
-
-	nextBlock(data: Uint8Array, isFinal: boolean): Chunk[] {
-		const chunks: Chunk[] = [];
-		let pos: i32 = 0;
-
-		while (pos < data.length) {
-			const result = this.next(data.subarray(pos), isFinal);
-			if (result.chunk) {
-				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-				chunks.push(result.chunk!);
-			}
-			pos += result.bytesConsumed;
-		}
-
-		return chunks;
-	}
-
-	finish(): Chunk | null {
-		return this.next(new Uint8Array(0), true).chunk;
-	}
-}
-
-export function createChunker(targetChunkSize: i32 = TARGET_CHUNK_SIZE): XetChunker {
-	const chunker = new XetChunker(targetChunkSize);
-
-	return chunker;
-}
-
-export function nextBlock(chunker: XetChunker, data: Uint8Array): Chunk[] {
-	return chunker.nextBlock(data, false);
-}
-
-export function finalize(chunker: XetChunker): Chunk | null {
-	return chunker.finish();
-}
-
-export function getChunks(data: Uint8Array, targetChunkSize: i32 = TARGET_CHUNK_SIZE): Chunk[] {
-	// console.log(`getChunks: ${targetChunkSize} ${data.length}`);
-	const chunker = createChunker(targetChunkSize);
-	return chunker.nextBlock(data, true);
-}

build/release.d.ts

-/** Exported memory */
-export declare const memory: WebAssembly.Memory;
-/**
- * assembly/xet-chunker/createChunker
- * @param targetChunkSize `i32`
- * @returns `assembly/xet-chunker/XetChunker`
- */
-export declare function createChunker(targetChunkSize?: number): __Internref7;
-/**
- * assembly/xet-chunker/finalize
- * @param chunker `assembly/xet-chunker/XetChunker`
- * @returns `assembly/xet-chunker/Chunk | null`
- */
-export declare function finalize(chunker: __Internref7): __Record9<never> | null;
-/**
- * assembly/xet-chunker/nextBlock
- * @param chunker `assembly/xet-chunker/XetChunker`
- * @param data `~lib/typedarray/Uint8Array`
- * @returns `~lib/array/Array<assembly/xet-chunker/Chunk>`
- */
-export declare function nextBlock(chunker: __Internref7, data: Uint8Array): Array<__Record9<never>>;
-/**
- * assembly/xet-chunker/getChunks
- * @param data `~lib/typedarray/Uint8Array`
- * @param targetChunkSize `i32`
- * @returns `~lib/array/Array<assembly/xet-chunker/Chunk>`
- */
-export declare function getChunks(data: Uint8Array, targetChunkSize?: number): Array<__Record9<never>>;
-/** assembly/xet-chunker/XetChunker */
-declare class __Internref7 extends Number {
-  private __nominal7: symbol;
-  private __nominal0: symbol;
-}
-/** assembly/xet-chunker/Chunk */
-declare interface __Record9<TOmittable> {
-  /** @type `~lib/typedarray/Uint8Array` */
-  hash: Uint8Array;
-  /** @type `i32` */
-  length: number | TOmittable;
-}

build/release.js

-async function instantiate(module, imports = {}) {
-  const adaptedImports = {
-    env: Object.assign(Object.create(globalThis), imports.env || {}, {
-      abort(message, fileName, lineNumber, columnNumber) {
-        // ~lib/builtins/abort(~lib/string/String | null?, ~lib/string/String | null?, u32?, u32?) => void
-        message = __liftString(message >>> 0);
-        fileName = __liftString(fileName >>> 0);
-        lineNumber = lineNumber >>> 0;
-        columnNumber = columnNumber >>> 0;
-        (() => {
-          // @external.js
-          throw Error(`${message} in ${fileName}:${lineNumber}:${columnNumber}`);
-        })();
-      },
-    }),
-  };
-  const { exports } = await WebAssembly.instantiate(module, adaptedImports);
-  const memory = exports.memory || imports.env.memory;
-  const adaptedExports = Object.setPrototypeOf({
-    createChunker(targetChunkSize) {
-      // assembly/xet-chunker/createChunker(i32?) => assembly/xet-chunker/XetChunker
-      exports.__setArgumentsLength(arguments.length);
-      return __liftInternref(exports.createChunker(targetChunkSize) >>> 0);
-    },
-    finalize(chunker) {
-      // assembly/xet-chunker/finalize(assembly/xet-chunker/XetChunker) => assembly/xet-chunker/Chunk | null
-      chunker = __lowerInternref(chunker) || __notnull();
-      return __liftRecord9(exports.finalize(chunker) >>> 0);
-    },
-    nextBlock(chunker, data) {
-      // assembly/xet-chunker/nextBlock(assembly/xet-chunker/XetChunker, ~lib/typedarray/Uint8Array) => ~lib/array/Array<assembly/xet-chunker/Chunk>
-      chunker = __retain(__lowerInternref(chunker) || __notnull());
-      data = __lowerTypedArray(Uint8Array, 8, 0, data) || __notnull();
-      try {
-        return __liftArray(pointer => __liftRecord9(__getU32(pointer)), 2, exports.nextBlock(chunker, data) >>> 0);
-      } finally {
-        __release(chunker);
-      }
-    },
-    getChunks(data, targetChunkSize) {
-      // assembly/xet-chunker/getChunks(~lib/typedarray/Uint8Array, i32?) => ~lib/array/Array<assembly/xet-chunker/Chunk>
-      data = __lowerTypedArray(Uint8Array, 8, 0, data) || __notnull();
-      exports.__setArgumentsLength(arguments.length);
-      return __liftArray(pointer => __liftRecord9(__getU32(pointer)), 2, exports.getChunks(data, targetChunkSize) >>> 0);
-    },
-  }, exports);
-  function __liftRecord9(pointer) {
-    // assembly/xet-chunker/Chunk
-    // Hint: Opt-out from lifting as a record by providing an empty constructor
-    if (!pointer) return null;
-    return {
-      hash: __liftTypedArray(Uint8Array, __getU32(pointer + 0)),
-      length: __getI32(pointer + 4),
-    };
-  }
-  function __liftString(pointer) {
-    if (!pointer) return null;
-    const
-      end = pointer + new Uint32Array(memory.buffer)[pointer - 4 >>> 2] >>> 1,
-      memoryU16 = new Uint16Array(memory.buffer);
-    let
-      start = pointer >>> 1,
-      string = "";
-    while (end - start > 1024) string += String.fromCharCode(...memoryU16.subarray(start, start += 1024));
-    return string + String.fromCharCode(...memoryU16.subarray(start, end));
-  }
-  function __liftArray(liftElement, align, pointer) {
-    if (!pointer) return null;
-    const
-      dataStart = __getU32(pointer + 4),
-      length = __dataview.getUint32(pointer + 12, true),
-      values = new Array(length);
-    for (let i = 0; i < length; ++i) values[i] = liftElement(dataStart + (i << align >>> 0));
-    return values;
-  }
-  function __liftTypedArray(constructor, pointer) {
-    if (!pointer) return null;
-    return new constructor(
-      memory.buffer,
-      __getU32(pointer + 4),
-      __dataview.getUint32(pointer + 8, true) / constructor.BYTES_PER_ELEMENT
-    ).slice();
-  }
-  function __lowerTypedArray(constructor, id, align, values) {
-    if (values == null) return 0;
-    const
-      length = values.length,
-      buffer = exports.__pin(exports.__new(length << align, 1)) >>> 0,
-      header = exports.__new(12, id) >>> 0;
-    __setU32(header + 0, buffer);
-    __dataview.setUint32(header + 4, buffer, true);
-    __dataview.setUint32(header + 8, length << align, true);
-    new constructor(memory.buffer, buffer, length).set(values);
-    exports.__unpin(buffer);
-    return header;
-  }
-  class Internref extends Number {}
-  const registry = new FinalizationRegistry(__release);
-  function __liftInternref(pointer) {
-    if (!pointer) return null;
-    const sentinel = new Internref(__retain(pointer));
-    registry.register(sentinel, pointer);
-    return sentinel;
-  }
-  function __lowerInternref(value) {
-    if (value == null) return 0;
-    if (value instanceof Internref) return value.valueOf();
-    throw TypeError("internref expected");
-  }
-  const refcounts = new Map();
-  function __retain(pointer) {
-    if (pointer) {
-      const refcount = refcounts.get(pointer);
-      if (refcount) refcounts.set(pointer, refcount + 1);
-      else refcounts.set(exports.__pin(pointer), 1);
-    }
-    return pointer;
-  }
-  function __release(pointer) {
-    if (pointer) {
-      const refcount = refcounts.get(pointer);
-      if (refcount === 1) exports.__unpin(pointer), refcounts.delete(pointer);
-      else if (refcount) refcounts.set(pointer, refcount - 1);
-      else throw Error(`invalid refcount '${refcount}' for reference '${pointer}'`);
-    }
-  }
-  function __notnull() {
-    throw TypeError("value must not be null");
-  }
-  let __dataview = new DataView(memory.buffer);
-  function __setU32(pointer, value) {
-    try {
-      __dataview.setUint32(pointer, value, true);
-    } catch {
-      __dataview = new DataView(memory.buffer);
-      __dataview.setUint32(pointer, value, true);
-    }
-  }
-  function __getI32(pointer) {
-    try {
-      return __dataview.getInt32(pointer, true);
-    } catch {
-      __dataview = new DataView(memory.buffer);
-      return __dataview.getInt32(pointer, true);
-    }
-  }
-  function __getU32(pointer) {
-    try {
-      return __dataview.getUint32(pointer, true);
-    } catch {
-      __dataview = new DataView(memory.buffer);
-      return __dataview.getUint32(pointer, true);
-    }
-  }
-  return adaptedExports;
-}
-export const {
-  memory,
-  createChunker,
-  finalize,
-  nextBlock,
-  getChunks,
-} = await (async url => instantiate(
-  await (async () => {
-    const isNodeOrBun = typeof process != "undefined" && process.versions != null && (process.versions.node != null || process.versions.bun != null);
-    if (isNodeOrBun) { return globalThis.WebAssembly.compile(await (await import("node:fs/promises")).readFile(url)); }
-    else { return await globalThis.WebAssembly.compileStreaming(globalThis.fetch(url)); }
-  })(), {
-  }
-))(new URL("release.wasm", import.meta.url));

LICENSE

-MIT License
-
-Copyright (c) 2023 Hugging Face
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Fixed alerts

Network access

Supply chain risk

This module accesses the network.

No contributors or author data

Maintenance

Package does not specify a list of contributors or an author in package.json.

Improved metrics

Dev dependency count

1

-50%

Number of package files

28

133.33%

Lines of code

771

115.36%

Number of low maintenance alerts

1

-50%

Number of lines in readme file

63

125%

Worsened metrics

Total package byte prevSize

39140

-93.77%

Dependency changes

• + Added

  @huggingface/blake3-jit@^0.0.1

• + Added

  gearhash-jit@workspace:*

• + Added

  @huggingface/blake3-jit@0.0.1
  (transitive)

• - Removed

  @huggingface/blake3-wasm@0.0.2

• - Removed

  @huggingface/gearhash-wasm@0.0.1

• - Removed

  @huggingface/blake3-wasm@0.0.2
  (transitive)

• - Removed

  @huggingface/gearhash-wasm@0.0.1
  (transitive)