@huggingface/xetchunk-wasm - npm Package Compare versions

Comparing version 0.0.3 to 0.0.4

src/hash-utils.ts

+import { Hasher } from "@huggingface/blake3-jit";
+import type { Chunk } from "./xet-chunker.js";
+import { xorbHash } from "./xorb-hash.js";
+
+const ZERO_KEY = new Uint8Array(32);
+
+const VERIFICATION_KEY = new Uint8Array([
+	127, 24, 87, 214, 206, 86, 237, 102, 18, 127, 249, 19, 231, 165, 195, 243, 164, 205, 38, 213, 181, 219, 73, 230,
+	65, 36, 152, 127, 40, 251, 148, 195,
+]);
+
+const fileHasher = Hasher.newKeyed(ZERO_KEY);
+const verificationHasher = Hasher.newKeyed(VERIFICATION_KEY);
+
+/**
+ * file_hash = hmac(xorb_hash(chunks), zero_key)
+ *
+ * Matches Rust's `merklehash::file_hash` which calls
+ * `file_hash_with_salt(chunks, &[0; 32])`.
+ */
+export function fileHash(chunks: Chunk[]): Uint8Array {
+	const xorb = xorbHash(chunks);
+	return fileHasher.reset().update(xorb).finalize(32);
+}
+
+/**
+ * HMAC: blake3_keyed_hash(key_bytes, hash_bytes)
+ *
+ * Both inputs are 32-byte Uint8Arrays.
+ * Matches Rust's `DataHash::hmac`.
+ *
+ * Uses a fresh hasher per call since the key varies.
+ */
+export function hmac(hash: Uint8Array, key: Uint8Array): Uint8Array {
+	return Hasher.newKeyed(key).update(hash).finalize(32);
+}
+
+/**
+ * Verification hash for a range of chunk hashes.
+ * Concatenates all 32-byte hashes and applies blake3_keyed_hash
+ * with VERIFICATION_KEY.
+ *
+ * Matches Rust's `chunk_verification::range_hash_from_chunks`.
+ */
+export function verificationHash(chunkHashes: Uint8Array[]): Uint8Array {
+	const combined = new Uint8Array(chunkHashes.length * 32);
+	for (let i = 0; i < chunkHashes.length; i++) {
+		combined.set(chunkHashes[i], i * 32);
+	}
+	return verificationHasher.reset().update(combined).finalize(32);
+}

src/index.ts

+export { createChunker, finalize, nextBlock, getChunks, hashToHex, hexToBytes, type Chunk } from "./xet-chunker.js";
+export { xorbHash } from "./xorb-hash.js";
+export { fileHash, hmac, verificationHash } from "./hash-utils.js";

src/xet-chunker.ts

+import { Hasher } from "gearhash-jit";
+import { createKeyed, Hasher as Blake3Hasher } from "@huggingface/blake3-jit";
+
+const TARGET_CHUNK_SIZE = 64 * 1024; // 64KB
+const MINIMUM_CHUNK_DIVISOR = 8;
+const MAXIMUM_CHUNK_MULTIPLIER = 2;
+const HASH_WINDOW_SIZE = 64;
+
+const BLAKE3_DATA_KEY = new Uint8Array([
+	102, 151, 245, 119, 91, 149, 80, 222, 49, 53, 203, 172, 165, 151, 24, 28, 157, 228, 33, 16, 155, 235, 43, 88, 180,
+	208, 176, 75, 147, 173, 242, 41,
+]);
+
+export interface Chunk {
+	hash: Uint8Array;
+	length: number;
+}
+
+interface NextResult {
+	chunk: Chunk | null;
+	bytesConsumed: number;
+}
+
+class XetChunker {
+	private minimumChunk: number;
+	private maximumChunk: number;
+	private chunkBuf: Uint8Array;
+	private curChunkLen: number;
+	private gear: Hasher;
+	private blake3: Blake3Hasher;
+
+	constructor(targetChunkSize: number = TARGET_CHUNK_SIZE) {
+		if (targetChunkSize <= 0) {
+			throw new Error("Target chunk size must be greater than 0");
+		}
+		if ((targetChunkSize & (targetChunkSize - 1)) !== 0) {
+			throw new Error("Target chunk size must be a power of 2");
+		}
+		if (targetChunkSize <= HASH_WINDOW_SIZE) {
+			throw new Error("Target chunk size must be greater than hash window size");
+		}
+		if (targetChunkSize >= Number.MAX_SAFE_INTEGER) {
+			throw new Error("Target chunk size must be less than Number.MAX_SAFE_INTEGER");
+		}
+
+		let mask = BigInt(targetChunkSize - 1);
+		let leadingZeros = 0;
+		for (let i = 63; i >= 0; i--) {
+			if ((mask & (1n << BigInt(i))) !== 0n) {
+				break;
+			}
+			leadingZeros++;
+		}
+		mask = mask << BigInt(leadingZeros);
+
+		const maximumChunk = targetChunkSize * MAXIMUM_CHUNK_MULTIPLIER;
+
+		this.minimumChunk = targetChunkSize / MINIMUM_CHUNK_DIVISOR;
+		this.maximumChunk = maximumChunk;
+		this.chunkBuf = new Uint8Array(maximumChunk);
+		this.curChunkLen = 0;
+		this.gear = new Hasher(mask);
+		this.blake3 = Blake3Hasher.newKeyed(BLAKE3_DATA_KEY);
+	}
+
+	/**
+	 * Streaming entry point: accepts an arbitrary slice of data, accumulates
+	 * it, and emits a chunk when a boundary (or max size) is reached.
+	 * Data is copied into an internal buffer because it may span calls.
+	 */
+	next(data: Uint8Array, isFinal: boolean): NextResult {
+		const nBytes = data.length;
+		let createChunk = false;
+		let consumeLen = 0;
+
+		if (nBytes !== 0) {
+			if (this.curChunkLen + HASH_WINDOW_SIZE < this.minimumChunk) {
+				const maxAdvance = Math.min(this.minimumChunk - this.curChunkLen - HASH_WINDOW_SIZE - 1, nBytes - consumeLen);
+				consumeLen += maxAdvance;
+				this.curChunkLen += maxAdvance;
+			}
+
+			const readEnd = Math.min(nBytes, consumeLen + this.maximumChunk - this.curChunkLen);
+
+			let bytesToNextBoundary: number;
+			const position = this.gear.nextMatch(data.subarray(consumeLen, readEnd));
+
+			if (position !== -1) {
+				bytesToNextBoundary = position;
+				createChunk = true;
+			} else {
+				bytesToNextBoundary = readEnd - consumeLen;
+			}
+
+			if (bytesToNextBoundary + this.curChunkLen >= this.maximumChunk) {
+				bytesToNextBoundary = this.maximumChunk - this.curChunkLen;
+				createChunk = true;
+			}
+
+			this.curChunkLen += bytesToNextBoundary;
+			consumeLen += bytesToNextBoundary;
+
+			this.chunkBuf.set(data.subarray(0, consumeLen), this.curChunkLen - consumeLen);
+		}
+
+		if (createChunk || (isFinal && this.curChunkLen > 0)) {
+			const chunkData = this.chunkBuf.subarray(0, this.curChunkLen);
+			const hash = this.blake3.reset().update(chunkData).finalize(32);
+			const chunk: Chunk = {
+				length: chunkData.length,
+				hash: hash,
+			};
+			this.curChunkLen = 0;
+			this.gear.resetHash();
+			return {
+				chunk,
+				bytesConsumed: consumeLen,
+			};
+		}
+
+		return {
+			chunk: null,
+			bytesConsumed: consumeLen,
+		};
+	}
+
+	/**
+	 * Batch entry point: processes a large contiguous buffer and returns all
+	 * complete chunks. Hashes directly from `data` — no intermediate copy
+	 * to chunkBuf — for every chunk whose bytes are fully within `data`.
+	 */
+	nextBlock(data: Uint8Array, isFinal: boolean): Chunk[] {
+		const chunks: Chunk[] = [];
+		let pos = 0;
+
+		// Drain any leftover from a previous nextBlock / next call.
+		while (pos < data.length && this.curChunkLen > 0) {
+			const result = this.next(data.subarray(pos), false);
+			if (result.chunk) chunks.push(result.chunk);
+			pos += result.bytesConsumed;
+		}
+
+		const minSkip = this.minimumChunk > HASH_WINDOW_SIZE
+			? this.minimumChunk - HASH_WINDOW_SIZE - 1
+			: 0;
+
+		while (pos < data.length) {
+			const chunkStart = pos;
+			const scanStart = Math.min(pos + minSkip, data.length);
+			const scanEnd = Math.min(data.length, pos + this.maximumChunk);
+
+			const position = this.gear.nextMatch(data.subarray(scanStart, scanEnd));
+
+			let chunkEnd: number;
+			let foundBoundary: boolean;
+
+			if (position !== -1 && scanStart + position - chunkStart <= this.maximumChunk) {
+				chunkEnd = scanStart + position;
+				foundBoundary = true;
+			} else if (scanEnd - chunkStart >= this.maximumChunk) {
+				chunkEnd = chunkStart + this.maximumChunk;
+				foundBoundary = true;
+			} else {
+				foundBoundary = false;
+				chunkEnd = scanEnd;
+			}
+
+			if (foundBoundary) {
+				const hash = this.blake3.reset()
+					.update(data.subarray(chunkStart, chunkEnd))
+					.finalize(32);
+				chunks.push({ length: chunkEnd - chunkStart, hash });
+				pos = chunkEnd;
+				this.gear.resetHash();
+			} else if (isFinal) {
+				const hash = this.blake3.reset()
+					.update(data.subarray(chunkStart))
+					.finalize(32);
+				chunks.push({ length: data.length - chunkStart, hash });
+				pos = data.length;
+			} else {
+				this.chunkBuf.set(data.subarray(chunkStart), 0);
+				this.curChunkLen = data.length - chunkStart;
+				pos = data.length;
+			}
+		}
+
+		return chunks;
+	}
+
+	finish(): Chunk | null {
+		if (this.curChunkLen > 0) {
+			const chunkData = this.chunkBuf.subarray(0, this.curChunkLen);
+			const hash = this.blake3.reset().update(chunkData).finalize(32);
+			const chunk: Chunk = { length: this.curChunkLen, hash };
+			this.curChunkLen = 0;
+			this.gear.resetHash();
+			return chunk;
+		}
+		return null;
+	}
+}
+
+export function createChunker(targetChunkSize: number = TARGET_CHUNK_SIZE): XetChunker {
+	return new XetChunker(targetChunkSize);
+}
+
+export function nextBlock(chunker: XetChunker, data: Uint8Array): Chunk[] {
+	return chunker.nextBlock(data, false);
+}
+
+export function finalize(chunker: XetChunker): Chunk | null {
+	return chunker.finish();
+}
+
+export function getChunks(data: Uint8Array, targetChunkSize: number = TARGET_CHUNK_SIZE): Chunk[] {
+	const chunker = createChunker(targetChunkSize);
+	return chunker.nextBlock(data, true);
+}
+
+export function hashToHex(hash: Uint8Array): string {
+	const view = new DataView(hash.buffer, hash.byteOffset, hash.byteLength);
+	const u64 = view.getBigUint64(0, true);
+	const u64_2 = view.getBigUint64(8, true);
+	const u64_3 = view.getBigUint64(16, true);
+	const u64_4 = view.getBigUint64(24, true);
+
+	return (
+		u64.toString(16).padStart(16, "0") +
+		u64_2.toString(16).padStart(16, "0") +
+		u64_3.toString(16).padStart(16, "0") +
+		u64_4.toString(16).padStart(16, "0")
+	);
+}
+
+export function hexToBytes(hex: string): Uint8Array {
+	const bytes = new Uint8Array(32);
+	const view = new DataView(bytes.buffer);
+	view.setBigUint64(0, BigInt("0x" + hex.slice(0, 16)), true);
+	view.setBigUint64(8, BigInt("0x" + hex.slice(16, 32)), true);
+	view.setBigUint64(16, BigInt("0x" + hex.slice(32, 48)), true);
+	view.setBigUint64(24, BigInt("0x" + hex.slice(48, 64)), true);
+	return bytes;
+}

src/xorb-hash.ts

+import { Hasher } from "@huggingface/blake3-jit";
+import type { Chunk } from "./xet-chunker.js";
+import { hashToHex } from "./xet-chunker.js";
+
+const MEAN_CHUNK_PER_NODE = 4;
+
+const BLAKE3_NODE_KEY = new Uint8Array([
+	1, 126, 197, 199, 165, 71, 41, 150, 253, 148, 102, 102, 180, 138, 2, 230, 93, 221, 83, 111, 55, 199, 109, 210, 248,
+	99, 82, 230, 74, 83, 113, 63,
+]);
+
+const INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH = 3 * 8;
+
+const nodeHasher = Hasher.newKeyed(BLAKE3_NODE_KEY);
+
+export function xorbHash(chunks: Chunk[]): Uint8Array {
+	if (chunks.length === 0) {
+		return new Uint8Array(32);
+	}
+
+	let currentChunks = chunks;
+
+	while (currentChunks.length > 1) {
+		const nodes: Chunk[] = [];
+		let currentIndex = 0;
+		let numOfChildrenSoFar = 0;
+
+		for (let i = 0; i < currentChunks.length; i++) {
+			if (
+				i === currentChunks.length - 1 ||
+				numOfChildrenSoFar === 2 * MEAN_CHUNK_PER_NODE ||
+				(numOfChildrenSoFar >= 2 &&
+					currentChunks[i].hash[INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH] % MEAN_CHUNK_PER_NODE === 0)
+			) {
+				nodes.push(mergedHashOfSequence(currentChunks.slice(currentIndex, i + 1)));
+				currentIndex = i + 1;
+				numOfChildrenSoFar = 0;
+			} else {
+				numOfChildrenSoFar++;
+			}
+		}
+		currentChunks = nodes;
+	}
+
+	return currentChunks[0].hash;
+}
+
+/**
+ * Matches Rust's `merged_hash_of_sequence`: serializes each entry as
+ * "{hash_hex} : {length_decimal}\n" then hashes with BLAKE3_NODE_KEY.
+ */
+function mergedHashOfSequence(chunks: Chunk[]): Chunk {
+	let text = "";
+	let totalLength = 0;
+	for (const chunk of chunks) {
+		text += hashToHex(chunk.hash) + " : " + chunk.length + "\n";
+		totalLength += chunk.length;
+	}
+	const bytes = new Uint8Array(text.length);
+	for (let i = 0; i < text.length; i++) {
+		bytes[i] = text.charCodeAt(i);
+	}
+	const hash = nodeHasher.reset().update(bytes).finalize(32);
+	return { hash, length: totalLength };
+}

package.json

 {
 	"name": "@huggingface/xetchunk-wasm",
-	"version": "0.0.3",
+	"version": "0.0.4",
 	"description": "Content-defined chunking and hashing for Hugging Face Xet storage",
@@ -17,2 +17,3 @@ "keywords": [
 		"dist",
+		"src",
 		"README.md"
@@ -33,3 +34,3 @@ ],
 		"gearhash-jit": "workspace:*",
-		"@huggingface/blake3-jit": "^0.0.1"
+		"@huggingface/blake3-jit": "workspace:*"
 	},
@@ -36,0 +37,0 @@ "devDependencies": {

Fixed alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Improved metrics

Total package byte prevSize

50293

28.5%

Number of package files

32

14.29%

Lines of code

1082

40.34%

Number of medium supply chain risk alerts

0

-100%

Dependency changes

• - Removed

  @huggingface/blake3-jit@0.0.1
  (transitive)

• Updated

  @huggingface/blake3-jit@workspace:*