@web3-storage/data-segment - npm Package Compare versions

Comparing version 2.1.0 to 2.2.0

dist/src/api.d.ts

 import type { Link, ToString } from 'multiformats/link';
+import { MultihashDigest } from 'multiformats';
 export { ToString };
@@ -40,2 +41,37 @@ /**
 }
+export interface StreamDigest<Code extends MulticodecCode, Size extends number> extends MultihashDigest<Code> {
+    size: Size;
+}
+export interface StreamingHasher<Code extends MulticodecCode, Size extends number, Digest = StreamDigest<Code, Size>> {
+    size: Size;
+    code: Code;
+    name: string;
+    /**
+     * Number of bytes currently consumed.
+     */
+    count(): bigint;
+    /**
+     * Returns multihash digest of the bytes written so far.
+     */
+    digest(): Digest;
+    /**
+     * Computes the digest of the given input and writes it into the given output
+     * at the given offset. Unless `asMultihash` is `false` multihash is
+     * written otherwise only the digest (without multihash prefix) is written.
+     */
+    digestInto(output: Uint8Array, offset?: number, asMultihash?: boolean): this;
+    /**
+     * Writes bytes to be digested.
+     */
+    write(bytes: Uint8Array): this;
+    /**
+     * Resets this hasher to its initial state.
+     */
+    reset(): this;
+    /**
+     * Disposes this hasher and frees up any resources it may be holding on to.
+     * After this is called this hasher should not be used.
+     */
+    dispose(): void;
+}
 type Poll<T, X> = Variant<{
@@ -116,4 +152,9 @@ ok: T;
 }
-export interface PieceInfoView extends PieceInfo {
+export interface PieceView {
     /**
+     * Commitment to the data segment (Merkle node which is the root of the
+     * subtree containing all the nodes making up the data segment)
+     */
+    link: PieceLink;
+    /**
      * Height of the perfect binary merkle tree representing
@@ -124,2 +165,4 @@ * this piece.
 }
+export interface PieceInfoView extends PieceInfo, PieceView {
+}
 /**
@@ -301,2 +344,9 @@ * Represents a piece tree and underlying merkle tree.
 /**
+ * [Multicodec code] usually used to tag [multiformat].
+ *
+ * [multiformat]:https://multiformats.io/
+ * [multicodec code]:https://github.com/multiformats/multicodec/blob/master/table.csv
+ */
+export type MulticodecCode<Code extends number = number, Name extends string = string> = Code & Phantom<Name>;
+/**
  * Utility type for defining a [keyed union] type as in IPLD Schema. In practice
@@ -303,0 +353,0 @@ * this just works around typescript limitation that requires discriminant field

dist/src/fr32.d.ts

@@ -6,10 +6,6 @@ /**
  *
- * @param {number} sourceSize - The size of the original resource
+ * @param {number} payloadSize - The size of the payload.
  * @returns {number}
  */
-export function toZeroPaddedSize(sourceSize: number): number;
-/**
- * The smallest amount of data for which FR32 padding has a defined result.
- */
-export const MIN_PIECE_SIZE: 65;
+export function toZeroPaddedSize(payloadSize: number): number;
 export function toPieceSize(size: number): number;
@@ -16,0 +12,0 @@ export function fromPieceSize(size: number): number;

dist/src/lib.d.ts

 export * from "./api.js";
+export * from "./constant.js";
 export * as API from "./api.js";
@@ -3,0 +4,0 @@ export * as Node from "./node.js";

dist/src/node.d.ts

@@ -1,6 +0,7 @@
-export const Size: 32;
+export { Size };
 export function of(...bytes: number[]): API.MerkleTreeNode;
 export function from(bytes: Iterable<number>): API.MerkleTreeNode;
 export function empty(): API.MerkleTreeNode;
+import { NODE_SIZE as Size } from './constant.js';
 import * as API from './api.js';
 //# sourceMappingURL=node.d.ts.map

dist/src/piece/padded-size.d.ts

@@ -5,3 +5,4 @@ export function from(size: number | API.uint64): API.PaddedPieceSize;
 export function toHeight(size: API.PaddedPieceSize): number;
+export function fromHeight(height: number): bigint;
 import * as API from '../api.js';
 //# sourceMappingURL=padded-size.d.ts.map

package.json

 {
   "name": "@web3-storage/data-segment",
   "description": "Implementation of [FRC-0058](https://github.com/filecoin-project/FIPs/blob/master/FRCs/frc-0058.md) verifiable aggregation scheme",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "keywords": [
@@ -68,5 +68,5 @@ "FRC-0058",
     "test": "c8 --check-coverage --branches 0 --functions 0 --lines 0 entail test/**/*.spec.js",
-    "coverage": "c8 --reporter=html mocha test/**/*.spec.js && npm_config_yes=true npx st -d coverage -p 8080",
+    "coverage": "c8 report -r html && open coverage/index.html",
     "check": "tsc --build"
   }
 }

src/aggregate.js

@@ -12,3 +12,3 @@ import * as API from './api.js'
 const EntrySize = Number(Index.EntrySize)
-export const MAX_CAPACITY = 2n ** BigInt(Tree.MAX_HEIGHT) * NodeSize
+export const MAX_CAPACITY = Piece.PaddedSize.fromHeight(Tree.MAX_HEIGHT)
 
@@ -15,0 +15,0 @@ /**

src/api.ts

@@ -1,3 +0,4 @@
-import exp from 'constants'
 import type { Link, ToString } from 'multiformats/link'
+import { MultihashDigest } from 'multiformats'
+import { Digest } from 'multiformats/hashes/digest'
 
@@ -43,2 +44,49 @@ export { ToString }
 
+export interface StreamDigest<Code extends MulticodecCode, Size extends number>
+  extends MultihashDigest<Code> {
+  size: Size
+}
+
+export interface StreamingHasher<
+  Code extends MulticodecCode,
+  Size extends number,
+  Digest = StreamDigest<Code, Size>
+> {
+  size: Size
+  code: Code
+  name: string
+  /**
+   * Number of bytes currently consumed.
+   */
+  count(): bigint
+
+  /**
+   * Returns multihash digest of the bytes written so far.
+   */
+  digest(): Digest
+
+  /**
+   * Computes the digest of the given input and writes it into the given output
+   * at the given offset. Unless `asMultihash` is `false` multihash is
+   * written otherwise only the digest (without multihash prefix) is written.
+   */
+  digestInto(output: Uint8Array, offset?: number, asMultihash?: boolean): this
+
+  /**
+   * Writes bytes to be digested.
+   */
+  write(bytes: Uint8Array): this
+
+  /**
+   * Resets this hasher to its initial state.
+   */
+  reset(): this
+
+  /**
+   * Disposes this hasher and frees up any resources it may be holding on to.
+   * After this is called this hasher should not be used.
+   */
+  dispose(): void
+}
+
 type Poll<T, X> = Variant<{
@@ -133,4 +181,10 @@ ok: T
 
-export interface PieceInfoView extends PieceInfo {
+export interface PieceView {
   /**
+   * Commitment to the data segment (Merkle node which is the root of the
+   * subtree containing all the nodes making up the data segment)
+   */
+  link: PieceLink
+
+  /**
    * Height of the perfect binary merkle tree representing
@@ -142,2 +196,4 @@ * this piece.
 
+export interface PieceInfoView extends PieceInfo, PieceView {}
+
 /**
@@ -347,3 +403,15 @@ * Represents a piece tree and underlying merkle tree.
 export interface Unit {}
+
 /**
+ * [Multicodec code] usually used to tag [multiformat].
+ *
+ * [multiformat]:https://multiformats.io/
+ * [multicodec code]:https://github.com/multiformats/multicodec/blob/master/table.csv
+ */
+export type MulticodecCode<
+  Code extends number = number,
+  Name extends string = string
+> = Code & Phantom<Name>
+
+/**
  * Utility type for defining a [keyed union] type as in IPLD Schema. In practice
@@ -350,0 +418,0 @@ * this just works around typescript limitation that requires discriminant field

src/fr32.js

 import * as API from './api.js'
+import {
+  OUT_BYTES_PER_QUAD,
+  FR_RATIO,
+  IN_BYTES_PER_QUAD,
+  MIN_PAYLOAD_SIZE,
+} from './constant.js'
 
 /**
- * The smallest amount of data for which FR32 padding has a defined result.
- */
-export const MIN_PIECE_SIZE = 65
-
-/**
- * Number of bits per byte
- */
-const BITS_PER_BYTE = 8
-
-/**
- * The number of Frs per Block.
- */
-const FRS_PER_QUAD = 4
-/**
- * The amount of bits in an Fr when not padded.
- */
-const IN_BITS_FR = 254
-/**
- * The amount of bits in an Fr when padded.
- */
-const OUT_BITS_FR = 256
-
-const IN_BYTES_PER_QUAD =
-  /** @type {127} */
-  ((FRS_PER_QUAD * IN_BITS_FR) / BITS_PER_BYTE)
-
-const OUT_BYTES_PER_QUAD =
-  /** @type {128} */
-  ((FRS_PER_QUAD * OUT_BITS_FR) / BITS_PER_BYTE)
-
-const BYTES_PER_FR =
-  /** @type {32} */
-  OUT_BYTES_PER_QUAD / FRS_PER_QUAD
-
-const FR_RATIO = IN_BITS_FR / OUT_BITS_FR
-
-/**
  * Determine the additional bytes of zeroed padding to append to the
@@ -45,7 +14,7 @@ * end of a resource of `size` length in order to fit within a pow2 piece while
  *
- * @param {number} sourceSize - The size of the original resource
+ * @param {number} payloadSize - The size of the payload.
  * @returns {number}
  */
-export function toZeroPaddedSize(sourceSize) {
-  const size = Math.max(sourceSize, MIN_PIECE_SIZE)
+export function toZeroPaddedSize(payloadSize) {
+  const size = Math.max(payloadSize, MIN_PAYLOAD_SIZE)
   const highestBit = Math.floor(Math.log2(size))
@@ -65,4 +34,3 @@
  */
-export const toPieceSize = (size) =>
-  (toZeroPaddedSize(size) / IN_BYTES_PER_QUAD) * OUT_BYTES_PER_QUAD
+export const toPieceSize = (size) => toZeroPaddedSize(size) / FR_RATIO
 
@@ -74,4 +42,3 @@ /**
  */
-export const fromPieceSize = (size) =>
-  (size / OUT_BYTES_PER_QUAD) * IN_BYTES_PER_QUAD
+export const fromPieceSize = (size) => size * FR_RATIO
 
@@ -78,0 +45,0 @@ /**

src/lib.js

 export * from './api.js'
+export * from './constant.js'
 export * as API from './api.js'
@@ -3,0 +4,0 @@ export * as Node from './node.js'

src/node.js

 import * as API from './api.js'
+import { NODE_SIZE as Size } from './constant.js'
 
-export const Size = 32
+export { Size }
 
@@ -5,0 +6,0 @@ /**

src/piece.js

 import * as API from './api.js'
 import * as Fr32 from './fr32.js'
-import { Size as NodeSize } from './node.js'
 import * as Digest from 'multiformats/hashes/digest'
@@ -9,3 +8,3 @@ import * as Link from 'multiformats/link'
 import * as PaddedSize from './piece/padded-size.js'
-import { log2Ceil } from './uint64.js'
+import { FR_RATIO, NODE_SIZE, MIN_PAYLOAD_SIZE } from './constant.js'
 
@@ -28,3 +27,3 @@ export { Tree }
  */
-export const MAX_PIECE_SIZE = Tree.MAX_LEAF_COUNT * NodeSize
+export const MAX_PIECE_SIZE = Tree.MAX_LEAF_COUNT * NODE_SIZE
 
@@ -34,3 +33,3 @@ /**
  */
-export const MAX_PAYLOAD_SIZE = (MAX_PIECE_SIZE / 128) * 127
+export const MAX_PAYLOAD_SIZE = MAX_PIECE_SIZE * FR_RATIO
 
@@ -53,3 +52,3 @@ export { UnpaddedSize, PaddedSize }
   get size() {
-    return 2n ** BigInt(this.height) * BigInt(NodeSize)
+    return PaddedSize.fromHeight(this.height)
   }
@@ -130,5 +129,5 @@ toJSON() {
 export const build = (source) => {
-  if (source.length < Fr32.MIN_PIECE_SIZE) {
+  if (source.length < MIN_PAYLOAD_SIZE) {
     throw new RangeError(
-      `Piece is not defined for payloads smaller than ${Fr32.MIN_PIECE_SIZE} bytes`
+      `Piece is not defined for payloads smaller than ${MIN_PAYLOAD_SIZE} bytes`
     )
@@ -135,0 +134,0 @@ }

src/piece/padded-size.js

 import * as API from '../api.js'
 import { onesCount64, log2Ceil } from '../uint64.js'
-import * as Node from '../node.js'
+import { OUT_BYTES_PER_QUAD, NODE_SIZE } from '../constant.js'
 
-const NODE_SIZE = BigInt(Node.Size)
+const BYTES_PER_NODE = BigInt(NODE_SIZE)
+const BYTES_PER_QUAD = BigInt(OUT_BYTES_PER_QUAD)
 
@@ -37,4 +38,6 @@ /**
 export const validate = (size) => {
-  if (size < 128) {
-    return { error: RangeError('minimum padded piece size is 128 bytes') }
+  if (size < BYTES_PER_QUAD) {
+    return {
+      error: RangeError(`minimum padded piece size is ${BYTES_PER_QUAD} bytes`),
+    }
   }
@@ -54,3 +57,3 @@
  */
-export const toUnpaddedSize = (size) => size - size / 128n
+export const toUnpaddedSize = (size) => size - size / BYTES_PER_QUAD
 
@@ -62,2 +65,9 @@ /**
  */
-export const toHeight = (size) => log2Ceil(size / NODE_SIZE)
+export const toHeight = (size) => log2Ceil(size / BYTES_PER_NODE)
+
+/**
+ * Calculates the padded size of the piece tree from height.
+ *
+ * @param {number} height
+ */
+export const fromHeight = (height) => 2n ** BigInt(height) * BYTES_PER_NODE

src/piece/unpadded-size.js

 import * as API from '../api.js'
 import { trailingZeros64, log2Ceil } from '../uint64.js'
-import * as Node from '../node.js'
+import { IN_BYTES_PER_QUAD, NODE_SIZE } from '../constant.js'
 
-const NODE_SIZE = BigInt(Node.Size)
+const BYTES_PER_NODE = BigInt(NODE_SIZE)
+const BYTES_PER_QUAD = BigInt(IN_BYTES_PER_QUAD)
 
@@ -36,10 +37,12 @@ /**
 export const validate = (size) => {
-  if (size < 127) {
-    return { error: new Error('Minimum piece size is 127 bytes') }
+  if (size < BYTES_PER_QUAD) {
+    return {
+      error: new Error(`Minimum piece size is ${BYTES_PER_QUAD} bytes`),
+    }
   }
 
-  if (size >> BigInt(trailingZeros64(size)) !== 127n) {
+  if (size >> BigInt(trailingZeros64(size)) !== BYTES_PER_QUAD) {
     return {
       error: new Error(
-        `Unpadded piece size must be a power of 2 multiple of 127, got ${size} instead`
+        `Unpadded piece size must be a power of 2 multiple of ${BYTES_PER_QUAD}, got ${size} instead`
       ),
@@ -66,3 +69,3 @@ }
  */
-export const toPaddedSize = (size) => size + size / 127n
+export const toPaddedSize = (size) => size + size / BYTES_PER_QUAD
 
@@ -74,2 +77,2 @@ /**
  */
-export const toHeight = (size) => log2Ceil(toPaddedSize(size) / NODE_SIZE)
+export const toHeight = (size) => log2Ceil(toPaddedSize(size) / BYTES_PER_NODE)

No alert changes

Improved metrics

Total package byte prevSize

112407

increased by20.72%

Number of package files

57

increased by11.76%

Lines of code

3053

increased by20.53%

No dependency changes