@aresrpg/sui-checkpoint-reader - npm Package Compare versions

Comparing version 2.2.7 to 3.0.0

package.json

 {
   "name": "@aresrpg/sui-checkpoint-reader",
-  "version": "2.2.7",
+  "version": "3.0.0",
   "description": "",
@@ -14,2 +14,12 @@ "main": "src/index.js",
   },
+  "exports": {
+    "./reader": {
+      "import": "./src/index.js",
+      "types": "./types/index.d.ts"
+    },
+    "./snapshot": {
+      "import": "./src/snapshot/index.js",
+      "types": "./types/snapshot/index.d.ts"
+    }
+  },
   "repository": {
@@ -26,4 +36,6 @@ "type": "git",
   "dependencies": {
+    "@mongodb-js/zstd": "^1.2.0",
     "@mysten/bcs": "^1.0.2",
-    "chokidar": "^3.6.0"
+    "chokidar": "^3.6.0",
+    "classic-level": "^1.4.1"
   },
@@ -30,0 +42,0 @@ "devDependencies": {

README.md

@@ -19,2 +19,4 @@ <p align=center>
 
+It also allows building a local leveldb from Sui's formal snapshots, which you can then process to initialize your indexer without the need to read checkpoints history for past epochs
+
 > **Note:** This is a work in progress and might not be suited for production usage.
@@ -33,3 +35,3 @@
 ```js
-import { read_checkpoints } from '@aresrpg/sui-checkpoint-reader'
+import { read_checkpoints } from '@aresrpg/sui-checkpoint-reader/reader'
 
@@ -54,3 +56,3 @@ async function get_remote_checkpoint(num) {
   known_types, // BCS types you want to parse
-  checkpoints_folder: '', // Local folder for checkpoint files
+  checkpoints_folder: '', // Local folder for checkpoint files (nesting is supported at 1 level)
   cleanup_checkpoints: false, // Clean up processed checkpoint files
@@ -61,2 +63,27 @@ process_checkpoint, // Function to process each checkpoint
 
+And here is how to get your leveldb from the formal snapshot
+
+```js
+import { download_and_store_objects } from '@aresrpg/sui-checkpoint-reader/snapshot'
+
+// this will start download the formal snapshot of the specified epoch
+// and save any objects recognized by your known types into leveldb (key: id, value: object)
+await download_and_store_objects({
+  network: 'testnet',
+  epoch: 440, // the epoch of the snapshot
+  known_types, // same types as for checkpoint reading
+  save_objects: false, // if you want to also save the .obj files locally under /epoch_X/*.obj
+  start_bucket = 1, // optional start files 1_1.obj
+  start_part = 1,
+  db_folder = './sui-formal-objects', // the leveldb folder
+})
+
+// and here you can iterate easily on your leveldb, note that you can use anything else to read those
+for await (const [db, object] of read_snapshot_objects('./sui-formal-objects')) {
+  // process all objects here, for example use db.get(object.contents.kiosk_id)
+  // to save something like { object, object_kiosk } in your storage of choice
+  console.dir(object, { depth: Infinity })
+}
+```
+
 ## ⚙️ How It Works
@@ -63,0 +90,0 @@

src/bcs.js

@@ -669,2 +669,48 @@ import { bcs } from '@mysten/bcs'
 
+// https://github.com/MystenLabs/sui/blob/c1b1e1e74c82b950e8d531f1b84c605d1ea957ca/crates/sui-snapshot/src/lib.rs#L142-L145
+const FileType = bcs.enum('FileType', {
+  Object: null,
+  Reference: null,
+})
+
+// https://github.com/MystenLabs/sui/blob/c1b1e1e74c82b950e8d531f1b84c605d1ea957ca/crates/sui-storage/src/lib.rs#L56-L59
+const FileCompression = bcs.enum('FileCompression', {
+  None: null,
+  Zstd: null,
+})
+
+// https://github.com/MystenLabs/sui/blob/c1b1e1e74c82b950e8d531f1b84c605d1ea957ca/crates/sui-snapshot/src/lib.rs#L148-L154
+const FileMetadata = bcs.struct('FileMetadata', {
+  file_type: FileType,
+  bucket_num: bcs.u32(),
+  part_num: bcs.u32(),
+  file_compression: FileCompression,
+  sha3_digest: bcs.fixedArray(32, bcs.u8()),
+})
+
+// https://github.com/MystenLabs/sui/blob/c1b1e1e74c82b950e8d531f1b84c605d1ea957ca/crates/sui-snapshot/src/lib.rs#L173-L178
+const ManifestV1 = bcs.struct('ManifestV1', {
+  snapshot_version: bcs.u8(),
+  address_length: bcs.u64(),
+  file_metadata: bcs.vector(FileMetadata),
+  epoch: bcs.u64(),
+})
+
+// https://github.com/MystenLabs/sui/blob/main/crates/sui-graphql-rpc/src/types/object.rs#L139-L142
+const ObjectKey = bcs.struct('ObjectKey', {
+  object_id: SuiAddress,
+  version: bcs.u64(),
+})
+
+// https://github.com/MystenLabs/sui/blob/c1b1e1e74c82b950e8d531f1b84c605d1ea957ca/crates/sui-core/src/authority/authority_store_tables.rs#L501-L504
+export const LiveObject = bcs.enum('LiveObject', {
+  Normal: SuiObject,
+  Wrapped: ObjectKey,
+})
+
+// https://github.com/MystenLabs/sui/blob/c1b1e1e74c82b950e8d531f1b84c605d1ea957ca/crates/sui-snapshot/src/lib.rs#L181
+export const Manifest = bcs.enum('Manifest', {
+  V1: ManifestV1,
+})
+
 // https://github.com/MystenLabs/sui/blob/testnet-v1.28.3/crates/sui-types/src/full_checkpoint_content.rs#L14-L18
@@ -671,0 +717,0 @@ export const CheckpointData = bcs.struct('CheckpointData', {

src/index.js

@@ -85,3 +85,3 @@ import { setInterval, setTimeout } from 'timers/promises'
 
-function format_objects(objects, known_types) {
+export function format_objects(objects, known_types) {
   return objects.map(object => {
@@ -443,2 +443,23 @@ const {
 
+export function premap_transaction(transaction) {
+  return mapper(transaction, {
+    sender: to_address,
+    address: to_address,
+    digest: toHEX,
+    owner: o => (Array.isArray(o) ? to_address(o) : o),
+    transaction_digest: to_address,
+    AddressOwner: to_address,
+    ObjectOwner: to_address,
+    previous_transaction: to_address,
+    package_id: to_address,
+    consensus_commit_digest: to_address,
+    ObjectWrite: ([key, value]) => ({ [to_address(key)]: value }),
+    changed_objects: entries_array =>
+      Object.fromEntries(
+        entries_array.map(([key, value]) => [to_address(key), value]),
+      ),
+    dependencies: dependencies => dependencies.map(to_address),
+  })
+}
+
 function read_checkpoint(buffer, known_types, object_filter) {
@@ -465,20 +486,3 @@ const { encoding, data } = read_blob(buffer)
     transactions: transactions.map(transaction => {
-      const mapped = mapper(transaction, {
-        sender: to_address,
-        address: to_address,
-        digest: toHEX,
-        owner: o => (Array.isArray(o) ? to_address(o) : o),
-        transaction_digest: to_address,
-        AddressOwner: to_address,
-        ObjectOwner: to_address,
-        previous_transaction: to_address,
-        package_id: to_address,
-        consensus_commit_digest: to_address,
-        ObjectWrite: ([key, value]) => ({ [to_address(key)]: value }),
-        changed_objects: entries_array =>
-          Object.fromEntries(
-            entries_array.map(([key, value]) => [to_address(key), value]),
-          ),
-        dependencies: dependencies => dependencies.map(to_address),
-      })
+      const mapped = premap_transaction(transaction)
 
@@ -485,0 +489,0 @@ return {

New alerts

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

81988

increased by13.17%

Number of package files

18

increased by20%

Lines of code

2201

increased by12.24%

Number of lines in readme file

189

increased by16.67%

Worsened metrics

Dependency count

4

increased by100%

Number of low supply chain risk alerts

6

increased by50%

Number of medium supply chain risk alerts

4

increased by100%

Dependency changes

• + Added@mongodb-js/zstd@^1.2.0

• + Addedclassic-level@^1.4.1

• + Added@mongodb-js/zstd@1.2.2(transitive)

• + Added@mongodb-js/zstd-darwin-arm64@1.2.2(transitive)

• + Added@mongodb-js/zstd-darwin-x64@1.2.2(transitive)

• + Added@mongodb-js/zstd-linux-arm64-gnu@1.2.2(transitive)

• + Added@mongodb-js/zstd-linux-arm64-musl@1.2.2(transitive)

• + Added@mongodb-js/zstd-linux-x64-gnu@1.2.2(transitive)

• + Added@mongodb-js/zstd-linux-x64-musl@1.2.2(transitive)

• + Added@mongodb-js/zstd-win32-x64-msvc@1.2.2(transitive)

• + Addedabstract-level@1.0.4(transitive)

• + Addedbase64-js@1.5.1(transitive)

• + Addedbuffer@6.0.3(transitive)

• + Addedcatering@2.1.1(transitive)

• + Addedclassic-level@1.4.1(transitive)

• + Addedieee754@1.2.1(transitive)

• + Addedis-buffer@2.0.5(transitive)

• + Addedlevel-supports@4.0.1(transitive)

• + Addedlevel-transcoder@1.0.1(transitive)

• + Addedmodule-error@1.0.2(transitive)

• + Addednapi-macros@2.2.2(transitive)

• + Addednode-gyp-build@4.8.4(transitive)

• + Addedqueue-microtask@1.2.3(transitive)