@snapshot-labs/checkpoint - npm Package Compare versions

Comparing version 0.1.0-beta.18 to 0.1.0-beta.19

dist/src/bin/index.d.ts

@@ -0,1 +1,2 @@
+#!/usr/bin/env node
 export {};

dist/src/bin/index.js

@@ -0,1 +1,2 @@
+#!/usr/bin/env node
 "use strict";
@@ -2,0 +3,0 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {

dist/src/checkpoint.d.ts

@@ -7,3 +7,3 @@ /// <reference types="node" />
 import { AsyncMySqlPool } from './mysql';
-import { ContractSourceConfig, CheckpointConfig, CheckpointOptions, CheckpointWriters } from './types';
+import { CheckpointConfig, CheckpointOptions, CheckpointWriters } from './types';
 export default class Checkpoint {
@@ -22,2 +22,3 @@ config: CheckpointConfig;
     private checkpointsStore?;
+    private activeTemplates;
     private sourceContracts;
@@ -43,2 +44,12 @@ private cpBlocksCache;
     }) => Promise<void>;
+    getCurrentSources(blockNumber: number): {
+        start: number;
+        contract: string;
+        events: {
+            name: string;
+            fn: string;
+        }[];
+        abi?: string | undefined;
+        deploy_fn?: string | undefined;
+    }[];
     /**
@@ -74,7 +85,7 @@ * Starts the indexer.
     resetMetadata(): Promise<void>;
-    addSource(source: ContractSourceConfig): void;
+    private addSource;
     executeTemplate(name: string, { contract, start }: {
         contract: string;
         start: number;
-    }): void;
+    }, persist?: boolean): Promise<void>;
     /**
@@ -81,0 +92,0 @@ * Registers the blocks where a contracts event can be found.

dist/src/checkpoint.js

@@ -34,2 +34,3 @@ "use strict";
 const controller_1 = require("./graphql/controller");
+const checkpoints_1 = require("./stores/checkpoints");
 const providers_1 = require("./providers");
@@ -42,4 +43,4 @@ const logger_1 = require("./utils/logger");
 const pg_1 = require("./pg");
+const schemas_1 = require("./schemas");
 const register_1 = require("./register");
-const checkpoints_1 = require("./stores/checkpoints");
 const DEFAULT_FETCH_INTERVAL = 7000;
@@ -59,5 +60,10 @@ class Checkpoint {
     checkpointsStore;
+    activeTemplates = [];
     sourceContracts;
     cpBlocksCache;
     constructor(config, writer, schema, opts) {
+        const validationResult = schemas_1.checkpointConfigSchema.safeParse(config);
+        if (validationResult.success === false) {
+            throw new Error(`Checkpoint config is invalid: ${validationResult.error.message}`);
+        }
         this.config = config;
@@ -127,2 +133,7 @@ this.writer = writer;
     }
+    getCurrentSources(blockNumber) {
+        if (!this.config.sources)
+            return [];
+        return this.config.sources.filter(source => source.start <= blockNumber);
+    }
     /**
@@ -138,2 +149,9 @@ * Starts the indexer.
         await this.validateStore();
+        const templateSources = await this.store.getTemplateSources();
+        await bluebird_1.default.all(templateSources.map(source => {
+            this.executeTemplate(source.template, {
+                contract: source.contractAddress,
+                start: source.startBlock
+            }, false);
+        }));
         const blockNum = await this.getStartBlockNum();
@@ -179,3 +197,3 @@ return await this.next(blockNum);
     }
-    executeTemplate(name, { contract, start }) {
+    async executeTemplate(name, { contract, start }, persist = true) {
         const template = this.config.templates?.[name];
@@ -186,2 +204,11 @@ if (!template) {
         }
+        const existingTemplate = this.activeTemplates.find(template => template.template === name &&
+            template.contractAddress === contract &&
+            template.startBlock === start);
+        if (existingTemplate)
+            return;
+        this.activeTemplates.push({ template: name, contractAddress: contract, startBlock: start });
+        if (persist) {
+            await this.store.insertTemplateSource(contract, start, name);
+        }
         this.addSource({
@@ -188,0 +215,0 @@ contract,

dist/src/providers/base.d.ts

 import { Pool as PgPool } from 'pg';
-import type { Logger } from '../utils/logger';
-import type Checkpoint from '../checkpoint';
-import type { AsyncMySqlPool } from '../mysql';
-import type { CheckpointConfig, CheckpointWriters } from '../types';
+import { Logger } from '../utils/logger';
+import Checkpoint from '../checkpoint';
+import { AsyncMySqlPool } from '../mysql';
+import { CheckpointConfig, CheckpointWriters, ContractSourceConfig } from '../types';
 type Instance = {
     writer: CheckpointWriters;
     config: CheckpointConfig;
+    getCurrentSources(blockNumber: number): ContractSourceConfig[];
     setLastIndexedBlock(blockNum: number): any;
@@ -10,0 +11,0 @@ insertCheckpoints(checkpoints: {

dist/src/providers/starknet/index.js

@@ -120,3 +120,3 @@ "use strict";
         }
-        for (const source of this.instance.config.sources || []) {
+        for (const source of this.instance.getCurrentSources(blockNumber)) {
             let foundContractData = false;
@@ -123,0 +123,0 @@ const contract = (0, starknet_1.validateAndParseAddress)(source.contract);

dist/src/stores/checkpoints.d.ts

 import { Knex } from 'knex';
 import { Logger } from '../utils/logger';
+import { TemplateSource } from '../types';
 type ToString = {
@@ -65,3 +66,5 @@ toString: () => string;
     getNextCheckpointBlocks(block: number, contracts: string[], limit?: number): Promise<number[]>;
+    insertTemplateSource(contractAddress: string, startBlock: number, template: string): Promise<void>;
+    getTemplateSources(): Promise<TemplateSource[]>;
 }
 export {};

dist/src/stores/checkpoints.js

@@ -30,3 +30,4 @@ "use strict";
     Checkpoints: '_checkpoints',
-    Metadata: '_metadatas' // using plural names to confirm with standards entities
+    Metadata: '_metadatas',
+    TemplateSources: '_template_sources'
 };
@@ -42,2 +43,8 @@ const Fields = {
         Value: 'value'
+    },
+    TemplateSources: {
+        Id: 'id',
+        ContractAddress: 'contract_address',
+        StartBlock: 'start_block',
+        Template: 'template'
     }
@@ -89,2 +96,3 @@ };
         const hasMetadataTable = await this.knex.schema.hasTable(Table.Metadata);
+        const hasTemplateSourcesTable = await this.knex.schema.hasTable(Table.TemplateSources);
         let builder = this.knex.schema;
@@ -104,2 +112,12 @@ if (!hasCheckpointsTable) {
         }
+        if (!hasTemplateSourcesTable) {
+            builder = builder
+                .dropTableIfExists(Table.TemplateSources)
+                .createTable(Table.TemplateSources, t => {
+                t.increments(Fields.TemplateSources.Id);
+                t.string(Fields.TemplateSources.ContractAddress, 66);
+                t.bigint(Fields.TemplateSources.StartBlock).notNullable();
+                t.string(Fields.TemplateSources.Template, 128).notNullable();
+            });
+        }
         await builder;
@@ -120,2 +138,3 @@ this.log.debug('checkpoints tables created');
         const hasMetadataTable = await this.knex.schema.hasTable(Table.Metadata);
+        const hasTemplateSourcesTable = await this.knex.schema.hasTable(Table.TemplateSources);
         if (hasCheckpointsTable) {
@@ -127,2 +146,5 @@ await this.knex(Table.Checkpoints).truncate();
         }
+        if (hasTemplateSourcesTable) {
+            await this.knex(Table.TemplateSources).truncate();
+        }
         this.log.debug('checkpoints tables truncated');
@@ -194,3 +216,20 @@ }
     }
+    async insertTemplateSource(contractAddress, startBlock, template) {
+        return this.knex.table(Table.TemplateSources).insert({
+            [Fields.TemplateSources.ContractAddress]: contractAddress,
+            [Fields.TemplateSources.StartBlock]: startBlock,
+            [Fields.TemplateSources.Template]: template
+        });
+    }
+    async getTemplateSources() {
+        const data = await this.knex
+            .select(Fields.TemplateSources.ContractAddress, Fields.TemplateSources.StartBlock, Fields.TemplateSources.Template)
+            .from(Table.TemplateSources);
+        return data.map(row => ({
+            contractAddress: row[Fields.TemplateSources.ContractAddress],
+            startBlock: row[Fields.TemplateSources.StartBlock],
+            template: row[Fields.TemplateSources.Template]
+        }));
+    }
 }
 exports.CheckpointsStore = CheckpointsStore;

dist/src/types.d.ts

@@ -0,7 +1,9 @@
+import { z } from 'zod';
 import { Pool as PgPool } from 'pg';
+import { RPC } from 'starknet';
+import Checkpoint from './checkpoint';
 import { AsyncMySqlPool } from './mysql';
 import { LogLevel } from './utils/logger';
-import type { RPC } from 'starknet';
-import type Checkpoint from './checkpoint';
-import type { BaseProvider } from './providers';
+import { BaseProvider } from './providers';
+import { contractSourceConfigSchema, contractTemplateSchema, checkpointConfigSchema } from './schemas';
 export type Block = RPC.GetBlockWithTxs;
@@ -21,2 +23,7 @@ export type Transaction = RPC.Transaction;
 export type ParsedEvent = Record<string, any>;
+export type TemplateSource = {
+    contractAddress: string;
+    startBlock: number;
+    template: string;
+};
 export interface CheckpointOptions {
@@ -31,34 +38,5 @@ resetOnConfigChange?: boolean;
 }
-export interface ContractEventConfig {
-    name: string;
-    fn: string;
-}
-export interface ContractSourceConfig {
-    contract: string;
-    abi?: string;
-    start: number;
-    deploy_fn?: string;
-    events: ContractEventConfig[];
-}
-export type ContractTemplate = {
-    abi?: string;
-    events: ContractEventConfig[];
-};
-export interface CheckpointConfig {
-    network_node_url: string;
-    optimistic_indexing?: boolean;
-    decimal_types?: {
-        [key: string]: {
-            p: number;
-            d: number;
-        };
-    };
-    start?: number;
-    tx_fn?: string;
-    global_events?: ContractEventConfig[];
-    sources?: ContractSourceConfig[];
-    templates?: {
-        [key: string]: ContractTemplate;
-    };
-}
+export type ContractSourceConfig = z.infer<typeof contractSourceConfigSchema>;
+export type ContractTemplate = z.infer<typeof contractTemplateSchema>;
+export type CheckpointConfig = z.infer<typeof checkpointConfigSchema>;
 /**
@@ -65,0 +43,0 @@ * Callback function invoked by checkpoint when a contract event

package.json

 {
   "name": "@snapshot-labs/checkpoint",
-  "version": "0.1.0-beta.18",
+  "version": "0.1.0-beta.19",
   "license": "MIT",
@@ -39,3 +39,4 @@ "bin": {
     "starknet": "^4.13.1",
-    "yargs": "^17.7.2"
+    "yargs": "^17.7.2",
+    "zod": "^3.21.4"
   },
@@ -42,0 +43,0 @@ "devDependencies": {

README.md

@@ -10,3 +10,3 @@ # Checkpoint
 ```tsx
-npm install @snapshot-labs/checkpoint
+npm install @snapshot-labs/checkpoint@beta
 ```
@@ -13,0 +13,0 @@

src/bin/index.ts

@@ -0,1 +1,3 @@
+#!/usr/bin/env node
+
 import path from 'path';
@@ -2,0 +4,0 @@ import fs from 'fs/promises';

src/checkpoint.ts

@@ -8,2 +8,3 @@ import Promise from 'bluebird';
 import { GqlEntityController } from './graphql/controller';
+import { CheckpointRecord, CheckpointsStore, MetadataId } from './stores/checkpoints';
 import { BaseProvider, StarknetProvider, BlockNotFoundError } from './providers';
@@ -16,2 +17,3 @@ import { createLogger, Logger, LogLevel } from './utils/logger';
 import { createPgPool } from './pg';
+import { checkpointConfigSchema } from './schemas';
 import { register } from './register';
@@ -22,5 +24,5 @@ import {
   CheckpointOptions,
-  CheckpointWriters
+  CheckpointWriters,
+  TemplateSource
 } from './types';
-import { CheckpointRecord, CheckpointsStore, MetadataId } from './stores/checkpoints';
 
@@ -44,2 +46,3 @@ const DEFAULT_FETCH_INTERVAL = 7000;
   private checkpointsStore?: CheckpointsStore;
+  private activeTemplates: TemplateSource[] = [];
   private sourceContracts: string[];
@@ -54,2 +57,7 @@ private cpBlocksCache: number[] | null;
   ) {
+    const validationResult = checkpointConfigSchema.safeParse(config);
+    if (validationResult.success === false) {
+      throw new Error(`Checkpoint config is invalid: ${validationResult.error.message}`);
+    }
+
     this.config = config;
@@ -136,2 +144,8 @@ this.writer = writer;
 
+  public getCurrentSources(blockNumber: number) {
+    if (!this.config.sources) return [];
+
+    return this.config.sources.filter(source => source.start <= blockNumber);
+  }
+
   /**
@@ -149,2 +163,16 @@ * Starts the indexer.
 
+    const templateSources = await this.store.getTemplateSources();
+    await Promise.all(
+      templateSources.map(source => {
+        this.executeTemplate(
+          source.template,
+          {
+            contract: source.contractAddress,
+            start: source.startBlock
+          },
+          false
+        );
+      })
+    );
+
     const blockNum = await this.getStartBlockNum();
@@ -189,3 +217,3 @@ return await this.next(blockNum);
 
-  public addSource(source: ContractSourceConfig) {
+  private addSource(source: ContractSourceConfig) {
     if (!this.config.sources) this.config.sources = [];
@@ -198,3 +226,7 @@
 
-  public executeTemplate(name: string, { contract, start }: { contract: string; start: number }) {
+  public async executeTemplate(
+    name: string,
+    { contract, start }: { contract: string; start: number },
+    persist = true
+  ) {
     const template = this.config.templates?.[name];
@@ -207,2 +239,16 @@
 
+    const existingTemplate = this.activeTemplates.find(
+      template =>
+        template.template === name &&
+        template.contractAddress === contract &&
+        template.startBlock === start
+    );
+
+    if (existingTemplate) return;
+    this.activeTemplates.push({ template: name, contractAddress: contract, startBlock: start });
+
+    if (persist) {
+      await this.store.insertTemplateSource(contract, start, name);
+    }
+
     this.addSource({
@@ -209,0 +255,0 @@ contract,

src/graphql/controller.ts

@@ -33,3 +33,3 @@ import {
 } from '../utils/graphql';
-import { CheckpointConfig, CheckpointOptions } from '../types';
+import { CheckpointConfig } from '../types';
 import { querySingle, queryMulti, ResolverContext, getNestedResolver } from './resolvers';
@@ -36,0 +36,0 @@

src/providers/base.ts

 import { Pool as PgPool } from 'pg';
-import type { Logger } from '../utils/logger';
-import type Checkpoint from '../checkpoint';
-import type { AsyncMySqlPool } from '../mysql';
-import type { CheckpointConfig, CheckpointWriters } from '../types';
+import { Logger } from '../utils/logger';
+import Checkpoint from '../checkpoint';
+import { AsyncMySqlPool } from '../mysql';
+import { CheckpointConfig, CheckpointWriters, ContractSourceConfig } from '../types';
 
@@ -10,2 +10,3 @@ type Instance = {
   config: CheckpointConfig;
+  getCurrentSources(blockNumber: number): ContractSourceConfig[];
   setLastIndexedBlock(blockNum: number);
@@ -12,0 +13,0 @@ insertCheckpoints(checkpoints: { blockNumber: number; contractAddress: string }[]);

src/providers/starknet/index.ts

@@ -183,3 +183,3 @@ import { RpcProvider, hash, validateAndParseAddress } from 'starknet';
 
-    for (const source of this.instance.config.sources || []) {
+    for (const source of this.instance.getCurrentSources(blockNumber)) {
       let foundContractData = false;
@@ -186,0 +186,0 @@ const contract = validateAndParseAddress(source.contract);

src/stores/checkpoints.ts

 import * as crypto from 'crypto';
 import { Knex } from 'knex';
 import { Logger } from '../utils/logger';
+import { TemplateSource } from '../types';
 
 const Table = {
   Checkpoints: '_checkpoints',
-  Metadata: '_metadatas' // using plural names to confirm with standards entities
+  Metadata: '_metadatas', // using plural names to confirm with standards entities
+  TemplateSources: '_template_sources'
 };
@@ -19,2 +21,8 @@
     Value: 'value'
+  },
+  TemplateSources: {
+    Id: 'id',
+    ContractAddress: 'contract_address',
+    StartBlock: 'start_block',
+    Template: 'template'
   }
@@ -78,2 +86,3 @@ };
     const hasMetadataTable = await this.knex.schema.hasTable(Table.Metadata);
+    const hasTemplateSourcesTable = await this.knex.schema.hasTable(Table.TemplateSources);
 
@@ -97,2 +106,13 @@ let builder = this.knex.schema;
 
+    if (!hasTemplateSourcesTable) {
+      builder = builder
+        .dropTableIfExists(Table.TemplateSources)
+        .createTable(Table.TemplateSources, t => {
+          t.increments(Fields.TemplateSources.Id);
+          t.string(Fields.TemplateSources.ContractAddress, 66);
+          t.bigint(Fields.TemplateSources.StartBlock).notNullable();
+          t.string(Fields.TemplateSources.Template, 128).notNullable();
+        });
+    }
+
     await builder;
@@ -117,2 +137,3 @@
     const hasMetadataTable = await this.knex.schema.hasTable(Table.Metadata);
+    const hasTemplateSourcesTable = await this.knex.schema.hasTable(Table.TemplateSources);
 
@@ -127,2 +148,6 @@ if (hasCheckpointsTable) {
 
+    if (hasTemplateSourcesTable) {
+      await this.knex(Table.TemplateSources).truncate();
+    }
+
     this.log.debug('checkpoints tables truncated');
@@ -212,2 +237,30 @@ }
   }
+
+  public async insertTemplateSource(
+    contractAddress: string,
+    startBlock: number,
+    template: string
+  ): Promise<void> {
+    return this.knex.table(Table.TemplateSources).insert({
+      [Fields.TemplateSources.ContractAddress]: contractAddress,
+      [Fields.TemplateSources.StartBlock]: startBlock,
+      [Fields.TemplateSources.Template]: template
+    });
+  }
+
+  public async getTemplateSources(): Promise<TemplateSource[]> {
+    const data = await this.knex
+      .select(
+        Fields.TemplateSources.ContractAddress,
+        Fields.TemplateSources.StartBlock,
+        Fields.TemplateSources.Template
+      )
+      .from(Table.TemplateSources);
+
+    return data.map(row => ({
+      contractAddress: row[Fields.TemplateSources.ContractAddress],
+      startBlock: row[Fields.TemplateSources.StartBlock],
+      template: row[Fields.TemplateSources.Template]
+    }));
+  }
 }

src/types.ts

@@ -0,7 +1,13 @@
+import { z } from 'zod';
 import { Pool as PgPool } from 'pg';
+import { RPC } from 'starknet';
+import Checkpoint from './checkpoint';
 import { AsyncMySqlPool } from './mysql';
 import { LogLevel } from './utils/logger';
-import type { RPC } from 'starknet';
-import type Checkpoint from './checkpoint';
-import type { BaseProvider } from './providers';
+import { BaseProvider } from './providers';
+import {
+  contractSourceConfigSchema,
+  contractTemplateSchema,
+  checkpointConfigSchema
+} from './schemas';
 
@@ -21,2 +27,8 @@ // Shortcuts to starknet types.
 
+export type TemplateSource = {
+  contractAddress: string;
+  startBlock: number;
+  template: string;
+};
+
 export interface CheckpointOptions {
@@ -43,41 +55,6 @@ // Setting to true will trigger reset of database on config changes.
 
-export interface ContractEventConfig {
-  // name of event in the contract
-  name: string;
-  // callback function in writer
-  fn: string;
-}
+export type ContractSourceConfig = z.infer<typeof contractSourceConfigSchema>;
+export type ContractTemplate = z.infer<typeof contractTemplateSchema>;
+export type CheckpointConfig = z.infer<typeof checkpointConfigSchema>;
 
-export interface ContractSourceConfig {
-  // contract address
-  contract: string;
-  // abi name
-  abi?: string;
-  // start block number
-  start: number;
-  // callback function in writer to handle deployment
-  deploy_fn?: string;
-  events: ContractEventConfig[];
-}
-
-export type ContractTemplate = {
-  // abi name
-  abi?: string;
-  events: ContractEventConfig[];
-};
-
-// Configuration used to initialize Checkpoint
-export interface CheckpointConfig {
-  network_node_url: string;
-  optimistic_indexing?: boolean;
-  // Configuration for decimal types
-  // defaults to Decimal(10, 2), BigDecimal(20, 8)
-  decimal_types?: { [key: string]: { p: number; d: number } };
-  start?: number;
-  tx_fn?: string;
-  global_events?: ContractEventConfig[];
-  sources?: ContractSourceConfig[];
-  templates?: { [key: string]: ContractTemplate };
-}
-
 /**
@@ -84,0 +61,0 @@ * Callback function invoked by checkpoint when a contract event

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

199464

increased by7.49%

Number of package files

75

increased by4.17%

Lines of code

5353

increased by8.32%

Worsened metrics

Dependency count

19

increased by5.56%

Dependency changes

• + Addedzod@^3.21.4

• + Addedzod@3.23.8(transitive)