@chainsafe/lodestar-db - npm Package Compare versions

Comparing version 0.39.0-dev.719d936cc3 to 0.39.0-dev.83aa2ccff5

lib/abstractRepository.d.ts

@@ -5,3 +5,2 @@ import { IChainForkConfig } from "@chainsafe/lodestar-config";
 import { Db } from "./controller/interface.js";
-import { DbMetricCounter, IDbMetrics } from "./metrics.js";
 import { Bucket } from "./schema.js";
@@ -21,6 +20,6 @@ export declare type Id = Uint8Array | string | number | bigint;
     protected bucket: Bucket;
+    private readonly bucketId;
+    private readonly dbReqOpts;
     protected type: Type<T>;
-    protected dbReadsMetrics?: ReturnType<DbMetricCounter["labels"]>;
-    protected dbWriteMetrics?: ReturnType<DbMetricCounter["labels"]>;
-    protected constructor(config: IChainForkConfig, db: Db, bucket: Bucket, type: Type<T>, metrics?: IDbMetrics);
+    protected constructor(config: IChainForkConfig, db: Db, bucket: Bucket, type: Type<T>);
     encodeValue(value: T): Uint8Array;
@@ -27,0 +26,0 @@ decodeValue(data: Uint8Array): T;

lib/abstractRepository.js

@@ -13,9 +13,9 @@ import { BUCKET_LENGTH } from "./const.js";
 export class Repository {
-    constructor(config, db, bucket, type, metrics) {
+    constructor(config, db, bucket, type) {
         this.config = config;
         this.db = db;
         this.bucket = bucket;
+        this.bucketId = getBucketNameByValue(bucket);
+        this.dbReqOpts = { bucketId: this.bucketId };
         this.type = type;
-        this.dbReadsMetrics = metrics === null || metrics === void 0 ? void 0 : metrics.dbReads.labels({ bucket: getBucketNameByValue(bucket) });
-        this.dbWriteMetrics = metrics === null || metrics === void 0 ? void 0 : metrics.dbWrites.labels({ bucket: getBucketNameByValue(bucket) });
     }
@@ -35,5 +35,3 @@ encodeValue(value) {
     async get(id) {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        const value = await this.db.get(this.encodeKey(id));
+        const value = await this.db.get(this.encodeKey(id), this.dbReqOpts);
         if (!value)
@@ -44,5 +42,3 @@ return null;
     async getBinary(id) {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        const value = await this.db.get(this.encodeKey(id));
+        const value = await this.db.get(this.encodeKey(id), this.dbReqOpts);
         if (!value)
@@ -56,15 +52,9 @@ return null;
     async put(id, value) {
-        var _a;
-        (_a = this.dbWriteMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        await this.db.put(this.encodeKey(id), this.encodeValue(value));
+        await this.db.put(this.encodeKey(id), this.encodeValue(value), this.dbReqOpts);
     }
     async putBinary(id, value) {
-        var _a;
-        (_a = this.dbWriteMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        await this.db.put(this.encodeKey(id), value);
+        await this.db.put(this.encodeKey(id), value, this.dbReqOpts);
     }
     async delete(id) {
-        var _a;
-        (_a = this.dbWriteMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        await this.db.delete(this.encodeKey(id));
+        await this.db.delete(this.encodeKey(id), this.dbReqOpts);
     }
@@ -82,22 +72,16 @@ // The Id can be inferred from the value
     async batchPut(items) {
-        var _a;
-        (_a = this.dbWriteMetrics) === null || _a === void 0 ? void 0 : _a.inc();
         await this.db.batchPut(Array.from({ length: items.length }, (_, i) => ({
             key: this.encodeKey(items[i].key),
             value: this.encodeValue(items[i].value),
-        })));
+        })), this.dbReqOpts);
     }
     // Similar to batchPut but we support value as Uint8Array
     async batchPutBinary(items) {
-        var _a;
-        (_a = this.dbWriteMetrics) === null || _a === void 0 ? void 0 : _a.inc();
         await this.db.batchPut(Array.from({ length: items.length }, (_, i) => ({
             key: this.encodeKey(items[i].key),
             value: items[i].value,
-        })));
+        })), this.dbReqOpts);
     }
     async batchDelete(ids) {
-        var _a;
-        (_a = this.dbWriteMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        await this.db.batchDelete(Array.from({ length: ids.length }, (_, i) => this.encodeKey(ids[i])));
+        await this.db.batchDelete(Array.from({ length: ids.length }, (_, i) => this.encodeKey(ids[i])), this.dbReqOpts);
     }
@@ -114,4 +98,2 @@ async batchAdd(values) {
     async keys(opts) {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
         const data = await this.db.keys(this.dbFilterOptions(opts));
@@ -121,4 +103,2 @@ return (data !== null && data !== void 0 ? data : []).map((data) => this.decodeKey(data));
     async *keysStream(opts) {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
         const keysStream = this.db.keysStream(this.dbFilterOptions(opts));
@@ -131,4 +111,2 @@ const decodeKey = this.decodeKey.bind(this);
     async values(opts) {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
         const data = await this.db.values(this.dbFilterOptions(opts));
@@ -138,4 +116,2 @@ return (data !== null && data !== void 0 ? data : []).map((data) => this.decodeValue(data));
     async *valuesStream(opts) {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
         const valuesStream = this.db.valuesStream(this.dbFilterOptions(opts));
@@ -148,9 +124,5 @@ const decodeValue = this.decodeValue.bind(this);
     async *binaryEntriesStream(opts) {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
         yield* this.db.entriesStream(this.dbFilterOptions(opts));
     }
     async entries(opts) {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
         const data = await this.db.entries(this.dbFilterOptions(opts));
@@ -163,4 +135,2 @@ return (data !== null && data !== void 0 ? data : []).map((data) => ({
     async *entriesStream(opts) {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
         const entriesStream = this.db.entriesStream(this.dbFilterOptions(opts));
@@ -177,5 +147,4 @@ const decodeKey = this.decodeKey.bind(this);
     async firstKey() {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        const keys = await this.keys({ limit: 1 });
+        // Metrics accounted in this.keys()
+        const keys = await this.keys({ limit: 1, bucketId: this.bucketId });
         if (!keys.length) {
@@ -187,5 +156,4 @@ return null;
     async lastKey() {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        const keys = await this.keys({ limit: 1, reverse: true });
+        // Metrics accounted in this.keys()
+        const keys = await this.keys({ limit: 1, reverse: true, bucketId: this.bucketId });
         if (!keys.length) {
@@ -197,5 +165,4 @@ return null;
     async firstValue() {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        const values = await this.values({ limit: 1 });
+        // Metrics accounted in this.values()
+        const values = await this.values({ limit: 1, bucketId: this.bucketId });
         if (!values.length) {
@@ -207,5 +174,4 @@ return null;
     async lastValue() {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        const values = await this.values({ limit: 1, reverse: true });
+        // Metrics accounted in this.values()
+        const values = await this.values({ limit: 1, reverse: true, bucketId: this.bucketId });
         if (!values.length) {
@@ -217,5 +183,4 @@ return null;
     async firstEntry() {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        const entries = await this.entries({ limit: 1 });
+        // Metrics accounted in this.entries()
+        const entries = await this.entries({ limit: 1, bucketId: this.bucketId });
         if (!entries.length) {
@@ -227,5 +192,4 @@ return null;
     async lastEntry() {
-        var _a;
-        (_a = this.dbReadsMetrics) === null || _a === void 0 ? void 0 : _a.inc();
-        const entries = await this.entries({ limit: 1, reverse: true });
+        // Metrics accounted in this.entries()
+        const entries = await this.entries({ limit: 1, reverse: true, bucketId: this.bucketId });
         if (!entries.length) {
@@ -243,2 +207,3 @@ return null;
             lt: _encodeKey(this.bucket + 1, Buffer.alloc(0)),
+            bucketId: this.bucketId,
         };
@@ -245,0 +210,0 @@ if (opts) {

lib/controller/index.d.ts

@@ -6,2 +6,3 @@ /**
 export { LevelDbController } from "./level.js";
+export { ILevelDbControllerMetrics } from "./metrics.js";
 //# sourceMappingURL=index.d.ts.map

lib/controller/interface.d.ts

 /**
  * @module db/controller
  */
+import { ILevelDbControllerMetrics } from "./metrics.js";
 /** Shortcut for Uint8Array based IDatabaseController */
@@ -16,3 +17,9 @@ export declare type Db = IDatabaseController<Uint8Array, Uint8Array>;
     limit?: number;
+    /** For metrics */
+    bucketId?: string;
 }
+export declare type DbReqOpts = {
+    /** For metrics */
+    bucketId?: string;
+};
 export interface IKeyValue<K, V> {
@@ -25,7 +32,9 @@ key: K;
     stop(): Promise<void>;
-    get(key: K): Promise<V | null>;
-    put(key: K, value: V): Promise<void>;
-    delete(key: K): Promise<void>;
-    batchPut(items: IKeyValue<K, V>[]): Promise<void>;
-    batchDelete(keys: K[]): Promise<void>;
+    /** To inject metrics after CLI initialization */
+    setMetrics(metrics: ILevelDbControllerMetrics): void;
+    get(key: K, opts?: DbReqOpts): Promise<V | null>;
+    put(key: K, value: V, opts?: DbReqOpts): Promise<void>;
+    delete(key: K, opts?: DbReqOpts): Promise<void>;
+    batchPut(items: IKeyValue<K, V>[], opts?: DbReqOpts): Promise<void>;
+    batchDelete(keys: K[], opts?: DbReqOpts): Promise<void>;
     keysStream(opts?: IFilterOptions<K>): AsyncIterable<K>;
@@ -32,0 +41,0 @@ keys(opts?: IFilterOptions<K>): Promise<K[]>;

lib/controller/level.d.ts

@@ -6,6 +6,11 @@ /**
 import { ILogger } from "@chainsafe/lodestar-utils";
-import { IDatabaseController, IDatabaseOptions, IFilterOptions, IKeyValue } from "./interface.js";
+import { DbReqOpts, IDatabaseController, IDatabaseOptions, IFilterOptions, IKeyValue } from "./interface.js";
+import { ILevelDbControllerMetrics } from "./metrics.js";
 export interface ILevelDBOptions extends IDatabaseOptions {
     db?: LevelUp;
 }
+export declare type LevelDbControllerModules = {
+    logger: ILogger;
+    metrics?: ILevelDbControllerMetrics | null;
+};
 /**
@@ -17,15 +22,16 @@ * The LevelDB implementation of DB
     private db;
-    private opts;
-    private logger;
-    constructor(opts: ILevelDBOptions, { logger }: {
-        logger: ILogger;
-    });
+    private readonly opts;
+    private readonly logger;
+    private metrics;
+    constructor(opts: ILevelDBOptions, { logger, metrics }: LevelDbControllerModules);
     start(): Promise<void>;
     stop(): Promise<void>;
+    /** To inject metrics after CLI initialization */
+    setMetrics(metrics: ILevelDbControllerMetrics): void;
     clear(): Promise<void>;
-    get(key: Uint8Array): Promise<Uint8Array | null>;
-    put(key: Uint8Array, value: Uint8Array): Promise<void>;
-    delete(key: Uint8Array): Promise<void>;
-    batchPut(items: IKeyValue<Uint8Array, Uint8Array>[]): Promise<void>;
-    batchDelete(keys: Uint8Array[]): Promise<void>;
+    get(key: Uint8Array, opts?: DbReqOpts): Promise<Uint8Array | null>;
+    put(key: Uint8Array, value: Uint8Array, opts?: DbReqOpts): Promise<void>;
+    delete(key: Uint8Array, opts?: DbReqOpts): Promise<void>;
+    batchPut(items: IKeyValue<Uint8Array, Uint8Array>[], opts?: DbReqOpts): Promise<void>;
+    batchDelete(keys: Uint8Array[], opts?: DbReqOpts): Promise<void>;
     keysStream(opts?: IFilterOptions<Uint8Array>): AsyncGenerator<Uint8Array>;
@@ -32,0 +38,0 @@ valuesStream(opts?: IFilterOptions<Uint8Array>): AsyncGenerator<Uint8Array>;

lib/controller/level.js

@@ -11,2 +11,3 @@ /**
 })(Status || (Status = {}));
+const BUCKET_ID_UNKNOWN = "unknown";
 /**
@@ -16,6 +17,7 @@ * The LevelDB implementation of DB
 export class LevelDbController {
-    constructor(opts, { logger }) {
+    constructor(opts, { logger, metrics }) {
         this.status = Status.stopped;
         this.opts = opts;
         this.logger = logger;
+        this.metrics = metrics !== null && metrics !== void 0 ? metrics : null;
         // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-assignment
@@ -37,7 +39,19 @@ this.db = opts.db || level(opts.name || "beaconchain", { keyEncoding: "binary", valueEncoding: "binary" });
     }
+    /** To inject metrics after CLI initialization */
+    setMetrics(metrics) {
+        if (this.metrics !== null) {
+            throw Error("metrics can only be set once");
+        }
+        else {
+            this.metrics = metrics;
+        }
+    }
     async clear() {
         await this.db.clear();
     }
-    async get(key) {
+    async get(key, opts) {
+        var _a, _b, _c, _d;
         try {
+            (_a = this.metrics) === null || _a === void 0 ? void 0 : _a.dbReadReq.inc({ bucket: (_b = opts === null || opts === void 0 ? void 0 : opts.bucketId) !== null && _b !== void 0 ? _b : BUCKET_ID_UNKNOWN }, 1);
+            (_c = this.metrics) === null || _c === void 0 ? void 0 : _c.dbReadItems.inc({ bucket: (_d = opts === null || opts === void 0 ? void 0 : opts.bucketId) !== null && _d !== void 0 ? _d : BUCKET_ID_UNKNOWN }, 1);
             return (await this.db.get(key));
@@ -52,9 +66,18 @@ }
     }
-    async put(key, value) {
+    async put(key, value, opts) {
+        var _a, _b, _c, _d;
+        (_a = this.metrics) === null || _a === void 0 ? void 0 : _a.dbWriteReq.inc({ bucket: (_b = opts === null || opts === void 0 ? void 0 : opts.bucketId) !== null && _b !== void 0 ? _b : BUCKET_ID_UNKNOWN }, 1);
+        (_c = this.metrics) === null || _c === void 0 ? void 0 : _c.dbWriteItems.inc({ bucket: (_d = opts === null || opts === void 0 ? void 0 : opts.bucketId) !== null && _d !== void 0 ? _d : BUCKET_ID_UNKNOWN }, 1);
         await this.db.put(key, value);
     }
-    async delete(key) {
+    async delete(key, opts) {
+        var _a, _b, _c, _d;
+        (_a = this.metrics) === null || _a === void 0 ? void 0 : _a.dbWriteReq.inc({ bucket: (_b = opts === null || opts === void 0 ? void 0 : opts.bucketId) !== null && _b !== void 0 ? _b : BUCKET_ID_UNKNOWN }, 1);
+        (_c = this.metrics) === null || _c === void 0 ? void 0 : _c.dbWriteItems.inc({ bucket: (_d = opts === null || opts === void 0 ? void 0 : opts.bucketId) !== null && _d !== void 0 ? _d : BUCKET_ID_UNKNOWN }, 1);
         await this.db.del(key);
     }
-    async batchPut(items) {
+    async batchPut(items, opts) {
+        var _a, _b, _c, _d;
+        (_a = this.metrics) === null || _a === void 0 ? void 0 : _a.dbWriteReq.inc({ bucket: (_b = opts === null || opts === void 0 ? void 0 : opts.bucketId) !== null && _b !== void 0 ? _b : BUCKET_ID_UNKNOWN }, 1);
+        (_c = this.metrics) === null || _c === void 0 ? void 0 : _c.dbWriteItems.inc({ bucket: (_d = opts === null || opts === void 0 ? void 0 : opts.bucketId) !== null && _d !== void 0 ? _d : BUCKET_ID_UNKNOWN }, items.length);
         const batch = this.db.batch();
@@ -65,3 +88,6 @@ for (const item of items)
     }
-    async batchDelete(keys) {
+    async batchDelete(keys, opts) {
+        var _a, _b, _c, _d;
+        (_a = this.metrics) === null || _a === void 0 ? void 0 : _a.dbWriteReq.inc({ bucket: (_b = opts === null || opts === void 0 ? void 0 : opts.bucketId) !== null && _b !== void 0 ? _b : BUCKET_ID_UNKNOWN }, 1);
+        (_c = this.metrics) === null || _c === void 0 ? void 0 : _c.dbWriteItems.inc({ bucket: (_d = opts === null || opts === void 0 ? void 0 : opts.bucketId) !== null && _d !== void 0 ? _d : BUCKET_ID_UNKNOWN }, keys.length);
         const batch = this.db.batch();
@@ -100,6 +126,8 @@ for (const key of keys)
     async *iterator(keysOpts, getValue, opts) {
+        var _a, _b, _c, _d;
+        (_a = this.metrics) === null || _a === void 0 ? void 0 : _a.dbWriteReq.inc({ bucket: (_b = opts === null || opts === void 0 ? void 0 : opts.bucketId) !== null && _b !== void 0 ? _b : BUCKET_ID_UNKNOWN }, 1);
+        let itemsRead = 0;
         // Entries = { keys: true, values: true }
         // Keys =    { keys: true, values: false }
         // Values =  { keys: false, values: true }
-        var _a;
         const iterator = this.db.iterator({
@@ -109,3 +137,3 @@ ...opts,
             // TODO: Test if this is necessary. It's in https://github.com/Level/iterator-stream but may be stale
-            limit: (_a = opts === null || opts === void 0 ? void 0 : opts.limit) !== null && _a !== void 0 ? _a : -1,
+            limit: (_c = opts === null || opts === void 0 ? void 0 : opts.limit) !== null && _c !== void 0 ? _c : -1,
         });
@@ -127,2 +155,4 @@ try {
                 }
+                // Count metrics after done condition
+                itemsRead++;
                 yield getValue(key, value);
@@ -132,2 +162,3 @@ }
         finally {
+            (_d = this.metrics) === null || _d === void 0 ? void 0 : _d.dbWriteItems.inc(itemsRead);
             // TODO: Should we await here?
@@ -134,0 +165,0 @@ await new Promise((resolve, reject) => {

lib/databaseService.d.ts

 import { IChainForkConfig } from "@chainsafe/lodestar-config";
+import { ILevelDbControllerMetrics } from "./controller/metrics.js";
 import { Db } from "./controller/index.js";
-import { IDbMetrics } from "./metrics.js";
 export interface IDatabaseApiOptions {
     config: IChainForkConfig;
     controller: Db;
-    metrics?: IDbMetrics;
 }
@@ -15,3 +14,5 @@ export declare abstract class DatabaseService {
     stop(): Promise<void>;
+    /** To inject metrics after CLI initialization */
+    setMetrics(metrics: ILevelDbControllerMetrics): void;
 }
 //# sourceMappingURL=databaseService.d.ts.map

lib/databaseService.js

@@ -12,3 +12,7 @@ export class DatabaseService {
     }
+    /** To inject metrics after CLI initialization */
+    setMetrics(metrics) {
+        this.db.setMetrics(metrics);
+    }
 }
 //# sourceMappingURL=databaseService.js.map

lib/index.d.ts

@@ -9,3 +9,2 @@ /**
 export * from "./const.js";
-export * from "./metrics.js";
 //# sourceMappingURL=index.d.ts.map

lib/index.js

@@ -9,3 +9,2 @@ /**
 export * from "./const.js";
-export * from "./metrics.js";
 //# sourceMappingURL=index.js.map

package.json

 {
   "name": "@chainsafe/lodestar-db",
-  "version": "0.39.0-dev.719d936cc3",
+  "version": "0.39.0-dev.83aa2ccff5",
   "description": "DB modules of Lodestar",
@@ -42,4 +42,4 @@ "author": "ChainSafe Systems",
   "dependencies": {
-    "@chainsafe/lodestar-config": "0.39.0-dev.719d936cc3",
-    "@chainsafe/lodestar-utils": "0.39.0-dev.719d936cc3",
+    "@chainsafe/lodestar-config": "0.39.0-dev.83aa2ccff5",
+    "@chainsafe/lodestar-utils": "0.39.0-dev.83aa2ccff5",
     "@chainsafe/ssz": "^0.9.2",
@@ -55,3 +55,3 @@ "@types/levelup": "^4.3.3",
   },
-  "gitHead": "85cc929ea94a499091ef0fcb2da24fbe6b6ab94b"
+  "gitHead": "03f499ee0f7abf44dfa218c6458fb71e4310da32"
 }

No alert changes

Improved metrics

Total package byte prevSize

63335

increased by6.75%

Lines of code

770

increased by2.12%

Dependency changes

• + Added@chainsafe/lodestar-config@0.39.0-dev.83aa2ccff5(transitive)

• + Added@chainsafe/lodestar-params@0.39.0-dev.83aa2ccff5(transitive)

• + Added@chainsafe/lodestar-types@0.39.0-dev.83aa2ccff5(transitive)

• + Added@chainsafe/lodestar-utils@0.39.0-dev.83aa2ccff5(transitive)

• - Removed@chainsafe/lodestar-config@0.39.0-dev.719d936cc3(transitive)

• - Removed@chainsafe/lodestar-params@0.39.0-dev.719d936cc3(transitive)

• - Removed@chainsafe/lodestar-types@0.39.0-dev.719d936cc3(transitive)

• - Removed@chainsafe/lodestar-utils@0.39.0-dev.719d936cc3(transitive)

• Updated@chainsafe/lodestar-config@0.39.0-dev.83aa2ccff5

• Updated@chainsafe/lodestar-utils@0.39.0-dev.83aa2ccff5