jembadb - npm Package Compare versions

Comparing version 2.3.0 to 3.0.0

package.json

 {
   "name": "jembadb",
-  "version": "2.3.0",
+  "version": "3.0.0",
   "description": "Json Embeddable Appendable Database",
@@ -5,0 +5,0 @@ "main": "./src/index.js",

src/JembaDb.js

@@ -5,4 +5,5 @@ 'use strict';
 
-const Table = require('./Table');
-const TableMem = require('./TableMem');
+const MemoryTable = require('./MemoryTable');
+const ShardedTable = require('./ShardedTable');
+const BasicTable = require('./BasicTable');
 
@@ -56,5 +57,6 @@ const LockQueue = require('./LockQueue');
         tableDefaults: {
-            type: 'basic' | 'memory' | 'huge', default 'basic'
+            type: 'basic' | 'memory' | 'sharded', default 'basic'
             cacheSize: Number, 5
-            blockSize: Number, 1000000, for huge table only
+            cacheShards: Number, 1, for sharded table only
+            autoShardSize: Number, 1000000, for sharded table only
             compressed: Number, {0..9}, 0
@@ -64,3 +66,2 @@ recreate: Boolean, false,
             forceFileClosing: Boolean, false,
-            lazyOpen: Boolean, false,
             typeCompatMode: Boolean, false,
@@ -150,5 +151,6 @@ },
 
-        type: 'basic' | 'memory' | 'huge', default 'basic'
+        type: 'basic' | 'memory' | 'sharded', default 'basic'
         cacheSize: Number, 5
-        blockSize: Number, 1000000, for huge table only
+        cacheShards: Number, 1, for sharded table only
+        autoShardSize: Number, 1000000, for sharded table only
         compressed: Number, {0..9}, 0
@@ -158,3 +160,2 @@ recreate: Boolean, false,
         forceFileClosing: Boolean, false,
-        lazyOpen: Boolean, false,
         typeCompatMode: Boolean, false,
@@ -247,2 +248,3 @@
             this._tableLock(table).ret();
+            this.tableLockMap.delete(table);
         }
@@ -307,3 +309,3 @@ }
                 if (tableInstance.type === 'memory') {
-                    const newTableInstance = new TableMem();
+                    const newTableInstance = new MemoryTable();
 
@@ -318,2 +320,3 @@ const opts = Object.assign({}, this.tableOpenDefaults);
                     const toTable = `${table}___temporary_truncating`;
+                    await fs.rmdir(`${this.dbPath}/${toTable}`, { recursive: true });
 
@@ -374,3 +377,3 @@ await this._clone({table, toTable, filter: 'nodata'});
             if (tableInstance.type === 'memory') {
-                const newTableInstance = new TableMem();
+                const newTableInstance = new MemoryTable();
 
@@ -397,2 +400,13 @@ const opts = Object.assign({}, this.tableOpenDefaults);
 
+    async _getTableType(query) {
+        let result = 'basic';
+
+        const typePath = `${this.dbPath}/${query.table}/type`;
+        if (await utils.pathExists(typePath)) {
+            result = await fs.readFile(typePath, 'utf8');
+        }
+
+        return result;
+    }
+
     /*
@@ -403,5 +417,6 @@ query = {
 
-        type: 'basic' | 'memory' | 'huge', default 'basic'
+        type: 'basic' | 'memory' | 'sharded', default 'basic'
         cacheSize: Number, 5
-        blockSize: Number, 1000000, for huge table only
+        cacheShards: Number, 1, for sharded table only
+        autoShardSize: Number, 1000000, for sharded table only
         compressed: Number, {0..9}, 0
@@ -411,3 +426,2 @@ recreate: Boolean, false,
         forceFileClosing: Boolean, false,
-        lazyOpen: Boolean, false,
         typeCompatMode: Boolean, false,
@@ -422,11 +436,17 @@ }
 
-        if (await this.tableExists({table: query.table}) || query.create) {
+        const tableExists = await this.tableExists({table: query.table});
+        if (tableExists || query.create) {
             let tableInstance = this.table.get(query.table);
 
             if (!tableInstance || !tableInstance.opened) {
+                let type = query.type;
+                if (tableExists)
+                    type = await this._getTableType(query);
 
-                if (query.type === 'memory') {
-                    tableInstance = new TableMem();
+                if (type === 'memory') {
+                    tableInstance = new MemoryTable();
+                } else if (type === 'sharded') {
+                    tableInstance = new ShardedTable();
                 } else {
-                    tableInstance = new Table();
+                    tableInstance = new BasicTable();
                 }
@@ -471,5 +491,6 @@ this.table.set(query.table, tableInstance);
     query = {
-        type: 'basic' | 'memory' | 'huge', default 'basic'
+        type: 'basic' | 'memory' | 'sharded', default 'basic'
         cacheSize: Number, 5
-        blockSize: Number, 1000000, for huge table only
+        cacheShards: Number, 1, for sharded table only
+        autoShardSize: Number, 1000000, for sharded table only
         compressed: Number, {0..9}, 0
@@ -479,3 +500,2 @@ recreate: Boolean, false,
         forceFileClosing: Boolean, false,
-        lazyOpen: Boolean, false,
     }
@@ -633,2 +653,4 @@ */
     (!) table: 'tableName',
+        shards: ['shard1', 'shard2', ...] || '(s) => (s == 'shard1')', //for sharded table only
+        persistent: Boolean,//for sharded table only, do not unload shard while persistent == true
         count: Boolean,
@@ -705,2 +727,3 @@ where: `@@index('field1', 10, 20)`,
         replace: Boolean,
+        shardGen: '(r) => r.date',//for sharded table only
     (!) rows: Array,
@@ -731,2 +754,3 @@ }
     (!) mod: '(r) => r.count++',
+        shards: ['shard1', 'shard2', ...] || '(s) => (s == 'shard1')', //for sharded table only
         where: `@@index('field1', 10, 20)`,
@@ -783,2 +807,3 @@ sort: '(a, b) => a.id - b.id',
     query = {
+    (!) table: 'tableName',
         message: String,
@@ -785,0 +810,0 @@ }

src/TableReducer.js

@@ -1035,2 +1035,19 @@ 'use strict';
 
+    async limit(ids, lim, ofs = 0) {
+        const result = new Set();
+        for (const id of ids) {
+            if (ofs > 0) {
+                ofs--;
+                continue;
+            }
+
+            if (lim <= 0)
+                break;
+
+            result.add(id);
+            lim--;
+        }
+        return result;
+    }
+
     async and() {
@@ -1037,0 +1054,0 @@ const result = [];

src/TableRowsFile.js

@@ -11,4 +11,2 @@ 'use strict';
 const maxFileDumpSize = 50*1024*1024;//bytes
-const defragAfter = 10;
-const defragBlockCountAtOnce = 10;//better >= defragAfter
 
@@ -26,2 +24,3 @@ class TableRowsFile {
         this.blockList = new Map();
+        this.blockSetDefrag = new Set();
         this.blocksNotFinalized = new Map();//indexes of blocks
@@ -31,3 +30,2 @@ this.loadedBlocks = [];
 
-        this.defragCounter = 0;
         this.destroyed = false;
@@ -81,2 +79,3 @@
                 block.delCount++;
+                this.blockSetDefrag.add(block.index);
                 delta.blockList.push([block.index, 1]);
@@ -330,2 +329,3 @@ }
             this.blocksNotFinalized.delete(index);
+            this.blockSetDefrag.add(index);
         }
@@ -371,26 +371,24 @@ }
 
-        //check all blocks fragmentation
+        //check all blocks fragmentation & defragment if needed
         if (!this.defragCandidates)
             this.defragCandidates = [];
 
-        if (!this.defragCandidates.length) {
-            if (this.defragCounter >= defragAfter) {
-                for (const block of this.blockList.values()) {
-                    if (!block.final)
-                        continue;
+        if (!this.defragCandidates.length && this.blockSetDefrag.size) {
+            for (const index of this.blockSetDefrag) {
+                const block = this.blockList.get(index);
+                if (!block || !block.final)
+                    continue;
 
-                    if (block.addCount - block.delCount < block.rowsLength/2 || block.size < maxBlockSize/2) {
-                        this.defragCandidates.push(block);
-                    }
+                if ( (block.delCount > 0 && block.addCount - block.delCount < block.rowsLength*0.6)
+                    || block.size < maxBlockSize/2
+                    ) {
+                    this.defragCandidates.push(block);
                 }
+            }
 
-                this.defragCounter = 0;
-            } else {
-                this.defragCounter++;
-            }
+            this.blockSetDefrag = new Set();
         }
 
-        let defragmented = 0;
         while (this.defragCandidates.length) {
-            if (defragmented >= defragBlockCountAtOnce || this.destroyed)
+            if (this.destroyed)
                 break;
@@ -420,3 +418,2 @@
 
-            defragmented++;
 //console.log(`defragmented block ${block.index}, size: ${block.size}, addCount: ${block.addCount}, delCount: ${block.delCount}, rowsLength: ${block.rowsLength}`);
@@ -580,2 +577,3 @@ }
         for (const block of this.blockList.values()) {
+            this.blockSetDefrag.add(block.index);
             if (!block.final)
@@ -582,0 +580,0 @@ this.blocksNotFinalized.set(block.index, 1);

New alerts

Uses eval

Supply chain risk

Package uses dynamic code execution (e.g., eval()), which is a dangerous practice. This can prevent the code from running in certain environments and increases the risk that the code may contain exploits or malicious behavior.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

180344

increased by25.05%

Number of package files

20

increased by5.26%

Lines of code

4739

increased by24.78%

Worsened metrics

Number of low supply chain risk alerts

6

increased by20%

Number of medium supply chain risk alerts

15

increased by25%

No dependency changes