🎩 You're Invited:Meet the Socket team at Black Hat in Las Vegas, August 3-6.RSVP
Sign In

lmdb

Package Overview
Dependencies
Maintainers
3
Versions
197
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

lmdb - npm Package Compare versions

Comparing version
3.4.2
to
3.4.3
+316
-151
open.js

@@ -1,3 +0,18 @@

import { Compression, getAddress, arch, fs, path as pathModule, lmdbError, EventEmitter, MsgpackrEncoder, Env,
Dbi, tmpdir, os, nativeAddon, version, isLittleEndian } from './native.js';
import {
Compression,
getAddress,
arch,
fs,
path as pathModule,
lmdbError,
EventEmitter,
MsgpackrEncoder,
Env,
Dbi,
tmpdir,
os,
nativeAddon,
version,
isLittleEndian,
} from './native.js';
import { CachingStore, setGetLastVersion } from './caching.js';

@@ -15,7 +30,6 @@ import { addReadMethods, makeReusableBuffer } from './read.js';

const buffers = [];
const { onExit, getEnvsPointer, setEnvsPointer, getEnvFlags, setJSFlags } = nativeAddon;
if (globalThis.__lmdb_envs__)
setEnvsPointer(globalThis.__lmdb_envs__);
else
globalThis.__lmdb_envs__ = getEnvsPointer();
const { onExit, getEnvsPointer, setEnvsPointer, getEnvFlags, setJSFlags } =
nativeAddon;
if (globalThis.__lmdb_envs__) setEnvsPointer(globalThis.__lmdb_envs__);
else globalThis.__lmdb_envs__ = getEnvsPointer();

@@ -29,3 +43,3 @@ // this is hard coded as an upper limit because it is important assumption of the fixed buffers in writing instructions

const DEFAULT_COMMIT_DELAY = 0;
const DEFAULT_BEGINNING_KEY = Buffer.from([5]); // the default starting key for iteration, which excludes symbols/metadata
export const allDbs = new Map();

@@ -50,3 +64,4 @@ let defaultCompression;

}
if (!keyBytes) // TODO: Consolidate get buffer and key buffer (don't think we need both)
if (!keyBytes)
// TODO: Consolidate get buffer and key buffer (don't think we need both)
allocateFixedBuffer();

@@ -61,9 +76,15 @@ if (typeof path == 'object' && !options) {

if (path == null) {
options = Object.assign({
deleteOnClose: true,
noSync: true,
}, options);
path = tmpdir() + '/' + Math.floor(Math.random() * 2821109907455).toString(36) + '.mdb'
} else if (!options)
options = {};
options = Object.assign(
{
deleteOnClose: true,
noSync: true,
},
options,
);
path =
tmpdir() +
'/' +
Math.floor(Math.random() * 2821109907455).toString(36) +
'.mdb';
} else if (!options) options = {};
let extension = pathModule.extname(path);

@@ -73,19 +94,32 @@ let name = pathModule.basename(path, extension);

let isLegacyLMDB = version.patch < 90;
let remapChunks = (options.remapChunks || options.encryptionKey || (options.mapSize ?
(is32Bit && options.mapSize > 0x100000000) : // larger than fits in address space, must use dynamic maps
is32Bit)) && !isLegacyLMDB; // without a known map size, we default to being able to handle large data correctly/well*/
let remapChunks =
(options.remapChunks ||
options.encryptionKey ||
(options.mapSize
? is32Bit && options.mapSize > 0x100000000 // larger than fits in address space, must use dynamic maps
: is32Bit)) &&
!isLegacyLMDB; // without a known map size, we default to being able to handle large data correctly/well*/
let userMapSize = options.mapSize;
options = Object.assign({
noSubdir: Boolean(extension),
isRoot: true,
maxDbs: 12,
remapChunks,
keyBytes,
overlappingSync: (options.noSync || options.readOnly) ? false : (os != 'win32'),
// default map size limit of 4 exabytes when using remapChunks, since it is not preallocated and we can
// make it super huge.
mapSize: remapChunks ? 0x10000000000000 :
isLegacyLMDB ? is32Bit ? 0x1000000 : 0x100000000 : 0x20000, // Otherwise we start small with 128KB
safeRestore: process.env.LMDB_RESTORE == 'safe',
}, options);
options = Object.assign(
{
noSubdir: Boolean(extension),
isRoot: true,
maxDbs: 12,
remapChunks,
keyBytes,
overlappingSync:
options.noSync || options.readOnly ? false : os != 'win32',
// default map size limit of 4 exabytes when using remapChunks, since it is not preallocated and we can
// make it super huge.
mapSize: remapChunks
? 0x10000000000000
: isLegacyLMDB
? is32Bit
? 0x1000000
: 0x100000000
: 0x20000, // Otherwise we start small with 128KB
safeRestore: process.env.LMDB_RESTORE == 'safe',
},
options,
);
options.path = path;

@@ -95,3 +129,8 @@ if (options.asyncTransactionOrder == 'strict') {

}
if (nativeAddon.version.major + nativeAddon.version.minor / 100 + nativeAddon.version.patch / 10000 < 0.0980) {
if (
nativeAddon.version.major +
nativeAddon.version.minor / 100 +
nativeAddon.version.patch / 10000 <
0.098
) {
options.overlappingSync = false; // not support on older versions

@@ -106,18 +145,27 @@ options.trackMetrics = false;

if (!exists(options.noSubdir ? pathModule.dirname(path) : path))
fs.mkdirSync(options.noSubdir ? pathModule.dirname(path) : path, { recursive: true }
);
fs.mkdirSync(options.noSubdir ? pathModule.dirname(path) : path, {
recursive: true,
});
function makeCompression(compressionOptions) {
if (compressionOptions instanceof Compression)
return compressionOptions;
if (compressionOptions instanceof Compression) return compressionOptions;
let useDefault = typeof compressionOptions != 'object';
if (useDefault && defaultCompression)
return defaultCompression;
compressionOptions = Object.assign({
threshold: 1000,
dictionary: fs.readFileSync(new URL('./dict/dict.txt', import.meta.url.replace(/dist[\\\/]index.cjs$/, ''))),
getValueBytes: makeReusableBuffer(0),
}, compressionOptions);
let compression = Object.assign(new Compression(compressionOptions), compressionOptions);
if (useDefault)
defaultCompression = compression;
if (useDefault && defaultCompression) return defaultCompression;
compressionOptions = Object.assign(
{
threshold: 1000,
dictionary: fs.readFileSync(
new URL(
'./dict/dict.txt',
import.meta.url.replace(/dist[\\\/]index.cjs$/, ''),
),
),
getValueBytes: makeReusableBuffer(0),
},
compressionOptions,
);
let compression = Object.assign(
new Compression(compressionOptions),
compressionOptions,
);
if (useDefault) defaultCompression = compression;
return compression;

@@ -127,3 +175,7 @@ }

// legacy LMDB, turn off these options
Object.assign(options, { overlappingSync: false, remapChunks: false, safeRestore: false });
Object.assign(options, {
overlappingSync: false,
remapChunks: false,
safeRestore: false,
});
}

@@ -148,3 +200,4 @@ if (options.compression)

let env = new Env();
let jsFlags = (options.overlappingSync ? 0x1000 : 0) |
let jsFlags =
(options.overlappingSync ? 0x1000 : 0) |
(options.separateFlushed ? 1 : 0) |

@@ -154,7 +207,9 @@ (options.deleteOnClose ? 2 : 0);

env.path = path;
if (rc)
lmdbError(rc);
delete options.keyBytes // no longer needed, don't copy to stores
if (rc) lmdbError(rc);
delete options.keyBytes; // no longer needed, don't copy to stores
let maxKeySize = env.getMaxKeySize();
maxKeySize = Math.min(maxKeySize, options.pageSize ? MAX_KEY_SIZE : DEFAULT_MAX_KEY_SIZE);
maxKeySize = Math.min(
maxKeySize,
options.pageSize ? MAX_KEY_SIZE : DEFAULT_MAX_KEY_SIZE,
);
flags = getEnvFlags(env.address); // re-retrieve them, they are not necessarily the same if we are connecting to an existing env

@@ -164,3 +219,5 @@ if (flags & 0x1000) {

env.close();
throw new Error('Can not set noSync on a database that was opened with overlappingSync');
throw new Error(
'Can not set noSync on a database that was opened with overlappingSync',
);
}

@@ -170,3 +227,5 @@ } else if (options.overlappingSync) {

env.close();
throw new Error('Can not enable overlappingSync on a database that was opened without this flag');
throw new Error(
'Can not enable overlappingSync on a database that was opened without this flag',
);
}

@@ -179,3 +238,7 @@ options.overlappingSync = false;

env.readerCheck(); // clear out any stale entries
if ((options.overlappingSync || options.deleteOnClose) && !hasRegisteredOnExit && process.on) {
if (
(options.overlappingSync || options.deleteOnClose) &&
!hasRegisteredOnExit &&
process.on
) {
hasRegisteredOnExit = true;

@@ -189,5 +252,11 @@ process.on('exit', onExit);

if (dbName === undefined)
throw new Error('Database name must be supplied in name property (may be null for root database)');
throw new Error(
'Database name must be supplied in name property (may be null for root database)',
);
if (options.compression && dbOptions.compression !== false && typeof dbOptions.compression != 'object')
if (
options.compression &&
dbOptions.compression !== false &&
typeof dbOptions.compression != 'object'
)
dbOptions.compression = options.compression; // use the parent compression if available

@@ -198,16 +267,21 @@ else if (dbOptions.compression)

if (dbOptions.dupSort && (dbOptions.useVersions || dbOptions.cache)) {
throw new Error('The dupSort flag can not be combined with versions or caching');
throw new Error(
'The dupSort flag can not be combined with versions or caching',
);
}
let keyIsBuffer = dbOptions.keyIsBuffer
let keyIsBuffer = dbOptions.keyIsBuffer;
this.defaultBeginningKey = DEFAULT_BEGINNING_KEY;
if (dbOptions.keyEncoding == 'uint32') {
dbOptions.keyIsUint32 = true;
this.defaultBeginningKey = Buffer.from([0]);
} else if (dbOptions.keyEncoder) {
if (dbOptions.keyEncoder.enableNullTermination) {
dbOptions.keyEncoder.enableNullTermination()
} else
keyIsBuffer = true;
dbOptions.keyEncoder.enableNullTermination();
} else keyIsBuffer = true;
} else if (dbOptions.keyEncoding == 'binary') {
keyIsBuffer = true;
this.defaultBeginningKey = Buffer.from([0]);
}
let flags = (dbOptions.reverseKey ? 0x02 : 0) |
let flags =
(dbOptions.reverseKey ? 0x02 : 0) |
(dbOptions.dupSort ? 0x04 : 0) |

@@ -219,5 +293,9 @@ (dbOptions.dupFixed ? 0x10 : 0) |

(dbOptions.useVersions ? 0x100 : 0);
let keyType = (dbOptions.keyIsUint32 || dbOptions.keyEncoding == 'uint32') ? 2 : keyIsBuffer ? 3 : 0;
if (keyType == 2)
flags |= 0x08; // integer key
let keyType =
dbOptions.keyIsUint32 || dbOptions.keyEncoding == 'uint32'
? 2
: keyIsBuffer
? 3
: 0;
if (keyType == 2) flags |= 0x08; // integer key

@@ -234,11 +312,21 @@ if (options.readOnly) {

} else {
this.transactionSync(() => {
this.db = new Dbi(env, flags, dbName, keyType, dbOptions.compression);
}, options.overlappingSync ? 0x10002 : 2); // no flush-sync, but synchronously commit
this.transactionSync(
() => {
this.db = new Dbi(
env,
flags,
dbName,
keyType,
dbOptions.compression,
);
},
options.overlappingSync ? 0x10002 : 2,
); // no flush-sync, but synchronously commit
}
this._commitReadTxn(); // current read transaction becomes invalid after opening another db
if (!this.db || this.db.dbi == 0xffffffff) {// not found
throw new Error('Database not found')
if (!this.db || this.db.dbi == 0xffffffff) {
// not found
throw new Error('Database not found');
}
this.dbAddress = this.db.address
this.dbAddress = this.db.address;
this.db.name = dbName || null;

@@ -257,7 +345,12 @@ this.name = dbName;

this.commitDelay = DEFAULT_COMMIT_DELAY;
Object.assign(this, { // these are the options that are inherited
path: options.path,
encoding: options.encoding,
strictAsyncOrder: options.strictAsyncOrder,
}, dbOptions);
Object.assign(
this,
{
// these are the options that are inherited
path: options.path,
encoding: options.encoding,
strictAsyncOrder: options.strictAsyncOrder,
},
dbOptions,
);
let Encoder;

@@ -268,11 +361,50 @@ if (this.encoder && this.encoder.Encoder) {

}
if (!Encoder && !(this.encoder && this.encoder.encode) && (!this.encoding || this.encoding == 'msgpack' || this.encoding == 'cbor')) {
Encoder = (this.encoding == 'cbor' ? moduleRequire('cbor-x').Encoder : MsgpackrEncoder);
if (
!Encoder &&
!(this.encoder && this.encoder.encode) &&
(!this.encoding ||
this.encoding == 'msgpack' ||
this.encoding == 'cbor')
) {
Encoder =
this.encoding == 'cbor'
? moduleRequire('cbor-x').Encoder
: MsgpackrEncoder;
}
if (Encoder) {
this.encoder = new Encoder(Object.assign(
assignConstrainedProperties(['copyBuffers', 'getStructures', 'saveStructures', 'useFloat32', 'useRecords', 'structuredClone', 'variableMapSize', 'useTimestamp32', 'largeBigIntToFloat', 'encodeUndefinedAsNil', 'int64AsNumber', 'onInvalidDate', 'mapsAsObjects', 'useTag259ForMaps', 'pack', 'maxSharedStructures', 'shouldShareStructure', 'randomAccessStructure', 'freezeData'],
this.sharedStructuresKey !== undefined ? this.setupSharedStructures() : {
copyBuffers: true, // need to copy any embedded buffers that are found since we use unsafe buffers
}, options, dbOptions), this.encoder));
this.encoder = new Encoder(
Object.assign(
assignConstrainedProperties(
[
'copyBuffers',
'getStructures',
'saveStructures',
'useFloat32',
'useRecords',
'structuredClone',
'variableMapSize',
'useTimestamp32',
'largeBigIntToFloat',
'encodeUndefinedAsNil',
'int64AsNumber',
'onInvalidDate',
'mapsAsObjects',
'useTag259ForMaps',
'pack',
'maxSharedStructures',
'shouldShareStructure',
'randomAccessStructure',
'freezeData',
],
this.sharedStructuresKey !== undefined
? this.setupSharedStructures()
: {
copyBuffers: true, // need to copy any embedded buffers that are found since we use unsafe buffers
},
options,
dbOptions,
),
this.encoder,
),
);
}

@@ -285,3 +417,3 @@ if (this.encoding == 'json') {

this.decoder = this.encoder;
this.decoderCopies = !this.encoder.needsStableBuffer
this.decoderCopies = !this.encoder.needsStableBuffer;
}

@@ -294,15 +426,15 @@ this.maxKeySize = maxKeySize;

if (this.dupSort && this.name == null)
throw new Error('Can not open named databases if the main database is dupSort')
throw new Error(
'Can not open named databases if the main database is dupSort',
);
if (typeof dbName == 'object' && !dbOptions) {
dbOptions = dbName;
dbName = dbOptions.name;
} else
dbOptions = dbOptions || {};
} else dbOptions = dbOptions || {};
try {
return dbOptions.cache ?
new (CachingStore(LMDBStore, env))(dbName, dbOptions) :
new LMDBStore(dbName, dbOptions);
} catch(error) {
if (error.message == 'Database not found')
return; // return undefined to indicate db not found
return dbOptions.cache
? new (CachingStore(LMDBStore, env))(dbName, dbOptions)
: new LMDBStore(dbName, dbOptions);
} catch (error) {
if (error.message == 'Database not found') return; // return undefined to indicate db not found
if (error.message.indexOf('MDB_DBS_FULL') > -1) {

@@ -316,17 +448,17 @@ error.message += ' (increase your maxDbs option)';

let db = this.openDB(dbOptions);
if (callback)
callback(null, db);
if (callback) callback(null, db);
return db;
}
backup(path, compact) {
if (noFSAccess)
return;
if (noFSAccess) return;
fs.mkdirSync(pathModule.dirname(path), { recursive: true });
return new Promise((resolve, reject) => env.copy(path, compact, (error) => {
if (error) {
reject(error);
} else {
resolve();
}
}));
return new Promise((resolve, reject) =>
env.copy(path, compact, (error) => {
if (error) {
reject(error);
} else {
resolve();
}
}),
);
}

@@ -337,7 +469,10 @@ isOperational() {

sync(callback) {
return env.sync(callback || function(error) {
if (error) {
console.error(error);
}
});
return env.sync(
callback ||
function (error) {
if (error) {
console.error(error);
}
},
);
}

@@ -349,11 +484,15 @@ deleteDB() {

dropSync() {
this.transactionSync(() =>
this.db.drop({
justFreePages: false
}), options.overlappingSync ? 0x10002 : 2);
this.transactionSync(
() =>
this.db.drop({
justFreePages: false,
}),
options.overlappingSync ? 0x10002 : 2,
);
}
clear(callback) {
if (typeof callback == 'function')
return this.clearAsync(callback);
console.warn('clear() is deprecated, use clearAsync or clearSync instead');
if (typeof callback == 'function') return this.clearAsync(callback);
console.warn(
'clear() is deprecated, use clearAsync or clearSync instead',
);
this.clearSync();

@@ -363,11 +502,12 @@ }

if (this.encoder) {
if (this.encoder.clearSharedData)
this.encoder.clearSharedData()
else if (this.encoder.structures)
this.encoder.structures = []
if (this.encoder.clearSharedData) this.encoder.clearSharedData();
else if (this.encoder.structures) this.encoder.structures = [];
}
this.transactionSync(() =>
this.db.drop({
justFreePages: true
}), options.overlappingSync ? 0x10002 : 2);
this.transactionSync(
() =>
this.db.drop({
justFreePages: true,
}),
options.overlappingSync ? 0x10002 : 2,
);
}

@@ -383,7 +523,5 @@ readerCheck() {

let lastVersion; // because we are doing a read here, we may need to save and restore the lastVersion from the last read
if (this.useVersions)
lastVersion = getLastVersion();
if (this.useVersions) lastVersion = getLastVersion();
let buffer = this.getBinary(this.sharedStructuresKey);
if (this.useVersions)
setLastVersion(lastVersion);
if (this.useVersions) setLastVersion(lastVersion);
return buffer && this.decoder.decode(buffer);

@@ -393,11 +531,21 @@ };

saveStructures: (structures, isCompatible) => {
return this.transactionSync(() => {
let existingStructuresBuffer = this.getBinary(this.sharedStructuresKey);
let existingStructures = existingStructuresBuffer && this.decoder.decode(existingStructuresBuffer);
if (typeof isCompatible == 'function' ?
!isCompatible(existingStructures) :
(existingStructures && existingStructures.length != isCompatible))
return false; // it changed, we need to indicate that we couldn't update
this.put(this.sharedStructuresKey, structures);
}, options.overlappingSync ? 0x10000 : 0);
return this.transactionSync(
() => {
let existingStructuresBuffer = this.getBinary(
this.sharedStructuresKey,
);
let existingStructures =
existingStructuresBuffer &&
this.decoder.decode(existingStructuresBuffer);
if (
typeof isCompatible == 'function'
? !isCompatible(existingStructures)
: existingStructures &&
existingStructures.length != isCompatible
)
return false; // it changed, we need to indicate that we couldn't update
this.put(this.sharedStructuresKey, structures);
},
options.overlappingSync ? 0x10000 : 0,
);
},

@@ -412,6 +560,17 @@ getStructures,

const removeSync = LMDBStore.prototype.removeSync;
addReadMethods(LMDBStore, { env, maxKeySize, keyBytes, keyBytesView, getLastVersion });
addReadMethods(LMDBStore, {
env,
maxKeySize,
keyBytes,
keyBytesView,
getLastVersion,
});
if (!options.readOnly)
addWriteMethods(LMDBStore, { env, maxKeySize, fixedBuffer: keyBytes,
resetReadTxn: LMDBStore.prototype.resetReadTxn, ...options });
addWriteMethods(LMDBStore, {
env,
maxKeySize,
fixedBuffer: keyBytes,
resetReadTxn: LMDBStore.prototype.resetReadTxn,
...options,
});
LMDBStore.prototype.supports = {

@@ -425,3 +584,3 @@ permanence: true,

deferredOpen: true,
openCallback: true,
openCallback: true,
};

@@ -454,17 +613,24 @@ let Class = options.cache ? CachingStore(LMDBStore, env) : LMDBStore;

function allocateFixedBuffer() {
keyBytes = typeof Buffer != 'undefined' ? Buffer.allocUnsafeSlow(KEY_BUFFER_SIZE) : new Uint8Array(KEY_BUFFER_SIZE);
keyBytes =
typeof Buffer != 'undefined'
? Buffer.allocUnsafeSlow(KEY_BUFFER_SIZE)
: new Uint8Array(KEY_BUFFER_SIZE);
const keyBuffer = keyBytes.buffer;
keyBytesView = keyBytes.dataView || (keyBytes.dataView = new DataView(keyBytes.buffer, 0, KEY_BUFFER_SIZE)); // max key size is actually 4026
keyBytesView =
keyBytes.dataView ||
(keyBytes.dataView = new DataView(keyBytes.buffer, 0, KEY_BUFFER_SIZE)); // max key size is actually 4026
keyBytes.uint32 = new Uint32Array(keyBuffer, 0, KEY_BUFFER_SIZE >> 2);
keyBytes.float64 = new Float64Array(keyBuffer, 0, KEY_BUFFER_SIZE >> 3);
keyBytes.uint32.address = keyBytes.address = keyBuffer.address = getAddress(keyBuffer);
keyBytes.uint32.address =
keyBytes.address =
keyBuffer.address =
getAddress(keyBuffer);
}
function exists(path) {
if (fs.existsSync)
return fs.existsSync(path);
if (fs.existsSync) return fs.existsSync(path);
try {
return fs.statSync(path);
} catch (error) {
return false
return false;
}

@@ -477,4 +643,3 @@ }

for (let key in source) {
if (allowedProperties.includes(key))
target[key] = source[key];
if (allowedProperties.includes(key)) target[key] = source[key];
}

@@ -481,0 +646,0 @@ }

{
"name": "lmdb",
"author": "Kris Zyp",
"version": "3.4.2",
"version": "3.4.3",
"description": "Simple, efficient, scalable, high-performance LMDB interface",

@@ -114,10 +114,10 @@ "license": "MIT",

"optionalDependencies": {
"@lmdb/lmdb-darwin-arm64": "3.4.2",
"@lmdb/lmdb-darwin-x64": "3.4.2",
"@lmdb/lmdb-linux-arm": "3.4.2",
"@lmdb/lmdb-linux-arm64": "3.4.2",
"@lmdb/lmdb-linux-x64": "3.4.2",
"@lmdb/lmdb-win32-arm64": "3.4.2",
"@lmdb/lmdb-win32-x64": "3.4.2"
"@lmdb/lmdb-darwin-arm64": "3.4.3",
"@lmdb/lmdb-darwin-x64": "3.4.3",
"@lmdb/lmdb-linux-arm": "3.4.3",
"@lmdb/lmdb-linux-arm64": "3.4.3",
"@lmdb/lmdb-linux-x64": "3.4.3",
"@lmdb/lmdb-win32-arm64": "3.4.3",
"@lmdb/lmdb-win32-x64": "3.4.3"
}
}

@@ -26,7 +26,6 @@ import { RangeIterable } from './util/RangeIterable.js';

unlock,
isLittleEndian
isLittleEndian,
} from './native.js';
import { saveKey } from './keys.js';
const IF_EXISTS = 3.542694326329068e-103;
const DEFAULT_BEGINNING_KEY = Buffer.from([5]); // the default starting key for iteration, which excludes symbols/metadata
const ITERATOR_DONE = { done: true, value: undefined };

@@ -537,3 +536,3 @@ const Uint8ArraySlice = Uint8Array.prototype.slice;

? options.start
: DEFAULT_BEGINNING_KEY;
: this.defaultBeginningKey;
let count = 0;

@@ -674,3 +673,3 @@ let cursor, cursorRenewId, cursorAddress;

reverse && !('end' in options)
? DEFAULT_BEGINNING_KEY
? store.defaultBeginningKey
: options.end,

@@ -677,0 +676,0 @@ store.writeKey,

Sorry, the diff of this file is too big to display

Sorry, the diff of this file is too big to display

Sorry, the diff of this file is too big to display