aerospike - npm Package Compare versions

Comparing version 3.16.0 to 3.16.1

benchmarks/main.js

 // *****************************************************************************
-// Copyright 2013-2019 Aerospike, Inc.
+// Copyright 2013-2020 Aerospike, Inc.
 //
@@ -21,9 +21,11 @@ // Licensed under the Apache License, Version 2.0 (the "License")
 
-var aerospike = require('aerospike')
-var cluster = require('cluster')
-var winston = require('winston')
-var stats = require('./stats')
-var alerts = require('./alerts')
-var argv = require('./config.json')
+const { format } = require('util')
 
+const aerospike = require('aerospike')
+const cluster = require('cluster')
+const winston = require('winston')
+const stats = require('./stats')
+const alerts = require('./alerts')
+const argv = require('./config.json')
+
 // *****************************************************************************
@@ -203,16 +205,28 @@ // Globals
   if (rwWorkers > 0) {
-    logger.info('%s read(tps=%d timeouts=%d errors=%d) write(tps=%d timeouts=%d errors=%d) ',
+    logger.info('%s read(tps=%d timeouts=%d errors=%d) write(tps=%d timeouts=%d errors=%d) mem(%s)',
       new Date().toString(), intervalStats[0][0], intervalStats[0][1], intervalStats[0][2],
-      intervalStats[1][0], intervalStats[1][1], intervalStats[1][2])
+      intervalStats[1][0], intervalStats[1][1], intervalStats[1][2],
+      memUsage())
   }
   if (queryWorkers) {
-    logger.info('%s query(records = %d timeouts = %d errors = %d)',
-      new Date().toString(), intervalStats[2][0], intervalStats[2][1], intervalStats[2][2])
+    logger.info('%s query(records = %d timeouts = %d errors = %d) mem(%s)',
+      new Date().toString(), intervalStats[2][0], intervalStats[2][1], intervalStats[2][2],
+      memUsage())
   }
   if (scanWorkers) {
-    logger.info('%s scan(records = %d timeouts = %d errors = %d)',
-      new Date().toString(), intervalStats[3][0], intervalStats[3][1], intervalStats[3][2])
+    logger.info('%s scan(records = %d timeouts = %d errors = %d) mem(%s)',
+      new Date().toString(), intervalStats[3][0], intervalStats[3][1], intervalStats[3][2],
+      memUsage())
   }
 }
 
+const MEGA = 1024 * 1024 // bytes in a MB
+function memUsage () {
+  const memUsage = process.memoryUsage()
+  const rss = Math.round(memUsage.rss / MEGA)
+  const heapUsed = Math.round(memUsage.heapUsed / MEGA)
+  const heapTotal = Math.round(memUsage.heapTotal / MEGA)
+  return format('%d MB, heap: %d / %d MB', rss, heapUsed, heapTotal)
+}
+
 function workerResultsIteration (worker, opStats) {
@@ -219,0 +233,0 @@ stats.iteration(opStats)

CHANGELOG.md

@@ -7,2 +7,7 @@ # Changelog
 
+## [3.16.1] - 2020-06-30
+
+* **Bug Fixes**
+  * Fix memory leaks when running secondary index queries with string filter predicates. [#370](https://github.com/aerospike/aerospike-client-nodejs/issues/370)
+
 ## [3.16.0] - 2020-05-18
@@ -16,3 +21,3 @@
 * **Updates**
-  * *BREAKING*: The client no longer supports the percent-based scan sampling for server versions 4.9 or later. Use the new max records scan policy option instead.
+  * *BREAKING*: The client no longer supports the percent-based scan sampling for server versions 4.9 or later. Use the new max records scan policy option instead. See [API Changes](https://www.aerospike.com/docs/client/nodejs/usage/incompatible.html#version-3-16-0) for details.
   * Update C client library to [v4.6.16](http://www.aerospike.com/download/client/c/notes.html#4.6.16).
@@ -48,2 +53,3 @@ * Update dependencies with potential vulnerabilities by running npm audit fix. [#367](https://github.com/aerospike/aerospike-client-nodejs/pull/367)
 * **Updates**
+  * Add support for RHEL/CentOS 8.
   * Update C client library to [v4.6.10](http://www.aerospike.com/download/client/c/notes.html#4.6.10).
@@ -263,3 +269,3 @@ * Minor API document updates. [#342](https://github.com/aerospike/aerospike-client-nodejs/pull/342)
   * Support CDT List Increment operation. Requires Aerospike server version 3.15 or later.
-  * Improved timeout handling and automatic transaction retries - see [detailed API changes](https://github.com/aerospike/aerospike-client-nodejs/blob/master/docs/api-changes.md#version-300) for more info.
+  * Improved timeout handling and automatic transaction retries - see [detailed API changes](https://www.aerospike.com/docs/client/nodejs/usage/incompatible.html#version-3-0-0) for more info.
   * Support gen policy for apply UDF operation.
@@ -278,3 +284,3 @@
 
-Please refer to the full list of [backward incompatible API changes](https://github.com/aerospike/aerospike-client-nodejs/blob/master/docs/api-changes.md#version-300)
+Please refer to the full list of [backward incompatible API changes](https://www.aerospike.com/docs/client/nodejs/usage/incompatible.html#version-3-0-0)
 for further details.
@@ -311,3 +317,3 @@
   * Update C client library to [v4.1.6](http://www.aerospike.com/download/client/c/notes.html#4.1.6).
-  * The Client#udfRegisterWait and Info#parseInfo functions have been marked deprecated. See [API Changes](https://github.com/aerospike/aerospike-client-nodejs/blob/master/docs/api-changes.md) for details.
+  * The Client#udfRegisterWait and Info#parseInfo functions have been marked deprecated. See [API Changes](https://www.aerospike.com/docs/client/nodejs/usage/incompatible.html#version-2-6-0) for details.
 
@@ -404,3 +410,3 @@ ## [2.5.2] - 2017-04-20
   * Shared memory layout has changed in v4.1.0 of the C client library. See
-    [backward incompatible API changes](https://github.com/aerospike/aerospike-client-nodejs/blob/master/docs/api-changes.md#version-240)
+    [backward incompatible API changes](https://www.aerospike.com/docs/client/nodejs/usage/incompatible.html#version-2-4-0)
     for more details.
@@ -459,3 +465,3 @@ * The <code>retry</code> policy value for read, write, remove and operate
   * The `aerospike.operator` module has been split up into two seperate modules `aerospike.operations` and `aerospike.lists` for operations on scalar and
-    list data types respectively. See detailed list of [API changes](https://github.com/aerospike/aerospike-client-nodejs/blob/master/docs/api-changes.md#version-210)
+    list data types respectively. See detailed list of [API changes](https://www.aerospike.com/docs/client/nodejs/usage/incompatible.html#version-2-1-0)
     for further details.
@@ -560,3 +566,3 @@
     was executed successfully. See
-    [backward incompatible API changes](https://github.com/aerospike/aerospike-client-nodejs/blob/master/docs/api-changes.md)
+    [backward incompatible API changes](https://www.aerospike.com/docs/client/nodejs/usage/incompatible.html#version-2-0-0-alpha-1)
     for more details. [#105](https://github.com/aerospike/aerospike-client-nodejs/issues/105),
@@ -563,0 +569,0 @@ [PR #106](https://github.com/aerospike/aerospike-client-nodejs/pull/106). Thanks to

package.json

 {
   "name": "aerospike",
-  "version": "3.16.0",
+  "version": "3.16.1",
   "description": "Aerospike Client Library",
@@ -59,2 +59,3 @@ "keywords": [
     "nyc": "^15.0.0",
+    "p-throttle": "^3.1.0",
     "semver": "^7.1.1",
@@ -61,0 +62,0 @@ "standard": "^14.3.1",

README.md

@@ -15,3 +15,3 @@ # Aerospike Node.js Client [![travis][travis-image]][travis-url] [![codecov][codecov-image]][codecov-url] [![npm][npm-image]][npm-url] [![downloads][downloads-image]][downloads-url]
 This module is compatible with Node.js v8.x (LTS), v10.x (LTS), and v12.x
-(LTS). It supports the following operating systems: CentOS/RHEL 6/7, Debian
+(LTS). It supports the following operating systems: CentOS/RHEL 6/7/8, Debian
 8/9/10, Ubuntu 14.04/16.04/18.04, as well as many Linux destributions compatible
@@ -18,0 +18,0 @@ with one of these OS releases. macOS and Windows are also supported.

test/batch_exists.js

@@ -34,8 +34,9 @@ // *****************************************************************************
   it('should successfully find 10 records', function () {
-    var numberOfRecords = 10
-    var kgen = keygen.string(helper.namespace, helper.set, { prefix: 'test/batch_exists/10/', random: false })
-    var mgen = metagen.constant({ ttl: 1000 })
-    var rgen = recgen.record({ i: valgen.integer(), s: valgen.string(), b: valgen.bytes() })
-
-    return putgen.put(numberOfRecords, kgen, rgen, mgen)
+    const numberOfRecords = 10
+    const generators = {
+      keygen: keygen.string(helper.namespace, helper.set, { prefix: 'test/batch_exists/10/', random: false }),
+      recgen: recgen.record({ i: valgen.integer(), s: valgen.string(), b: valgen.bytes() }),
+      metagen: metagen.constant({ ttl: 1000 })
+    }
+    return putgen.put(numberOfRecords, generators)
       .then(records => {
@@ -42,0 +43,0 @@ const keys = records.map(record => record.key)

test/batch_get.js

@@ -34,8 +34,9 @@ // *****************************************************************************
   it('should successfully read 10 records', function () {
-    var numberOfRecords = 10
-    var kgen = keygen.string(helper.namespace, helper.set, { prefix: 'test/batch_get/success', random: false })
-    var mgen = metagen.constant({ ttl: 1000 })
-    var rgen = recgen.record({ i: valgen.integer(), s: valgen.string(), b: valgen.bytes() })
-
-    return putgen.put(numberOfRecords, kgen, rgen, mgen)
+    const numberOfRecords = 10
+    const generators = {
+      keygen: keygen.string(helper.namespace, helper.set, { prefix: 'test/batch_get/success', random: false }),
+      recgen: recgen.record({ i: valgen.integer(), s: valgen.string(), b: valgen.bytes() }),
+      metagen: metagen.constant({ ttl: 1000 })
+    }
+    return putgen.put(numberOfRecords, generators)
       .then(records => {
@@ -42,0 +43,0 @@ const keys = records.map(record => record.key)

test/batch_read.js

@@ -38,11 +38,13 @@ // *****************************************************************************
     const nrecords = 10
-    const kgen = keygen.string(helper.namespace, helper.set, { prefix: 'test/batch_read/', random: false })
-    const mgen = metagen.constant({ ttl: 1000 })
-    const rgen = recgen.record({
-      i: valgen.integer(),
-      s: valgen.string(),
-      l: () => [1, 2, 3],
-      m: () => { return { a: 1, b: 2, c: 3 } }
-    })
-    return putgen.put(nrecords, kgen, rgen, mgen)
+    const generators = {
+      keygen: keygen.string(helper.namespace, helper.set, { prefix: 'test/batch_read/', random: false }),
+      recgen: recgen.record({
+        i: valgen.integer(),
+        s: valgen.string(),
+        l: () => [1, 2, 3],
+        m: () => { return { a: 1, b: 2, c: 3 } }
+      }),
+      metagen: metagen.constant({ ttl: 1000 })
+    }
+    return putgen.put(nrecords, generators)
   })
@@ -49,0 +51,0 @@

test/batch_select.js

@@ -34,8 +34,9 @@ // *****************************************************************************
   it('should successfully read bins from 10 records', function () {
-    var numberOfRecords = 10
-    var kgen = keygen.string(helper.namespace, helper.set, { prefix: 'test/batch_get/success', random: false })
-    var mgen = metagen.constant({ ttl: 1000 })
-    var rgen = recgen.record({ i: valgen.integer(), s: valgen.string(), b: valgen.bytes() })
-
-    return putgen.put(numberOfRecords, kgen, rgen, mgen)
+    const numberOfRecords = 10
+    const generators = {
+      keygen: keygen.string(helper.namespace, helper.set, { prefix: 'test/batch_get/success', random: false }),
+      recgen: recgen.record({ i: valgen.integer(), s: valgen.string(), b: valgen.bytes() }),
+      metagen: metagen.constant({ ttl: 1000 })
+    }
+    return putgen.put(numberOfRecords, generators)
       .then(records => {
@@ -42,0 +43,0 @@ const keys = records.map(record => record.key)

test/generators/put.js

 // *****************************************************************************
-// Copyright 2013-2019 Aerospike, Inc.
+// Copyright 2013-2020 Aerospike, Inc.
 //
@@ -19,2 +19,4 @@ // Licensed under the Apache License, Version 2.0 (the "License")
 
+const pThrottle = require('p-throttle')
+
 const Aerospike = require('../../lib/aerospike')
@@ -24,3 +26,3 @@ const Record = require('../../lib/record')
 
-function createRecords (client, generator, recordsToCreate, maxConcurrent, callback) {
+function createRecords (putCall, generator, recordsToCreate, maxConcurrent, callback) {
   var currentRecordNo = 0
@@ -44,3 +46,3 @@ var inFlight = 0
       const meta = { ttl: record.ttl, gen: record.gen }
-      client.put(record.key, record.bins, meta, policy, putCb)
+      putCall(record.key, record.bins, meta, policy, putCb)
       inFlight++
@@ -57,25 +59,27 @@ } else if (currentRecordNo > recordsToCreate && inFlight === 0) {
 
-function put (n, keygen, recgen, metagen, policy, callback) {
-  if (typeof policy === 'function') {
-    callback = policy
-    policy = null
+function put (n, options, callback) {
+  const policy = options.policy || new Aerospike.WritePolicy({
+    totalTimeout: 1000,
+    exists: Aerospike.policy.exists.CREATE_OR_REPLACE
+  })
+
+  const generator = {
+    key: options.keygen,
+    bins: options.recgen,
+    metadata: options.metagen,
+    policy: function () { return policy }
   }
-  if (typeof policy === 'undefined') {
-    policy = new Aerospike.WritePolicy({
-      totalTimeout: 1000,
-      exists: Aerospike.policy.exists.CREATE_OR_REPLACE
-    })
+
+  let putCall = helper.client.put.bind(helper.client)
+  if (options.throttle) {
+    const { limit, interval } = options.throttle
+    putCall = pThrottle(putCall, limit, interval)
   }
-  var generator = {
-    key: keygen,
-    bins: recgen,
-    metadata: metagen,
-    policy: function () { return policy }
-  }
+
   if (callback) {
-    createRecords(helper.client, generator, n, 200, callback)
+    createRecords(putCall, generator, n, 200, callback)
   } else {
     return new Promise((resolve, reject) => {
       const records = []
-      createRecords(helper.client, generator, n, 200, record => {
+      createRecords(putCall, generator, n, 200, record => {
         if (record) {
@@ -82,0 +86,0 @@ records.push(record)

test/predexp.js

@@ -113,6 +113,8 @@ // *****************************************************************************
       const entries = samples.entries()
-      const rgen = () => entries.next().value[1]
-      const kgen = keygen.string(helper.namespace, testSet, { prefix: 'test/predexp/query', random: false })
-      const mgen = metagen.constant({ ttl: 300 })
-      return putgen.put(samples.length, kgen, rgen, mgen)
+      const generators = {
+        keygen: keygen.string(helper.namespace, testSet, { prefix: 'test/predexp/query', random: false }),
+        recgen: () => entries.next().value[1],
+        metagen: metagen.constant({ ttl: 300 })
+      }
+      return putgen.put(samples.length, generators)
     })
@@ -119,0 +121,0 @@

test/query.js

@@ -115,7 +115,9 @@ // *****************************************************************************
   before(() => {
-    const sampleGen = () => samples.pop()
-    const kgen = keygen.string(helper.namespace, testSet, { prefix: 'test/query/', random: false })
-    const mgen = metagen.constant({ ttl: 300 })
+    const generators = {
+      keygen: keygen.string(helper.namespace, testSet, { prefix: 'test/query/', random: false }),
+      recgen: () => samples.pop(),
+      metagen: metagen.constant({ ttl: 300 })
+    }
     return Promise.all([
-      putgen.put(numberOfSamples, kgen, sampleGen, mgen)
+      putgen.put(numberOfSamples, generators)
         .then((records) => { keys = records.map((rec) => rec.key) })
@@ -122,0 +124,0 @@ .then(() => Promise.all(indexes.map(idx =>

test/scan.js

@@ -43,11 +43,13 @@ // *****************************************************************************
     .then(() => {
-      const kgen = keygen.string(helper.namespace, testSet, { prefix: 'test/scan/', random: false })
-      const rgen = recgen.record({ i: valgen.integer(), s: valgen.string() })
-      const mgen = metagen.constant({ ttl: 300 })
-      const policy = new Aerospike.WritePolicy({
-        totalTimeout: 1000,
-        key: Aerospike.policy.key.SEND,
-        exists: Aerospike.policy.exists.CREATE_OR_REPLACE
-      })
-      return putgen.put(numberOfRecords, kgen, rgen, mgen, policy)
+      const config = {
+        keygen: keygen.string(helper.namespace, testSet, { prefix: 'test/scan/', random: false }),
+        recgen: recgen.record({ i: valgen.integer(), s: valgen.string() }),
+        metagen: metagen.constant({ ttl: 300 }),
+        policy: new Aerospike.WritePolicy({
+          totalTimeout: 1000,
+          key: Aerospike.policy.key.SEND,
+          exists: Aerospike.policy.exists.CREATE_OR_REPLACE
+        })
+      }
+      return putgen.put(numberOfRecords, config)
         .then((records) => { keys = records.map((rec) => rec.key) })
@@ -54,0 +56,0 @@ }))

test/stress/perfdata.js

 // *****************************************************************************
-// Copyright 2013-2019 Aerospike, Inc.
+// Copyright 2013-2020 Aerospike, Inc.
 //
@@ -19,2 +19,4 @@ // Licensed under the Apache License, Version 2.0 (the "License")
 
+const { format } = require('util')
+
 const helper = require('../test_helper')
@@ -45,18 +47,24 @@
 function generate (ns, set, numberOfRecords, recordSize, done) {
-  var numBinsPerRecord = recordSize[0]
-  var sizePerBin = recordSize[1]
-  var kgen = keygen.string(ns, set, { length: { min: 20, max: 20 } })
-  var bins = { id: valgen.integer({ random: false, min: 0 }) }
-  for (var i = 0; i < numBinsPerRecord; i++) {
+  const numBinsPerRecord = recordSize[0]
+  const sizePerBin = recordSize[1]
+  const bins = { id: valgen.integer({ random: false, min: 0 }) }
+  for (let i = 0; i < numBinsPerRecord; i++) {
     bins['b' + i] = valgen.bytes({ length: { min: sizePerBin, max: sizePerBin } })
   }
-  var rgen = recgen.record(bins)
-  var mgen = metagen.constant({})
-  var keysCreated = 0
-  var uniqueKeys = new Set()
-  var timer = interval(10 * 1000, function (ms) {
-    var throughput = Math.round(1000 * keysCreated / ms)
-    console.info('%s ms: %d records created (%d records / second)', ms, keysCreated, throughput)
+  const generators = {
+    keygen: keygen.string(ns, set, { length: { min: 20, max: 20 } }),
+    recgen: recgen.record(bins),
+    metagen: metagen.constant({}),
+    throttle: {
+      limit: 5000,
+      interval: 1000
+    }
+  }
+  let keysCreated = 0
+  const uniqueKeys = new Set()
+  const timer = interval(10 * 1000, function (ms) {
+    const throughput = Math.round(1000 * keysCreated / ms)
+    console.info('%s ms: %d records created (%d records / second) - %s', ms, keysCreated, throughput, memoryUsage())
   })
-  putgen.put(numberOfRecords, kgen, rgen, mgen, function (key) {
+  putgen.put(numberOfRecords, generators, function (key) {
     if (key) {
@@ -73,5 +81,15 @@ keysCreated++
 
+const MEGA = 1024 * 1024 // bytes in a MB
+function memoryUsage () {
+  const memUsage = process.memoryUsage()
+  const rss = Math.round(memUsage.rss / MEGA)
+  const heapUsed = Math.round(memUsage.heapUsed / MEGA)
+  const heapTotal = Math.round(memUsage.heapTotal / MEGA)
+  return format('mem: %d MB, heap: %d / %d MB', rss, heapUsed, heapTotal)
+}
+
 module.exports = {
   interval,
-  generate
+  generate,
+  memoryUsage
 }

test/stress/query.js

 // *****************************************************************************
-// Copyright 2013-2019 Aerospike, Inc.
+// Copyright 2013-2020 Aerospike, Inc.
 //
@@ -27,4 +27,2 @@ // Licensed under the Apache License, Version 2.0 (the "License")
 
-const mega = 1024 * 1024 // bytes in a MB
-
 describe('client.query()', function () {
@@ -46,9 +44,5 @@ this.enableTimeouts(false)
     var timer = perfdata.interval(10000, function (ms) {
-      var throughput = Math.round(1000 * received / ms)
-      var memUsage = process.memoryUsage()
-      var rss = Math.round(memUsage.rss / mega)
-      var heapUsed = Math.round(memUsage.heapUsed / mega)
-      var heapTotal = Math.round(memUsage.heapTotal / mega)
-      console.log('%d ms: %d records received (%d rps; mem: %d MB, heap: %d / %d MB)',
-        ms, received, throughput, rss, heapUsed, heapTotal)
+      const throughput = Math.round(1000 * received / ms)
+      console.log('%d ms: %d records received (%d rps; %s)',
+        ms, received, throughput, perfdata.memoryUsage())
     })
@@ -100,4 +94,4 @@
         // perf test data already exists
-        numberOfRecords = record.norec
-        testSet = record.set
+        numberOfRecords = record.bins.norec
+        testSet = record.bins.set
         console.info('using performance test data from set %s (%d records)', testSet, numberOfRecords)
@@ -104,0 +98,0 @@ done()

test/stress/scan.js

 // *****************************************************************************
-// Copyright 2013-2019 Aerospike, Inc.
+// Copyright 2013-2020 Aerospike, Inc.
 //
@@ -48,9 +48,5 @@ // Licensed under the Apache License, Version 2.0 (the "License")
     var timer = perfdata.interval(reportingInterval, function (ms) {
-      var throughput = Math.round(1000 * received / ms)
-      var memUsage = process.memoryUsage()
-      var rss = Math.round(memUsage.rss / mega)
-      var heapUsed = Math.round(memUsage.heapUsed / mega)
-      var heapTotal = Math.round(memUsage.heapTotal / mega)
-      console.log('%d ms: %d records received (%d rps; mem: %d MB, heap: %d / %d MB)',
-        ms, received, throughput, rss, heapUsed, heapTotal)
+      const throughput = Math.round(1000 * received / ms)
+      console.log('%d ms: %d records received (%d rps; %s)',
+        ms, received, throughput, perfdata.memUsage())
     })
@@ -57,0 +53,0 @@

test/truncate.js

@@ -37,3 +37,3 @@ // *****************************************************************************
 
-  var client = helper.client
+  const client = helper.client
 
@@ -43,7 +43,9 @@ // Generates a number of records; the callback function is called with a list
   function genRecords (kgen, noRecords, done) {
-    var mgen = metagen.constant({ ttl: 300 })
-    var rgen = recgen.constant({ a: 'foo', b: 'bar' })
-    putgen.put(noRecords, kgen, rgen, mgen)
+    const generators = {
+      keygen: kgen,
+      recgen: recgen.constant({ a: 'foo', b: 'bar' }),
+      metagen: metagen.constant({ ttl: 300 })
+    }
+    putgen.put(noRecords, generators)
       .then(done)
-      .catch(err => { throw err })
   }
@@ -50,0 +52,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

1493176

Lines of code

25908

increased by0.14%

Worsened metrics

Dev dependency count

15

increased by7.14%

No dependency changes