rate-limiter-flexible
Advanced tools
Comparing version 0.15.5 to 0.16.0
@@ -6,4 +6,7 @@ const RateLimiterStoreAbstract = require('./RateLimiterStoreAbstract'); | ||
/** | ||
* @callback callback | ||
* @param {Object} err | ||
* | ||
* @param {Object} opts | ||
* @param {callback} cb | ||
* Defaults { | ||
@@ -13,25 +16,43 @@ * ... see other in RateLimiterStoreAbstract | ||
* storeClient: anySqlClient, | ||
* storeType: 'knex', // required only for Knex instance | ||
* dbName: 'string', | ||
* tableName: 'string', | ||
* } | ||
*/ | ||
constructor(opts) { | ||
constructor(opts, cb = null) { | ||
super(opts); | ||
this.client = opts.storeClient; | ||
this.clientType = opts.storeType; | ||
this.dbName = opts.dbName; | ||
this.tableName = opts.tableName; | ||
this.clearExpiredByTimeout = opts.clearExpiredByTimeout; | ||
this._tableCreated = false; | ||
this.client.query(`CREATE DATABASE IF NOT EXISTS ${this.dbName};`, (errDb) => { | ||
if (errDb) { | ||
throw errDb; | ||
} else { | ||
this.client.query(this._getCreateTableStmt(), (err) => { | ||
if (err) { | ||
throw err; | ||
} else { | ||
this._tableCreated = true; | ||
this._clearExpiredHourAgo(); | ||
} | ||
}); | ||
} | ||
this._createDbAndTable() | ||
.then(() => { | ||
this._tableCreated = true; | ||
if (this.clearExpiredByTimeout) { | ||
this._clearExpiredHourAgo(); | ||
} | ||
if (typeof cb === 'function') { | ||
cb(); | ||
} | ||
}) | ||
.catch((err) => { | ||
if (typeof cb === 'function') { | ||
cb(err); | ||
} else { | ||
throw err; | ||
} | ||
}); | ||
} | ||
clearExpired(expire) { | ||
return new Promise((resolve) => { | ||
this.client.query(`DELETE FROM ${this.tableName} WHERE expire < ?`, [expire], () => { | ||
resolve(); | ||
}); | ||
}); | ||
@@ -41,7 +62,10 @@ } | ||
_clearExpiredHourAgo() { | ||
if (this._clearExpiredTimeoutId) { | ||
clearTimeout(this._clearExpiredTimeoutId); | ||
} | ||
this._clearExpiredTimeoutId = setTimeout(() => { | ||
const expire = Date.now() - 3600000; | ||
this.client.query(`DELETE FROM ${this.tableName} WHERE expire < ?`, [expire], () => { | ||
this._clearExpiredHourAgo(); | ||
}); | ||
this.clearExpired(Date.now() - 3600000) | ||
.then(() => { | ||
this._clearExpiredHourAgo(); | ||
}); | ||
}, 300000); | ||
@@ -51,2 +75,71 @@ this._clearExpiredTimeoutId.unref(); | ||
/** | ||
* | ||
* @return Promise<any> | ||
* @private | ||
*/ | ||
_getConnection() { | ||
switch (this.clientType) { | ||
case 'pool': | ||
return new Promise((resolve, reject) => { | ||
this.client.getConnection((errConn, conn) => { | ||
if (errConn) { | ||
return reject(errConn); | ||
} | ||
resolve(conn); | ||
}); | ||
}); | ||
case 'sequelize': | ||
return this.client.connectionManager.getConnection(); | ||
case 'knex': | ||
return this.client.client.acquireConnection(); | ||
default: | ||
return Promise.resolve(this.client); | ||
} | ||
} | ||
_releaseConnection(conn) { | ||
switch (this.clientType) { | ||
case 'pool': | ||
return conn.release(); | ||
case 'sequelize': | ||
return this.client.connectionManager.releaseConnection(conn); | ||
case 'knex': | ||
return this.client.client.releaseConnection(conn); | ||
default: | ||
return true; | ||
} | ||
} | ||
/** | ||
* | ||
* @returns {Promise<any>} | ||
* @private | ||
*/ | ||
_createDbAndTable() { | ||
return new Promise((resolve, reject) => { | ||
this._getConnection() | ||
.then((conn) => { | ||
conn.query(`CREATE DATABASE IF NOT EXISTS ${this.dbName};`, (errDb) => { | ||
if (errDb) { | ||
this._releaseConnection(conn); | ||
return reject(errDb); | ||
} | ||
conn.query(this._getCreateTableStmt(), (err) => { | ||
if (err) { | ||
this._releaseConnection(conn); | ||
return reject(err); | ||
} | ||
this._releaseConnection(conn); | ||
resolve(); | ||
}); | ||
}); | ||
}) | ||
.catch((err) => { | ||
reject(err); | ||
}); | ||
}); | ||
} | ||
_getCreateTableStmt() { | ||
@@ -61,2 +154,21 @@ return `CREATE TABLE IF NOT EXISTS ${this.tableName} (` + | ||
get clientType() { | ||
return this._clientType; | ||
} | ||
set clientType(value) { | ||
if (typeof value === 'undefined') { | ||
if (this.client.constructor.name === 'Connection') { | ||
value = 'connection'; | ||
} else if (this.client.constructor.name === 'Pool') { | ||
value = 'pool'; | ||
} else if (this.client.constructor.name === 'Sequelize') { | ||
value = 'sequelize'; | ||
} else { | ||
throw new Error('clientType is not defined'); | ||
} | ||
} | ||
this._clientType = value.toLowerCase(); | ||
} | ||
get dbName() { | ||
@@ -78,2 +190,10 @@ return this._dbName; | ||
get clearExpiredByTimeout() { | ||
return this._clearExpiredByTimeout; | ||
} | ||
set clearExpiredByTimeout(value) { | ||
this._clearExpiredByTimeout = typeof value === 'undefined' ? true : Boolean(value); | ||
} | ||
_getRateLimiterRes(rlKey, changedPoints, result) { | ||
@@ -92,76 +212,60 @@ const res = new RateLimiterRes(); | ||
_upsertTransaction(isPool, conn, resolve, reject, key, points, msDuration, forceExpire) { | ||
conn.query('BEGIN', (errBegin) => { | ||
if (errBegin) { | ||
conn.rollback(() => { | ||
if (isPool) { | ||
conn.release(); | ||
} | ||
}); | ||
_upsertTransaction(conn, key, points, msDuration, forceExpire) { | ||
return new Promise((resolve, reject) => { | ||
conn.query('BEGIN', (errBegin) => { | ||
if (errBegin) { | ||
conn.rollback(); | ||
return reject(errBegin); | ||
} | ||
return reject(errBegin); | ||
} | ||
const dateNow = Date.now(); | ||
const newExpire = dateNow + msDuration; | ||
const dateNow = Date.now(); | ||
const newExpire = dateNow + msDuration; | ||
let q; | ||
let values; | ||
if (forceExpire) { | ||
q = `INSERT INTO ?? VALUES (?, ?, ?) | ||
let q; | ||
let values; | ||
if (forceExpire) { | ||
q = `INSERT INTO ?? VALUES (?, ?, ?) | ||
ON DUPLICATE KEY UPDATE | ||
points = ?, | ||
expire = ?;`; | ||
values = [ | ||
this.tableName, key, points, newExpire, | ||
points, | ||
newExpire, | ||
]; | ||
} else { | ||
q = `INSERT INTO ?? VALUES (?, ?, ?) | ||
values = [ | ||
this.tableName, key, points, newExpire, | ||
points, | ||
newExpire, | ||
]; | ||
} else { | ||
q = `INSERT INTO ?? VALUES (?, ?, ?) | ||
ON DUPLICATE KEY UPDATE | ||
points = IF(expire <= ?, ?, points + (?)), | ||
expire = IF(expire <= ?, ?, expire);`; | ||
values = [ | ||
this.tableName, key, points, newExpire, | ||
dateNow, points, points, | ||
dateNow, newExpire, | ||
]; | ||
} | ||
values = [ | ||
this.tableName, key, points, newExpire, | ||
dateNow, points, points, | ||
dateNow, newExpire, | ||
]; | ||
} | ||
conn.query(q, values, (errUpsert) => { | ||
if (errUpsert) { | ||
conn.rollback(() => { | ||
if (isPool) { | ||
conn.release(); | ||
conn.query(q, values, (errUpsert) => { | ||
if (errUpsert) { | ||
conn.rollback(); | ||
return reject(errUpsert); | ||
} | ||
conn.query('SELECT points, expire FROM ?? WHERE `key` = ?;', [this.tableName, key], (errSelect, res) => { | ||
if (errSelect) { | ||
conn.rollback(); | ||
return reject(errSelect); | ||
} | ||
}); | ||
return reject(errUpsert); | ||
} | ||
conn.query('SELECT points, expire FROM ?? WHERE `key` = ?;', [this.tableName, key], (errSelect, res) => { | ||
if (errSelect) { | ||
conn.rollback(() => { | ||
if (isPool) { | ||
conn.release(); | ||
conn.query('COMMIT', (err) => { | ||
if (err) { | ||
conn.rollback(); | ||
return reject(err); | ||
} | ||
resolve(res); | ||
}); | ||
return reject(errSelect); | ||
} | ||
conn.query('COMMIT', (err) => { | ||
if (err) { | ||
conn.rollback(() => { | ||
if (isPool) { | ||
conn.release(); | ||
} | ||
}); | ||
return reject(err); | ||
} | ||
if (isPool) { | ||
conn.release(); | ||
} | ||
resolve(res); | ||
}); | ||
@@ -179,14 +283,18 @@ }); | ||
return new Promise((resolve, reject) => { | ||
// Pool support | ||
if (typeof this.client.getConnection === 'function') { | ||
this.client.getConnection((errConn, conn) => { | ||
if (errConn) { | ||
return reject(errConn); | ||
} | ||
this._upsertTransaction(true, conn, resolve, reject, key, points, msDuration, forceExpire); | ||
this._getConnection() | ||
.then((conn) => { | ||
this._upsertTransaction(conn, key, points, msDuration, forceExpire) | ||
.then((res) => { | ||
resolve(res); | ||
}) | ||
.catch((err) => { | ||
reject(err); | ||
}) | ||
.finally(() => { | ||
this._releaseConnection(conn); | ||
}); | ||
}) | ||
.catch((err) => { | ||
reject(err); | ||
}); | ||
} else { | ||
this._upsertTransaction(false, this.client, resolve, reject, key, points, msDuration, forceExpire); | ||
} | ||
}); | ||
@@ -200,18 +308,24 @@ } | ||
const q = 'SELECT points, expire FROM ?? WHERE `key` = ? AND `expire` > ?'; | ||
return new Promise((resolve, reject) => { | ||
this._getConnection() | ||
.then((conn) => { | ||
conn.query( | ||
'SELECT points, expire FROM ?? WHERE `key` = ? AND `expire` > ?', | ||
[this.tableName, rlKey, Date.now()], | ||
(err, res) => { | ||
if (err) { | ||
reject(err); | ||
} else if (res.length === 0) { | ||
resolve(null); | ||
} else { | ||
resolve(res); | ||
} | ||
return new Promise((resolve, reject) => { | ||
this.client.query( | ||
q, | ||
[this.tableName, rlKey, Date.now()], | ||
(err, res) => { | ||
if (err) { | ||
reject(err); | ||
} else if (res.length === 0) { | ||
resolve(null); | ||
} else { | ||
resolve(res); | ||
} | ||
} // eslint-disable-line | ||
); | ||
this._releaseConnection(conn); | ||
} // eslint-disable-line | ||
); | ||
}) | ||
.catch((err) => { | ||
reject(err); | ||
}); | ||
}); | ||
@@ -218,0 +332,0 @@ } |
@@ -6,4 +6,7 @@ const RateLimiterStoreAbstract = require('./RateLimiterStoreAbstract'); | ||
/** | ||
* @callback callback | ||
* @param {Object} err | ||
* | ||
* @param {Object} opts | ||
* @param {callback} cb | ||
* Defaults { | ||
@@ -13,24 +16,32 @@ * ... see other in RateLimiterStoreAbstract | ||
* storeClient: postgresClient, | ||
* storeType: 'knex', // required only for Knex instance | ||
* tableName: 'string', | ||
* } | ||
*/ | ||
constructor(opts) { | ||
constructor(opts, cb = null) { | ||
super(opts); | ||
this.client = opts.storeClient; | ||
this.clientType = opts.storeType; | ||
this.tableName = opts.tableName; | ||
this.clearExpiredByTimeout = opts.clearExpiredByTimeout; | ||
this._tableCreated = false; | ||
this.client.query(`${this._getCreateTableStmt()}`) | ||
this._createTable() | ||
.then(() => { | ||
this._tableCreated = true; | ||
this._clearExpiredHourAgo(); | ||
if (this.clearExpiredByTimeout) { | ||
this._clearExpiredHourAgo(); | ||
} | ||
if (typeof cb === 'function') { | ||
cb(); | ||
} | ||
}) | ||
.catch((err) => { | ||
if (err.code === '23505') { | ||
// Error: duplicate key value violates unique constraint "pg_type_typname_nsp_index" | ||
// Postgres doesn't handle concurrent table creation | ||
// It is supposed, that table is created by another worker | ||
this._tableCreated = true; | ||
this._clearExpiredHourAgo(); | ||
if (typeof cb === 'function') { | ||
cb(err); | ||
} else { | ||
throw err; | ||
} | ||
@@ -40,2 +51,19 @@ }); | ||
clearExpired(expire) { | ||
return new Promise((resolve) => { | ||
const q = { | ||
name: 'rlflx-clear-expired', | ||
text: `DELETE FROM ${this.tableName} WHERE expire < $1`, | ||
values: [expire], | ||
}; | ||
this._query(q) | ||
.catch(() => { | ||
// Deleting expired query is not critical | ||
}) | ||
.finally(() => { | ||
resolve(); | ||
}); | ||
}); | ||
} | ||
/** | ||
@@ -47,16 +75,9 @@ * Delete all rows expired 1 hour ago once per 5 minutes | ||
_clearExpiredHourAgo() { | ||
if (this._clearExpiredTimeoutId) { | ||
clearTimeout(this._clearExpiredTimeoutId); | ||
} | ||
this._clearExpiredTimeoutId = setTimeout(() => { | ||
const expire = Date.now() - 3600000; | ||
const q = { | ||
name: 'rlflx-clear-expired', | ||
text: `DELETE FROM ${this.tableName} WHERE expire < $1`, | ||
values: [expire], | ||
}; | ||
this.client.query(q) | ||
this.clearExpired(Date.now() - 3600000) | ||
.then(() => { | ||
this._clearExpiredHourAgo(); | ||
}) | ||
.catch(() => { | ||
// Deleting expired query is not critical | ||
this._clearExpiredHourAgo(); | ||
}); | ||
@@ -67,2 +88,57 @@ }, 300000); | ||
/** | ||
* | ||
* @return Promise<any> | ||
* @private | ||
*/ | ||
_getConnection() { | ||
switch (this.clientType) { | ||
case 'pool': | ||
return Promise.resolve(this.client); | ||
case 'sequelize': | ||
return this.client.connectionManager.getConnection(); | ||
case 'knex': | ||
return this.client.client.acquireConnection(); | ||
default: | ||
return Promise.resolve(this.client); | ||
} | ||
} | ||
_releaseConnection(conn) { | ||
switch (this.clientType) { | ||
case 'pool': | ||
return true; | ||
case 'sequelize': | ||
return this.client.connectionManager.releaseConnection(conn); | ||
case 'knex': | ||
return this.client.client.releaseConnection(conn); | ||
default: | ||
return true; | ||
} | ||
} | ||
/** | ||
* | ||
* @returns {Promise<any>} | ||
* @private | ||
*/ | ||
_createTable() { | ||
return new Promise((resolve, reject) => { | ||
this._query(this._getCreateTableStmt()) | ||
.then(() => { | ||
resolve(); | ||
}) | ||
.catch((err) => { | ||
if (err.code === '23505') { | ||
// Error: duplicate key value violates unique constraint "pg_type_typname_nsp_index" | ||
// Postgres doesn't handle concurrent table creation | ||
// It is supposed, that table is created by another worker | ||
resolve(); | ||
} else { | ||
reject(err); | ||
} | ||
}); | ||
}); | ||
} | ||
_getCreateTableStmt() { | ||
@@ -76,2 +152,21 @@ return `CREATE TABLE IF NOT EXISTS ${this.tableName} ( | ||
get clientType() { | ||
return this._clientType; | ||
} | ||
set clientType(value) { | ||
if (typeof value === 'undefined') { | ||
if (this.client.constructor.name === 'Client') { | ||
value = 'client'; | ||
} else if (this.client.constructor.name === 'Pool') { | ||
value = 'pool'; | ||
} else if (this.client.constructor.name === 'Sequelize') { | ||
value = 'sequelize'; | ||
} else { | ||
throw new Error('clientType is not defined'); | ||
} | ||
} | ||
this._clientType = value.toLowerCase(); | ||
} | ||
get tableName() { | ||
@@ -85,2 +180,10 @@ return this._tableName; | ||
get clearExpiredByTimeout() { | ||
return this._clearExpiredByTimeout; | ||
} | ||
set clearExpiredByTimeout(value) { | ||
this._clearExpiredByTimeout = typeof value === 'undefined' ? true : Boolean(value); | ||
} | ||
_getRateLimiterRes(rlKey, changedPoints, result) { | ||
@@ -99,2 +202,23 @@ const res = new RateLimiterRes(); | ||
_query(q) { | ||
return new Promise((resolve, reject) => { | ||
this._getConnection() | ||
.then((conn) => { | ||
conn.query(q) | ||
.then((res) => { | ||
resolve(res); | ||
}) | ||
.catch((err) => { | ||
reject(err); | ||
}) | ||
.finally(() => { | ||
this._releaseConnection(conn); | ||
}); | ||
}) | ||
.catch((err) => { | ||
reject(err); | ||
}); | ||
}); | ||
} | ||
_upsert(key, points, msDuration, forceExpire = false) { | ||
@@ -112,3 +236,4 @@ if (!this._tableCreated) { | ||
END `; | ||
const q = { | ||
return this._query({ | ||
name: forceExpire ? 'rlflx-upsert-force' : 'rlflx-upsert', | ||
@@ -125,5 +250,3 @@ text: ` | ||
values: [key, points, newExpire, Date.now()], | ||
}; | ||
return this.client.query(q); | ||
}); | ||
} | ||
@@ -137,3 +260,3 @@ | ||
return new Promise((resolve, reject) => { | ||
const q = { | ||
this._query({ | ||
name: 'rlflx-get', | ||
@@ -143,5 +266,3 @@ text: ` | ||
values: [rlKey, Date.now()], | ||
}; | ||
this.client.query(q) | ||
}) | ||
.then((res) => { | ||
@@ -148,0 +269,0 @@ if (res.rowCount === 0) { |
35
MYSQL.md
## RateLimiterMySQL | ||
It supports `mysql2` and `mysql` single connection and pool. | ||
It supports `mysql2`, `mysql`, `sequilize` and `knex`. | ||
@@ -19,2 +19,6 @@ **Note**: It takes 50-150 ms per request on more than 1000 concurrent requests per second | ||
It is recommended to provide `ready` callback as the second option of ` new RateLimiterMySQL(opts, ready)` | ||
to react on errors during creating database or table(s) for rate limiters. See example below. | ||
`ready` callback can be omitted, if process is exit on unhandled errors. | ||
### Usage | ||
@@ -41,3 +45,12 @@ | ||
const rateLimiter = new RateLimiterMySQL(opts); | ||
const ready = (err) => { | ||
if (err) { | ||
// log or/and process exit | ||
} else { | ||
// db and table checked/created | ||
} | ||
}; | ||
// if second parameter is not a function or not provided, it may throw unhandled error on creation db or table | ||
const rateLimiter = new RateLimiterMySQL(opts, ready); | ||
rateLimiter.consume(key) | ||
@@ -52,2 +65,20 @@ .then((rateLimiterRes) => { | ||
#### Sequelize and Knex support | ||
It gets internal connection from Sequelize or Knex to make raw queries. | ||
Connection is released after any query or transaction, so workflow is clean. | ||
```javascript | ||
const rateLimiter = new RateLimiterMySQL({ | ||
storeClient: sequelizeInstance, | ||
}, ready); | ||
const rateLimiter = new RateLimiterMySQL({ | ||
storeClient: knexInstance, | ||
storeType: `knex`, // knex requires this option | ||
}, ready); | ||
``` | ||
[See detailed options description here](https://github.com/animir/node-rate-limiter-flexible#options) | ||
### Benchmark | ||
@@ -54,0 +85,0 @@ |
{ | ||
"name": "rate-limiter-flexible", | ||
"version": "0.15.5", | ||
"version": "0.16.0", | ||
"description": "Flexible API rate limiter backed by Redis for distributed node.js applications", | ||
@@ -5,0 +5,0 @@ "main": "index.js", |
@@ -19,3 +19,5 @@ ## RateLimiterPostgres | ||
[See detailed options description here](https://github.com/animir/node-rate-limiter-flexible#options) | ||
It is recommended to provide `ready` callback as the second option of ` new RateLimiterMySQL(opts, ready)` | ||
to react on errors during creating table(s) for rate limiters. See example below. | ||
`ready` callback can be omitted, if process is exit on unhandled errors. | ||
@@ -57,4 +59,14 @@ ```javascript | ||
const rateLimiter = new RateLimiterPostgres(opts); | ||
const ready = (err) => { | ||
if (err) { | ||
// log or/and process exit | ||
} else { | ||
// table checked/created | ||
} | ||
}; | ||
// if second parameter is not a function or not provided, it may throw unhandled error on creation db or table | ||
const rateLimiter = new RateLimiterPostgres(opts, ready); | ||
rateLimiter.consume(userIdOrIp) | ||
@@ -81,45 +93,20 @@ .then((rateLimiterRes) => { | ||
### Knex option | ||
#### Sequelize and Knex support | ||
It gets internal connection from Sequelize or Knex to make raw queries. | ||
Connection is released after any query or transaction, so workflow is clean. | ||
```javascript | ||
knex.client.acquireRawConnection() | ||
.then((connection) => { | ||
const opts = { | ||
storeClient: connection, | ||
tableName: 'mytable', | ||
points: 5, // Number of points | ||
duration: 1, // Per second(s) | ||
}; | ||
const rateLimiter = new RateLimiterPostgres({ | ||
storeClient: sequelizeInstance, | ||
}, ready); | ||
const rateLimiter = new RateLimiterPostgres(opts); | ||
/* ... */ | ||
}) | ||
.catch((err) => { | ||
}); | ||
const rateLimiter = new RateLimiterPostgres({ | ||
storeClient: knexInstance, | ||
storeType: `knex`, // knex requires this option | ||
}, ready); | ||
``` | ||
### Sequelize option | ||
[See detailed options description here](https://github.com/animir/node-rate-limiter-flexible#options) | ||
```javascript | ||
// sequelize is connected postgres instance | ||
sequelize.connectionManager.getConnection() | ||
.then((connection) => { | ||
const opts = { | ||
storeClient: connection, | ||
tableName: 'mytable', | ||
points: 5, // Number of points | ||
duration: 1, // Per second(s) | ||
}; | ||
const rateLimiter = new RateLimiterPostgres(opts); | ||
/* ... */ | ||
}) | ||
.catch((err) => { | ||
}); | ||
``` | ||
### Benchmark | ||
@@ -126,0 +113,0 @@ |
@@ -48,5 +48,5 @@ [![Build Status](https://travis-ci.org/animir/node-rate-limiter-flexible.png)](https://travis-ci.org/animir/node-rate-limiter-flexible) | ||
Average latency during test pure NodeJS endpoint in cluster of 4 workers with everything set up on one server by | ||
Average latency during test pure NodeJS endpoint in cluster of 4 workers with everything set up on one server. | ||
1000 concurrent clients with maximum 2000 requests per sec during 30 seconds. | ||
1000 concurrent clients with maximum 2000 requests per sec during 30 seconds. | ||
@@ -117,2 +117,4 @@ ```text | ||
* `storeType` `Default: storeClient.constructor.name` It is required only for Knex and have to be set to 'knex' | ||
#### Options specific to MySQL | ||
@@ -119,0 +121,0 @@ |
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
102388
28
1683
491