Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

logzio-nodejs

Package Overview
Dependencies
Maintainers
4
Versions
52
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

logzio-nodejs - npm Package Compare versions

Comparing version 0.4.17 to 0.4.18

test.js
test/test-logger.js
test/test-udp.js

11

.eslintrc.js
module.exports = {
"env": {
"node": true,
"es6": true,
},
"extends": "airbnb-base",
"extends": "eslint:recommended",
"parserOptions": {
"ecmaVersion": 5
},
"rules": {

@@ -23,7 +25,4 @@ "indent": [

"always"
],
"no-param-reassign": 1,
"no-underscore-dangle": 0,
"linebreak-style": 0
]
}
};

479

lib/logzio-nodejs.js

@@ -1,319 +0,280 @@

const request = require('request-promise');
const stringifySafe = require('json-stringify-safe');
const assign = require('lodash.assign');
const dgram = require('dgram');
const zlib = require('zlib');
var request = require('request');
var stringifySafe = require('json-stringify-safe');
var _assign = require('lodash.assign');
var dgram = require('dgram');
exports.version = require('../package.json').version;
const jsonToString = (json) => {
try {
return JSON.stringify(json);
} catch (ex) {
return stringifySafe(json, null, null, () => { });
var LogzioLogger = function (options) {
if (!options || !options.token) {
throw new Error('You are required to supply a token for logging.');
}
};
const messagesToBody = messages => messages.reduce((body, msg) => `${body}${jsonToString(msg)}\n`, '');
this.token = options.token;
this.host = options.host || 'listener.logz.io';
this.userAgent = 'Logzio-Logger NodeJS';
this.type = options.type || 'nodejs';
this.sendIntervalMs = options.sendIntervalMs || 10 * 1000;
this.bufferSize = options.bufferSize || 100;
this.debug = options.debug || false;
this.numberOfRetries = options.numberOfRetries || 3;
this.timer = null;
this.closed = false;
this.supressErrors = options.supressErrors || false;
this.addTimestampWithNanoSecs = options.addTimestampWithNanoSecs || false;
const UNAVAILABLE_CODES = ['ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ECONNABORTED'];
var protocolToPortMap = {
'udp': 5050,
'http': 8070,
'https': 8071
};
this.protocol = options.protocol || 'http';
if (!protocolToPortMap.hasOwnProperty(this.protocol)) {
throw new Error('Invalid protocol defined. Valid options are : ' + JSON.stringify(Object.keys(protocolToPortMap)));
}
this.port = options.port || protocolToPortMap[this.protocol];
const zlibPromised = body => new Promise(((resolve, reject) => {
zlib.gzip(body, (err, res) => {
if (err) return reject(err);
return resolve(res);
});
}));
if (this.protocol === 'udp') {
this.udpClient = dgram.createSocket('udp4');
}
const protocolToPortMap = {
udp: 5050,
http: 8070,
https: 8071,
};
/*
Callback method executed on each bulk of messages sent to logzio.
If the bulk failed, it will be called: callback(exception), otherwise upon
success it will called as callback()
*/
this.callback = options.callback || this._defaultCallback;
const USER_AGENT = 'Logzio-Logger NodeJS';
/*
* the read/write/connection timeout in milliseconds of the outgoing HTTP request
*/
this.timeout = options.timeout;
class LogzioLogger {
constructor({
token,
host = 'listener.logz.io',
type = 'nodejs',
sendIntervalMs = 10 * 1000,
bufferSize = 100,
debug = false,
numberOfRetries = 3,
supressErrors = false,
addTimestampWithNanoSecs = false,
compress = false,
internalLogger = console,
protocol = 'http',
port,
timeout,
sleepUntilNextRetry = 2 * 1000,
callback = this._defaultCallback,
extraFields = {},
}) {
if (!token) {
throw new Error('You are required to supply a token for logging.');
}
// build the url for logging
this.url = this.protocol + '://' + this.host + ':' + this.port + '?token=' + this.token;
this.token = token;
this.host = host;
this.type = type;
this.sendIntervalMs = sendIntervalMs;
this.bufferSize = bufferSize;
this.debug = debug;
this.numberOfRetries = numberOfRetries;
this.supressErrors = supressErrors;
this.addTimestampWithNanoSecs = addTimestampWithNanoSecs;
this.compress = compress;
this.internalLogger = internalLogger;
this.sleepUntilNextRetry = sleepUntilNextRetry;
this.messages = [];
this.bulkId = 1;
this.extraFields = options.extraFields || {};
};
this.timer = null;
this.closed = false;
exports.createLogger = function (options) {
var l = new LogzioLogger(options);
l._timerSend();
return l;
};
this.protocol = protocol;
this._setProtocol(port);
var jsonToString = exports.jsonToString = function(json) {
try {
return JSON.stringify(json);
}
catch(ex) {
return stringifySafe(json, null, null, function() { });
}
};
/*
Callback method executed on each bulk of messages sent to logzio.
If the bulk failed, it will be called: callback(exception), otherwise upon
success it will called as callback()
*/
this.callback = callback;
LogzioLogger.prototype._defaultCallback = function(err) {
if (err && !this.supressErrors) {
console.error('logzio-logger error: ' + err, err);
}
};
/*
* the read/write/connection timeout in milliseconds of the outgoing HTTP request
*/
this.timeout = timeout;
LogzioLogger.prototype.sendAndClose = function(callback){
this.callback = callback || this._defaultCallback;
this._debug('Sending last messages and closing...');
this._popMsgsAndSend();
clearTimeout(this.timer);
// build the url for logging
this.url = `${this.protocol}://${this.host}:${this.port}?token=${this.token}`;
if (this.protocol === 'udp') {
this.udpClient.close();
}
};
this.messages = [];
this.bulkId = 1;
this.extraFields = extraFields;
LogzioLogger.prototype._timerSend = function() {
if (this.messages.length > 0) {
this._debug('Woke up and saw ' + this.messages.length + ' messages to send. Sending now...');
this._popMsgsAndSend();
}
_setProtocol(port) {
if (!protocolToPortMap[this.protocol]) {
throw new Error(`Invalid protocol defined. Valid options are : ${JSON.stringify(Object.keys(protocolToPortMap))}`);
}
this.port = port || protocolToPortMap[this.protocol];
var self = this;
this.timer = setTimeout(function() {
self._timerSend();
}, this.sendIntervalMs);
};
if (this.protocol === 'udp') {
this.udpClient = dgram.createSocket('udp4');
}
}
LogzioLogger.prototype._sendMessagesUDP = function() {
var messagesLength = this.messages.length;
var self = this;
_defaultCallback(err) {
if (err && !this.supressErrors) {
this.internalLogger.log(`logzio-logger error: ${err}`, err);
var udpSentCallback = function(err, bytes) {
if (err) {
self._debug('Error while sending udp packets. err = ' + err);
self.callback(new Error('Failed to send udp log message. err = ' + err));
}
}
};
sendAndClose(callback) {
this.callback = callback || this._defaultCallback;
this._debug('Sending last messages and closing...');
this._popMsgsAndSend();
clearTimeout(this.timer);
for (var i=0; i<messagesLength; i++) {
if (this.protocol === 'udp') {
this.udpClient.close();
}
var msg = this.messages[i];
msg.token = this.token;
var buff = new Buffer(stringifySafe(msg));
self._debug('Starting to send messages via udp.');
this.udpClient.send(buff, 0, buff.length, this.port, this.host, udpSentCallback);
}
};
_timerSend() {
if (this.messages.length > 0) {
this._debug(`Woke up and saw ${this.messages.length} messages to send. Sending now...`);
this._popMsgsAndSend();
}
LogzioLogger.prototype.close = function () {
// clearing the timer allows the node event loop to quit when needed
clearTimeout(this.timer);
this.timer = setTimeout(() => {
this._timerSend();
}, this.sendIntervalMs);
// send pending messages, if any
if (this.messages.length > 0) {
this._debug('Closing, purging messages.');
this._popMsgsAndSend();
}
_sendMessagesUDP() {
const udpSentCallback = (err) => {
if (err) {
this._debug(`Error while sending udp packets. err = ${err}`);
this.callback(new Error(`Failed to send udp log message. err = ${err}`));
}
};
this.messages.forEach((message) => {
const msg = message;
msg.token = this.token;
const buff = new Buffer(stringifySafe(msg));
this._debug('Starting to send messages via udp.');
this.udpClient.send(buff, 0, buff.length, this.port, this.host, udpSentCallback);
});
if (this.protocol === 'udp') {
this.udpClient.close();
}
close() {
// clearing the timer allows the node event loop to quit when needed
clearTimeout(this.timer);
// no more logging allowed
this.closed = true;
};
// send pending messages, if any
if (this.messages.length > 0) {
this._debug('Closing, purging messages.');
this._popMsgsAndSend();
}
/**
* Attach a timestamp to the log record. If @timestamp already exists, use it. Else, use current time.
* The same goes for @timestamp_nano
* @param msg - The message (Object) to append the timestamp to.
* @private
*/
LogzioLogger.prototype._addTimestamp = function(msg) {
var now = (new Date()).toISOString();
msg['@timestamp'] = msg['@timestamp'] || now;
if (this.protocol === 'udp') {
this.udpClient.close();
}
if (this.addTimestampWithNanoSecs) {
var time = process.hrtime();
msg['@timestamp_nano'] = msg['@timestamp_nano'] || [now, time[0].toString(), time[1].toString()].join('-');
}
};
// no more logging allowed
this.closed = true;
LogzioLogger.prototype.log = function(msg) {
if (this.closed === true) {
throw new Error('Logging into a logger that has been closed!');
}
if (typeof msg === 'string') {
msg = { message: msg };
if (this.type) msg.type = this.type;
}
msg = _assign(msg, this.extraFields);
msg.type = this.type;
/**
* Attach a timestamp to the log record.
* If @timestamp already exists, use it. Else, use current time.
* The same goes for @timestamp_nano
* @param msg - The message (Object) to append the timestamp to.
* @private
*/
_addTimestamp(msg) {
const now = (new Date()).toISOString();
msg['@timestamp'] = msg['@timestamp'] || now;
this._addTimestamp(msg);
if (this.addTimestampWithNanoSecs) {
const time = process.hrtime();
msg['@timestamp_nano'] = msg['@timestamp_nano'] || [now, time[0].toString(), time[1].toString()].join('-');
}
this.messages.push(msg);
if (this.messages.length >= this.bufferSize) {
this._debug('Buffer is full - sending bulk');
this._popMsgsAndSend();
}
};
log(msg) {
if (this.closed === true) {
throw new Error('Logging into a logger that has been closed!');
}
if (typeof msg === 'string') {
msg = {
message: msg,
};
}
msg = assign(msg, this.extraFields);
if (!msg.type) {
msg.type = this.type;
}
this._addTimestamp(msg);
this.messages.push(msg);
if (this.messages.length >= this.bufferSize) {
this._debug('Buffer is full - sending bulk');
this._popMsgsAndSend();
}
LogzioLogger.prototype._popMsgsAndSend = function() {
if (this.protocol === 'udp') {
this._debug('Sending messages via udp');
this._sendMessagesUDP();
}
else {
var bulk = this._createBulk(this.messages);
this._debug('Sending bulk #' + bulk.id);
this._send(bulk);
}
_popMsgsAndSend() {
if (this.protocol === 'udp') {
this._debug('Sending messages via udp');
this._sendMessagesUDP();
} else {
const bulk = this._createBulk(this.messages);
this._debug(`Sending bulk #${bulk.id}`);
this._send(bulk);
}
this.messages = [];
};
this.messages = [];
}
LogzioLogger.prototype._createBulk = function(msgs) {
var bulk = {};
// creates a new copy of the array. Objects references are copied (no deep copy)
bulk.msgs = msgs.slice();
bulk.attemptNumber = 1;
bulk.sleepUntilNextRetry = 2*1000;
bulk.id = this.bulkId++;
_createBulk(msgs) {
const bulk = {};
// creates a new copy of the array. Objects references are copied (no deep copy)
bulk.msgs = msgs.slice();
bulk.attemptNumber = 1;
bulk.sleepUntilNextRetry = this.sleepUntilNextRetry;
bulk.id = this.bulkId; // TODO test
this.bulkId += 1;
return bulk;
};
return bulk;
LogzioLogger.prototype._messagesToBody = function(msgs) {
var body = '';
for (var i = 0; i < msgs.length; i++) {
body = body + jsonToString(msgs[i]) + '\n';
}
return body;
};
_debug(msg) {
if (this.debug) this.internalLogger.log(`logzio-nodejs: ${msg}`);
}
LogzioLogger.prototype._debug = function(msg) {
if (this.debug) console.log('logzio-nodejs: ' + msg);
};
_tryAgainIn(sleepTimeMs, bulk) {
this._debug(`Bulk #${bulk.id} - Trying again in ${sleepTimeMs}[ms], attempt no. ${bulk.attemptNumber}`);
setTimeout(() => {
this._send(bulk);
LogzioLogger.prototype._send = function(bulk) {
var self = this;
function tryAgainIn(sleepTimeMs) {
self._debug('Bulk #' + bulk.id + ' - Trying again in ' + sleepTimeMs + '[ms], attempt no. ' + bulk.attemptNumber);
setTimeout(function() {
self._send(bulk);
}, sleepTimeMs);
}
_send(bulk) {
const body = messagesToBody(bulk.msgs);
const options = {
uri: this.url,
headers: {
host: this.host,
accept: '*/*',
'user-agent': USER_AGENT,
'content-type': 'text/plain',
},
};
if (typeof this.timeout !== 'undefined') {
options.timeout = this.timeout;
var body = this._messagesToBody(bulk.msgs);
var options = {
uri: this.url,
body: body,
headers: {
'host': this.host,
'accept': '*/*',
'user-agent': this.userAgent,
'content-type': 'text/plain',
'content-length': Buffer.byteLength(body)
}
return Promise.resolve()
.then(() => {
if (this.compress) {
options.headers['content-encoding'] = 'gzip';
return zlibPromised(body);
}
return body;
})
.then((finalBody) => {
options.body = finalBody;
this._tryToSend(options, bulk);
});
};
if (typeof this.timeout !== 'undefined') {
options.timeout = this.timeout;
}
_tryToSend(options, bulk) {
return request.post(options)
.then(() => {
this._debug(`Bulk #${bulk.id} - sent successfully`);
this.callback();
})
.catch((err) => {
var callback = this.callback;
try {
request.post(options, function (err, res, body) {
if (err) {
// In rare cases server is busy
const errorCode = err.cause && err.cause.code;
if (UNAVAILABLE_CODES.includes(errorCode)) {
if (bulk.attemptNumber >= this.numberOfRetries) {
return this.callback(new Error(`Failed after ${bulk.attemptNumber} retries on error = ${err}`));
if (err.code === 'ETIMEDOUT' || err.code === 'ECONNRESET' || err.code === 'ESOCKETTIMEDOUT' || err.code === 'ECONNABORTED') {
if (bulk.attemptNumber >= self.numberOfRetries) {
callback(new Error('Failed after ' + bulk.attemptNumber + ' retries on error = ' + err, err));
} else {
self._debug('Bulk #' + bulk.id + ' - failed on error: ' + err);
var sleepTimeMs = bulk.sleepUntilNextRetry;
bulk.sleepUntilNextRetry = bulk.sleepUntilNextRetry * 2;
bulk.attemptNumber++;
tryAgainIn(sleepTimeMs);
}
this._debug(`Bulk #${bulk.id} - failed on error: ${err}`);
const sleepTimeMs = bulk.sleepUntilNextRetry;
bulk.sleepUntilNextRetry *= 2;
bulk.attemptNumber += 1;
return this._tryAgainIn(sleepTimeMs, bulk);
}
if (err.statusCode !== 200) {
return this.callback(new Error(`There was a problem with the request.\nResponse: ${err.statusCode}: ${err.message}`));
else {
callback(err);
}
}
else {
var responseCode = res.statusCode.toString();
if (responseCode !== '200') {
callback(new Error('There was a problem with the request.\nResponse: ' + responseCode + ': ' + body.toString()));
}
else {
self._debug('Bulk #' + bulk.id + ' - sent successfully');
callback();
}
}
return this.callback(err);
});
});
}
}
const createLogger = (options) => {
const l = new LogzioLogger(options);
l._timerSend();
return l;
catch (ex) {
callback(ex);
}
};
module.exports = {
jsonToString,
createLogger,
};

20

package.json
{
"name": "logzio-nodejs",
"description": "A nodejs implementation for sending logs to Logz.IO cloud service",
"version": "0.4.17",
"version": "0.4.18",
"author": "Gilly Barr <gilly@logz.io>",

@@ -38,4 +38,3 @@ "contributors": [{

"moment": "^2.22.1",
"request": "^2.88.0",
"request-promise": "^4.2.2"
"request": "^2.87.0"
},

@@ -45,21 +44,16 @@ "devDependencies": {

"async": "1.4.2",
"eslint": "^5.9.0",
"eslint-config-airbnb-base": "^13.1.0",
"eslint-plugin-import": "^2.14.0",
"eslint": "^5.0.1",
"mocha": "^5.2.0",
"jest": "^23.6.0",
"nock": "^10.0.2",
"nock": "^9.4.0",
"should": "^7.1.0",
"sinon": "^7.1.1"
"sinon": "^1.17.1"
},
"main": "./lib/logzio-nodejs",
"engines": {
"node": ">= 6.0.0"
"node": ">= 0.8.0"
},
"license": "(Apache-2.0)",
"scripts": {
"test": "jest",
"detached": "jest --detectOpenHandles",
"watch": "jest --watch"
"test": "node_modules/.bin/mocha"
}
}

24

README.md
![Build Status](https://travis-ci.org/logzio/logzio-nodejs.svg?branch=master)
# logzio-nodejs
NodeJS logger for Logz.io.
NodeJS logger for LogzIO.
The logger stashes the log messages you send into an array which is sent as a bulk once it reaches its size limit (100 messages) or time limit (10 sec) in an async fashion.
It contains a simple retry mechanism which upon connection reset (server side) or client timeout, wait a bit (default interval of 2 seconds), and try this bulk again. It does not block other messages from being accumulated and sent (async). The interval increases by a factor of 2 between each retry until it reaches the maximum allowed attempts (3).
It contains a simple retry mechanism which upon connection reset (server side) or client timeout, wait a bit (default interval of 2 seconds), and try this bulk again (it does not block other messages from being accumulated and sent (async). The interval increases by a factor of 2 between each retry, until we reached the maximum allowed attempts (3).
By default, any error is logged to the console. This can be changed by supplying a callback function.
By default any error is logged to the console. This can be changed by supplying a callback function.

@@ -48,20 +48,14 @@

* **addTimestampWithNanoSecs** - Add a timestamp with nano seconds granularity. This is needed when many logs are sent in the same millisecond, so you can properly order the logs in kibana. The added timestamp field will be `@timestamp_nano` Default: `false`
* **compress** - If true the the logs are compressed in gzip format. Default: `false`
* **internalLogger** - set internal logger that supports the function log. Default: console.
## Using UDP
A few notes are worth mentioning regarding the use of the UDP protocol:
* UDP has some limitations, and therefore it is not the recommended protocol:
A few notes are worth mentioning regarding the use of the UDP protocol :
* UDP has some limitations, and therefore it is not the recommended protocol :
* There is no guarantee that the logs have been received.
* UDP can't take advantage of the bulk API, so performance is sub-optimal.
* When using UDP, each message is sent separately, and not using the bulk API. This means that the meaning of `bufferSize` is slightly different in this case. The messages will still be sent separately, but the logger will wait for the buffer to reach the size specified before sending out all the messages. If you want each message to be sent out immediately, then set `bufferSize = 1`.
* UDP can't take advantage of the bulk api and therefore performance is sub-optimal.
* When using UDP, each message is being sent separately, and not using the bulk api. This means that the meaning of `bufferSize` is slightly
different in this case. The messages will still be sent separately, but the logger will wait for the buffer to reach the size specified before
sending out all the messages. If you want each message to be sent out immediately, then set `bufferSize = 1`.
## Update log
**1.0.1**
- ES6
- support node greater than node 6
- added gzip compress option
- added internal logger option
**0.4.14**

@@ -68,0 +62,0 @@ - UDP callback bug fix + tests

2

test/.eslintrc.js
module.exports = {
"env": {
"jest": true,
"mocha": true,
},
};
.vscode/settings.json
example.js
test/logger.test.js
test/udp.test.js
.travis.yml

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc