@janiscommerce/log - npm Package Compare versions

Comparing version 3.1.0 to 3.1.1

CHANGELOG.md

@@ -8,2 +8,8 @@ # Changelog
 
+## [Unreleased]
+
+## [3.1.1] - 2020-04-23
+### Added
+- Multiple logs support
+
 ## [3.1.0] - 2020-03-04
@@ -10,0 +16,0 @@ ### Added

lib/aws-wrappers.js

@@ -12,5 +12,5 @@ 'use strict';
 	/* istanbul ignore next */
-	// AWS generates the Firehose class on the fly, the putRecord method do not exists before creating the insance
-	putRecord(record) {
-		return this._firehose.putRecord(record).promise();
+	// AWS generates the Firehose class on the fly, the putRecordBatch method do not exists before creating the insance
+	putRecordBatch(records) {
+		return this._firehose.putRecordBatch(records).promise();
 	}
@@ -17,0 +17,0 @@ }

lib/log.js

 'use strict';
 
-const { struct } = require('superstruct');
-
 const EventEmmiter = require('events');
-const UUID = require('uuid/v4');
 
+const { STS, Firehose } = require('./aws-wrappers');
+const { arrayChunk } = require('./helpers/utils');
+
+const Validator = require('./helpers/validator');
 const LogError = require('./log-error');
 
-const { STS, Firehose } = require('./aws-wrappers');
-
 const ARN_DURATION = 1800; // 30 min
@@ -16,2 +15,3 @@ const MAX_ATTEMPTS = 3;
 const DELIVERY_STREAM_PREFIX = 'JanisTraceFirehose';
+const LOGS_BATCH_LIMIT = 500;
 
@@ -24,15 +24,15 @@ const sts = new STS();
 
-	static get _serviceName() {
+	static get serviceName() {
 		return process.env.JANIS_SERVICE_NAME;
 	}
 
-	static get _env() {
+	static get env() {
 		return process.env.JANIS_ENV;
 	}
 
-	static get _roleArn() {
+	static get roleArn() {
 		return process.env.LOG_ROLE_ARN;
 	}
 
-	static get _envs() {
+	static get envs() {
 
@@ -50,3 +50,3 @@ return {
 		if(!this._deliveryStreamName)
-			this._deliveryStreamName = `${DELIVERY_STREAM_PREFIX}${this._getFormattedEnv()}`;
+			this._deliveryStreamName = `${DELIVERY_STREAM_PREFIX}${this.formattedEnv}`;
 
@@ -56,6 +56,25 @@ return this._deliveryStreamName;
 
+	static get formattedEnv() {
+
+		if(this.env && this.envs[this.env])
+			return this.envs[this.env];
+
+		throw new LogError('Unknown environment', LogError.codes.NO_ENVIRONMENT);
+	}
+
 	/**
-	 * Put a log into Firehose
+	 * Sets a callback for the specified event name
+	 * @param {String} event The event name
+	 * @param {Function} callback The event callback
+	 * @example
+	 * on('create-error', (log, err) => {...});
+	 */
+	static on(event, callback) {
+		emitter.on(event, callback);
+	}
+
+	/**
+	 * Put logs into Firehose
 	 * @param {String} client The client code who created the log
-	 * @param {Object} log The log object
+	 * @param {Object|Array.<object>} logs The log object or log objects array
 	 * @example
@@ -72,62 +91,64 @@ * add('some-client', {
 	 */
-	static async add(client, log) {
+	static async add(client, logs) {
 
+		// For local development
+		if(this.env === 'local')
+			return true;
+
+		if(!Array.isArray(logs))
+			logs = [logs];
+
+		let validLogs;
+
 		try {
 
-			log = this._validateLog(log, client);
+			validLogs = logs.map(log => this.validateLog(log, client));
 
 		} catch(err) {
-			return emitter.emit('create-error', log, err);
+			return emitter.emit('create-error', logs, err);
 		}
 
-		return this._add(log);
+		const logsBatches = this.createLogsBatches(validLogs);
+		return this._add(logsBatches);
 	}
 
-	/**
-	 * Sets a callback for the specified event name
-	 * @param {String} event The event name
-	 * @param {Function} callback The event callback
-	 * @example
-	 * on('create-error', (log, err) => {...});
-	 */
-	static on(event, callback) {
-		emitter.on(event, callback);
+	static validateLog(log, client) {
+		return Validator.validate(log, client, this.serviceName);
 	}
 
-	static _validateLog(rawLog, client) {
+	static createLogsBatches(logs) {
+		return arrayChunk(logs, LOGS_BATCH_LIMIT);
+	}
 
-		const logStruct = struct.partial({
-			id: 'string',
-			service: 'string',
-			entity: 'string',
-			entityId: 'string?|number?',
-			type: 'string',
-			log: 'object?|array?',
-			message: 'string?',
-			client: 'string',
-			userCreated: 'string?'
-		}, {
-			id: UUID(),
-			service: this._serviceName,
-			client
-		});
+	static async _add(logsBatches, attempts = 0) {
 
 		try {
 
-			const validLog = logStruct(rawLog);
+			const firehose = await this.getFirehoseInstance();
 
-			if(validLog.log)
-				validLog.log = JSON.stringify(validLog.log);
+			return Promise.all(
+				logsBatches.map(logs => firehose.putRecordBatch(
+					{
+						DeliveryStreamName: this.deliveryStreamName,
+						Records: logs.map(log => ({
+							Data: Buffer.from(JSON.stringify(log))
+						}))
+					}
+				))
+			);
 
-			return {
-				...validLog,
-				dateCreated: new Date().toISOString()
-			};
+		} catch(err) {
 
-		} catch(err) {
-			throw new LogError(err.message, LogError.codes.INVALID_LOG);
+			attempts++;
+
+			if(attempts >= MAX_ATTEMPTS) {
+				return emitter.emit('create-error', logsBatches,
+					new LogError(`Unable to put the logs into firehose, max attempts reached: ${err.message}`, LogError.codes.FIREHOSE_ERROR));
+			}
+
+			return this._add(logsBatches, attempts);
 		}
 	}
 
-	static async _getFirehoseInstance() {
+	static async getFirehoseInstance() {
 
@@ -144,6 +165,4 @@ const hasExpired = this._credentialsExpiration < new Date();
 
-		if(this._roleArn) {
-
-			firehoseParams.credentials = await this._getCredentials();
-
+		if(this.roleArn) {
+			firehoseParams.credentials = await this.getCredentials();
 			this._credentialsExpiration = firehoseParams.credentials.expiration;
@@ -153,11 +172,10 @@ }
 		this._firehose = new Firehose(firehoseParams);
-
 		return this._firehose;
 	}
 
-	static async _getCredentials() {
+	static async getCredentials() {
 
 		const assumedRole = await sts.assumeRole({
-			RoleArn: this._roleArn,
-			RoleSessionName: this._serviceName,
+			RoleArn: this.roleArn,
+			RoleSessionName: this.serviceName,
 			DurationSeconds: ARN_DURATION
@@ -178,38 +196,4 @@ });
 	}
-
-	static _getFormattedEnv() {
-
-		if(this._env && this._envs[this._env])
-			return this._envs[this._env];
-
-		throw new LogError('Unknown environment', LogError.codes.NO_ENVIRONMENT);
-	}
-
-	static async _add(log, attempts = 0) {
-
-		try {
-
-			const firehose = await this._getFirehoseInstance();
-
-			await firehose.putRecord({
-				DeliveryStreamName: this.deliveryStreamName,
-				Record: {
-					Data: Buffer.from(JSON.stringify(log))
-				}
-			});
-
-		} catch(err) {
-
-			attempts++;
-
-			if(attempts >= MAX_ATTEMPTS) {
-				return emitter.emit('create-error', log,
-					new LogError(`Unable to put the log into firehose, max attempts reached: ${err.message}`, LogError.codes.FIREHOSE_ERROR));
-			}
-
-			return this._add(log, attempts);
-		}
-	}
 }
 
 module.exports = Log;

package.json

 {
   "name": "@janiscommerce/log",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "A package for creating logs in Firehose",
@@ -21,9 +21,9 @@ "main": "lib/log.js",
   "devDependencies": {
-    "eslint": "^5.16.0",
+    "eslint": "^6.8.0",
     "eslint-config-airbnb-base": "^13.1.0",
-    "eslint-plugin-import": "^2.17.3",
-    "husky": "^3.0.1",
+    "eslint-plugin-import": "^2.20.2",
+    "husky": "^4.2.5",
     "md5": "^2.2.1",
-    "mocha": "^5.2.0",
-    "nyc": "^13.1.0",
+    "mocha": "^7.1.1",
+    "nyc": "^15.0.1",
     "sinon": "^7.3.2"
@@ -38,6 +38,6 @@ },
   "dependencies": {
+    "@janiscommerce/superstruct": "^1.1.1",
     "aws-sdk": "^2.498.0",
-    "uuid": "^3.3.2",
-    "superstruct": "0.6.2"
+    "uuid": "^7.0.3"
   }
 }

README.md

@@ -16,9 +16,9 @@ # log
 **`JANIS_SERVICE_NAME`** (required): The name of the service that will create the log.  
-**`JANIS_ENV`** (required): The name stage that will used as suffix for janis-trace-service bucket.
+**`JANIS_ENV`** (required): The stage name that will used as prefix for trace firehose delivery stream.
 **`LOG_ROLE_ARN`** (required): The ARN to assume the trace role in order to put records in Firehose.
 
 ## API
-### **`add(clientCode, log)`**  
-Parameters: `clientCode [String]`, `log [Object]`  
-Puts the recieved log into the janis-trace-firehose
+### **`add(clientCode, logs)`**  
+Parameters: `clientCode [String]`, `logs [Object] or [Object array]`  
+Puts the recieved log or logs into the janis-trace-firehose
 
@@ -73,3 +73,3 @@ ### Log structure
 
-In case of error while creating your log into S3, this package will emit an event called `create-error`, you can handle it using the `on()` method.
+- In case of error while sending your logs to Firehose, this package will emit an event called `create-error`, you can handle it using the `on()` method.
 
@@ -80,13 +80,75 @@ ## Usage
 
-Log.add('some-client', {
-	type: 1,
-	entity: 'api',
-	entityId: 'product',
-	message: '[GET] Request from 0.0.0.0 of custom_data'
-	// ...
+// Single log send
+await Log.add('some-client', {
+    service: "oms",
+    entity: "api",
+    entityId: "order",
+    type: "api-request",
+    dateCreated: "2020-04-21T17:16:01.324Z",
+    log: {
+        api: {
+            endpoint: "order/5ea1c7f48efca3c21654d4a3/pick-items",
+            httpMethod: "post"
+        },
+        request: {
+            headers: {
+                accept: "application/json",
+                "content-type": "application/json",
+                Host: "oms.host.com",
+                "janis-client": "some-client",
+                "X-Amzn-Trace-Id": "Root=1-fca3c2-5ea1c7f48efca3c21654d4a3",
+                "X-Forwarded-For": "12.354.67.890",
+                "X-Forwarded-Port": "123",
+                "X-Forwarded-Proto": "https"
+            },
+            data: {
+                0: {
+                    pickedQuantity: 1,
+                    pickingSessionId: "5ea1c88463d91e9758f2c1b8",
+                    pickerId: "5ea1c8895ebb38d472ccd8c3",
+                    id: "5EA1C88D6E94BC19F7FC1612",
+                    pickedEans: [
+                        "1234567890"
+                    ]
+                }
+            }
+        },
+        response: {
+            code: 200,
+            headers: {},
+            body: {}
+        },
+        executionTime: 868.251946
+    }
+}
 });
 
+// Multiple logs send
+await Log.add('some-client', [
+  {
+    service: "catalog",
+    entity: "account",
+    entityId: "5ea1c8c53fdac68fb60eac9e",
+    type: "upserted",
+    dateCreated: "2020-04-22T22:03:50.507Z",
+    log: {
+      id: "5ea1c8c53fdac68fb60eac9e",
+      referenceId: "rv-000005"
+    }
+  },
+  {
+    service: "catalog",
+    entity: "account",
+    entityId: "5ea1c8cd11f82560a364cbd4",
+    type: "upserted",
+    dateCreated: "2020-04-22T22:03:50.507Z",
+    log: {
+      id: "5ea1c8cd11f82560a364cbd4",
+      referenceId: "rf-00752"
+    }
+  }
+]);
+
 Log.on('create-error', (log, err) => {
 	console.error(`An error occurred while creating the log ${err.message}`);
-});
-```
+});

No alert changes

Improved metrics

Total package byte prevSize

13712

increased by28.07%

Number of package files

8

increased by33.33%

Lines of code

238

increased by21.43%

Number of lines in readme file

151

increased by69.66%

Dependency changes

• + Added@janiscommerce/superstruct@^1.1.1

• + Added@janiscommerce/superstruct@1.2.1(transitive)

• + Addeduuid@7.0.3(transitive)

• - Removedsuperstruct@0.6.2

• - Removeduuid@3.4.0(transitive)

• Updateduuid@^7.0.3