@twec/serverless-es-logs - npm Package Compare versions

Comparing version 2.0.0 to 2.1.0

dist/index.js

@@ -222,8 +222,7 @@ "use strict";
         const dirPath = path_1.default.join(this.serverless.config.servicePath, this.logProcesserDir);
+        const filePath = path_1.default.join(dirPath, 'index.js');
         const handler = `${this.logProcesserDir}/index.handler`;
         const name = `${this.serverless.service.service}-${this.options.stage}-es-logs-plugin`;
         fs_extra_1.default.ensureDirSync(dirPath);
-        fs_extra_1.default.copySync(path_1.default.resolve(__dirname, '../templates/code/logsToEs.js'), path_1.default.join(dirPath, 'index.js'));
-        fs_extra_1.default.copySync(path_1.default.resolve(__dirname, '../templates/code/request.js'), path_1.default.join(dirPath, 'request.js'));
-        fs_extra_1.default.copySync(path_1.default.resolve(__dirname, '../templates/code/transform.js'), path_1.default.join(dirPath, 'transform.js'));
+        fs_extra_1.default.copySync(path_1.default.resolve(__dirname, '../templates/code/logsToEs.js'), filePath);
         this.serverless.service.functions[this.logProcesserName] = {
@@ -244,3 +243,3 @@ description: 'Serverless ES Logs Plugin',
             },
-            runtime: 'nodejs8.10',
+            runtime: 'nodejs10.x',
             timeout: 60,
@@ -247,0 +246,0 @@ tracing: false,

package.json

 {
   "name": "@twec/serverless-es-logs",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
   "description": "A Serverless plugin to transport logs to ElasticSearch",
   "author": "Daniel Cottone <[e-mail]> (https://github.com/daniel-cottone)",
-  "contributors": [
-    "Luis Couto <[e-mail]> (https://github.com/Couto)"
-  ],
   "repository": {
@@ -19,3 +19,3 @@ "type": "git",
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=6.10.0"
   },
@@ -22,0 +22,0 @@ "main": "dist/index.js",

src/index.ts

@@ -256,8 +256,7 @@ import fs from 'fs-extra';
     const dirPath = path.join(this.serverless.config.servicePath, this.logProcesserDir);
+    const filePath = path.join(dirPath, 'index.js');
     const handler = `${this.logProcesserDir}/index.handler`;
     const name = `${this.serverless.service.service}-${this.options.stage}-es-logs-plugin`;
     fs.ensureDirSync(dirPath);
-    fs.copySync(path.resolve(__dirname, '../templates/code/logsToEs.js'), path.join(dirPath, 'index.js'));
-    fs.copySync(path.resolve(__dirname, '../templates/code/request.js'), path.join(dirPath, 'request.js'));
-    fs.copySync(path.resolve(__dirname, '../templates/code/transform.js'), path.join(dirPath, 'transform.js'));
+    fs.copySync(path.resolve(__dirname, '../templates/code/logsToEs.js'), filePath);
     this.serverless.service.functions[this.logProcesserName] = {
@@ -278,3 +277,3 @@ description: 'Serverless ES Logs Plugin',
       },
-      runtime: 'nodejs8.10',
+      runtime: 'nodejs10.x',
       timeout: 60,
@@ -281,0 +280,0 @@ tracing: false,

templates/code/logsToEs.js

@@ -1,24 +0,248 @@
-const zlib = require('zlib');
-const utils = require('util');
-const request = require('./request');
-const transform = require('./transform');
+// v1.1.2
+var https = require('https');
+var zlib = require('zlib');
+var crypto = require('crypto');
 
-const gunZip = utils.promisify(zlib.gunzip);
-const toUTF8 = data => data.toString('utf8');
-const toJSON = data => JSON.parse(data);
-const ifTruthy = fn => data => data && fn(data);
+var endpoint = process.env.ES_ENDPOINT;
+var indexPrefix = process.env.INDEX_PREFIX;
 
-exports.handler = async (input, context) =>
-  gunZip(new Buffer(input.awslogs.data, 'base64'))
-    .then(toUTF8)
-    .then(toJSON)
-    .then(transform)
-    .then(ifTruthy(request))
-    .then(success => {
-      console.log('Response:', JSON.stringify(success));
-      context.succeed('Success');
-    })
-    .catch(err => {
-      console.log('Error:', JSON.stringify(err));
-      context.fail(err);
+exports.handler = function(input, context) {
+    // decode input from base64
+    var zippedInput = new Buffer(input.awslogs.data, 'base64');
+
+    // decompress the input
+    zlib.gunzip(zippedInput, function(error, buffer) {
+        if (error) { context.fail(error); return; }
+
+        // parse the input from JSON
+        var awslogsData = JSON.parse(buffer.toString('utf8'));
+
+        // transform the input to Elasticsearch documents
+        var elasticsearchBulkData = transform(awslogsData);
+
+        // skip control messages
+        if (!elasticsearchBulkData) {
+            console.log('Received a control message');
+            context.succeed('Control message handled successfully');
+            return;
+        }
+
+        // post documents to the Amazon Elasticsearch Service
+        post(elasticsearchBulkData, function(error, success, statusCode, failedItems) {
+            console.log('Response: ' + JSON.stringify({ 
+                "statusCode": statusCode 
+            }));
+
+            if (error) { 
+                console.log('Error: ' + JSON.stringify(error, null, 2));
+
+                if (failedItems && failedItems.length > 0) {
+                    console.log("Failed Items: " +
+                        JSON.stringify(failedItems, null, 2));
+                }
+
+                context.fail(JSON.stringify(error));
+            } else {
+                console.log('Success: ' + JSON.stringify(success));
+                context.succeed('Success');
+            }
+        });
     });
+};
+
+function transform(payload) {
+    if (payload.messageType === 'CONTROL_MESSAGE') {
+        return null;
+    }
+
+    var bulkRequestBody = '';
+
+    payload.logEvents.forEach(function(logEvent) {
+        var timestamp = new Date(1 * logEvent.timestamp);
+
+        // index name format: cwl-YYYY.MM.DD
+        var indexName = [
+            indexPrefix + '-' + timestamp.getUTCFullYear(),   // year
+            ('0' + (timestamp.getUTCMonth() + 1)).slice(-2),  // month
+            ('0' + timestamp.getUTCDate()).slice(-2)          // day
+        ].join('.');
+
+        var source = buildSource(logEvent.message, logEvent.extractedFields);
+        source['@id'] = logEvent.id;
+        source['@timestamp'] = new Date(1 * logEvent.timestamp).toISOString();
+        source['@message'] = logEvent.message;
+        source['@owner'] = payload.owner;
+        source['@log_group'] = payload.logGroup;
+        source['@log_stream'] = payload.logStream;
+
+        var action = { "index": {} };
+        action.index._index = indexName;
+        action.index._type = 'serverless-es-logs';
+        action.index._id = logEvent.id;
+        
+        bulkRequestBody += [ 
+            JSON.stringify(action), 
+            JSON.stringify(source),
+        ].join('\n') + '\n';
+    });
+    return bulkRequestBody;
+}
+
+function buildSource(message, extractedFields) {
+    if (extractedFields) {
+        var source = {};
+
+        for (var key in extractedFields) {
+            if (extractedFields.hasOwnProperty(key) && extractedFields[key]) {
+                var value = extractedFields[key];
+
+                if (isNumeric(value)) {
+                    source[key] = 1 * value;
+                    continue;
+                }
+
+                jsonSubString = extractJson(value);
+                if (jsonSubString !== null) {
+                    source['$' + key] = JSON.parse(jsonSubString);
+                }
+
+                source[key] = (key === 'apigw_request_id') ? value.slice(1, value.length - 1) : value;
+            }
+        }
+        return source;
+    }
+
+    jsonSubString = extractJson(message);
+    if (jsonSubString !== null) { 
+        return JSON.parse(jsonSubString); 
+    }
+
+    return {};
+}
+
+function extractJson(message) {
+    var jsonStart = message.indexOf('{');
+    if (jsonStart < 0) return null;
+    var jsonSubString = message.substring(jsonStart);
+    return isValidJson(jsonSubString) ? jsonSubString : null;
+}
+
+function isValidJson(message) {
+    try {
+        JSON.parse(message);
+    } catch (e) { return false; }
+    return true;
+}
+
+function isNumeric(n) {
+    return !isNaN(parseFloat(n)) && isFinite(n);
+}
+
+function post(body, callback) {
+    var requestParams = buildRequest(endpoint, body);
+
+    var request = https.request(requestParams, function(response) {
+        var responseBody = '';
+        response.on('data', function(chunk) {
+            responseBody += chunk;
+        });
+        response.on('end', function() {
+            var info = JSON.parse(responseBody);
+            var failedItems;
+            var success;
+            
+            if (response.statusCode >= 200 && response.statusCode < 299) {
+                failedItems = info.items.filter(function(x) {
+                    return x.index.status >= 300;
+                });
+
+                success = { 
+                    "attemptedItems": info.items.length,
+                    "successfulItems": info.items.length - failedItems.length,
+                    "failedItems": failedItems.length
+                };
+            }
+
+            var error = response.statusCode !== 200 || info.errors === true ? {
+                "statusCode": response.statusCode,
+                "responseBody": responseBody
+            } : null;
+
+            callback(error, success, response.statusCode, failedItems);
+        });
+    }).on('error', function(e) {
+        callback(e);
+    });
+    request.end(requestParams.body);
+}
+
+function buildRequest(endpoint, body) {
+    var endpointParts = endpoint.match(/^([^\.]+)\.?([^\.]*)\.?([^\.]*)\.amazonaws\.com$/);
+    console.log('endpoint', endpoint);
+    console.log('endpointParts', endpointParts);
+    var region = endpointParts[2];
+    var service = endpointParts[3];
+    var datetime = (new Date()).toISOString().replace(/[:\-]|\.\d{3}/g, '');
+    var date = datetime.substr(0, 8);
+    var kDate = hmac('AWS4' + process.env.AWS_SECRET_ACCESS_KEY, date);
+    var kRegion = hmac(kDate, region);
+    var kService = hmac(kRegion, service);
+    var kSigning = hmac(kService, 'aws4_request');
+    
+    var request = {
+        host: endpoint,
+        method: 'POST',
+        path: '/_bulk',
+        body: body,
+        headers: { 
+            'Content-Type': 'application/json',
+            'Host': endpoint,
+            'Content-Length': Buffer.byteLength(body),
+            'X-Amz-Security-Token': process.env.AWS_SESSION_TOKEN,
+            'X-Amz-Date': datetime
+        }
+    };
+
+    var canonicalHeaders = Object.keys(request.headers)
+        .sort(function(a, b) { return a.toLowerCase() < b.toLowerCase() ? -1 : 1; })
+        .map(function(k) { return k.toLowerCase() + ':' + request.headers[k]; })
+        .join('\n');
+
+    var signedHeaders = Object.keys(request.headers)
+        .map(function(k) { return k.toLowerCase(); })
+        .sort()
+        .join(';');
+
+    var canonicalString = [
+        request.method,
+        request.path, '',
+        canonicalHeaders, '',
+        signedHeaders,
+        hash(request.body, 'hex'),
+    ].join('\n');
+
+    var credentialString = [ date, region, service, 'aws4_request' ].join('/');
+
+    var stringToSign = [
+        'AWS4-HMAC-SHA256',
+        datetime,
+        credentialString,
+        hash(canonicalString, 'hex')
+    ] .join('\n');
+
+    request.headers.Authorization = [
+        'AWS4-HMAC-SHA256 Credential=' + process.env.AWS_ACCESS_KEY_ID + '/' + credentialString,
+        'SignedHeaders=' + signedHeaders,
+        'Signature=' + hmac(kSigning, stringToSign, 'hex')
+    ].join(', ');
+
+    return request;
+}
+
+function hmac(key, str, encoding) {
+    return crypto.createHmac('sha256', key).update(str, 'utf8').digest(encoding);
+}
+
+function hash(str, encoding) {
+    return crypto.createHash('sha256').update(str, 'utf8').digest(encoding);
+}

test/support/ServerlessBuilder.ts

@@ -63,3 +63,3 @@ import _ from 'lodash';
         region: 'us-east-1',
-        runtime: 'nodejs8.10',
+        runtime: 'nodejs10.x',
         stage: 'dev',
@@ -66,0 +66,0 @@ },

New alerts

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 1 instance in 1 package

Worsened metrics

Total package byte prevSize

89741

decreased by-70.69%

Number of package files

43

decreased by-4.44%

Lines of code

1691

decreased by-0.24%

Number of low supply chain risk alerts

6

increased by20%

Number of medium supply chain risk alerts

2

increased by100%

No dependency changes