logagent-js - npm Package Compare versions

Comparing version 1.2.1 to 1.2.2

bin/logagent.js

@@ -43,2 +43,4 @@ #!/bin/sh
 var count = 0
+var logsShipped = 0
+var httpFailed = 0
 var emptyLines = 0
@@ -142,6 +144,7 @@ var bytes = 0
     logger.on('log', function (data) {
-      // console.log(data)
+      logsShipped += (Number(data.count) || 0)
     })
     logger.on('error', function (err) {
-      console.error('Error in Logsene request:' + err.message)
+      httpFailed++
+      console.error('Error in Logsene request: ' + JSON.stringify(err))
     })
@@ -156,6 +159,9 @@ if (process.env.LOG_NEW_TOKENS) {
 
+function _logToLogsene () {
+  var logger = getLogger(this.token, this.type)
+  var data = this.data
+  logger.log(data.level || data.severity || 'info', data.message || data.msg || data.MESSAGE, data) 
+}
 function logToLogsene (token, type, data) {
-  var logger = getLogger(token, type)
-  //console.log('token:' + token + ' message: ' + JSON.stringify(data))
-  logger.log(data.level || data.severity || 'info', data.message || data.msg || data.MESSAGE, data)
+  setImmediate(_logToLogsene.bind({token: token, data: data, type:type}))
 }
@@ -405,2 +411,3 @@
 }
+
 function parseLine (line, sourceName, cbf) {
@@ -418,10 +425,12 @@ if (!line && cbf) {
 
+
 function readStdIn () {
-  var rl = readline.createInterface({
-    terminal: false,
-    input: process.stdin
-  })
-  rl.on('line', parseLine)
-  rl.on('close', terminate)
-  rl.on('finish', terminate)
+  var tr = require('through2')
+  var split = require('split')
+  process.stdin.on('end', terminate)
+  process.stdin.pipe(split()).pipe(tr(function (chunk, enc, callback) {
+    parseLine(chunk.toString())
+    setImmediate(callback)
+  }))
+  
 }
@@ -449,9 +458,13 @@
   console.error('pid['+process.pid + ']' + ' ' + duration + ' ms ' + count + ' lines parsed.  ' + throughput.toFixed(0) + ' lines/s ' + throughputBytes.toFixed(3) + ' MB/s - empty lines: ' + emptyLines)
-  console.error('Tokens used: ' + Object.keys(loggers).length)
-  console.error('Heap Used: ' + (process.memoryUsage().heapUsed / (1024 * 1024)) + ' MB')
-  console.error('Heap Total: ' + (process.memoryUsage().heapTotal / (1024 * 1024)) + ' MB')
-  console.error('Memory RSS: ' + (process.memoryUsage().rss / (1024 * 1024)) + ' MB')
+  console.error('Tokens used:\t' + Object.keys(loggers).length)
+  console.error('Logs shipped:\t' + logsShipped)
+  console.error('HTTP failed:\t' + httpFailed)
+  console.error('Heap Used:\t' + (process.memoryUsage().heapUsed / (1024 * 1024)) + ' MB')
+  console.error('Heap Total:\t' + (process.memoryUsage().heapTotal / (1024 * 1024)) + ' MB')
+  console.error('Memory RSS:\t' + (process.memoryUsage().rss / (1024 * 1024)) + ' MB')
   begin = now
   count = 0
   bytes = 0
+  logsShipped=0
+  httpFailed=0
 }
@@ -458,0 +471,0 @@

lib/index.js

@@ -289,8 +289,11 @@ /*
     var br = this.getMultiLineReader(source, function (data) {
-        try {
-          this._parseLine(data, source, cbf)
-        } catch (err) {
+        setImmediate (function () {
+          try {
+            this._parseLine(data, source, cbf)
+          } catch (err) {
           // console.log('Error parsing logs from ' + source +  ': ' + err + ' originalLine: ' + line)
-          cbf(err, data)
-        }
+            cbf(err, data)
+          }  
+        }.bind(this))
+        
     }.bind(this))
@@ -297,0 +300,0 @@ br.add(line)

package.json

 {
   "name": "logagent-js",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "Smart log parser written in Node",
@@ -14,8 +14,7 @@ "main": "lib/index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "mocha",
     "start": "node bin/logagent.js -s --heroku $PORT",
     "pubdocs": "mkdocs gh-deploy --clean",
     "docs": "mkdocs serve",
-    "mkdocs": "pip install mkdocs",
-    "test": "mocha"
+    "mkdocs": "pip install mkdocs"
   },
@@ -52,5 +51,7 @@ "repository": {
     "prettyjson": "^1.1.2",
+    "split": "^1.0.0",
     "syslogd": "^1.1.2",
     "tail-forever": "^0.3.11",
     "throng": "^4.0.0",
+    "through2": "^2.0.1",
     "yamljs": "^0.2.3"
@@ -57,0 +58,0 @@ },

site/mkdocs/search_index.json

@@ -100,3 +100,3 @@ {
             "location": "/cli-parameters/", 
-            "text": "Command Line Parameters for logagent\n\n\nSynopsis\n\n\nlogagent [options] [file list]\n\n\n\n\n\n\n\n\nOptions\n\n\nDescription\n\n\n\n\n\n\n\n\n\n\n-f\n patterns.yml\n\n\nfile with pattern definitions\n\n\n\n\n\n\n-y\n\n\nprints parsed messages in YAML format to stdout\n\n\n\n\n\n\n-p\n\n\npretty json output\n\n\n\n\n\n\n-s\n\n\nsilent, print no logs, only throughput and memory usage on exit\n\n\n\n\n\n\n--print_stats\n\n\nprint processing stats in the given interval in seconds, e.g. \n--print_stats 30\n. Usefull with -s to see logagent activity on the console without printing the parsed logs.\n\n\n\n\n\n\n-t\n token\n\n\nLogsene\n App Token to insert parsed records into Logsene.\n\n\n\n\n\n\n-g\n glob-pattern\n\n\nuse a \nglob\n pattern to watch log files e.g. \n-g \"{/var/log/*.log,/Users/stefan/myapp/*.log}\"\n. The complete glob expression must be quoted, to avoid interpretation of special characters by the linux shell.\n\n\n\n\n\n\n--logsene-tmp-dir\n  path\n\n\ndirectory to store buffered logs during network outage\n\n\n\n\n\n\n-u\n UDP_PORT\n\n\nstarts a syslogd UDP listener on the given port to act as syslogd\n\n\n\n\n\n\n-n\n name\n\n\nname for the source only when stdin is used, important to make multi-line patterns working on stdin because the status is tracked by the source name.\n\n\n\n\n\n\n--heroku\n PORT\n\n\nlistens for Heroku logs (http drain / framed syslog over http)\n\n\n\n\n\n\n--cfhttp\n PORT\n\n\nlistens for Cloud Foundry logs (syslog over http)\n\n\n\n\n\n\n--rtail-port\n\n\nforwards logs via UDP to \nrtail\n server\n\n\n\n\n\n\n--rtail-host\n hostname\n\n\nrtail\n server (UI for realtime logs), default: localhost\n\n\n\n\n\n\nlist of files\n\n\nEvery argument after the options list is interpreted as file name. All files in the file list (e.g. /var/log/*.log) are watched by \ntail-forever\n starting at end of file\n\n\n\n\n\n\n\n\nThe default output is line delimited JSON for parsed log lines, as long as no format options like -yml (YAML format), -p (pretty JSON), or -s (silent, no output to console) are specified. \n\n\nEnvironment variables\n\n\n\n\n\n\n\n\nVariable\n\n\nDescription\n\n\n\n\n\n\n\n\n\n\nLOGSENE_TMP_DIR\n\n\nDirectory to store failed bulk requests, for later re-transmission.\n\n\n\n\n\n\nLOGSENE_LOG_INTERVAL\n\n\nTime to batch logs before a bulk request is done. Default 10000 ms (10 seconds)\n\n\n\n\n\n\nLOGSENE_BULK_SIZE\n\n\nMaximum size of a bulk request. Default 1000.\n\n\n\n\n\n\nLOGSENE_URL\n\n\nURL for the Logsene receiver. For a local Elasticsearch server or for On-Premise version of Logsene. Defaults to Sematext Logsene SaaS receiver https://logsene-receiver.sematext.com/_bulk. Example for Elasticsearch: \nLOGSENE_URL=http://localhost:9200/_bulk\n\n\n\n\n\n\nHTTPS_PROXY\n\n\nProxy URL for HTTPS endpoints, like Logsene receiver. \nexport HTTPS_PROXY=http://my-proxy.example\n\n\n\n\n\n\nHTTP_PROXY\n\n\nProxy URL for HTTP endpoints (e.g. On-Premises or local Elasticsearch). \nexport HTTP_PROXY=http://my-proxy.example\n\n\n\n\n\n\nLOGAGENT_CONFIG\n\n\nFilename to read logagent CLI parameters from a file, defaults to \n`/etc/sematext/logagent.conf\n\n\n\n\n\n\n\n\nCommand Line Examples\n\n\n# Be Evil: parse all logs \n# stream logs to Logsene 1-Click ELK stack \nlogagent -t LOGSENE_TOKEN /var/log/*.log \n\n# Act as syslog server on UDP and forward messages to Logsene\nlogagent -u 514 -t LOGSENE_TOKEN  \n\n# Act as syslog server on UDP and write YAML formatted messages to console\nlogagent -u 514 -y  \n\n\n\n\nUse a \nglob\n pattern to build the file list \n\n\nlogagent -t LOGSENE_TOKEN -g \n{/var/log/*.log,/opt/myapp/*.log}\n \n\n\n\n\nWatch selective log output on console by passing logs via stdin and format in YAML\n\n\ntail -f /var/log/access.log | logagent -y \ntail -f /var/log/system.log | logagent -f my_own_patterns.yml  -y \n\n\n\n\nShip logs to rtail and Logsene to view logs in real-time in rtail and store logs in Logsene\n\n\n# rtail don't need to be installed, logagent uses the rtail protocol\nlogagent -t $LOGSENE_TOKEN --rtail-host myrtailserver --rtail-port 9999 /var/log/*.log\n\n\n\n\nLogagent can start the rtail web-server (in-process, saving memory), open browser with http://localhost:8080\n\n\n# logagent has no dependency to rtail, to keep the package small\nsudo npm i rtail -g\nlogagent -s -t $LOGSENE_TOKEN --rtail-web-port 8080 --rtail-port 9999 /var/log/*.log\n\n\n\n\nAnd of course you can combine rtail and Logagent in the traditional way, simply connect both via unix pipes. An example with rtail and Logsene storage and charts:", 
+            "text": "Command Line Parameters for logagent\n\n\nSynopsis\n\n\nlogagent [options] [file list]\n\n\n\n\n\n\n\n\nOptions\n\n\nDescription\n\n\n\n\n\n\n\n\n\n\n-f\n patterns.yml\n\n\nfile with pattern definitions\n\n\n\n\n\n\n-y\n\n\nprints parsed messages in YAML format to stdout\n\n\n\n\n\n\n-p\n\n\npretty json output\n\n\n\n\n\n\n-s\n\n\nsilent, print no logs, only throughput and memory usage on exit\n\n\n\n\n\n\n--print_stats\n\n\nprint processing stats in the given interval in seconds, e.g. \n--print_stats 30\n. Usefull with -s to see logagent activity on the console without printing the parsed logs.\n\n\n\n\n\n\n-t\n token\n\n\nLogsene\n App Token to insert parsed records into Logsene.\n\n\n\n\n\n\n-g\n glob-pattern\n\n\nuse a \nglob\n pattern to watch log files e.g. \n-g \"{/var/log/*.log,/Users/stefan/myapp/*.log}\"\n. The complete glob expression must be quoted, to avoid interpretation of special characters by the linux shell.\n\n\n\n\n\n\n--logsene-tmp-dir\n  path\n\n\ndirectory to store buffered logs during network outage\n\n\n\n\n\n\n-u\n UDP_PORT\n\n\nstarts a syslogd UDP listener on the given port to act as syslogd\n\n\n\n\n\n\n-n\n name\n\n\nname for the source only when stdin is used, important to make multi-line patterns working on stdin because the status is tracked by the source name.\n\n\n\n\n\n\n--heroku\n PORT\n\n\nlistens for Heroku logs (http drain / framed syslog over http)\n\n\n\n\n\n\n--cfhttp\n PORT\n\n\nlistens for Cloud Foundry logs (syslog over http)\n\n\n\n\n\n\n--rtail-port\n\n\nforwards logs via UDP to \nrtail\n server\n\n\n\n\n\n\n--rtail-host\n hostname\n\n\nrtail\n server (UI for realtime logs), default: localhost\n\n\n\n\n\n\nlist of files\n\n\nEvery argument after the options list is interpreted as file name. All files in the file list (e.g. /var/log/*.log) are watched by \ntail-forever\n starting at end of file\n\n\n\n\n\n\n\n\nThe default output is line delimited JSON for parsed log lines, as long as no format options like -yml (YAML format), -p (pretty JSON), or -s (silent, no output to console) are specified. \n\n\nEnvironment variables\n\n\n\n\n\n\n\n\nVariable\n\n\nDescription\n\n\n\n\n\n\n\n\n\n\nLOGSENE_TMP_DIR\n\n\nDirectory to store failed bulk requests, for later re-transmission.\n\n\n\n\n\n\nLOGSENE_LOG_INTERVAL\n\n\nTime to batch logs before a bulk request is done. Default 10000 ms (10 seconds)\n\n\n\n\n\n\nLOGSENE_BULK_SIZE\n\n\nMaximum size of a bulk request. Default 1000.\n\n\n\n\n\n\nLOGSENE_URL\n\n\nURL for the Logsene receiver. For a local Elasticsearch server or for On-Premise version of Logsene. Defaults to Sematext Logsene SaaS receiver https://logsene-receiver.sematext.com/_bulk. Example for Elasticsearch: \nLOGSENE_URL=http://localhost:9200/_bulk\n\n\n\n\n\n\nHTTPS_PROXY\n\n\nProxy URL for HTTPS endpoints, like Logsene receiver. \nexport HTTPS_PROXY=http://my-proxy.example\n\n\n\n\n\n\nHTTP_PROXY\n\n\nProxy URL for HTTP endpoints (e.g. On-Premises or local Elasticsearch). \nexport HTTP_PROXY=http://my-proxy.example\n\n\n\n\n\n\nLOGAGENT_CONFIG\n\n\nFilename to read logagent CLI parameters from a file, defaults to \n`/etc/sematext/logagent.conf\n\n\n\n\n\n\nPATTERN_MATCHING_ENABLED\n\n\nDefault is 'true'. The value 'false' disables parsing of logs.\n\n\n\n\n\n\n\n\nCommand Line Examples\n\n\n# Be Evil: parse all logs \n# stream logs to Logsene 1-Click ELK stack \nlogagent -t LOGSENE_TOKEN /var/log/*.log \n\n# Act as syslog server on UDP and forward messages to Logsene\nlogagent -u 514 -t LOGSENE_TOKEN  \n\n# Act as syslog server on UDP and write YAML formatted messages to console\nlogagent -u 514 -y  \n\n\n\n\nUse a \nglob\n pattern to build the file list \n\n\nlogagent -t LOGSENE_TOKEN -g \n{/var/log/*.log,/opt/myapp/*.log}\n \n\n\n\n\nWatch selective log output on console by passing logs via stdin and format in YAML\n\n\ntail -f /var/log/access.log | logagent -y \ntail -f /var/log/system.log | logagent -f my_own_patterns.yml  -y \n\n\n\n\nShip logs to rtail and Logsene to view logs in real-time in rtail and store logs in Logsene\n\n\n# rtail don't need to be installed, logagent uses the rtail protocol\nlogagent -t $LOGSENE_TOKEN --rtail-host myrtailserver --rtail-port 9999 /var/log/*.log\n\n\n\n\nLogagent can start the rtail web-server (in-process, saving memory), open browser with http://localhost:8080\n\n\n# logagent has no dependency to rtail, to keep the package small\nsudo npm i rtail -g\nlogagent -s -t $LOGSENE_TOKEN --rtail-web-port 8080 --rtail-port 9999 /var/log/*.log\n\n\n\n\nAnd of course you can combine rtail and Logagent in the traditional way, simply connect both via unix pipes. An example with rtail and Logsene storage and charts:", 
             "title": "Command Line Options"
@@ -116,3 +116,3 @@ },
             "location": "/cli-parameters/#environment-variables", 
-            "text": "Variable  Description      LOGSENE_TMP_DIR  Directory to store failed bulk requests, for later re-transmission.    LOGSENE_LOG_INTERVAL  Time to batch logs before a bulk request is done. Default 10000 ms (10 seconds)    LOGSENE_BULK_SIZE  Maximum size of a bulk request. Default 1000.    LOGSENE_URL  URL for the Logsene receiver. For a local Elasticsearch server or for On-Premise version of Logsene. Defaults to Sematext Logsene SaaS receiver https://logsene-receiver.sematext.com/_bulk. Example for Elasticsearch:  LOGSENE_URL=http://localhost:9200/_bulk    HTTPS_PROXY  Proxy URL for HTTPS endpoints, like Logsene receiver.  export HTTPS_PROXY=http://my-proxy.example    HTTP_PROXY  Proxy URL for HTTP endpoints (e.g. On-Premises or local Elasticsearch).  export HTTP_PROXY=http://my-proxy.example    LOGAGENT_CONFIG  Filename to read logagent CLI parameters from a file, defaults to  `/etc/sematext/logagent.conf", 
+            "text": "Variable  Description      LOGSENE_TMP_DIR  Directory to store failed bulk requests, for later re-transmission.    LOGSENE_LOG_INTERVAL  Time to batch logs before a bulk request is done. Default 10000 ms (10 seconds)    LOGSENE_BULK_SIZE  Maximum size of a bulk request. Default 1000.    LOGSENE_URL  URL for the Logsene receiver. For a local Elasticsearch server or for On-Premise version of Logsene. Defaults to Sematext Logsene SaaS receiver https://logsene-receiver.sematext.com/_bulk. Example for Elasticsearch:  LOGSENE_URL=http://localhost:9200/_bulk    HTTPS_PROXY  Proxy URL for HTTPS endpoints, like Logsene receiver.  export HTTPS_PROXY=http://my-proxy.example    HTTP_PROXY  Proxy URL for HTTP endpoints (e.g. On-Premises or local Elasticsearch).  export HTTP_PROXY=http://my-proxy.example    LOGAGENT_CONFIG  Filename to read logagent CLI parameters from a file, defaults to  `/etc/sematext/logagent.conf    PATTERN_MATCHING_ENABLED  Default is 'true'. The value 'false' disables parsing of logs.", 
             "title": "Environment variables"
@@ -119,0 +119,0 @@ },

No alert changes

Improved metrics

Total package byte prevSize

1101807

increased by0.07%

Lines of code

3773

increased by0.35%

Worsened metrics

Dependency count

16

increased by14.29%

Dependency changes

• + Addedsplit@^1.0.0

• + Addedthrough2@^2.0.1

• + Addedcore-util-is@1.0.3(transitive)

• + Addedisarray@1.0.0(transitive)

• + Addedprocess-nextick-args@2.0.1(transitive)

• + Addedreadable-stream@2.3.8(transitive)

• + Addedsafe-buffer@5.1.2(transitive)

• + Addedsplit@1.0.1(transitive)

• + Addedstring_decoder@1.1.1(transitive)

• + Addedthrough@2.3.8(transitive)

• + Addedthrough2@2.0.5(transitive)

• + Addedutil-deprecate@1.0.2(transitive)

• + Addedxtend@4.0.2(transitive)