logagent-js - npm Package Compare versions

Comparing version 1.1.20 to 1.1.21

app.json

 {
-  "name": "logagent-js",
-  "website": "https://sematext.com/logsene/",
-  "description": "Logagent as Log Drain. Setup: heroku drains:add https://mylogagent.herokuapp.com/LOGSENE_TOKEN -a myapp",
-  "repository": "https://github.com/sematext/logagent-js.git",
-  "logo": "https://node-js-sample.herokuapp.com/node.svg",
-  "keywords": ["node", "logs", "logging", "search", "solr", "lucene", "elasticsearch"]
-}
+    "name": "Logagent-js",
+    "description": "Ships Heroku Logs to Logsene / ELK. Setup: heroku drains:add https://mylogagent.herokuapp.com/LOGSENE_TOKEN -a myapp",
+    "website": "https://sematext.com/logsene/",
+    "repository": "https://github.com/sematext/logagent-js.git",
+    "logo": "https://pbs.twimg.com/profile_images/502764375774486528/mxRBQgzI.png",
+    "success_url": "/",
+    "keywords": [
+        "log management",
+        "log analytics",
+        "log shipper",
+        "log shipping",
+        "log agent",
+        "centralized logging",
+        "elk",
+        "kibana",
+        "log drain",
+        "elasticsearch",
+        "splunk alternative"
+    ],
+    "generator": "https://www.expeditedssl.com/heroku-button-maker"
+}

bin/logagent.js

@@ -6,3 +6,3 @@ #!/usr/bin/env node
  *
- * @licence logparser-js is free-to-use, proprietary software.
+ * @licence logagent-js is free-to-use, proprietary software.
  * THIS IS PROPRIETARY SOURCE CODE OF Sematext Group, Inc. (Sematext)
@@ -13,3 +13,2 @@ * This source code may not be copied, reverse engineered, or altered for any purpose.
  */
-
 var argv = require('minimist')(process.argv.slice(2))
@@ -34,2 +33,5 @@ var prettyjson = require('prettyjson')
 var WORKERS = process.env.WEB_CONCURRENCY || 1
+var dgram = require('dgram');
+var udpClient = dgram.createSocket('udp4') 
+var flat = require('flat')
 
@@ -88,5 +90,5 @@ process.on('beforeExit', function () {})
       log(err, msg)
-      data.ts = null
+      delete data.ts
       //delete data.ts
-      // data['_type'] = type
+      data['_type'] = type
       var msg = data
@@ -224,2 +226,3 @@ if (type === 'heroku') {
   }
+  
   if (argv.p) {
@@ -232,4 +235,28 @@ console.log(JSON.stringify(data, null, '\t'))
   }
+  if(argv['rtail-port']) {
+    var ts = data['@timestamp'] 
+    delete data['@timestamp']
+    delete data['originalLine']
+    delete data['ts']
+    var type = data['_type']
+    delete data['_type']
+    var message = new Buffer(JSON.stringify({
+      timestamp: ts || new Date(), 
+      content: data, //prettyJs(data),
+      id: type || argv.n || 'logs'
+    })) 
+    udpClient.send(message, 0, message.length, argv['rtail-port'], argv['rtail-host']||'localhost', function (err) {
+      // udpClient.close();
+    })
+  }
 }
 
+function prettyJs(o) {
+  var rv = ''
+  var f = flat(o)
+  Object.keys(f).forEach(function(key, i) {
+    rv += key + ': ' + f[key] + ' '
+  })
+  return rv
+}
 function parseLine (line, sourceName, cbf) {
@@ -251,2 +278,15 @@ bytes += line.length
 
+function rtailServer() {
+  // console.log(process.argv)
+  try {
+    process.argv = [process.argv[0], process.argv[1], '--web-port', String(argv['rtail-web-port'])]
+    require('rtail/cli/rtail-server.js')
+  } catch (err) {
+    console.log(err)
+    console.log('rtail is not installed. To start rtail server with logagent run:')
+    console.log('    npm i rtail -g')
+    setTimeout(process.exit, 300)
+  }
+}
+
 function terminate (reason) {
@@ -277,27 +317,36 @@ if (argv.heroku && reason !== 'exitWorker') {
 }
-if (argv.cfhttp) {
-  getHttpServer(argv.cfhttp, cloudFoundryHandler)
-}
-if (argv.heroku) {
-  throng(start, {
-    workers: WORKERS,
-    lifetime: Infinity
-  })
-}
-if (argv._.length > 0) {
-  // tail files
-  tailFiles(argv._)
-} else if (globPattern) {
-  // checks for file list and start tail for all files
-  console.log('using glob pattern: ' + globPattern)
-  tailFilesFromGlob(globPattern)
-} else if (argv.u) {
-  try {
-    getSyslogServer(logseneToken, argv.u)
-  } catch (err) {
-    console.error(err)
-    process.exit(-1)
+
+function cli() {
+  if (argv['rtail-web-port']) {
+    console.log('loading rtail')
+    rtailServer()
   }
-} else {
-  readStdIn()
+  if (argv.cfhttp) {
+    getHttpServer(argv.cfhttp, cloudFoundryHandler)
+  }
+  if (argv.heroku) {
+    throng(start, {
+      workers: WORKERS,
+      lifetime: Infinity
+    })
+  }
+  if (argv._.length > 0) {
+    // tail files
+    tailFiles(argv._)
+  } else if (globPattern) {
+    // checks for file list and start tail for all files
+    console.log('using glob pattern: ' + globPattern)
+    tailFilesFromGlob(globPattern)
+  } else if (argv.u) {
+    try {
+      getSyslogServer(logseneToken, argv.u)
+    } catch (err) {
+      console.error(err)
+      process.exit(-1)
+    }
+  } else {
+    readStdIn()
+  }
 }
+cli()
+

lib/index.js

@@ -153,2 +153,3 @@ /*
         }
+        
         return 1
@@ -173,8 +174,16 @@ }
   parseLine: function (line, source, cbf) {
-    var self = this
-    var br = self.getMultiLineReader(source, function (data) {
-      self._parseLine(data, source, cbf)
-    })
+    var br = this.getMultiLineReader(source, function (data) {
+      this._parseLine(data, source, cbf)
+    }.bind(this))
     br.add(line)
   },
+  globalTransform: function (source, parsed) {
+    if (this.cfg.globalTransform) {
+      try {
+        this.cfg.globalTransform(source, parsed)
+      } catch (ex) {
+        console.error('Error in gloabalTransform():' + ex)
+      }
+    } 
+  }, 
   _parseLine: function (line, source, cbf) {
@@ -197,2 +206,3 @@ if (line === null || line === '') {
           }
+          delete parsed.ts
         }
@@ -202,2 +212,3 @@ if (!parsed.message && parsed.msg) {
         }
+        this.globalTransform(source, parsed)
         return cbf(null, parsed)
@@ -208,12 +219,12 @@ } catch (ex) {
     }
-    var self = this
-    var patternList = self.getPatternsForSource(source) || this.patterns
+    var patternList = this.getPatternsForSource(source) || this.patterns
     for (var k in patternList) {
       var patterns = patternList[k]
       for (var i = 0; i < patterns.match.length; i++) {
-        if (self.matchPatterns(patterns.match[i], parsed, line)) {
-          self.bubbleUp(patternList, k)
+        if (this.matchPatterns(patterns.match[i], parsed, line)) {
+          this.bubbleUp(patternList, k)
           if (!parsed._type) {
             parsed._type = patternList[k].type
           }
+          this.globalTransform(source, parsed)
           return cbf(null, parsed)
@@ -220,0 +231,0 @@ }

lib/multiLine.js

@@ -11,14 +11,15 @@ 'use strict'
   this.tid = 0
-  var self = this
   if (delimiter) {
-    setInterval(function () {
-      if (self.lines.length > 0 && (new Date().getTime() - self.lastCall.getTime()) > self.opt.timeout) {
-        self.consumer(self.lines.join('\n'))
-        self.lines.length = 0
-        self.state = 0
+    setInterval(function lineTimeout() {
+      if (this.lines.length > 0 && (new Date().getTime() - this.lastCall.getTime()) > this.opt.timeout) {
+        this.consumer(this.lines.join('\n'))
+        this.lines.length = 0
+        this.state = 0
       }
-    }, 310)
+    }.bind(this), 310)
   }
 }
 
+MultiLine.prototype.intervalHandler = function () {}
+
 MultiLine.prototype.add = function (line) {
@@ -25,0 +26,0 @@ if (!this.opt.delimiter) {

package.json

 {
   "name": "logagent-js",
-  "version": "1.1.20",
+  "version": "1.1.21",
   "description": "Smart log parser written in Node",
@@ -11,3 +11,3 @@ "main": "lib/index.js",
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "node bin/logagent.js --heroku $PORT"
+    "start": "node bin/logagent.js -s --heroku $PORT"
   },
@@ -34,2 +34,3 @@ "repository": {
   "dependencies": {
+    "flat": "^2.0.0",
     "glob": "^5.0.14",
@@ -36,0 +37,0 @@ "js-yaml": "^3.3.1",

README.md

@@ -1,2 +0,2 @@
-[![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy?template=https://github.com/sematext/logagent-js)
+[![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy?template=https://github.com/sematext/logagent-js) - [read more](http://blog.sematext.com/2016/02/18/how-to-ship-heroku-logs-to-logsene-managed-elk-stack/)
 
@@ -43,61 +43,3 @@ # logagent-js
 
-# Pattern definitions
 
-The default pattern definition file include already patterns for:
-- MongoDB, 
-  - MySQL, 
-  - Nginx, 
-  - Redis, 
-  - Elasticsearch
-  - Apache 
-    - Webserver (httpd), 
-    - Zookeeper, 
-    - Cassandra, 
-    - Kafka,
-    - HBase HDFS Data Node,
-    - HBase Region Server,
-    - Hadoop YARN Node Manager, 
-    - Apache SOLR,
-  - various Linux/Mac OS X system log files 
-
-The file format is based on [JS-YAML](https://nodeca.github.io/js-yaml/), in short:
-
-- - indicates an  array
-- !js/regexp - indicates a JS regular expression
-- !!js/function > - indicates a JS function 
-
-Properties:
-- patterns - the list of patterns, each pattern starts with "-"
-- match: A group of patterns for a specific log source
-- regex: a JS regular expression 
-- fields: the field list of extracted match groups from the regex
-- type: the type used in Logsene (Elasticsearch Mapping)
-- dateFormat: the format of the special fields 'ts', if the date format matches, a new field @timestamp is generated
-- transform: a JS function to manipulate the result of regex and date parsing
-
-Example:
-
-```
-# Sensitive data can be replaced with a hashcode (sha1)
-# it applies to fields matching the field names by a regular expression
-# Note: this function is not optimized (yet) and might take 10-15% of performance
-# autohash: !!js/regexp /user|client_ip|password|email|credit_card_number|payment_info/i
-
-patterns: 
-  - # APACHE  Web Logs
-  sourceName: httpd
-  match: 
-    # Common Log Format
-    - regex:        !!js/regexp /([0-9a-f.:]+)\s+(-|.+?)\s+(-|.+?)\s+\[([0-9]{2}\/[a-z]{3}\/[0-9]{4}\:[0-9]{2}:[0-9]{2}:[0-9]{2}[^\]]*)\] \"(\S+?)\s(\S*?)\s{0,1}(\S+?)\" ([0-9|\-]+) ([0-9|\-]+)/i
-      type: apache_access_common
-      fields:       [client_ip,remote_id,user,ts,method,path,http_version,status_code,size]
-      dateFormat: DD/MMM/YYYY:HH:mm:ss ZZ
-      transform: !!js/function >
-        function (p) {
-          p.message = p.method + ' ' + p.path
-        }
-```
-
-The default patterns are [here](/patterns.yml) - contributions are welcome.
-
 # Use logagent-js in Node
@@ -131,6 +73,6 @@
 
+Official Node.js [downloads and instructions](https://nodejs.org/en/download/).
+E.g. for Debian/Ubuntu:
 ```
-# Note the new setup script name for Node.js v0.12
-curl -sL https://deb.nodesource.com/setup_0.12 | sudo bash -
-# Then install with:
+curl -sL https://deb.nodesource.com/setup_4.x | sudo -E bash -
 sudo apt-get install -y nodejs
@@ -142,3 +84,3 @@ ```
 npm i -g logagent-js
-# ship all your logs to logsene, parsed, timestamped - displyed on console in YAML format (-y)
+# ship all your logs to Logsene, parsed with timestamps - output on console in YAML format (-y)
 logagent -t LOGSENE_TOKEN -y /var/log/*.log
@@ -149,14 +91,16 @@ ```
 
-- -f file with pattern definitions 
-- -y prints parsed messages in YAML format
-- -p pretty json output
-- -s silent, print only throughput on exit
-- -t token [Logsene](http://sematext.com/logsene) App Token to insert parsed records into Logsene
-- -g use a [glob](https://www.npmjs.com/package/glob) pattern to watch log files e.g. -g "{/var/log/*.log,/Users/stefan/*/*.log}" 
-- -u UDP_PORT starts a syslogd UDP listener on the given port to act as syslogd
-- -n name for the source only when stdin is used (e.g. cat zookeeper.log | logagent -n zookeeper), important to make
+- __-f__ file with pattern definitions 
+- __-y__ prints parsed messages in YAML format
+- __-p__ pretty json output
+- __-s__ silent, print no logss, only throughput and memory usage on exit
+- __-t__ token [Logsene](http://sematext.com/logsene) App Token to insert parsed records into Logsene.
+- __-g__ use a [glob](https://www.npmjs.com/package/glob) pattern to watch log files e.g. -g "{/var/log/*.log,/Users/stefan/*/*.log}" 
+- __-u__ UDP_PORT starts a syslogd UDP listener on the given port to act as syslogd
+- __-n__ name for the source only when stdin is used (e.g. cat zookeeper.log | logagent -n zookeeper), important to make
   multi-line patterns working on stdin because the status is tracked by the source name. 
-- --heroku PORT listens for heroku logs (http drain / framed syslog over http) 
-- --cfhttp PORT listens for CloudFoundry logs (syslog over http)
-- list of files, watched by tail-forver starting at end of file to watch
+- __--heroku__ PORT listens for heroku logs (http drain / framed syslog over http) 
+- __--cfhttp__ PORT listens for CloudFoundry logs (syslog over http)
+- __--rtail-port__ forwards logs via udp to [rtail](http://rtail.org/) server 
+- __--rtail-host__ hostname [rtail](http://rtail.org/) server (UI for realtime logs), default: localhost
+- __list of files__, e.g. /var/log/*.log watched by tail-forver starting at end of file to watch
 
@@ -188,2 +132,20 @@ The default output is line delimited JSON.
 ```
+
+Ship logs to rtail and Logsene to view logs in real-time in rtail and store logs in Logsene
+
+```
+# rtail don't need to be installed, logagent uses the rtail protocol
+logagent -t $LOGSENE_TOKEN --rtail-host myrtailserver --rtail-port 9999 /var/log/*.log
+```
+
+Logagent can start the rtail web-server (in-process, saving memory), open browser with http://localhost:8080
+```
+# logagent has no dependency to rtail, to keep the package small
+sudo npm i rtail -g
+logagent -s -t $LOGSENE_TOKEN --rtail-web-port 8080 --rtail-port 9999 /var/log/*.log
+```
+
+And of course you can combine rtail and Logagent in the traditional way, simply connect both via unix pipes. An example with rtail and Logsene storage and charts:
+![](http://g.recordit.co/usjLitb3Dd.gif)
+
 # Logagent as Heroku log drain
@@ -279,2 +241,62 @@
 
+# Pattern definitions
+
+The default pattern definition file include already patterns for:
+- MongoDB, 
+  - MySQL, 
+  - Nginx, 
+  - Redis, 
+  - Elasticsearch
+  - Apache 
+    - Webserver (httpd), 
+    - Zookeeper, 
+    - Cassandra, 
+    - Kafka,
+    - HBase HDFS Data Node,
+    - HBase Region Server,
+    - Hadoop YARN Node Manager, 
+    - Apache SOLR,
+  - various Linux/Mac OS X system log files 
+
+The file format is based on [JS-YAML](https://nodeca.github.io/js-yaml/), in short:
+
+- - indicates an  array
+- !js/regexp - indicates a JS regular expression
+- !!js/function > - indicates a JS function 
+
+Properties:
+- patterns - the list of patterns, each pattern starts with "-"
+- match: A group of patterns for a specific log source
+- regex: a JS regular expression 
+- fields: the field list of extracted match groups from the regex
+- type: the type used in Logsene (Elasticsearch Mapping)
+- dateFormat: the format of the special fields 'ts', if the date format matches, a new field @timestamp is generated
+- transform: a JS function to manipulate the result of regex and date parsing
+
+Example:
+
+```
+# Sensitive data can be replaced with a hashcode (sha1)
+# it applies to fields matching the field names by a regular expression
+# Note: this function is not optimized (yet) and might take 10-15% of performance
+# autohash: !!js/regexp /user|client_ip|password|email|credit_card_number|payment_info/i
+
+patterns: 
+  - # APACHE  Web Logs
+  sourceName: httpd
+  match: 
+    # Common Log Format
+    - regex:        !!js/regexp /([0-9a-f.:]+)\s+(-|.+?)\s+(-|.+?)\s+\[([0-9]{2}\/[a-z]{3}\/[0-9]{4}\:[0-9]{2}:[0-9]{2}:[0-9]{2}[^\]]*)\] \"(\S+?)\s(\S*?)\s{0,1}(\S+?)\" ([0-9|\-]+) ([0-9|\-]+)/i
+      type: apache_access_common
+      fields:       [client_ip,remote_id,user,ts,method,path,http_version,status_code,size]
+      dateFormat: DD/MMM/YYYY:HH:mm:ss ZZ
+      transform: !!js/function >
+        function (p) {
+          p.message = p.method + ' ' + p.path
+        }
+```
+
+The default patterns are [here](/patterns.yml) - contributions are welcome.
+
+
 # Related packages
@@ -281,0 +303,0 @@

New alerts

Network access

Supply chain risk

This module accesses the network.

Found 1 instance in 1 package

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

64852

increased by6.43%

Lines of code

616

increased by13.03%

Number of lines in readme file

313

increased by7.56%

Worsened metrics

Dependency count

11

increased by10%

Number of medium supply chain risk alerts

3

increased by200%

Dependency changes

• + Addedflat@^2.0.0

• + Addedflat@2.0.2(transitive)

• + Addedis-buffer@1.1.6(transitive)