logagent-js - npm Package Compare versions

Comparing version 1.1.60 to 1.1.61

bin/logagent.js

@@ -57,2 +57,9 @@ #!/bin/sh
 var flat = require('flat')
+var logseneDiskBufferDir = argv['logsene-tmp-dir'] || process.env.LOGSENE_TMP_DIR || require('os').tmpdir()
+var mkpath = require('mkpath')
+mkpath(logseneDiskBufferDir, function (err) {
+  if (err) {
+    console.error('ERROR: create directory LOGSENE_TMP_DIR (' + logseneDiskBufferDir + '): ' + err.message)
+  }
+})
 var la = new LogAnalyzer(argv.f, {}, function () {
@@ -124,8 +131,11 @@ cli()
 
+  
+
 function getLogger (token, type) {
   var key = token + type
   // console.log(token)
+
   if (!loggers[key]) {
     var logger = new Logsene(token, type, null,
-      argv['logsene-tmp-dir'] || process.env.LOGSENE_TMP_DIR || require('os').tmpdir())
+      logseneDiskBufferDir)
     logger.on('log', function (data) {
@@ -132,0 +142,0 @@ // console.log(data)

docs/cli-parameters.md

@@ -22,2 +22,13 @@ ## Command Line Parameters for logagent
 
+## Environment variables
+|Variable|Description|
+|--------|-----------|
+|LOGSENE_TMP_DIR | Directory to store failed bulk requests, for later re-transmission.|
+|LOGSENE_LOG_INTERVAL | Time to collect logs before a bulk request is done. Default 10000 ms (10 seconds)|
+|LOGSENE_BULK_SIZE | Maximum size of a bulk request. Default 1000.|
+|LOGSENE_URL | URL for the Logsene receiver. For a local Elasticsearch server or for On-Premise version of Logsene. Defaults to Sematext Logsene SaaS receiver https://logsene-receiver.sematext.com/_bulk. Example for Elasticsearch: ```LOGSENE_URL=http://localhost:9200/_bulk```|
+|HTTPS_PROXY|Proxy URL for https endpoints, like Logsene receiver. ```export HTTPS_PROXY=http://my-proxy.example```|
+|HTTP_PROXY|Proxy URL for HTTP endpoints (e.g. On-Premises or local Elasticsearch). ```export HTTP_PROXY=http://my-proxy.example```|
+
+
 ## Command Line Examples 
@@ -29,5 +40,5 @@ ```
 # Act as syslog server on UDP and forward messages to Logsene
-logagent -t LOGSENE_TOKEN -u 1514 
-# Act as syslog server on UDP and write YAML formated messages to console
-logagent -u 1514 -y  
+logagent -u 514 -t LOGSENE_TOKEN  
+# Act as syslog server on UDP and write YAML formatted messages to console
+logagent -u 514 -y  
 ```
@@ -34,0 +45,0 @@

docs/index.md

@@ -5,7 +5,7 @@ [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy?template=https://github.com/sematext/logagent-js) - [read more](http://blog.sematext.com/2016/02/18/how-to-ship-heroku-logs-to-logsene-managed-elk-stack/)
 
-Smart Log Parser and Log Shipper written in Node. 
+Smart and lightweight Log Parser and Log Shipper written in Node. It can ship logs to Elasticsearch and thus also to [Logsene](http://www.sematext.com/logsene/). See [Documentation](http://sematext.github.io/logagent-js/).
 
 # Features
 
-This project contains a library and patterns for log parsing and cli tools and installers to use logagent-js as log shipper with following features: 
+This project contains a library and patterns for log parsing and cli tools and installers to use logagent-js as log shipper with the following features: 
 
@@ -15,4 +15,4 @@ ## Parser
 - pattern library included 
+- easy to extend with custom patterns and JS transform functions
 - recognition of Date and Number fields
-- easy to extend with custom patterns and JS transform functions
 - replace sensitive data with SHA-1 hash codes
@@ -23,4 +23,4 @@ - GeoIP lookup with automatic GeoIP db updates (maxmind geopip-lite files)
 
-- log format converter (e.g. text to JSON, line delimited JSON or YAML) 
-- Log shipper for [Logsene](http://www.sematext.com/logsene/)
+- log format converter (e.g. text to JSON, line delimited JSON or YAML)
+- log shipper for [Logsene](http://www.sematext.com/logsene/)
 
@@ -31,33 +31,29 @@ - including cli, launchd (Mac OS X), upstart and systemd (Linux) service installer
 ## Inputs
-- Standart input (stdin) taking the output stream from any Linux cli tool
-  - patterns are applied to each incomming text lines, including support for multi-line patters, e.g. for Java Stack Traces and JSON parser. 
-- Syslog Server (UDP) - reception of Syslog messages via UDP. The parser is applied to the message field. 
-- [Heroku Log Drain](https://github.com/sematext/logagent-js#logagent-as-heroku-log-drain)
-- CloudFoundry Log Drain
+- Standard input (stdin) that can read the output stream from any Linux cli tool
+  - patterns are applied to each incoming text line; includes support for multi-line patters, e.g. for Java Stack Traces and JSON input.
+- Syslog Server (UDP) listener - logagent-js can also act as a syslog server and receive Syslog messages via UDP. The parser is applied to the message field. 
+- [Heroku Log Drain](https://github.com/sematext/logagent-js#logagent-as-heroku-log-drain) makes it easy to ship Heroku logs to Elasticsearch or [Logsene](http://www.sematext.com/logsene/)
+- Cloud Foundry Log Drain
 
 ## Processing
-- logagent-js applies the patterns defined in ```patterns.yml``` to all logs to create structured output from plain text lines. Patterns are defined for input sources with regular expressions. The parsed logs can be post-processed with node.js transform function e.g. to enrich data or perform complex parser operations. 
-- GeoIP lookups for IP adress fields, including download and update of the GeoIP lite database from Maxmind
+- logagent-js applies patterns defined in patterns.yml to all logs and creates structured logs from plain-text log lines
+- GeoIP lookups for IP address fields, including automatic download and update of the GeoIP lite database from Maxmind
 
-## Security
-- Masking sensitive data - Logagent can relace field content with SHA-1 hash codes to mask sensitive data. The advantage of hash codes is that data is still searchable when you hash the original value before you start a search.  In addtion it is possible to exclude the original log line from shipping to avoid that sensitive data gets shipped in the field "originalLogLine".
-- Shipping logs to Logsene is done via https by default 
-- Support of proxy servers if the logging server is behind a firewall
-
 ## Reliable log shipping with disk buffer
 
-Logagent stores parsed logs to disk in case the network connection to the Elasticsearch API fails. Logagent retries to ship the logs later, when the network or Elasticsearch server is available again.  
+Logagent doesn't lose data.  It stores parsed logs to a disk buffer if the network connection to the Elasticsearch API fails.  Logagent retries shipping logs later, when the network or Elasticsearch is available again.  
 
 ## Outputs
 - bulk inserts to [Logsene](http://sematext.com/logsene) / Elasticsearch API
-- JSON, line delimited JSON and YML to stadard output  
+- JSON, line delimited JSON and YML to standard output  
 
 ## Deployment options
-- Deployable as system service: systemd, upstart (Linux) launchd (Mac OS X)setups 
+- Deployable as a system service: systemd, upstart (Linux), or launchd (Mac OS X)
 - Docker Container to receive logs via syslog
 - Deployement to Heroku as Heroku Log drain
+- Deployement to Cloud Foundry as Cloud Foundry Log drain (thus usable with Pivotal, Bluemix, etc.)
 
 ## API 
 - Node.js module to integrate parsers into Node.js programs
-- logagent-js is part of [SPM for Docker](https://github.com/sematext/spm-agent-docker) to parse Container Logs
+- logagent-js is a part of [SPM for Docker](https://github.com/sematext/sematext-agent-docker) to parse Container Logs
 
@@ -75,4 +71,4 @@
 
-- Twitter: [@sematext](http://www.twitter.com/sematext)
-- Blog: [blog.sematext.com](http://blog.sematext.com)
-- Homepage: [www.sematext.com](http://www.sematext.com)
+- Twitter: [@sematext](http://twitter.com/sematext)
+- Blog: [sematext.com/blog](http://sematext.com/blog)
+- Homepage: [sematext.com](http://sematext.com)

lib/index.js

@@ -174,3 +174,2 @@ /*
       } else {
-        console.log('MULTILINE!!!!!!!!!!!')
         this.sources[sourceName].reader = new MultiLine(/^\S+/, parser)
@@ -177,0 +176,0 @@ return this.sources[sourceName].reader

package.json

 {
   "name": "logagent-js",
-  "version": "1.1.60",
+  "version": "1.1.61",
   "description": "Smart log parser written in Node",
@@ -44,2 +44,3 @@ "main": "lib/index.js",
     "minimist": "^1.1.2",
+    "mkpath": "^1.0.0",
     "moment": "^2.10.3",
@@ -46,0 +47,0 @@ "prettyjson": "^1.1.2",

README.md

@@ -5,7 +5,7 @@ [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy?template=https://github.com/sematext/logagent-js) - [read more](http://blog.sematext.com/2016/02/18/how-to-ship-heroku-logs-to-logsene-managed-elk-stack/)
 
-Smart Log Parser and Log Shipper written in Node. See [Documentation](http://sematext.github.io/logagent-js/).
+Smart and lightweight Log Parser and Log Shipper written in Node. It can ship logs to Elasticsearch and thus also to [Logsene](http://www.sematext.com/logsene/). See [Documentation](http://sematext.github.io/logagent-js/).
 
 # Features
 
-This project contains a library and patterns for log parsing and cli tools and installers to use logagent-js as log shipper with following features: 
+This project contains a library and patterns for log parsing and cli tools and installers to use logagent-js as log shipper with the following features: 
 
@@ -15,4 +15,4 @@ ## Parser
 - pattern library included 
+- easy to extend with custom patterns and JS transform functions
 - recognition of Date and Number fields
-- easy to extend with custom patterns and JS transform functions
 - replace sensitive data with SHA-1 hash codes
@@ -23,4 +23,4 @@ - GeoIP lookup with automatic GeoIP db updates (maxmind geopip-lite files)
 
-- log format converter (e.g. text to JSON, line delimited JSON or YAML) 
-- Log shipper for [Logsene](http://www.sematext.com/logsene/)
+- log format converter (e.g. text to JSON, line delimited JSON or YAML)
+- log shipper for [Logsene](http://www.sematext.com/logsene/)
 
@@ -31,28 +31,29 @@ - including cli, launchd (Mac OS X), upstart and systemd (Linux) service installer
 ## Inputs
-- Standart input (stdin) taking the output stream from any Linux cli tool
-  - patterns are applied to each incomming text lines, including support for multi-line patters, e.g. for Java Stack Traces and JSON parser. 
-- Syslog Server (UDP) - reception of Syslog messages via UDP. The parser is applied to the message field. 
-- [Heroku Log Drain](https://github.com/sematext/logagent-js#logagent-as-heroku-log-drain)
-- CloudFoundry Log Drain
+- Standard input (stdin) that can read the output stream from any Linux cli tool
+  - patterns are applied to each incoming text line; includes support for multi-line patters, e.g. for Java Stack Traces and JSON input.
+- Syslog Server (UDP) listener - logagent-js can also act as a syslog server and receive Syslog messages via UDP. The parser is applied to the message field. 
+- [Heroku Log Drain](https://github.com/sematext/logagent-js#logagent-as-heroku-log-drain) makes it easy to ship Heroku logs to Elasticsearch or [Logsene](http://www.sematext.com/logsene/)
+- Cloud Foundry Log Drain
 
 ## Processing
-- logagent-js applies the patterns defined in ```patterns.yml' to all logs to create structured output from plain text lines
-- GeoIP lookups for IP adress fields, including download and update of the GeoIP lite database from Maxmind
+- logagent-js applies patterns defined in patterns.yml to all logs and creates structured logs from plain-text log lines
+- GeoIP lookups for IP address fields, including automatic download and update of the GeoIP lite database from Maxmind
 
 ## Reliable log shipping with disk buffer
 
-Logagent stores parsed logs to disk in case the network connection to the Elasticsearch API fails. Logagent retries to ship the logs later, when the network or Elasticsearch server is available again.  
+Logagent doesn't lose data.  It stores parsed logs to a disk buffer if the network connection to the Elasticsearch API fails.  Logagent retries shipping logs later, when the network or Elasticsearch is available again.  
 
 ## Outputs
 - bulk inserts to [Logsene](http://sematext.com/logsene) / Elasticsearch API
-- JSON, line delimited JSON and YML to stadard output  
+- JSON, line delimited JSON and YML to standard output  
 
 ## Deployment options
-- Deployable as system service: systemd, upstart (Linux) launchd (Mac OS X)setups 
+- Deployable as a system service: systemd, upstart (Linux), or launchd (Mac OS X)
 - Docker Container to receive logs via syslog
 - Deployement to Heroku as Heroku Log drain
+- Deployement to Cloud Foundry as Cloud Foundry Log drain (thus usable with Pivotal, Bluemix, etc.)
 
 ## API 
 - Node.js module to integrate parsers into Node.js programs
-- logagent-js is part of [SPM for Docker](https://github.com/sematext/spm-agent-docker) to parse Container Logs
+- logagent-js is a part of [SPM for Docker](https://github.com/sematext/sematext-agent-docker) to parse Container Logs
 
@@ -66,3 +67,3 @@
 
-## Preparation: Install Node.js 
+## Install Node.js 
 
@@ -79,12 +80,17 @@ Official Node.js [downloads and instructions](https://nodejs.org/en/download/).
 npm i -g logagent-js
+
 # Be Evil: parse all logs 
 # stream logs to Logsene 1-Click ELK stack 
 logagent -t LOGSENE_TOKEN /var/log/*.log 
-# Act as syslog server on UDP and write YAML formated messages to console
+
+# Act as syslog server on UDP and write YAML formatted messages to console
 logagent -u 514 -y  
+
 # Act as syslog server on UDP and forward messages to Logsene
-logagent -t LOGSENE_TOKEN -u 514 
+logagent -u 514 -t LOGSENE_TOKEN
+
 # Install the service (Linux, Mac OS X)
 sudo logagent-setup LOGSENE_TOKEN
-# Change CLI parameters to your needs
+
+# Adjust CLI parameters for your needs
 vi /etc/sematext/logagent.conf
@@ -103,4 +109,4 @@ ```
 
-- Twitter: [@sematext](http://www.twitter.com/sematext)
-- Blog: [blog.sematext.com](http://blog.sematext.com)
-- Homepage: [www.sematext.com](http://www.sematext.com)
+- Twitter: [@sematext](http://twitter.com/sematext)
+- Blog: [sematext.com/blog](http://sematext.com/blog)
+- Homepage: [sematext.com](http://sematext.com)

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Fixed alerts

Uses eval

Supply chain risk

Package uses dynamic code execution (e.g., eval()), which is a dangerous practice. This can prevent the code from running in certain environments and increases the risk that the code may contain exploits or malicious behavior.

Found 1 instance in 1 package

Dynamic require

Supply chain risk

Dynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.

Found 1 instance in 1 package

Minified code

Quality

This package contains minified code. This may be harmless in some cases where minified code is included in packaged libraries, however packages on npm should not minify code.

Found 1 instance in 1 package

Improved metrics

Number of low quality alerts

1

decreased by-50%

Number of lines in readme file

106

increased by7.07%

Number of low supply chain risk alerts

27

decreased by-10%

Number of medium supply chain risk alerts

8

decreased by-11.11%

Worsened metrics

Total package byte prevSize

92332

decreased by-91.53%

Dependency count

14

increased by7.69%

Number of package files

21

decreased by-64.41%

Lines of code

934

decreased by-74.98%

Dependency changes

• + Addedmkpath@^1.0.0