bunyan-stream-elasticsearch
Advanced tools
Comparing version 0.6.1 to 1.0.0
140
index.js
@@ -0,3 +1,4 @@ | ||
'use strict'; | ||
const Writable = require('stream').Writable; | ||
const util = require('util'); | ||
const elasticsearch = require('elasticsearch'); | ||
@@ -8,3 +9,3 @@ const moment = require('moment'); | ||
let levels = { | ||
const levels = { | ||
10: 'trace', | ||
@@ -23,3 +24,3 @@ 20: 'debug', | ||
function callOrString(value, entry) { | ||
if (typeof(value) === 'function') { | ||
if (typeof (value) === 'function') { | ||
return value(entry); | ||
@@ -51,87 +52,82 @@ } | ||
class ElasticsearchStream extends Writable { | ||
constructor(options) { | ||
super(options); | ||
options = options || {}; | ||
this._client = options.client || new elasticsearch.Client(options); | ||
this._type = options.type || 'logs'; | ||
let indexPattern = options.indexPattern || '[logstash-]YYYY.MM.DD'; | ||
this._index = options.index || generateIndexName.bind(null, indexPattern); | ||
this._writeCallback = options.writeCallback; | ||
this._template = !options.template || options.template === true ? defaultTemplate : options.template; | ||
function ElasticsearchStream(options) { | ||
options = options || {}; | ||
this._client = options.client || new elasticsearch.Client(options); | ||
this._type = options.type || 'logs'; | ||
let indexPattern = options.indexPattern || '[logstash-]YYYY.MM.DD'; | ||
this._index = options.index || generateIndexName.bind(null, indexPattern); | ||
this._writeCallback = options.writeCallback || undefined; | ||
this._template = options.template || defaultTemplate; | ||
// async | ||
this.initTemplate(options.index || indexPattern, this._template); | ||
} | ||
// async | ||
this.initTemplate(options.index || indexPattern, this._template); | ||
initTemplate(name, template) { | ||
if (template === false) | ||
return; | ||
Writable.call(this, options); | ||
} | ||
let tpl = generateRawTemplateName(name); | ||
util.inherits(ElasticsearchStream, Writable); | ||
template.template = tpl.template; | ||
return this._client.indices.putTemplate({ | ||
name: tpl.name, | ||
create: false, // can replace a previous one | ||
body: template | ||
}); | ||
} | ||
ElasticsearchStream.prototype.initTemplate = function (name, template) { | ||
if (template === false) | ||
return; | ||
_write(entry, encoding, callback) { | ||
const client = this._client; | ||
const index = this._index; | ||
const type = this._type; | ||
let tpl = generateRawTemplateName(name); | ||
const input = JSON.parse(entry.toString('utf8')); | ||
template.template = tpl.template; | ||
// Reassign these fields so them match what the default Kibana dashboard | ||
// expects to see. | ||
let output = { | ||
// The _timestamp field is deprecated. Instead, use a normal date field and set its value explicitly. | ||
'date': input.time, | ||
'level_int': input.level, | ||
'level': levels[input.level], | ||
'message': input.msg, | ||
'datestamp': moment(input.time).format('YYYY.MM.DD'), | ||
}; | ||
return this._client.indices.putTemplate({ | ||
name: tpl.name, | ||
create: false, // can be replace a previous one | ||
body: template | ||
}); | ||
}; | ||
// merge | ||
output = _.defaults(output, input); | ||
ElasticsearchStream.prototype._write = function (entry, encoding, callback) { | ||
delete output.msg; | ||
delete output.v; | ||
delete output.time; | ||
let client = this._client; | ||
let index = this._index; | ||
let type = this._type; | ||
if (input.err) { | ||
output.error = input.err; | ||
if (!output.message) output.message = output.error.message; | ||
} | ||
let input = JSON.parse(entry.toString('utf8')); | ||
if (this._writeCallback) { | ||
output = this._writeCallback(output, input) || output; | ||
} | ||
// Reassign these fields so them match what the default Kibana dashboard | ||
// expects to see. | ||
let output = { | ||
// The _timestamp field is deprecated. Instead, use a normal date field and set its value explicitly. | ||
'date': input.time, | ||
'level_int': input.level, | ||
'level': levels[input.level], | ||
'message': input.msg, | ||
'datestamp': moment(input.time).format('YYYY.MM.DD'), | ||
}; | ||
const options = { | ||
index: callOrString(index, entry), | ||
type: callOrString(type, entry), | ||
body: output | ||
}; | ||
// merge | ||
output = _.defaults(output, input); | ||
delete output.msg; | ||
delete output.v; | ||
delete output.time; | ||
if (input.err) { | ||
output.error = input.err; | ||
if (!output.message) output.message = output.error.message; | ||
const self = this; | ||
client.index(options, function (err) { | ||
if (err) { | ||
self.emit('error', err); | ||
} | ||
callback(); | ||
}); | ||
} | ||
} | ||
if (this._writeCallback) { | ||
this._writeCallback(output, input); | ||
} | ||
let options = { | ||
index: callOrString(index, entry), | ||
type: callOrString(type, entry), | ||
body: output | ||
}; | ||
let self = this; | ||
client.index(options, function (err) { | ||
if (err) { | ||
self.emit('error', err); | ||
} | ||
callback(); | ||
}); | ||
}; | ||
module.exports = ElasticsearchStream; |
{ | ||
"name": "bunyan-stream-elasticsearch", | ||
"version": "0.6.1", | ||
"version": "1.0.0", | ||
"description": "A Bunyan stream for sending log data to Elasticsearch with custom entry function", | ||
@@ -14,3 +14,3 @@ "main": "index.js", | ||
"engines": { | ||
"node": ">= 7.0.0" | ||
"node": ">= 4.0.0" | ||
}, | ||
@@ -24,2 +24,5 @@ "keywords": [ | ||
"author": "bloublou2014", | ||
"contributors": [ | ||
"BendingBender(https://github.com/BendingBender)" | ||
], | ||
"license": "MIT", | ||
@@ -31,13 +34,13 @@ "bugs": { | ||
"dependencies": { | ||
"elasticsearch": "^12.1.3", | ||
"elasticsearch": "^13.3.1", | ||
"lodash": "^4.17.4", | ||
"moment": "^2.17.1" | ||
"moment": "^2.19.1" | ||
}, | ||
"devDependencies": { | ||
"bluebird": "^3.4.7", | ||
"bluebird": "^3.5.1", | ||
"lodash": "^4.17.4", | ||
"mocha": "^3.2.0", | ||
"require-new": "^1.1.1", | ||
"should": "^11.2.0" | ||
"should": "^13.1.2" | ||
} | ||
} |
bunyan-stream-elasticsearch | ||
=========================== | ||
[![npm version](https://badge.fury.io/js/bloublou2014/bunyan-stream-elasticsearch.svg)](http://badge.fury.io/js/bunyan-stream-elasticsearch) | ||
[![npm version](https://badge.fury.io/js/bunyan-stream-elasticsearch.svg)](https://badge.fury.io/js/bunyan-stream-elasticsearch) | ||
[![Build Status](https://travis-ci.org/bloublou2014/bunyan-stream-elasticsearch.svg)](https://travis-ci.org/bloublou2014/bunyan-stream-elasticsearch) | ||
[![Dependency Status](https://david-dm.org/bloublou2014/bunyan-stream-elasticsearch.svg)](https://david-dm.org/bloublou2014/bunyan-stream-elasticsearch) | ||
[![Dev Dependency Status](https://david-dm.org/bloublou2014/bunyan-stream-elasticsearch/dev-status.svg)](https://david-dm.org/bloublou2014/bunyan-stream-elasticsearch#info=devDependencies) | ||
[![Dev Dependency Status](https://david-dm.org/bloublou2014/bunyan-stream-elasticsearch/dev-status.svg)](https://david-dm.org/bloublou2014/bunyan-stream-elasticsearch#type=dev) | ||
[![Known Vulnerabilities](https://snyk.io/test/github/bloublou2014/bunyan-stream-elasticsearch/badge.svg)](https://snyk.io/test/github/bloublou2014/bunyan-stream-elasticsearch) | ||
@@ -12,2 +12,4 @@ | ||
V1.0 requires ES6. | ||
## Install | ||
@@ -21,37 +23,36 @@ | ||
By default bunyan-stream-elasticsearch will create an index with a specific mapping template for your indexPattern. Template name will be 'template-logstash-' with default settings. | ||
If your index pattern is for example '[test-]YYYY.MM.DD[-pattern]', template name will be 'template-test--pattern'. Each time node app is starting, template is overwrite. | ||
You can disabled it via option `template:false` or overwrite it via `template:{es template}`. | ||
By default bunyan-stream-elasticsearch will create an index with a specific mapping template for your `indexPattern`. Template name will be `template-logstash-` with default settings. | ||
If your index pattern is for example `[test-]YYYY.MM.DD[-pattern]` the template name will be `template-test--pattern`. Each time an instance of this stream is created, the template will be overwritten. | ||
You can disable it by passing the option `template: false` or provide your own via `template: {elastic template}`. | ||
## Custom Write function | ||
## Custom Write Function | ||
You can add or modify elasticsearch document providing a `write(entry)` callback option. | ||
This allow a fine tuning on how document will be defined. Do not forget to override default template if you add new fields. | ||
This allows a fine tuning on how the document will be defined. Do not forget to override the default template if you add new fields. | ||
## Example | ||
``` | ||
let bunyan = require('bunyan'); | ||
let bunyanStreamElasticsearch = require('bunyan-elasticsearch'); | ||
```js | ||
const bunyan = require('bunyan'); | ||
const ElasticsearchStream = require('bunyan-stream-elasticsearch'); | ||
let writeEntryCallback = function(entry) { | ||
const writeCallback = entry => { | ||
// modify entry values | ||
entry.myProperty = 'my value'; | ||
return entry; | ||
}; | ||
let esStream = new bunyanStreamElasticsearch({ | ||
const esStream = new ElasticsearchStream({ | ||
indexPattern: '[logstash-]YYYY.MM.DD', | ||
type: 'logs', | ||
host: 'localhost:9200', | ||
defaultTemplate:true, | ||
writeCallback : writeEntryCallback | ||
defaultTemplate: true, | ||
writeCallback, | ||
}); | ||
// manage error case | ||
esStream.on('error', function (err) { | ||
console.log('Buyan Stream Elasticsearch Error:', err.stack); | ||
}); | ||
esStream.on('error', err => console.log('Buyan Stream Elasticsearch Error:', err.stack)); | ||
// Create the logger itself | ||
let logger = bunyan.createLogger({ | ||
const logger = bunyan.createLogger({ | ||
name: "My Application", | ||
@@ -77,5 +78,16 @@ streams: [ | ||
* `index` {string|function}: Elasticsearch index. Defaults to index generated using index pattern | ||
* `template` {json|boolean}: Elasticsearch Template to push to elasticseach at each start. if `false` no template will be pushed, if `{...}` will act as template remplacement. | ||
* `writeCallback` {function} : Custom write callback to modify entry before pushing it to Elasticsearch. | ||
* `template` {object|boolean}: Elasticsearch Template to push to elasticseach at each start. If `false` no template will be pushed, if `{...}` will act as template replacement. | ||
* `writeCallback` {function} : Custom write callback to modify the log entry before pushing it to Elasticsearch. | ||
Options `type` and `index` can be either a string or function. For these options, when the option is set to a function, the function is passed the log entry object as an argument | ||
## Contributors | ||
- [Bending Bender](https://github.com/BendingBender) | ||
## Changelog | ||
### 2017/10/22 | ||
- refactoring in ES6 class format | ||
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
No v1
QualityPackage is not semver >=1. This means it is not stable and does not support ^ ranges.
Found 1 instance in 1 package
147
0
91
31078
13
+ Addedagentkeepalive@2.2.0(transitive)
+ Addedelasticsearch@13.3.1(transitive)
+ Addedlodash@2.4.2(transitive)
+ Addedlodash.get@4.4.2(transitive)
+ Addedlodash.isempty@4.4.0(transitive)
+ Addedlodash.trimend@4.5.1(transitive)
- Removedasap@2.0.6(transitive)
- Removedelasticsearch@12.1.3(transitive)
- Removedforever-agent@0.6.1(transitive)
- Removedpromise@7.3.1(transitive)
Updatedelasticsearch@^13.3.1
Updatedmoment@^2.19.1