Comparing version 1.0.0-pre2 to 1.0.0-pre3
var split = require('split'); | ||
var through = require('through'); | ||
var Readable = require('stream').Readable; | ||
var PassThrough = require('stream').PassThrough; | ||
var logfmt = require('../logfmt'); | ||
@@ -10,28 +12,13 @@ exports = module.exports = function(options){ | ||
return function(req, res, next) { | ||
//setup | ||
//honor already parsed bodies | ||
if (req._body) return next(); | ||
//mime-type check | ||
var is_mime = req.header('content-type') === mime; | ||
if (!is_mime) return next(); | ||
req._body = true; | ||
req.body = new PassThrough({objectMode: true}); | ||
req.pipe(logfmt.streamParser()).pipe(req.body); | ||
//define Readable body Stream | ||
req.body = new Readable({ objectMode: true }); | ||
req.body._read = function(n) { | ||
req.body._paused = false; | ||
}; | ||
function parseLine(line) { | ||
if(line) { | ||
var parsedLine = options.parser(line); | ||
if(!req.body._paused) req.body._paused = !req.body.push(parsedLine); | ||
} | ||
} | ||
function end() { req.body.push(null); } | ||
function trimNewline(line) { | ||
return line.replace(/\r?\n/, ''); | ||
} | ||
req.pipe(split(/\b\r?\n\b/, trimNewline, null)).pipe(through(parseLine, end)); | ||
return next(); | ||
@@ -38,0 +25,0 @@ } |
var _ = require('lodash'); | ||
exports.log = function(data, stream) { | ||
this.stream = this.stream || process.stdout; | ||
if(stream == undefined) stream = this.stream; | ||
var logData = _.extend({}, this.defaultData, data); | ||
@@ -19,4 +21,7 @@ | ||
var label = label || 'elapsed'; | ||
var timer = new logfmt(this.stream, this.defaultData, | ||
{key: label, now: startTime}); | ||
var timer = new logfmt(); | ||
timer.stream = this.stream; | ||
timer.defaultData = this.defaultData; | ||
timer.timerKey = label; | ||
timer.timerNow = startTime; | ||
return timer; | ||
@@ -27,5 +32,8 @@ } | ||
var logfmt = require('../logfmt'); | ||
var namespaced = new logfmt() | ||
var namespace = _.extend({}, this.defaultData, object); | ||
var namespaced = new logfmt(this.stream, namespace, | ||
{key: this.timerKey, now: this.timerNow}); | ||
namespaced.stream = this.stream; | ||
namespaced.defaultData = namespace | ||
namespaced.timerKey = this.timerKey; | ||
namespaced.timerNow = this.timerNow; | ||
return namespaced; | ||
@@ -35,2 +43,3 @@ } | ||
exports.error = function(err, id) { | ||
this.maxErrorLines = this.maxErrorLines || 10; | ||
if (id === undefined) { | ||
@@ -42,5 +51,5 @@ id = Math.random().toString().slice(2, 12); | ||
for (var line in stack) { | ||
if (line >= (this.maxErrorLines || 10)) break; | ||
if (line >= this.maxErrorLines) break; | ||
this.log({ error:true, id:id, line:line, trace:stack[line] }); | ||
} | ||
} |
@@ -14,5 +14,5 @@ exports.stringify = function(data){ | ||
var needs_quoting = value.indexOf(' ') > -1 || value.indexOf('=') > -1; | ||
var needs_escaping = value.indexOf('"') > -1; | ||
var needs_escaping = value.indexOf('"') > -1 || value.indexOf("\\") > -1; | ||
if(needs_escaping) value = value.replace(/"/g, '\\"'); | ||
if(needs_escaping) value = value.replace(/["\\]/g, '\\$&'); | ||
if(needs_quoting) value = '"' + value + '"'; | ||
@@ -19,0 +19,0 @@ if(value === '' && !is_null) value = '""'; |
@@ -0,2 +1,8 @@ | ||
//constructor | ||
function logfmt() { | ||
} | ||
module.exports = logfmt; | ||
var _ = require('lodash'); | ||
var streaming = require('./lib/streaming'); | ||
var bodyParser = require('./lib/body_parser'); | ||
@@ -9,15 +15,5 @@ var bodyParserStream = require('./lib/body_parser_stream'); | ||
function logfmt(stream, defaultData, timer) { | ||
this.stream = stream || process.stdout; | ||
this.defaultData = defaultData || {}; | ||
if(timer){ | ||
this.timerKey = timer.key; | ||
this.timerNow = timer.now; | ||
} | ||
this.maxErrorLines = 10; | ||
} | ||
//Build up logfmt prototype | ||
_.extend(logfmt.prototype, logger); | ||
_.extend(logfmt.prototype, streaming); | ||
@@ -38,3 +34,3 @@ logfmt.prototype.stringify = serializer.stringify; | ||
var mime = options.contentType || "application/logplex-1"; | ||
return bodyParserStream({ contentType: mime, parser: this.parse }); | ||
return bodyParserStream({ contentType: mime }); | ||
}; | ||
@@ -49,3 +45,1 @@ | ||
_.extend(logfmt, logfmt.prototype); | ||
module.exports = logfmt; | ||
logfmt.call(module.exports) |
{ | ||
"name": "logfmt", | ||
"version": "1.0.0-pre2", | ||
"version": "1.0.0-pre3", | ||
"description": "key=value logger and parser", | ||
@@ -5,0 +5,0 @@ "main": "logfmt.js", |
421
README.md
@@ -10,5 +10,6 @@ # node-logfmt | ||
for logging objects to a stream in logfmt format. | ||
It provides a logfmt parser, a logging facility, | ||
and both streaming and non-streaming body parsers for express. | ||
It provides a logfmt parser, logfmt stringifier, a logging facility, | ||
and both streaming and non-streaming body parsers for express and restify. | ||
You should use this library if you're trying to write structured logs or | ||
@@ -23,33 +24,76 @@ if you're consuming them (especially if you're writing a logplex drain). | ||
## node.js | ||
The `logfmt` module is a singleton that works directly from require. | ||
```javascript | ||
var logfmt = require('logfmt'); | ||
logfmt.stringify({foo: 'bar'}); | ||
// 'foo=bar' | ||
logfmt.parse('foo=bar'); | ||
// {foo: 'bar'} | ||
``` | ||
Parse a line in logfmt format | ||
It is also a constructor function, so you can use `new logfmt` to create | ||
a new `logfmt` object so you can configure it differently. | ||
```javascript | ||
logfmt.parse('foo=bar a=14 baz="hello kitty"') | ||
//=> { foo: "bar", a: 14, baz: 'hello kitty'} | ||
var logfmt2 = new logfmt; | ||
// replace our stringify with JSON's | ||
logfmt2.stringify = JSON.stringify | ||
// now we log JSON! | ||
logfmt2.log({foo: 'bar'}) | ||
// {"foo":"bar"} | ||
// and the original logfmt is untouched | ||
logfmt.log({foo: 'bar'}) | ||
// foo=bar | ||
``` | ||
## command line | ||
### logfmt | ||
accepts lines on STDIN and converts them to json | ||
> echo "foo=bar a=14 baz=\"hello kitty\" cool%story=bro f %^asdf" | logfmt | ||
{ "foo": "bar", "a": 14, "baz": "hello kitty", "cool%story": "bro", "f": true, "%^asdf": true } | ||
### logfmt -r (reverse) | ||
accepts JSON on STDIN and converts them to logfmt | ||
> echo '{ "foo": "bar", "a": 14, "baz": "hello kitty", \ | ||
"cool%story": "bro", "f": true, "%^asdf": true }' | logfmt -r | ||
foo=bar a=14 baz="hello kitty" cool%story=bro f=true %^asdf=true | ||
round trips for free! | ||
> echo "foo=bar a=14 baz=\"hello kitty\" cool%story=bro f %^asdf" | logfmt | logfmt -r | logfmt | ||
{ "foo": "bar", "a": 14, "baz": "hello kitty", "cool%story": "bro", "f": true, "%^asdf": true } | ||
# API | ||
## stringifying | ||
Serialize an object to logfmt format | ||
```javascript | ||
logfmt.serialize({foo: "bar", a: 14, baz: 'hello kitty'}) | ||
//=> 'foo=bar a=14 baz="hello kitty"' | ||
``` | ||
### `logfmt.stringify(object)` | ||
Log an object to `logfmt.stream` (defaults to STDOUT) | ||
Serializes a single object. | ||
```javascript | ||
logfmt.log({foo: "bar", a: 14, baz: 'hello kitty'}) | ||
// foo=bar a=14 baz="hello kitty" | ||
//=> undefined | ||
logfmt.stringify({foo: "bar", a: 14, baz: 'hello kitty'}) | ||
//> 'foo=bar a=14 baz="hello kitty"' | ||
``` | ||
The `logfmt` module is a singleton that works directly from require. | ||
Because it is also a function, you can use the idiom `new logfmt` to create | ||
a new `logfmt` object. | ||
## parsing | ||
## parser | ||
Parse a line in logfmt format | ||
@@ -59,6 +103,4 @@ ### `logfmt.parse(string)` | ||
```javascript | ||
var logfmt = require('logfmt'); | ||
logfmt.parse("foo=bar a=14 baz=\"hello kitty\" cool%story=bro f %^asdf code=H12") | ||
//=>{ "foo": "bar", "a": '14', "baz": "hello kitty", "cool%story": "bro", "f": true, "%^asdf": true, "code" : "H12" } | ||
//> { "foo": "bar", "a": '14', "baz": "hello kitty", "cool%story": "bro", "f": true, "%^asdf": true, "code" : "H12" } | ||
``` | ||
@@ -70,6 +112,184 @@ | ||
## Streaming | ||
### `logfmt.streamStringify([options])` | ||
Pipe objects into the stream and it will write logfmt. | ||
You can customize the delimiter via the `options` object, which | ||
defaults to `\n` (newlines). | ||
```javascript | ||
var parseJSON = function(line) { | ||
if(!line) return; | ||
this.queue(JSON.parse(line.trim())) | ||
} | ||
process.stdin | ||
.pipe(split()) | ||
.pipe(through(parseJSON)) | ||
.pipe(logfmt.streamStringify()) | ||
.pipe(process.stdout) | ||
``` | ||
### `logfmt.streamParser()` | ||
Creates a streaming parser that will automatically split and parse incoming lines and | ||
emit javascript objects. | ||
```javascript | ||
process.stdin | ||
.pipe(logfmt.streamParser()) | ||
``` | ||
```javascript | ||
req.pipe(logfmt.streamParser()) | ||
``` | ||
#### Example | ||
Example command line of parsing logfmt and echoing JSON: | ||
```javascript | ||
var jsonOut = through(function(line){ | ||
this.queue(JSON.stringify(line)) | ||
}, function(){ | ||
this.queue(null) | ||
}) | ||
process.stdin | ||
.pipe(logfmt.streamParser()) | ||
.pipe(jsonOut) | ||
.pipe(process.stdout) | ||
``` | ||
Example HTTP request parsing logfmt and echoing JSON: | ||
``` | ||
server.post('/logs', function(req, res, next){ | ||
var jsonOut = through(function(line){ | ||
this.queue(JSON.stringify(line)) | ||
}, function(){ | ||
this.queue(null) | ||
}) | ||
req.pipe(logfmt.streamParser()) | ||
.pipe(jsonOut) | ||
.pipe(process.stdout); | ||
return next(); | ||
}) | ||
``` | ||
## express/restify parsing middleware | ||
```javascript | ||
// streaming | ||
app.use(logfmt.bodyParserStream()); | ||
// buffering | ||
app.use(logfmt.bodyParser()); | ||
``` | ||
#### `logfmt.bodyParserStream([opts])` | ||
Valid Options: | ||
- `contentType`: defaults to `application/logplex-1` | ||
If you use the `logfmt.bodyParserStream()` for a body parser, | ||
you will have a `req.body` that is a readable stream. | ||
Pipes FTW: | ||
```javascript | ||
var app = require('express')(); | ||
var http = require('http'); | ||
var through = require('through'); | ||
var logfmt = require('logfmt'); | ||
app.use(logfmt.bodyParserStream()); | ||
app.post('/logs', function(req, res){ | ||
if(!req.body) return res.send('OK'); | ||
req.body.pipe(through(function(line){ | ||
console.dir(line); | ||
})) | ||
res.send('OK'); | ||
}) | ||
http.createServer(app).listen(3000); | ||
``` | ||
Or you can just use the `readable` event: | ||
```javascript | ||
var app = require('express')(); | ||
var http = require('http'); | ||
var logfmt = require('logfmt'); | ||
app.use(logfmt.bodyParserStream()); | ||
// req.body is now a Readable Stream | ||
app.post('/logs', function(req, res){ | ||
req.body.on('readable', function(){ | ||
var parsedLine = req.body.read(); | ||
if(parsedLine) console.log(parsedLine); | ||
else res.send('OK'); | ||
}) | ||
}) | ||
http.createServer(app).listen(3000); | ||
``` | ||
### Non-Streaming | ||
#### `logfmt.bodyParser([opts])` | ||
Valid Options: | ||
- `contentType`: defaults to `application/logplex-1` | ||
If you use the `logfmt.bodyParser()` for a body parser, | ||
you will have a `req.body` that is an array of objects. | ||
```javascript | ||
var logfmt = require('logfmt'); | ||
app.use(logfmt.bodyParser()); | ||
// req.body is now an array of objects | ||
app.post('/logs', function(req, res){ | ||
console.log('BODY: ' + JSON.stringify(req.body)); | ||
req.body.forEach(function(data){ | ||
console.log(data); | ||
}); | ||
res.send('OK'); | ||
}) | ||
http.createServer(app).listen(3000); | ||
``` | ||
test it: | ||
```bash | ||
curl -X POST --header 'Content-Type: application/logplex-1' -d "foo=bar a=14 baz=\"hello kitty\" cool%story=bro f %^asdf" http://localhost:3000/logs | ||
``` | ||
## logging | ||
Uses the `logfmt.serialize` function to write the result to `logfmt.stream` | ||
Log an object to `logfmt.stream` (defaults to STDOUT) | ||
Uses the `logfmt.stringify` function to write the result to `logfmt.stream` | ||
```javascript | ||
logfmt.log({foo: "bar", a: 14, baz: 'hello kitty'}) | ||
//=> foo=bar a=14 baz="hello kitty" | ||
//> undefined | ||
``` | ||
### `logfmt.log(object, [stream])` | ||
@@ -80,3 +300,20 @@ | ||
```javascript | ||
logfmt.log({ "foo": "bar", "a": 14, baz: 'hello kitty'}) | ||
//=> foo=bar a=14 baz="hello kitty" | ||
``` | ||
#### customizing logging location | ||
`logfmt.log()` Accepts as 2nd argument anything that responds to `write(string)` | ||
```javascript | ||
var logfmt = require('logfmt'); | ||
logfmt.log({ "foo": "bar", "a": 14, baz: 'hello kitty'}, process.stderr) | ||
//=> foo=bar a=14 baz="hello kitty" | ||
``` | ||
Overwrite the default global location by setting `logfmt.stream` | ||
```javascript | ||
var logfmt = require('logfmt'); | ||
logfmt.stream = process.stderr | ||
logfmt.log({ "foo": "bar", "a": 14, baz: 'hello kitty'}) | ||
@@ -150,23 +387,8 @@ //=> foo=bar a=14 baz="hello kitty" | ||
### customizing logging location | ||
### `logfmt.error(error)` | ||
`logfmt.log()` and `logger.log()` Accepts as 2nd argument anything that responds to `write(string)` | ||
```javascript | ||
var logfmt = require('logfmt'); | ||
logfmt.log({ "foo": "bar", "a": 14, baz: 'hello kitty'}, process.stderr) | ||
//=> foo=bar a=14 baz="hello kitty" | ||
``` | ||
Accepts a Javascript `Error` object and converts it to logfmt format. | ||
Overwrite the default global location by setting `logfmt.stream` | ||
```javascript | ||
var logfmt = require('logfmt'); | ||
logfmt.stream = process.stderr | ||
It will print up to `logfmt.maxErrorLines` lines. | ||
logfmt.log({ "foo": "bar", "a": 14, baz: 'hello kitty'}) | ||
//=> foo=bar a=14 baz="hello kitty" | ||
``` | ||
### logging errors | ||
`logfmt.error()` Accepts a Javascript `Error` object and converts it to logfmt format | ||
```javascript | ||
@@ -244,122 +466,1 @@ var logfmt = require('logfmt'); | ||
``` | ||
## express/restify parsing middleware | ||
```javascript | ||
// streaming | ||
app.use(logfmt.bodyParserStream()); | ||
// buffering | ||
app.use(logfmt.bodyParser()); | ||
``` | ||
### Streaming | ||
#### `logfmt.bodyParserStream([opts])` | ||
Valid Options: | ||
- `contentType`: defaults to `application/logplex-1` | ||
If you use the `logfmt.bodyParserStream()` for a body parser, | ||
you will have a `req.body` that is a readable stream. | ||
Pipes FTW: | ||
```javascript | ||
var app = require('express')(); | ||
var http = require('http'); | ||
var through = require('through'); | ||
var logfmt = require('logfmt'); | ||
app.use(logfmt.bodyParserStream()); | ||
app.post('/logs', function(req, res){ | ||
if(!req.body) return res.send('OK'); | ||
req.body.pipe(through(function(line){ | ||
console.dir(line); | ||
})) | ||
res.send('OK'); | ||
}) | ||
http.createServer(app).listen(3000); | ||
``` | ||
Or you can just use the `readable` event: | ||
```javascript | ||
var app = require('express')(); | ||
var http = require('http'); | ||
var logfmt = require('logfmt'); | ||
app.use(logfmt.bodyParserStream()); | ||
// req.body is now a Readable Stream | ||
app.post('/logs', function(req, res){ | ||
req.body.on('readable', function(){ | ||
var parsedLine = req.body.read(); | ||
if(parsedLine) console.log(parsedLine); | ||
else res.send('OK'); | ||
}) | ||
}) | ||
http.createServer(app).listen(3000); | ||
``` | ||
### Non-Streaming | ||
#### `logfmt.bodyParser([opts])` | ||
Valid Options: | ||
- `contentType`: defaults to `application/logplex-1` | ||
If you use the `logfmt.bodyParser()` for a body parser, | ||
you will have a `req.body` that is an array of objects. | ||
```javascript | ||
var logfmt = require('logfmt'); | ||
app.use(logfmt.bodyParser()); | ||
// req.body is now an array of objects | ||
app.post('/logs', function(req, res){ | ||
console.log('BODY: ' + JSON.stringify(req.body)); | ||
req.body.forEach(function(data){ | ||
console.log(data); | ||
}); | ||
res.send('OK'); | ||
}) | ||
http.createServer(app).listen(3000); | ||
``` | ||
test it: | ||
```bash | ||
curl -X POST --header 'Content-Type: application/logplex-1' -d "foo=bar a=14 baz=\"hello kitty\" cool%story=bro f %^asdf" http://localhost:3000/logs | ||
``` | ||
## command line | ||
### logfmt | ||
accepts lines on STDIN and converts them to json | ||
> echo "foo=bar a=14 baz=\"hello kitty\" cool%story=bro f %^asdf" | logfmt | ||
{ "foo": "bar", "a": 14, "baz": "hello kitty", "cool%story": "bro", "f": true, "%^asdf": true } | ||
### logfmt -r (reverse) | ||
accepts JSON on STDIN and converts them to logfmt | ||
> echo '{ "foo": "bar", "a": 14, "baz": "hello kitty", "cool%story": "bro", "f": true, "%^asdf": true }' | logfmt -r | ||
foo=bar a=14 baz="hello kitty" cool%story=bro f=true %^asdf=true | ||
> echo "foo=bar a=14 baz=\"hello kitty\" cool%story=bro f %^asdf" | logfmt | logfmt -r | logfmt | ||
{ "foo": "bar", "a": 14, "baz": "hello kitty", "cool%story": "bro", "f": true, "%^asdf": true } |
var logfmt = require('../logfmt'), | ||
through = require('through'), | ||
stream = require('stream'), | ||
@@ -120,9 +121,10 @@ assert = require('assert'); | ||
var matches = [{path: '/'}, {foo: 'bar'}, {hello: 'kitty'}]; | ||
mockReq.body.on('readable', function(){ | ||
var data = mockReq.body.read(); | ||
mockReq.body.pipe(through(function(data){ | ||
assert.deepEqual(data, matches.pop()) | ||
if(matches.length == 0) done(); | ||
}) | ||
},function(){ | ||
done() | ||
})) | ||
}) | ||
}) |
@@ -24,10 +24,2 @@ var logfmt = require('../logfmt'), | ||
}) | ||
test('constructor accepts a stream', function(){ | ||
var stream = new OutStream; | ||
var logfmt2 = new logfmt(stream); | ||
var data = {foo: 'bar', a: 14} | ||
logfmt2.log(data); | ||
assert.equal("foo=bar a=14\n", stream.logline) | ||
}) | ||
}) |
@@ -25,2 +25,8 @@ var logfmt = require('../logfmt'), | ||
test("backslahes are restored", function(){ | ||
var data = {foo: 'why would you use \\LaTeX?'} | ||
assert.deepEqual(data, logfmt.parse(logfmt.stringify(data))) | ||
}) | ||
test("escaped strings are restored", function(){ | ||
@@ -27,0 +33,0 @@ var data = {foo: 'hello my "friend"'} |
@@ -18,6 +18,2 @@ var logfmt = require('../logfmt'), | ||
}) | ||
test('maxErrorLines is configured', function(){ | ||
assert.equal(10, logfmt.maxErrorLines); | ||
}) | ||
}) |
@@ -32,2 +32,7 @@ var logfmt = require('../logfmt'), | ||
test("escapes backslahes within strings", function(){ | ||
var data = {foo: 'why would you use \\LaTeX?'} | ||
assert.equal('foo="why would you use \\\\LaTeX?"', logfmt.stringify(data)) | ||
}) | ||
test("undefined is nothing", function(){ | ||
@@ -34,0 +39,0 @@ var data = {foo: undefined} |
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
61530
51
1424
460