aws-log-replay
Advanced tools
Comparing version 2.2.3-0 to 2.3.0
173
index.js
var request = require('requestretry'); | ||
var s3scan = require('s3scan'); | ||
var split = require('split'); | ||
var zlib = require('zlib'); | ||
var url = require('url'); | ||
var stream = require('stream'); | ||
var queue = require('queue-async'); | ||
var crypto = require('crypto'); | ||
var parallel = require('parallel-stream'); | ||
@@ -21,10 +18,10 @@ module.exports = {}; | ||
function cloudFrontDecode(path) { | ||
var whitelist = ['3C', '3E', '22', '23', '25', '7B', '7D', '7C', '5C', '5E', '7E', '5B', '5D', '60', '27', '20']; | ||
return path.replace(/%([\dA-F]{2})/g, function(match, hex) { | ||
var code = parseInt(hex, 16); | ||
if ((code < 32) || (code > 127) || (whitelist.indexOf(hex) !== -1)) | ||
return String.fromCharCode(code); | ||
else | ||
var whitelist = ['3C', '3E', '22', '23', '25', '7B', '7D', '7C', '5C', '5E', '7E', '5B', '5D', '60', '27', '20']; | ||
return path.replace(/%([\dA-F]{2})/g, function(match, hex) { | ||
var code = parseInt(hex, 16); | ||
if ((code < 32) || (code > 127) || (whitelist.indexOf(hex) !== -1)) | ||
return String.fromCharCode(code); | ||
else | ||
return match; | ||
}); | ||
}); | ||
} | ||
@@ -38,27 +35,27 @@ | ||
function GeneratePath(type) { | ||
var generatePath = new stream.Transform({ objectMode: true }); | ||
generatePath._transform = function(line, enc, callback) { | ||
if (!line) return callback(); | ||
if (Buffer.isBuffer(line)) line = line.toString('utf-8'); | ||
if (type.toLowerCase() == 'cloudfront') { | ||
var parts = line.split(/\s+/g); | ||
if (parts.length > 7) { | ||
if (parts[11] && parts[11] !== "-") { | ||
generatePath.push(cloudFrontDecode(parts[7] + "?" + parts[11])); | ||
} else { | ||
generatePath.push(cloudFrontDecode(parts[7])); | ||
} | ||
} | ||
} else if (type.toLowerCase() == 'lb') { | ||
if (line.indexOf('Amazon Route 53 Health Check Service') > -1) return callback(); | ||
var parts = line.split(/\s+/g); | ||
if (parts.length < 12) return callback(); | ||
var path = parts.length === 18 ? parts[12] : parts[13]; | ||
path = path.split(/:[0-9]\w+/g)[1]; | ||
if (!path) return callback(); | ||
generatePath.push(path); | ||
var generatePath = new stream.Transform({ objectMode: true }); | ||
generatePath._transform = function(line, enc, callback) { | ||
if (!line) return callback(); | ||
if (Buffer.isBuffer(line)) line = line.toString('utf-8'); | ||
if (type.toLowerCase() == 'cloudfront') { | ||
var parts = line.split(/\s+/g); | ||
if (parts.length > 7) { | ||
if (parts[11] && parts[11] !== '-') { | ||
generatePath.push(cloudFrontDecode(parts[7] + '?' + parts[11])); | ||
} else { | ||
generatePath.push(cloudFrontDecode(parts[7])); | ||
} | ||
callback(); | ||
}; | ||
return generatePath; | ||
} | ||
} else if (type.toLowerCase() == 'lb') { | ||
if (line.indexOf('Amazon Route 53 Health Check Service') > -1) return callback(); | ||
parts = line.split(/\s+/g); | ||
if (parts.length < 12) return callback(); | ||
var path = parts.length === 18 ? parts[12] : parts[13]; | ||
path = path.split(/:[0-9]\w+/g)[1]; | ||
if (!path) return callback(); | ||
generatePath.push(path); | ||
} | ||
callback(); | ||
}; | ||
return generatePath; | ||
} | ||
@@ -73,51 +70,33 @@ | ||
function RequestStream(options) { | ||
options = options || {}; | ||
if (!options.baseurl) throw new Error('options.baseurl should be an http:// or https:// baseurl for replay requests'); | ||
if (!options.hwm) options.hwm = 100; | ||
options = options || {}; | ||
if (!options.baseurl) throw new Error('options.baseurl should be an http:// or https:// baseurl for replay requests'); | ||
if (!options.hwm) options.hwm = 100; | ||
var starttime = Date.now(); | ||
var requestStream = new stream.Transform(options); | ||
requestStream._readableState.objectMode = true; | ||
requestStream.got = 0; | ||
requestStream.pending = 0; | ||
requestStream.queue = queue(options.hwm); | ||
requestStream.queue.awaitAll(function(err) { | ||
if (err) requestStream.emit('error', err); | ||
}); | ||
function transform(pathname, enc, callback) { | ||
if (this._closed) return setImmediate(callback); | ||
if (typeof pathname !== 'string') pathname = pathname.toString('utf8'); | ||
if (!pathname || pathname.indexOf('/') !== 0) return callback(); | ||
requestStream._transform = function(pathname, enc, callback) { | ||
if (typeof pathname !== 'string') pathname = pathname.toString('utf8'); | ||
if (!pathname || pathname.indexOf('/') !== 0) return callback(); | ||
var uri = url.parse(pathname, true); | ||
var requrl = options.baseurl + url.format(uri); | ||
var uri = url.parse(pathname, true); | ||
request({ | ||
agent: options.agent, | ||
encoding: null, | ||
uri: requrl | ||
}, (err, res, body) => { | ||
if (err) return callback(err); | ||
if (res.statusCode !== 200) return callback(); | ||
this.push({ url: requrl, body: body }); | ||
callback(); | ||
}); | ||
} | ||
if (requestStream.pending > (options.hwm * 1.5)) { | ||
return setImmediate(requestStream._transform.bind(requestStream), pathname, enc, callback); | ||
} | ||
var requestStream = parallel.transform(transform, { concurrency: options.hwm, objectMode: true }); | ||
requestStream.close = function() { | ||
requestStream._closed = true; | ||
}; | ||
var requrl = options.baseurl + url.format(uri); | ||
requestStream.pending++; | ||
requestStream.queue.defer(function(next) { | ||
request({ | ||
agent: options.agent, | ||
encoding: null, | ||
uri: requrl, | ||
time: true | ||
}, function(err, res, body) { | ||
requestStream.pending--; | ||
if (err) return requestStream.emit('error', err); | ||
if (res.statusCode !== 200) return next(); | ||
requestStream.push({ url: requrl, elapsedTime: res.elapsedTime, body: body }); | ||
requestStream.got++; | ||
next(); | ||
}); | ||
}); | ||
callback(); | ||
}; | ||
requestStream._flush = function(callback) { | ||
requestStream.queue.awaitAll(callback); | ||
}; | ||
return requestStream; | ||
return requestStream; | ||
} | ||
@@ -134,25 +113,25 @@ | ||
function SampleStream(options) { | ||
options = options || {}; | ||
if (!options.rate) throw new Error('must specify a sample rate (0 < sample < 1)'); | ||
if ((parseFloat(options.rate) <= 0) || (parseFloat(options.rate) >= 1)) throw new Error('rate must be between 0 and 1'); | ||
options = options || {}; | ||
if (!options.rate) throw new Error('must specify a sample rate (0 < sample < 1)'); | ||
if ((parseFloat(options.rate) <= 0) || (parseFloat(options.rate) >= 1)) throw new Error('rate must be between 0 and 1'); | ||
var sampleStream = new stream.Transform({ objectMode: true }); | ||
sampleStream.count = 0; | ||
sampleStream.threshold = Math.round(parseFloat(options.rate) * Math.pow(2, 16)); | ||
if (options.filter) sampleStream.filter = new RegExp(options.filter); | ||
sampleStream._transform = function(line, enc, callback) { | ||
if (!line) return callback(); | ||
var sampleStream = new stream.Transform({ objectMode: true }); | ||
sampleStream.count = 0; | ||
sampleStream.threshold = Math.round(parseFloat(options.rate) * Math.pow(2, 16)); | ||
if (options.filter) sampleStream.filter = new RegExp(options.filter); | ||
sampleStream._transform = function(line, enc, callback) { | ||
if (!line) return callback(); | ||
if ((sampleStream.filter) && (!sampleStream.filter.test(line))) | ||
return callback(); | ||
if ((sampleStream.filter) && (!sampleStream.filter.test(line))) | ||
return callback(); | ||
var hash = crypto.createHash('md5').update('cloudfront-log-read-salt-' + sampleStream.count).digest().readUInt16LE(0); | ||
if (hash < sampleStream.threshold) | ||
sampleStream.push(line); | ||
sampleStream.count++; | ||
var hash = crypto.createHash('md5').update('cloudfront-log-read-salt-' + sampleStream.count).digest().readUInt16LE(0); | ||
if (hash < sampleStream.threshold) | ||
sampleStream.push(line); | ||
sampleStream.count++; | ||
callback(); | ||
}; | ||
callback(); | ||
}; | ||
return sampleStream; | ||
return sampleStream; | ||
} |
{ | ||
"name": "aws-log-replay", | ||
"version": "2.2.3-0", | ||
"version": "2.3.0", | ||
"description": "Transform and replay CloudFront, ELB Classic, and ALB logs.", | ||
@@ -13,2 +13,3 @@ "main": "index.js", | ||
"minimist": "^1.2.0", | ||
"parallel-stream": "^1.1.2", | ||
"queue-async": "1.0.x", | ||
@@ -22,2 +23,3 @@ "requestretry": "1.5.x", | ||
"documentation": "2.0.x", | ||
"eslint": "^3.11.1", | ||
"tape": "3.5.x" | ||
@@ -31,4 +33,5 @@ }, | ||
"scripts": { | ||
"pretest": "eslint test bin index.js", | ||
"test": "tape test/*.test.js" | ||
} | ||
} |
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
New author
Supply chain riskA new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
No v1
QualityPackage is not semver >=1. This means it is not stable and does not support ^ ranges.
Found 1 instance in 1 package
1
11381
9
3
117
1
+ Addedparallel-stream@^1.1.2
+ Addedparallel-stream@1.1.2(transitive)