replacestream
Advanced tools
Comparing version 0.3.0 to 1.0.1
128
index.js
@@ -7,22 +7,30 @@ var through = require('through'); | ||
var totalMatches = 0; | ||
var isRegex = search instanceof RegExp; | ||
options = options || {}; | ||
options.limit = options.limit || Infinity; | ||
options.encoding = options.encoding || 'utf8'; | ||
options.regExpOptions = options.regExpOptions || 'gim'; | ||
options.max_match_len = options.max_match_len || 100; | ||
if (!isRegex) | ||
options.regExpOptions = options.regExpOptions || 'gmi'; | ||
var replaceFn = replace; | ||
if (typeof replace !== 'function') { | ||
replaceFn = function () { | ||
return replace; | ||
}; | ||
replaceFn = createReplaceFn(replace, isRegex); | ||
var match; | ||
if (isRegex) { | ||
match = matchFromRegex(search, options) | ||
} else { | ||
match = matchFromString(search, options); | ||
options.max_match_len = search.length; | ||
} | ||
var match = permuteMatch(search, options); | ||
function write(buf) { | ||
var matches; | ||
var lastPos = 0; | ||
var runningMatch = ''; | ||
var matchCount = 0; | ||
var rewritten = ''; | ||
var remaining = buf; | ||
var haystack = tail + buf.toString(options.encoding); | ||
@@ -36,15 +44,17 @@ tail = ''; | ||
var before = haystack.slice(lastPos, matches.index); | ||
var regexMatch = matches[0]; | ||
lastPos = matches.index + regexMatch.length; | ||
var regexMatch = matches; | ||
lastPos = matches.index + regexMatch[0].length; | ||
var dataToAppend = getDataToAppend(before,regexMatch); | ||
rewritten += dataToAppend; | ||
if (lastPos == haystack.length && regexMatch[0].length < options.max_match_len) { | ||
tail = regexMatch[0] | ||
} else { | ||
var dataToAppend = getDataToAppend(before,regexMatch); | ||
rewritten += dataToAppend; | ||
} | ||
} | ||
if (matchCount) | ||
remaining = haystack.slice(lastPos, haystack.length); | ||
else if (tail) | ||
remaining = haystack; | ||
if (tail.length < 1) | ||
tail = haystack.slice(lastPos) > options.max_match_len ? haystack.slice(lastPos).slice(0 - options.max_match_len) : haystack.slice(lastPos) | ||
var dataToQueue = getDataToQueue(matchCount,remaining,rewritten); | ||
var dataToQueue = getDataToQueue(matchCount,haystack,rewritten,lastPos); | ||
this.queue(dataToQueue); | ||
@@ -56,31 +66,22 @@ } | ||
if (tail) | ||
dataToAppend = tail + dataToAppend; | ||
totalMatches++; | ||
if (match.length < search.length) { | ||
tail = match; | ||
return dataToAppend; | ||
} | ||
dataToAppend += isRegex ? replaceFn(match) : replaceFn(match[0]); | ||
tail = ''; | ||
totalMatches++; | ||
dataToAppend += replaceFn(match); | ||
return dataToAppend; | ||
} | ||
function getDataToQueue(matchCount, remaining, rewritten) { | ||
var dataToQueue = remaining; | ||
function getDataToQueue(matchCount, haystack, rewritten, lastPos) { | ||
var dataToQueue; | ||
if (matchCount) { | ||
if ((tail.length + remaining.length) < search.length) { | ||
tail += remaining; | ||
return rewritten; | ||
if (matchCount > 0) { | ||
if (haystack.length > tail.length) { | ||
dataToQueue = rewritten + haystack.slice(lastPos, haystack.length - tail.length) | ||
} else { | ||
dataToQueue = rewritten | ||
} | ||
dataToQueue = rewritten +tail + remaining; | ||
} else { | ||
dataToQueue = haystack.slice(0, haystack.length - tail.length) | ||
} | ||
tail = ''; | ||
return dataToQueue; | ||
@@ -98,2 +99,27 @@ } | ||
function createReplaceFn(replace, isRegEx) { | ||
var regexReplaceFunction = function (match) { | ||
var newReplace = replace; | ||
// ability to us $1 with captures | ||
match.forEach(function(m, index) { | ||
newReplace = newReplace.replace('$' + index, m || '') | ||
}); | ||
return newReplace; | ||
}; | ||
var stringReplaceFunction = function () { | ||
return replace; | ||
}; | ||
if (isRegEx && !(replace instanceof Function)) { | ||
return regexReplaceFunction; | ||
} | ||
if (!(replace instanceof Function)) { | ||
return stringReplaceFunction | ||
} | ||
return replace | ||
} | ||
function escapeRegExp(s) { | ||
@@ -103,21 +129,17 @@ return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'); | ||
function permute(s) { | ||
var ret = []; | ||
var acc = ''; | ||
for (var i = 0, len = s.length; i < len; i++) { | ||
acc += s[i]; | ||
ret.push(acc); | ||
function matchFromRegex(s, options) { | ||
if (options.regExpOptions) { | ||
return new RegExp(s.source, options.regExpOptions) | ||
} else { | ||
var flags = s.toString().replace(/\/[^\/].*\//, '') | ||
// If there is no global flag then there can only be one match | ||
if (flags.indexOf('g') < 0) { | ||
options.limit = 1; | ||
} | ||
return new RegExp(s.source, flags) | ||
} | ||
return ret; | ||
} | ||
function permuteMatch(s, options) { | ||
var match = | ||
permute(s) | ||
.map(function (permute, i, arr) { | ||
return '(' + escapeRegExp(permute) + | ||
((i < arr.length - 1) ? '$' : '') + ')'; | ||
}) | ||
.join('|'); | ||
return new RegExp(match, options.regExpOptions); | ||
function matchFromString(s, options) { | ||
return new RegExp(escapeRegExp(s), options.regExpOptions); | ||
} |
{ | ||
"name": "replacestream", | ||
"version": "0.3.0", | ||
"version": "1.0.1", | ||
"description": "A node.js through stream that does basic streaming text search and replace and is chunk boundary friendly", | ||
@@ -5,0 +5,0 @@ "main": "index.js", |
104
README.md
@@ -94,2 +94,87 @@ # replacestream | ||
### Search and replace with Regular Expressions | ||
Here's the same example, but with RegEx: | ||
``` | ||
// happybirthday.txt | ||
Happy birthday to you! | ||
Happy birthday to you! | ||
Happy birthday to dear Liza! | ||
Happy birthday to you! | ||
``` | ||
``` js | ||
var replaceStream = require('replacestream') | ||
, fs = require('fs') | ||
, path = require('path'); | ||
// Replace any word that has an 'o' with 'oh' | ||
fs.createReadStream(path.join(__dirname, 'happybirthday.txt')) | ||
.pipe(replaceStream(/\w*o\w*/g, 'oh')) | ||
.pipe(process.stdout); | ||
``` | ||
Running this will print out: | ||
``` bash | ||
$ node simple.js | ||
Happy birthday oh oh! | ||
Happy birthday oh oh! | ||
Happy birthday oh dear Liza! | ||
Happy birthday oh oh! | ||
``` | ||
You can also insert captures using the $1 ($index) notation. This is similar the built in method [replace](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#Specifying_a_string_as_a_parameter). | ||
``` | ||
// happybirthday.txt | ||
Happy birthday to you! | ||
Happy birthday to you! | ||
Happy birthday to dear Liza! | ||
Happy birthday to you! | ||
``` | ||
``` js | ||
var replaceStream = require('replacestream') | ||
, fs = require('fs') | ||
, path = require('path'); | ||
// Replace any word that has an 'o' with 'oh' | ||
fs.createReadStream(path.join(__dirname, 'happybirthday.txt')) | ||
.pipe(replaceStream(/(dear) (Liza!)/, 'my very good and $1 friend $2')) | ||
.pipe(process.stdout); | ||
``` | ||
Running this will print: | ||
``` bash | ||
$ node simple.js | ||
Happy birthday to you! | ||
Happy birthday to you! | ||
Happy birthday to my very good and dear friend Liza! | ||
Happy birthday to you! | ||
``` | ||
You can also pass in a replacement function. The function will be passed the the match array as returned by the built-in method [exec](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/exec): | ||
``` js | ||
function replaceFn(match) { | ||
return match[2] + ' to ' + match[1] | ||
} | ||
fs.createReadStream(path.join(__dirname, 'happybirthday.txt')) | ||
.pipe(replaceStream(/(birt\w*)\sto\s(you)/g, replaceFn)) | ||
.pipe(process.stdout); | ||
``` | ||
Which would output: | ||
``` bash | ||
$ node simple.js | ||
Happy you to birthday! | ||
Happy you to birthday! | ||
Happy birthday to dear Liza! | ||
Happy you to birthday! | ||
``` | ||
### Web server search and replace over a test file | ||
@@ -151,1 +236,20 @@ | ||
By default the encoding will be set to 'utf8'. | ||
## Contributing | ||
replacestream is an **OPEN Open Source Project**. This means that: | ||
> Individuals making significant and valuable contributions are given commit-access to the project to contribute as they see fit. This project is more like an open wiki than a standard guarded open source project. | ||
See the [CONTRIBUTING.md](https://github.com/eugeneware/replacestream/blob/master/CONTRIBUTING.md) file for more details. | ||
### Contributors | ||
replacestream is only possible due to the excellent work of the following contributors: | ||
<table><tbody> | ||
<tr><th align="left">Eugene Ware</th><td><a href="https://github.com/eugeneware">GitHub/eugeneware</a></td></tr> | ||
<tr><th align="left">Ryan Mehta</th><td><a href="https://github.com/mehtaphysical">GitHub/mehtaphysical</a></td></tr> | ||
<tr><th align="left">Tim Chaplin</th><td><a href="https://github.com/tjchaplin">GitHub/tjchaplin</a></td></tr> | ||
<tr><th align="left">Romain</th><td><a href="https://github.com/Filirom1">GitHub/Filirom1</a></td></tr> | ||
</tbody></table> |
@@ -485,2 +485,501 @@ var expect = require('chai').expect | ||
}); | ||
it('should be able to replace within a chunk using regex', function (done) { | ||
var haystack = [ | ||
'<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </head>', | ||
' <body>', | ||
' <h1>Head</h1>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'); | ||
var acc = ''; | ||
var inject = script(fs.readFileSync('./test/fixtures/inject.js')); | ||
var replace = replaceStream(/<\/head>/, inject + '</head>'); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
expect(acc).to.include(inject); | ||
done(); | ||
}); | ||
replace.write(haystack); | ||
replace.end(); | ||
}); | ||
it('should be able to replace between chunks using regex', function (done) { | ||
var haystacks = [ | ||
[ '<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </head>', | ||
' <body>', | ||
' <h1>I love feeeee' | ||
].join('\n'), | ||
[ 'eeeeeeeeeed</h1>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'), | ||
]; | ||
var acc = ''; | ||
var replace = replaceStream(/fe+d/, 'foooooooood'); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
expect(acc).to.include('foooooooood'); | ||
done(); | ||
}); | ||
haystacks.forEach(function (haystack) { | ||
replace.write(haystack); | ||
}); | ||
replace.end(); | ||
}); | ||
it('should be able to handle no matches using regex', function (done) { | ||
var haystacks = [ | ||
[ '<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </de' | ||
].join('\n'), | ||
[ 'ad>', | ||
' <body>', | ||
' <h1>Head</h1>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'), | ||
]; | ||
var acc = ''; | ||
var inject = script(fs.readFileSync('./test/fixtures/inject.js')); | ||
var replace = replaceStream(/<\/head>/, inject + '</head>'); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
expect(acc).to.not.include(inject); | ||
done(); | ||
}); | ||
haystacks.forEach(function (haystack) { | ||
replace.write(haystack); | ||
}); | ||
replace.end(); | ||
}); | ||
it('should be able to handle dangling tails using regex', function (done) { | ||
var haystacks = [ | ||
[ '<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </he' | ||
].join('\n') | ||
]; | ||
var acc = ''; | ||
var inject = script(fs.readFileSync('./test/fixtures/inject.js')); | ||
var replace = replaceStream(/<\/head>/, inject + '</head>'); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
expect(acc).to.include('</he'); | ||
done(); | ||
}); | ||
haystacks.forEach(function (haystack) { | ||
replace.write(haystack); | ||
}); | ||
replace.end(); | ||
}); | ||
it('should be able to handle multiple searches and replaces using regex', | ||
function (done) { | ||
var haystacks = [ | ||
[ '<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </head>', | ||
' <body>', | ||
' <p> Hello 1</p>', | ||
' <p> Hello 2</' | ||
].join('\n'), | ||
[ 'p>', | ||
' <p> Hello 3</p>', | ||
' <p> Hello 4</p>', | ||
' <p> Hello 5</p>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'), | ||
]; | ||
var acc = ''; | ||
var inject = script(fs.readFileSync('./test/fixtures/inject.js')); | ||
var replace = replaceStream(/<\/p>/g, ', world</p>'); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
var expected = [ | ||
'<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </head>', | ||
' <body>', | ||
' <p> Hello 1, world</p>', | ||
' <p> Hello 2, world</p>', | ||
' <p> Hello 3, world</p>', | ||
' <p> Hello 4, world</p>', | ||
' <p> Hello 5, world</p>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'); | ||
expect(acc).to.equal(expected); | ||
done(); | ||
}); | ||
haystacks.forEach(function (haystack) { | ||
replace.write(haystack); | ||
}); | ||
replace.end(); | ||
}); | ||
it('should be able to handle a limited searches and replaces using regex', | ||
function (done) { | ||
var haystacks = [ | ||
[ '<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </head>', | ||
' <body>', | ||
' <p> Hello 1</p>', | ||
' <p> Hello 2</' | ||
].join('\n'), | ||
[ 'p>', | ||
' <p> Hello 3</p>', | ||
' <p> Hello 4</p>', | ||
' <p> Hello 5</p>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'), | ||
]; | ||
var acc = ''; | ||
var inject = script(fs.readFileSync('./test/fixtures/inject.js')); | ||
var replace = replaceStream(/<\/p>/g, ', world</p>', { limit: 3 }); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
var expected = [ | ||
'<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </head>', | ||
' <body>', | ||
' <p> Hello 1, world</p>', | ||
' <p> Hello 2, world</p>', | ||
' <p> Hello 3, world</p>', | ||
' <p> Hello 4</p>', | ||
' <p> Hello 5</p>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'); | ||
expect(acc).to.equal(expected); | ||
done(); | ||
}); | ||
haystacks.forEach(function (haystack) { | ||
replace.write(haystack); | ||
}); | ||
replace.end(); | ||
}); | ||
it('should be able to customize the regexp options using regex', | ||
function (done) { | ||
var haystacks = [ | ||
[ '<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </head>', | ||
' <body>', | ||
' <P> Hello 1</P>', | ||
' <P> Hello 2</' | ||
].join('\n'), | ||
[ 'P>', | ||
' <P> Hello 3</P>', | ||
' <p> Hello 4</p>', | ||
' <p> Hello 5</p>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'), | ||
]; | ||
var acc = ''; | ||
var inject = script(fs.readFileSync('./test/fixtures/inject.js')); | ||
var replace = replaceStream(/<\/P>/, ', world</P>', { regExpOptions: 'gm' }); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
var expected = [ | ||
'<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </head>', | ||
' <body>', | ||
' <P> Hello 1, world</P>', | ||
' <P> Hello 2, world</P>', | ||
' <P> Hello 3, world</P>', | ||
' <p> Hello 4</p>', | ||
' <p> Hello 5</p>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'); | ||
expect(acc).to.equal(expected); | ||
done(); | ||
}); | ||
haystacks.forEach(function (haystack) { | ||
replace.write(haystack); | ||
}); | ||
replace.end(); | ||
}); | ||
it('should replace characters specified and not modify partial matches using regex', function (done) { | ||
var haystack = [ | ||
'ab', | ||
'a', | ||
'a', | ||
'b' | ||
].join('\n'); | ||
var acc = ''; | ||
var replace = replaceStream('ab','Z'); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
var expected = [ | ||
'Z', | ||
'a', | ||
'a', | ||
'b' | ||
].join('\n'); | ||
expect(acc).to.equal(expected); | ||
done(); | ||
}); | ||
replace.write(haystack); | ||
replace.end(); | ||
}); | ||
it('should handle partial matches between complete matches using regex', function (done) { | ||
var haystack = [ | ||
"ab", | ||
'a', | ||
'ab', | ||
'b' | ||
].join('\n'); | ||
var acc = ''; | ||
var replace = replaceStream(/ab/g,'Z'); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
var expected = [ | ||
'Z', | ||
'a', | ||
'Z', | ||
'b' | ||
].join('\n'); | ||
expect(acc).to.equal(expected); | ||
done(); | ||
}); | ||
replace.write(haystack); | ||
replace.end(); | ||
}); | ||
it('should only replace characters specified using regex', function (done) { | ||
var haystack = [ | ||
'ab', | ||
'a', | ||
'b' | ||
].join('\n'); | ||
var acc = ''; | ||
var replace = replaceStream(/ab/,'Z'); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
var expected = [ | ||
'Z', | ||
'a', | ||
'b' | ||
].join('\n'); | ||
expect(acc).to.equal(expected); | ||
done(); | ||
}); | ||
replace.write(haystack); | ||
replace.end(); | ||
}); | ||
it('should be able to use a replace function using regex', function (done) { | ||
var haystacks = [ | ||
[ '<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </he' | ||
].join('\n'), | ||
[ 'ad>', | ||
' <body>', | ||
' <h1>Head</h1>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'), | ||
]; | ||
var acc = ''; | ||
var inject = script(fs.readFileSync('./test/fixtures/inject.js')); | ||
function replaceFn(match) { | ||
expect(match[0]).to.equal('</head>'); | ||
return inject + '</head>'; | ||
} | ||
var replace = replaceStream(/<\/head>/, replaceFn); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
expect(acc).to.include(inject); | ||
done(); | ||
}); | ||
haystacks.forEach(function (haystack) { | ||
replace.write(haystack); | ||
}); | ||
replace.end(); | ||
}); | ||
it('should be able to change each replacement value with a function using regex', | ||
function (done) { | ||
var haystacks = [ | ||
[ '<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </head>', | ||
' <body>', | ||
' <p> Hello 1</p>', | ||
' <p> Hello 2</' | ||
].join('\n'), | ||
[ 'p>', | ||
' <p> Hello 3</p>', | ||
' <p> Hello 4</p>', | ||
' <p> Hello 5</p>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'), | ||
]; | ||
var acc = ''; | ||
var greetings = ['Hi', 'Hey', 'Gday', 'Bonjour', 'Greetings']; | ||
function replaceFn(match) { | ||
return greetings.shift(); | ||
} | ||
var replace = replaceStream(/Hello/g, replaceFn); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
var expected = [ | ||
'<!DOCTYPE html>', | ||
'<html>', | ||
' <head>', | ||
' <title>Test</title>', | ||
' </head>', | ||
' <body>', | ||
' <p> Hi 1</p>', | ||
' <p> Hey 2</p>', | ||
' <p> Gday 3</p>', | ||
' <p> Bonjour 4</p>', | ||
' <p> Greetings 5</p>', | ||
' </body>', | ||
'</html>' | ||
].join('\n'); | ||
expect(acc).to.equal(expected); | ||
done(); | ||
}); | ||
haystacks.forEach(function (haystack) { | ||
replace.write(haystack); | ||
}); | ||
replace.end(); | ||
}); | ||
it('should be able to replace captures using $1 notation', function (done) { | ||
var haystack = [ | ||
"ab", | ||
'a', | ||
'ab', | ||
'b' | ||
].join('\n'); | ||
var acc = ''; | ||
var replace = replaceStream(/(a)(b)/g,'this is $1 and this is $2'); | ||
replace.on('data', function (data) { | ||
acc += data; | ||
}); | ||
replace.on('end', function () { | ||
var expected = [ | ||
'this is a and this is b', | ||
'a', | ||
'this is a and this is b', | ||
'b' | ||
].join('\n'); | ||
expect(acc).to.equal(expected); | ||
done(); | ||
}); | ||
replace.write(haystack); | ||
replace.end(); | ||
}) | ||
}); |
No v1
QualityPackage is not semver >=1. This means it is not stable and does not support ^ ranges.
Found 1 instance in 1 package
39811
13
1044
1
254