You're Invited:Meet the Socket Team at BlackHat and DEF CON in Las Vegas, Aug 4-6.RSVP →
Socket
Book a DemoInstallSign in
Socket
Book a DemoInstallSign in

template-replace-stream

Package Overview
Dependencies
Maintainers
1
Versions
6
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

template-replace-stream - npm Package Compare versions

Comparing version

2.1.1
to
2.1.2

178

dist/index.js

@@ -5,20 +5,3 @@ "use strict";

var __getOwnPropNames = Object.getOwnPropertyNames;
var __getOwnPropSymbols = Object.getOwnPropertySymbols;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __propIsEnum = Object.prototype.propertyIsEnumerable;
var __knownSymbol = (name, symbol) => {
return (symbol = Symbol[name]) ? symbol : Symbol.for("Symbol." + name);
};
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
var __spreadValues = (a, b) => {
for (var prop in b || (b = {}))
if (__hasOwnProp.call(b, prop))
__defNormalProp(a, prop, b[prop]);
if (__getOwnPropSymbols)
for (var prop of __getOwnPropSymbols(b)) {
if (__propIsEnum.call(b, prop))
__defNormalProp(a, prop, b[prop]);
}
return a;
};
var __export = (target, all) => {

@@ -37,32 +20,9 @@ for (var name in all)

var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var __async = (__this, __arguments, generator) => {
return new Promise((resolve, reject) => {
var fulfilled = (value) => {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
};
var rejected = (value) => {
try {
step(generator.throw(value));
} catch (e) {
reject(e);
}
};
var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
step((generator = generator.apply(__this, __arguments)).next());
});
};
var __forAwait = (obj, it, method) => (it = obj[__knownSymbol("asyncIterator")]) ? it.call(obj) : (obj = obj[__knownSymbol("iterator")](), it = {}, method = (key, fn) => (fn = obj[key]) && (it[key] = (arg) => new Promise((yes, no, done) => (arg = fn.call(obj, arg), done = arg.done, Promise.resolve(arg.value).then((value) => yes({ value, done }), no)))), method("next"), method("return"), it);
// src/index.ts
var src_exports = {};
__export(src_exports, {
// index.ts
var template_replace_stream_exports = {};
__export(template_replace_stream_exports, {
TemplateReplaceStream: () => TemplateReplaceStream
});
module.exports = __toCommonJS(src_exports);
// src/template-replace-stream.ts
module.exports = __toCommonJS(template_replace_stream_exports);
var import_node_stream = require("stream");

@@ -78,2 +38,10 @@ var DEFAULT_OPTIONS = {

var TemplateReplaceStream = class extends import_node_stream.Transform {
_stack = Buffer.alloc(0);
_state = 0 /* SEARCHING_START_PATTERN */;
_matchCount = 0;
_stackIndex = 0;
_startPattern;
_endPattern;
_resolveVariable;
_options;
/**

@@ -89,3 +57,3 @@ * Creates a new instance of the {@link TemplateReplaceStream}.

constructor(variables, options = {}) {
const _options = __spreadValues(__spreadValues({}, DEFAULT_OPTIONS), options);
const _options = { ...DEFAULT_OPTIONS, ...options };
if (_options.maxVariableNameLength <= 0) {

@@ -99,6 +67,2 @@ throw new Error("The maximum variable name length must be greater than 0");

super(_options.streamOptions);
this._stack = Buffer.alloc(0);
this._state = 0 /* SEARCHING_START_PATTERN */;
this._matchCount = 0;
this._stackIndex = 0;
this._options = _options;

@@ -109,46 +73,44 @@ this._startPattern = this.toBuffer(_options.startPattern);

}
_transform(chunk, encoding, callback) {
return __async(this, null, function* () {
if (typeof chunk === "string")
chunk = Buffer.from(chunk, encoding);
try {
if (chunk instanceof Buffer) {
if (this._stack.length === 0) {
this._stack = chunk;
} else {
this._stack = Buffer.concat([this._stack, chunk]);
}
while (this._stackIndex < this._stack.length) {
switch (this._state) {
case 0 /* SEARCHING_START_PATTERN */:
this.findStartPattern();
this.releaseStack(this._stackIndex - this._matchCount);
break;
case 1 /* PROCESSING_VARIABLE */:
this.findVariableEnd();
break;
case 2 /* SEARCHING_END_PATTERN */:
if (this.findEndPattern()) {
const variableNameBuffer = this._stack.subarray(this._startPattern.length, this._stackIndex - this._endPattern.length);
const value = this.getValueOfVariable(variableNameBuffer);
if (value) {
this._stack = this._stack.subarray(this._stackIndex);
this._stackIndex = 0;
yield this.writeToOutput(value);
} else {
this.releaseStack(this._stackIndex);
}
async _transform(chunk, encoding, callback) {
if (typeof chunk === "string")
chunk = Buffer.from(chunk, encoding);
try {
if (chunk instanceof Buffer) {
if (this._stack.length === 0) {
this._stack = chunk;
} else {
this._stack = Buffer.concat([this._stack, chunk]);
}
while (this._stackIndex < this._stack.length) {
switch (this._state) {
case 0 /* SEARCHING_START_PATTERN */:
this.findStartPattern();
this.releaseStack(this._stackIndex - this._matchCount);
break;
case 1 /* PROCESSING_VARIABLE */:
this.findVariableEnd();
break;
case 2 /* SEARCHING_END_PATTERN */:
if (this.findEndPattern()) {
const variableNameBuffer = this._stack.subarray(this._startPattern.length, this._stackIndex - this._endPattern.length);
const value = this.getValueOfVariable(variableNameBuffer);
if (value) {
this._stack = this._stack.subarray(this._stackIndex);
this._stackIndex = 0;
await this.writeToOutput(value);
} else {
this.releaseStack(this._stackIndex);
}
break;
}
}
break;
}
} else {
this.handleUnknownChunkType(chunk);
}
} catch (e) {
callback(e instanceof Error ? e : new Error(`${e}`));
return;
} else {
this.handleUnknownChunkType(chunk);
}
callback();
});
} catch (e) {
callback(e instanceof Error ? e : new Error(`${e}`));
return;
}
callback();
}

@@ -279,30 +241,14 @@ _flush(callback) {

*/
writeToOutput(stringSource) {
return __async(this, null, function* () {
if (stringSource instanceof import_node_stream.Readable) {
yield this.writeStreamToOutput(stringSource);
} else {
this.push(this.toBuffer(stringSource));
}
});
async writeToOutput(stringSource) {
if (stringSource instanceof import_node_stream.Readable) {
await this.writeStreamToOutput(stringSource);
} else {
this.push(this.toBuffer(stringSource));
}
}
writeStreamToOutput(stream) {
return __async(this, null, function* () {
try {
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield iter.next()).done; more = false) {
const chunk = temp.value;
if (!this.push(chunk))
yield new Promise((resolve) => this.once("drain", resolve));
}
} catch (temp) {
error = [temp];
} finally {
try {
more && (temp = iter.return) && (yield temp.call(iter));
} finally {
if (error)
throw error[0];
}
}
});
async writeStreamToOutput(stream) {
for await (const chunk of stream) {
if (!this.push(chunk))
await new Promise((resolve) => this.once("drain", resolve));
}
}

@@ -309,0 +255,0 @@ toBuffer(stringLike) {

14

package.json
{
"name": "template-replace-stream",
"version": "2.1.1",
"version": "2.1.2",
"description": "A high performance template replace stream working on binary or string streams",

@@ -12,5 +12,8 @@ "main": "dist/index.js",

"scripts": {
"build": "tsup && npm run build:readme",
"build:readme": "ts-node examples/create-readme.ts",
"test": "jest"
"build": "tsup && (npm run build:readme)",
"build:all": "(npm run build:examples) && (npm run build:tests) && (npm run build)",
"build:readme": "cd examples && ts-node generate-readme.ts",
"build:tests": "cd tests && npm install",
"build:examples": "cd examples && npm install",
"test": "cd tests && npm test"
},

@@ -41,4 +44,3 @@ "repository": {

"devDependencies": {
"jest": "^29.7.0",
"ts-jest": "^29.1.3",
"@tsconfig/node20": "^20.1.4",
"tsup": "^8.0.2",

@@ -45,0 +47,0 @@ "typescript": "^5.4.5"

184

README.md
# template-replace-stream
[![GitHub Actions CI](https://github.com/SoulKa/template-replace-stream/actions/workflows/node.js.yml/badge.svg)](https://github.com/SoulKa/template-replace-stream/actions/workflows/node.js.yml)
[![npm version](https://badge.fury.io/js/template-replace-stream.svg)](https://www.npmjs.com/package/template-replace-stream)
[![Downloads](https://img.shields.io/npm/dm/template-replace-stream.svg)](https://www.npmjs.com/package/template-replace-stream)
A high performance `{{ template }}` replace stream working on binary or string streams.
This module is written in pure TypeScript, consists of only 189 lines of code (including type definitions) and has no other dependencies. It is flexible and allows replacing an arbitrary wide range of template variables while being extremely fast (we reached over 20GiB/s, see [Benchmarks](#benchmarks)).
This module is written in pure TypeScript, consists of only 189 lines of code (including type
definitions) and has no other dependencies. It is flexible and allows replacing an arbitrary wide
range of template variables while being extremely fast (we reached over 20GiB/s,
see [Benchmarks](#benchmarks)).

@@ -11,7 +18,17 @@ ## Install

This module can be imported via `require()` or `import` in JavaScript
This module contains type definitions and also an `.mjs` file for maximum compatibility.
### Supported Node.js Versions
The following Node.js versions are tested to work with the package. Older versions are not tested but should still be able to use it.
| 16.x | 18.x | 20.x | 22.x |
| --- | --- | --- | --- |
| [![CI](https://github.com/SoulKa/template-replace-stream/actions/workflows/node.js.yml/badge.svg?branch=main)](https://github.com/SoulKa/template-replace-stream/actions/workflows/node.js.yml) | [![CI](https://github.com/SoulKa/template-replace-stream/actions/workflows/node.js.yml/badge.svg?branch=main)](https://github.com/SoulKa/template-replace-stream/actions/workflows/node.js.yml) | [![CI](https://github.com/SoulKa/template-replace-stream/actions/workflows/node.js.yml/badge.svg?branch=main)](https://github.com/SoulKa/template-replace-stream/actions/workflows/node.js.yml) | [![CI](https://github.com/SoulKa/template-replace-stream/actions/workflows/node.js.yml/badge.svg?branch=main)](https://github.com/SoulKa/template-replace-stream/actions/workflows/node.js.yml) |
## Usage
You create a `TemplateReplaceStream` by passing a source of template variables and their replacement values to the constructor. This may either be a map containing key-value pairs, or a function that returns a replacement value for a given template string.
You create a `TemplateReplaceStream` by passing a source of template variables and their replacement
values to the constructor. This may either be a map containing key-value pairs, or a function that
returns a replacement value for a given template string.

@@ -21,3 +38,3 @@ ### JavaScript

```js
const {TemplateReplaceStream} = require("../dist");
const {TemplateReplaceStream} = require("template-replace-stream");
const fs = require("node:fs");

@@ -42,3 +59,3 @@ const path = require("node:path");

```ts
import {TemplateReplaceStream} from "../src";
import {TemplateReplaceStream} from "template-replace-stream";
import fs from "node:fs";

@@ -63,4 +80,8 @@ import path from "node:path";

#### Readable Stream as Replacement Value Source
It's also possible to pass another `Readable` as replacement value source to the `TemplateReplaceStream`. In fact, the README you are just reading was created using this feature. This makes it possible to replace template variables with whole files without reading them into a stream before.
It's also possible to pass another `Readable` as replacement value source to
the `TemplateReplaceStream`. In fact, the README you are just reading was created using this
feature. This makes it possible to replace template variables with whole files without reading them
into a stream before.
<details>

@@ -70,45 +91,54 @@ <summary>Advanced Example Code</summary>

```ts
import {StringSource, TemplateReplaceStream} from "../src";
import fs from "fs";
import path from "path";
import sloc from "sloc"
import {StringSource, TemplateReplaceStream} from "template-replace-stream";
import fs from "node:fs";
import path from "node:path";
import sloc from "sloc";
import {Project, ts} from "ts-morph";
const rootDir = path.join(__dirname, "..");
const exampleFiles = ["javascript-example.js", "typescript-example.ts", "create-readme.ts"];
const exampleFiles = ["javascript-example.js", "typescript-example.ts", "generate-readme.ts"];
const codeInfo = sloc(fs.readFileSync(path.join(rootDir, "src", "template-replace-stream.ts"), "utf8"), "ts");
const outputFilePath = path.join(rootDir, "README.md");
const sourceFilePath = path.join(rootDir, "index.ts");
const codeInfo = sloc(fs.readFileSync(sourceFilePath, "utf8"), "ts");
const loc = codeInfo.total - codeInfo.comment - codeInfo.empty;
const optionsDefinition = extractTypeDefinition("TemplateReplaceStreamOptions", sourceFilePath);
/**
* Opens a file stream and replaces the import paths in the examples. This is used to
* have module imports in the README but still local imports in the examples.
*
* @param file The file to read.
*/
function openExampleStream(file: string) {
const replaceStream = new TemplateReplaceStream(
new Map([
[`../src`, `"template-replace-stream"`],
[`../dist`, `"template-replace-stream"`]
]),
{
startPattern: '"',
endPattern: '"'
}
);
return fs.createReadStream(path.join(__dirname, file)).pipe(replaceStream);
}
// the map of example files and their read streams and further template variables
const templateMap = new Map<string, StringSource>(exampleFiles.map((file) => [file, openExampleStream(file)]));
templateMap.set("loc", loc.toString());
templateMap.set("options-definition", optionsDefinition);
// create the streams
const readmeReadStream = fs.createReadStream(path.join(rootDir, "README.template.md"));
const readmeWriteStream = fs.createWriteStream(path.join(rootDir, "README.md"));
const readmeReadStream = fs.createReadStream(path.join(rootDir, "template.md"));
const readmeWriteStream = fs.createWriteStream(outputFilePath);
// connect the streams and put the template replace stream in the middle
readmeReadStream.pipe(new TemplateReplaceStream(templateMap)).pipe(readmeWriteStream);
readmeWriteStream.on("finish", () => console.log("Finished writing README.md"));
readmeWriteStream.on("finish", () => console.log(`Created ${outputFilePath}`));
/**
* Opens a file stream to the given source file.
*
* @param file The file to read.
*/
function openExampleStream(file: string) {
return fs.createReadStream(path.join(__dirname, file));
}
/**
* Extracts the type definition from the given source file.
*
* @param typeName The name of the type to extract.
* @param filePath The full path to the source file.
*/
function extractTypeDefinition(typeName: string, filePath: string) {
const sourceFile = new Project().addSourceFileAtPath(filePath);
const typeNode = sourceFile.getTypeAlias(typeName)?.compilerNode;
if (!typeNode) throw new Error(`Type alias ${typeName} not found.`);
const printer = ts.createPrinter({removeComments: false});
return printer.printNode(ts.EmitHint.Unspecified, typeNode, sourceFile.compilerNode);
}
```
</details>

@@ -119,24 +149,27 @@

```ts
type TemplateReplaceStreamOptions = {
/** Default: `false`. If true, the stream creates logs on debug level */
log: boolean;
/**
* Default: `false`. If true, the stream throws an error when a template variable has no
* replacement value
*/
throwOnUnmatchedTemplate: boolean;
/**
* Default: `100`. The maximum length of a variable name between a start and end pattern including
* whitespaces around it. Any variable name longer than this length is ignored, i.e. the search
* for the end pattern canceled and the stream looks for the next start pattern.
* Note that a shorter length improves performance but may not find all variables.
*/
maxVariableNameLength: number;
/** Default: `'{{'`. The start pattern of a template string either as string or buffer */
startPattern: string | Buffer;
/** Default: `'}}'`. The end pattern of a template string either as string or buffer */
endPattern: string | Buffer;
/** Any options for the lower level {@link Transform} stream. Do not replace transform or flush */
streamOptions?: TransformOptions;
}
/**
* Options for the template replace stream.
*/
export type TemplateReplaceStreamOptions = {
/** Default: `false`. If true, the stream creates logs on debug level */
log: boolean;
/**
* Default: `false`. If true, the stream throws an error when a template variable has no
* replacement value. Takes precedence over `removeUnmatchedTemplate`.
*/
throwOnUnmatchedTemplate: boolean;
/**
* Default: `100`. The maximum length of a variable name between a start and end pattern including
* whitespaces around it. Any variable name longer than this length is ignored, i.e. the search
* for the end pattern canceled and the stream looks for the next start pattern.
* Note that a shorter length improves performance but may not find all variables.
*/
maxVariableNameLength: number;
/** Default: `'{{'`. The start pattern of a template string either as string or buffer */
startPattern: string | Buffer;
/** Default: `'}}'`. The end pattern of a template string either as string or buffer */
endPattern: string | Buffer;
/** Any options for the lower level {@link Transform} stream. Do not replace transform or flush */
streamOptions?: TransformOptions;
};
```

@@ -146,3 +179,6 @@

The benchmarks were run on my MacBook Pro with an Apple M1 Pro Chip. The data source were virtual files generated from- and to memory to omit any bottleneck due to the file system. The "native" data refers to reading a files from disk without doing anything else with it (native `fs.Readable` streams). So they are the absolute highest possible.
The benchmarks were run on my MacBook Pro with an Apple M1 Pro Chip. The data source were virtual
files generated from- and to memory to omit any bottleneck due to the file system. The "native" data
refers to reading a virtual file without doing anything else with it (native `fs.Readable` streams).
So they are the absolute highest possible.

@@ -153,7 +189,12 @@ ## Replacing a single Template Variable in a large File

Like the raw file system stream, a `TemplateReplaceStream` becomes faster with an increasing source file size. It is more than 20x faster than the `replace-stream` when processing large files. The throughput of the `TemplateReplaceStream` was more than 20GiB/s when replacing a single variable in a 100MiB file.
Like the raw file system stream, a `TemplateReplaceStream` becomes faster with an increasing source
file size. It is more than 20x faster than the `replace-stream` when processing large files. The
throughput of the `TemplateReplaceStream` was more than 20GiB/s when replacing a single variable in
a 100MiB file.
![Duration vs File Size when replacing a single Variable](benchmarks/plots/size-vs-duration-with-one-replacement.png)
Replacing a single variable in a 100MiB file takes only 6ms using a `TemplateReplaceStream`. Reading the whole file from the disk alone takes already more than 1ms. The `stream-replace-string` packages was omitted im this graph, as it took over 16s to process the 100MiB file.
Replacing a single variable in a 100MiB file takes only 6ms using a `TemplateReplaceStream`. Reading
the whole file from the disk alone takes already more than 1ms. The `stream-replace-string` packages
was omitted im this graph, as it took over 16s to process the 100MiB file.

@@ -164,11 +205,20 @@ ## Replacing 10 thousand Template Variables in a large File

You can see that the performance declines when working with more replacements. Note that one reason is the virtually generated workload (see "native" in the graph). `TemplateReplaceStream` still reaches 10GiB/s.
You can see that the performance declines when working with more replacements. Note that one reason
is the virtually generated workload (see "native" in the graph). `TemplateReplaceStream` still
reaches 10GiB/s.
![Duration vs File Size when replacing a 10K Variables](benchmarks/plots/size-vs-duration-with-10k-replacement.png)
To replace ten thousand template variables in a 100MiB file, the `TemplateReplaceStream` takes around 10ms. Since this duration is similar for smaller file sizes, we can see that it does not perform too well in the 1MiB file. We will keep optimizing for that.
To replace ten thousand template variables in a 100MiB file, the `TemplateReplaceStream` takes
around 10ms. Since this duration is similar for smaller file sizes, we can see that it does not
perform too well in the 1MiB file. We will keep optimizing for that.
## Changelog
### 2.1.2
- Add CI to repository
- Update README
### 2.1.1
- Fix stream ending when replacing a template with another stream during the last chunk of data

@@ -178,7 +228,11 @@ - Update README

### 2.1.0
- Further improve performance by using `Buffer.indexOf()` to find the end of a template variable, too
- Further improve performance by using `Buffer.indexOf()` to find the end of a template variable,
too
- Add more benchmarks
### 2.0.0
- Drastically improve performance (by ~10x) by using `Buffer.indexOf()` instead of iterating over the buffer myself
- Drastically improve performance (by ~10x) by using `Buffer.indexOf()` instead of iterating over
the buffer myself
- Rename option `throwOnMissingVariable` to `throwOnUnmatchedTemplate`

@@ -188,5 +242,7 @@ - Add benchmarks

### 1.0.1
- Update README
### 1.0.0
- Initial Release
README.template.md
dist/index.js.map

Sorry, the diff of this file is not supported yet

dist/index.mjs

Sorry, the diff of this file is not supported yet

dist/index.mjs.map

Sorry, the diff of this file is not supported yet

LICENSE

Sorry, the diff of this file is not supported yet