winston-newrelic-logs-transport - npm Package Compare versions

Comparing version 1.1.0 to 1.2.0

dist/index.js

@@ -6,29 +6,97 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultBatchThrottle = exports.defaultBatchSize = void 0;
 const axios_1 = __importDefault(require("axios"));
 const winston_transport_1 = __importDefault(require("winston-transport"));
-const triple_beam_1 = require("triple-beam");
+const lodash_throttle_1 = __importDefault(require("lodash.throttle"));
+const lodash_clonedeep_1 = __importDefault(require("lodash.clonedeep"));
+exports.defaultBatchSize = 100;
+exports.defaultBatchThrottle = 1000;
 class WinstonNewrelicLogsTransport extends winston_transport_1.default {
     constructor(options) {
-        var _a;
+        var _a, _b, _c;
         super();
-        this.axiosClient = axios_1.default.create(Object.assign(Object.assign({ timeout: 5000 }, options.axiosOptions), { baseURL: options.apiUrl, headers: Object.assign(Object.assign({}, (_a = options.axiosOptions) === null || _a === void 0 ? void 0 : _a.headers), { "X-License-Key": options.licenseKey, "Content-Type": "application/json" }) }));
+        this.logs = [];
+        this.axiosClient = axios_1.default.create(Object.assign(Object.assign({ timeout: 5000 }, options.axiosOptions), { baseURL: options.apiUrl, headers: Object.assign(Object.assign({}, (_a = options.axiosOptions) === null || _a === void 0 ? void 0 : _a.headers), { 'Api-Key': options.licenseKey, 'Content-Type': 'application/json' }) }));
+        if (options.batchSize !== undefined ||
+            options.batchThrottle !== undefined) {
+            this.batchSize =
+                options.batchSize === true
+                    ? exports.defaultBatchSize
+                    : (_b = options.batchSize) !== null && _b !== void 0 ? _b : exports.defaultBatchSize;
+            this.batchThrottle =
+                options.batchThrottle === true
+                    ? exports.defaultBatchThrottle
+                    : (_c = options.batchThrottle) !== null && _c !== void 0 ? _c : exports.defaultBatchThrottle;
+            if (this.batchSize <= 0) {
+                throw new Error('Expected a batchSize greater than 0');
+            }
+            if (this.batchThrottle <= 0) {
+                throw new Error('Expected a batchThrottle greater than 0');
+            }
+            this.throttledBatchPost = (0, lodash_throttle_1.default)(this.batchPost.bind(this), this.batchThrottle, { leading: false, trailing: true });
+        }
     }
-    log(info, callback) {
-        this.axiosClient
-            .post("/log/v1", {
-            timestamp: Date.now(),
-            message: info[triple_beam_1.MESSAGE],
-            logtype: info[triple_beam_1.LEVEL],
-        })
-            .then(() => {
-            this.emit("logged", info);
+    batchPost() {
+        try {
+            const logs = this.logs.slice();
+            this.logs = [];
+            const data = [
+                {
+                    logs,
+                },
+            ];
+            this.axiosClient
+                .post('/log/v1', data)
+                .then(() => {
+                for (const log of logs) {
+                    this.emit('logged', log);
+                }
+            })
+                .catch((err) => {
+                this.emit('error', err);
+            });
+        }
+        catch (err) {
+            this.emit('error', err);
+        }
+    }
+    log(data, callback) {
+        const entry = validateData(data);
+        if (!entry.timestamp) {
+            entry.timestamp = Date.now();
+        }
+        if (this.throttledBatchPost && this.batchSize && this.batchSize > 0) {
+            this.logs.push(entry);
+            this.throttledBatchPost();
+            if (this.logs.length >= this.batchSize) {
+                this.throttledBatchPost.flush();
+            }
             callback(null);
-        })
-            .catch((err) => {
-            this.emit("error", err);
-            callback(err);
-        });
+        }
+        else {
+            this.axiosClient
+                .post('/log/v1', entry)
+                .then(() => {
+                this.emit('logged', entry);
+                callback(null);
+            })
+                .catch((err) => {
+                this.emit('error', err);
+                callback(err);
+            });
+        }
     }
 }
 exports.default = WinstonNewrelicLogsTransport;
+function validateData(data) {
+    if (data === null) {
+        return {};
+    }
+    else if (typeof data !== 'object') {
+        return { metadata: data };
+    }
+    else {
+        return (0, lodash_clonedeep_1.default)(data);
+    }
+}
 //# sourceMappingURL=index.js.map

index.test.ts

@@ -1,3 +0,6 @@
-import WinstonNewrelicLogsTransport from "index";
-import { beforeAll, expect, test, vi } from "vitest";
+import WinstonNewrelicLogsTransport, {
+  defaultBatchSize,
+  defaultBatchThrottle,
+} from "index";
+import { beforeEach, describe, expect, test, vi } from "vitest";
 import { LEVEL, MESSAGE } from "triple-beam";
@@ -15,3 +18,3 @@ import axios, { AxiosInstance } from "axios";
 
-beforeAll(() => {
+beforeEach(() => {
   axiosCreateSpy.mockReturnValue(mockAxiosClient);
@@ -27,6 +30,8 @@
   );
+
+  vi.clearAllTimers();
 });
 
-test("should post and fire callback", async () => {
-  const transport = new WinstonNewrelicLogsTransport({
+test("should setup axios", async () => {
+  new WinstonNewrelicLogsTransport({
     apiUrl: "logs.foo.com",
@@ -44,34 +49,214 @@ licenseKey: "000000",
       "Content-Type": "application/json",
-      "X-License-Key": "000000",
+      "Api-Key": "000000",
     },
     timeout: 123456,
   });
+});
 
-  const cb = vi.fn();
+describe("single mode", () => {
+  test("should post and fire callback", async () => {
+    const transport = new WinstonNewrelicLogsTransport({
+      apiUrl: "logs.foo.com",
+      licenseKey: "000000",
+      axiosOptions: {
+        timeout: 123456,
+      },
+    });
 
-  transport.log({ [MESSAGE]: "Some message", [LEVEL]: "info" }, cb);
+    const cb = vi.fn();
 
-  await vi.runAllTimersAsync();
-  expect(cb).toHaveBeenCalledWith(null);
+    transport.log({ [MESSAGE]: "Some message", [LEVEL]: "info" }, cb);
+
+    expect(mockPost).toHaveBeenCalledTimes(1);
+    expect(mockPost).toHaveBeenCalledWith("/log/v1", {
+      timestamp: expect.any(Number),
+      [MESSAGE]: "Some message",
+      [LEVEL]: "info",
+    });
+
+    await vi.runAllTimersAsync();
+
+    expect(cb).toHaveBeenCalledWith(null);
+  });
+
+  test("should handle errors", async () => {
+    const errorEventHandler = vi.fn();
+
+    const transport = new WinstonNewrelicLogsTransport({
+      apiUrl: "logs.foo.com",
+      licenseKey: "000000",
+    });
+    transport.on("error", errorEventHandler);
+
+    const cb = vi.fn();
+
+    const err = new Error("A timeout or something");
+    mockPost.mockRejectedValue(err);
+
+    transport.log({ [MESSAGE]: "Some message", [LEVEL]: "info" }, cb);
+
+    await vi.advanceTimersByTimeAsync(100);
+
+    expect(cb).toHaveBeenCalledWith(err);
+    expect(errorEventHandler).toHaveBeenCalledWith(err);
+  });
 });
 
-test("should handle errors", async () => {
-  const errorEventHandler = vi.fn();
+describe("batch mode", () => {
+  test("should throw with bad batch size", () => {
+    expect(
+      () =>
+        new WinstonNewrelicLogsTransport({
+          apiUrl: "logs.foo.com",
+          licenseKey: "000000",
+          batchSize: -1,
+        })
+    ).toThrowErrorMatchingInlineSnapshot(
+      '"Expected a batchSize greater than 0"'
+    );
+  });
 
-  const transport = new WinstonNewrelicLogsTransport({
-    apiUrl: "logs.foo.com",
-    licenseKey: "000000",
+  test("should throw with bad throttle", () => {
+    expect(
+      () =>
+        new WinstonNewrelicLogsTransport({
+          apiUrl: "logs.foo.com",
+          licenseKey: "000000",
+          batchThrottle: -4000,
+        })
+    ).toThrowErrorMatchingInlineSnapshot(
+      '"Expected a batchThrottle greater than 0"'
+    );
   });
-  transport.on("error", errorEventHandler);
 
-  const cb = vi.fn();
+  test("should setup defaults", () => {
+    const transport = new WinstonNewrelicLogsTransport({
+      apiUrl: "logs.foo.com",
+      licenseKey: "000000",
+      batchThrottle: true,
+    });
 
-  const err = new Error("A timeout or something");
-  mockPost.mockRejectedValue(err);
+    expect(transport.batchSize).toEqual(defaultBatchSize);
+    expect(transport.batchThrottle).toEqual(defaultBatchThrottle);
+  });
 
-  transport.log({ [MESSAGE]: "Some message", [LEVEL]: "info" }, cb);
+  test("should batch send", async () => {
+    const transport = new WinstonNewrelicLogsTransport({
+      apiUrl: "logs.foo.com",
+      licenseKey: "000000",
+      batchThrottle: true,
+    });
 
-  await vi.runAllTimersAsync();
-  expect(cb).toHaveBeenCalledWith(err);
-  expect(errorEventHandler).toHaveBeenCalledWith(err);
+    const cb = vi.fn();
+
+    for (let i = 0; i < 4; i++) {
+      transport.log(
+        { [MESSAGE]: `Some message ${i + 1}`, [LEVEL]: "info" },
+        cb
+      );
+    }
+
+    expect(mockPost).not.toHaveBeenCalledTimes(1);
+
+    await vi.advanceTimersByTimeAsync(500);
+
+    expect(cb).toHaveBeenCalledWith(null);
+
+    await vi.advanceTimersByTimeAsync(1000);
+
+    expect(mockPost).toHaveBeenCalledTimes(1);
+    expect(mockPost).toHaveBeenCalledWith("/log/v1", [
+      {
+        logs: [
+          {
+            timestamp: expect.any(Number),
+            [MESSAGE]: "Some message 1",
+            [LEVEL]: "info",
+          },
+          {
+            timestamp: expect.any(Number),
+            [MESSAGE]: "Some message 2",
+            [LEVEL]: "info",
+          },
+          {
+            timestamp: expect.any(Number),
+            [MESSAGE]: "Some message 3",
+            [LEVEL]: "info",
+          },
+          {
+            timestamp: expect.any(Number),
+            [MESSAGE]: "Some message 4",
+            [LEVEL]: "info",
+          },
+        ],
+      },
+    ]);
+  });
+
+  test("should batch send immediately if the size limit is breached", async () => {
+    const transport = new WinstonNewrelicLogsTransport({
+      apiUrl: "logs.foo.com",
+      licenseKey: "000000",
+      batchThrottle: true,
+      batchSize: 3,
+    });
+
+    const cb = vi.fn();
+
+    for (let i = 0; i < 5; i++) {
+      transport.log(
+        { [MESSAGE]: `Some message ${i + 1}`, [LEVEL]: "info" },
+        cb
+      );
+    }
+
+    expect(mockPost).toHaveBeenCalledTimes(1);
+    expect(mockPost.mock.calls[0][1][0].logs).toHaveLength(3);
+    expect(mockPost).toHaveBeenCalledWith("/log/v1", [
+      {
+        logs: [
+          {
+            timestamp: expect.any(Number),
+            [MESSAGE]: "Some message 1",
+            [LEVEL]: "info",
+          },
+          {
+            timestamp: expect.any(Number),
+            [MESSAGE]: "Some message 2",
+            [LEVEL]: "info",
+          },
+          {
+            timestamp: expect.any(Number),
+            [MESSAGE]: "Some message 3",
+            [LEVEL]: "info",
+          },
+        ],
+      },
+    ]);
+
+    await vi.advanceTimersByTimeAsync(500);
+
+    expect(cb).toHaveBeenCalledWith(null);
+
+    await vi.advanceTimersByTimeAsync(1000);
+
+    expect(mockPost).toHaveBeenCalledTimes(2);
+    expect(mockPost.mock.calls[1][1][0].logs).toHaveLength(2);
+    expect(mockPost).toHaveBeenCalledWith("/log/v1", [
+      {
+        logs: [
+          {
+            timestamp: expect.any(Number),
+            [MESSAGE]: "Some message 4",
+            [LEVEL]: "info",
+          },
+          {
+            timestamp: expect.any(Number),
+            [MESSAGE]: "Some message 5",
+            [LEVEL]: "info",
+          },
+        ],
+      },
+    ]);
+  });
 });

index.ts

 import axios, { AxiosInstance, AxiosRequestConfig } from "axios";
 import TransportStream from "winston-transport";
-import { LEVEL, MESSAGE } from "triple-beam";
+import throttle from "lodash.throttle";
+import cloneDeep from "lodash.clonedeep";
 
 export interface WinstonNewrelicLogsTransportOptions {
+  /**
+   * Your NewRelic key.
+   *
+   * @see https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#auth-header.
+   */
   licenseKey: string;
+  /**
+   * The URL to send data to.
+   *
+   * @see https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#endpoint.
+   */
   apiUrl: string;
+  /**
+   * Options to use when sending data via Axios.
+   */
   axiosOptions?: AxiosRequestConfig;
+  /**
+   * How many log items you would like to bundle together before posting to loggly.
+   */
+  batchSize?: number | true;
+  /**
+   * The maximum frequency the batch posting should occur unless the batch size is exceeded.
+   */
+  batchThrottle?: number | true;
 }
 
+type LogDataType = Record<string | symbol, string | number>;
+
+export const defaultBatchSize = 100;
+export const defaultBatchThrottle = 1000;
+
 /**
  * Transport for reporting errors to newrelic.
- * 
+ *
  * @type {WinstonNewrelicLogsTransport}
- * @extends {TransportStream}
+ * @augments {TransportStream}
  */
 export default class WinstonNewrelicLogsTransport extends TransportStream {
-  axiosClient: AxiosInstance;
+  private axiosClient: AxiosInstance;
+  private logs: LogDataType[];
+  public readonly batchSize?: number;
+  public readonly batchThrottle?: number;
+  private readonly throttledBatchPost: ReturnType<typeof throttle> | undefined;
 
+  /**
+   * Contrusctor for the WinstonNewrelicLogsTransport.
+   *
+   * @param options - Options.
+   */
   constructor(options: WinstonNewrelicLogsTransportOptions) {
     super();
 
+    this.logs = [];
     this.axiosClient = axios.create({
@@ -29,30 +66,123 @@ timeout: 5000,
         ...options.axiosOptions?.headers,
-        "X-License-Key": options.licenseKey,
-        "Content-Type": "application/json",
+        'Api-Key': options.licenseKey,
+        'Content-Type': 'application/json',
       },
     });
+
+    if (
+      options.batchSize !== undefined ||
+      options.batchThrottle !== undefined
+    ) {
+      this.batchSize =
+        options.batchSize === true
+          ? defaultBatchSize
+          : options.batchSize ?? defaultBatchSize;
+      this.batchThrottle =
+        options.batchThrottle === true
+          ? defaultBatchThrottle
+          : options.batchThrottle ?? defaultBatchThrottle;
+
+      if (this.batchSize <= 0) {
+        throw new Error('Expected a batchSize greater than 0');
+      }
+      if (this.batchThrottle <= 0) {
+        throw new Error('Expected a batchThrottle greater than 0');
+      }
+
+      this.throttledBatchPost = throttle(
+        this.batchPost.bind(this),
+        this.batchThrottle,
+        { leading: false, trailing: true }
+      );
+    }
   }
 
-  log(info: { [x in symbol]: string }, callback: (error?: Error) => void) {
+  /**
+   * Performs the batch posting operations.
+   */
+  private batchPost() {
+    try {
+      const logs = this.logs.slice();
+      this.logs = [];
+
+      const data = [
+        {
+          logs,
+        },
+      ];
+
+      this.axiosClient
+        .post('/log/v1', data)
+        .then(() => {
+          for (const log of logs) {
+            this.emit('logged', log);
+          }
+        })
+        .catch((err) => {
+          this.emit('error', err);
+        });
+    } catch (err) {
+      this.emit('error', err);
+    }
+  }
+
+  /**
+   * Logs data to Newrelic either directly or via batching as configured.
+   *
+   * @param data - Info to log.
+   * @param callback - Logging callback.
+   */
+  public log(data: LogDataType, callback: (error?: Error | null) => void) {
     // The implementation of log callbacks isn't documented and the exported type
     // definitions appear to be wrong too. This implementation has been compied
     // https://github.com/winstonjs/winston-mongodb/blob/master/lib/winston-mongodb.js#L229-L235
-    // However I don't know what the second argument for callback is supposed to 
+    // However I don't know what the second argument for callback is supposed to
     // indicate.
 
-    this.axiosClient
-      .post("/log/v1", {
-        timestamp: Date.now(),
-        message: info[MESSAGE],
-        logtype: info[LEVEL],
-      })
-      .then(() => {
-        this.emit("logged", info);
-        callback(null);
-      })
-      .catch((err) => {
-        this.emit("error", err);
-        callback(err);
-      });
+    const entry = validateData(data);
+
+    // https://docs.newrelic.com/docs/logs/log-api/log-event-data/
+    if (!entry.timestamp) {
+      entry.timestamp = Date.now();
+    }
+
+    if (this.throttledBatchPost && this.batchSize && this.batchSize > 0) {
+      this.logs.push(entry);
+      this.throttledBatchPost();
+
+      if (this.logs.length >= this.batchSize) {
+        this.throttledBatchPost.flush();
+      }
+
+      callback(null);
+    } else {
+      this.axiosClient
+        .post('/log/v1', entry)
+        .then(() => {
+          this.emit('logged', entry);
+          callback(null);
+        })
+        .catch((err) => {
+          this.emit('error', err);
+          callback(err);
+        });
+    }
   }
 }
+
+/**
+ * Checks the incoming meta data and makes it safe for sending.
+ *
+ * @param data - Data to check.
+ * @returns Checked and cloned data.
+ */
+// eslint-disable-next-line @typescript-eslint/ban-types
+function validateData(data: LogDataType): LogDataType {
+  if (data === null) {
+    return {};
+  } else if (typeof data !== 'object') {
+    return { metadata: data };
+  } else {
+    return cloneDeep(data);
+  }
+}

package.json

 {
   "name": "winston-newrelic-logs-transport",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "NewRelic Logs API Transport for Winston",
@@ -12,3 +12,3 @@ "main": "dist/index.js",
   "scripts": {
-    "build": "tsc",
+    "build": "tsc --project tsconfig.build.json",
     "prepublish": "tsc",
@@ -19,2 +19,4 @@ "test": "vitest"
     "axios": "^0.21.1",
+    "lodash.clonedeep": "^4.5.0",
+    "lodash.throttle": "^4.1.1",
     "logform": "^2.5.1",
@@ -24,2 +26,4 @@ "winston-transport": "^4.2.0"
   "devDependencies": {
+    "@types/lodash.clonedeep": "^4.5.7",
+    "@types/lodash.throttle": "^4.1.7",
     "@types/triple-beam": "^1.3.2",
@@ -26,0 +30,0 @@ "tslint": "^6.1.3",

README.md

 # winston-newrelic-logs-transport
 
-A [newrelic][http://newrelic.com/] Logs API transport for [winston][https://github.com/flatiron/winston].
+A [newrelic](http://newrelic.com/) Logs API transport for [winston](https://github.com/flatiron/winston).
 
@@ -30,1 +30,8 @@ ## Installation
 * __apiUrl__: New Relic Log Base API URL.
+* __axiosOptions__: Options passed to Axios when sending data. (Optional)
+* __batchSize__:  How many log items you would like to bundle together before posting to loggly. (Optional, positive integer or true, default 100)
+* __batchThrottle__: The maximum frequency the batch posting should occur unless the batch size is exceeded. (Optional, positive integer or true, default 1000)
+
+### Batching
+
+If either batching option is set without the other, or simply set as `true` then default values are used as specified. 

tsconfig.build.json

 {
-    "extends": "./tsconfig.json",
-    "exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
-  }
-  
+  "extends": "./tsconfig.json",
+  "exclude": [
+    "node_modules",
+    "test",
+    "dist",
+    "**/*spec.ts",
+    "**/*.test.ts",
+    "vitest.config.ts"
+  ]
+}

vitest.config.ts

@@ -9,3 +9,8 @@ import { defineConfig } from 'vitest/config';
     clearMocks: true,
+    exclude: [
+      // Comment if you want to run the integration tests.
+      '**/*.integration.test.*',
+      'dist/**/*'
+    ]
   },
 });

New alerts

Deprecated

Maintenance

The maintainer of the package marked it as deprecated. This could indicate that a single version should not be used, or that the package is no longer maintained and any new vulnerabilities will not be fixed.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

21629

increased by155%

Number of package files

12

increased by9.09%

Lines of code

585

increased by242.11%

Number of lines in readme file

37

increased by23.33%

Worsened metrics

Dependency count

5

increased by66.67%

Dev dependency count

6

increased by50%

Number of high maintenance alerts

1

increased byInfinity%

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Addedlodash.clonedeep@^4.5.0

• + Addedlodash.throttle@^4.1.1

• + Addedlodash.clonedeep@4.5.0(transitive)

• + Addedlodash.throttle@4.1.1(transitive)