protons - npm Package Compare versions

Comparing version 7.3.4 to 7.4.0

dist/bin/protons.js

@@ -17,3 +17,2 @@ #! /usr/bin/env node
 `, {
-        // @ts-expect-error importMeta is missing from the types
         importMeta: import.meta,
@@ -20,0 +19,0 @@ flags: {

dist/src/index.js

 /* eslint-disable max-depth */
+/**
+ * @packageDocumentation
+ *
+ * `protons` is a high performance implementation of [Protocol Buffers v3](https://protobuf.dev/programming-guides/proto3/).
+ *
+ * It transpiles code to TypeScript and supports BigInts for 64 bit types.
+ *
+ * The `protons` module contains the code to compile `.proto` files to `.ts` files and `protons-runtime` contains the code to do serialization/deserialization to `Uint8Array`s during application execution.
+ *
+ * Please ensure you declare them as the correct type of dependencies:
+ *
+ * ```console
+ * $ npm install --save-dev protons
+ * $ npm install --save protons-runtime
+ * ```
+ *
+ * ## Usage
+ *
+ * First generate your `.ts` files:
+ *
+ * ```console
+ * $ protons ./path/to/foo.proto ./path/to/output.ts
+ * ```
+ *
+ * Then run tsc over them as normal:
+ *
+ * ```console
+ * $ tsc
+ * ```
+ *
+ * In your code import the generated classes and use them to transform to/from bytes:
+ *
+ * ```js
+ * import { Foo } from './foo.js'
+ *
+ * const foo = {
+ *   message: 'hello world'
+ * }
+ *
+ * const encoded = Foo.encode(foo)
+ * const decoded = Foo.decode(encoded)
+ *
+ * console.info(decoded.message)
+ * // 'hello world'
+ * ```
+ *
+ * ## Differences from protobuf.js
+ *
+ * This module uses the internal reader/writer from `protobuf.js` as it is highly optimised and there's no point reinventing the wheel.
+ *
+ * It does have one or two differences:
+ *
+ * 1. Supports `proto3` semantics only
+ * 2. All 64 bit values are represented as `BigInt`s and not `Long`s (e.g. `int64`, `uint64`, `sint64` etc)
+ * 3. Unset `optional` fields are set on the deserialized object forms as `undefined` instead of the default values
+ * 4. `singular` fields set to default values are not serialized and are set to default values when deserialized if not set - protobuf.js [diverges from the language guide](https://github.com/protobufjs/protobuf.js/issues/1468#issuecomment-745177012) around this feature
+ * 5. `map` fields can have keys of any type - protobufs.js [only supports strings](https://github.com/protobufjs/protobuf.js/issues/1203#issuecomment-488637338)
+ * 6. `map` fields are deserialized as ES6 `Map`s - protobuf.js uses `Object`s
+ *
+ * ## Extra features
+ *
+ * ### Limiting the size of repeated/map elements
+ *
+ * To protect decoders from malicious payloads, it's possible to limit the maximum size of repeated/map elements.
+ *
+ * You can either do this at compile time by using the [protons.options](https://github.com/protocolbuffers/protobuf/blob/6f1d88107f268b8ebdad6690d116e74c403e366e/docs/options.md?plain=1#L490-L493) extension:
+ *
+ * ```protobuf
+ * message MyMessage {
+ *   // repeatedField cannot have more than 10 entries
+ *   repeated uint32 repeatedField = 1 [(protons.options).limit = 10];
+ *
+ *   // stringMap cannot have more than 10 keys
+ *   map<string, string> stringMap = 2 [(protons.options).limit = 10];
+ * }
+ * ```
+ *
+ * Or at runtime by passing objects to the `.decode` function of your message:
+ *
+ * ```TypeScript
+ * const message = MyMessage.decode(buf, {
+ *   limits: {
+ *     repeatedField: 10,
+ *     stringMap: 10
+ *   }
+ * })
+ * ```
+ *
+ * ### Overriding 64 bit types
+ *
+ * By default 64 bit types are implemented as [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt)s.
+ *
+ * Sometimes this is undesirable due to [performance issues](https://betterprogramming.pub/the-downsides-of-bigints-in-javascript-6350fd807d) or code legibility.
+ *
+ * It's possible to override the JavaScript type 64 bit fields will deserialize to:
+ *
+ * ```protobuf
+ * message MyMessage {
+ *   repeated int64 bigintField = 1;
+ *   repeated int64 numberField = 2 [jstype = JS_NUMBER];
+ *   repeated int64 stringField = 3 [jstype = JS_STRING];
+ * }
+ * ```
+ *
+ * ```TypeScript
+ * const message = MyMessage.decode(buf)
+ *
+ * console.info(typeof message.bigintField) // bigint
+ * console.info(typeof message.numberField) // number
+ * console.info(typeof message.stringField) // string
+ * ```
+ *
+ * ## Missing features
+ *
+ * Some features are missing `OneOf`s, etc due to them not being needed so far in ipfs/libp2p. If these features are important to you, please open PRs implementing them along with tests comparing the generated bytes to `protobuf.js` and `pbjs`.
+ */
 import fs from 'fs/promises';
@@ -404,2 +520,3 @@ import path from 'path';
     moduleDef.addTypeImport('protons-runtime', 'Codec');
+    moduleDef.addTypeImport('protons-runtime', 'DecodeOptions');
     moduleDef.addTypeImport('uint8arraylist', 'Uint8ArrayList');
@@ -546,6 +663,10 @@ const interfaceFields = defineFields(fields, messageDef, moduleDef)
             if (fieldDef.map) {
-                let limit = '';
+                moduleDef.addImport('protons-runtime', 'CodeError');
+                let limit = `
+              if (opts.limits?.${fieldName} != null && obj.${fieldName}.size === opts.limits.${fieldName}) {
+                throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_SIZE')
+              }
+`;
                 if (fieldDef.lengthLimit != null) {
-                    moduleDef.addImport('protons-runtime', 'CodeError');
-                    limit = `
+                    limit += `
               if (obj.${fieldName}.size === ${fieldDef.lengthLimit}) {
@@ -563,6 +684,10 @@ throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_SIZE')
             else if (fieldDef.repeated) {
-                let limit = '';
+                moduleDef.addImport('protons-runtime', 'CodeError');
+                let limit = `
+              if (opts.limits?.${fieldName} != null && obj.${fieldName}.length === opts.limits.${fieldName}) {
+                throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_LENGTH')
+              }
+`;
                 if (fieldDef.lengthLimit != null) {
-                    moduleDef.addImport('protons-runtime', 'CodeError');
-                    limit = `
+                    limit += `
               if (obj.${fieldName}.length === ${fieldDef.lengthLimit}) {
@@ -599,3 +724,3 @@ throw new CodeError('decode error - repeated field "${fieldName}" had too many elements', 'ERR_MAX_LENGTH')
         }
-      }, (reader, length) => {
+      }, (reader, length, opts = {}) => {
         const obj: any = {${createDefaultObject(fields, messageDef, moduleDef)}}
@@ -627,4 +752,4 @@
 
-  export const decode = (buf: Uint8Array | Uint8ArrayList): ${messageDef.name} => {
-    return decodeMessage(buf, ${messageDef.name}.codec())
+  export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<${messageDef.name}>): ${messageDef.name} => {
+    return decodeMessage(buf, ${messageDef.name}.codec(), opts)
   }`;
@@ -631,0 +756,0 @@ return `

package.json

 {
   "name": "protons",
-  "version": "7.3.4",
+  "version": "7.4.0",
   "description": "Protobuf to ts transpiler",
@@ -14,2 +14,6 @@ "license": "Apache-2.0 OR MIT",
   },
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
   "bin": {
@@ -39,87 +43,2 @@ "protons": "./dist/bin/protons.js"
   },
-  "release": {
-    "branches": [
-      "main"
-    ],
-    "plugins": [
-      [
-        "@semantic-release/commit-analyzer",
-        {
-          "preset": "conventionalcommits",
-          "releaseRules": [
-            {
-              "breaking": true,
-              "release": "major"
-            },
-            {
-              "revert": true,
-              "release": "patch"
-            },
-            {
-              "type": "feat",
-              "release": "minor"
-            },
-            {
-              "type": "fix",
-              "release": "patch"
-            },
-            {
-              "type": "docs",
-              "release": "patch"
-            },
-            {
-              "type": "test",
-              "release": "patch"
-            },
-            {
-              "type": "deps",
-              "release": "patch"
-            },
-            {
-              "scope": "no-release",
-              "release": false
-            }
-          ]
-        }
-      ],
-      [
-        "@semantic-release/release-notes-generator",
-        {
-          "preset": "conventionalcommits",
-          "presetConfig": {
-            "types": [
-              {
-                "type": "feat",
-                "section": "Features"
-              },
-              {
-                "type": "fix",
-                "section": "Bug Fixes"
-              },
-              {
-                "type": "chore",
-                "section": "Trivial Changes"
-              },
-              {
-                "type": "docs",
-                "section": "Documentation"
-              },
-              {
-                "type": "deps",
-                "section": "Dependencies"
-              },
-              {
-                "type": "test",
-                "section": "Tests"
-              }
-            ]
-          }
-        }
-      ],
-      "@semantic-release/changelog",
-      "@semantic-release/npm",
-      "@semantic-release/github",
-      "@semantic-release/git"
-    ]
-  },
   "scripts": {
@@ -131,7 +50,6 @@ "clean": "aegir clean",
     "test": "aegir test -t node",
-    "test:node": "aegir test -t node --cov",
-    "release": "aegir release"
+    "test:node": "aegir test -t node --cov"
   },
   "dependencies": {
-    "meow": "^13.0.0",
+    "meow": "^13.1.0",
     "protobufjs-cli": "^1.0.0"
@@ -138,0 +56,0 @@ },

README.md

@@ -10,19 +10,11 @@ # protons <!-- omit in toc -->
 
-## Table of contents <!-- omit in toc -->
+# About
 
-- [Install](#install)
-- [Usage](#usage)
-- [Differences from protobuf.js](#differences-from-protobufjs)
-- [Missing features](#missing-features)
-- [API Docs](#api-docs)
-- [License](#license)
-- [Contribute](#contribute)
+`protons` is a high performance implementation of [Protocol Buffers v3](https://protobuf.dev/programming-guides/proto3/).
 
-## Install
+It transpiles code to TypeScript and supports BigInts for 64 bit types.
 
-```console
-$ npm i protons
-```
+The `protons` module contains the code to compile `.proto` files to `.ts` files and `protons-runtime` contains the code to do serialization/deserialization to `Uint8Array`s during application execution.
 
-`protons` contains the code to compile `.proto` files to `.ts` files and `protons-runtime` contains the code to do serialization/deserialization to `Uint8Array`s during application execution.
+Please ensure you declare them as the correct type of dependencies:
 
@@ -77,2 +69,55 @@ ```console
 
+## Extra features
+
+### Limiting the size of repeated/map elements
+
+To protect decoders from malicious payloads, it's possible to limit the maximum size of repeated/map elements.
+
+You can either do this at compile time by using the [protons.options](https://github.com/protocolbuffers/protobuf/blob/6f1d88107f268b8ebdad6690d116e74c403e366e/docs/options.md?plain=1#L490-L493) extension:
+
+```protobuf
+message MyMessage {
+  // repeatedField cannot have more than 10 entries
+  repeated uint32 repeatedField = 1 [(protons.options).limit = 10];
+
+  // stringMap cannot have more than 10 keys
+  map<string, string> stringMap = 2 [(protons.options).limit = 10];
+}
+```
+
+Or at runtime by passing objects to the `.decode` function of your message:
+
+```TypeScript
+const message = MyMessage.decode(buf, {
+  limits: {
+    repeatedField: 10,
+    stringMap: 10
+  }
+})
+```
+
+### Overriding 64 bit types
+
+By default 64 bit types are implemented as [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt)s.
+
+Sometimes this is undesirable due to [performance issues](https://betterprogramming.pub/the-downsides-of-bigints-in-javascript-6350fd807d) or code legibility.
+
+It's possible to override the JavaScript type 64 bit fields will deserialize to:
+
+```protobuf
+message MyMessage {
+  repeated int64 bigintField = 1;
+  repeated int64 numberField = 2 [jstype = JS_NUMBER];
+  repeated int64 stringField = 3 [jstype = JS_STRING];
+}
+```
+
+```TypeScript
+const message = MyMessage.decode(buf)
+
+console.info(typeof message.bigintField) // bigint
+console.info(typeof message.numberField) // number
+console.info(typeof message.stringField) // string
+```
+
 ## Missing features
@@ -82,7 +127,13 @@
 
-## API Docs
+# Install
 
+```console
+$ npm i protons
+```
+
+# API Docs
+
 - <https://ipfs.github.io/protons/modules/protons.html>
 
-## License
+# License
 
@@ -94,3 +145,3 @@ Licensed under either of
 
-## Contribute
+# Contribute
 
@@ -97,0 +148,0 @@ Contributions welcome! Please check out [the issues](https://github.com/ipfs/protons/issues).

src/index.ts

 /* eslint-disable max-depth */
 
+/**
+ * @packageDocumentation
+ *
+ * `protons` is a high performance implementation of [Protocol Buffers v3](https://protobuf.dev/programming-guides/proto3/).
+ *
+ * It transpiles code to TypeScript and supports BigInts for 64 bit types.
+ *
+ * The `protons` module contains the code to compile `.proto` files to `.ts` files and `protons-runtime` contains the code to do serialization/deserialization to `Uint8Array`s during application execution.
+ *
+ * Please ensure you declare them as the correct type of dependencies:
+ *
+ * ```console
+ * $ npm install --save-dev protons
+ * $ npm install --save protons-runtime
+ * ```
+ *
+ * ## Usage
+ *
+ * First generate your `.ts` files:
+ *
+ * ```console
+ * $ protons ./path/to/foo.proto ./path/to/output.ts
+ * ```
+ *
+ * Then run tsc over them as normal:
+ *
+ * ```console
+ * $ tsc
+ * ```
+ *
+ * In your code import the generated classes and use them to transform to/from bytes:
+ *
+ * ```js
+ * import { Foo } from './foo.js'
+ *
+ * const foo = {
+ *   message: 'hello world'
+ * }
+ *
+ * const encoded = Foo.encode(foo)
+ * const decoded = Foo.decode(encoded)
+ *
+ * console.info(decoded.message)
+ * // 'hello world'
+ * ```
+ *
+ * ## Differences from protobuf.js
+ *
+ * This module uses the internal reader/writer from `protobuf.js` as it is highly optimised and there's no point reinventing the wheel.
+ *
+ * It does have one or two differences:
+ *
+ * 1. Supports `proto3` semantics only
+ * 2. All 64 bit values are represented as `BigInt`s and not `Long`s (e.g. `int64`, `uint64`, `sint64` etc)
+ * 3. Unset `optional` fields are set on the deserialized object forms as `undefined` instead of the default values
+ * 4. `singular` fields set to default values are not serialized and are set to default values when deserialized if not set - protobuf.js [diverges from the language guide](https://github.com/protobufjs/protobuf.js/issues/1468#issuecomment-745177012) around this feature
+ * 5. `map` fields can have keys of any type - protobufs.js [only supports strings](https://github.com/protobufjs/protobuf.js/issues/1203#issuecomment-488637338)
+ * 6. `map` fields are deserialized as ES6 `Map`s - protobuf.js uses `Object`s
+ *
+ * ## Extra features
+ *
+ * ### Limiting the size of repeated/map elements
+ *
+ * To protect decoders from malicious payloads, it's possible to limit the maximum size of repeated/map elements.
+ *
+ * You can either do this at compile time by using the [protons.options](https://github.com/protocolbuffers/protobuf/blob/6f1d88107f268b8ebdad6690d116e74c403e366e/docs/options.md?plain=1#L490-L493) extension:
+ *
+ * ```protobuf
+ * message MyMessage {
+ *   // repeatedField cannot have more than 10 entries
+ *   repeated uint32 repeatedField = 1 [(protons.options).limit = 10];
+ *
+ *   // stringMap cannot have more than 10 keys
+ *   map<string, string> stringMap = 2 [(protons.options).limit = 10];
+ * }
+ * ```
+ *
+ * Or at runtime by passing objects to the `.decode` function of your message:
+ *
+ * ```TypeScript
+ * const message = MyMessage.decode(buf, {
+ *   limits: {
+ *     repeatedField: 10,
+ *     stringMap: 10
+ *   }
+ * })
+ * ```
+ *
+ * ### Overriding 64 bit types
+ *
+ * By default 64 bit types are implemented as [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt)s.
+ *
+ * Sometimes this is undesirable due to [performance issues](https://betterprogramming.pub/the-downsides-of-bigints-in-javascript-6350fd807d) or code legibility.
+ *
+ * It's possible to override the JavaScript type 64 bit fields will deserialize to:
+ *
+ * ```protobuf
+ * message MyMessage {
+ *   repeated int64 bigintField = 1;
+ *   repeated int64 numberField = 2 [jstype = JS_NUMBER];
+ *   repeated int64 stringField = 3 [jstype = JS_STRING];
+ * }
+ * ```
+ *
+ * ```TypeScript
+ * const message = MyMessage.decode(buf)
+ *
+ * console.info(typeof message.bigintField) // bigint
+ * console.info(typeof message.numberField) // number
+ * console.info(typeof message.stringField) // string
+ * ```
+ *
+ * ## Missing features
+ *
+ * Some features are missing `OneOf`s, etc due to them not being needed so far in ipfs/libp2p. If these features are important to you, please open PRs implementing them along with tests comparing the generated bytes to `protobuf.js` and `pbjs`.
+ */
+
 import fs from 'fs/promises'
@@ -521,2 +638,3 @@ import path from 'path'
   moduleDef.addTypeImport('protons-runtime', 'Codec')
+  moduleDef.addTypeImport('protons-runtime', 'DecodeOptions')
   moduleDef.addTypeImport('uint8arraylist', 'Uint8ArrayList')
@@ -695,8 +813,12 @@
         if (fieldDef.map) {
-          let limit = ''
+          moduleDef.addImport('protons-runtime', 'CodeError')
 
+          let limit = `
+              if (opts.limits?.${fieldName} != null && obj.${fieldName}.size === opts.limits.${fieldName}) {
+                throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_SIZE')
+              }
+`
+
           if (fieldDef.lengthLimit != null) {
-            moduleDef.addImport('protons-runtime', 'CodeError')
-
-            limit = `
+            limit += `
               if (obj.${fieldName}.size === ${fieldDef.lengthLimit}) {
@@ -714,8 +836,12 @@ throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_SIZE')
         } else if (fieldDef.repeated) {
-          let limit = ''
+          moduleDef.addImport('protons-runtime', 'CodeError')
 
+          let limit = `
+              if (opts.limits?.${fieldName} != null && obj.${fieldName}.length === opts.limits.${fieldName}) {
+                throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_LENGTH')
+              }
+`
+
           if (fieldDef.lengthLimit != null) {
-            moduleDef.addImport('protons-runtime', 'CodeError')
-
-            limit = `
+            limit += `
               if (obj.${fieldName}.length === ${fieldDef.lengthLimit}) {
@@ -756,3 +882,3 @@ throw new CodeError('decode error - repeated field "${fieldName}" had too many elements', 'ERR_MAX_LENGTH')
         }
-      }, (reader, length) => {
+      }, (reader, length, opts = {}) => {
         const obj: any = {${createDefaultObject(fields, messageDef, moduleDef)}}
@@ -784,4 +910,4 @@
 
-  export const decode = (buf: Uint8Array | Uint8ArrayList): ${messageDef.name} => {
-    return decodeMessage(buf, ${messageDef.name}.codec())
+  export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<${messageDef.name}>): ${messageDef.name} => {
+    return decodeMessage(buf, ${messageDef.name}.codec(), opts)
   }`
@@ -788,0 +914,0 @@

No alert changes

Improved metrics

Total package byte prevSize

110633

10.12%

Number of package files

13

8.33%

Lines of code

2054

14.24%

Number of lines in readme file

153

50%

Dependency changes

• Updated

  meow@^13.1.0