fast-json-stringify - npm Package Compare versions

Comparing version 3.0.3 to 3.1.0

bench.js

@@ -6,2 +6,6 @@ 'use strict'
 
+const STR_LEN = 1e4
+const LARGE_ARRAY_SIZE = 2e4
+const MULTI_ARRAY_LENGHT = 1e3
+
 const schema = {
@@ -93,3 +97,4 @@ title: 'Example Schema',
 
-const multiArray = []
+const multiArray = new Array(MULTI_ARRAY_LENGHT)
+const largeArray = new Array(LARGE_ARRAY_SIZE)
 
@@ -104,3 +109,6 @@ const CJS = require('compile-json-stringify')
 const stringify = FJS(schema)
-const stringifyArray = FJS(arraySchema)
+const stringifyArrayDefault = FJS(arraySchema)
+const stringifyArrayJSONStringify = FJS(arraySchema, {
+  largeArrayMechanism: 'json-stringify'
+})
 const stringifyDate = FJS(dateFormatSchema)
@@ -116,4 +124,26 @@ const stringifyString = FJS({ type: 'string' })
 
+const getRandomString = (length) => {
+  if (!Number.isInteger(length)) {
+    throw new Error('Expected integer length')
+  }
+
+  const validCharacters = 'abcdefghijklmnopqrstuvwxyz'
+  const nValidCharacters = 26
+
+  let result = ''
+  for (let i = 0; i < length; ++i) {
+    result += validCharacters[Math.floor(Math.random() * nValidCharacters)]
+  }
+
+  return result[0].toUpperCase() + result.slice(1)
+}
+
 // eslint-disable-next-line
-for (var i = 0; i < 10000; i++) {
+for (let i = 0; i < STR_LEN; i++) {
+  largeArray[i] = {
+    firstName: getRandomString(8),
+    lastName: getRandomString(6),
+    age: Math.ceil(Math.random() * 99)
+  }
+
   str += i
@@ -125,6 +155,14 @@ if (i % 100 === 0) {
 
+for (let i = STR_LEN; i < LARGE_ARRAY_SIZE; ++i) {
+  largeArray[i] = {
+    firstName: getRandomString(10),
+    lastName: getRandomString(4),
+    age: Math.ceil(Math.random() * 99)
+  }
+}
+
 Number(str)
 
-for (i = 0; i < 1000; i++) {
-  multiArray.push(obj)
+for (let i = 0; i < MULTI_ARRAY_LENGHT; i++) {
+  multiArray[i] = obj
 }
@@ -146,6 +184,10 @@
 
-suite.add('fast-json-stringify array', function () {
-  stringifyArray(multiArray)
+suite.add('fast-json-stringify array default', function () {
+  stringifyArrayDefault(multiArray)
 })
 
+suite.add('fast-json-stringify array json-stringify', function () {
+  stringifyArrayJSONStringify(multiArray)
+})
+
 suite.add('compile-json-stringify array', function () {
@@ -159,2 +201,22 @@ CJSStringifyArray(multiArray)
 
+suite.add('JSON.stringify large array', function () {
+  JSON.stringify(largeArray)
+})
+
+suite.add('fast-json-stringify large array default', function () {
+  stringifyArrayDefault(largeArray)
+})
+
+suite.add('fast-json-stringify large array json-stringify', function () {
+  stringifyArrayJSONStringify(largeArray)
+})
+
+suite.add('compile-json-stringify large array', function () {
+  CJSStringifyArray(largeArray)
+})
+
+suite.add('AJV Serialize large array', function () {
+  ajvSerializeArray(largeArray)
+})
+
 suite.add('JSON.stringify long string', function () {
@@ -161,0 +223,0 @@ JSON.stringify(str)

example.js

 const moment = require('moment')
-const fastJson = require('fast-json-stringify')
+const fastJson = require('.')
 const stringify = fastJson({
@@ -4,0 +4,0 @@ title: 'Example Schema',

index.js

@@ -6,2 +6,3 @@ 'use strict'
 const Ajv = require('ajv')
+const fastUri = require('fast-uri')
 const ajvFormats = require('ajv-formats')
@@ -14,3 +15,10 @@ const merge = require('deepmerge')
 const validate = require('./schema-validator')
+
+let largeArraySize = 2e4
 let stringSimilarity = null
+let largeArrayMechanism = 'default'
+const validLargeArrayMechanisms = [
+  'default',
+  'json-stringify'
+]
 
@@ -57,3 +65,3 @@ const addComma = `
 
-  ajvInstance = new Ajv({ ...options.ajv, strictSchema: false })
+  ajvInstance = new Ajv({ ...options.ajv, strictSchema: false, uriResolver: fastUri })
   ajvFormats(ajvInstance)
@@ -78,2 +86,18 @@
 
+  if (options.largeArrayMechanism) {
+    if (validLargeArrayMechanisms.includes(options.largeArrayMechanism)) {
+      largeArrayMechanism = options.largeArrayMechanism
+    } else {
+      throw new Error(`Unsupported large array mechanism ${options.rounding}`)
+    }
+  }
+
+  if (options.largeArraySize) {
+    if (!Number.isNaN(Number.parseInt(options.largeArraySize, 10))) {
+      largeArraySize = options.largeArraySize
+    } else {
+      throw new Error(`Unsupported large array size. Expected integer-like, got ${options.largeArraySize}`)
+    }
+  }
+
   /* eslint no-new-func: "off" */
@@ -196,3 +220,3 @@ let code = `
  * Infer type based on keyword in order to generate optimized code
- * https://json-schema.org/latest/json-schema-validation.html#rfc.section.6
+ * https://datatracker.ietf.org/doc/html/draft-handrews-json-schema-validation-01#section-6
  */
@@ -1036,2 +1060,7 @@ function inferTypeByKeyword (schema) {
     var l = obj.length
+    if (l && l >= ${largeArraySize}) {`
+
+  const concatSnippet = `
+    }
+
     var jsonOutput= ''
@@ -1048,3 +1077,21 @@ for (var i = 0; i < l; i++) {
     return \`[\${jsonOutput}]\`
+  }`
+
+  switch (largeArrayMechanism) {
+    case 'default':
+      code += `
+      return \`[\${obj.map(${result.mapFnName}).join(',')}]\``
+      break
+
+    case 'json-stringify':
+      code += `
+      return JSON.stringify(obj)`
+      break
+
+    default:
+      throw new Error(`Unsupported large array mechanism ${largeArrayMechanism}`)
   }
+
+  code += `
+  ${concatSnippet}
   ${result.laterCode}
@@ -1157,2 +1204,3 @@ `
     case 'null':
+      funcName = '$asNull'
       code += `
@@ -1163,2 +1211,3 @@ json += $asNull()
     case 'string': {
+      funcName = '$asString'
       const stringSerializer = getStringSerializer(schema.format)
@@ -1169,8 +1218,11 @@ code += nullable ? `json += obj${accessor} === null ? null : ${stringSerializer}(obj${accessor})` : `json += ${stringSerializer}(obj${accessor})`
     case 'integer':
+      funcName = '$asInteger'
       code += nullable ? `json += obj${accessor} === null ? null : $asInteger(obj${accessor})` : `json += $asInteger(obj${accessor})`
       break
     case 'number':
+      funcName = '$asNumber'
       code += nullable ? `json += obj${accessor} === null ? null : $asNumber(obj${accessor})` : `json += $asNumber(obj${accessor})`
       break
     case 'boolean':
+      funcName = '$asBoolean'
       code += nullable ? `json += obj${accessor} === null ? null : $asBoolean(obj${accessor})` : `json += $asBoolean(obj${accessor})`
@@ -1193,2 +1245,3 @@ break
     case undefined:
+      funcName = '$asNull'
       if ('anyOf' in schema) {
@@ -1332,3 +1385,4 @@ // beware: dereferenceOfRefs has side effects and changes schema.anyOf
     code,
-    laterCode
+    laterCode,
+    mapFnName: funcName
   }
@@ -1349,2 +1403,4 @@ }
 
+module.exports.validLargeArrayMechanisms = validLargeArrayMechanisms
+
 module.exports.restore = function ({ code, ajv }) {
@@ -1351,0 +1407,0 @@ // eslint-disable-next-line

package.json

 {
   "name": "fast-json-stringify",
-  "version": "3.0.3",
+  "version": "3.1.0",
   "description": "Stringify your JSON at max speed",
@@ -42,3 +42,3 @@ "main": "index.js",
     "standard": "^16.0.1",
-    "tap": "^15.0.0",
+    "tap": "^16.0.1",
     "typescript": "^4.0.2",
@@ -48,5 +48,6 @@ "webpack": "^5.40.0"
   "dependencies": {
-    "ajv": "^8.6.2",
+    "ajv": "^8.10.0",
     "ajv-formats": "^2.1.1",
     "deepmerge": "^4.2.2",
+    "fast-uri": "^1.0.1",
     "rfdc": "^1.2.0",
@@ -53,0 +54,0 @@ "string-similarity": "^4.0.1"

README.md

@@ -59,3 +59,3 @@ # fast-json-stringify
  - <a href="#additionalProperties">`Additional Properties`</a>
- - <a href="#anyof">`AnyOf` and `OneOf`</a>
+ - <a href="#AnyOf-and-OneOf">`AnyOf` and `OneOf`</a>
  - <a href="#ref">`Reuse - $ref`</a>
@@ -65,2 +65,3 @@ - <a href="#long">`Long integers`</a>
  - <a href="#nullable">`Nullable`</a>
+ - <a href="#largearrays">`Large Arrays`</a>
 - <a href="#security">`Security Notice`</a>
@@ -122,2 +123,4 @@ - <a href="#acknowledgements">`Acknowledgements`</a>
 - `rounding`: setup how the `integer` types will be rounded when not integers. [More details](#integer)
+- `largeArrayMechanism`: set the mechanism that should be used to handle large
+(by default `20000` or more items) arrays. [More details](#largearrays)
 
@@ -274,3 +277,3 @@
   matchfoo: 42,
-  otherfoo: 'str'
+  otherfoo: 'str',
   matchnum: 3
@@ -318,3 +321,3 @@ }
   matchfoo: 42,
-  otherfoo: 'str'
+  otherfoo: 'str',
   matchnum: 3,
@@ -590,2 +593,55 @@ nomatchstr: 'valar morghulis',
 
+<a name="largearrays"></a>
+#### Large Arrays
+
+Large arrays are, for the scope of this document, defined as arrays containing,
+by default, `20000` elements or more. That value can be adjusted via the option
+parameter `largeArraySize`.
+
+At some point the overhead caused by the default mechanism used by
+`fast-json-stringify` to handle arrays starts increasing exponentially, leading
+to slow overall executions.
+
+##### Settings
+
+In order to improve that the user can set the `largeArrayMechanism` and
+`largeArraySize` options.
+
+`largeArrayMechanism`'s default value is `default`. Valid values for it are:
+
+- `default` - This option is a compromise between performance and feature set by
+still providing the expected functionality out of this lib but giving up some
+possible performance gain. With this option set, **large arrays** would be
+stringified by joining their stringified elements using `Array.join` instead of
+string concatenation for better performance
+- `json-stringify` - This option will remove support for schema validation
+within **large arrays** completely. By doing so the overhead previously
+mentioned is nulled, greatly improving execution time. Mind there's no change
+in behavior for arrays not considered _large_
+
+`largeArraySize`'s default value is `20000`. Valid values for it are
+integer-like values, such as:
+
+- `20000`
+- `2e4`
+- `'20000'`
+- `'2e4'` - _note this will be converted to `2`, not `20000`_
+- `1.5` - _note this will be converted to `1`_
+
+##### Benchmarks
+
+For reference, here goes some benchmarks for comparison over the three
+mechanisms. Benchmarks conducted on an old machine.
+
+- Machine: `ST1000LM024 HN-M 1TB HDD, Intel Core i7-3610QM @ 2.3GHz, 12GB RAM, 4C/8T`.
+- Node.js `v16.13.1`
+
+```
+JSON.stringify large array x 157 ops/sec ±0.73% (86 runs sampled)
+fast-json-stringify large array default x 48.72 ops/sec ±4.92% (48 runs sampled)
+fast-json-stringify large array json-stringify x 157 ops/sec ±0.76% (86 runs sampled)
+compile-json-stringify large array x 175 ops/sec ±4.47% (79 runs sampled)
+AJV Serialize large array x 58.76 ops/sec ±4.59% (60 runs sampled)
+```
+
 <a name="security"></a>
@@ -592,0 +648,0 @@ ## Security notice

test/array.test.js

@@ -8,3 +8,3 @@ 'use strict'
 
-function buildTest (schema, toStringify) {
+function buildTest (schema, toStringify, options) {
   test(`render a ${schema.title} as JSON`, (t) => {
@@ -14,3 +14,3 @@ t.plan(3)
     const validate = validator(schema)
-    const stringify = build(schema)
+    const stringify = build(schema, options)
     const output = stringify(toStringify)
@@ -324,1 +324,45 @@
 })
+
+const largeArray = new Array(2e4).fill({ a: 'test', b: 1 })
+buildTest({
+  title: 'large array with default mechanism',
+  type: 'object',
+  properties: {
+    ids: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          a: { type: 'string' },
+          b: { type: 'number' }
+        }
+      }
+    }
+  }
+}, {
+  ids: largeArray
+}, {
+  largeArraySize: 2e4,
+  largeArrayMechanism: 'default'
+})
+
+buildTest({
+  title: 'large array with json-stringify mechanism',
+  type: 'object',
+  properties: {
+    ids: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          a: { type: 'string' },
+          b: { type: 'number' }
+        }
+      }
+    }
+  }
+}, {
+  ids: largeArray
+}, {
+  largeArrayMechanism: 'json-stringify'
+})

No alert changes

Improved metrics

Total package byte prevSize

275137

increased by2.36%

Lines of code

10063

increased by1.38%

Number of lines in readme file

687

increased by8.87%

Worsened metrics

Dependency count

6

increased by20%

Dependency changes

• + Addedfast-uri@^1.0.1

• + Addedfast-uri@1.0.1(transitive)

• Updatedajv@^8.10.0