mongo-cursor-pagination - npm Package Compare versions

Comparing version 8.1.0 to 8.1.1

CHANGELOG.md

@@ -0,1 +1,8 @@
+### [8.1.1](https://github.com/mixmaxhq/mongo-cursor-pagination/compare/v8.1.0...v8.1.1) (2022-08-26)
+
+
+### Bug Fixes
+
+* properly page through undefs and nulls ([0eb28e7](https://github.com/mixmaxhq/mongo-cursor-pagination/commit/0eb28e7f573511ef7fc8790e9c0fc84e5a997cef))
+
 ## [8.1.0](https://github.com/mixmaxhq/mongo-cursor-pagination/compare/v8.0.1...v8.1.0) (2022-08-25)
@@ -2,0 +9,0 @@

dist/node/aggregate.js

@@ -39,2 +39,3 @@ "use strict";
  *          aggregation steps added at the end of the pipeline to implement the paging can access it.
+          4. Consistent. All values (except undefined and null values) must be of the same type.
  *      The default is to use the Mongo built-in '_id' field, which satisfies the above criteria.
@@ -41,0 +42,0 @@ *      The only reason to NOT use the Mongo _id field is if you chose to implement your own ids.

dist/node/find.js

@@ -32,2 +32,3 @@ "use strict";
  *        3. Immutable. If the value changes between paged queries, it could appear twice.
+          4. Consistent. All values (except undefined and null values) must be of the same type.
  *      The default is to use the Mongo built-in '_id' field, which satisfies the above criteria.
@@ -34,0 +35,0 @@ *      The only reason to NOT use the Mongo _id field is if you chose to implement your own ids.

dist/node/utils/bsonUrlEncoding.js

@@ -7,10 +7,13 @@ "use strict";
 
-const base64url = require('base64-url');
+const base64url = require('base64-url'); // BSON can't encode undefined values, so we will use this value instead:
+
+
+const BSON_UNDEFINED = '__mixmax__undefined__';
 /**
- * These will take a BSON object (an database result returned by the MongoDB library) and
- * encode/decode as a URL-safe string.
+ * These will take a paging handle (`next` or `previous`) and encode/decode it
+ * as a string which can be passed in a URL.
  */
 
-
 module.exports.encode = function (obj) {
+  if (Array.isArray(obj) && obj[0] === undefined) obj[0] = BSON_UNDEFINED;
   return base64url.encode(EJSON.stringify(obj));
@@ -20,3 +23,5 @@ };
 module.exports.decode = function (str) {
-  return EJSON.parse(base64url.decode(str));
+  const obj = EJSON.parse(base64url.decode(str));
+  if (Array.isArray(obj) && obj[0] === BSON_UNDEFINED) obj[0] = undefined;
+  return obj;
 };

dist/node/utils/query.js

@@ -28,4 +28,9 @@ "use strict";
     let previousPaginatedField = objectPath.get(response.previous, params.paginatedField);
-    if (params.sortCaseInsensitive) previousPaginatedField = previousPaginatedField.toLowerCase();
 
+    if (params.sortCaseInsensitive) {
+      var _previousPaginatedFie, _previousPaginatedFie2, _previousPaginatedFie3;
+
+      previousPaginatedField = (_previousPaginatedFie = (_previousPaginatedFie2 = previousPaginatedField) === null || _previousPaginatedFie2 === void 0 ? void 0 : (_previousPaginatedFie3 = _previousPaginatedFie2.toLowerCase) === null || _previousPaginatedFie3 === void 0 ? void 0 : _previousPaginatedFie3.call(_previousPaginatedFie2)) !== null && _previousPaginatedFie !== void 0 ? _previousPaginatedFie : '';
+    }
+
     if (shouldSecondarySortOnId) {
@@ -40,4 +45,9 @@ response.previous = bsonUrlEncoding.encode([previousPaginatedField, response.previous._id]);
     let nextPaginatedField = objectPath.get(response.next, params.paginatedField);
-    if (params.sortCaseInsensitive) nextPaginatedField = nextPaginatedField.toLowerCase();
 
+    if (params.sortCaseInsensitive) {
+      var _nextPaginatedField$t, _nextPaginatedField, _nextPaginatedField$t2;
+
+      nextPaginatedField = (_nextPaginatedField$t = (_nextPaginatedField = nextPaginatedField) === null || _nextPaginatedField === void 0 ? void 0 : (_nextPaginatedField$t2 = _nextPaginatedField.toLowerCase) === null || _nextPaginatedField$t2 === void 0 ? void 0 : _nextPaginatedField$t2.call(_nextPaginatedField)) !== null && _nextPaginatedField$t !== void 0 ? _nextPaginatedField$t : '';
+    }
+
     if (shouldSecondarySortOnId) {
@@ -115,4 +125,3 @@ response.next = bsonUrlEncoding.encode([nextPaginatedField, response.next._id]);
     if (!params.next && !params.previous) return {};
-    const sortAsc = !params.sortAscending && params.previous || params.sortAscending && !params.previous;
-    const comparisonOp = sortAsc ? '$gt' : '$lt'; // a `next` cursor will have precedence over a `previous` cursor.
+    const sortAsc = !params.sortAscending && params.previous || params.sortAscending && !params.previous; // a `next` cursor will have precedence over a `previous` cursor.
 
@@ -122,23 +131,117 @@ const op = params.next || params.previous;
     if (params.paginatedField == '_id') {
-      return {
-        _id: {
-          [comparisonOp]: op
+      if (sortAsc) {
+        return {
+          _id: {
+            $gt: op
+          }
+        };
+      } else {
+        return {
+          _id: {
+            $lt: op
+          }
+        };
+      }
+    } else {
+      const field = params.sortCaseInsensitive ? '__lc' : params.paginatedField;
+      const notUndefined = {
+        [field]: {
+          $exists: true
         }
       };
-    } else {
-      const field = params.sortCaseInsensitive ? '__lc' : params.paginatedField;
-      return {
-        $or: [{
+      const onlyUndefs = {
+        [field]: {
+          $exists: false
+        }
+      };
+      const notNullNorUndefined = {
+        [field]: {
+          $ne: null
+        }
+      };
+      const nullOrUndefined = {
+        [field]: null
+      };
+      const onlyNulls = {
+        $and: [{
           [field]: {
-            [comparisonOp]: op[0]
+            $exists: true
           }
         }, {
-          [field]: {
-            $eq: op[0]
-          },
-          _id: {
-            [comparisonOp]: op[1]
-          }
+          [field]: null
         }]
       };
+      const [paginatedFieldValue, idValue] = op;
+
+      switch (paginatedFieldValue) {
+        case null:
+          if (sortAsc) {
+            return {
+              $or: [notNullNorUndefined, { ...onlyNulls,
+                _id: {
+                  $gt: idValue
+                }
+              }]
+            };
+          } else {
+            return {
+              $or: [onlyUndefs, { ...onlyNulls,
+                _id: {
+                  $lt: idValue
+                }
+              }]
+            };
+          }
+
+        case undefined:
+          if (sortAsc) {
+            return {
+              $or: [notUndefined, { ...onlyUndefs,
+                _id: {
+                  $gt: idValue
+                }
+              }]
+            };
+          } else {
+            return { ...onlyUndefs,
+              _id: {
+                $lt: idValue
+              }
+            };
+          }
+
+        default:
+          if (sortAsc) {
+            return {
+              $or: [{
+                [field]: {
+                  $gt: paginatedFieldValue
+                }
+              }, {
+                [field]: {
+                  $eq: paginatedFieldValue
+                },
+                _id: {
+                  $gt: idValue
+                }
+              }]
+            };
+          } else {
+            return {
+              $or: [{
+                [field]: {
+                  $lt: paginatedFieldValue
+                }
+              }, nullOrUndefined, {
+                [field]: {
+                  $eq: paginatedFieldValue
+                },
+                _id: {
+                  $lt: idValue
+                }
+              }]
+            };
+          }
+
+      }
     }
@@ -145,0 +248,0 @@ }

package.json

 {
   "name": "mongo-cursor-pagination",
-  "version": "8.1.0",
+  "version": "8.1.1",
   "description": "Make it easy to return cursor-paginated results from a Mongo collection",
@@ -42,3 +42,3 @@ "main": "index.js",
     "base64-url": "^2.2.0",
-    "bson": "^4.1.0",
+    "bson": "^4.7.0",
     "object-path": "^0.11.5",
@@ -65,3 +65,3 @@ "projection-utils": "^1.1.0",
     "mongodb-memory-server": "^5.2.11",
-    "mongoist": "2.3.0",
+    "mongoist": "^2.5.5",
     "mongoose": "5.11.10",
@@ -68,0 +68,0 @@ "prettier": "^1.19.1",

README.md

@@ -57,3 +57,3 @@ # mongo-cursor-pagination
           3. Immutable. If the value changes between paged queries, it could appear twice.
-          4. Complete. A value must exist for all documents.
+          4. Consistent. All values (except undefined and null values) must be of the same type.
         The default is to use the Mongo built-in '_id' field, which satisfies the above criteria.
@@ -60,0 +60,0 @@ The only reason to NOT use the Mongo _id field is if you chose to implement your own ids.

src/aggregate.js

@@ -31,2 +31,3 @@ const _ = require('underscore');
  *          aggregation steps added at the end of the pipeline to implement the paging can access it.
+          4. Consistent. All values (except undefined and null values) must be of the same type.
  *      The default is to use the Mongo built-in '_id' field, which satisfies the above criteria.
@@ -33,0 +34,0 @@ *      The only reason to NOT use the Mongo _id field is if you chose to implement your own ids.

src/find.js

@@ -23,2 +23,3 @@ const _ = require('underscore');
  *        3. Immutable. If the value changes between paged queries, it could appear twice.
+          4. Consistent. All values (except undefined and null values) must be of the same type.
  *      The default is to use the Mongo built-in '_id' field, which satisfies the above criteria.
@@ -25,0 +26,0 @@ *      The only reason to NOT use the Mongo _id field is if you chose to implement your own ids.

src/utils/bsonUrlEncoding.js

 const { EJSON } = require('bson');
 const base64url = require('base64-url');
 
+// BSON can't encode undefined values, so we will use this value instead:
+const BSON_UNDEFINED = '__mixmax__undefined__';
+
 /**
- * These will take a BSON object (an database result returned by the MongoDB library) and
- * encode/decode as a URL-safe string.
+ * These will take a paging handle (`next` or `previous`) and encode/decode it
+ * as a string which can be passed in a URL.
  */
 
 module.exports.encode = function(obj) {
+  if (Array.isArray(obj) && obj[0] === undefined) obj[0] = BSON_UNDEFINED;
   return base64url.encode(EJSON.stringify(obj));
@@ -14,3 +18,5 @@ };
 module.exports.decode = function(str) {
-  return EJSON.parse(base64url.decode(str));
+  const obj = EJSON.parse(base64url.decode(str));
+  if (Array.isArray(obj) && obj[0] === BSON_UNDEFINED) obj[0] = undefined;
+  return obj;
 };

src/utils/query.js

@@ -24,3 +24,5 @@ const bsonUrlEncoding = require('./bsonUrlEncoding');
     let previousPaginatedField = objectPath.get(response.previous, params.paginatedField);
-    if (params.sortCaseInsensitive) previousPaginatedField = previousPaginatedField.toLowerCase();
+    if (params.sortCaseInsensitive) {
+      previousPaginatedField = previousPaginatedField?.toLowerCase?.() ?? '';
+    }
     if (shouldSecondarySortOnId) {
@@ -34,3 +36,5 @@ response.previous = bsonUrlEncoding.encode([previousPaginatedField, response.previous._id]);
     let nextPaginatedField = objectPath.get(response.next, params.paginatedField);
-    if (params.sortCaseInsensitive) nextPaginatedField = nextPaginatedField.toLowerCase();
+    if (params.sortCaseInsensitive) {
+      nextPaginatedField = nextPaginatedField?.toLowerCase?.() ?? '';
+    }
     if (shouldSecondarySortOnId) {
@@ -117,3 +121,2 @@ response.next = bsonUrlEncoding.encode([nextPaginatedField, response.next._id]);
       (!params.sortAscending && params.previous) || (params.sortAscending && !params.previous);
-    const comparisonOp = sortAsc ? '$gt' : '$lt';
 
@@ -124,28 +127,83 @@ // a `next` cursor will have precedence over a `previous` cursor.
     if (params.paginatedField == '_id') {
-      return {
-        _id: {
-          [comparisonOp]: op,
-        },
-      };
+      if (sortAsc) {
+        return { _id: { $gt: op } };
+      } else {
+        return { _id: { $lt: op } };
+      }
     } else {
       const field = params.sortCaseInsensitive ? '__lc' : params.paginatedField;
-      return {
-        $or: [
-          {
-            [field]: {
-              [comparisonOp]: op[0],
-            },
-          },
-          {
-            [field]: {
-              $eq: op[0],
-            },
-            _id: {
-              [comparisonOp]: op[1],
-            },
-          },
-        ],
-      };
+
+      const notUndefined = { [field]: { $exists: true } };
+      const onlyUndefs = { [field]: { $exists: false } };
+      const notNullNorUndefined = { [field]: { $ne: null } };
+      const nullOrUndefined = { [field]: null };
+      const onlyNulls = { $and: [{ [field]: { $exists: true } }, { [field]: null }] };
+
+      const [paginatedFieldValue, idValue] = op;
+      switch (paginatedFieldValue) {
+        case null:
+          if (sortAsc) {
+            return {
+              $or: [
+                notNullNorUndefined,
+                {
+                  ...onlyNulls,
+                  _id: { $gt: idValue },
+                },
+              ],
+            };
+          } else {
+            return {
+              $or: [
+                onlyUndefs,
+                {
+                  ...onlyNulls,
+                  _id: { $lt: idValue },
+                },
+              ],
+            };
+          }
+        case undefined:
+          if (sortAsc) {
+            return {
+              $or: [
+                notUndefined,
+                {
+                  ...onlyUndefs,
+                  _id: { $gt: idValue },
+                },
+              ],
+            };
+          } else {
+            return {
+              ...onlyUndefs,
+              _id: { $lt: idValue },
+            };
+          }
+        default:
+          if (sortAsc) {
+            return {
+              $or: [
+                { [field]: { $gt: paginatedFieldValue } },
+                {
+                  [field]: { $eq: paginatedFieldValue },
+                  _id: { $gt: idValue },
+                },
+              ],
+            };
+          } else {
+            return {
+              $or: [
+                { [field]: { $lt: paginatedFieldValue } },
+                nullOrUndefined,
+                {
+                  [field]: { $eq: paginatedFieldValue },
+                  _id: { $lt: idValue },
+                },
+              ],
+            };
+          }
+      }
     }
   },
 };

No alert changes

Improved metrics

Total package byte prevSize

91873

6.75%

Lines of code

1638

11.28%

Dependency changes

• Updated

  bson@^4.7.0