couchdb-iterator - npm Package Compare versions

Comparing version 1.1.0 to 2.0.0

index.js

@@ -10,2 +10,3 @@ 'use strict';
 const iteratorCaller = require('./lib/iteratorCaller');
+const iteratorCallerBulk = require('./lib/iteratorCallerBulk');
 
@@ -61,4 +62,4 @@ const allowedQueryOptions = [
 
-function getQueryOptions(options) {
-    const queryOptions = mapKeys(omit(options, ['nano', 'concurrency']), (value, key) => snakeCase(key));
+function getQueryOptions(options, extra) {
+    const queryOptions = mapKeys(omit(options, extra), (value, key) => snakeCase(key));
     const invalidQueryOption = find(Object.keys(queryOptions), (queryOption) => allowedQueryOptions.indexOf(queryOption) === -1);
@@ -82,7 +83,9 @@
 
-    options = Object.assign({ limit: 500, concurrency: 50 }, options);
+    options = Object.assign({ concurrency: 50 }, options);
+    options.limit = options.limit || options.concurrency * 10;
+
     return new Promise((resolve, reject) => {
         const couchdb = getCouchDb(couchdbAddr, options);
         const queryFn = getQueryFn(couchdb, view);
-        const queryOptions = getQueryOptions(options);
+        const queryOptions = getQueryOptions(options, ['nano', 'concurrency']);
 
@@ -105,2 +108,34 @@ // Start the iteration!
 
+function couchdbIteratorBulk(couchdbAddr, view, iterator, options) {
+    if (typeof view === 'function') {
+        options = iterator;
+        iterator = view;
+        view = null;
+    }
+
+    options = Object.assign({ bulkSize: 50 }, options);
+    options.limit = options.limit || options.bulkSize * 10;
+
+    return new Promise((resolve, reject) => {
+        const couchdb = getCouchDb(couchdbAddr, options);
+        const queryFn = getQueryFn(couchdb, view);
+        const queryOptions = getQueryOptions(options, ['nano', 'bulkSize']);
+
+        // Start the iteration!
+        const rowsReaderStream = rowsReader(queryFn, queryOptions);
+        const iteratorCallerStream = iteratorCallerBulk(iterator, options.bulkSize);
+
+        rowsReaderStream
+        .on('error', reject)
+        .pipe(iteratorCallerStream)
+        .on('error', reject)
+        .on('end', () => resolve(iteratorCallerStream.getCount()));
+
+        iteratorCallerStream.on('readable', () => {
+            while (iteratorCallerStream.read() !== null) { /* do nothing */ }
+        });
+    });
+}
+
 module.exports = couchdbIterator;
+module.exports.bulk = couchdbIteratorBulk;

lib/iteratorCaller.js

@@ -20,8 +20,8 @@ 'use strict';
     _transform(data, encoding, callback) {
+        data.index = this._count++;  // eslint-disable-line no-plusplus
+
         // Execute the iterator
         // Note that the iterator can throw synchronously as well as return non-promise values
-        return new Promise((resolve, reject) => {
-            Promise.resolve(this._iterator(data, this._count++))  // eslint-disable-line no-plusplus
-            .then(resolve, reject);
-        })
+        return Promise.resolve()
+        .then(() => this._iterator(data))
         .then(() => callback(null, data), (err) => callback(err))
@@ -28,0 +28,0 @@ // Equivalent to .done()

package.json

 {
   "name": "couchdb-iterator",
-  "version": "1.1.0",
+  "version": "2.0.0",
   "description": "A fast and easy to ease CouchDB iterator for views and all documents",
@@ -5,0 +5,0 @@ "main": "index.js",

README.md

@@ -27,5 +27,7 @@ # couchdb-iterator
 
-`.couchdbIterator(couchdbAddr, [view], iterator, [options])`
+### Concurrent row iteration
 
-Calls `iterator` for all rows of the database referenced by `couchdbAddr`.
+`couchdbIterator(couchdbAddr, [view], iterator, [options])`
+
+Calls `iterator` for each row of the database referenced by `couchdbAddr`.
 If a `view` is supplied, iterates only over that view's rows.
@@ -42,3 +44,3 @@
 // Iterate over all rows of a database
-couchdbIterator('http://localhost:5984/my-db', 'my-designdoc/my-view', (row, index) => {
+couchdbIterator('http://localhost:5984/my-db', (row, index) => {
     console.log(index, row.id, row.key, row.value);
@@ -54,3 +56,3 @@ // Do something with `row`; you may return a promise here
 // Iterate over all rows of a view
-couchdbIterator('http://localhost:5984/my-db', 'my-designdoc/my-view', (row, index) => {
+couchdbIterator('http://localhost:5984/my-db', 'my-design-doc/my-view', (row, index) => {
     console.log(index, row.id, row.key, row.value);
@@ -66,3 +68,3 @@ // Do something with `row`; you may return a promise here
 
-The `couchdbAddr` argument must be connection string with protocol, host, port and database path (e.g.: http://localhost:5984/my-db) or a [nano](https://www.npmjs.com/package/nano) instance. The `view` argument is a string in the form of `designdoc/view` (e.g.: app/byUser).
+The `couchdbAddr` argument must be a connection string with protocol, host, port and database path (e.g.: http://localhost:5984/my-db) or a [nano](https://www.npmjs.com/package/nano) instance. The `view` argument is a string in the form of `design-doc/view-name` (e.g.: app/byUser).
 
@@ -75,5 +77,57 @@ Available options:
 
-All querying options have no default value, except for `limit` which is `500`. Also, `stale` is automatically set to `ok` after the first iteration to further improve performance.
+All querying options have no default value, except for `limit` which is `concurrency * 10`. Also, `stale` is automatically set to `ok` after the first iteration to further improve performance.
 
 
+### Bulk iteration
+
+`couchdbIterator.bulk(couchdbAddr, [view], iterator, [options])`
+
+Calls `iterator` for a bulk of rows of the database referenced by `couchdbAddr`.
+If a `view` is supplied, iterates only over that view's rows.
+
+This method is similar to `couchdbIterator()` but iterates in bulks and it respects the order of rows. The order is respected because since a bulk is ordered and the next bulk only comes when the current bulk is handled.
+
+Examples:
+
+```js
+const couchdbIterator = require('couchdb-iterator');
+
+// Iterate over all rows of a database
+couchdbIterator.bulk('http://localhost:5984/my-db', (rows) => {
+    rows.forEach((row) => {
+        console.log(row.index, row.id, row.key, row.value);
+    });
+    // Do something with `rows`; you may return a promise here
+})
+.then((rowsCount) => {
+    console.log(`Iteration completed! ${rowsCount}`);
+}, (err) => {
+    console.log('Iteration failed', err);
+});
+
+// Iterate over all rows of a view
+couchdbIterator.bulk('http://localhost:5984/my-db', 'my-design-doc/my-view', (rows) => {
+    rows.forEach((row) => {
+        console.log(row.index, row.id, row.key, row.value);
+    });
+    // Do something with `rows`; you may return a promise here
+})
+.then((rowsCount) => {
+    console.log(`Iteration completed! ${rowsCount}`);
+}, (err) => {
+    console.log('Iteration failed', err);
+});
+```
+
+The `couchdbAddr` argument must be a connection string with protocol, host, port and database path (e.g.: http://localhost:5984/my-db) or a [nano](https://www.npmjs.com/package/nano) instance. The `view` argument is a string in the form of `design-doc/view-name` (e.g.: app/byUser).
+
+Available options:
+
+- `bulkSize`: The bulkSize, defaults to `50`.
+- `nano`: Custom options to be used when creating the [nano]((https://www.npmjs.com/package/nano)) instance, defaults to `null`.
+- The following [querying options](https://wiki.apache.org/couchdb/HTTP_view_API) are available: `limit`, `skip`, `stale`, `descending`, `startkey`, `startkey_docid`, `endkey`, `endkey_docid`, `include_docs` and `inclusive_end` (can be camelCased).
+
+All querying options have no default value, except for `limit` which is `bulkSize * 10`. Also, `stale` is automatically set to `ok` after the first iteration to further improve performance.
+
+
 ## Tests
@@ -84,5 +138,7 @@
 
+The tests expect a running CouchDB in `http://localhost:5984` but you may specify a different address with `COUCHDB`, e.g.: `$ COUCHDB=http://admin:admin@localhost:5984 npm test`.
 
+
 ## License
 
 Released under the [MIT License](http://www.opensource.org/licenses/mit-license.php).

test/test.js

@@ -12,3 +12,3 @@ 'use strict';
 
-describe('couchdb-iterator', () => {
+describe('couchdbIterator()', () => {
     let prepared;
@@ -26,7 +26,7 @@
 
-        return couchdbIterator(prepared.couchdbAddr, (row, index) => {
-            expect(index).to.equal(count);
+        return couchdbIterator(prepared.couchdbAddr, (row) => {
+            expect(row.index).to.equal(count);
             count += 1;
 
-            expect(row).to.have.all.keys('id', 'key', 'value');
+            expect(row).to.have.all.keys('index', 'id', 'key', 'value');
             expect(row.id).to.be.a('string');
@@ -46,7 +46,7 @@ expect(row.key).to.be.a('string');
 
-        return couchdbIterator(prepared.couchdbAddr, 'test/view-1', (row, index) => {
-            expect(index).to.equal(count);
+        return couchdbIterator(prepared.couchdbAddr, 'test/view-1', (row) => {
+            expect(row.index).to.equal(count);
             count += 1;
 
-            expect(row).to.have.all.keys('id', 'key', 'value');
+            expect(row).to.have.all.keys('index', 'id', 'key', 'value');
             expect(row.id).to.be.a('string');
@@ -159,1 +159,151 @@ expect(row.key).to.be.a('array');
 });
+
+describe('couchdbIterator.bulk()', () => {
+    let prepared;
+
+    before(() => {
+        return prepare()
+        .then((prepared_) => { prepared = prepared_; });
+    });
+
+    after(() => prepared.destroy());
+
+    it('should iterate all rows, respecting the options.bulkSize', () => {
+        let count = 0;
+
+        return couchdbIterator.bulk(prepared.couchdbAddr, (rows) => {
+            count < prepared.documents.length - 5 && expect(rows).to.have.length(5);
+            count += rows.length;
+
+            rows.forEach((row) => {
+                expect(row).to.have.all.keys('index', 'id', 'key', 'value');
+                expect(row.id).to.be.a('string');
+                expect(row.key).to.be.a('string');
+                expect(row.value).to.be.an('object');
+                expect(row.key).to.equal(row.id);
+            });
+        }, { bulkSize: 5 })
+        .then((rowsCount) => {
+            expect(count).to.equal(prepared.documents.length);
+            expect(rowsCount).to.equal(count);
+        });
+    });
+
+    it('should iterate all rows from a view, respecting the options.bulkSize', () => {
+        let count = 0;
+
+        return couchdbIterator.bulk(prepared.couchdbAddr, 'test/view-1', (rows) => {
+            count < prepared.documents.length - 5 && expect(rows).to.have.length(5);
+            count += rows.length;
+
+            rows.forEach((row) => {
+                expect(row).to.have.all.keys('index', 'id', 'key', 'value');
+                expect(row.id).to.be.a('string');
+                expect(row.key).to.be.a('array');
+                expect(row.value).to.be.an('object');
+                expect(row.key).to.eql([row.value.$type, row.id]);
+            });
+        }, { bulkSize: 5 })
+        .then((rowsCount) => {
+            expect(count).to.equal(prepared.documents.length - 1);  // Design doc doesn't count
+            expect(rowsCount).to.equal(count);
+        });
+    });
+
+    it('should wait for iterator to fulfill', () => {
+        let count = 0;
+        let waited = 0;
+        const userDocuments = prepared.documents.filter((doc) => doc.$type === 'user');
+
+        return couchdbIterator.bulk(prepared.couchdbAddr, 'test/view-1', () => {
+            count += 1;
+
+            return Promise.delay(50)
+            .then(() => { waited += 1; });
+        }, {
+            startkey: ['user', ''],
+            endkey: ['user', '\ufff0'],
+            bulkSize: 1,
+        })
+        .then((rowsCount) => {
+            expect(waited).to.equal(userDocuments.length);
+            expect(count).to.equal(userDocuments.length);
+            expect(rowsCount).to.equal(count);
+        });
+    });
+
+    it('should fail if the iterator fails', () => {
+        return couchdbIterator.bulk(prepared.couchdbAddr, () => {
+            throw new Error('foo');
+        })
+        .then(() => {
+            throw new Error('Expected to fail');
+        }, (err) => {
+            expect(err).to.be.an.instanceOf(Error);
+            expect(err.message).to.equal('foo');
+        });
+    });
+
+    describe('arguments', () => {
+        it('should fail if couchdb does not point to a db', () => {
+            return couchdbIterator.bulk('http://localhost:5984', () => {})
+            .then(() => {
+                throw new Error('Expected to fail');
+            }, (err) => {
+                expect(err).to.be.an.instanceOf(Error);
+                expect(err.message).to.match(/no database is selected/i);
+            });
+        });
+
+        it('should accept a nano instance', () => {
+            return couchdbIterator.bulk(prepared.couchdb, () => {})
+            .then((rowsCount) => {
+                expect(rowsCount).to.equal(prepared.documents.length);
+            });
+        });
+
+        it('should use options as view params', () => {
+            let count = 0;
+            const userDocuments = prepared.documents.filter((doc) => doc.$type === 'user');
+
+            return couchdbIterator.bulk(prepared.couchdbAddr, 'test/view-1', (rows) => {
+                count += rows.length;
+            }, {
+                startkey: ['user', ''],
+                endkey: ['user', '\ufff0'],
+            })
+            .then((rowsCount) => {
+                expect(count).to.equal(userDocuments.length);
+                expect(rowsCount).to.equal(count);
+            });
+        });
+
+        it('should snakeCase view params', () => {
+            let count = 0;
+            const userDocuments = prepared.documents.filter((doc) => doc.$type === 'user');
+
+            return couchdbIterator.bulk(prepared.couchdbAddr, 'test/view-1', (rows) => {
+                count += rows.length;
+                rows.forEach((row) => expect(row.doc).to.be.an('object'));
+            }, {
+                startkey: ['user', ''],
+                endkey: ['user', '\ufff0'],
+                includeDocs: true,
+            })
+            .then((rowsCount) => {
+                expect(count).to.equal(userDocuments.length);
+                expect(rowsCount).to.equal(count);
+            });
+        });
+
+        it('should fail on invalid options', () => {
+            return couchdbIterator.bulk(prepared.couchdbAddr, () => {}, { some: 'option' })
+            .then(() => {
+                throw new Error('Expected to fail');
+            }, (err) => {
+                expect(err).to.be.an.instanceOf(Error);
+                expect(err.message).to.match(/option `some` is not allowed/i);
+            });
+        });
+    });
+});

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

30944

increased by54.1%

Number of package files

14

increased by7.69%

Lines of code

583

increased by55.88%

Number of lines in readme file

139

increased by67.47%

No dependency changes