db-streamer - npm Package Compare versions

Comparing version 0.2.0 to 0.3.0

lib/inserters/pgInserter.js

@@ -45,3 +45,8 @@ var fs = require('fs');
   this.defer = config.deferUntilEnd;
+  this.client = config.client;
 
+  if(this.client) {
+    this.setModel();
+  }
+
 };
@@ -53,2 +58,8 @@
   
+  if(this.client) {
+    this.setModel();
+    callback();
+    return;
+  }
+
   this.client = new pg.Client(this.dbConnString);
@@ -61,3 +72,3 @@ var self = this;
     if(callback) {
-      callback(err);
+      callback(err, self.client);
     }
@@ -64,0 +75,0 @@ });

package.json

 {
   "name": "db-streamer",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "A library to stream data into a SQL database.",
@@ -5,0 +5,0 @@ "main": "index.js",

README.md

@@ -11,3 +11,3 @@ # db-streamer
 
-## Example Usage
+## Usage
 
@@ -24,3 +24,3 @@ var dbStreamer = require('db-streamer');
     // establish connection
-    inserter.connect(function() {
+    inserter.connect(function(err, client) {
 
@@ -32,4 +32,20 @@ // push some rows
 
+      // create child table inserter using deferring strategy
+      // this is useful to avoid missing foreign key conflicts as a result of race conditions
+      var childInserter = dbStreamer.getInserter({
+        tableName: 'child_table',
+        columns: ['a', 'd', 'e'],
+        client: client,
+        deferUntilEnd: true
+      });
+
+      childInserter.push({a: 2, d: 'asdf', e: new Date() });
+      childInserter.push({a: 3, d: 'ghjk', e: new Date() });
+
+      childInserter.setEndHandler(callback);
+
       // set end callback
-      inserter.setEndHandler(callback);
+      inserter.setEndHandler(function() {
+        childInserter.end();
+      });
 
@@ -40,3 +56,21 @@ // announce end
     });
+    
+### Inserter Config
 
+| Key | Description |
+| --- | --- |
+| dbConnString | A database connection string. |
+| tableName | The tablename to insert into. |
+| columns | Array of column names. |
+| client | Optional.  A database client.  Provide this in place of dbConnString. |
+| deferUntilEnd | Boolean (default=false).  Stream output to temporary file which is then streamed in all at once into table upon calling `end`. |
+
+### Inserter Config (Sequelize Bulk Insert alternative)
+
+| Key | Description |
+| --- | --- |
+| useSequelizeBulkInsert | Boolean.  Perform the insert using a combination of [async.cargo](https://github.com/caolan/async#cargo) and [sequelize bulkInsert](http://docs.sequelizejs.com/en/latest/api/model/#bulkcreaterecords-options-promisearrayinstance).  Must provide `sequelizeModel` parameter too. |
+| sequelizeModel | The sequelize model to perform a bulk insert with. |
+| deferUntilEnd | Boolean (default=false).  Pause all cargo iterations until calling `end`. |
+
 ## PostgreSQL
@@ -43,0 +77,0 @@

tests/testMain.js

@@ -12,3 +12,3 @@ var dbStreamer = require('../index.js');
     // establish connection
-    inserter.connect(function() {
+    inserter.connect(function(err, client) {
 
@@ -20,4 +20,14 @@ // push some rows
 
+      // create defered inserter
+      streamerConfig.client = client;
+      streamerConfig.deferUntilEnd = true;
+      var deferedInserter = dbStreamer.getInserter(streamerConfig);
+
+      deferedInserter.push({a: 4, b: 'four', c: new Date() });
+      deferedInserter.setEndHandler(callback);
+
       // set end callback
-      inserter.setEndHandler(callback);
+      inserter.setEndHandler(function() {
+        deferedInserter.end();
+      });
 
@@ -24,0 +34,0 @@ // announce end

tests/testPg.js

@@ -18,8 +18,2 @@ var Promise = require('bluebird');
   });
-
-  it('deferred copy stream should load', function() {
-    var testMainPromise = Promise.promisify(testMain);
-    streamerConfig.deferUntilEnd = true;
-    return testMainPromise(testModel(sequelizeConfig), streamerConfig);
-  });
 });

tests/testSequelizeBulkInsert.js

@@ -9,5 +9,2 @@ var Promise = require('bluebird');
   streamerConfig = {
-    dbConnString: sequelizeConfig,
-    tableName: 'test_table',
-    columns: ['a', 'b', 'c'],
     useSequelizeBulkInsert: true,
@@ -18,3 +15,2 @@ sequelizeModel: testModel
 describe('sequelize-bulk-insert-load', function() {
-  
   it('bulkInsert should load', function() {
@@ -24,9 +20,2 @@ var testMainPromise = Promise.promisify(testMain);
   });
-
-  it('deferred bulkInsert should load', function() {
-    var testMainPromise = Promise.promisify(testMain);
-    streamerConfig.deferUntilEnd = true;
-    return testMainPromise(testModel, streamerConfig);
-  });
-
 });

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

13851

increased by12.62%

Lines of code

264

increased by1.54%

Number of lines in readme file

81

increased by72.34%

No dependency changes