mysql-import - npm Package Compare versions

Comparing version 4.0.24 to 5.0.19

Gruntfile.js

@@ -55,3 +55,3 @@
 					replacements: [{
-						pattern: /\d+.\d+.\d+/g,
+						pattern: /\d+\.\d+\.\d+/g,
 						replacement: '<%= pkg.version %>'
@@ -58,0 +58,0 @@ }]

mysql-import.js

 /**
- * mysql-import - v4.0.24
+ * mysql-import - v5.0.19
  * Import .sql into a MySQL database with Node.
@@ -14,2 +14,3 @@ * @author Rob Parham
 const path = require("path");
+const stream = require('stream');
 
@@ -19,3 +20,3 @@
  * mysql-import - Importer class
- * @version 4.0.24
+ * @version 5.0.19
  * https://github.com/Pamblam/mysql-import
@@ -35,2 +36,6 @@ */
 		this._imported = [];
+		this._progressCB = ()=>{};
+		this._dumpCompletedCB = ()=>{};
+		this._total_files = 0;
+		this._current_file_no = 0;
 	}
@@ -77,2 +82,3 @@
 				this._connection_settings.database = database;
+				resolve();
 				return;
@@ -91,2 +97,32 @@ }
 	/**
+	 * Set a progress callback
+	 * @param {Function} cb - Callback function is called whenever a chunk of
+	 *		the stream is read. It is provided an object with the folling properties:
+	 *			- total_files: The total files in the queue. 
+	 *			- file_no: The number of the current dump file in the queue. 
+	 *			- bytes_processed: The number of bytes of the file processed.
+	 *			- total_bytes: The size of the dump file.
+	 *			- file_path: The full path to the dump file.
+	 * @returns {undefined}
+	 */
+	onProgress(cb){
+		if(typeof cb !== 'function') return;
+		this._progressCB = cb;
+	}
+	
+	/**
+	 * Set a progress callback
+	 * @param {Function} cb - Callback function is called whenever a dump
+	 *		file has finished processing.
+	 *			- total_files: The total files in the queue. 
+	 *			- file_no: The number of the current dump file in the queue. 
+	 *			- file_path: The full path to the dump file.
+	 * @returns {undefined}
+	 */
+	onDumpCompleted(cb){
+		if(typeof cb !== 'function') return;
+		this._dumpCompletedCB = cb;
+	}
+	
+	/**
 	 * Import (an) .sql file(s).
@@ -101,4 +137,8 @@ * @param string|array input - files or paths to scan for .sql files
 				var files = await this._getSQLFilePaths(...input);
+				this._total_files = files.length;
+				this._current_file_no = 0;
+				
 				var error = null;
 				await slowLoop(files, (file, index, next)=>{
+					this._current_file_no++;
 					if(error){
@@ -157,33 +197,53 @@ next();
 	 * Import a single .sql file into the database
-	 * @param {type} filepath
+	 * @param {object} fileObj - Object containing the following properties:
+	 *		- file: The full path to the file
+	 *		- size: The size of the file in bytes
 	 * @returns {Promise}
 	 */
-	_importSingleFile(filepath){
+	_importSingleFile(fileObj){
 		return new Promise((resolve, reject)=>{
-			fs.readFile(filepath, this._encoding, (err, queriesString) => {
-				if(err){
-					reject(err);
-					return;
+			
+			var parser = new queryParser({
+				db_connection: this._conn,
+				encoding: this._encoding,
+				onProgress: (progress) => {
+					this._progressCB({
+						total_files: this._total_files, 
+						file_no: this._current_file_no, 
+						bytes_processed: progress, 
+						total_bytes: fileObj.size,
+						file_path: fileObj.file
+					});
 				}
-				var queries = new queryParser(queriesString).queries;
-				var error = null;
-				slowLoop(queries, (query, index, next)=>{
-					if(error){
-						next();
-						return;
-					}
-					this._conn.query(query, err=>{
-						if (err) error = err;
-						next();
-					});
-				}).then(()=>{
-					if(error){
-						reject(error);
-					}else{
-						this._imported.push(filepath);
-						resolve();
-					}
-				});
-				
 			});
+			
+			const dumpCompletedCB = (err) => this._dumpCompletedCB({
+				total_files: this._total_files, 
+				file_no: this._current_file_no, 
+				file_path: fileObj.file,
+				error: err
+			});
+			
+			parser.on('finish', ()=>{
+				this._imported.push(fileObj.file);
+				dumpCompletedCB(null);
+				resolve();
+			});
+			
+			
+			parser.on('error', (err)=>{
+				dumpCompletedCB(err);
+				reject(err);
+			});
+			
+			var readerStream = fs.createReadStream(fileObj.file);
+			readerStream.setEncoding(this._encoding);
+			
+			/* istanbul ignore next */
+			readerStream.on('error', (err)=>{
+				dumpCompletedCB(err);
+				reject(err);
+			});
+			
+			readerStream.pipe(parser);
 		});
@@ -285,3 +345,6 @@ }
 						if(filepath.toLowerCase().substring(filepath.length-4) === '.sql'){
-							full_paths.push(path.resolve(filepath));
+							full_paths.push({
+								file: path.resolve(filepath),
+								size: stat.size
+							});
 						}
@@ -296,2 +359,3 @@ next();
 					}else{
+						/* istanbul ignore next */
 						next();
@@ -317,3 +381,3 @@ }
  */
-Importer.version = '4.0.24';
+Importer.version = '5.0.19';
 
@@ -337,2 +401,3 @@ module.exports = Importer;
 	return new Promise(f => {
+		/* istanbul ignore next */
 		if(!items.length) return f();
@@ -347,9 +412,21 @@ let done = arguments[2] || f;
 
-class queryParser{
+class queryParser extends stream.Writable{
 	
-	constructor(queriesString){
+	constructor(options){
+		/* istanbul ignore next */
+		options = options || {};
+		super(options);
 		
-		// Input string containing SQL queries
-		this.queriesString = queriesString.trim();
+		// The number of bytes processed so far
+		this.processed_size = 0;
 		
+		// The progress callback
+		this.onProgress = options.onProgress || (() => {});
+		
+		// the encoding of the file being read
+		this.encoding = options.encoding || 'utf8';
+		
+		// the encoding of the database connection
+		this.db_connection = options.db_connection;
+		
 		// The quote type (' or ") if the parser 
@@ -359,5 +436,2 @@ // is currently inside of a quote, else false
 		
-		// An array of complete queries
-		this.queries = [];
-		
 		// An array of chars representing the substring
@@ -375,24 +449,51 @@ // the is currently being parsed
 		this.seekingDelimiter = false;
-
-		// Does the sql set change delimiter?
-		this.hasDelimiter = queriesString.toLowerCase().includes('delimiter ');
-
-		// Iterate over each char in the string
-		for (let i = 0; i < this.queriesString.length; i++) {
-			let char = this.queriesString[i];
-			this.parseChar(char);
+		
+	}
+	
+	////////////////////////////////////////////////////////////////////////////
+	// "Private" methods" //////////////////////////////////////////////////////
+	////////////////////////////////////////////////////////////////////////////
+	
+	// handle piped data
+	async _write(chunk, enc, next) {
+		var query;
+		chunk = chunk.toString(this.encoding);
+		var error = null;
+		for (let i = 0; i < chunk.length; i++) {
+			let char = chunk[i];
+			query = this.parseChar(char);
+			try{
+				if(query) await this.executeQuery(query);
+			}catch(e){
+				error = e;
+				break;
+			}
 		}
+		this.processed_size += chunk.length;
+		this.onProgress(this.processed_size);
+		next(error);
 	}
 	
+	// Execute a query, return a Promise
+	executeQuery(query){
+		return new Promise((resolve, reject)=>{
+			this.db_connection.query(query, err=>{
+				if (err){
+					reject(err);
+				}else{
+					resolve();
+				}
+			});
+		});
+	}
+	
 	// Parse the next char in the string
+	// return a full query if one is detected after parsing this char
+	// else return false.
 	parseChar(char){
 		this.checkEscapeChar();
 		this.buffer.push(char);
-
-		if (this.hasDelimiter) {
-			this.checkNewDelimiter(char);
-		}
-
+		this.checkNewDelimiter(char);
 		this.checkQuote(char);
-		this.checkEndOfQuery();
+		return this.checkEndOfQuery();
 	}
@@ -438,3 +539,5 @@
 	// Check if we're at the end of the query
+	// return the query if so, else return false;
 	checkEndOfQuery(){
+		var query = false;
 		var demiliterFound = false;
@@ -448,6 +551,8 @@ if(!this.quoteType && this.buffer.length >= this.delimiter.length){
 			this.buffer.splice(-this.delimiter.length, this.delimiter.length);
-			this.queries.push(this.buffer.join('').trim());
+			query = this.buffer.join('').trim();
 			this.buffer = [];
 		}
+		
+		return query;
 	}
 }

package.json

@@ -41,6 +41,7 @@ {
     "build": "grunt",
-    "test": "node_modules/.bin/nyc --reporter=html --reporter=text node_modules/.bin/mocha --timeout 15000",
+    "test": "node_modules/.bin/nyc --reporter=html --reporter=text node_modules/.bin/mocha ./test/test.js --timeout 15000",
+    "memory-test": "node_modules/.bin/mocha ./test/memory-stress-test.js --timeout 0",
     "coverage": "nyc report --reporter=text-lcov | coveralls"
   },
-  "version": "4.0.24"
+  "version": "5.0.19"
 }

README.md

 
+
 <p align="center">
@@ -6,3 +7,3 @@ <img src='https://i.imgur.com/AOfuTLA.png'>
 
-*Version 4.0.24* ([NPM](https://www.npmjs.com/package/mysql-import)) ([Github](https://github.com/Pamblam/mysql-import/))
+*Version 5.0.19* ([NPM](https://www.npmjs.com/package/mysql-import)) ([Github](https://github.com/Pamblam/mysql-import/))
 
@@ -24,2 +25,4 @@ [![Build Status](https://api.travis-ci.org/Pamblam/mysql-import.svg?branch=master)](https://travis-ci.org/Pamblam/mysql-import/) [![Coverage Status](https://coveralls.io/repos/github/Pamblam/mysql-import/badge.svg?branch=master)](https://coveralls.io/github/Pamblam/mysql-import?branch=master)
    - [`importer.disconnect(graceful=true)`](#importerprototypedisconnectgracefultrue)
+   - [`importer.onProgress(callback)`](#importerprototypeonprogresscallback)
+   - [`importer.onDumpCompleted(callback)`](#importerprototypeondumpcompletedcallback)
  - [Contributing](#contributing)
@@ -39,3 +42,3 @@
 
-```
+```js
 const host = 'localhost';
@@ -46,9 +49,14 @@ const user = 'root';
 
-const mysql_import = require('mysql-import');
-const Importer = require('../mysql-import.js');
+const Importer = require('mysql-import');
 const importer = new Importer({host, user, password, database});
 
+// New onProgress method, added in version 5.0!
+importer.onProgress(progress=>{
+  var percent = Math.floor(progress.bytes_processed / progress.total_bytes * 10000) / 100;
+  console.log(`${percent}% Completed`);
+});
+
 importer.import('path/to/dump.sql').then(()=>{
   var files_imported = importer.getImported();
-  console.log('${files_imported.length} SQL file(s) imported.');
+  console.log(`${files_imported.length} SQL file(s) imported.`);
 }).catch(err=>{
@@ -60,20 +68,39 @@ console.error(err);
 
-#### new Importer({host, user, password[, database]})
+### `new Importer({host, user, password[, database]})`
 
 The constructor requires an object with a `host`, `user`, and `password` parameter. Passing in a database parameter is optional.
 
-#### Importer.prototype.getImported()
+### `Importer.prototype.getImported()`
 
 Get an array of files imported.
 
-#### Importer.prototype.setEncoding(encoding)
+### `Importer.prototype.setEncoding(encoding)`
 
 Set the encoding to use when reading import files. Supported arguments are: `utf8`, `ucs2`, `utf16le`, `latin1`, `ascii`, `base64`, or `hex`.
 
-#### Importer.prototype.use(database)
+### `Importer.prototype.use(database)`
 
 Set or change the database to import to.
 
-#### Importer.prototype.import(...input)
+### `Importer.prototype.onProgress(callback)`
 
+*(New in v. 5.0!) -* Set a callback to be called as the importer processes chunks of the dump file. Callback is provided an object with the following properties:
+
+ - `total_files`: The total files in the queue. 
+ - `file_no`: The number of the current dump file in the queue. 
+ - `bytes_processed`: The number of bytes of the file processed.
+ - `total_bytes`: The size of the dump file.
+ - `file_path`: The full path to the dump file.
+
+### `Importer.prototype.onDumpCompleted(callback)`
+
+*(New in v. 5.0!) -* Set a callback to be called after each dump file has completed processing. Callback is provided an object with the following properties:
+
+ - `total_files`: The total files in the queue. 
+ - `file_no`: The number of the current dump file in the queue. 
+ - `file_path`: The full path to the dump file.
+ - `error`: If there was an error, the error object; if no errors, this will be `null`.
+
+### `Importer.prototype.import(...input)`
+
 Import an `.sql` file or files into the database. This method will take...
@@ -95,3 +122,3 @@
 
-#### Importer.prototype.disconnect(graceful=true)
+### `Importer.prototype.disconnect(graceful=true)`
 
@@ -98,0 +125,0 @@ Disconnects the connection. If `graceful` is switched to false it will force close any connections. This is called automatically after files are imported so typically *this method should never be required*.

src/header.js

@@ -6,1 +6,2 @@ 'use strict';
 const path = require("path");
+const stream = require('stream');

src/importer.js

@@ -19,2 +19,6 @@
 		this._imported = [];
+		this._progressCB = ()=>{};
+		this._dumpCompletedCB = ()=>{};
+		this._total_files = 0;
+		this._current_file_no = 0;
 	}
@@ -61,2 +65,3 @@
 				this._connection_settings.database = database;
+				resolve();
 				return;
@@ -75,2 +80,32 @@ }
 	/**
+	 * Set a progress callback
+	 * @param {Function} cb - Callback function is called whenever a chunk of
+	 *		the stream is read. It is provided an object with the folling properties:
+	 *			- total_files: The total files in the queue. 
+	 *			- file_no: The number of the current dump file in the queue. 
+	 *			- bytes_processed: The number of bytes of the file processed.
+	 *			- total_bytes: The size of the dump file.
+	 *			- file_path: The full path to the dump file.
+	 * @returns {undefined}
+	 */
+	onProgress(cb){
+		if(typeof cb !== 'function') return;
+		this._progressCB = cb;
+	}
+	
+	/**
+	 * Set a progress callback
+	 * @param {Function} cb - Callback function is called whenever a dump
+	 *		file has finished processing.
+	 *			- total_files: The total files in the queue. 
+	 *			- file_no: The number of the current dump file in the queue. 
+	 *			- file_path: The full path to the dump file.
+	 * @returns {undefined}
+	 */
+	onDumpCompleted(cb){
+		if(typeof cb !== 'function') return;
+		this._dumpCompletedCB = cb;
+	}
+	
+	/**
 	 * Import (an) .sql file(s).
@@ -85,4 +120,8 @@ * @param string|array input - files or paths to scan for .sql files
 				var files = await this._getSQLFilePaths(...input);
+				this._total_files = files.length;
+				this._current_file_no = 0;
+				
 				var error = null;
 				await slowLoop(files, (file, index, next)=>{
+					this._current_file_no++;
 					if(error){
@@ -141,33 +180,53 @@ next();
 	 * Import a single .sql file into the database
-	 * @param {type} filepath
+	 * @param {object} fileObj - Object containing the following properties:
+	 *		- file: The full path to the file
+	 *		- size: The size of the file in bytes
 	 * @returns {Promise}
 	 */
-	_importSingleFile(filepath){
+	_importSingleFile(fileObj){
 		return new Promise((resolve, reject)=>{
-			fs.readFile(filepath, this._encoding, (err, queriesString) => {
-				if(err){
-					reject(err);
-					return;
+			
+			var parser = new queryParser({
+				db_connection: this._conn,
+				encoding: this._encoding,
+				onProgress: (progress) => {
+					this._progressCB({
+						total_files: this._total_files, 
+						file_no: this._current_file_no, 
+						bytes_processed: progress, 
+						total_bytes: fileObj.size,
+						file_path: fileObj.file
+					});
 				}
-				var queries = new queryParser(queriesString).queries;
-				var error = null;
-				slowLoop(queries, (query, index, next)=>{
-					if(error){
-						next();
-						return;
-					}
-					this._conn.query(query, err=>{
-						if (err) error = err;
-						next();
-					});
-				}).then(()=>{
-					if(error){
-						reject(error);
-					}else{
-						this._imported.push(filepath);
-						resolve();
-					}
-				});
-				
 			});
+			
+			const dumpCompletedCB = (err) => this._dumpCompletedCB({
+				total_files: this._total_files, 
+				file_no: this._current_file_no, 
+				file_path: fileObj.file,
+				error: err
+			});
+			
+			parser.on('finish', ()=>{
+				this._imported.push(fileObj.file);
+				dumpCompletedCB(null);
+				resolve();
+			});
+			
+			
+			parser.on('error', (err)=>{
+				dumpCompletedCB(err);
+				reject(err);
+			});
+			
+			var readerStream = fs.createReadStream(fileObj.file);
+			readerStream.setEncoding(this._encoding);
+			
+			/* istanbul ignore next */
+			readerStream.on('error', (err)=>{
+				dumpCompletedCB(err);
+				reject(err);
+			});
+			
+			readerStream.pipe(parser);
 		});
@@ -269,3 +328,6 @@ }
 						if(filepath.toLowerCase().substring(filepath.length-4) === '.sql'){
-							full_paths.push(path.resolve(filepath));
+							full_paths.push({
+								file: path.resolve(filepath),
+								size: stat.size
+							});
 						}
@@ -280,2 +342,3 @@ next();
 					}else{
+						/* istanbul ignore next */
 						next();
@@ -282,0 +345,0 @@ }

src/parser.js

 
-class queryParser{
+class queryParser extends stream.Writable{
 	
-	constructor(queriesString){
+	constructor(options){
+		/* istanbul ignore next */
+		options = options || {};
+		super(options);
 		
-		// Input string containing SQL queries
-		this.queriesString = queriesString.trim();
+		// The number of bytes processed so far
+		this.processed_size = 0;
 		
+		// The progress callback
+		this.onProgress = options.onProgress || (() => {});
+		
+		// the encoding of the file being read
+		this.encoding = options.encoding || 'utf8';
+		
+		// the encoding of the database connection
+		this.db_connection = options.db_connection;
+		
 		// The quote type (' or ") if the parser 
@@ -13,5 +25,2 @@ // is currently inside of a quote, else false
 		
-		// An array of complete queries
-		this.queries = [];
-		
 		// An array of chars representing the substring
@@ -29,24 +38,51 @@ // the is currently being parsed
 		this.seekingDelimiter = false;
-
-		// Does the sql set change delimiter?
-		this.hasDelimiter = queriesString.toLowerCase().includes('delimiter ');
-
-		// Iterate over each char in the string
-		for (let i = 0; i < this.queriesString.length; i++) {
-			let char = this.queriesString[i];
-			this.parseChar(char);
+		
+	}
+	
+	////////////////////////////////////////////////////////////////////////////
+	// "Private" methods" //////////////////////////////////////////////////////
+	////////////////////////////////////////////////////////////////////////////
+	
+	// handle piped data
+	async _write(chunk, enc, next) {
+		var query;
+		chunk = chunk.toString(this.encoding);
+		var error = null;
+		for (let i = 0; i < chunk.length; i++) {
+			let char = chunk[i];
+			query = this.parseChar(char);
+			try{
+				if(query) await this.executeQuery(query);
+			}catch(e){
+				error = e;
+				break;
+			}
 		}
+		this.processed_size += chunk.length;
+		this.onProgress(this.processed_size);
+		next(error);
 	}
 	
+	// Execute a query, return a Promise
+	executeQuery(query){
+		return new Promise((resolve, reject)=>{
+			this.db_connection.query(query, err=>{
+				if (err){
+					reject(err);
+				}else{
+					resolve();
+				}
+			});
+		});
+	}
+	
 	// Parse the next char in the string
+	// return a full query if one is detected after parsing this char
+	// else return false.
 	parseChar(char){
 		this.checkEscapeChar();
 		this.buffer.push(char);
-
-		if (this.hasDelimiter) {
-			this.checkNewDelimiter(char);
-		}
-
+		this.checkNewDelimiter(char);
 		this.checkQuote(char);
-		this.checkEndOfQuery();
+		return this.checkEndOfQuery();
 	}
@@ -92,3 +128,5 @@
 	// Check if we're at the end of the query
+	// return the query if so, else return false;
 	checkEndOfQuery(){
+		var query = false;
 		var demiliterFound = false;
@@ -102,6 +140,8 @@ if(!this.quoteType && this.buffer.length >= this.delimiter.length){
 			this.buffer.splice(-this.delimiter.length, this.delimiter.length);
-			this.queries.push(this.buffer.join('').trim());
+			query = this.buffer.join('').trim();
 			this.buffer = [];
 		}
+		
+		return query;
 	}
 }

src/slowloop.js

@@ -16,2 +16,3 @@ /**
 	return new Promise(f => {
+		/* istanbul ignore next */
 		if(!items.length) return f();
@@ -18,0 +19,0 @@ let done = arguments[2] || f;

test/test.js

@@ -24,2 +24,33 @@
 
+// For coverage
+importer.onProgress('Not a function');
+importer.onDumpCompleted('Not a function');
+
+importer.onProgress(progress=>{
+	var percent = Math.floor(progress.bytes_processed / progress.total_bytes * 10000) / 100;
+	var filename = progress.file_path.split("/").pop();
+	var message = `\tFile ${progress.file_no} of ${progress.total_files}: `+
+			`processing ${filename} - ${percent}% Complete`;
+	process.stdout.clearLine();
+	process.stdout.cursorTo(0);
+	process.stdout.write(message);
+});
+
+importer.onDumpCompleted(status=>{
+	var filename = status.file_path.split("/").pop();
+	var message;
+	if(status.error){
+		message = `\tFile ${status.file_no} of ${status.total_files}: `+
+			`Was not processed.\n`;
+	}else{
+		message = `\tFile ${status.file_no} of ${status.total_files}: `+
+			`Completed processing ${filename}\n`;
+	}
+	process.stdout.clearLine();
+	process.stdout.cursorTo(0);
+	process.stdout.write(message);
+});
+
+importer.setEncoding('utf8');
+
 const start_time = new Date();
@@ -29,8 +60,7 @@
 	
-	before(async ()=>{
+	before(async function(){	
 		await createTestDB('mysql-import-test-db-1');
-		await createTestDB('mysql-import-test-db-2');
+		await createTestDB('mysql-import-test-db-2');		
 		query("USE `mysql-import-test-db-1`");
 		await importer.import(__dirname+'/sample_dump_files/test.sql');
-		importer.setEncoding('utf8');
 	});
@@ -88,2 +118,7 @@
 	
+	it('Test imported', async ()=>{
+		var files = importer.getImported();
+		expect(files.length).to.equal(10);
+	});
+	
 	it('Test unsupported encoding', ()=>{
@@ -90,0 +125,0 @@ var error;

No alert changes

Improved metrics

Total package byte prevSize

171765

increased by8.2%

Number of package files

23

increased by9.52%

Lines of code

1415

increased by30.78%

Number of lines in readme file

124

increased by27.84%

Worsened metrics

Number of low supply chain risk alerts

5

increased by66.67%

No dependency changes