ftp-folder-sync - npm Package Compare versions

Comparing version 1.0.6 to 1.0.7

index.js

@@ -11,3 +11,3 @@ "use strict";
 
-var FtpSync = function({ username, password, host }) {
+var FtpSync = function ({ username, password, host }) {
     this.options = {
@@ -44,8 +44,13 @@ dir: null,
     var this_ = this;
-    this.fileIndexer.onAccessError = function(file) {
+    this.fileIndexer.onAccessError = function (file) {
         this_.onFileError('LOCKED', file);
     };
+    this.ftpPurgeList = [];
+    this.removeCounter = 0;
+    this.ftpScanCounter = 0;
+    this.removeDirList = [];
+    this.dirRemovedErrorInfo = {};
 };
 
-FtpSync.prototype.log = function(text) {
+FtpSync.prototype.log = function (text) {
     if (this.enableLog) {
@@ -56,15 +61,15 @@ console.log(text);
 
-FtpSync.prototype.setExcludedDirs = function(dirs) {
+FtpSync.prototype.setExcludedDirs = function (dirs) {
     this.excludeDirs = dirs;
 };
 
-FtpSync.prototype.setEndFileList = function(files) {
+FtpSync.prototype.setEndFileList = function (files) {
     this.endFileList = files;
 };
 
-FtpSync.prototype.setBlockedFileList = function(files) {
+FtpSync.prototype.setBlockedFileList = function (files) {
     this.blockedList = files;
 };
 
-FtpSync.prototype.reset = async function() {
+FtpSync.prototype.reset = async function () {
     await this.fileIndexer.reset();
@@ -88,5 +93,7 @@ await this.closeDb();
     this.scanning = false;
+    this.ftpPurgeList = [];
+    this.dirRemovedErrorInfo = {};
 };
 
-FtpSync.prototype.check = function() {
+FtpSync.prototype.check = function () {
     return this.client.access({
@@ -100,3 +107,3 @@ host: this.options.host,
 
-FtpSync.prototype.connect = async function() {
+FtpSync.prototype.connect = async function () {
     try {
@@ -118,20 +125,20 @@ await this.client.access({
 
-FtpSync.prototype.onFileError = function(type, file) {
+FtpSync.prototype.onFileError = function (type, file) {
 
 }
 
-FtpSync.prototype.disconnect = function() {
+FtpSync.prototype.disconnect = function () {
     this.client.close();
 };
 
-FtpSync.prototype.reconnect = async function() {
-    await this.client.close();
+FtpSync.prototype.reconnect = async function () {
+    this.client.close();
     await this.connect();
 };
 
-FtpSync.prototype.onConnectError = function(err) {
+FtpSync.prototype.onConnectError = function (err) {
     throw err;
 };
 
-FtpSync.prototype.onUploadError = function(err, i) {
+FtpSync.prototype.onUploadError = function (err, i, purgeError) {
     //reset of connection of firewall
@@ -147,3 +154,3 @@ if (err.code === "ECONNRESET") {
     //file removed
-    if (err.code === "ENOENT") {
+    if (err.code === "ENOENT" && !purgeError) {
         this.fileIndexer.fileErrors.push(this.fileIndexer.changeList[i].fullpath);
@@ -155,3 +162,3 @@ this.onFileError('REMOVED', this.fileIndexer.changeList[i].fullpath);
     //file locked
-    if (err.code === "EBUSY") {
+    if (err.code === "EBUSY" && !purgeError) {
         this.fileIndexer.fileErrors.push(this.fileIndexer.changeList[i].fullpath);
@@ -163,3 +170,3 @@ this.onFileError('LOCKED', this.fileIndexer.changeList[i].fullpath);
     //permission issue
-    if (err.code === "EPERM") {
+    if (err.code === "EPERM" && !purgeError) {
         this.fileIndexer.fileErrors.push(this.fileIndexer.changeList[i].fullpath);
@@ -182,3 +189,5 @@ this.onFileError('PERMISSION', this.fileIndexer.changeList[i].fullpath);
     if (!this.isFtpError(err)) {
-        this.onNoneFtpError(this.fileIndexer.changeList[i], err);
+        if (!purgeError) {
+            this.onNoneFtpError(this.fileIndexer.changeList[i], err);
+        }
         return i + 1;
@@ -191,11 +200,11 @@ }
 
-FtpSync.prototype.onNoneFtpError = function(file, err) {
+FtpSync.prototype.onNoneFtpError = function (file, err) {
 
 }
 
-FtpSync.prototype.isFtpError = function(err) {
+FtpSync.prototype.isFtpError = function (err) {
     return this.isConnectionError(err) || this.isUserNamePasswordError(err) || this.isStorageFull(err);
 }
 
-FtpSync.prototype.isConnectionError = function(error) {
+FtpSync.prototype.isConnectionError = function (error) {
     if (typeof error.code !== 'undefined') {
@@ -215,3 +224,3 @@ return (error.code == 421 ||
 
-FtpSync.prototype.isUserNamePasswordError = function(error) {
+FtpSync.prototype.isUserNamePasswordError = function (error) {
     if (typeof error.code !== 'undefined') {
@@ -228,3 +237,3 @@ return (error.code == 331 ||
 
-FtpSync.prototype.isStorageFull = function(error) {
+FtpSync.prototype.isStorageFull = function (error) {
     if (typeof error.code !== 'undefined') {
@@ -235,6 +244,7 @@ return (error.code == 452);
 
-FtpSync.prototype.setDir = function(dir) {
+FtpSync.prototype.setDir = function (dir) {
     for (let i = 0; i < dir.length; i++) {
         dir[i] = pathLib.normalize(dir[i]);
         if (!fs.existsSync(dir[i])) {
+            this.dirRemovedErrorInfo = { dir: dir[i] };
             throw "Path does not exist";
@@ -248,11 +258,11 @@ }
 
-FtpSync.prototype.setRemotePath = function(dir) {
+FtpSync.prototype.setRemotePath = function (dir) {
     this.ftpBaseDir = dir;
 };
 
-FtpSync.prototype.setDatabase = function(databaseName) {
+FtpSync.prototype.setDatabase = function (databaseName) {
     this.options.databaseName = databaseName;
 };
 
-FtpSync.prototype.initDb = async function(name) {
+FtpSync.prototype.initDb = async function (name) {
     return new Promise((resolve, reject) => {
@@ -270,3 +280,3 @@ if (this.db == null) {
 
-FtpSync.prototype.closeDb = async function(name) {
+FtpSync.prototype.closeDb = async function (name) {
     return new Promise((resolve, reject) => {
@@ -284,7 +294,7 @@ if (this.db !== null) {
 
-FtpSync.prototype.statusUpdate = function(info) {};
-FtpSync.prototype.statusStart = function(info) {};
-FtpSync.prototype.statusFileDone = function(info) {};
+FtpSync.prototype.statusUpdate = function (info) { };
+FtpSync.prototype.statusStart = function (info) { };
+FtpSync.prototype.statusFileDone = function (info) { };
 
-FtpSync.prototype.updateStatus = function() {
+FtpSync.prototype.updateStatus = function () {
     this.statusUpdate({
@@ -304,3 +314,3 @@ current: this.status,
 
-FtpSync.prototype.sync = async function() {
+FtpSync.prototype.sync = async function () {
     if (this.options.databaseName === null || this.batches.length === 0) {
@@ -321,3 +331,2 @@ throw "Options not set";
     for (let i = 0; i < this.batches.length; i++) {
-        this.log("Scanning " + this.batches[i].dir);
         this.fileIndexer.setDir(this.batches[i].dir);
@@ -421,3 +430,3 @@ await this.fileIndexer.scan();
 
-FtpSync.prototype.halt = function() {
+FtpSync.prototype.halt = function () {
     if (this.stop) {
@@ -431,3 +440,3 @@ this.disconnect();
 
-FtpSync.prototype.cacheCheck = function(dir) {
+FtpSync.prototype.cacheCheck = function (dir) {
     for (let i = 0; i < this.dirCreationCache.length; i++) {
@@ -442,3 +451,3 @@ if (this.dirCreationCache[i] === dir) {
 
-FtpSync.prototype.createDir = async function(remoteDirPath) {
+FtpSync.prototype.createDir = async function (remoteDirPath) {
     const names = remoteDirPath.split("/").filter((name) => name !== "");
@@ -458,7 +467,68 @@ let dircache = '/';
 
-FtpSync.prototype._openDir = async function(dir) {
+FtpSync.prototype.getPurgeList = async function (localPath, remoteDirPath, level) {
+    if (this.halt()) {
+        return null;
+    }
+
+    if (level == undefined) { level = 0; }
+    if (level == 0) {
+        this.ftpPurgeList = [];
+        if (!this.fileExists(localPath)) {
+            console.warn("The directory " + localPath + " not avaliable");
+            return;
+        }
+    }
+    let list = [];
+
+    try {
+        await this.client.cd(remoteDirPath);
+        list = await this.client.list();
+    } catch (err) {
+        let error = this.onUploadError(err, 1, true);
+        return error > -1 ? error : null;
+    }
+
+    if (Array.isArray(list)) {
+        for (let i = 0; i < list.length; i++) {
+            let ftpItem = list[i];
+            if (ftpItem.type == 2) {
+                let next = level + 1;
+                let returnCode = 0;
+                while (null !== returnCode) {
+                    returnCode = await this.getPurgeList(localPath + "/" + ftpItem.name, remoteDirPath + "/" + ftpItem.name, next);
+                    if (returnCode !== null) {
+                        await this.reconnect();
+                    }
+                }
+
+                if (returnCode !== null) {
+                    return returnCode;
+                }
+            } else if (ftpItem.type == 1) {
+                this.ftpScanCounter++;
+                this.FTPPurgeStatus({
+                    action: "ftp-purge-status",
+                    count: this.removeCounter,
+                    total: this.ftpScanCounter
+                });
+                if (!fs.existsSync(localPath + "/" + ftpItem.name)) {
+                    this.ftpPurgeList.push({
+                        file: localPath + "/" + ftpItem.name,
+                        remoteFile: remoteDirPath + "/" + ftpItem.name
+                    });
+                }
+            }
+        }
+    } else {
+        console.log('Noting in the ftp dir: ' + remoteDirPath);
+    }
+
+    return null;
+};
+
+FtpSync.prototype._openDir = async function (dir) {
     await this.client.sendIgnoringError("MKD " + dir);
 };
 
-FtpSync.prototype.updateLastmodified = async function(path, info) {
+FtpSync.prototype.updateLastmodified = async function (path, info) {
     let mod = new Date(info.mtimeMs);
@@ -469,7 +539,7 @@ path = await this.client.protectWhitespace(path);
 
-FtpSync.prototype.formatTime = function(time) {
+FtpSync.prototype.formatTime = function (time) {
     return Math.trunc(time / 1000);
 };
 
-FtpSync.prototype.lastmodifiedChanged = async function(path, info) {
+FtpSync.prototype.lastmodifiedChanged = async function (path, info) {
     let lastchange = await this.client.lastMod(path);
@@ -480,3 +550,3 @@ let mod = new Date(info.mtimeMs);
 
-Date.prototype.yyyymmdd = function() {
+Date.prototype.yyyymmdd = function () {
     var yyyy = this.getFullYear();
@@ -488,3 +558,3 @@ var mm = this.getUTCMonth() < 9 ? "0" + (this.getUTCMonth() + 1) : (this.getUTCMonth() + 1); // getMonth() is zero-based
 
-Date.prototype.yyyymmddhhmm = function() {
+Date.prototype.yyyymmddhhmm = function () {
     var yyyymmdd = this.yyyymmdd();
@@ -496,3 +566,3 @@ var hh = this.getUTCHours() < 10 ? "0" + this.getUTCHours() : this.getUTCHours();
 
-Date.prototype.yyyymmddhhmmss = function() {
+Date.prototype.yyyymmddhhmmss = function () {
     var yyyymmddhhmm = this.yyyymmddhhmm();
@@ -503,3 +573,3 @@ var ss = this.getUTCSeconds() < 10 ? "0" + this.getUTCSeconds() : this.getUTCSeconds();
 
-FtpSync.prototype.resumeUpload = async function(from, to, info) {
+FtpSync.prototype.resumeUpload = async function (from, to, info) {
     this.log("Get file size");
@@ -536,3 +606,3 @@ let size = 0;
 
-FtpSync.prototype.removeFileFromResume = function(src) {
+FtpSync.prototype.removeFileFromResume = function (src) {
     for (let i = 0; i < this.continueUpload.length; i++) {
@@ -546,3 +616,3 @@ if (this.continueUpload[i] === src) {
 
-FtpSync.prototype.syncFiles = async function(i) {
+FtpSync.prototype.syncFiles = async function (i) {
     if (this.halt()) {
@@ -667,3 +737,3 @@ return null;
 
-FtpSync.prototype.fileExists = function(file) {
+FtpSync.prototype.fileExists = function (file) {
     try {
@@ -679,3 +749,3 @@ if (fs.existsSync(file)) {
 
-FtpSync.prototype.getBaseDir = function(dir) {
+FtpSync.prototype.getBaseDir = function (dir) {
     if (this.batches.length === 1) {
@@ -685,2 +755,3 @@ return this.batches[0].dir;
 
+    let bestMatch = "";
     for (let index = 0; index < this.batches.length; index++) {
@@ -690,10 +761,29 @@ if (dir.substr(0, this.batches[index].dir.length).toUpperCase() == this.batches[index].dir.toUpperCase()) {
             let dirto = split.slice(0, split.length - 1).join(pathLib.sep) + pathLib.sep;
-            return dirto;
+            if(bestMatch.length < dirto.length){
+                bestMatch = dirto;
+            }
         }
     }
 
-    return null;
+    return bestMatch === "" ? null : bestMatch;
 }
 
-FtpSync.prototype.remove = async function(file) {
+FtpSync.prototype.getMainDir = function (dir) {
+    if (this.batches.length === 1) {
+        return this.batches[0].dir;
+    }
+
+    let bestMatch = "";
+    for (let index = 0; index < this.batches.length; index++) {
+        if (dir.substr(0, this.batches[index].dir.length).toUpperCase() == this.batches[index].dir.toUpperCase()) {
+            if(bestMatch.length < this.batches[index].dir.length){
+                bestMatch = this.batches[index].dir;
+            }
+        }
+    }
+
+    return bestMatch === "" ? null : bestMatch;
+}
+
+FtpSync.prototype.remove = async function (file) {
     if (!this.fileExists(file)) {
@@ -711,15 +801,22 @@ this.log('Ftp removing:' + file);
                 await this.removeErrorHandel(error);
+                return false;
             }
         }
+
+        this.removeCounter++;
+        this.FTPPurgeStatus({
+            action: "ftp-purge-status",
+            count: this.removeCounter,
+            total: this.ftpScanCounter
+        });
+
+        await this.fileIndexer.db.del(file);
+
         try {
             if (!this.fileExists(pathInfo.dir)) {
-                try {
-                    await this.client.removeDir(ftpDir);
-                } catch (error) {
-                    if (error.code !== 550) {
-                        await this.removeErrorHandel(error);
-                    }
+                if (!this.removeDirList.includes(ftpDir)) {
+                    this.removeDirList.push(ftpDir);
+                    this.purgeDirCheck(pathInfo.dir);
                 }
             }
-            await this.fileIndexer.db.del(file);
         } catch (error) {
@@ -729,5 +826,55 @@ await this.removeErrorHandel(error);
     }
+
+    return true;
 }
 
-FtpSync.prototype.removeErrorHandel = async function(error) {
+FtpSync.prototype.purgeDirCheck = function (dir) {
+    try {
+        let dirNotEmpty = true;
+        let counter = 0;
+        let ftpDir = "";
+        let baseDir = null;
+
+        while (dirNotEmpty) {
+            dir = pathLib.dirname(dir);
+            baseDir = this.getBaseDir(dir);
+            if(baseDir == null){ break; }
+            if(baseDir == dir){ break; }
+            
+            ftpDir = this.ftpBaseDir + "/" + upath.toUnix(pathLib.relative(baseDir, dir));
+
+            if (!this.fileExists(dir)) {
+                if (!this.removeDirList.includes(ftpDir)) {
+                    this.removeDirList.push(ftpDir);
+                }
+            }
+
+            counter++;
+            if (counter > 100) {
+                dirNotEmpty = false;
+                throw "Error pruging directory base path mismatch."
+            }
+        }
+    } catch (e) {
+        console.log(e);
+    }
+};
+
+FtpSync.prototype.purgeDirs = async function () {
+    let errorCounter = 0;
+    for (let i = 0; i < this.removeDirList.length; i++) {
+        try {
+            await this.client.removeDir(this.removeDirList[i]);
+        } catch (error) {
+            if (error.code !== 550) {
+                await this.removeErrorHandel(error);
+                errorCounter++;
+                i--;
+                continue;
+            }
+        }
+    }
+};
+
+FtpSync.prototype.removeErrorHandel = async function (error) {
     if (error.code !== 550) {
@@ -738,3 +885,3 @@ if (error.code === "ECONNRESET" ||
         } else {
-            throw err;
+            throw error;
         }
@@ -744,6 +891,17 @@ }
 
-FtpSync.prototype.purgeList = async function(files) {
+FtpSync.prototype.purgeList = async function (files) {
+    let response = false;
+    let maxErrorCounter = 0;
     for (let i = 0; i < files.length; i++) {
         try {
-            await this.remove(files[i]);
+            maxErrorCounter = 0;
+            response = false;
+            while (!response) {
+                response = await this.remove(files[i]);
+                maxErrorCounter++;
+
+                if (maxErrorCounter > 10) {
+                    throw 'Failed to remove file stopping task.';
+                }
+            }
         } catch (err) {
@@ -753,11 +911,16 @@ return err;
     }
+
+    await this.purgeDirs();
 }
 
-FtpSync.prototype.purge = async function(callback) {
+FtpSync.prototype.purge = async function (callback) {
 
     await this.fileIndexer.initDb();
     let files = [];
+    let mainDir = "";
+
     this.fileIndexer.db.createReadStream({ keys: true, values: false })
         .on('data', (file) => {
-            if (this.fileExists(this.getBaseDir(file))) {
+            mainDir = this.getMainDir(file);
+            if (this.fileExists(mainDir)) {
                 if (!this.fileExists(file)) {
@@ -768,2 +931,3 @@ files.push(file);
                 callback(files);
+                this.dirRemovedErrorInfo = { dir: mainDir };
                 throw 'Path does not exist';
@@ -779,3 +943,3 @@ }
         })
-        .on('end', async() => {
+        .on('end', async () => {
             callback(files);
@@ -785,2 +949,36 @@ });
 
+FtpSync.prototype.FTPPurge = async function () {
+    let response = false;
+    let maxErrorCounter = 0;
+    let ftpDir = "";
+
+    for (let i = 0; i < this.batches.length; i++) {
+        ftpDir = this.ftpBaseDir + "/" + upath.toUnix(pathLib.relative(this.getBaseDir(this.batches[i].dir), this.batches[i].dir));
+
+        await this.getPurgeList(this.batches[i].dir, ftpDir, 0);
+        for (let i2 = 0; i2 < this.ftpPurgeList.length; i2++) {
+            maxErrorCounter = 0;
+            response = false;
+            while (!response) {
+                if (!this.fileExists(this.batches[i].dir)) { 
+                    this.dirRemovedErrorInfo = { dir: this.batches[i].dir };
+                    throw 'Path does not exist';
+                }
+                response = await this.remove(this.ftpPurgeList[i2].file);
+                maxErrorCounter++;
+
+                if (maxErrorCounter > 10) {
+                    throw 'Failed to remove file stopping task.';
+                }
+            }
+        }
+    }
+
+    await this.purgeDirs();
+};
+
+FtpSync.prototype.FTPPurgeStatus = function (status) {
+
+}
+
 module.exports = FtpSync;

package.json

 {
     "name": "ftp-folder-sync",
-    "version": "1.0.6",
+    "version": "1.0.7",
     "description": "",
@@ -5,0 +5,0 @@ "main": "index.js",

No alert changes

Improved metrics

Total package byte prevSize

36904

increased by20.49%

Lines of code

1042

increased by19.36%

No dependency changes