easy-sftp-deploy - npm Package Compare versions

Comparing version 1.3.9 to 1.4.0

dist/config.d.ts

@@ -33,2 +33,4 @@ import { z } from 'zod';
     credentialsID: z.ZodString;
+    useFastPut: z.ZodDefault<z.ZodBoolean>;
+    parallel: z.ZodDefault<z.ZodNumber>;
     port: z.ZodOptional<z.ZodNumber>;
@@ -39,3 +41,7 @@ }, "strict", z.ZodTypeAny, {
     credentialsID: string;
+    useFastPut: boolean;
+    parallel: number;
 }, {
+    useFastPut?: boolean | undefined;
+    parallel?: number | undefined;
     port?: number | undefined;
@@ -48,2 +54,4 @@ host: string;
     credentialsID: z.ZodString;
+    useFastPut: z.ZodDefault<z.ZodBoolean>;
+    parallel: z.ZodDefault<z.ZodNumber>;
     port: z.ZodOptional<z.ZodNumber>;
@@ -54,3 +62,7 @@ }, "strict", z.ZodTypeAny, {
     credentialsID: string;
+    useFastPut: boolean;
+    parallel: number;
 }, {
+    useFastPut?: boolean | undefined;
+    parallel?: number | undefined;
     port?: number | undefined;
@@ -100,12 +112,12 @@ host: string;
     dstFolder: z.ZodString;
-    overwrite: z.ZodOptional<z.ZodBoolean>;
-    clear: z.ZodOptional<z.ZodBoolean>;
-    dryRun: z.ZodOptional<z.ZodBoolean>;
+    overwrite: z.ZodDefault<z.ZodBoolean>;
+    clear: z.ZodDefault<z.ZodBoolean>;
+    dryRun: z.ZodDefault<z.ZodBoolean>;
 }, "strict", z.ZodTypeAny, {
-    overwrite?: boolean | undefined;
-    clear?: boolean | undefined;
-    dryRun?: boolean | undefined;
     hostID: string;
     srcID: string;
     dstFolder: string;
+    overwrite: boolean;
+    clear: boolean;
+    dryRun: boolean;
 }, {
@@ -137,2 +149,4 @@ overwrite?: boolean | undefined;
         credentialsID: z.ZodString;
+        useFastPut: z.ZodDefault<z.ZodBoolean>;
+        parallel: z.ZodDefault<z.ZodNumber>;
         port: z.ZodOptional<z.ZodNumber>;
@@ -143,3 +157,7 @@ }, "strict", z.ZodTypeAny, {
         credentialsID: string;
+        useFastPut: boolean;
+        parallel: number;
     }, {
+        useFastPut?: boolean | undefined;
+        parallel?: number | undefined;
         port?: number | undefined;
@@ -169,12 +187,12 @@ host: string;
         dstFolder: z.ZodString;
-        overwrite: z.ZodOptional<z.ZodBoolean>;
-        clear: z.ZodOptional<z.ZodBoolean>;
-        dryRun: z.ZodOptional<z.ZodBoolean>;
+        overwrite: z.ZodDefault<z.ZodBoolean>;
+        clear: z.ZodDefault<z.ZodBoolean>;
+        dryRun: z.ZodDefault<z.ZodBoolean>;
     }, "strict", z.ZodTypeAny, {
-        overwrite?: boolean | undefined;
-        clear?: boolean | undefined;
-        dryRun?: boolean | undefined;
         hostID: string;
         srcID: string;
         dstFolder: string;
+        overwrite: boolean;
+        clear: boolean;
+        dryRun: boolean;
     }, {
@@ -198,2 +216,4 @@ overwrite?: boolean | undefined;
         credentialsID: string;
+        useFastPut: boolean;
+        parallel: number;
     }>;
@@ -207,8 +227,8 @@ sourceFolders: Record<string, {
     deployments: {
-        overwrite?: boolean | undefined;
-        clear?: boolean | undefined;
-        dryRun?: boolean | undefined;
         hostID: string;
         srcID: string;
         dstFolder: string;
+        overwrite: boolean;
+        clear: boolean;
+        dryRun: boolean;
     }[];
@@ -222,2 +242,4 @@ }, {
     hosts: Record<string, {
+        useFastPut?: boolean | undefined;
+        parallel?: number | undefined;
         port?: number | undefined;
@@ -224,0 +246,0 @@ host: string;

dist/config.js

@@ -6,17 +6,24 @@ "use strict";
 // Credentials
-const sftpCredentialConfig = zod_1.z.object({
+const sftpCredentialConfig = zod_1.z
+    .object({
     username: zod_1.z.string(),
     password: zod_1.z.string().optional(),
     privateKey: zod_1.z.string().optional(),
-}).strict();
+})
+    .strict();
 const sftpCredentialsConfig = zod_1.z.record(sftpCredentialConfig);
 // Hosts
-const sftpHostConfig = zod_1.z.object({
+const sftpHostConfig = zod_1.z
+    .object({
     host: zod_1.z.string(),
     credentialsID: zod_1.z.string(),
+    useFastPut: zod_1.z.boolean().default(false),
+    parallel: zod_1.z.number().int().min(1).default(10),
     port: zod_1.z.number().optional(),
-}).strict();
+})
+    .strict();
 const sftpHostsConfig = zod_1.z.record(sftpHostConfig);
 // Sources
-const sftpSrcConfig = zod_1.z.object({
+const sftpSrcConfig = zod_1.z
+    .object({
     folder: zod_1.z.string(),
@@ -26,15 +33,19 @@ filters: zod_1.z.array(zod_1.z.string()).optional(),
     includeAllFolders: zod_1.z.boolean().optional(),
-}).strict();
+})
+    .strict();
 const sftpSrcsConfig = zod_1.z.record(sftpSrcConfig);
 // Deployments
-const sftpDeployConfig = zod_1.z.object({
+const sftpDeployConfig = zod_1.z
+    .object({
     hostID: zod_1.z.string(),
     srcID: zod_1.z.string(),
     dstFolder: zod_1.z.string(),
-    overwrite: zod_1.z.boolean().optional(),
-    clear: zod_1.z.boolean().optional(),
-    dryRun: zod_1.z.boolean().optional(),
-}).strict();
+    overwrite: zod_1.z.boolean().default(false),
+    clear: zod_1.z.boolean().default(false),
+    dryRun: zod_1.z.boolean().default(false),
+})
+    .strict();
 // Main config
-exports.sftpConfig = zod_1.z.object({
+exports.sftpConfig = zod_1.z
+    .object({
     credentials: sftpCredentialsConfig,
@@ -44,2 +55,3 @@ hosts: sftpHostsConfig,
     deployments: zod_1.z.array(sftpDeployConfig),
-}).strict();
+})
+    .strict();

dist/deployment.js

@@ -16,6 +16,7 @@ "use strict";
 exports.ExecuteDeployment = void 0;
+const fs_1 = __importDefault(require("fs"));
 const micromatch_1 = __importDefault(require("micromatch"));
-const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const ssh2_sftp_client_1 = __importDefault(require("ssh2-sftp-client"));
+const parallelizor_1 = __importDefault(require("./parallelizor"));
 const recursive_readdir_1 = __importDefault(require("./recursive-readdir"));
@@ -88,3 +89,4 @@ function ExecuteDeployment(config, deploymentID, log) {
             })
-            : filteredSrcFiles.map((file) => path_1.default.dirname(file))
+            : filteredSrcFiles
+                .map((file) => path_1.default.dirname(file))
                 .filter((folder) => folder != '.')
@@ -111,2 +113,3 @@ .filter((folder, index, self) => self.indexOf(folder) === index);
         const sftp = new ssh2_sftp_client_1.default();
+        sftp.client.setMaxListeners(host.parallel + 10);
         yield sftp.connect({
@@ -128,21 +131,29 @@ host: host.host,
         }
+        const parallelizor = new parallelizor_1.default(host.parallel);
+        // Clear files ===========================================================
         if (deployment.clear) {
             log(`Clearing target directory ${dstAbsDir}`, { color: 'cyan' });
-            try {
-                const found = yield sftp.list(dstAbsDir);
-                for (let file of found) {
-                    const p = path_1.default.resolve(dstAbsDir, file.name);
-                    log(`Deleting ${p}`, { color: 'yellow' });
-                    if (file.type === 'd') {
-                        yield sftp.rmdir(p, true);
+            const found = yield sftp.list(dstAbsDir);
+            for (let i = 0; i < found.length; i++) {
+                const p = path_1.default.resolve(dstAbsDir, found[i].name);
+                log(`Deleting ${p}`, { color: 'yellow' });
+                const result = yield parallelizor.run(() => __awaiter(this, void 0, void 0, function* () {
+                    try {
+                        if (found[i].type === 'd') {
+                            yield sftp.rmdir(p, true);
+                        }
+                        else {
+                            yield sftp.delete(p);
+                        }
+                        return true;
                     }
-                    else {
-                        yield sftp.delete(p);
+                    catch (e) {
+                        log(`Could not delete ${p}`, { type: 'error' });
+                        return false;
                     }
+                }), i === found.length - 1);
+                if (result.includes(false)) {
+                    return false;
                 }
             }
-            catch (e) {
-                log(`Could not clear directory ${dstAbsDir}`, { type: 'error' });
-                return false;
-            }
         }
@@ -165,2 +176,6 @@ log(`Creating folders on ${dstAbsDir}`, { color: 'cyan' });
         log(`Uploading files to ${dstAbsDir}`, { color: 'cyan' });
+        if (host.parallel > 1)
+            log(`  - Parallel uploads: ${host.parallel}`, { color: 'cyan' });
+        if (host.useFastPut)
+            log(`  - Using fastPut`, { color: 'cyan' });
         for (let i = 0; i < filteredSrcFiles.length; i++) {
@@ -172,13 +187,23 @@ const srcFile = path_1.default.resolve(srcAbsDir, filteredSrcFiles[i]);
             });
-            try {
-                if (yield sftp.exists(dstFile)) {
-                    if (!deployment.overwrite) {
-                        log(`  Skipping, already exists`, { type: 'error' });
-                        continue;
+            const results = yield parallelizor.run(() => __awaiter(this, void 0, void 0, function* () {
+                try {
+                    if (yield sftp.exists(dstFile)) {
+                        if (!deployment.overwrite) {
+                            log(`Skipping ${srcFile}, already exists`, { type: 'error' });
+                            return true;
+                        }
                     }
+                    if (host.useFastPut)
+                        yield sftp.fastPut(srcFile, dstFile);
+                    else
+                        yield sftp.put(srcFile, dstFile);
+                    return true;
                 }
-                yield sftp.fastPut(srcFile, dstFile);
-            }
-            catch (e) {
-                log(`Could not upload ${srcFile}`, { type: 'error' });
+                catch (e) {
+                    console.error(e);
+                    log(`Could not upload ${srcFile}`, { type: 'error' });
+                    return false;
+                }
+            }), i === filteredSrcFiles.length - 1);
+            if (results.includes(false)) {
                 return false;
@@ -185,0 +210,0 @@ }

package.json

 {
   "name": "easy-sftp-deploy",
-  "version": "1.3.9",
+  "version": "1.4.0",
   "description": "Easily deploy files and directories to a server with sftp",
@@ -5,0 +5,0 @@ "repository": "https://github.com/rubikscraft/easy-sftp-deploy",

README.md

@@ -1,2 +0,2 @@
-# easy-sftp-deploy
+# Easy SFTP deploy
 
@@ -28,4 +28,4 @@ Thusfar every sftp deploy package I've tried has been terrible, so I thought I'd make my own better one.
 
-```txt
-credentials: { [credtialID]: credentialConfig }
+```yml
+credentials: { [credentialID]: credentialConfig }
 hosts: { [hostID]: hostConfig }
@@ -40,3 +40,3 @@ sourceFolders: { [sourceFolderID]: sourceFolderConfig }
 
-```txt
+```yml
 username: string
@@ -50,8 +50,12 @@ password: string (optional)
 For the host configuration you can specify the hostname and port as expected.
+There are also options for speeding up the connection using either fastPut for large files and/or parallelization for many files. Do keep in mind that both of these options might not work on all servers.
 But for the credentials, you need to reference one of the configured credentials. This allows you to more easily reuse credentials between multiple hosts without a giant configuration file.
 
-```txt
+```yml
 hostname: string
 port: number (optional, default is 22)
 
+useFastPut: boolean (optional, default is false)
+parallel: number (optional, default is 10)
+
 credentialsID: string
@@ -67,3 +71,3 @@ ```
 
-```txt
+```yml
 folder: string
@@ -88,3 +92,3 @@ filters: [string] (optional)
 
-```txt
+```yml
 hostID: string
@@ -111,4 +115,5 @@ sourceFolderID: string
 ```js
+// ES6 Style
 import { DeployToSftp } from 'easy-sftp-deploy';
-// or
+// CommonJS Style
 const { DeployToSftp } = require('easy-sftp-deploy');
@@ -157,2 +162,5 @@ ```
       credentialsID: 'stevelogin',
+
+      useFastPut: true,
+      parallel: 50,
     },
@@ -224,2 +232,4 @@ },
 
+* [1.4.0]
+  * Added support for parallel uploads
 * [1.3.3 - 1.3.9]
@@ -226,0 +236,0 @@ * Update dependencies

No alert changes

Improved metrics

Total package byte prevSize

74284

increased by5.81%

Number of package files

19

increased by11.76%

Lines of code

780

increased by13.7%

Number of lines in readme file

248

increased by4.2%

No dependency changes