@cloudcannon/asset-uploader - npm Package Compare versions

Comparing version 0.0.1-rc3 to 0.1.0

oclif.manifest.json

@@ -1,1 +0,1 @@
-{"version":"0.0.1-rc3","commands":{"analyse":{"id":"analyse","description":"Provide an output with the list of assets found to be uploaded.","pluginName":"@cloudcannon/asset-uploader","pluginType":"core","aliases":[],"flags":{"source":{"name":"source","type":"option","char":"n","description":"source of the website to be parsed"},"output":{"name":"output","type":"option","char":"n","description":"output changes"}},"args":[]},"complete":{"id":"complete","description":"Scan source directory, upload and rewrite source files with the new assets location.","pluginName":"@cloudcannon/asset-uploader","pluginType":"core","aliases":[],"flags":{"source":{"name":"source","type":"option","char":"n","description":"source of the website to be parsed"},"folder":{"name":"folder","type":"option","char":"n","description":"folder where to uploade the files"},"output":{"name":"output","type":"option","char":"n","description":"output changes"}},"args":[]},"upload":{"id":"upload","description":"Scan source directory and upload assets to cloudinary.","pluginName":"@cloudcannon/asset-uploader","pluginType":"core","aliases":[],"flags":{"source":{"name":"source","type":"option","char":"n","description":"source of the website to be parsed"},"folder":{"name":"folder","type":"option","char":"n","description":"folder where to upload the files on cloudinary"},"output":{"name":"output","type":"option","char":"n","description":"output changes"}},"args":[]}}}
+{"version":"0.1.0","commands":{"analyse":{"id":"analyse","description":"Provide an output with the list of assets found to be uploaded.","pluginName":"@cloudcannon/asset-uploader","pluginType":"core","aliases":[],"flags":{"source":{"name":"source","type":"option","char":"s","description":"source of the website to be parsed"},"output":{"name":"output","type":"option","char":"o","description":"output changes"}},"args":[]},"complete":{"id":"complete","description":"Scan source directory, upload and rewrite source files with the new assets location.","pluginName":"@cloudcannon/asset-uploader","pluginType":"core","aliases":[],"flags":{"source":{"name":"source","type":"option","char":"s","description":"source of the website to be parsed"},"folder":{"name":"folder","type":"option","char":"f","description":"folder where to uploade the files"},"output":{"name":"output","type":"option","char":"o","description":"output changes"}},"args":[]},"upload":{"id":"upload","description":"Scan source directory and upload assets to cloudinary.","pluginName":"@cloudcannon/asset-uploader","pluginType":"core","aliases":[],"flags":{"source":{"name":"source","type":"option","char":"s","description":"source of the website to be parsed"},"folder":{"name":"folder","type":"option","char":"f","description":"folder where to upload the files on cloudinary"},"output":{"name":"output","type":"option","char":"o","description":"output changes"}},"args":[]}}}

package.json

 {
   "name": "@cloudcannon/asset-uploader",
-  "version": "0.0.1-rc3",
+  "version": "0.1.0",
   "author": "Jonatah Benatti @jonatahb",
@@ -14,3 +14,5 @@ "bin": {
     "ansi-colors": "^4.1.1",
+    "cli-progress": "^3.8.2",
     "cloudinary": "^1.22.0",
+    "es6-promise-pool": "^2.5.0",
     "fs": "0.0.1-security",
@@ -32,3 +34,3 @@ "js-yaml": "^3.14.0",
     "globby": "^10.0.2",
-    "mocha": "^8.0.1",
+    "mocha": "^8.1.1",
     "nyc": "^14.1.1"
@@ -61,3 +63,3 @@ },
   },
-  "repository": "CloudCannon/asset-uploader",
+  "repository": "CloudCannon/assets-uploader",
   "scripts": {
@@ -64,0 +66,0 @@ "postpack": "rm -f oclif.manifest.json",

README.md

@@ -23,3 +23,3 @@ asset-uploader
 $ asset-uploader (-v|--version|version)
-@cloudcannon/asset-uploader/0.0.1-rc3 darwin-x64 node-v10.19.0
+@cloudcannon/asset-uploader/0.1.0 darwin-x64 node-v10.19.0
 $ asset-uploader --help [COMMAND]
@@ -58,7 +58,7 @@ USAGE
 OPTIONS
-  -n, --output=output  output changes
-  -n, --source=source  source of the website to be parsed
+  -o, --output=output  output changes
+  -s, --source=source  source of the website to be parsed
 ```
 
-_See code: [src/commands/analyse.js](https://github.com/CloudCannon/asset-uploader/blob/v0.0.1-rc3/src/commands/analyse.js)_
+_See code: [src/commands/analyse.js](https://github.com/CloudCannon/assets-uploader/blob/v0.1.0/src/commands/analyse.js)_
 
@@ -74,8 +74,8 @@ ## `asset-uploader complete`
 OPTIONS
-  -n, --folder=folder  folder where to uploade the files
-  -n, --output=output  output changes
-  -n, --source=source  source of the website to be parsed
+  -f, --folder=folder  folder where to uploade the files
+  -o, --output=output  output changes
+  -s, --source=source  source of the website to be parsed
 ```
 
-_See code: [src/commands/complete.js](https://github.com/CloudCannon/asset-uploader/blob/v0.0.1-rc3/src/commands/complete.js)_
+_See code: [src/commands/complete.js](https://github.com/CloudCannon/assets-uploader/blob/v0.1.0/src/commands/complete.js)_
 
@@ -108,8 +108,8 @@ ## `asset-uploader help [COMMAND]`
 OPTIONS
-  -n, --folder=folder  folder where to upload the files on cloudinary
-  -n, --output=output  output changes
-  -n, --source=source  source of the website to be parsed
+  -f, --folder=folder  folder where to upload the files on cloudinary
+  -o, --output=output  output changes
+  -s, --source=source  source of the website to be parsed
 ```
 
-_See code: [src/commands/upload.js](https://github.com/CloudCannon/asset-uploader/blob/v0.0.1-rc3/src/commands/upload.js)_
+_See code: [src/commands/upload.js](https://github.com/CloudCannon/assets-uploader/blob/v0.1.0/src/commands/upload.js)_
 <!-- commandsstop -->

src/commands/analyse.js

@@ -20,6 +20,6 @@ const { Command, flags } = require('@oclif/command');
 AnalyseCommand.flags = {
-	source: flags.string({ char: 'n', description: 'source of the website to be parsed' }),
-	output: flags.string({ char: 'n', description: 'output changes' })
+	source: flags.string({ char: 's', description: 'source of the website to be parsed' }),
+	output: flags.string({ char: 'o', description: 'output changes' })
 };
 
 module.exports = AnalyseCommand;

src/commands/complete.js

@@ -20,7 +20,7 @@ const { Command, flags } = require('@oclif/command');
 CompleteCommand.flags = {
-	source: flags.string({ char: 'n', description: 'source of the website to be parsed' }),
-	folder: flags.string({ char: 'n', description: 'folder where to uploade the files' }),
-	output: flags.string({ char: 'n', description: 'output changes' })
+	source: flags.string({ char: 's', description: 'source of the website to be parsed' }),
+	folder: flags.string({ char: 'f', description: 'folder where to uploade the files' }),
+	output: flags.string({ char: 'o', description: 'output changes' })
 };
 
 module.exports = CompleteCommand;

src/commands/upload.js

@@ -20,7 +20,7 @@ const { Command, flags } = require('@oclif/command');
 UploadCommand.flags = {
-	source: flags.string({ char: 'n', description: 'source of the website to be parsed' }),
-	folder: flags.string({ char: 'n', description: 'folder where to upload the files on cloudinary' }),
-	output: flags.string({ char: 'n', description: 'output changes' })
+	source: flags.string({ char: 's', description: 'source of the website to be parsed' }),
+	folder: flags.string({ char: 'f', description: 'folder where to upload the files on cloudinary' }),
+	output: flags.string({ char: 'o', description: 'output changes' })
 };
 
 module.exports = UploadCommand;

src/lib/cloudinaryHelper.js

@@ -5,4 +5,7 @@ const cloudinary = require('cloudinary').v2;
 	upload: function (filePath, config, folder) {
+		if (!config.cloudinary.cloud_name || !config.cloudinary.api_key || !config.cloudinary.api_secret) {
+			throw new Error('Invalid configuration for Cloudinary API Keys!');
+		}
 		this.setCloudinaryConfig(config);
-		return cloudinary.uploader.upload(filePath,
+		const upload = cloudinary.uploader.upload(filePath,
 			{
@@ -14,2 +17,3 @@ use_filename: true,
 			});
+		return upload;
 	},
@@ -16,0 +20,0 @@

src/lib/config/cloudinary.js

@@ -5,6 +5,2 @@ const CLOUD_NAME = process.env.CLOUD_NAME ? process.env.CLOUD_NAME : '';
 
-if (!CLOUD_NAME || !API_KEY || !API_SECRET) {
-	throw new Error('Invalid configuration for Cloudfinary API Keys!');
-}
-
 module.exports = {
@@ -11,0 +7,0 @@ cloud_name: CLOUD_NAME,

src/lib/frontMatterHelper.js

@@ -56,4 +56,7 @@ const jsyaml = require('js-yaml');
 		}
-
-		return this.wrap(this.stringifyYaml(frontMatterObj));
+		const safeDump = this.stringifyYaml(frontMatterObj);
+		if (!safeDump) {
+			return this.empty;
+		}
+		return this.wrap(safeDump);
 	},
@@ -71,5 +74,11 @@
 
-		return jsyaml
-			.safeDump(frontMatterObj)
-			.replace(/([^"][:-]) (\[\]|''|null)(\n|$)/g, '$1\n'); // Normalises junk empty values produced by safeDump
+		try {
+			return jsyaml
+				.safeDump(frontMatterObj)
+				.replace(/([^"][:-]) (\[\]|''|null)(\n|$)/g, '$1\n'); // Normalises junk empty values produced by safeDump
+		} catch {
+			console.log('Failed to safeDump:');
+			console.log(frontMatterObj);
+			return null;
+		}
 	},
@@ -76,0 +85,0 @@

src/lib/ParserInterface.js

@@ -13,2 +13,16 @@ // const c = require('ansi-colors');
 }
+
+function traverse(o, fn) {
+	Object.keys(o).forEach((i) => {
+		const updatedValue = fn.apply(this, [i, o[i]]);
+		if (updatedValue !== o[i]) {
+			o[i] = updatedValue;
+		}
+
+		if (o[i] !== null && typeof (o[i]) === 'object') {
+			traverse(o[i], fn);
+		}
+	});
+}
+
 function handleHTMLFile(parser) {
@@ -30,15 +44,17 @@ return through(function transform(file, encoding, callback) {
 		if (frontMatter) {
-			Object.keys(frontMatter).forEach((key) => {
-				if (frontMatter[key] && frontMatterHelper.isLocalImage(key, frontMatter[key])) {
-					frontMatter[key] = parser.processFrontMatter(key, frontMatter[key], file.sitePath);
+			traverse(frontMatter, (k, v) => {
+				if (v && frontMatterHelper.isLocalImage(k, v)) {
+					v = parser.processFrontMatter(k, v, file.sitePath);
 					foundImage = true;
 				}
+				return v;
 			});
 			if (parser.updateFrontMatter && foundImage) {
-				const newContent = frontMatterHelper.stringify(frontMatter)
-									+ frontMatterHelper.parseContent(fileContent);
+				const newfrontMatter = frontMatterHelper.stringify(frontMatter);
+				if (newfrontMatter) {
+					const newContent = newfrontMatter + frontMatterHelper.parseContent(fileContent);
+					file.contents = Buffer.from(newContent);
 
-				file.contents = Buffer.from(newContent);
-
-				this.push(file);
+					this.push(file);
+				}
 			}
@@ -50,2 +66,3 @@ }
 }
+
 module.exports = class ParserInterface {
@@ -52,0 +69,0 @@ constructor(options) {

src/lib/runner.js

@@ -8,4 +8,4 @@ const config = require('./config/config.js');
 	analyse: async function (options) {
-		console.log('Options');
-		console.log(options);
+		// console.log('Options');
+		// console.log(options);
 		const reader = new Reader(options);
@@ -18,7 +18,9 @@ await reader.readFiles();
 	upload: async function (options) {
-		console.log('Options');
-		console.log(options);
+		// console.log('Options');
+		// console.log(options);
 		const reader = new Reader(options);
 		await reader.readFiles();
 
+		await reader.outputResult();
+
 		const uploader = new Uploader(options, reader.images, config);
@@ -31,10 +33,14 @@ await uploader.upload();
 	complete: async function (options) {
-		console.log('Options');
-		console.log(options);
+		// console.log('Options');
+		// console.log(options);
 		const reader = new Reader(options);
 		await reader.readFiles();
 
+		await reader.outputResult();
+
 		const uploader = new Uploader(options, reader.images, config);
 		await uploader.upload();
 
+		await reader.outputResult(uploader.images);
+
 		const writer = new Writer(options, uploader.images);
@@ -41,0 +47,0 @@ await writer.updateFiles();

src/lib/Uploader.js

 const path = require('path');
+const PromisePool = require('es6-promise-pool');
+const cliProgress = require('cli-progress');
 const cloudinary = require('./cloudinaryHelper');
 
+const CLOUDINARY_BATCH_SIZE = 100;
 module.exports = class Uploader {
@@ -11,17 +14,30 @@ constructor(options, images, config) {
 
-	upload() {
-		const promises = [];
+	// eslint-disable-next-line class-methods-use-this
+	async upload() {
+		// create a new progress bar instance and use shades_classic theme
+		const bar1 = new cliProgress.SingleBar({ etaBuffer: CLOUDINARY_BATCH_SIZE }, cliProgress.Presets.shades_classic);
 
-		Object.keys(this.images).forEach((filePath) => {
+		let complete = 0;
+		const errors = [];
+		// Create array of functions that will return a promise when executed
+		const promises = Object.keys(this.images).map((filePath) => {
 			const fullPath = path.join(this.options.source, filePath);
 			const dirName = path.dirname(filePath);
 			const uploadFolder = path.join(this.options.folder, dirName);
-			console.log(`Uploading: ${fullPath}`);
-			const promise = cloudinary.upload(fullPath, this.config, uploadFolder)
+			// console.log(`Uploading: ${fullPath}`);
+			const upload = () => cloudinary.upload(fullPath, this.config, uploadFolder)
 				.then((result) => {
-					console.log(`Image uploaded: ${filePath} -> ${result.url}`);
-					this.images[filePath].url = result.url;
+					// console.log(`Image uploaded: ${filePath} -> ${result.url}`);
+					this.images[filePath].url = result.secure_url || result.url;
+					complete += 1;
+					// update the current value in your application..
+					bar1.update(complete);
 				}, (error) => {
-					console.log('Error!!');
-					console.log(error);
+					complete += 1;
+					// update the current value in your application..
+					bar1.update(complete);
+					errors.push(error);
+					this.images[filePath].errors = error;
+					// console.log('Error!');
+					// console.log(error);
 				}).catch((reason) => {
@@ -31,8 +47,44 @@ console.log('Exception!!');
 				});
-			promises.push(promise);
+			// return upload;
+			return upload;
 		});
-		return Promise.all(promises).then(() => {
-			console.log('Finished uploading');
+		const totalImages = promises.length;
+
+		console.log(`Found ${totalImages} to upload!`);
+
+		// start the progress bar with a total value of 200 and start value of 0
+		bar1.start(totalImages, 0);
+
+		let count = 0;
+		const promiseProducer = function () {
+			if (count >= promises.length) {
+				return null;
+			}
+			const promise = promises[count];
+			count += 1;
+			return promise();
+		};
+
+		// The number of promises to process simultaneously.
+		const concurrency = CLOUDINARY_BATCH_SIZE;
+
+		// Create a pool.
+		const pool = new PromisePool(promiseProducer, concurrency);
+		// Start the pool.
+		const poolPromise = pool.start();
+
+		// Wait for the pool to settle.
+		await poolPromise.then(() => {
+			// console.log('All promises fulfilled');
+		}, (error) => {
+			console.log(`Some promise rejected: ${error.message}`);
 		});
+
+		// stop the progress bar
+		bar1.stop();
+		errors.forEach((error) => {
+			console.log('Erross encountered uploading files:');
+			console.log(error);
+		});
 	}
 };

src/lib/Writer.js

@@ -22,3 +22,3 @@ const vfs = require('vinyl-fs');
 	processFrontMatter(key, value, filePath) {
-		if (!this.images[value]) {
+		if (!this.images[value] || !this.images[value].url) {
 			return value;
@@ -25,0 +25,0 @@ }

New alerts

No repository

Supply chain risk

Package does not have a linked source code repository. Without this field, a package will have no reference to the location of the source code use to generate the package.

Found 1 instance in 1 package

Fixed alerts

No repository

Supply chain risk

Package does not have a linked source code repository. Without this field, a package will have no reference to the location of the source code use to generate the package.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

21630

increased by12.66%

Lines of code

455

increased by19.11%

Worsened metrics

Dependency count

13

increased by18.18%

Dependency changes

• + Addedcli-progress@^3.8.2

• + Addedes6-promise-pool@^2.5.0

• + Addedcli-progress@3.12.0(transitive)

• + Addedes6-promise-pool@2.5.0(transitive)

• + Addedms@2.1.2(transitive)

• + Addedtslib@2.6.3(transitive)

• - Removedtslib@2.6.2(transitive)