@hubspot/cms-lib - npm Package Compare versions

Comparing version 0.0.13 to 0.0.14

download.js

 const path = require('path');
 const fs = require('fs-extra');
 const prettier = require('prettier');
-const { downloadModule: downloadModuleApi } = require('./fileMapper');
-const { fetchBuiltinMapping } = require('./designManager');
+const { fetchBuiltinMapping } = require('./api/designManager');
+const { fetchModule } = require('./api/fileMapper');
 const { logger } = require('./logger');
@@ -60,3 +60,3 @@
   try {
-    response = await downloadModuleApi(portalId, moduleId);
+    response = await fetchModule(portalId, moduleId);
   } catch (error) {
@@ -63,0 +63,0 @@ logger.error('Failed to download %s', moduleId);

fileMapper.js

 const fs = require('fs-extra');
 const path = require('path');
-const http = require('./http');
 const { logger } = require('./logger');
 const { convertToLocalFileSystemPath } = require('./path');
+const { fetchFile, fetchFolder } = require('./api/fileMapper');
+const { default: PQueue } = require('p-queue');
 
-const FILE_MAPPER_API_PATH = 'content/filemapper/v1';
+const CONCURRENCY = 100;
 
-async function upload(portalId, src, dest) {
-  return http.post(portalId, {
-    uri: `${FILE_MAPPER_API_PATH}/upload/${encodeURIComponent(dest)}`,
-    formData: {
-      file: fs.createReadStream(path.resolve(process.env.INIT_CWD, src)),
-    },
-  });
-}
-
-async function downloadModule(portalId, moduleId) {
-  return http.get(portalId, {
-    uri: `${FILE_MAPPER_API_PATH}/modules/${moduleId}`,
-  });
-}
-
 /**
@@ -39,67 +25,62 @@ * TODO: Replace with TypeScript interface.
 /**
- * Download a file by file path.
- *
- * @async
- * @param {number} portalId
- * @param {string} filePath
- * @returns {Promise<FileMapperNode>}
+ * @private
+ * @param {FileMapperNode} node
+ * @throws {TypeError}
  */
-async function downloadFile(portalId, filePath) {
-  return http.get(portalId, {
-    uri: `${FILE_MAPPER_API_PATH}/download/${filePath}`,
-  });
+function validateFileMapperNode(node) {
+  if (node === Object(node)) return;
+  let json;
+  try {
+    json = JSON.stringify(node, null, 2);
+  } catch (err) {
+    json = node;
+  }
+  throw new TypeError(`Invalid FileMapperNode: ${json}`);
 }
 
 /**
- * Download a folder by folder path.
- *
- * @async
- * @param {number} portalId
- * @param {string} folderPath
- * @returns {Promise<FileMapperNode>}
+ * @callback recurseFileMapperNodeCallback
+ * @param {FileMapperNode} node
+ * @param {object}         meta
+ * @param {boolean}        meta.isRoot
+ * @param {number}         meta.depth
+ * @returns {boolean} `false` to exit recursion.
  */
-async function downloadFolder(portalId, folderPath) {
-  return http.get(portalId, {
-    uri: `${FILE_MAPPER_API_PATH}/download/folder/${folderPath}`,
-  });
-}
 
 /**
- * Download entire tree for portal.
+ * Recurse a FileMapperNode tree.
  *
- * @async
- * @param {number} portalId
- * @returns {Promise<FileMapperNode>}
+ * @param {FileMapperNode}                node
+ * @param {recurseFileMapperNodeCallback} callback
+ * @throws {Error}
  */
-async function downloadAll(portalId) {
-  return http.get(portalId, {
-    uri: `${FILE_MAPPER_API_PATH}/download/all`,
+function recurseFileMapperNode(node, callback, depth = 0) {
+  validateFileMapperNode(node);
+  const isRoot = depth === 0 && node.folder && node.path === '/';
+  let __break = callback(node, { isRoot, depth });
+  if (__break === false) return __break;
+  __break = node.children.every(childNode => {
+    __break = recurseFileMapperNode(childNode, callback, depth + 1);
+    return __break !== false;
   });
+  return depth === 0 ? undefined : __break;
 }
 
 /**
- * @param {FileMapperNode} node
- * @throws {TypeError}
- */
-function validateFileMapperNode(node) {
-  if (node !== Object(node)) {
-    throw new TypeError('Invalid filemapper node');
-  }
-}
-
-/**
- * Write file/folder data for an individual node to disk.
- *
+ * @private
  * @async
+ * @param {Object} input
+ * @param {number} input.portalId
+ * @param {stirng} input.path
+ * @param {string} input.dest
+ * @param {object} input.options
  * @param {FileMapperNode} node
- * @param {string}         dest
- * @param {boolean}        overwrite
  * @returns {Promise}
  */
-async function writeNode(node, dest, overwrite = false) {
-  let filepath = '';
+async function writeFileMapperNode(input, node) {
+  const { portalId, dest, options } = input;
+  let filepath;
   try {
-    validateFileMapperNode(node);
     filepath = convertToLocalFileSystemPath(path.join(dest, node.path));
-    if (!overwrite && (await fs.pathExists(filepath))) {
+    if (!options.overwrite && (await fs.pathExists(filepath))) {
       logger.log('Skipped existing "%s"', filepath);
@@ -112,3 +93,12 @@ return;
     } else {
-      await fs.ensureFile(filepath);
+      if (node.source != null) {
+        // File fetches will include `source`
+        await fs.ensureFile(filepath);
+      } else {
+        // Files from folder fetches will not include `source`
+        [, node] = await Promise.all([
+          fs.ensureFile(filepath),
+          fetchFile(portalId, node.path),
+        ]);
+      }
       if (typeof node.source === 'string') {
@@ -124,3 +114,3 @@ await fs.writeFile(filepath, node.source);
   } catch (err) {
-    logger.error('Error writing node "%s": %o', filepath, err);
+    logger.error('Error writing "%s": %o', filepath, err);
   }
@@ -130,27 +120,53 @@ }
 /**
- * Write file/folder data recursively to disk.
- *
+ * @private
  * @async
+ * @param {Object} input
+ * @param {number} input.portalId
+ * @param {stirng} input.path
+ * @param {string} input.dest
+ * @param {object} input.options
  * @param {FileMapperNode} node
- * @param {string}         dest
- * @param {Object}         options
- * @param {boolean}        options.overwrite - Overwrite existing nodes.
- * @returns {Promise}
  */
-async function writeFileMapping(node, dest, options = {}, __depth = 0) {
+async function recursiveWriteFileMapperNode(input, node) {
+  const { path: _path, dest } = input;
   try {
-    validateFileMapperNode(node);
-    const isRoot = __depth === 0 && node.folder && node.path === '/';
-    if (!isRoot) {
-      writeNode(node, dest, options.overwrite);
-    }
-    node.children.forEach(childNode => {
-      writeFileMapping(childNode, dest, options, __depth + 1);
+    const queue = new PQueue({
+      concurrency: CONCURRENCY,
     });
+    recurseFileMapperNode(node, (childNode, { isRoot }) => {
+      if (isRoot) return;
+      queue.add(() => writeFileMapperNode(input, childNode));
+    });
+    await queue.onIdle();
+    logger.log('Completed fetch "%s" to "%s"', _path, dest);
   } catch (err) {
-    logger.error('Error writing filemapper tree: %o', err);
-    return;
+    logger.error(
+      'Error writing filemapper tree from "%s" to "%s": %o',
+      _path,
+      dest,
+      err
+    );
   }
-  if (__depth === 0) {
-    logger.log('Completed writing filemapping to "%s"', dest);
+}
+
+/**
+ * Fetch a file/folder and write to local file system.
+ *
+ * @async
+ * @param {Object} input
+ * @param {number} input.portalId
+ * @param {stirng} input.path
+ * @param {string} input.dest
+ * @param {object} input.options
+ * @returns {Promise}
+ */
+async function downloadFileOrFolder(input) {
+  const { portalId, path: _path } = input;
+  try {
+    const download = path.extname(_path) ? fetchFile : fetchFolder;
+    const node = await download(portalId, _path);
+    logger.log('Fetched "%s" from portal %d successfully', _path, portalId);
+    recursiveWriteFileMapperNode(input, node);
+  } catch (err) {
+    logger.error('Error fetching "%s": %o', _path, err);
   }
@@ -160,8 +176,3 @@ }
 module.exports = {
-  upload,
-  downloadModule,
-  downloadFile,
-  downloadFolder,
-  downloadAll,
-  writeFileMapping,
+  downloadFileOrFolder,
 };

lib/config.js

@@ -77,5 +77,6 @@ const yaml = require('js-yaml');
   }
+  const env = environment.toUpperCase() === 'QA' ? 'QA' : undefined;
   const nextPortalConfig = {
     ...portalConfig,
-    env: environment.toUpperCase(),
+    env,
     portalId,
@@ -82,0 +83,0 @@ authType,

lib/sync.js

 const path = require('path');
 const { ALLOWED_EXTENSIONS } = require('./constants');
 const { logger } = require('../logger');
-const { upload } = require('../fileMapper');
+const { upload } = require('../api/fileMapper');
 const { walk } = require('./walk');
@@ -6,0 +6,0 @@ const escapeRegExp = require('./escapeRegExp');

lib/watch.js

@@ -7,3 +7,3 @@ const path = require('path');
 const { sync } = require('./sync');
-const { upload } = require('../fileMapper');
+const { upload } = require('../api/fileMapper');
 const escapeRegExp = require('./escapeRegExp');
@@ -10,0 +10,0 @@ const { convertToUnixPath } = require('../path');

package.json

 {
   "name": "@hubspot/cms-lib",
-  "version": "0.0.13",
+  "version": "0.0.14",
   "description": "Library for working with the HubSpot CMS",
@@ -10,4 +10,5 @@ "license": "Apache-2.0",
     "findup-sync": "^3.0.0",
-    "fs-extra": "^7.0.1",
+    "fs-extra": "^8.1.0",
     "js-yaml": "^3.12.2",
+    "p-queue": "^6.0.2",
     "prettier": "1.16.4",
@@ -24,3 +25,3 @@ "request": "^2.87.0",
   },
-  "gitHead": "6efa49006ea4ac81e854cfc02de2577d653ac1e8"
+  "gitHead": "7f6633b4ef5f1ff56cf6b45edc26828e569b8fb2"
 }

path.js

@@ -23,2 +23,9 @@ const path = require('path');
 
+const getCwd = () => {
+  if (process.env.INIT_CWD) {
+    return process.env.INIT_CWD;
+  }
+  return process.cwd();
+};
+
 module.exports = {
@@ -28,2 +35,3 @@ convertToUnixPath,
   convertToLocalFileSystemPath,
+  getCwd,
 };

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Number of package files

17

increased by6.25%

Lines of code

811

increased by13.59%

Worsened metrics

Total package byte prevSize

22772

decreased by-13.79%

Dependency count

9

increased by12.5%

Number of low supply chain risk alerts

8

increased by33.33%

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Addedp-queue@^6.0.2

• + Addedeventemitter3@4.0.7(transitive)

• + Addedfs-extra@8.1.0(transitive)

• + Addedp-finally@1.0.0(transitive)

• + Addedp-queue@6.6.2(transitive)

• + Addedp-timeout@3.2.0(transitive)

• - Removedfs-extra@7.0.1(transitive)

• Updatedfs-extra@^8.1.0