@hubspot/cms-lib - npm Package Compare versions

Comparing version 0.0.23 to 0.0.24

__tests__/fileMapper.js

@@ -7,3 +7,4 @@ const path = require('path');
   recurseFolder,
-  fetchFromApi,
+  fetchFolderFromApi,
+  getTypeDataFromPath,
 } = require('../fileMapper');
@@ -139,18 +140,11 @@ const apiFileMapper = require('../api/fileMapper');
 
-  describe('fetchFromApi()', () => {
-    const portalId = 67890;
-    describe('fetch file (stream)', () => {
-      const input = {
-        portalId,
-        src: '1234/test.html',
-      };
-      it('should execute the fetchFileStream client per the request input', async () => {
-        const spy = jest.spyOn(apiFileMapper, 'fetchFileStream');
-        await fetchFromApi(input);
-        expect(spy).toHaveBeenCalled();
-      });
-      it('should return file flags per the request input', async () => {
-        const { isFile, isModule, isFolder, isRoot } = await fetchFromApi(
-          input
-        );
+  describe('getTypeDataFromPath()', () => {
+    it('should return file flags per the request input', () => {
+      filePaths.forEach(async p => {
+        const {
+          isFile,
+          isModule,
+          isFolder,
+          isRoot,
+        } = await getTypeDataFromPath(p);
         expect(isFile).toBe(true);
@@ -161,15 +155,50 @@ expect(isModule).toBe(false);
       });
-      it('should return the file FileMapperNode per the request input', async () => {
-        const { node } = await fetchFromApi(input);
-        expect(node).toMatchObject({
-          path: `/${input.src}`,
-          name: path.basename(input.src),
-          folder: false,
-          children: [],
-        });
-        expect(typeof node.source).toBe('string');
-        expect(node.source.length).toBeTruthy();
+    });
+    it('should return folder flags per the request input', () => {
+      folderPaths.forEach(async p => {
+        const {
+          isFile,
+          isModule,
+          isFolder,
+          isRoot,
+        } = await getTypeDataFromPath(p);
+        expect(isFile).toBe(false);
+        expect(isModule).toBe(false);
+        expect(isFolder).toBe(true);
+        expect(isRoot).toBe(false);
       });
     });
+    it('should return root folder flags per the request input', () => {
+      rootPaths.forEach(async p => {
+        const {
+          isFile,
+          isModule,
+          isFolder,
+          isRoot,
+        } = await getTypeDataFromPath(p);
+        expect(isFile).toBe(false);
+        expect(isModule).toBe(false);
+        expect(isFolder).toBe(true);
+        expect(isRoot).toBe(true);
+      });
+    });
+    it('should return module folder flags per the request input', () => {
+      modulePaths.forEach(async p => {
+        const {
+          isFile,
+          isModule,
+          isFolder,
+          isRoot,
+        } = await getTypeDataFromPath(p);
+        expect(isFile).toBe(false);
+        expect(isModule).toBe(true);
+        expect(isFolder).toBe(true);
+        expect(isRoot).toBe(false);
+      });
+    });
+  });
 
+  describe('fetchFolderFromApi()', () => {
+    const portalId = 67890;
+
     describe('fetch folder', () => {
@@ -180,18 +209,9 @@ const input = {
       };
-      it('should execute the fetchFolder client per the request input', async () => {
-        const spy = jest.spyOn(apiFileMapper, 'fetchFolder');
-        await fetchFromApi(input);
+      it('should execute the download client per the request input', async () => {
+        const spy = jest.spyOn(apiFileMapper, 'download');
+        await fetchFolderFromApi(input);
         expect(spy).toHaveBeenCalled();
       });
-      it('should return folder flags per the request input', async () => {
-        const { isFile, isModule, isFolder, isRoot } = await fetchFromApi(
-          input
-        );
-        expect(isFile).toBe(false);
-        expect(isModule).toBe(false);
-        expect(isFolder).toBe(true);
-        expect(isRoot).toBe(false);
-      });
       it('should return the folder FileMapperNode per the request input', async () => {
-        const { node } = await fetchFromApi(input);
+        const node = await fetchFolderFromApi(input);
         expect(node).toMatchObject({
@@ -213,18 +233,9 @@ path: `/${input.src}`,
       };
-      it('should execute the fetchModuleFolder client per the request input', async () => {
-        const spy = jest.spyOn(apiFileMapper, 'fetchModuleFolder');
-        await fetchFromApi(input);
+      it('should execute the download client per the request input', async () => {
+        const spy = jest.spyOn(apiFileMapper, 'download');
+        await fetchFolderFromApi(input);
         expect(spy).toHaveBeenCalled();
       });
-      it('should return module folder flags per the request input', async () => {
-        const { isFile, isModule, isFolder, isRoot } = await fetchFromApi(
-          input
-        );
-        expect(isFile).toBe(false);
-        expect(isModule).toBe(true);
-        expect(isFolder).toBe(true);
-        expect(isRoot).toBe(false);
-      });
       it('should return the module FileMapperNode per the request input', async () => {
-        const { node } = await fetchFromApi(input);
+        const node = await fetchFolderFromApi(input);
         expect(node).toMatchObject({
@@ -251,18 +262,9 @@ path: `/${input.src}`,
       };
-      it('should execute the fetchAll client per the request input', async () => {
-        const spy = jest.spyOn(apiFileMapper, 'fetchAll');
-        await fetchFromApi(input);
+      it('should execute the download client per the request input', async () => {
+        const spy = jest.spyOn(apiFileMapper, 'download');
+        await fetchFolderFromApi(input);
         expect(spy).toHaveBeenCalled();
       });
-      it('should return root flags per the request input', async () => {
-        const { isFile, isModule, isFolder, isRoot } = await fetchFromApi(
-          input
-        );
-        expect(isFile).toBe(false);
-        expect(isModule).toBe(false);
-        expect(isFolder).toBe(true);
-        expect(isRoot).toBe(true);
-      });
       it('should return the root folder FileMapperNode per the request input', async () => {
-        const { node } = await fetchFromApi(input);
+        const node = await fetchFolderFromApi(input);
         expect(node).toMatchObject({
@@ -269,0 +271,0 @@ path: '/',

api/__mocks__/fileMapper.js

@@ -129,18 +129,2 @@ const fileMapper = jest.genMockFromModule('../fileMapper');
 /**
- * GET /content/filemapper/v1/stream/1234/test.html
- * `hscms fetch 1234/test.html`
- * Note: Represents node value processed from stream response.
- */
-const fileStreamRequestData = {
-  source:
-    '<!--\n    templateType: page\n    isAvailableForNewContent: false\n-->\n<div>\n\thello new stuff\n\thello new stuff\n\thello new stuff\n</div>\n',
-  path: '/1234/test.html',
-  createdAt: 0,
-  updatedAt: 1565214001268,
-  name: 'test.html',
-  folder: false,
-  children: [],
-};
-
-/**
  * GET /content/filemapper/v1/download/all
@@ -165,10 +149,15 @@ * `hscms fetch /`
 
-fileMapper.fetchFolder = jest.fn(async () => folderRequestData);
+fileMapper.download = jest.fn(async (portalId, filepath) => {
+  if (filepath === '@root') {
+    return allRequestData;
+  }
+  if (moduleFileRequestData.path.includes(filepath)) {
+    return moduleFileRequestData;
+  }
+  if (folderRequestData.path.includes(filepath)) {
+    return folderRequestData;
+  }
+  throw new Error({ statusCode: 404 });
+});
 
-fileMapper.fetchModuleFolder = jest.fn(async () => moduleFileRequestData);
-
-fileMapper.fetchFileStream = jest.fn(async () => fileStreamRequestData);
-
-fileMapper.fetchAll = jest.fn(async () => allRequestData);
-
 module.exports = fileMapper;

api/__tests__/fileMapper.js

 const fileStreamResponse = require('./fixtures/fileStreamResponse');
-const { createFileMapperNodeStreamTransform } = require('../fileMapper');
+const { createFileMapperNodeFromStreamResponse } = require('../fileMapper');
 
 describe('cms-lib/api/fileMapper', () => {
-  describe('createFileMapperNodeStreamTransform()', () => {
+  describe('createFileMapperNodeFromStreamResponse()', () => {
     const src = '1234/test.html';
-    it('should return request#tranform function', () => {
-      const transform = createFileMapperNodeStreamTransform(src);
-      expect(typeof transform).toBe('function');
-    });
     it('should return request#tranform to create a FileMapperNode from the octet-stream response', () => {
-      const node = createFileMapperNodeStreamTransform(src)(
-        fileStreamResponse.body,
+      const node = createFileMapperNodeFromStreamResponse(
+        src,
         fileStreamResponse
       );
       expect(node).toEqual({
-        source:
-          '<!--\n    templateType: page\n    isAvailableForNewContent: false\n-->\n<div>\n\thello new stuff\n\thello new stuff\n\thello new stuff\n</div>\n',
+        source: null,
         path: '/1234/test.html',
@@ -20,0 +15,0 @@ createdAt: 0,

api/fileMapper.js

@@ -14,6 +14,3 @@ const fs = require('fs-extra');
  */
-const createFileMapperNodeStreamTransform = filePath => (source, response) => {
-  const { parameters } = contentDisposition.parse(
-    response.headers['content-disposition']
-  );
+function createFileMapperNodeFromStreamResponse(filePath, response) {
   if (filePath[0] !== '/') {
@@ -25,24 +22,24 @@ filePath = `/${filePath}`;
   }
-  if (typeof source !== 'string') {
-    // JSON responses will have already been parsed.
-    // TODO: Directly write the stream so we don't lose fidelity like whitespace.
-    try {
-      source = JSON.stringify(source, null, 2);
-    } catch (err) {
-      throw new TypeError(
-        `Unable to parse "${filePath}" source: ${typeof source}`
-      );
-    }
-  }
   const node = {
-    source,
+    source: null,
     path: filePath,
+    name: path.basename(filePath),
+    folder: false,
+    children: [],
+    createdAt: 0,
+    updatedAt: 0,
+  };
+  if (!(response.headers && response.headers['content-disposition'])) {
+    return node;
+  }
+  const { parameters } = contentDisposition.parse(
+    response.headers['content-disposition']
+  );
+  return {
+    ...node,
     name: parameters.filename,
     createdAt: parseInt(parameters['creation-date'], 10) || 0,
     updatedAt: parseInt(parameters['modification-date'], 10) || 0,
-    folder: false,
-    children: [],
   };
-  return node;
-};
+}
 
@@ -85,18 +82,2 @@ /**
 /**
- * Fetch a *.module folder.
- *
- * @async
- * @param {number} portalId
- * @param {string} filePath
- * @param {object} options
- * @returns {Promise<FileMapperNode>}
- */
-async function fetchModuleFolder(portalId, filePath, options = {}) {
-  return http.get(portalId, {
-    uri: `${FILE_MAPPER_API_PATH}/download/${filePath}`,
-    ...options,
-  });
-}
-
-/**
  * Fetch a file by file path.
@@ -107,34 +88,30 @@ *
  * @param {string} filePath
+ * @param {stream.Writable} destination
  * @param {object} options
  * @returns {Promise<FileMapperNode>}
  */
-async function fetchFileStream(portalId, filePath, options = {}) {
-  // Note: Use `request` instead of `request-promise` to use Node streams
-  // https://github.com/request/request-promise#api-in-detail
-  // https://github.com/request/request#streaming
-  const { headers, ...opts } = options;
-  return http.get(portalId, {
-    uri: `${FILE_MAPPER_API_PATH}/stream/${filePath}`,
-    ...opts,
-    headers: {
-      ...headers,
-      'content-type': 'application/octet-stream',
-      accept: 'application/octet-stream',
+async function fetchFileStream(portalId, filePath, destination, options = {}) {
+  const response = await http.getOctetStream(
+    portalId,
+    {
+      uri: `${FILE_MAPPER_API_PATH}/stream/${filePath}`,
+      ...options,
     },
-    transform: createFileMapperNodeStreamTransform(filePath),
-  });
+    destination
+  );
+  return createFileMapperNodeFromStreamResponse(filePath, response);
 }
 
 /**
- * Fetch a folder by folder path.
+ * Fetch a folder or file node by path.
  *
  * @async
  * @param {number} portalId
- * @param {string} folderPath
+ * @param {string} filepath
  * @param {object} options
  * @returns {Promise<FileMapperNode>}
  */
-async function fetchFolder(portalId, folderPath, options = {}) {
+async function download(portalId, filepath, options = {}) {
   return http.get(portalId, {
-    uri: `${FILE_MAPPER_API_PATH}/download/folder/${folderPath}`,
+    uri: `${FILE_MAPPER_API_PATH}/download/${filepath}`,
     ...options,
@@ -145,17 +122,2 @@ });
 /**
- * Fetch entire tree for portal.
- *
- * @async
- * @param {number} portalId
- * @param {object} options
- * @returns {Promise<FileMapperNode>}
- */
-async function fetchAll(portalId, options = {}) {
-  return http.get(portalId, {
-    uri: `${FILE_MAPPER_API_PATH}/download/all`,
-    ...options,
-  });
-}
-
-/**
  * Delete file by path
@@ -228,12 +190,10 @@ *
 module.exports = {
-  createFileMapperNodeStreamTransform,
   deleteFile,
   deleteFolder,
-  fetchAll,
+  download,
   fetchFileStream,
-  fetchFolder,
   fetchModule,
-  fetchModuleFolder,
   trackUsage,
   upload,
+  createFileMapperNodeFromStreamResponse,
 };

errorHandlers.js

@@ -5,5 +5,4 @@ const { HubSpotAuthError } = require('@hubspot/api-auth-lib/Errors');
 const isApiStatusCodeError = err =>
-  err.name === 'StatusCodeError' &&
-  err.statusCode >= 100 &&
-  err.statusCode < 600;
+  err.name === 'StatusCodeError' ||
+  (err.statusCode >= 100 && err.statusCode < 600);
 const isApiUploadValidationError = err =>
@@ -10,0 +9,0 @@ !!(

fileMapper.js

 const fs = require('fs-extra');
 const path = require('path');
 const { default: PQueue } = require('p-queue');
-const prettier = require('prettier');
 const {
@@ -10,38 +9,14 @@ ApiErrorContext,
   logFileSystemErrorInstance,
-  logErrorInstance,
 } = require('./errorHandlers');
 const { logger } = require('./logger');
-const { getCwd, convertToLocalFileSystemPath } = require('./path');
 const {
-  fetchAll,
-  fetchFileStream,
-  fetchFolder,
-  fetchModuleFolder,
-} = require('./api/fileMapper');
+  getAllowedExtensions,
+  getCwd,
+  convertToLocalFileSystemPath,
+} = require('./path');
+const { fetchFileStream, download } = require('./api/fileMapper');
 const { Mode } = require('./lib/constants');
 
-const BUILTINS_NAMESPACE = '@hubspot';
-
 const MODULE_EXTENSION = '.module';
 
-const META_KEYS_WHITELIST = new Set([
-  'content_tags',
-  'css_assets',
-  'default',
-  'editable_contexts',
-  'external_js',
-  'extra_classes',
-  'global',
-  'help_text',
-  'host_template_types',
-  'icon',
-  'is_available_for_new_content',
-  'js_assets',
-  'label',
-  'master_language',
-  'other_assets',
-  'smart_type',
-  'tags',
-]);
-
 const queue = new PQueue({
@@ -53,3 +28,6 @@ concurrency: 100,
   if (typeof filepath !== 'string') return '';
-  return path.extname(filepath).trim();
+  return path
+    .extname(filepath)
+    .trim()
+    .toLowerCase();
 }
@@ -89,2 +67,15 @@
 /**
+ * @private
+ * @param {number} portalId
+ * @param {string} filepath
+ * @returns {boolean}
+ */
+function isAllowedExtension(portalId, filepath) {
+  let ext = getExt(filepath);
+  if (ext[0] !== '.') return false;
+  ext = ext.slice(1);
+  return getAllowedExtensions(portalId).has(ext);
+}
+
+/**
  * Determines API `buffer` param based on mode.
@@ -154,9 +145,15 @@ *
  * @private
- * @param {FileMapperNode} node
- * @returns {FileMapperNodeMeta}
+ * @param {string} src
+ * @returns {object<boolean, boolean, boolean, boolean}
  */
-function getFileMapperNodeMeta(node) {
+function getTypeDataFromPath(src) {
+  const isModule = isPathToModule(src);
+  const isFile = !isModule && isPathToFile(src);
+  const isRoot = !isModule && !isFile && isPathToRoot(src);
+  const isFolder = !isFile;
   return {
-    isModule: !!~node.path.indexOf('.module'),
-    isBuiltin: node.path.indexOf(BUILTINS_NAMESPACE) === 0,
+    isModule,
+    isFile,
+    isRoot,
+    isFolder,
   };
@@ -166,47 +163,2 @@ }
 /**
- * Clean the meta.json file from builtin custom modules.
- *
- * @private
- * @param {string} source
- * @returns {string}
- */
-function cleanMetaJson(source) {
-  const meta = JSON.parse(source);
-  const out = {};
-  META_KEYS_WHITELIST.forEach(key => {
-    if (meta[key]) {
-      out[key] = meta[key];
-    }
-  });
-  return JSON.stringify(out);
-}
-
-/**
- * Gets the `source` field from a file node and process if needed.
- *
- * @private
- * @param {FileMapperNode} node
- * @returns {string}
- */
-function getFileSource(node) {
-  let { source } = node;
-  if (node.folder || !source) {
-    return source;
-  }
-  const { isBuiltin, isModule } = getFileMapperNodeMeta(node);
-  if (isBuiltin && isModule) {
-    // Clean and format builtin module sources.
-    if (node.name === 'meta.json') {
-      source = cleanMetaJson(source);
-    }
-    if (path.extname(node.name) === '.json') {
-      source = prettier.format(source, {
-        parser: 'json',
-      });
-    }
-  }
-  return source;
-}
-
-/**
  * @typedef {Object} FileMapperInputArguments
@@ -274,2 +226,83 @@ * @property {number} portalId
 /**
+ * @private
+ * @async
+ * @param {FileMapperInputArguments} input
+ * @param {string} filepath
+ * @returns {Promise<boolean}
+ */
+async function skipExisting(input, filepath) {
+  if (input.options.overwrite) {
+    return false;
+  }
+  if (await fs.pathExists(filepath)) {
+    logger.log('Skipped existing "%s"', filepath);
+    return true;
+  }
+  return false;
+}
+
+/**
+ * @private
+ * @async
+ * @param {FileMapperInputArguments} input
+ * @param {string} srcPath - Server path to download.
+ * @param {string} filepath - Local path to write to.
+ */
+async function fetchAndWriteFileStream(input, srcPath, filepath) {
+  if (await skipExisting(input, filepath)) {
+    return null;
+  }
+  if (!isAllowedExtension(input.portalId, srcPath)) {
+    const message = `Invalid file type requested: "${srcPath}"`;
+    logger.error(message);
+    throw new Error(message);
+  }
+  const { portalId } = input;
+  const logFsError = err => {
+    logFileSystemErrorInstance(
+      err,
+      new FileSystemErrorContext({
+        filepath,
+        portalId,
+        write: true,
+      })
+    );
+  };
+  let writeStream;
+  try {
+    await fs.ensureFile(filepath);
+    writeStream = fs.createWriteStream(filepath, { encoding: 'binary' });
+  } catch (err) {
+    logFsError(err);
+    throw err;
+  }
+  let node;
+  try {
+    node = await fetchFileStream(portalId, srcPath, writeStream, {
+      qs: getFileMapperApiQueryFromMode(input.mode),
+    });
+  } catch (err) {
+    logApiErrorInstance(
+      err,
+      new ApiErrorContext({
+        portalId,
+        request: srcPath,
+      })
+    );
+    throw err;
+  }
+  return new Promise(async (resolve, reject) => {
+    writeStream.on('error', err => {
+      logFsError(err);
+      reject(err);
+    });
+    writeStream.on('close', async () => {
+      await writeUtimes(input, filepath, node);
+      logger.log('Wrote file "%s"', filepath);
+      resolve(node);
+    });
+  });
+}
+
+/**
  * Writes an individual file or folder (not recursive).  If file source is missing, the
@@ -286,32 +319,17 @@ * file is fetched.
 async function writeFileMapperNode(input, node, filepath) {
-  const { portalId, options } = input;
+  filepath = convertToLocalFileSystemPath(path.resolve(filepath));
+  if (await skipExisting(input, filepath)) {
+    return;
+  }
+  if (!node.folder) {
+    try {
+      await fetchAndWriteFileStream(input, node.path, filepath);
+    } catch (err) {
+      // Logging handled by handler
+    }
+    return;
+  }
   try {
-    filepath = convertToLocalFileSystemPath(path.resolve(filepath));
-    if (!options.overwrite && (await fs.pathExists(filepath))) {
-      logger.log('Skipped existing "%s"', filepath);
-      return;
-    }
-    if (node.folder) {
-      await fs.ensureDir(filepath);
-      logger.log('Wrote folder "%s"', filepath);
-    } else {
-      if (node.source != null) {
-        // File fetches will include `source`
-        await fs.ensureFile(filepath);
-      } else {
-        // Files from folder fetches will not include `source`
-        [, node] = await Promise.all([
-          fs.ensureFile(filepath),
-          fetchFileStream(portalId, node.path, {
-            qs: getFileMapperApiQueryFromMode(input.mode),
-          }),
-        ]);
-      }
-      if (typeof node.source === 'string') {
-        const source = getFileSource(node);
-        await fs.writeFile(filepath, source);
-      }
-      logger.log('Wrote file "%s"', filepath);
-    }
-    await writeUtimes(input, filepath, node);
+    await fs.ensureDir(filepath);
+    logger.log('Wrote folder "%s"', filepath);
   } catch (err) {
@@ -333,21 +351,55 @@ logFileSystemErrorInstance(
  * @param {FileMapperInputArguments} input
- * @returns {Promise<object<boolean, boolean, boolean, boolean, FileMapperNode>>}
+ * @returns {Promise}
  */
-async function fetchFromApi(input) {
+async function downloadFile(input) {
+  try {
+    const { src } = input;
+    const { isFile } = getTypeDataFromPath(src);
+    if (!isFile) {
+      throw new Error(`Invalid request for file: "${src}"`);
+    }
+    const dest = path.resolve(input.dest);
+    const cwd = getCwd();
+    let filepath;
+    if (dest === cwd) {
+      // Dest: CWD
+      filepath = path.resolve(cwd, path.basename(src));
+    } else if (isPathToFile(dest)) {
+      // Dest: file path
+      filepath = path.isAbsolute(dest) ? dest : path.resolve(cwd, dest);
+    } else {
+      // Dest: folder path
+      const name = path.basename(src);
+      filepath = path.isAbsolute(dest)
+        ? path.resolve(dest, name)
+        : path.resolve(cwd, dest, name);
+    }
+    const localFsPath = convertToLocalFileSystemPath(filepath);
+    await fetchAndWriteFileStream(input, input.src, localFsPath);
+    await queue.onIdle();
+    logger.log('Completed fetch of file "%s" to "%s"', input.src, localFsPath);
+  } catch (err) {
+    logger.error('Failed fetch of file "%s" to "%s"', input.src, input.dest);
+  }
+}
+
+/**
+ * @private
+ * @async
+ * @param {FileMapperInputArguments} input
+ * @returns {Promise<FileMapperNode}
+ */
+async function fetchFolderFromApi(input) {
   const { portalId, src, mode } = input;
-  const isModule = isPathToModule(src);
-  const isFile = !isModule && isPathToFile(src);
-  const isRoot = !isModule && !isFile && isPathToRoot(src);
-  const isFolder = !isFile;
-  let node;
+  const { isRoot, isFolder } = getTypeDataFromPath(src);
+  if (!isFolder) {
+    throw new Error(`Invalid request for folder: "${src}"`);
+  }
   try {
-    const fetch =
-      (isModule && fetchModuleFolder) ||
-      (isFile && fetchFileStream) ||
-      (isRoot && fetchAll) ||
-      fetchFolder;
-    node = await fetch(portalId, src, {
+    const srcPath = isRoot ? '@root' : src;
+    const node = await download(portalId, srcPath, {
       qs: getFileMapperApiQueryFromMode(mode),
     });
     logger.log('Fetched "%s" from portal %d successfully', src, portalId);
+    return node;
   } catch (err) {
@@ -362,3 +414,3 @@ logApiErrorInstance(
   }
-  return { isFile, isFolder, isModule, isRoot, node };
+  return null;
 }
@@ -370,32 +422,12 @@
  * @param {FileMapperInputArguments} input
- * @param {FileMapperNode}           node
  * @returns {Promise}
  */
-async function writeFileDownload(input, node) {
-  let destPath;
+async function downloadFolder(input) {
   try {
+    const node = await fetchFolderFromApi(input);
+    if (!node) {
+      return;
+    }
     const dest = path.resolve(input.dest);
-    destPath = isPathToFile(dest)
-      ? dest
-      : convertToLocalFileSystemPath(path.resolve(dest, node.name));
-    await queue.add(() => writeFileMapperNode(input, node, destPath));
-  } catch (err) {
-    logErrorInstance(err, {
-      portalId: input.portalId,
-    });
-  }
-}
-
-/**
- * @private
- * @async
- * @param {FileMapperInputArguments} input
- * @param {FileMapperNode}           node
- * @returns {Promise}
- */
-async function writeFolderDownload(input, node) {
-  let rootPath;
-  try {
-    const dest = path.resolve(input.dest);
-    rootPath =
+    const rootPath =
       dest === getCwd()
@@ -411,6 +443,6 @@ ? convertToLocalFileSystemPath(path.resolve(dest, node.name))
     );
+    await queue.onIdle();
+    logger.log('Completed fetch of folder "%s" to "%s"', input.src, input.dest);
   } catch (err) {
-    logErrorInstance(err, {
-      portalId: input.portalId,
-    });
+    logger.error('Failed fetch of folder "%s" to "%s"', input.src, input.dest);
   }
@@ -428,17 +460,13 @@ }
   try {
-    const { isFile, node } = await fetchFromApi(input);
-    if (node) {
-      if (isFile) {
-        await writeFileDownload(input, node);
-      } else {
-        await writeFolderDownload(input, node);
-      }
-      await queue.onIdle();
-      logger.log('Completed fetch of "%s" to "%s"', input.src, input.dest);
+    if (!(input && input.src)) {
+      return;
     }
+    const { isFile } = getTypeDataFromPath(input.src);
+    if (isFile) {
+      await downloadFile(input);
+    } else {
+      await downloadFolder(input);
+    }
   } catch (err) {
-    logger.error('Error fetching "%s" to "%s"', input.src, input.dest);
-    logErrorInstance(err, {
-      portalId: input.portalId,
-    });
+    // Specific handlers provide logging.
   }
@@ -454,3 +482,4 @@ }
   getFileMapperApiQueryFromMode,
-  fetchFromApi,
+  fetchFolderFromApi,
+  getTypeDataFromPath,
 };

http.js

@@ -1,2 +0,3 @@
-const request = require('request-promise-native');
+const request = require('request');
+const requestPN = require('request-promise-native');
 const { getPortalConfig, getConfig } = require('./lib/config');
@@ -83,21 +84,64 @@ const { getOauthManager } = require('./oauth');
   const requestOptions = addQueryParams(rest, query);
-  return request.get(await withAuth(portalId, requestOptions));
+  return requestPN.get(await withAuth(portalId, requestOptions));
 };
 
 const postRequest = async (portalId, options) => {
-  return request.post(await withAuth(portalId, options));
+  return requestPN.post(await withAuth(portalId, options));
 };
 
 const putRequest = async (portalId, options) => {
-  return request.put(await withAuth(portalId, options));
+  return requestPN.put(await withAuth(portalId, options));
 };
 
 const deleteRequest = async (portalId, options) => {
-  return request.del(await withAuth(portalId, options));
+  return requestPN.del(await withAuth(portalId, options));
 };
 
+const createGetRequestStream = ({ contentType }) => async (
+  portalId,
+  options,
+  destination
+) => {
+  const { query, ...rest } = options;
+  const requestOptions = addQueryParams(rest, query);
+  // Using `request` instead of `request-promise` per the docs so
+  // the response can be piped.
+  // https://github.com/request/request-promise#api-in-detail
+  return new Promise(async (resolve, reject) => {
+    try {
+      const { headers, ...opts } = await withAuth(portalId, requestOptions);
+      const req = request.get({
+        ...opts,
+        headers: {
+          ...headers,
+          'content-type': contentType,
+          accept: contentType,
+        },
+        json: false,
+      });
+      let response;
+      req
+        .on('error', reject)
+        .on('response', r => {
+          if (r.statusCode >= 200 && r.statusCode < 300) {
+            response = r;
+          } else {
+            reject(r);
+          }
+        })
+        .on('end', () => resolve(response))
+        .pipe(destination);
+    } catch (err) {
+      reject(err);
+    }
+  });
+};
+
 module.exports = {
   getRequestOptions,
-  request,
+  request: requestPN,
   get: getRequest,
+  getOctetStream: createGetRequestStream({
+    contentType: 'application/octet-stream',
+  }),
   post: postRequest,
@@ -104,0 +148,0 @@ put: putRequest,

package.json

 {
   "name": "@hubspot/cms-lib",
-  "version": "0.0.23",
+  "version": "0.0.24",
   "description": "Library for working with the HubSpot CMS",
@@ -26,3 +26,3 @@ "license": "Apache-2.0",
   },
-  "gitHead": "e330f0d5d1dd991aa26cdffaf2aab4877015aaac"
+  "gitHead": "475e5339b6af5b748d7966392cd431088558123b"
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Network access

Supply chain risk

This module accesses the network.

Found 1 instance in 1 package

Improved metrics

Lines of code

2694

increased by0.86%

Number of medium supply chain risk alerts

0

decreased by-100%

Worsened metrics

Total package byte prevSize

86152

decreased by-0.46%

No dependency changes