@sanity/export - npm Package Compare versions

Comparing version 0.126.0 to 0.126.2

lib/AssetHandler.js

@@ -8,2 +8,3 @@ 'use strict';
 const path = require('path');
+const fse = require('fs-extra');
 const miss = require('mississippi');
@@ -17,25 +18,39 @@ const PQueue = require('p-queue');
 const ACTION_REWRITE = 'rewrite';
-const precompressedExts = ['.zip', '.gz', '.rar', '.png', '.jpeg', '.jpg', '.gif'];
 
 class AssetHandler {
   constructor(options) {
-    this.rewriteAssets = miss.through.obj((doc, enc, callback) => {
-      if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
-        const type = doc._type === 'sanity.imageAsset' ? 'image' : 'file';
-        const filePath = `${type}s/${generateFilename(doc._id)}`;
-        this.queueAssetDownload(doc, filePath);
-        callback();
-        return;
-      }
+    var _this = this;
 
-      callback(null, this.findAndModify(doc, ACTION_REWRITE));
-    });
-    this.stripAssets = miss.through.obj((doc, enc, callback) => {
-      if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
-        callback();
-        return;
-      }
+    this.rewriteAssets = miss.through.obj((() => {
+      var _ref = _asyncToGenerator(function* (doc, enc, callback) {
+        if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
+          const type = doc._type === 'sanity.imageAsset' ? 'image' : 'file';
+          const filePath = `${type}s/${generateFilename(doc._id)}`;
+          _this.assetsSeen.set(doc._id, type);
+          _this.queueAssetDownload(doc, filePath);
+          callback();
+          return;
+        }
 
-      callback(null, this.findAndModify(doc, ACTION_REMOVE));
-    });
+        callback(null, (yield _this.findAndModify(doc, ACTION_REWRITE)));
+      });
+
+      return function (_x, _x2, _x3) {
+        return _ref.apply(this, arguments);
+      };
+    })());
+    this.stripAssets = miss.through.obj((() => {
+      var _ref2 = _asyncToGenerator(function* (doc, enc, callback) {
+        if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
+          callback();
+          return;
+        }
+
+        callback(null, (yield _this.findAndModify(doc, ACTION_REMOVE)));
+      });
+
+      return function (_x4, _x5, _x6) {
+        return _ref2.apply(this, arguments);
+      };
+    })());
     this.skipAssets = miss.through.obj((doc, enc, callback) => {
@@ -52,41 +67,73 @@ const isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type);
 
-    this.findAndModify = (item, action) => {
-      if (Array.isArray(item)) {
-        return item.map(child => this.findAndModify(child, action)).filter(Boolean);
-      }
+    this.findAndModify = (() => {
+      var _ref3 = _asyncToGenerator(function* (item, action) {
+        if (Array.isArray(item)) {
+          const children = yield Promise.all(item.map(function (child) {
+            return _this.findAndModify(child, action);
+          }));
+          return children.filter(Boolean);
+        }
 
-      if (!item || typeof item !== 'object') {
-        return item;
-      }
+        if (!item || typeof item !== 'object') {
+          return item;
+        }
 
-      const assetType = getAssetType(item);
-      if (assetType) {
-        if (action === ACTION_REMOVE) {
+        const isAsset = isAssetField(item);
+        if (isAsset && action === ACTION_REMOVE) {
           return undefined;
         }
 
-        if (action === ACTION_REWRITE) {
-          const filePath = `${assetType}s/${generateFilename(item.asset._ref)}`;
-          return {
-            _sanityAsset: `${assetType}@file://./${filePath}`
-          };
+        if (isAsset && action === ACTION_REWRITE) {
+          const assetId = item.asset._ref;
+          if (isModernAsset(assetId)) {
+            const assetType = getAssetType(item);
+            const filePath = `${assetType}s/${generateFilename(assetId)}`;
+            return { _sanityAsset: `${assetType}@file://./${filePath}` };
+          }
+
+          // Legacy asset
+          const type = _this.assetsSeen.get(assetId) || (yield _this.lookupAssetType(assetId));
+          const filePath = `${type}s/${generateFilename(assetId)}`;
+          return { _sanityAsset: `${type}@file://./${filePath}` };
         }
-      }
 
-      return Object.keys(item).reduce((acc, key) => {
-        const value = item[key];
-        acc[key] = this.findAndModify(value, action);
+        const newItem = {};
+        const keys = Object.keys(item);
+        for (let i = 0; i < keys.length; i++) {
+          const key = keys[i];
+          const value = item[key];
 
-        if (typeof acc[key] === 'undefined') {
-          delete acc[key];
+          // eslint-disable-next-line no-await-in-loop
+          newItem[key] = yield _this.findAndModify(value, action);
+
+          if (typeof newItem[key] === 'undefined') {
+            delete newItem[key];
+          }
         }
 
-        return acc;
-      }, {});
-    };
+        return newItem;
+      });
 
+      return function (_x7, _x8) {
+        return _ref3.apply(this, arguments);
+      };
+    })();
+
+    this.lookupAssetType = (() => {
+      var _ref4 = _asyncToGenerator(function* (assetId) {
+        const docType = yield _this.client.fetch('*[_id == $id][0]._type', { id: assetId });
+        return docType === 'sanity.imageAsset' ? 'image' : 'file';
+      });
+
+      return function (_x9) {
+        return _ref4.apply(this, arguments);
+      };
+    })();
+
     this.client = options.client;
-    this.archive = options.archive;
-    this.archivePrefix = options.prefix;
+    this.tmpDir = options.tmpDir;
+    this.assetDirsCreated = false;
 
+    this.assetsSeen = new Map();
+    this.filesWritten = 0;
     this.queueSize = 0;
@@ -100,2 +147,3 @@ this.queue = options.queue || new PQueue({ concurrency: 3 });
   clear() {
+    this.assetsSeen.clear();
     this.queue.clear();
@@ -125,2 +173,7 @@ this.queueSize = 0;
   queueAssetDownload(assetDoc, dstPath) {
+    if (!assetDoc.url) {
+      debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id);
+      return;
+    }
+
     debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath);
@@ -132,3 +185,3 @@ this.queueSize++;
   downloadAsset(url, dstPath) {
-    var _this = this;
+    var _this2 = this;
 
@@ -138,22 +191,31 @@ return _asyncToGenerator(function* () {
       const stream = yield requestStream({ url, headers });
-      const store = precompressedExts.includes(path.extname(dstPath));
 
       if (stream.statusCode !== 200) {
-        _this.archive.abort();
-        _this.queue.clear();
-        _this.reject(new Error(`Referenced asset URL "${url}" returned HTTP ${stream.statusCode}`));
+        _this2.queue.clear();
+        _this2.reject(new Error(`Referenced asset URL "${url}" returned HTTP ${stream.statusCode}`));
         return;
       }
 
-      debug('Asset stream ready, appending to archive at %s', dstPath);
-      _this.archive.append(stream, {
-        name: path.basename(dstPath),
-        prefix: [_this.archivePrefix, path.dirname(dstPath)].join('/'),
-        store
-      });
+      if (!_this2.assetDirsCreated) {
+        /* eslint-disable no-sync */
+        fse.ensureDirSync(path.join(_this2.tmpDir, 'files'));
+        fse.ensureDirSync(path.join(_this2.tmpDir, 'images'));
+        /* eslint-enable no-sync */
+        _this2.assetDirsCreated = true;
+      }
+
+      debug('Asset stream ready, writing to filesystem at %s', dstPath);
+      yield writeStream(path.join(_this2.tmpDir, dstPath), stream);
+
+      _this2.filesWritten++;
     })();
   }
 
+  // eslint-disable-next-line complexity
 }
 
+function isAssetField(item) {
+  return item.asset && item.asset._ref;
+}
+
 function getAssetType(item) {
@@ -164,6 +226,6 @@ if (!item.asset || typeof item.asset._ref !== 'string') {
 
-  var _ref = item.asset._ref.match(/^(image|file)-/) || [],
-      _ref2 = _slicedToArray(_ref, 2);
+  var _ref5 = item.asset._ref.match(/^(image|file)-/) || [],
+      _ref6 = _slicedToArray(_ref5, 2);
 
-  const type = _ref2[1];
+  const type = _ref6[1];
 
@@ -173,17 +235,24 @@ return type || null;
 
+function isModernAsset(assetId) {
+  return (/^(image|file)/.test(assetId)
+  );
+}
+
 function generateFilename(assetId) {
-  var _assetId$match = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/),
-      _assetId$match2 = _slicedToArray(_assetId$match, 4);
+  var _ref7 = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [],
+      _ref8 = _slicedToArray(_ref7, 4);
 
-  const asset = _assetId$match2[2],
-        ext = _assetId$match2[3];
+  const asset = _ref8[2],
+        ext = _ref8[3];
 
   const extension = (ext || 'bin').replace(/^-/, '');
-  return `${asset}.${extension}`;
+  return asset ? `${asset}.${extension}` : `${assetId}.bin`;
 }
 
-function lookupAssetUrl(client, assetId) {
-  return client.fetch('*[_id == $id][0].url', { id: assetId });
+function writeStream(filePath, stream) {
+  return new Promise((resolve, reject) => miss.pipe(stream, fse.createWriteStream(filePath), err => {
+    return err ? reject(err) : resolve();
+  }));
 }
 
 module.exports = AssetHandler;

lib/export.js

@@ -5,4 +5,6 @@ 'use strict';
 
-const fs = require('fs');
+const os = require('os');
+const path = require('path');
 const zlib = require('zlib');
+const fse = require('fs-extra');
 const miss = require('mississippi');
@@ -34,6 +36,12 @@ const split = require('split2');
   const prefix = `${opts.dataset}-export-${slugDate}`;
-  const assetHandler = new AssetHandler({ client: options.client, archive, prefix });
+  const tmpDir = path.join(os.tmpdir(), prefix);
+  const assetHandler = new AssetHandler({
+    client: options.client,
+    tmpDir,
+    prefix
+  });
 
+  debug('Outputting assets (temporarily) to %s', tmpDir);
   debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath);
-  const outputStream = options.outputPath === '-' ? process.stdout : fs.createWriteStream(options.outputPath);
+  const outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);
 
@@ -47,2 +55,22 @@ let assetStreamHandler = assetHandler.noop;
     var _ref = _asyncToGenerator(function* (resolve, reject) {
+      let onComplete = (() => {
+        var _ref3 = _asyncToGenerator(function* (err) {
+          onProgress({ step: 'Clearing temporary files...' });
+          yield fse.remove(tmpDir);
+
+          if (!err) {
+            resolve();
+            return;
+          }
+
+          debug('Error during streaming: %s', err.stack);
+          assetHandler.clear();
+          reject(err);
+        });
+
+        return function onComplete(_x4) {
+          return _ref3.apply(this, arguments);
+        };
+      })();
+
       miss.finished(archive, function (archiveErr) {
@@ -56,3 +84,2 @@ if (archiveErr) {
         debug('Archive finished!');
-        resolve();
       });
@@ -62,4 +89,25 @@
       onProgress({ step: 'Exporting documents...' });
+
+      let documentCount = 0;
+      let lastReported = Date.now();
+      const reportDocumentCount = function reportDocumentCount(chunk, enc, cb) {
+        ++documentCount;
+
+        const now = Date.now();
+        if (now - lastReported > 50) {
+          onProgress({
+            step: 'Exporting documents...',
+            current: documentCount,
+            total: '?',
+            update: true
+          });
+
+          lastReported = now;
+        }
+
+        cb(null, chunk);
+      };
+
       const inputStream = yield getDocumentsStream(options.client, options.dataset);
-      const jsonStream = miss.pipeline(inputStream, split(JSON.parse), rejectOnApiError, filterSystemDocuments, assetStreamHandler, filterDocumentTypes(options.types), options.drafts ? miss.through.obj() : filterDrafts, stringifyStream);
+      const jsonStream = miss.pipeline(inputStream, split(JSON.parse), rejectOnApiError, filterSystemDocuments, assetStreamHandler, filterDocumentTypes(options.types), options.drafts ? miss.through.obj() : filterDrafts, stringifyStream, miss.through(reportDocumentCount));
 
@@ -72,14 +120,28 @@ miss.finished(jsonStream, (() => {
 
+          onProgress({
+            step: 'Exporting documents...',
+            current: documentCount,
+            total: documentCount,
+            update: true
+          });
+
           if (!options.raw && options.assets) {
             onProgress({ step: 'Downloading assets...' });
+          }
 
-            archive.on('progress', function ({ entries }) {
-              onProgress({
-                step: 'Downloading assets...',
-                current: entries.processed,
-                total: Math.max(assetHandler.queueSize, entries.processed),
-                update: true
-              });
+          let prevCompleted = 0;
+          const progressInterval = setInterval(function () {
+            const completed = assetHandler.queueSize - assetHandler.queue.size;
+            if (prevCompleted === completed) {
+              return;
+            }
+
+            prevCompleted = completed;
+            onProgress({
+              step: 'Downloading assets...',
+              current: completed,
+              total: assetHandler.queueSize,
+              update: true
             });
-          }
+          }, 500);
 
@@ -89,3 +151,5 @@ debug('Waiting for asset handler to complete downloads');
             yield assetHandler.finish();
+            clearInterval(progressInterval);
           } catch (assetErr) {
+            clearInterval(progressInterval);
             reject(assetErr);
@@ -95,3 +159,8 @@ return;
 
+          // Add all downloaded assets to archive
+          archive.directory(path.join(tmpDir, 'files'), `${prefix}/files`, { store: true });
+          archive.directory(path.join(tmpDir, 'images'), `${prefix}/images`, { store: true });
+
           debug('Finalizing archive, flushing streams');
+          onProgress({ step: 'Adding assets to archive...' });
           archive.finalize();
@@ -105,14 +174,8 @@ });
 
+      archive.on('warning', function (err) {
+        debug('Archive warning: %s', err.message);
+      });
+
       archive.append(jsonStream, { name: 'data.ndjson', prefix });
       miss.pipe(archive, outputStream, onComplete);
-
-      function onComplete(err) {
-        if (!err) {
-          return;
-        }
-
-        debug('Error during streaming: %s', err.stack);
-        assetHandler.clear();
-        reject(err);
-      }
     });
@@ -119,0 +182,0 @@

package.json

 {
   "name": "@sanity/export",
-  "version": "0.126.0",
+  "version": "0.126.2",
   "description": "Export Sanity documents and assets",
@@ -27,2 +27,3 @@ "main": "lib/export.js",
     "debug": "^3.1.0",
+    "fs-extra": "^5.0.0",
     "lodash": "^4.17.4",
@@ -29,0 +30,0 @@ "mississippi": "^2.0.0",

src/AssetHandler.js

 const path = require('path')
+const fse = require('fs-extra')
 const miss = require('mississippi')
@@ -10,3 +11,2 @@ const PQueue = require('p-queue')
 const ACTION_REWRITE = 'rewrite'
-const precompressedExts = ['.zip', '.gz', '.rar', '.png', '.jpeg', '.jpg', '.gif']
 
@@ -16,5 +16,7 @@ class AssetHandler {
     this.client = options.client
-    this.archive = options.archive
-    this.archivePrefix = options.prefix
+    this.tmpDir = options.tmpDir
+    this.assetDirsCreated = false
 
+    this.assetsSeen = new Map()
+    this.filesWritten = 0
     this.queueSize = 0
@@ -28,2 +30,3 @@ this.queue = options.queue || new PQueue({concurrency: 3})
   clear() {
+    this.assetsSeen.clear()
     this.queue.clear()
@@ -42,6 +45,7 @@ this.queueSize = 0
   // placeholder asset references (_sanityAsset: 'image@file:///local/path')
-  rewriteAssets = miss.through.obj((doc, enc, callback) => {
+  rewriteAssets = miss.through.obj(async (doc, enc, callback) => {
     if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
       const type = doc._type === 'sanity.imageAsset' ? 'image' : 'file'
       const filePath = `${type}s/${generateFilename(doc._id)}`
+      this.assetsSeen.set(doc._id, type)
       this.queueAssetDownload(doc, filePath)
@@ -52,3 +56,3 @@ callback()
 
-    callback(null, this.findAndModify(doc, ACTION_REWRITE))
+    callback(null, await this.findAndModify(doc, ACTION_REWRITE))
   })
@@ -58,3 +62,3 @@
   // as well as references to assets (*.asset._ref ^= (image|file)-)
-  stripAssets = miss.through.obj((doc, enc, callback) => {
+  stripAssets = miss.through.obj(async (doc, enc, callback) => {
     if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
@@ -65,3 +69,3 @@ callback()
 
-    callback(null, this.findAndModify(doc, ACTION_REMOVE))
+    callback(null, await this.findAndModify(doc, ACTION_REMOVE))
   })
@@ -84,2 +88,7 @@
   queueAssetDownload(assetDoc, dstPath) {
+    if (!assetDoc.url) {
+      debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id)
+      return
+    }
+
     debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath)
@@ -93,6 +102,4 @@ this.queueSize++
     const stream = await requestStream({url, headers})
-    const store = precompressedExts.includes(path.extname(dstPath))
 
     if (stream.statusCode !== 200) {
-      this.archive.abort()
       this.queue.clear()
@@ -103,13 +110,21 @@ this.reject(new Error(`Referenced asset URL "${url}" returned HTTP ${stream.statusCode}`))
 
-    debug('Asset stream ready, appending to archive at %s', dstPath)
-    this.archive.append(stream, {
-      name: path.basename(dstPath),
-      prefix: [this.archivePrefix, path.dirname(dstPath)].join('/'),
-      store
-    })
+    if (!this.assetDirsCreated) {
+      /* eslint-disable no-sync */
+      fse.ensureDirSync(path.join(this.tmpDir, 'files'))
+      fse.ensureDirSync(path.join(this.tmpDir, 'images'))
+      /* eslint-enable no-sync */
+      this.assetDirsCreated = true
+    }
+
+    debug('Asset stream ready, writing to filesystem at %s', dstPath)
+    await writeStream(path.join(this.tmpDir, dstPath), stream)
+
+    this.filesWritten++
   }
 
-  findAndModify = (item, action) => {
+  // eslint-disable-next-line complexity
+  findAndModify = async (item, action) => {
     if (Array.isArray(item)) {
-      return item.map(child => this.findAndModify(child, action)).filter(Boolean)
+      const children = await Promise.all(item.map(child => this.findAndModify(child, action)))
+      return children.filter(Boolean)
     }
@@ -121,29 +136,48 @@
 
-    const assetType = getAssetType(item)
-    if (assetType) {
-      if (action === ACTION_REMOVE) {
-        return undefined
+    const isAsset = isAssetField(item)
+    if (isAsset && action === ACTION_REMOVE) {
+      return undefined
+    }
+
+    if (isAsset && action === ACTION_REWRITE) {
+      const assetId = item.asset._ref
+      if (isModernAsset(assetId)) {
+        const assetType = getAssetType(item)
+        const filePath = `${assetType}s/${generateFilename(assetId)}`
+        return {_sanityAsset: `${assetType}@file://./${filePath}`}
       }
 
-      if (action === ACTION_REWRITE) {
-        const filePath = `${assetType}s/${generateFilename(item.asset._ref)}`
-        return {
-          _sanityAsset: `${assetType}@file://./${filePath}`
-        }
-      }
+      // Legacy asset
+      const type = this.assetsSeen.get(assetId) || (await this.lookupAssetType(assetId))
+      const filePath = `${type}s/${generateFilename(assetId)}`
+      return {_sanityAsset: `${type}@file://./${filePath}`}
     }
 
-    return Object.keys(item).reduce((acc, key) => {
+    const newItem = {}
+    const keys = Object.keys(item)
+    for (let i = 0; i < keys.length; i++) {
+      const key = keys[i]
       const value = item[key]
-      acc[key] = this.findAndModify(value, action)
 
-      if (typeof acc[key] === 'undefined') {
-        delete acc[key]
+      // eslint-disable-next-line no-await-in-loop
+      newItem[key] = await this.findAndModify(value, action)
+
+      if (typeof newItem[key] === 'undefined') {
+        delete newItem[key]
       }
+    }
 
-      return acc
-    }, {})
+    return newItem
   }
+
+  lookupAssetType = async assetId => {
+    const docType = await this.client.fetch('*[_id == $id][0]._type', {id: assetId})
+    return docType === 'sanity.imageAsset' ? 'image' : 'file'
+  }
 }
 
+function isAssetField(item) {
+  return item.asset && item.asset._ref
+}
+
 function getAssetType(item) {
@@ -158,12 +192,20 @@ if (!item.asset || typeof item.asset._ref !== 'string') {
 
+function isModernAsset(assetId) {
+  return /^(image|file)/.test(assetId)
+}
+
 function generateFilename(assetId) {
-  const [, , asset, ext] = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/)
+  const [, , asset, ext] = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || []
   const extension = (ext || 'bin').replace(/^-/, '')
-  return `${asset}.${extension}`
+  return asset ? `${asset}.${extension}` : `${assetId}.bin`
 }
 
-function lookupAssetUrl(client, assetId) {
-  return client.fetch('*[_id == $id][0].url', {id: assetId})
+function writeStream(filePath, stream) {
+  return new Promise((resolve, reject) =>
+    miss.pipe(stream, fse.createWriteStream(filePath), err => {
+      return err ? reject(err) : resolve()
+    })
+  )
 }
 
 module.exports = AssetHandler

src/export.js

@@ -1,3 +0,5 @@
-const fs = require('fs')
+const os = require('os')
+const path = require('path')
 const zlib = require('zlib')
+const fse = require('fs-extra')
 const miss = require('mississippi')
@@ -32,7 +34,13 @@ const split = require('split2')
   const prefix = `${opts.dataset}-export-${slugDate}`
-  const assetHandler = new AssetHandler({client: options.client, archive, prefix})
+  const tmpDir = path.join(os.tmpdir(), prefix)
+  const assetHandler = new AssetHandler({
+    client: options.client,
+    tmpDir,
+    prefix
+  })
 
+  debug('Outputting assets (temporarily) to %s', tmpDir)
   debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath)
   const outputStream =
-    options.outputPath === '-' ? process.stdout : fs.createWriteStream(options.outputPath)
+    options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath)
 
@@ -53,3 +61,2 @@ let assetStreamHandler = assetHandler.noop
       debug('Archive finished!')
-      resolve()
     })
@@ -59,2 +66,23 @@
     onProgress({step: 'Exporting documents...'})
+
+    let documentCount = 0
+    let lastReported = Date.now()
+    const reportDocumentCount = (chunk, enc, cb) => {
+      ++documentCount
+
+      const now = Date.now()
+      if (now - lastReported > 50) {
+        onProgress({
+          step: 'Exporting documents...',
+          current: documentCount,
+          total: '?',
+          update: true
+        })
+
+        lastReported = now
+      }
+
+      cb(null, chunk)
+    }
+
     const inputStream = await getDocumentsStream(options.client, options.dataset)
@@ -69,3 +97,4 @@ const jsonStream = miss.pipeline(
       options.drafts ? miss.through.obj() : filterDrafts,
-      stringifyStream
+      stringifyStream,
+      miss.through(reportDocumentCount)
     )
@@ -78,14 +107,28 @@
 
+      onProgress({
+        step: 'Exporting documents...',
+        current: documentCount,
+        total: documentCount,
+        update: true
+      })
+
       if (!options.raw && options.assets) {
         onProgress({step: 'Downloading assets...'})
+      }
 
-        archive.on('progress', ({entries}) => {
-          onProgress({
-            step: 'Downloading assets...',
-            current: entries.processed,
-            total: Math.max(assetHandler.queueSize, entries.processed),
-            update: true
-          })
+      let prevCompleted = 0
+      const progressInterval = setInterval(() => {
+        const completed = assetHandler.queueSize - assetHandler.queue.size
+        if (prevCompleted === completed) {
+          return
+        }
+
+        prevCompleted = completed
+        onProgress({
+          step: 'Downloading assets...',
+          current: completed,
+          total: assetHandler.queueSize,
+          update: true
         })
-      }
+      }, 500)
 
@@ -95,3 +138,5 @@ debug('Waiting for asset handler to complete downloads')
         await assetHandler.finish()
+        clearInterval(progressInterval)
       } catch (assetErr) {
+        clearInterval(progressInterval)
         reject(assetErr)
@@ -101,11 +146,24 @@ return
 
+      // Add all downloaded assets to archive
+      archive.directory(path.join(tmpDir, 'files'), `${prefix}/files`, {store: true})
+      archive.directory(path.join(tmpDir, 'images'), `${prefix}/images`, {store: true})
+
       debug('Finalizing archive, flushing streams')
+      onProgress({step: 'Adding assets to archive...'})
       archive.finalize()
     })
 
+    archive.on('warning', err => {
+      debug('Archive warning: %s', err.message)
+    })
+
     archive.append(jsonStream, {name: 'data.ndjson', prefix})
     miss.pipe(archive, outputStream, onComplete)
 
-    function onComplete(err) {
+    async function onComplete(err) {
+      onProgress({step: 'Clearing temporary files...'})
+      await fse.remove(tmpDir)
+
       if (!err) {
+        resolve()
         return
@@ -112,0 +170,0 @@ }

New alerts

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Fixed alerts

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

34946

increased by24.58%

Lines of code

846

increased by28.96%

Worsened metrics

Dependency count

8

increased by14.29%

Number of low supply chain risk alerts

4

increased by100%

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Addedfs-extra@^5.0.0

• + Addedfs-extra@5.0.0(transitive)

• + Addedjsonfile@4.0.0(transitive)

• + Addeduniversalify@0.1.2(transitive)