Socket
Socket
Sign inDemoInstall

@sanity/export

Package Overview
Dependencies
68
Maintainers
7
Versions
1040
Alerts
File Explorer

Advanced tools

Install Socket

Detect and block malicious and high-risk dependencies

Install

Comparing version 0.133.0-realtime-block-editor.33c2edde to 0.133.0-realtime-block-editor.3566387d

153

lib/AssetHandler.js

@@ -1,23 +0,40 @@

'use strict';
"use strict";
var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }();
function _slicedToArray(arr, i) { return _arrayWithHoles(arr) || _iterableToArrayLimit(arr, i) || _nonIterableRest(); }
var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; };
function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance"); }
function _objectWithoutProperties(obj, keys) { var target = {}; for (var i in obj) { if (keys.indexOf(i) >= 0) continue; if (!Object.prototype.hasOwnProperty.call(obj, i)) continue; target[i] = obj[i]; } return target; }
function _iterableToArrayLimit(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"] != null) _i["return"](); } finally { if (_d) throw _e; } } return _arr; }
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
function _arrayWithHoles(arr) { if (Array.isArray(arr)) return arr; }
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; var ownKeys = Object.keys(source); if (typeof Object.getOwnPropertySymbols === 'function') { ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function (sym) { return Object.getOwnPropertyDescriptor(source, sym).enumerable; })); } ownKeys.forEach(function (key) { _defineProperty(target, key, source[key]); }); } return target; }
function _objectWithoutProperties(source, excluded) { if (source == null) return {}; var target = _objectWithoutPropertiesLoose(source, excluded); var key, i; if (Object.getOwnPropertySymbols) { var sourceSymbolKeys = Object.getOwnPropertySymbols(source); for (i = 0; i < sourceSymbolKeys.length; i++) { key = sourceSymbolKeys[i]; if (excluded.indexOf(key) >= 0) continue; if (!Object.prototype.propertyIsEnumerable.call(source, key)) continue; target[key] = source[key]; } } return target; }
function _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
const path = require('path');
const crypto = require('crypto');
const fse = require('fs-extra');
const miss = require('mississippi');
const PQueue = require('p-queue');
var _require = require('lodash');
const _require = require('lodash'),
omit = _require.omit;
const omit = _require.omit;
const pkg = require('../package.json');
const pkg = require('../package.json');
const requestStream = require('./requestStream');
const debug = require('./debug');

@@ -33,3 +50,5 @@

this.rewriteAssets = miss.through.obj((() => {
_defineProperty(this, "rewriteAssets", miss.through.obj(
/*#__PURE__*/
function () {
var _ref = _asyncToGenerator(function* (doc, enc, callback) {

@@ -39,4 +58,7 @@ if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {

const filePath = `${type}s/${generateFilename(doc._id)}`;
_this.assetsSeen.set(doc._id, type);
_this.queueAssetDownload(doc, filePath, type);
callback();

@@ -52,4 +74,7 @@ return;

};
})());
this.stripAssets = miss.through.obj((() => {
}()));
_defineProperty(this, "stripAssets", miss.through.obj(
/*#__PURE__*/
function () {
var _ref2 = _asyncToGenerator(function* (doc, enc, callback) {

@@ -67,5 +92,7 @@ if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {

};
})());
this.skipAssets = miss.through.obj((doc, enc, callback) => {
}()));
_defineProperty(this, "skipAssets", miss.through.obj((doc, enc, callback) => {
const isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type);
if (isAsset) {

@@ -77,11 +104,12 @@ callback();

callback(null, doc);
});
this.noop = miss.through.obj((doc, enc, callback) => callback(null, doc));
}));
this.findAndModify = (() => {
_defineProperty(this, "noop", miss.through.obj((doc, enc, callback) => callback(null, doc)));
_defineProperty(this, "findAndModify",
/*#__PURE__*/
function () {
var _ref3 = _asyncToGenerator(function* (item, action) {
if (Array.isArray(item)) {
const children = yield Promise.all(item.map(function (child) {
return _this.findAndModify(child, action);
}));
const children = yield Promise.all(item.map(child => _this.findAndModify(child, action)));
return children.filter(Boolean);

@@ -95,2 +123,3 @@ }

const isAsset = isAssetField(item);
if (isAsset && action === ACTION_REMOVE) {

@@ -102,15 +131,20 @@ return undefined;

const asset = item.asset,
other = _objectWithoutProperties(item, ['asset']);
other = _objectWithoutProperties(item, ["asset"]);
const assetId = asset._ref;
if (isModernAsset(assetId)) {
const assetType = getAssetType(item);
const filePath = `${assetType}s/${generateFilename(assetId)}`;
return _extends({ _sanityAsset: `${assetType}@file://./${filePath}` }, other);
}
return _objectSpread({
_sanityAsset: `${assetType}@file://./${filePath}`
}, other);
} // Legacy asset
// Legacy asset
const type = _this.assetsSeen.get(assetId) || (yield _this.lookupAssetType(assetId));
const filePath = `${type}s/${generateFilename(assetId)}`;
return _extends({ _sanityAsset: `${type}@file://./${filePath}` }, other);
return _objectSpread({
_sanityAsset: `${type}@file://./${filePath}`
}, other);
}

@@ -120,7 +154,7 @@

const keys = Object.keys(item);
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
const value = item[key];
const value = item[key]; // eslint-disable-next-line no-await-in-loop
// eslint-disable-next-line no-await-in-loop
newItem[key] = yield _this.findAndModify(value, action);

@@ -139,7 +173,11 @@

};
})();
}());
this.lookupAssetType = (() => {
_defineProperty(this, "lookupAssetType",
/*#__PURE__*/
function () {
var _ref4 = _asyncToGenerator(function* (assetId) {
const docType = yield _this.client.fetch('*[_id == $id][0]._type', { id: assetId });
const docType = yield _this.client.fetch('*[_id == $id][0]._type', {
id: assetId
});
return docType === 'sanity.imageAsset' ? 'image' : 'file';

@@ -151,3 +189,3 @@ });

};
})();
}());

@@ -157,3 +195,2 @@ this.client = options.client;

this.assetDirsCreated = false;
this.assetsSeen = new Map();

@@ -163,3 +200,6 @@ this.assetMap = {};

this.queueSize = 0;
this.queue = options.queue || new PQueue({ concurrency: 3 });
this.queue = options.queue || new PQueue({
concurrency: 3
});
this.reject = () => {

@@ -181,16 +221,6 @@ throw new Error('Asset handler errored before `finish()` was called');

});
}
// Called when we want to download all assets to local filesystem and rewrite documents to hold
} // Called when we want to download all assets to local filesystem and rewrite documents to hold
// placeholder asset references (_sanityAsset: 'image@file:///local/path')
// Called in the case where we don't _want_ assets, so basically just remove all asset documents
// as well as references to assets (*.asset._ref ^= (image|file)-)
// Called when we are using raw export mode along with `assets: false`, where we simply
// want to skip asset documents but retain asset references (useful for data mangling)
queueAssetDownload(assetDoc, dstPath, type) {

@@ -212,9 +242,15 @@ if (!assetDoc.url) {

const url = assetDoc.url;
const headers = {
'User-Agent': `${pkg.name}@${pkg.version}`
};
const stream = yield requestStream({
url,
headers
});
const headers = { 'User-Agent': `${pkg.name}@${pkg.version}` };
const stream = yield requestStream({ url, headers });
if (stream.statusCode !== 200) {
_this2.queue.clear();
_this2.reject(new Error(`Referenced asset URL "${url}" returned HTTP ${stream.statusCode}`));
return;

@@ -228,2 +264,3 @@ }

/* eslint-enable no-sync */
_this2.assetDirsCreated = true;

@@ -236,4 +273,4 @@ }

const id = `${type}-${hash}`;
const metaProps = omit(assetDoc, EXCLUDE_PROPS);
const metaProps = omit(assetDoc, EXCLUDE_PROPS);
if (Object.keys(metaProps).length > 0) {

@@ -245,5 +282,5 @@ _this2.assetMap[id] = metaProps;

})();
}
} // eslint-disable-next-line complexity
// eslint-disable-next-line complexity
}

@@ -260,7 +297,6 @@

var _ref5 = item.asset._ref.match(/^(image|file)-/) || [],
_ref6 = _slicedToArray(_ref5, 2);
const _ref5 = item.asset._ref.match(/^(image|file)-/) || [],
_ref6 = _slicedToArray(_ref5, 2),
type = _ref6[1];
const type = _ref6[1];
return type || null;

@@ -270,11 +306,9 @@ }

function isModernAsset(assetId) {
return (/^(image|file)/.test(assetId)
);
return /^(image|file)/.test(assetId);
}
function generateFilename(assetId) {
var _ref7 = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [],
_ref8 = _slicedToArray(_ref7, 4);
const asset = _ref8[2],
const _ref7 = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [],
_ref8 = _slicedToArray(_ref7, 4),
asset = _ref8[2],
ext = _ref8[3];

@@ -292,3 +326,2 @@

});
return new Promise((resolve, reject) => miss.pipe(stream, hasher, fse.createWriteStream(filePath), err => {

@@ -295,0 +328,0 @@ return err ? reject(err) : resolve(hash.digest('hex'));

@@ -1,3 +0,3 @@

'use strict';
"use strict";
module.exports = require('debug')('sanity:export');

@@ -1,20 +0,37 @@

'use strict';
"use strict";
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
const os = require('os');
const path = require('path');
const zlib = require('zlib');
const fse = require('fs-extra');
const miss = require('mississippi');
const split = require('split2');
const archiver = require('archiver');
const debug = require('./debug');
const AssetHandler = require('./AssetHandler');
const stringifyStream = require('./stringifyStream');
const validateOptions = require('./validateOptions');
const rejectOnApiError = require('./rejectOnApiError');
const getDocumentsStream = require('./getDocumentsStream');
const filterSystemDocuments = require('./filterSystemDocuments');
const filterDocumentTypes = require('./filterDocumentTypes');
const filterDrafts = require('./filterDrafts');

@@ -29,7 +46,7 @@

gzip: true,
gzipOptions: { level: options.compress ? zlib.Z_DEFAULT_COMPRESSION : zlib.Z_NO_COMPRESSION }
gzipOptions: {
level: options.compress ? zlib.Z_DEFAULT_COMPRESSION : zlib.Z_NO_COMPRESSION
}
});
const slugDate = new Date().toISOString().replace(/[^a-z0-9]/gi, '-').toLowerCase();
const prefix = `${opts.dataset}-export-${slugDate}`;

@@ -42,8 +59,7 @@ const tmpDir = path.join(os.tmpdir(), prefix);

});
debug('Outputting assets (temporarily) to %s', tmpDir);
debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath);
const outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);
let assetStreamHandler = assetHandler.noop;
let assetStreamHandler = assetHandler.noop;
if (!options.raw) {

@@ -53,25 +69,7 @@ assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets;

return new Promise((() => {
return new Promise(
/*#__PURE__*/
function () {
var _ref = _asyncToGenerator(function* (resolve, reject) {
let onComplete = (() => {
var _ref3 = _asyncToGenerator(function* (err) {
onProgress({ step: 'Clearing temporary files...' });
yield fse.remove(tmpDir);
if (!err) {
resolve();
return;
}
debug('Error during streaming: %s', err.stack);
assetHandler.clear();
reject(err);
});
return function onComplete(_x4) {
return _ref3.apply(this, arguments);
};
})();
miss.finished(archive, function (archiveErr) {
miss.finished(archive, archiveErr => {
if (archiveErr) {

@@ -85,12 +83,13 @@ debug('Archiving errored! %s', archiveErr.stack);

});
debug('Getting dataset export stream');
onProgress({ step: 'Exporting documents...' });
onProgress({
step: 'Exporting documents...'
});
let documentCount = 0;
let lastReported = Date.now();
const reportDocumentCount = function reportDocumentCount(chunk, enc, cb) {
const reportDocumentCount = (chunk, enc, cb) => {
++documentCount;
const now = Date.now();
const now = Date.now();
if (now - lastReported > 50) {

@@ -103,3 +102,2 @@ onProgress({

});
lastReported = now;

@@ -113,4 +111,5 @@ }

const jsonStream = miss.pipeline(inputStream, split(JSON.parse), rejectOnApiError, filterSystemDocuments, assetStreamHandler, filterDocumentTypes(options.types), options.drafts ? miss.through.obj() : filterDrafts, stringifyStream, miss.through(reportDocumentCount));
miss.finished(jsonStream, (() => {
miss.finished(jsonStream,
/*#__PURE__*/
function () {
var _ref2 = _asyncToGenerator(function* (err) {

@@ -129,8 +128,11 @@ if (err) {

if (!options.raw && options.assets) {
onProgress({ step: 'Downloading assets...' });
onProgress({
step: 'Downloading assets...'
});
}
let prevCompleted = 0;
const progressInterval = setInterval(function () {
const progressInterval = setInterval(() => {
const completed = assetHandler.queueSize - assetHandler.queue.size;
if (prevCompleted === completed) {

@@ -148,7 +150,10 @@ return;

}, 500);
debug('Waiting for asset handler to complete downloads');
debug('Waiting for asset handler to complete downloads');
try {
const assetMap = yield assetHandler.finish();
archive.append(JSON.stringify(assetMap), { name: 'assets.json', prefix });
archive.append(JSON.stringify(assetMap), {
name: 'assets.json',
prefix
});
clearInterval(progressInterval);

@@ -159,10 +164,15 @@ } catch (assetErr) {

return;
}
} // Add all downloaded assets to archive
// Add all downloaded assets to archive
archive.directory(path.join(tmpDir, 'files'), `${prefix}/files`, { store: true });
archive.directory(path.join(tmpDir, 'images'), `${prefix}/images`, { store: true });
archive.directory(path.join(tmpDir, 'files'), `${prefix}/files`, {
store: true
});
archive.directory(path.join(tmpDir, 'images'), `${prefix}/images`, {
store: true
});
debug('Finalizing archive, flushing streams');
onProgress({ step: 'Adding assets to archive...' });
onProgress({
step: 'Adding assets to archive...'
});
archive.finalize();

@@ -174,10 +184,34 @@ });

};
})());
archive.on('warning', function (err) {
}());
archive.on('warning', err => {
debug('Archive warning: %s', err.message);
});
archive.append(jsonStream, {
name: 'data.ndjson',
prefix
});
miss.pipe(archive, outputStream, onComplete);
archive.append(jsonStream, { name: 'data.ndjson', prefix });
miss.pipe(archive, outputStream, onComplete);
function onComplete(_x4) {
return _onComplete.apply(this, arguments);
}
function _onComplete() {
_onComplete = _asyncToGenerator(function* (err) {
onProgress({
step: 'Clearing temporary files...'
});
yield fse.remove(tmpDir);
if (!err) {
resolve();
return;
}
debug('Error during streaming: %s', err.stack);
assetHandler.clear();
reject(err);
});
return _onComplete.apply(this, arguments);
}
});

@@ -188,5 +222,5 @@

};
})());
}());
}
module.exports = exportDataset;

@@ -1,2 +0,2 @@

'use strict';
"use strict";

@@ -7,2 +7,3 @@ const miss = require('mississippi');

const type = doc && doc._type;
if (allowedTypes.includes(type)) {

@@ -9,0 +10,0 @@ callback(null, doc);

@@ -1,2 +0,2 @@

'use strict';
"use strict";

@@ -3,0 +3,0 @@ const miss = require('mississippi');

@@ -1,4 +0,5 @@

'use strict';
"use strict";
const miss = require('mississippi');
const debug = require('./debug');

@@ -5,0 +6,0 @@

@@ -1,4 +0,5 @@

'use strict';
"use strict";
const pkg = require('../package.json');
const requestStream = require('./requestStream');

@@ -14,4 +15,6 @@

};
return requestStream({ url, headers });
return requestStream({
url,
headers
});
};

@@ -1,2 +0,2 @@

'use strict';
"use strict";

@@ -3,0 +3,0 @@ const miss = require('mississippi');

@@ -1,2 +0,2 @@

'use strict';
"use strict";

@@ -3,0 +3,0 @@ const simpleGet = require('simple-get');

@@ -1,2 +0,2 @@

'use strict';
"use strict";

@@ -3,0 +3,0 @@ const miss = require('mississippi');

@@ -1,2 +0,2 @@

'use strict';
"use strict";

@@ -30,2 +30,3 @@ const defaults = require('lodash/defaults');

const missing = clientMethods.find(key => typeof options.client[key] !== 'function');
if (missing) {

@@ -36,2 +37,3 @@ throw new Error(`\`options.client\` is not a valid @sanity/client instance - no "${missing}" method found`);

const clientConfig = options.client.config();
if (!clientConfig.token) {

@@ -38,0 +40,0 @@ throw new Error('Client is not instantiated with a `token`');

{
"name": "@sanity/export",
"version": "0.133.0-realtime-block-editor.33c2edde",
"version": "0.133.0-realtime-block-editor.3566387d",
"description": "Export Sanity documents and assets",

@@ -27,13 +27,13 @@ "main": "lib/export.js",

"debug": "^3.1.0",
"fs-extra": "^5.0.0",
"fs-extra": "^6.0.1",
"lodash": "^4.17.4",
"mississippi": "^2.0.0",
"p-queue": "^2.3.0",
"simple-get": "^2.7.0",
"simple-get": "^3.0.2",
"split2": "^2.1.1"
},
"devDependencies": {
"@sanity/client": "0.133.0-realtime-block-editor.33c2edde",
"@sanity/client": "0.133.0-realtime-block-editor.3566387d",
"decompress": "^4.2.0",
"jest": "^22.1.4",
"jest": "^22.4.3",
"rimraf": "^2.6.2",

@@ -40,0 +40,0 @@ "string-to-stream": "^1.1.0"

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc