Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

chi-datapackage

Package Overview
Dependencies
Maintainers
1
Versions
17
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

chi-datapackage - npm Package Compare versions

Comparing version 3.1.0 to 4.0.0

dist/lib/load.js
src/lib/load.js

5

CHANGELOG.md

@@ -9,2 +9,7 @@ CHANGELOG

## 4.0.0 (2016-09-12)
* read and fetch as seperate options for Loader
* Add seperate JSON and JSON5 types
* Improved error handling
## 3.1.0 (2016-09-06)

@@ -11,0 +16,0 @@ _(none)_

18

dist/lib/json.js
'use strict';
var parse = require('json5').parse;
var JSON5 = require('json5');
module.exports = function fromJson(json) {
module.exports.json5 = function (json) {
if (typeof json === 'string' || /* istanbul ignore next */json instanceof Buffer) {
try {
json = parse(json);
json = JSON5.parse(json);
} catch (err) {

@@ -15,2 +15,14 @@ /* istanbul ignore next */

return json;
};
module.exports.json = function (json) {
if (typeof json === 'string' || /* istanbul ignore next */json instanceof Buffer) {
try {
json = JSON.parse(json);
} catch (err) {
/* istanbul ignore next */
json = {};
}
}
return json;
};

5

dist/lib/mime.json

@@ -24,4 +24,7 @@ {

"compressible": true,
"extensions": ["json", "json5", "map"]
"extensions": ["json"]
},
"application/json5": {
"extensions": ["json5", "map"]
},
"text/matrix": {

@@ -28,0 +31,0 @@ "source": "custom",

8

dist/lib/translators.js

@@ -11,3 +11,3 @@ 'use strict';

var jsonParse = require('./json');
var JSON5 = require('json5');
var matrixParse = require('./matrix');

@@ -47,4 +47,8 @@

'application/json': function applicationJson(resource) {
return { data: jsonParse(resource.content) };
return { data: JSON.parse(resource.content) };
},
'application/json5': function applicationJson5(resource) {
return { data: JSON5.parse(resource.content) };
}
};

49

dist/loader.js

@@ -7,9 +7,10 @@ 'use strict';

var path = require('path');
var assert = require('assert');
var debug = require('debug')('Loader');
var identifier = require('datapackage-identifier');
var parse = require('json5').parse;
var JSON5 = require('json5');
// const url = require('url');
var identifier = require('datapackage-identifier');
var absURLRegEx = /^([^\/]+:\/\/|\/)/;

@@ -31,3 +32,2 @@ // const forwardSlashPattern = /\//g;

}
var path = require('path');

@@ -41,2 +41,6 @@ return function (url) {

function isFilePath(url) {
return !process.browser && url.indexOf('file://') === 0;
}
var Loader = function () {

@@ -46,6 +50,16 @@ function Loader(opts) {

assert(opts && typeof opts.fetch === 'function', 'opts.fetch is required.');
this.fetch = opts.fetch;
this.read = opts.read;
}
_createClass(Loader, [{
key: 'load',
value: function load(pathOrUrl) {
if (isFilePath(pathOrUrl)) {
return this.read(pathOrUrl.replace('file://', ''));
}
return this.fetch(pathOrUrl);
}
}, {
key: 'datapackage',

@@ -61,3 +75,7 @@ value: function datapackage(_datapackage) {

return this.fetch(url).catch(function (err) {
if (!url) {
return Promise.resolve(_datapackage);
}
return this.load(url).catch(function (err) {
if (err.code === 'ENOENT') {

@@ -68,3 +86,3 @@ throw new Error('No DataPackage at path \'' + url + '\'');

throw err;
}).then(parse).then(function (res) {
}).then(JSON5.parse).then(function (res) {
return Object.assign(res, _datapackage, id);

@@ -87,8 +105,11 @@ });

debug('Loading resource', _resource);
if (!_resource.url) {
var url = _resource.url;
if (!url) {
return Promise.resolve(_resource);
}
return this.fetch(_resource.url).catch(function (err) {
return this.load(url).catch(function (err) {
if (err.code === 'ENOENT') {
throw new Error('No DataPackage resource at path \'' + _resource.url + '\'');
throw new Error('No DataPackage resource at path \'' + url + '\'');
}

@@ -112,3 +133,11 @@ /* istanbul ignore next */

url = url.replace(backSlashPattern, '/');
url = url.match(absURLRegEx) ? url : resolvePath(url);
if (!url.match(absURLRegEx)) {
url = resolvePath(url);
}
if (url.indexOf('file://') === 0) {
return {
url: url,
dataPackageJsonUrl: url
};
}
return identifier.parse(url);

@@ -115,0 +144,0 @@ }

11

dist/normalizer.js
'use strict';
var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol ? "symbol" : typeof obj; };
var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();

@@ -7,4 +9,5 @@

var assert = require('assert');
var urijs = require('urijs');
var deepExtend = require('deep-extend');
var merge = require('lodash.merge');
var cuid = require('cuid');

@@ -16,3 +19,3 @@

opts = opts || {};
assert(opts && _typeof(opts.mimeLookup) === 'object', 'opts.mimeLookup is required.');
this.mime = opts.mimeLookup;

@@ -29,3 +32,3 @@ }

var normalized = deepExtend({
var normalized = merge({
path: path,

@@ -55,3 +58,3 @@ base: base,

_resource = deepExtend({}, _resource);
_resource = merge({}, _resource);

@@ -58,0 +61,0 @@ if (_resource.path || _resource.url) {

32

dist/processor.js
'use strict';
var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol ? "symbol" : typeof obj; };
var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();

@@ -7,3 +9,4 @@

var deepExtend = require('deep-extend');
var assert = require('assert');
var merge = require('lodash.merge');

@@ -14,4 +17,5 @@ var Processor = function () {

opts = opts || {};
this.translators = deepExtend({}, opts.translators);
assert(opts && _typeof(opts.translators) === 'object', 'opts.translators is required.');
assert(opts && _typeof(opts.schemaProcessor) === 'object', 'opts.schemaProcessor is required.');
this.translators = merge({}, opts.translators);
this.schemaProcessor = opts.schemaProcessor;

@@ -34,13 +38,19 @@ }

value: function resource(_resource) {
var r = Object.assign({}, _resource);
if (r.content) {
var translator = this.translators[r.mediatype];
if (translator) {
Object.assign(r, translator(r));
var mediatype = _resource.mediatype;
var result = Object.assign({}, _resource, {
$error: null,
errors: []
});
if (_resource.content && mediatype) {
if (!Object.prototype.hasOwnProperty.call(this.translators, mediatype)) {
throw new Error('Unknown media type ' + _resource.mediatype);
}
var translator = this.translators[mediatype];
Object.assign(result, translator(result));
}
if (r.schema) {
return Object.assign(r, this.schemaProcessor.process(r));
if (_resource.schema) {
Object.assign(result, this.schemaProcessor.process(result));
}
return r;
return result;
}

@@ -47,0 +57,0 @@ }]);

38

dist/schema.js
'use strict';
var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol ? "symbol" : typeof obj; };
var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();

@@ -7,3 +9,4 @@

var deepExtend = require('deep-extend');
var assert = require('assert');
var merge = require('lodash.merge');
var pointers = require('./lib/pointers');

@@ -29,4 +32,4 @@

opts = opts || {};
deepExtend(this, opts);
assert(opts && _typeof(opts.types) === 'object', 'opts.types is required.');
this.types = merge({}, opts.types);
}

@@ -110,3 +113,2 @@

resource = Object.assign({}, resource);
/* istanbul ignore if */

@@ -117,15 +119,18 @@ if (!Array.isArray(resource.data) || !resource.schema || !resource.schema.fields || resource.schema.fields.length === 0) {

resource.errors = resource.errors || [];
var resourceErrors = resource.errors || [];
var fields = resource.schema.fields;
resource.data = resource.data.map(function (d, i) {
d = Object.assign({}, d);
var processedData = [];
resource.data.forEach(function (d, i) {
var r = merge({}, d);
var errors = [];
fields.forEach(function (field) {
var key = field.name;
if (Object.prototype.hasOwnProperty.call(d, key)) {
if (Object.prototype.hasOwnProperty.call(r, key)) {
var $fn = field.$fn || (field.$fn = self.generateCastFn(field));
try {
d[key] = $fn(d[key]);
r[key] = $fn(r[key]);
} catch (err) {
resource.errors.push({
errors.push({
type: 'FieldMismatch',

@@ -138,3 +143,3 @@ code: 'InvalidType',

} else if (field.constraints && field.constraints.required) {
resource.errors.push({
errors.push({
type: 'ConstraintsError',

@@ -148,6 +153,13 @@ code: 'MissingField',

return d;
if (errors.length === 0) {
processedData.push(r);
} else {
resourceErrors = resourceErrors.concat(errors);
}
});
return resource;
return Object.assign({}, resource, {
errors: resourceErrors,
data: processedData
});
}

@@ -154,0 +166,0 @@ }]);

28

dist/service.js

@@ -17,4 +17,5 @@ 'use strict';

var types = require('./lib/types');
var fetch = require('./lib/fetch');
var load = require('./lib/load');
var DataPackageService = function () {

@@ -29,3 +30,3 @@ function DataPackageService() {

this.processor = new Processor({ translators: translators, schemaProcessor: this.schemaProcessor });
this.loader = new Loader({ fetch: fetch });
this.loader = new Loader(load);
}

@@ -90,7 +91,20 @@

value: function processResource(r) {
r.$error = undefined; // last error
r.errors = []; // list of all errors
try {
return Object.assign(r, this.processor.resource(r));
Object.assign(r, this.processor.resource(r));
if (r.errors && r.errors.length > 0) {
console.log('asdfasdfsdf');
r.$error = new Error('Errors processing resource ' + r.name);
}
} catch (err) {
return Object.assign(r, { $valid: false, $error: err });
r.$error = err;
r.errors = r.errors || [];
r.errors.shift({
code: 'Parsing',
type: err.name,
message: 'Parsing error: ' + err.message
});
}
return r;
}

@@ -100,3 +114,7 @@ }, {

value: function processPackage(p) {
Object.assign(p, this.processor.datapackage(p));
var _this3 = this;
p.resources = p.resources.map(function (r) {
return _this3.processResource(r);
});
p.$resourcesByName = Normalizer.index(p);

@@ -103,0 +121,0 @@ return p;

4

package.json
{
"name": "chi-datapackage",
"version": "3.1.0",
"version": "4.0.0",
"description": "Normalize datapackage and datapackage resources",

@@ -79,3 +79,2 @@ "main": "index.js",

"debug": "^2.2.0",
"deep-extend": "^0.4.1",
"isomorphic-fetch": "^2.2.1",

@@ -85,2 +84,3 @@ "js-yaml": "^3.6.1",

"jsonpointer": "^4.0.0",
"lodash.merge": "^4.6.0",
"mime-lookup": "0.0.2",

@@ -87,0 +87,0 @@ "parse-iso-duration": "^1.0.0",

18

src/lib/json.js

@@ -1,7 +0,7 @@

const parse = require('json5').parse;
const JSON5 = require('json5');
module.exports = function fromJson (json) {
module.exports.json5 = function (json) {
if (typeof json === 'string' || /* istanbul ignore next */ json instanceof Buffer) {
try {
json = parse(json);
json = JSON5.parse(json);
} catch (err) {

@@ -14,1 +14,13 @@ /* istanbul ignore next */

};
module.exports.json = function (json) {
if (typeof json === 'string' || /* istanbul ignore next */ json instanceof Buffer) {
try {
json = JSON.parse(json);
} catch (err) {
/* istanbul ignore next */
json = {};
}
}
return json;
};

5

src/lib/mime.json

@@ -24,4 +24,7 @@ {

"compressible": true,
"extensions": ["json", "json5", "map"]
"extensions": ["json"]
},
"application/json5": {
"extensions": ["json5", "map"]
},
"text/matrix": {

@@ -28,0 +31,0 @@ "source": "custom",

6

src/lib/translators.js

@@ -9,3 +9,3 @@ 'use strict';

const jsonParse = require('./json');
const JSON5 = require('json5');
const matrixParse = require('./matrix');

@@ -38,3 +38,5 @@

'application/json': resource => ({data: jsonParse(resource.content)})
'application/json': resource => ({data: JSON.parse(resource.content)}),
'application/json5': resource => ({data: JSON5.parse(resource.content)})
};

50

src/loader.js
'use strict';
const path = require('path');
const assert = require('assert');
const debug = require('debug')('Loader');
const identifier = require('datapackage-identifier');
const parse = require('json5').parse;
const JSON5 = require('json5');
// const url = require('url');
const identifier = require('datapackage-identifier');
const absURLRegEx = /^([^\/]+:\/\/|\/)/;

@@ -26,3 +27,2 @@ // const forwardSlashPattern = /\//g;

}
const path = require('path');

@@ -37,7 +37,20 @@ return url => {

function isFilePath (url) {
return (!process.browser && url.indexOf('file://') === 0);
}
class Loader {
constructor (opts) {
assert(opts && typeof opts.fetch === 'function', 'opts.fetch is required.');
this.fetch = opts.fetch;
this.read = opts.read;
}
load (pathOrUrl) {
if (isFilePath(pathOrUrl)) {
return this.read(pathOrUrl.replace('file://', ''));
}
return this.fetch(pathOrUrl);
}
datapackage (datapackage) {

@@ -52,4 +65,7 @@ debug('Loading datapackage', datapackage);

return this
.fetch(url)
if (!url) {
return Promise.resolve(datapackage);
}
return this.load(url)
.catch(err => {

@@ -62,3 +78,3 @@ if (err.code === 'ENOENT') {

})
.then(parse)
.then(JSON5.parse)
.then(res => Object.assign(res, datapackage, id));

@@ -75,10 +91,12 @@ }

debug('Loading resource', resource);
if (!resource.url) {
const url = resource.url;
if (!url) {
return Promise.resolve(resource);
}
return this
.fetch(resource.url)
return this.load(url)
.catch(err => {
if (err.code === 'ENOENT') {
throw new Error(`No DataPackage resource at path '${resource.url}'`);
throw new Error(`No DataPackage resource at path '${url}'`);
}

@@ -100,3 +118,11 @@ /* istanbul ignore next */

url = url.replace(backSlashPattern, '/');
url = url.match(absURLRegEx) ? url : resolvePath(url);
if (!url.match(absURLRegEx)) {
url = resolvePath(url);
}
if (url.indexOf('file://') === 0) {
return {
url,
dataPackageJsonUrl: url
};
}
return identifier.parse(url);

@@ -103,0 +129,0 @@ }

9

src/normalizer.js
'use strict';
const assert = require('assert');
const urijs = require('urijs');
const deepExtend = require('deep-extend');
const merge = require('lodash.merge');
const cuid = require('cuid');

@@ -9,3 +10,3 @@

constructor (opts) {
opts = opts || {};
assert(opts && typeof opts.mimeLookup === 'object', 'opts.mimeLookup is required.');
this.mime = opts.mimeLookup;

@@ -20,3 +21,3 @@ }

const normalized = deepExtend({
const normalized = merge({
path,

@@ -45,3 +46,3 @@ base,

resource = deepExtend({}, resource);
resource = merge({}, resource);

@@ -48,0 +49,0 @@ if (resource.path || resource.url) {

30

src/processor.js
'use strict';
const deepExtend = require('deep-extend');
const assert = require('assert');
const merge = require('lodash.merge');
class Processor {
constructor (opts) {
opts = opts || {};
this.translators = deepExtend({}, opts.translators);
assert(opts && typeof opts.translators === 'object', 'opts.translators is required.');
assert(opts && typeof opts.schemaProcessor === 'object', 'opts.schemaProcessor is required.');
this.translators = merge({}, opts.translators);
this.schemaProcessor = opts.schemaProcessor;

@@ -19,13 +21,19 @@ }

resource (resource) {
const r = Object.assign({}, resource);
if (r.content) {
const translator = this.translators[r.mediatype];
if (translator) {
Object.assign(r, translator(r));
const mediatype = resource.mediatype;
const result = Object.assign({}, resource, {
$error: null,
errors: []
});
if (resource.content && mediatype) {
if (!Object.prototype.hasOwnProperty.call(this.translators, mediatype)) {
throw new Error(`Unknown media type ${resource.mediatype}`);
}
const translator = this.translators[mediatype];
Object.assign(result, translator(result));
}
if (r.schema) {
return Object.assign(r, this.schemaProcessor.process(r));
if (resource.schema) {
Object.assign(result, this.schemaProcessor.process(result));
}
return r;
return result;
}

@@ -32,0 +40,0 @@ }

36

src/schema.js
'use strict';
const deepExtend = require('deep-extend');
const assert = require('assert');
const merge = require('lodash.merge');
const pointers = require('./lib/pointers');

@@ -22,4 +23,4 @@

constructor (opts) {
opts = opts || {};
deepExtend(this, opts);
assert(opts && typeof opts.types === 'object', 'opts.types is required.');
this.types = merge({}, opts.types);
}

@@ -94,3 +95,2 @@

resource = Object.assign({}, resource);
/* istanbul ignore if */

@@ -101,15 +101,18 @@ if (!Array.isArray(resource.data) || !resource.schema || !resource.schema.fields || resource.schema.fields.length === 0) {

resource.errors = resource.errors || [];
let resourceErrors = resource.errors || [];
const fields = resource.schema.fields;
resource.data = resource.data.map((d, i) => {
d = Object.assign({}, d);
const processedData = [];
resource.data.forEach((d, i) => {
const r = merge({}, d);
const errors = [];
fields.forEach(field => {
const key = field.name;
if (Object.prototype.hasOwnProperty.call(d, key)) {
if (Object.prototype.hasOwnProperty.call(r, key)) {
const $fn = field.$fn || (field.$fn = self.generateCastFn(field));
try {
d[key] = $fn(d[key]);
r[key] = $fn(r[key]);
} catch (err) {
resource.errors.push({
errors.push({
type: 'FieldMismatch',

@@ -122,3 +125,3 @@ code: 'InvalidType',

} else if (field.constraints && field.constraints.required) {
resource.errors.push({
errors.push({
type: 'ConstraintsError',

@@ -132,6 +135,13 @@ code: 'MissingField',

return d;
if (errors.length === 0) {
processedData.push(r);
} else {
resourceErrors = resourceErrors.concat(errors);
}
});
return resource;
return Object.assign({}, resource, {
errors: resourceErrors,
data: processedData
});
}

@@ -138,0 +148,0 @@ }

24

src/service.js

@@ -13,4 +13,5 @@ 'use strict';

const types = require('./lib/types');
const fetch = require('./lib/fetch');
const load = require('./lib/load');
class DataPackageService {

@@ -23,3 +24,3 @@ constructor () {

this.processor = new Processor({translators, schemaProcessor: this.schemaProcessor});
this.loader = new Loader({fetch});
this.loader = new Loader(load);
}

@@ -67,11 +68,24 @@

processResource (r) {
r.$error = undefined; // last error
r.errors = []; // list of all errors
try {
return Object.assign(r, this.processor.resource(r));
Object.assign(r, this.processor.resource(r));
if (r.errors && r.errors.length > 0) {
console.log('asdfasdfsdf');
r.$error = new Error(`Errors processing resource ${r.name}`);
}
} catch (err) {
return Object.assign(r, {$valid: false, $error: err});
r.$error = err;
r.errors = r.errors || [];
r.errors.shift({
code: 'Parsing',
type: err.name,
message: `Parsing error: ${err.message}`
});
}
return r;
}
processPackage (p) {
Object.assign(p, this.processor.datapackage(p));
p.resources = p.resources.map(r => this.processResource(r));
p.$resourcesByName = Normalizer.index(p);

@@ -78,0 +92,0 @@ return p;

src/lib/fetch.js
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc