chi-datapackage - npm Package Compare versions

Comparing version 2.2.0 to 3.0.0

CHANGELOG.md

@@ -9,2 +9,9 @@ CHANGELOG
 
+## 3.0.0 (2016-08-19)
+* Assign cuid to resources without name
+* Record errors while processing data
+* Add normalizeSchemas to DataPackageService
+* Now normalize schemas after resources
+* Generate missing readable title from field name
+
 ## 2.2.0 (2016-08-15)
@@ -11,0 +18,0 @@ * add support for missingValues

dist/lib/types.js

@@ -8,3 +8,3 @@ 'use strict';
 
-var INVALID_TYPE = 'chi-datapackage: Invalid type';
+var INVALID_TYPE = 'Invalid type';
 
@@ -15,3 +15,3 @@ function jsonParse(isArray) {
     if (Array.isArray(c) !== isArray) {
-      throw new Error(INVALID_TYPE);
+      throw new Error(INVALID_TYPE + ': expected JSON Array');
     }
@@ -27,3 +27,3 @@ return c;
     if (Number.isNaN(c) || c === null) {
-      throw new Error(INVALID_TYPE);
+      throw new Error(INVALID_TYPE + ': expected ' + fmt + ' formmated date');
     }
@@ -37,3 +37,3 @@ return c;
   if (isNaN(c.getTime())) {
-    throw new Error(INVALID_TYPE);
+    throw new Error(INVALID_TYPE + ': expected formmated date');
   }
@@ -56,3 +56,3 @@ return c;
     if (Number.isNaN(c)) {
-      throw new Error(INVALID_TYPE);
+      throw new Error(INVALID_TYPE + ': expected numeric value');
     }
@@ -74,3 +74,3 @@ return c;
   }
-  throw new Error(INVALID_TYPE);
+  throw new Error(INVALID_TYPE + ': expected integer value');
 }
@@ -91,3 +91,3 @@
   }
-  throw new Error(INVALID_TYPE);
+  throw new Error(INVALID_TYPE + ': expected boolean value');
 }
@@ -94,0 +94,0 @@

dist/loader.js

@@ -15,4 +15,4 @@ 'use strict';
 var resolvePath = function () {
+  /* istanbul ignore next , in browser*/
   if (typeof document !== 'undefined') {
-    // in browser
     return function resolve(url) {
@@ -29,14 +29,2 @@ var div = document.createElement('div');
 
-function normalizeDataPackageUrl(datapackage) {
-  if (typeof datapackage === 'string') {
-    datapackage = { url: datapackage };
-  }
-  if (!datapackage.dataPackageJsonUrl) {
-    var url = datapackage.path || datapackage.url;
-    url = url.match(absURLRegEx) ? url : resolvePath(url);
-    datapackage = Object.assign(datapackage, identifier.parse(url));
-  }
-  return datapackage;
-}
-
 var Loader = function () {
@@ -53,7 +41,12 @@ function Loader(opts) {
       debug('Loading datapackage', _datapackage);
-      _datapackage = normalizeDataPackageUrl(_datapackage);
-      var dataPackageJsonUrl = _datapackage.dataPackageJsonUrl;
-      return this.fetch(dataPackageJsonUrl).catch(function (err) {
+      if (typeof _datapackage === 'string') {
+        _datapackage = { path: _datapackage };
+      }
+      var id = getIdentifier(_datapackage);
+      var url = id.dataPackageJsonUrl;
+      id.base = id.url;
+
+      return this.fetch(url).catch(function (err) {
         if (err.code === 'ENOENT') {
-          throw new Error('No DataPackage at path \'' + dataPackageJsonUrl + '\'');
+          throw new Error('No DataPackage at path \'' + url + '\'');
         }
@@ -63,3 +56,3 @@ /* istanbul ignore next */
       }).then(parse).then(function (res) {
-        return Object.assign(_datapackage, res, _datapackage);
+        return Object.assign(res, _datapackage, id);
       });
@@ -69,10 +62,8 @@ }
     key: 'resources',
-    value: function resources(datapackage) {
+    value: function resources(_resources) {
       var _this = this;
 
-      return Promise.all(datapackage.resources.map(function (r) {
+      return Promise.all(_resources.map(function (r) {
         return _this.resource(r);
-      })).then(function () {
-        return datapackage;
-      });
+      }));
     }
@@ -82,2 +73,3 @@ }, {
     value: function resource(_resource) {
+      _resource = Object.assign({}, _resource);
       debug('Loading resource', _resource);
@@ -103,2 +95,36 @@ if (!_resource.url) {
 
+Loader.id = getIdentifier;
+
+function getIdentifier(datapackage) {
+  var url = datapackage.path || datapackage.url;
+  url = url.match(absURLRegEx) ? url : resolvePath(url);
+  return identifier.parse(url);
+}
+
+/* function getDataPackageJsonUrl (datapackage) {
+  let url = '';
+  if (datapackage.dataPackageJsonUrl) {
+    return datapackage.dataPackageJsonUrl;
+  }
+  if (typeof datapackage === 'string') {
+    url = datapackage;
+  } else {
+    url = datapackage.path || datapackage.url;
+  }
+  url = url.match(absURLRegEx) ? url : resolvePath(url);
+  return identifier.parse(url).dataPackageJsonUrl;
+}
+
+function normalizeDataPackageUrl (datapackage) {
+  if (typeof datapackage === 'string') {
+    datapackage = {url: datapackage};
+  }
+  if (!datapackage.dataPackageJsonUrl) {
+    let url = datapackage.path || datapackage.url;
+    url = url.match(absURLRegEx) ? url : resolvePath(url);
+    datapackage = Object.assign(datapackage, identifier.parse(url));
+  }
+  return datapackage;
+} */
+
 module.exports = Loader;

dist/normalizer.js

@@ -9,2 +9,3 @@ 'use strict';
 var deepExtend = require('deep-extend');
+var cuid = require('cuid');
 
@@ -33,7 +34,8 @@ var Normalizer = function () {
         homepage: base,
-        description: ''
+        description: '',
+        schemas: {}
       }, _datapackage);
 
       ['image', 'readme'].forEach(function (key) {
-        if ({}.hasOwnProperty.call(normalized, key)) {
+        if (Object.prototype.hasOwnProperty.call(normalized, key)) {
           normalized[key] = urijs(normalized[key], base).href();
@@ -43,30 +45,13 @@ }
 
-      if (normalized.schemas) {
-        var schemas = normalized.schemas;
-        for (var key in schemas) {
-          if ({}.hasOwnProperty.call(schemas, key)) {
-            schemas[key].key = schemas[key].key || key;
-          }
-        }
-      }
-
       return normalized;
     }
 
-    // TODO: 1.0.0-beta.15: only one of url, path, data present
+    /* resources (datapackage) {
+      if (!Array.isArray(datapackage.resources)) {
+        return [];
+      }
+      return datapackage.resources.map(resource => this.resource(datapackage, resource));
+    } */
 
   }, {
-    key: 'resources',
-    value: function resources(datapackage) {
-      var _this = this;
-
-      if (datapackage.resources) {
-        datapackage.resources = datapackage.resources.map(function (resource) {
-          return _this.resource(datapackage, resource);
-        });
-      }
-      Normalizer.index(datapackage);
-      return datapackage;
-    }
-  }, {
     key: 'resource',
@@ -78,2 +63,4 @@ value: function resource(datapackage, _resource) {
 
+      _resource = deepExtend({}, _resource);
+
       if (_resource.path || _resource.url) {
@@ -88,2 +75,4 @@ var uri = urijs(_resource.path || _resource.url);
 
+      _resource.name = _resource.name || cuid();
+
       if (!_resource.format && !_resource.content && _resource.data) {
@@ -97,5 +86,11 @@ _resource.format = 'json';
 
-      if (_resource.schema && typeof _resource.schema === 'string') {
-        _resource.schema = datapackage.schemas[_resource.schema]; // TODO: check for URLS, catch missing schemas
-      }
+      /* if (resource.schema) {
+        if (typeof resource.schema === 'string') {
+          resource.schema = datapackage.schemas[resource.schema]; // TODO: check for URLS, catch missing schemas
+        } else {
+          // maybe bad, datapackage should be immutable here
+          datapackage.schemas = datapackage.schemas || {};
+          datapackage.schemas[`@@${resource.name}:schema`] = resource.schema;
+        }
+      } */
 
@@ -109,12 +104,14 @@ return _resource;
 
-Normalizer.index = function index(datapackage) {
-  datapackage.$resourcesByName = {};
+function getResourceIndex(datapackage) {
+  var $resourcesByName = {};
   datapackage.resources.forEach(function (r) {
     if (r.name) {
-      datapackage.$resourcesByName[r.name] = r;
+      $resourcesByName[r.name] = r;
     }
   });
-  return datapackage;
-};
+  return $resourcesByName;
+}
 
+Normalizer.index = getResourceIndex;
+
 module.exports = Normalizer;

dist/processor.js

@@ -23,7 +23,7 @@ 'use strict';
 
-      // todo: process schemas?
-      var resources = dataPackage.resources.map(function (resource) {
+      dataPackage = Object.assign({}, dataPackage);
+      dataPackage.resources = dataPackage.resources.map(function (resource) {
         return _this.resource(resource);
       });
-      return Object.assign(dataPackage, { resources: resources });
+      return dataPackage;
     }
@@ -33,12 +33,13 @@ }, {
     value: function resource(_resource) {
-      if (_resource.content) {
-        var translator = this.translators[_resource.mediatype];
+      var r = Object.assign({}, _resource);
+      if (r.content) {
+        var translator = this.translators[r.mediatype];
         if (translator) {
-          Object.assign(_resource, translator(_resource));
+          Object.assign(r, translator(r));
         }
       }
-      if (_resource.schema) {
-        Object.assign(_resource, this.schemaProcessor.process(_resource));
+      if (r.schema) {
+        return Object.assign(r, this.schemaProcessor.process(r));
       }
-      return _resource;
+      return r;
     }
@@ -45,0 +46,0 @@ }]);

dist/schema.js

@@ -23,17 +23,2 @@ 'use strict';
 
-function normalizeField(input) {
-  var r = Object.assign({
-    format: 'default',
-    missingValues: input.type === 'string' ? undefined : [''],
-    pattern: null
-  }, input);
-  r.missingValues = Array.isArray(r.missingValues) ? r.missingValues : [r.missingValues];
-  if (r.format.indexOf(':') !== -1) {
-    var s = r.format.split(':');
-    r.format = s[0];
-    r.pattern = s[1];
-  }
-  return r;
-}
-
 var Schema = function () {
@@ -48,14 +33,38 @@ function Schema(opts) {
   _createClass(Schema, [{
-    key: 'generate',
-    value: function generate(schema) {
+    key: 'normalizeField',
+    value: function normalizeField(field) {
+      var type = field.type || 'string';
+
+      field = Object.assign({
+        type: type,
+        format: 'default',
+        missingValues: type === 'string' ? undefined : [''],
+        pattern: null,
+        constraints: {}
+      }, field);
+
+      field.missingValues = Array.isArray(field.missingValues) ? field.missingValues : [field.missingValues];
+      field.title = field.title || readableName(field.name);
+
+      if (field.format.indexOf(':') !== -1) {
+        var s = field.format.split(':');
+        field.format = s[0];
+        field.pattern = s[1];
+      }
+
+      field.$fn = field.$fn || this.generateCastFn(field);
+      return field;
+    }
+  }, {
+    key: 'normalizeSchema',
+    value: function normalizeSchema(schema) {
       var _this = this;
 
-      var castMap = {};
-      schema.fields.forEach(function (field) {
-        var fn = _this.generateCastFn(field);
-        if (fn) {
-          castMap[field.name] = fn;
-        }
+      schema = Object.assign({
+        fields: []
+      }, schema);
+      schema.fields = schema.fields.map(function (f) {
+        return _this.normalizeField(f);
       });
-      return castMap;
+      return schema;
     }
@@ -65,3 +74,2 @@ }, {
     value: function generateCastFn(field) {
-      field = normalizeField(field);
       if (!field.type || !Object.prototype.hasOwnProperty.call(this.types, field.type)) {
@@ -83,17 +91,43 @@ return null;
     value: function process(resource) {
+      var self = this;
+
+      resource = Object.assign({}, resource);
       /* istanbul ignore if */
-      if (!Array.isArray(resource.data)) {
-        return { data: resource.data };
+      if (!Array.isArray(resource.data) || !resource.schema || !resource.schema.fields || resource.schema.fields.length === 0) {
+        return resource;
       }
 
-      var castMap = resource.schema.$castMap || (resource.schema.$castMap = this.generate(resource.schema));
+      resource.errors = resource.errors || [];
+      var fields = resource.schema.fields;
 
-      return { data: resource.data.map(function (d) {
-          var r = {};
-          for (var key in d) {
-            /* eslint guard-for-in: 0 */
-            r[key] = key in castMap ? castMap[key](d[key]) : d[key];
+      resource.data = resource.data.map(function (d, i) {
+        d = Object.assign({}, d);
+        fields.forEach(function (field) {
+          var key = field.name;
+          if (Object.prototype.hasOwnProperty.call(d, key)) {
+            var $fn = field.$fn || (field.$fn = self.generateCastFn(field));
+            try {
+              d[key] = $fn(d[key]);
+            } catch (err) {
+              resource.errors.push({
+                type: 'FieldMismatch',
+                code: 'InvalidType',
+                message: err.message,
+                row: i
+              });
+            }
+          } else if (field.constraints && field.constraints.required) {
+            resource.errors.push({
+              type: 'ConstraintsError',
+              code: 'MissingField',
+              message: 'Missing field: the field "' + key + '" requires a value',
+              row: i
+            });
           }
-          return r;
-        }) };
+        });
+
+        return d;
+      });
+
+      return resource;
     }
@@ -105,2 +139,31 @@ }]);
 
+function readableName(columnName) {
+  // adapted from readableColumnName: https://github.com/angular-ui/ui-grid/blob/master/src/js/core/services/ui-grid-util.js
+  if (typeof columnName === 'undefined' || columnName === undefined || columnName === null) {
+    return columnName;
+  }
+
+  if (typeof columnName !== 'string') {
+    columnName = String(columnName);
+  }
+
+  columnName = columnName.split('.').pop();
+
+  return columnName
+  // Convert underscores to spaces
+  .replace(/_+/g, ' ')
+  // Replace a completely all-capsed word with a first-letter-capitalized version
+  .replace(/^[A-Z]+$/, function (match) {
+    return (match.charAt(0).toUpperCase() + match.slice(1)).toLowerCase();
+  })
+  // Capitalize the first letter of words
+  .replace(/([\w\u00C0-\u017F]+)/g, function (match) {
+    return match.charAt(0).toUpperCase() + match.slice(1);
+  })
+  // Put a space in between words that have partial capilizations (i.e. 'firstName' becomes 'First Name')
+  // .replace(/([A-Z]|[A-Z]\w+)([A-Z])/g, "$1 $2");
+  // .replace(/(\w+?|\w)([A-Z])/g, "$1 $2");
+  .replace(/(\w+?(?=[A-Z]))/g, '$1 ');
+}
+
 module.exports = Schema;

dist/service.js

@@ -24,6 +24,6 @@ 'use strict';
     var mimeLookup = new MimeLookup(mimeDb);
-    var schemaProcessor = new SchemaProcessor({ types: types });
 
+    this.schemaProcessor = new SchemaProcessor({ types: types });
     this.normalize = new Normalizer({ mimeLookup: mimeLookup });
-    this.processor = new Processor({ translators: translators, schemaProcessor: schemaProcessor });
+    this.processor = new Processor({ translators: translators, schemaProcessor: this.schemaProcessor });
     this.loader = new Loader({ fetch: fetch });
@@ -35,3 +35,3 @@ }
     value: function normalizePackage(p) {
-      return this.normalize.datapackage(p);
+      return Object.assign(p, this.normalize.datapackage(p));
     }
@@ -41,3 +41,3 @@ }, {
     value: function normalizeResource(p, r) {
-      return this.normalize.resource(p, r);
+      return Object.assign(r, this.normalize.resource(p, r));
     }
@@ -47,5 +47,39 @@ }, {
     value: function normalizeResources(p) {
-      return this.normalize.resources(p);
+      var _this = this;
+
+      p.resources = Array.isArray(p.resources) ? p.resources.map(function (r) {
+        return _this.normalize.resource(p, r);
+      }) : [];
+      p.$resourcesByName = Normalizer.index(p);
+      return p;
     }
   }, {
+    key: 'normalizeSchemas',
+    value: function normalizeSchemas(p) {
+      var _this2 = this;
+
+      var schemas = p.schemas;
+
+      p.resources.forEach(function (r) {
+        if (r.schema) {
+          if (typeof r.schema === 'string') {
+            r.schema = schemas[r.schema]; // TODO: check for URLS, catch missing schemas
+          } else {
+            schemas['@@' + r.name + ':schema'] = r.schema;
+          }
+        }
+      });
+
+      for (var key in p.schemas) {
+        if (Object.prototype.hasOwnProperty.call(p.schemas, key)) {
+          p.schemas[key].key = p.schemas[key].key || key;
+          p.schemas[key].fields = p.schemas[key].fields.map(function (f) {
+            return _this2.schemaProcessor.normalizeField(f);
+          });
+        }
+      }
+
+      return p;
+    }
+  }, {
     key: 'loadPackage',
@@ -56,7 +90,41 @@ value: function loadPackage(p) {
   }, {
+    key: 'loadResources',
+    value: function loadResources(p) {
+      return this.loader.resources(p.resources).then(function (r) {
+        p.resources = r;
+        return p;
+      });
+    }
+  }, {
     key: 'processResource',
     value: function processResource(r) {
-      return this.processor.resource(r);
+      try {
+        return Object.assign(r, this.processor.resource(r));
+      } catch (err) {
+        return Object.assign(r, { $valid: false, $error: err });
+      }
     }
   }, {
+    key: 'processPackage',
+    value: function processPackage(p) {
+      Object.assign(p, this.processor.datapackage(p));
+      p.$resourcesByName = Normalizer.index(p);
+      return p;
+    }
+  }, {
+    key: 'updateResource',
+    value: function updateResource(p, r) {
+      Object.assign(r, this.normalize.resource(p, r));
+      p.$resourcesByName = Normalizer.index(p);
+      return Object.assign(r, this.processor.resource(r));
+    }
+  }, {
+    key: 'addResource',
+    value: function addResource(p, r) {
+      p.resources.push(r);
+      Object.assign(r, this.normalize.resource(p, r));
+      p.$resourcesByName = Normalizer.index(p);
+      return Object.assign(r, this.processor.resource(r));
+    }
+  }, {
     key: 'load',
@@ -67,9 +135,11 @@ value: function load(datapackage) {
       return self.loader.datapackage(datapackage).then(function (p) {
-        return self.normalize.datapackage(p);
+        return self.normalizePackage(p);
       }).then(function (p) {
-        return self.normalize.resources(p);
+        return self.normalizeResources(p);
       }).then(function (p) {
-        return self.loader.resources(p);
+        return self.normalizeSchemas(p);
       }).then(function (p) {
-        return self.processor.datapackage(p);
+        return self.loadResources(p);
+      }).then(function (p) {
+        return self.processPackage(p);
       });
@@ -76,0 +146,0 @@ }

package.json

 {
   "name": "chi-datapackage",
-  "version": "2.2.0",
+  "version": "3.0.0",
   "description": "Normalize datapackage and datapackage resources",
@@ -24,2 +24,3 @@ "main": "index.js",
     "eslint-plugin-node": "^2.0.0",
+    "nock": "^8.0.0",
     "nyc": "^8.1.0",
@@ -75,2 +76,3 @@ "xo": "^0.16.0"
     "crlf-helper": "^0.1.0",
+    "cuid": "^1.3.8",
     "d3-time-format": "^2.0.2",
@@ -77,0 +79,0 @@ "datapackage-identifier": "^0.4.1",

src/lib/types.js

@@ -6,3 +6,3 @@ const d3time = require('d3-time-format');
 
-const INVALID_TYPE = 'chi-datapackage: Invalid type';
+const INVALID_TYPE = 'Invalid type';
 
@@ -13,3 +13,3 @@ function jsonParse (isArray) {
     if (Array.isArray(c) !== isArray) {
-      throw new Error(INVALID_TYPE);
+      throw new Error(`${INVALID_TYPE}: expected JSON Array`);
     }
@@ -25,3 +25,3 @@ return c;
     if (Number.isNaN(c) || c === null) {
-      throw new Error(INVALID_TYPE);
+      throw new Error(`${INVALID_TYPE}: expected ${fmt} formmated date`);
     }
@@ -35,3 +35,3 @@ return c;
   if (isNaN(c.getTime())) {
-    throw new Error(INVALID_TYPE);
+    throw new Error(`${INVALID_TYPE}: expected formmated date`);
   }
@@ -53,3 +53,3 @@ return c;
   if (Number.isNaN(c)) {
-    throw new Error(INVALID_TYPE);
+    throw new Error(`${INVALID_TYPE}: expected numeric value`);
   }
@@ -70,3 +70,3 @@ return c;
   }
-  throw new Error(INVALID_TYPE);
+  throw new Error(`${INVALID_TYPE}: expected integer value`);
 }
@@ -87,3 +87,3 @@
   }
-  throw new Error(INVALID_TYPE);
+  throw new Error(`${INVALID_TYPE}: expected boolean value`);
 }
@@ -90,0 +90,0 @@

src/loader.js

@@ -11,4 +11,4 @@ 'use strict';
 const resolvePath = (() => {
+  /* istanbul ignore next , in browser*/
   if (typeof document !== 'undefined') {
-    // in browser
     return function resolve (url) {
@@ -25,14 +25,2 @@ const div = document.createElement('div');
 
-function normalizeDataPackageUrl (datapackage) {
-  if (typeof datapackage === 'string') {
-    datapackage = {url: datapackage};
-  }
-  if (!datapackage.dataPackageJsonUrl) {
-    let url = datapackage.path || datapackage.url;
-    url = url.match(absURLRegEx) ? url : resolvePath(url);
-    datapackage = Object.assign(datapackage, identifier.parse(url));
-  }
-  return datapackage;
-}
-
 class Loader {
@@ -45,9 +33,14 @@ constructor (opts) {
     debug('Loading datapackage', datapackage);
-    datapackage = normalizeDataPackageUrl(datapackage);
-    const dataPackageJsonUrl = datapackage.dataPackageJsonUrl;
+    if (typeof datapackage === 'string') {
+      datapackage = {path: datapackage};
+    }
+    const id = getIdentifier(datapackage);
+    const url = id.dataPackageJsonUrl;
+    id.base = id.url;
+
     return this
-      .fetch(dataPackageJsonUrl)
+      .fetch(url)
       .catch(err => {
         if (err.code === 'ENOENT') {
-          throw new Error(`No DataPackage at path '${dataPackageJsonUrl}'`);
+          throw new Error(`No DataPackage at path '${url}'`);
         }
@@ -58,12 +51,12 @@ /* istanbul ignore next */
       .then(parse)
-      .then(res => Object.assign(datapackage, res, datapackage));
+      .then(res => Object.assign(res, datapackage, id));
   }
 
-  resources (datapackage) {
+  resources (resources) {
     return Promise
-      .all(datapackage.resources.map(r => this.resource(r)))
-      .then(() => datapackage);
+      .all(resources.map(r => this.resource(r)));
   }
 
   resource (resource) {
+    resource = Object.assign({}, resource);
     debug('Loading resource', resource);
@@ -89,2 +82,36 @@ if (!resource.url) {
 
+Loader.id = getIdentifier;
+
+function getIdentifier (datapackage) {
+  let url = datapackage.path || datapackage.url;
+  url = url.match(absURLRegEx) ? url : resolvePath(url);
+  return identifier.parse(url);
+}
+
+/* function getDataPackageJsonUrl (datapackage) {
+  let url = '';
+  if (datapackage.dataPackageJsonUrl) {
+    return datapackage.dataPackageJsonUrl;
+  }
+  if (typeof datapackage === 'string') {
+    url = datapackage;
+  } else {
+    url = datapackage.path || datapackage.url;
+  }
+  url = url.match(absURLRegEx) ? url : resolvePath(url);
+  return identifier.parse(url).dataPackageJsonUrl;
+}
+
+function normalizeDataPackageUrl (datapackage) {
+  if (typeof datapackage === 'string') {
+    datapackage = {url: datapackage};
+  }
+  if (!datapackage.dataPackageJsonUrl) {
+    let url = datapackage.path || datapackage.url;
+    url = url.match(absURLRegEx) ? url : resolvePath(url);
+    datapackage = Object.assign(datapackage, identifier.parse(url));
+  }
+  return datapackage;
+} */
+
 module.exports = Loader;

src/normalizer.js

@@ -5,2 +5,3 @@ 'use strict';
 const deepExtend = require('deep-extend');
+const cuid = require('cuid');
 
@@ -25,7 +26,8 @@ class Normalizer {
       homepage: base,
-      description: ''
+      description: '',
+      schemas: {}
     }, datapackage);
 
     ['image', 'readme'].forEach(key => {
-      if ({}.hasOwnProperty.call(normalized, key)) {
+      if (Object.prototype.hasOwnProperty.call(normalized, key)) {
         normalized[key] = urijs(normalized[key], base).href();
@@ -35,22 +37,11 @@ }
 
-    if (normalized.schemas) {
-      const schemas = normalized.schemas;
-      for (const key in schemas) {
-        if ({}.hasOwnProperty.call(schemas, key)) {
-          schemas[key].key = schemas[key].key || key;
-        }
-      }
-    }
-
     return normalized;
   }
 
-  // TODO: 1.0.0-beta.15: only one of url, path, data present
-  resources (datapackage) {
-    if (datapackage.resources) {
-      datapackage.resources = datapackage.resources.map(resource => this.resource(datapackage, resource));
+  /* resources (datapackage) {
+    if (!Array.isArray(datapackage.resources)) {
+      return [];
     }
-    Normalizer.index(datapackage);
-    return datapackage;
-  }
+    return datapackage.resources.map(resource => this.resource(datapackage, resource));
+  } */
 
@@ -62,2 +53,4 @@ resource (datapackage, resource) {
 
+    resource = deepExtend({}, resource);
+
     if (resource.path || resource.url) {
@@ -72,2 +65,4 @@ const uri = urijs(resource.path || resource.url);
 
+    resource.name = resource.name || cuid();
+
     if (!resource.format && !resource.content && resource.data) {
@@ -81,5 +76,11 @@ resource.format = 'json';
 
-    if (resource.schema && typeof resource.schema === 'string') {
-      resource.schema = datapackage.schemas[resource.schema]; // TODO: check for URLS, catch missing schemas
-    }
+    /* if (resource.schema) {
+      if (typeof resource.schema === 'string') {
+        resource.schema = datapackage.schemas[resource.schema]; // TODO: check for URLS, catch missing schemas
+      } else {
+        // maybe bad, datapackage should be immutable here
+        datapackage.schemas = datapackage.schemas || {};
+        datapackage.schemas[`@@${resource.name}:schema`] = resource.schema;
+      }
+    } */
 
@@ -90,12 +91,14 @@ return resource;
 
-Normalizer.index = function index (datapackage) {
-  datapackage.$resourcesByName = {};
+function getResourceIndex (datapackage) {
+  const $resourcesByName = {};
   datapackage.resources.forEach(r => {
     if (r.name) {
-      datapackage.$resourcesByName[r.name] = r;
+      $resourcesByName[r.name] = r;
     }
   });
-  return datapackage;
-};
+  return $resourcesByName;
+}
 
+Normalizer.index = getResourceIndex;
+
 module.exports = Normalizer;

src/processor.js

@@ -13,18 +13,19 @@ 'use strict';
   datapackage (dataPackage) {
-    // todo: process schemas?
-    const resources = dataPackage.resources.map(resource => this.resource(resource));
-    return Object.assign(dataPackage, {resources});
+    dataPackage = Object.assign({}, dataPackage);
+    dataPackage.resources = dataPackage.resources.map(resource => this.resource(resource));
+    return dataPackage;
   }
 
   resource (resource) {
-    if (resource.content) {
-      const translator = this.translators[resource.mediatype];
+    const r = Object.assign({}, resource);
+    if (r.content) {
+      const translator = this.translators[r.mediatype];
       if (translator) {
-        Object.assign(resource, translator(resource));
+        Object.assign(r, translator(r));
       }
     }
-    if (resource.schema) {
-      Object.assign(resource, this.schemaProcessor.process(resource));
+    if (r.schema) {
+      return Object.assign(r, this.schemaProcessor.process(r));
     }
-    return resource;
+    return r;
   }
@@ -31,0 +32,0 @@ }

src/schema.js

@@ -19,17 +19,2 @@ 'use strict';
 
-function normalizeField (input) {
-  const r = Object.assign({
-    format: 'default',
-    missingValues: input.type === 'string' ? undefined : [''],
-    pattern: null
-  }, input);
-  r.missingValues = Array.isArray(r.missingValues) ? r.missingValues : [r.missingValues];
-  if (r.format.indexOf(':') !== -1) {
-    const s = r.format.split(':');
-    r.format = s[0];
-    r.pattern = s[1];
-  }
-  return r;
-}
-
 class Schema {
@@ -41,15 +26,35 @@ constructor (opts) {
 
-  generate (schema) {
-    const castMap = {};
-    schema.fields.forEach(field => {
-      const fn = this.generateCastFn(field);
-      if (fn) {
-        castMap[field.name] = fn;
-      }
-    });
-    return castMap;
+  normalizeField (field) {
+    const type = field.type || 'string';
+
+    field = Object.assign({
+      type,
+      format: 'default',
+      missingValues: type === 'string' ? undefined : [''],
+      pattern: null,
+      constraints: {}
+    }, field);
+
+    field.missingValues = Array.isArray(field.missingValues) ? field.missingValues : [field.missingValues];
+    field.title = field.title || readableName(field.name);
+
+    if (field.format.indexOf(':') !== -1) {
+      const s = field.format.split(':');
+      field.format = s[0];
+      field.pattern = s[1];
+    }
+
+    field.$fn = field.$fn || this.generateCastFn(field);
+    return field;
   }
 
+  normalizeSchema (schema) {
+    schema = Object.assign({
+      fields: []
+    }, schema);
+    schema.fields = schema.fields.map(f => this.normalizeField(f));
+    return schema;
+  }
+
   generateCastFn (field) {
-    field = normalizeField(field);
     if (!field.type || !Object.prototype.hasOwnProperty.call(this.types, field.type)) {
@@ -70,19 +75,75 @@ return null;
   process (resource) {
+    const self = this;
+
+    resource = Object.assign({}, resource);
     /* istanbul ignore if */
-    if (!Array.isArray(resource.data)) {
-      return {data: resource.data};
+    if (!Array.isArray(resource.data) || !resource.schema || !resource.schema.fields || resource.schema.fields.length === 0) {
+      return resource;
     }
 
-    const castMap = resource.schema.$castMap || (resource.schema.$castMap = this.generate(resource.schema));
+    resource.errors = resource.errors || [];
+    const fields = resource.schema.fields;
 
-    return {data: resource.data.map(d => {
-      const r = {};
-      for (const key in d) { /* eslint guard-for-in: 0 */
-        r[key] = (key in castMap) ? castMap[key](d[key]) : d[key];
-      }
-      return r;
-    })};
+    resource.data = resource.data.map((d, i) => {
+      d = Object.assign({}, d);
+      fields.forEach(field => {
+        const key = field.name;
+        if (Object.prototype.hasOwnProperty.call(d, key)) {
+          const $fn = field.$fn || (field.$fn = self.generateCastFn(field));
+          try {
+            d[key] = $fn(d[key]);
+          } catch (err) {
+            resource.errors.push({
+              type: 'FieldMismatch',
+              code: 'InvalidType',
+              message: err.message,
+              row: i
+            });
+          }
+        } else if (field.constraints && field.constraints.required) {
+          resource.errors.push({
+            type: 'ConstraintsError',
+            code: 'MissingField',
+            message: `Missing field: the field "${key}" requires a value`,
+            row: i
+          });
+        }
+      });
+
+      return d;
+    });
+
+    return resource;
   }
 }
 
+function readableName (columnName) {
+  // adapted from readableColumnName: https://github.com/angular-ui/ui-grid/blob/master/src/js/core/services/ui-grid-util.js
+  if (typeof columnName === 'undefined' || columnName === undefined || columnName === null) {
+    return columnName;
+  }
+
+  if (typeof columnName !== 'string') {
+    columnName = String(columnName);
+  }
+
+  columnName = columnName.split('.').pop();
+
+  return columnName
+    // Convert underscores to spaces
+    .replace(/_+/g, ' ')
+    // Replace a completely all-capsed word with a first-letter-capitalized version
+    .replace(/^[A-Z]+$/, match => {
+      return (match.charAt(0).toUpperCase() + match.slice(1)).toLowerCase();
+    })
+    // Capitalize the first letter of words
+    .replace(/([\w\u00C0-\u017F]+)/g, match => {
+      return match.charAt(0).toUpperCase() + match.slice(1);
+    })
+    // Put a space in between words that have partial capilizations (i.e. 'firstName' becomes 'First Name')
+    // .replace(/([A-Z]|[A-Z]\w+)([A-Z])/g, "$1 $2");
+    // .replace(/(\w+?|\w)([A-Z])/g, "$1 $2");
+    .replace(/(\w+?(?=[A-Z]))/g, '$1 ');
+}
+
 module.exports = Schema;

src/service.js

@@ -18,6 +18,6 @@ 'use strict';
     const mimeLookup = new MimeLookup(mimeDb);
-    const schemaProcessor = new SchemaProcessor({types});
 
+    this.schemaProcessor = new SchemaProcessor({types});
     this.normalize = new Normalizer({mimeLookup});
-    this.processor = new Processor({translators, schemaProcessor});
+    this.processor = new Processor({translators, schemaProcessor: this.schemaProcessor});
     this.loader = new Loader({fetch});
@@ -27,13 +27,38 @@ }
   normalizePackage (p) {
-    return this.normalize.datapackage(p);
+    return Object.assign(p, this.normalize.datapackage(p));
   }
 
   normalizeResource (p, r) {
-    return this.normalize.resource(p, r);
+    return Object.assign(r, this.normalize.resource(p, r));
   }
 
   normalizeResources (p) {
-    return this.normalize.resources(p);
+    p.resources = (Array.isArray(p.resources)) ? p.resources.map(r => this.normalize.resource(p, r)) : [];
+    p.$resourcesByName = Normalizer.index(p);
+    return p;
   }
 
+  normalizeSchemas (p) {
+    const schemas = p.schemas;
+
+    p.resources.forEach(r => {
+      if (r.schema) {
+        if (typeof r.schema === 'string') {
+          r.schema = schemas[r.schema]; // TODO: check for URLS, catch missing schemas
+        } else {
+          schemas[`@@${r.name}:schema`] = r.schema;
+        }
+      }
+    });
+
+    for (const key in p.schemas) {
+      if (Object.prototype.hasOwnProperty.call(p.schemas, key)) {
+        p.schemas[key].key = p.schemas[key].key || key;
+        p.schemas[key].fields = p.schemas[key].fields.map(f => this.schemaProcessor.normalizeField(f));
+      }
+    }
+
+    return p;
+  }
+
   loadPackage (p) {
@@ -43,6 +68,37 @@ return this.loader.datapackage(p);
 
+  loadResources (p) {
+    return this.loader.resources(p.resources)
+      .then(r => {
+        p.resources = r;
+        return p;
+      });
+  }
+
   processResource (r) {
-    return this.processor.resource(r);
+    try {
+      return Object.assign(r, this.processor.resource(r));
+    } catch (err) {
+      return Object.assign(r, {$valid: false, $error: err});
+    }
   }
 
+  processPackage (p) {
+    Object.assign(p, this.processor.datapackage(p));
+    p.$resourcesByName = Normalizer.index(p);
+    return p;
+  }
+
+  updateResource (p, r) {
+    Object.assign(r, this.normalize.resource(p, r));
+    p.$resourcesByName = Normalizer.index(p);
+    return Object.assign(r, this.processor.resource(r));
+  }
+
+  addResource (p, r) {
+    p.resources.push(r);
+    Object.assign(r, this.normalize.resource(p, r));
+    p.$resourcesByName = Normalizer.index(p);
+    return Object.assign(r, this.processor.resource(r));
+  }
+
   load (datapackage) {
@@ -52,6 +108,7 @@ const self = this;
     return self.loader.datapackage(datapackage)
-      .then(p => self.normalize.datapackage(p))
-      .then(p => self.normalize.resources(p))
-      .then(p => self.loader.resources(p))
-      .then(p => self.processor.datapackage(p));
+      .then(p => self.normalizePackage(p))
+      .then(p => self.normalizeResources(p))
+      .then(p => self.normalizeSchemas(p))
+      .then(p => self.loadResources(p))
+      .then(p => self.processPackage(p));
   }
@@ -58,0 +115,0 @@ }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

52553

increased by23.4%

Lines of code

1405

increased by22.81%

Worsened metrics

Dependency count

13

increased by8.33%

Dev dependency count

7

increased by16.67%

Dependency changes

• + Addedcuid@^1.3.8

• + Addedbrowser-fingerprint@0.0.1(transitive)

• + Addedcore-js@1.2.7(transitive)

• + Addedcuid@1.3.8(transitive)

• + Addednode-fingerprint@0.0.2(transitive)