chi-datapackage - npm Package Compare versions

Comparing version 2.1.1 to 2.2.0

CHANGELOG.md

@@ -9,2 +9,7 @@ CHANGELOG
 
+## 2.2.0 (2016-08-15)
+* add support for missingValues
+* Support NaN, INF, and -INF in number and integer types
+* Types are now strict
+
 ## 2.1.1 (2016-08-09)
@@ -11,0 +16,0 @@ * Fixed bug in DataPackageService:normalizeResource

dist/lib/types.js

@@ -6,6 +6,84 @@ 'use strict';
 
-var jsonParse = require('./json');
+var parse = require('json5').parse;
 
+var INVALID_TYPE = 'chi-datapackage: Invalid type';
+
+function jsonParse(isArray) {
+  return function (d) {
+    var c = parse(d);
+    if (Array.isArray(c) !== isArray) {
+      throw new Error(INVALID_TYPE);
+    }
+    return c;
+  };
+}
+
+function utcParse(fmt) {
+  var fn = d3time.utcParse(fmt);
+  return function (d) {
+    var c = fn(d);
+    if (Number.isNaN(c) || c === null) {
+      throw new Error(INVALID_TYPE);
+    }
+    return c;
+  };
+}
+
+function dateParse(d) {
+  var c = new Date(d);
+  if (isNaN(c.getTime())) {
+    throw new Error(INVALID_TYPE);
+  }
+  return c;
+}
+
+var checkSpecialNumbers = function checkSpecialNumbers(fn) {
+  return function (d) {
+    if (d === 'INF' || d === Infinity) {
+      return Infinity;
+    }
+    if (d === '-INF' || d === -Infinity) {
+      return -Infinity;
+    }
+    if (d === 'NaN' || Number.isNaN(d)) {
+      return NaN;
+    }
+    var c = fn(d);
+    if (Number.isNaN(c)) {
+      throw new Error(INVALID_TYPE);
+    }
+    return c;
+  };
+};
+
+function castNumber(value) {
+  if (typeof value === 'number') {
+    return value;
+  }
+  return Number(value);
+}
+
+function castInt(value) {
+  if (/^(\-|\+)?([0-9]+|Infinity)$/.test(String(value))) {
+    return castNumber(value);
+  }
+  throw new Error(INVALID_TYPE);
+}
+
 var TRUE_VALUES = ['yes', 'y', 'true', 't', '1'];
+var FALSE_VALUES = ['no', 'n', 'false', 'f', '0'];
 
+function castBoolean(value) {
+  if (typeof value === 'boolean') {
+    return value;
+  }
+  if (TRUE_VALUES.indexOf(String(value).toLowerCase()) !== -1) {
+    return true;
+  }
+  if (FALSE_VALUES.indexOf(String(value).toLowerCase()) !== -1) {
+    return false;
+  }
+  throw new Error(INVALID_TYPE);
+}
+
 var typeToCast = {
@@ -17,29 +95,23 @@ string: {
   integer: {
-    default: parseInt
+    default: checkSpecialNumbers(castInt)
   },
   number: {
-    default: parseFloat
+    default: checkSpecialNumbers(castNumber)
     // todo: currency?
   },
   datetime: {
-    default: d3time.utcParse('%Y-%m-%dT%H:%M:%SZ'),
-    fmt: d3time.utcParse,
-    any: function any(d) {
-      return new Date(d);
-    }
+    default: utcParse('%Y-%m-%dT%H:%M:%SZ'),
+    fmt: utcParse,
+    any: dateParse
   },
   date: {
-    default: d3time.utcParse('%Y-%m-%d'),
-    fmt: d3time.utcParse,
-    yyyy: d3time.utcParse('%Y'),
-    any: function any(d) {
-      return new Date(d);
-    }
+    default: utcParse('%Y-%m-%d'),
+    fmt: utcParse,
+    yyyy: utcParse('%Y'),
+    any: dateParse
   },
   time: {
-    default: d3time.utcParse('%H:%M:%S'),
-    fmt: d3time.utcParse,
-    any: function any(d) {
-      return new Date(d);
-    }
+    default: utcParse('%H:%M:%S'),
+    fmt: utcParse,
+    any: dateParse
   },
@@ -50,13 +122,14 @@ duration: {
   boolean: {
-    default: function _default(d) {
-      return d === true || TRUE_VALUES.indexOf(String(d).toLowerCase()) !== -1;
-    }
+    default: castBoolean
   },
   object: {
-    default: jsonParse
+    default: jsonParse(false),
+    any: parse
+  },
+  array: {
+    default: jsonParse(true),
+    any: parse
   }
 };
 
-typeToCast.array = typeToCast.object;
-
 module.exports = typeToCast;

dist/normalizer.js

@@ -52,2 +52,5 @@ 'use strict';
     }
+
+    // TODO: 1.0.0-beta.15: only one of url, path, data present
+
   }, {
@@ -54,0 +57,0 @@ key: 'resources',

dist/schema.js

@@ -14,2 +14,26 @@ 'use strict';
 
+function nullCheck(mv, fn) {
+  return function (d) {
+    if (mv.indexOf(d) !== -1) {
+      return null;
+    }
+    return fn(d);
+  };
+}
+
+function normalizeField(input) {
+  var r = Object.assign({
+    format: 'default',
+    missingValues: input.type === 'string' ? undefined : [''],
+    pattern: null
+  }, input);
+  r.missingValues = Array.isArray(r.missingValues) ? r.missingValues : [r.missingValues];
+  if (r.format.indexOf(':') !== -1) {
+    var s = r.format.split(':');
+    r.format = s[0];
+    r.pattern = s[1];
+  }
+  return r;
+}
+
 var Schema = function () {
@@ -40,2 +64,3 @@ function Schema(opts) {
     value: function generateCastFn(field) {
+      field = normalizeField(field);
       if (!field.type || !Object.prototype.hasOwnProperty.call(this.types, field.type)) {
@@ -45,11 +70,10 @@ return null;
       var type = this.types[field.type];
-      var format = field.format || 'default';
-      var pattern = null;
-      if (format.indexOf(':') !== -1) {
-        var s = format.split(':');
-        format = s[0];
-        pattern = s[1];
+      var map = type[field.format];
+      if (field.pattern) {
+        map = map(field.pattern);
       }
-      var map = type[format] || type.default;
-      return pattern ? map(pattern) : map;
+      if (field.missingValues) {
+        return nullCheck(field.missingValues, map);
+      }
+      return map;
     }
@@ -59,6 +83,5 @@ }, {
     value: function process(resource) {
-      var data = resource.data;
       /* istanbul ignore if */
-      if (!Array.isArray(data)) {
-        return { data: data };
+      if (!Array.isArray(resource.data)) {
+        return { data: resource.data };
       }
@@ -68,12 +91,10 @@
 
-      data = data.map(function (d) {
-        var r = {};
-        for (var key in d) {
-          /* eslint guard-for-in: 0 */
-          r[key] = key in castMap ? castMap[key](d[key]) : d[key];
-        }
-        return r;
-      });
-
-      return { data: data };
+      return { data: resource.data.map(function (d) {
+          var r = {};
+          for (var key in d) {
+            /* eslint guard-for-in: 0 */
+            r[key] = key in castMap ? castMap[key](d[key]) : d[key];
+          }
+          return r;
+        }) };
     }
@@ -80,0 +101,0 @@ }]);

package.json

 {
   "name": "chi-datapackage",
-  "version": "2.1.1",
+  "version": "2.2.0",
   "description": "Normalize datapackage and datapackage resources",
@@ -24,3 +24,3 @@ "main": "index.js",
     "eslint-plugin-node": "^2.0.0",
-    "nyc": "^7.1.0",
+    "nyc": "^8.1.0",
     "xo": "^0.16.0"
@@ -27,0 +27,0 @@ },

src/lib/types.js

 const d3time = require('d3-time-format');
 const parseIsoDuration = require('parse-iso-duration');
 
-const jsonParse = require('./json');
+const parse = require('json5').parse;
 
+const INVALID_TYPE = 'chi-datapackage: Invalid type';
+
+function jsonParse (isArray) {
+  return function (d) {
+    const c = parse(d);
+    if (Array.isArray(c) !== isArray) {
+      throw new Error(INVALID_TYPE);
+    }
+    return c;
+  };
+}
+
+function utcParse (fmt) {
+  const fn = d3time.utcParse(fmt);
+  return function (d) {
+    const c = fn(d);
+    if (Number.isNaN(c) || c === null) {
+      throw new Error(INVALID_TYPE);
+    }
+    return c;
+  };
+}
+
+function dateParse (d) {
+  const c = new Date(d);
+  if (isNaN(c.getTime())) {
+    throw new Error(INVALID_TYPE);
+  }
+  return c;
+}
+
+const checkSpecialNumbers = fn => d => {
+  if (d === 'INF' || d === Infinity) {
+    return Infinity;
+  }
+  if (d === '-INF' || d === -Infinity) {
+    return -Infinity;
+  }
+  if (d === 'NaN' || Number.isNaN(d)) {
+    return NaN;
+  }
+  const c = fn(d);
+  if (Number.isNaN(c)) {
+    throw new Error(INVALID_TYPE);
+  }
+  return c;
+};
+
+function castNumber (value) {
+  if (typeof value === 'number') {
+    return value;
+  }
+  return Number(value);
+}
+
+function castInt (value) {
+  if (/^(\-|\+)?([0-9]+|Infinity)$/.test(String(value))) {
+    return castNumber(value);
+  }
+  throw new Error(INVALID_TYPE);
+}
+
 const TRUE_VALUES = ['yes', 'y', 'true', 't', '1'];
+const FALSE_VALUES = ['no', 'n', 'false', 'f', '0'];
 
+function castBoolean (value) {
+  if (typeof value === 'boolean') {
+    return value;
+  }
+  if (TRUE_VALUES.indexOf(String(value).toLowerCase()) !== -1) {
+    return true;
+  }
+  if (FALSE_VALUES.indexOf(String(value).toLowerCase()) !== -1) {
+    return false;
+  }
+  throw new Error(INVALID_TYPE);
+}
+
 const typeToCast = {
@@ -14,23 +90,23 @@ string: {
   integer: {
-    default: parseInt
+    default: checkSpecialNumbers(castInt)
   },
   number: {
-    default: parseFloat
+    default: checkSpecialNumbers(castNumber)
     // todo: currency?
   },
   datetime: {
-    default: d3time.utcParse('%Y-%m-%dT%H:%M:%SZ'),
-    fmt: d3time.utcParse,
-    any: d => new Date(d)
+    default: utcParse('%Y-%m-%dT%H:%M:%SZ'),
+    fmt: utcParse,
+    any: dateParse
   },
   date: {
-    default: d3time.utcParse('%Y-%m-%d'),
-    fmt: d3time.utcParse,
-    yyyy: d3time.utcParse('%Y'),
-    any: d => new Date(d)
+    default: utcParse('%Y-%m-%d'),
+    fmt: utcParse,
+    yyyy: utcParse('%Y'),
+    any: dateParse
   },
   time: {
-    default: d3time.utcParse('%H:%M:%S'),
-    fmt: d3time.utcParse,
-    any: d => new Date(d)
+    default: utcParse('%H:%M:%S'),
+    fmt: utcParse,
+    any: dateParse
   },
@@ -41,11 +117,14 @@ duration: {
   boolean: {
-    default: d => d === true || TRUE_VALUES.indexOf(String(d).toLowerCase()) !== -1
+    default: castBoolean
   },
   object: {
-    default: jsonParse
+    default: jsonParse(false),
+    any: parse
+  },
+  array: {
+    default: jsonParse(true),
+    any: parse
   }
 };
 
-typeToCast.array = typeToCast.object;
-
 module.exports = typeToCast;

src/normalizer.js

@@ -45,2 +45,3 @@ 'use strict';
 
+  // TODO: 1.0.0-beta.15: only one of url, path, data present
   resources (datapackage) {
@@ -47,0 +48,0 @@ if (datapackage.resources) {

src/schema.js

@@ -10,2 +10,26 @@ 'use strict';
 
+function nullCheck (mv, fn) {
+  return d => {
+    if (mv.indexOf(d) !== -1) {
+      return null;
+    }
+    return fn(d);
+  };
+}
+
+function normalizeField (input) {
+  const r = Object.assign({
+    format: 'default',
+    missingValues: input.type === 'string' ? undefined : [''],
+    pattern: null
+  }, input);
+  r.missingValues = Array.isArray(r.missingValues) ? r.missingValues : [r.missingValues];
+  if (r.format.indexOf(':') !== -1) {
+    const s = r.format.split(':');
+    r.format = s[0];
+    r.pattern = s[1];
+  }
+  return r;
+}
+
 class Schema {
@@ -29,2 +53,3 @@ constructor (opts) {
   generateCastFn (field) {
+    field = normalizeField(field);
     if (!field.type || !Object.prototype.hasOwnProperty.call(this.types, field.type)) {
@@ -34,18 +59,16 @@ return null;
     const type = this.types[field.type];
-    let format = field.format || 'default';
-    let pattern = null;
-    if (format.indexOf(':') !== -1) {
-      const s = format.split(':');
-      format = s[0];
-      pattern = s[1];
+    let map = type[field.format];
+    if (field.pattern) {
+      map = map(field.pattern);
     }
-    const map = type[format] || type.default;
-    return pattern ? map(pattern) : map;
+    if (field.missingValues) {
+      return nullCheck(field.missingValues, map);
+    }
+    return map;
   }
 
   process (resource) {
-    let data = resource.data;
     /* istanbul ignore if */
-    if (!Array.isArray(data)) {
-      return {data};
+    if (!Array.isArray(resource.data)) {
+      return {data: resource.data};
     }
@@ -55,3 +78,3 @@
 
-    data = data.map(d => {
+    return {data: resource.data.map(d => {
       const r = {};
@@ -62,5 +85,3 @@ for (const key in d) { /* eslint guard-for-in: 0 */
       return r;
-    });
-
-    return {data};
+    })};
   }
@@ -67,0 +88,0 @@ }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

42589

increased by12.28%

Number of package files

27

increased by3.85%

Lines of code

1144

increased by18.67%

No dependency changes