littlefork - npm Package Compare versions

Comparing version 0.9.2 to 0.9.3

_dist/index.js

@@ -6,6 +6,10 @@ 'use strict';
 });
-exports.utils = exports.envelope = exports.units = exports.unit = exports.plugins = exports.runner = undefined;
+exports.utils = exports.envelope = exports.data = exports.queries = exports.plugins = exports.runner = undefined;
 
+var _fp = require('lodash/fp');
+
 var _runner = require('./runner');
 
+var _runner2 = _interopRequireDefault(_runner);
+
 var _plugins = require('./plugins');
@@ -15,6 +19,14 @@
 
-var _envelope = require('./envelope');
+var _envelope = require('./data/envelope');
 
 var _envelope2 = _interopRequireDefault(_envelope);
 
+var _homonyms = require('./data/homonyms');
+
+var _homonyms2 = _interopRequireDefault(_homonyms);
+
+var _data = require('./data/data');
+
+var _data2 = _interopRequireDefault(_data);
+
 var _utils = require('./utils');
@@ -24,25 +36,35 @@
 
-var _unit = require('./data/unit');
+var _hasher = require('./utils/hasher');
 
-var _unit2 = _interopRequireDefault(_unit);
+var _hasher2 = _interopRequireDefault(_hasher);
 
-var _units = require('./data/units');
+var _assertions = require('./utils/assertions');
 
-var _units2 = _interopRequireDefault(_units);
+var _assertions2 = _interopRequireDefault(_assertions);
 
+var _combinators = require('./utils/combinators');
+
+var _combinators2 = _interopRequireDefault(_combinators);
+
+var _fs = require('./utils/fs');
+
+var _fs2 = _interopRequireDefault(_fs);
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
-exports.runner = _runner.runner;
+var utils = (0, _fp.merge)(_utils2.default, { hasher: _hasher2.default, assertions: _assertions2.default, combinators: _combinators2.default, fs: _fs2.default });
+
+exports.runner = _runner2.default;
 exports.plugins = _plugins2.default;
-exports.unit = _unit2.default;
-exports.units = _units2.default;
+exports.queries = _homonyms2.default;
+exports.data = _data2.default;
 exports.envelope = _envelope2.default;
-exports.utils = _utils2.default;
+exports.utils = utils;
 exports.default = {
-  runner: _runner.runner,
+  runner: _runner2.default,
   plugins: _plugins2.default,
-  unit: _unit2.default,
-  units: _units2.default,
+  queries: _homonyms2.default,
+  data: _data2.default,
   envelope: _envelope2.default,
-  utils: _utils2.default
+  utils: utils
 };

_dist/runner.js

@@ -6,3 +6,2 @@ 'use strict';
 });
-exports.runner = undefined;
 
@@ -21,7 +20,7 @@ var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }();
 
-var _envelope = require('./envelope');
+var _envelope = require('./data/envelope');
 
-var _unit = require('./data/unit');
+var _data = require('./data/data');
 
-var _unit2 = _interopRequireDefault(_unit);
+var _data2 = _interopRequireDefault(_data);
 
@@ -32,4 +31,2 @@ var _utils = require('./utils');
 
-var _hasher = require('./utils/hasher');
-
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
@@ -54,17 +51,14 @@
 
-var marker = (0, _fp.curry)(function (label, envelope) {
-  var inject = function inject(u) {
-    return (0, _fp.flow)([(0, _fp.getOr)([], '_lf_markers'), (0, _fp.concat)([label]), _fp.uniq, function (ms) {
-      return (0, _fp.merge)(u, { _lf_markers: ms });
-    }])(u);
-  };
-  return (0, _envelope.fmapData)(inject, envelope);
+var mark = (0, _fp.curry)(function (marker, envelope) {
+  return (0, _envelope.fmapData)(_data2.default.concatOne({ _lf_markers: [marker] }), envelope);
 });
 
-var unitDefaults = (0, _fp.curry)(function (defaults, envelope) {
-  return (0, _envelope.fmapData)(_unit2.default.concat(defaults), envelope);
+var unitDefaults = (0, _envelope.fmapData)(_data2.default.concatOne(_data2.default.emptyOne()));
+
+var hashData = (0, _envelope.fmapData)(_data2.default.hashOne);
+
+var source = (0, _fp.curry)(function (name, envelope) {
+  return (0, _envelope.fmapData)(_data2.default.concatOne({ _lf_source: name }), envelope);
 });
 
-var hash = (0, _envelope.fmapData)(_hasher.hashUnit);
-
 /**
@@ -107,4 +101,3 @@ * Create a runable littlefork object.
  */
-var runner = exports.runner = (0, _fp.curry)(function (cfg, queries) {
-  // FIXME: How to handle missing modules?
+var runner = (0, _fp.curry)(function (cfg, queries) {
   var _flow = (0, _fp.flow)([_plugins.list, _plugins.load])(),
@@ -116,3 +109,3 @@ _flow2 = _slicedToArray(_flow, 2),
   var stream = _baconjs2.default.Bus();
-  var label = (0, _shortid.generate)();
+  var marker = (0, _shortid.generate)();
 
@@ -125,3 +118,3 @@ // Check that all dependencies are installed.
 
-  // Make sure we have all requrested plugins.
+  // Make sure we have all requested plugins.
   var missingPlugins = (0, _fp.flow)([_fp.keys, (0, _fp.difference)(cfg.plugins)])(plugins);
@@ -161,3 +154,3 @@ if (!(0, _fp.isEmpty)(missingPlugins)) {
 
-        return (0, _plugins.liftManyA2)([start(stream, name), plugin, unitDefaults((0, _utils.dataUnitDefaults)()), hash, marker(label), stats(stream, name), end(stream, name)], envelope, { log: log, cfg: cfg });
+        return (0, _plugins.liftManyA2)([start(stream, name), plugin, unitDefaults, hashData, source(name), mark(marker), stats(stream, name), end(stream, name)], envelope, { log: log, cfg: (0, _fp.merge)({ marker: marker }, cfg) });
       }, (0, _envelope.concatQueries)(queries, (0, _envelope.empty)()), pipeline).catch(function (e) {
@@ -171,5 +164,8 @@ return stream.error(e);
 
-  return [run, stream];
+  run.marker = marker;
+  run.stream = stream;
+
+  return run;
 });
 
-exports.default = { runner: runner };
+exports.default = runner;

_dist/utils/assertions.js

@@ -10,3 +10,3 @@ 'use strict';
 
-var _envelope = require('../envelope');
+var _envelope = require('../data/envelope');
 
@@ -13,0 +13,0 @@ // TODO: Deprecated. Better use a sort of validation.

_dist/utils/combinators.js

@@ -6,3 +6,3 @@ 'use strict';
 });
-exports.retry = exports.promisify = exports.reduceP = exports.mapP = exports.flowP = exports.composeP = undefined;
+exports.mapQueriesAsync = exports.retry = exports.promisify = exports.reduceP = exports.mapP = exports.flowP = exports.composeP = undefined;
 
@@ -15,2 +15,8 @@ var _fp = require('lodash/fp');
 
+var _envelope = require('../data/envelope');
+
+var _data = require('../data/data');
+
+var _data2 = _interopRequireDefault(_data);
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
@@ -75,2 +81,14 @@
 
+var mapQueriesAsync = exports.mapQueriesAsync = (0, _fp.curry)(function (f, source, envelope) {
+  var e = (0, _envelope.filterQueries)(function (_ref) {
+    var type = _ref.type;
+    return type === source;
+  }, envelope);
+  return mapP(function (q) {
+    return f(q.term).then(_data2.default.fmap(_data2.default.concatOne({ _lf_queries: [q] })));
+  }, e.queries).then(function (ds) {
+    return (0, _envelope.concatData)(_data2.default.fold(ds), envelope);
+  });
+});
+
 exports.default = {
@@ -82,3 +100,4 @@ composeP: composeP,
   promisify: promisify,
-  retry: retry
+  retry: retry,
+  mapQueriesAsync: mapQueriesAsync
 };

_dist/utils/fs.js

@@ -6,4 +6,6 @@ 'use strict';
 });
-exports.sha256sum = exports.mkdirP = undefined;
+exports.md5sum = exports.sha256sum = exports.mkdirP = undefined;
 
+var _fp = require('lodash/fp');
+
 var _bluebird = require('bluebird');
@@ -42,5 +44,5 @@
 
-var sha256sum = exports.sha256sum = function sha256sum(target) {
+var hashFile = (0, _fp.curry)(function (algorithm, target) {
   var fd = _fs2.default.createReadStream(target);
-  var hash = _crypto2.default.createHash('sha256');
+  var hash = _crypto2.default.createHash(algorithm);
   hash.setEncoding('hex');
@@ -57,7 +59,11 @@
   });
-};
+});
 
+var sha256sum = exports.sha256sum = hashFile('sha256');
+var md5sum = exports.md5sum = hashFile('md5');
+
 exports.default = {
   mkdirP: mkdirP,
-  sha256sum: sha256sum
+  sha256sum: sha256sum,
+  md5sum: md5sum
 };

_dist/utils/hasher.js

@@ -6,3 +6,3 @@ 'use strict';
 });
-exports.hashUnit = exports.hashMap = exports.hashWithFields = exports.hash = exports.hashKeys = exports.sha256 = undefined;
+exports.hashWithField = exports.hashKeys = exports.sha256 = undefined;
 
@@ -15,10 +15,2 @@ var _fp = require('lodash/fp');
 
-var _unit = require('../data/unit');
-
-var _unit2 = _interopRequireDefault(_unit);
-
-var _units = require('../data/units');
-
-var _units2 = _interopRequireDefault(_units);
-
 var _ = require('./');
@@ -28,4 +20,2 @@
 
-function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-
 var sha256 = exports.sha256 = function sha256(s) {
@@ -36,45 +26,13 @@ return _crypto2.default.createHash('sha256').update(s).digest('hex');
 var hashKeys = exports.hashKeys = (0, _fp.curry)(function (keys, u) {
-  if ((0, _fp.isEmpty)(keys)) {
-    return u;
-  }
   return (0, _fp.flow)([(0, _fp.pick)(keys), _fp.values, _.stringify, sha256])(u);
 });
 
-var hash = exports.hash = (0, _fp.curry)(function (name, keys, u) {
-  return _unit2.default.concat(u, _defineProperty({}, '_lf_' + name + '_hash', hashKeys(keys, u)));
+var hashWithField = exports.hashWithField = (0, _fp.curry)(function (field, u) {
+  return hashKeys(u[field], u);
 });
 
-var hashWithFields = exports.hashWithFields = (0, _fp.curry)(function (name, u) {
-  if (!u['_lf_' + name + '_fields']) {
-    return u;
-  }
-  return hash(name, u['_lf_' + name + '_fields'], u);
-});
-
-var hashHref = hash('id', ['href']);
-var hashTypedHref = hash('id', ['type', 'href']);
-var hashTypedTerm = hash('id', ['type', 'term']);
-
-var hashMap = exports.hashMap = (0, _fp.curry)(function (hasher, field, u) {
-  if (!u[field] || (0, _fp.isEmpty)(u[field])) {
-    return u;
-  }
-  return _unit2.default.concat(u, _defineProperty({}, field, _units2.default.fmap(hasher, u[field])));
-});
-
-var hashId = hashWithFields('id');
-var hashContent = hashWithFields('content');
-var hashLinks = hashMap(hashHref, '_lf_links');
-var hashImages = hashMap(hashTypedHref, '_lf_images');
-var hashDownloads = hashMap(hashTypedHref, '_lf_downloads');
-var hashRelations = hashMap(hashTypedTerm, '_lf_relations');
-
-var hashUnit = exports.hashUnit = (0, _fp.flow)([hashId, hashContent, hashLinks, hashImages, hashDownloads, hashRelations]);
-
 exports.default = {
   sha256: sha256,
-  hash: hash,
-  hashWithFields: hashWithFields,
-  hashMap: hashMap,
-  hashUnit: hashUnit
+  hashKeys: hashKeys,
+  hashWithField: hashWithField
 };

_dist/utils/index.js

@@ -6,3 +6,3 @@ 'use strict';
 });
-exports.stringify = exports.printf = exports.tap = exports.defaultFields = exports.dataUnitDefaults = exports.now = undefined;
+exports.deepConcatWith = exports.arrayify = exports.stringify = exports.printf = exports.tap = exports.now = undefined;
 
@@ -17,24 +17,4 @@ var _fp = require('lodash/fp');
 
-var _hasher = require('./hasher');
-
-var _hasher2 = _interopRequireDefault(_hasher);
-
-var _assertions = require('./assertions');
-
-var _assertions2 = _interopRequireDefault(_assertions);
-
-var _combinators = require('./combinators');
-
-var _combinators2 = _interopRequireDefault(_combinators);
-
-var _fs = require('./fs');
-
-var _fs2 = _interopRequireDefault(_fs);
-
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
-function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-
-var reduceObj = _fp.reduce.convert({ cap: false });
-
 var now = exports.now = function now() {
@@ -44,19 +24,2 @@ return _moment2.default.utc().toDate();
 
-var dataUnitDefaults = exports.dataUnitDefaults = function dataUnitDefaults() {
-  return (0, _fp.merge)({}, {
-    _lf_links: [],
-    _lf_media: [],
-    _lf_pubdates: { fetch: now() },
-    _lf_relations: [],
-    _lf_downloads: []
-  });
-};
-
-// we make sure that we have a set of default fields set.
-var defaultFields = exports.defaultFields = function defaultFields(unit) {
-  return reduceObj(function (memo, v, k) {
-    return memo[k] ? memo : (0, _fp.merge)(memo, _defineProperty({}, k, v));
-  }, unit, dataUnitDefaults);
-};
-
 var tap = exports.tap = (0, _fp.curry)(function (f, x) {
@@ -82,12 +45,25 @@ f((0, _fp.cloneDeep)(x));
 
+var arrayify = exports.arrayify = function arrayify(a) {
+  if ((0, _fp.isArray)(a)) {
+    return a;
+  }
+  return [a];
+};
+
+var deepConcatWith = exports.deepConcatWith = (0, _fp.curry)(function (equals, merger, xs, ys) {
+  return (0, _fp.reduce)(function (memo, u) {
+    var i = (0, _fp.findIndex)(equals(u), memo);
+    if (i === -1) return (0, _fp.concat)(memo, u);
+    memo[i] = merger(memo[i], u); // eslint-disable-line no-param-reassign
+    return memo;
+  }, [], (0, _fp.concat)(xs, ys));
+});
+
 exports.default = {
-  hasher: _hasher2.default,
-  assertions: _assertions2.default,
-  combinators: _combinators2.default,
-  fs: _fs2.default,
-  defaultFields: defaultFields,
   now: now,
   tap: tap,
   printf: printf,
-  stringify: stringify
+  stringify: stringify,
+  arrayify: arrayify,
+  deepConcatWith: deepConcatWith
 };

package.json

 {
   "name": "littlefork",
-  "version": "0.9.2",
+  "version": "0.9.3",
   "description": "A sequential data processing pipeline.",
@@ -5,0 +5,0 @@ "main": "_dist/index.js",

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

114413

increased by6.62%

Lines of code

924

increased by19.38%

No dependency changes