map-filter-reduce
Advanced tools
Comparing version 1.1.0 to 2.0.0
@@ -48,6 +48,6 @@ var pull = require('pull-stream') | ||
return pull.reduce(reduce(q), null, cb) | ||
return SinkThrough(function (cb) { | ||
return pull(SinkThrough(function (cb) { | ||
return pull.reduce(reduce(q), null, cb) | ||
}) | ||
}), pull.flatten()) | ||
} | ||
{ | ||
"name": "map-filter-reduce", | ||
"description": "", | ||
"version": "1.1.0", | ||
"version": "2.0.0", | ||
"homepage": "https://github.com/dominictarr/map-filter-reduce", | ||
@@ -6,0 +6,0 @@ "repository": { |
132
README.md
@@ -56,3 +56,3 @@ # map-filter-reduce | ||
``` js | ||
{$prefix: 'o'} | ||
{$filter: {$prefix: 'o'}} | ||
``` | ||
@@ -67,3 +67,3 @@ this will match `"okay"` `"ok"` `"oh no"`, etc. | ||
``` js | ||
{$gte: 0, $lt: 1} | ||
{$filter: {$gte: 0, $lt: 1}} | ||
``` | ||
@@ -78,3 +78,3 @@ will filter numbers from 0 up to but not including 1. | ||
``` js | ||
{name: {$prefix: 'bob'}, age: {$gt: 10}} | ||
{$filter: {name: {$prefix: 'bob'}, age: {$gt: 10}}} | ||
``` | ||
@@ -85,5 +85,9 @@ select "bob", "bobby", and "bobalobadingdog", if their age is greater than 10. | ||
arrays are used treated somewhat like strings, in that you can also | ||
use the `$prefix` and `$gt, $lt, $gte, $lte` operators. | ||
arrays are like objects, but their keys are compared in order from left to right. | ||
TODO: implement prefix and ltgt ranges on arrays. | ||
{$filter: {$prefix: ['okay']}} | ||
@@ -99,5 +103,8 @@ ### map | ||
``` js | ||
{name: ['value', 'name']} | ||
{$map: {name: ['value', 'name']}} | ||
``` | ||
if the input was `{key: k, value: {foo: blah, name: 'bob'}}` | ||
the output would just be `'bob'`. | ||
#### key: strings, numbers | ||
@@ -108,3 +115,3 @@ | ||
``` js | ||
'value' | ||
{$map: 'value'} | ||
``` | ||
@@ -124,3 +131,3 @@ would return the value property of the input, and drop the rest. | ||
``` js | ||
{foo: 'bar', bar: 'foo'} | ||
{$map: {foo: 'bar', bar: 'foo'}} | ||
``` | ||
@@ -136,10 +143,24 @@ transform the input stream into a stream with `foo` and `bar` keys switched, | ||
``` js | ||
['value', 'name'] | ||
{$map: ['value', 'name']} | ||
``` | ||
get the value property, and then get the name property off that. | ||
#### wildcard | ||
to get _everything_ use `true` | ||
``` js | ||
{$map: true} | ||
``` | ||
say, we have an object with _many_ keys, and we want to reduce that to just | ||
`timestamp` and then put everything under `value` | ||
``` js | ||
{$map: {timestamp: 'timestamp', value: true}} | ||
``` | ||
### reduce | ||
reduce is used to aggregate many values into a representative value. | ||
`count`, `sum`, `min`, `max`, and `group` are all reduces. | ||
`count`, `sum`, `min`, `max`, are all reduces. | ||
@@ -150,3 +171,3 @@ #### count | ||
``` js | ||
{$count: true} | ||
{$reduce: {$count: true}} | ||
``` | ||
@@ -161,3 +182,3 @@ | ||
``` js | ||
{$sum: ['value', 'age']} | ||
{$reduce: {$sum: ['value', 'age']}} | ||
``` | ||
@@ -170,3 +191,3 @@ | ||
``` js | ||
{$min: ['value', 'age']} | ||
{$reduce: {$min: ['value', 'age']}} | ||
``` | ||
@@ -180,3 +201,3 @@ | ||
``` js | ||
{$collect: 'foo'} | ||
{$reduce: {$collect: 'foo'}} | ||
``` | ||
@@ -191,82 +212,36 @@ this may produce a large value if there are many items in the input stream. | ||
``` js | ||
{ | ||
{$reduce: { | ||
youngest: {$min: ['value','age']}, | ||
oldest: {$max: ['value', 'age']} | ||
} | ||
}} | ||
``` | ||
#### group | ||
#### groups | ||
group the input into sections. group is used with other reducers, | ||
first group splits the input into groups, and then runs the reducers | ||
on the elements in that group. | ||
To group, use the other reduce functions | ||
along side path expressions, as used under `$map` | ||
In the following query, | ||
``` js | ||
{ | ||
$group: 'country', | ||
$count: true | ||
} | ||
{$reduce: { | ||
country: 'country', | ||
population: {$count: true} | ||
}} | ||
``` | ||
count items per country. will give a single output, that is an object | ||
where each value for `country` in the input becomes a key. | ||
count items per country. Will give a stream of items each with | ||
a `{country, population}` fields. | ||
say, if the inputs values for country where `nz, us, de, au` then the | ||
output might be: | ||
``` js | ||
{ | ||
nz: 2, | ||
us: 4, | ||
de: 2, | ||
au: 1 | ||
} | ||
``` | ||
group can be applied to more than one field by using an array. | ||
``` js | ||
{$group: ['country', 'city'], $count: true} | ||
{$reduce: {country: 'country', city: 'city', population: {$count: true}}} | ||
``` | ||
would return an object nested to two levels, by country, then city | ||
This would return a sequence of {country, city, population} tripples. | ||
Note that in group, arrays are used for a list of groups, | ||
instead of paths, to get a path, use an array inside an array! | ||
Note that, like in `$map` arrays can be used to drill deep into | ||
an object. | ||
``` js | ||
{$group: [['value', 'name']], $count: true} | ||
{$reduce: {name: ['value', 'name'], posts: {$count: true}}} | ||
``` | ||
since groups are also just reduces, they can be nested by using | ||
object keys. this lets us apply aggregation on different levels. | ||
``` js | ||
{$group: 'country', | ||
count: {$count: true}, | ||
city: { | ||
$group: [['country', 'city']], | ||
$count: true | ||
} | ||
} | ||
``` | ||
the output might look like this: | ||
``` js | ||
{ | ||
nz: { | ||
count: 2, | ||
city: {akl: 1, wlg: 1}, | ||
}, | ||
us: { | ||
count: 4, | ||
city: {nyc: 1, aus: 1, sfo: 1, pdx: 1} | ||
}, | ||
de: { | ||
count: 2, city: {ber: 2} | ||
}, | ||
au: { | ||
count: 1, city: {syd: 1} | ||
} | ||
} | ||
``` | ||
TODO: group by time ranges (day, month, week, year, etc) | ||
@@ -278,6 +253,1 @@ | ||
var u = require('./util') | ||
var map = u.map | ||
var simple = require('./basic') | ||
var search = require('binary-search') | ||
@@ -26,18 +27,20 @@ function isFunction (f) { return 'function' === typeof f } | ||
function each(list, iter) { | ||
if(u.isString(list)) return iter(list) | ||
for(var i = 0; i < list.length; i++) | ||
iter(list[i], (list.length - i - 1)) | ||
} | ||
function group (g, reduce) { | ||
//instead of taking the query, | ||
//this should take a path, and a reduce function. | ||
function group (g, reduce) { | ||
function compare (a, b) { | ||
for(var i in g) { | ||
var x = u.get(a, g[i]) | ||
var y = u.get(b, g[i]) | ||
if(x != y) return x < y ? -1 : 1 | ||
} | ||
return 0 | ||
} | ||
return function (a, b) { | ||
var A = a = a || {} | ||
each(g, function (k, notLast) { | ||
var v = u.get(b, k) | ||
A[v] = !notLast ? reduce(A[v], b) : A[v] || {} | ||
A = A[v] | ||
}) | ||
var A = a = a || [] | ||
var i = search(A, b, compare) | ||
if(i >= 0) A[i] = reduce(A[i], b) | ||
else A.splice(~i, 0, reduce(undefined, b)) | ||
return a | ||
@@ -47,2 +50,3 @@ } | ||
function make (query) { | ||
@@ -54,8 +58,20 @@ var k = isSimple(query) | ||
if(k == '$group') return undefined | ||
return gmake(query[k]) | ||
return make(query[k]) | ||
})) | ||
else return function (a, b) { | ||
return b[query] | ||
} | ||
} | ||
function gmake (query) { | ||
return query.$group ? group(query.$group, make(query)) : make(query) | ||
if(isSimple(query)) return make(query) | ||
var paths = [] | ||
u.each(query, function traverse (value) { | ||
if(isSimple(value)) return | ||
else if(u.isObject(value)) each(value, traverse) | ||
else if(value) paths.push(value) | ||
}) | ||
return paths.length ? group(paths, make(query)) : make(query) | ||
} | ||
@@ -65,3 +81,1 @@ | ||
@@ -58,7 +58,10 @@ var tape = require('tape') | ||
objs.reduce(R({ | ||
$group: 'baz', $count: true | ||
baz: 'baz', count: {$count: true} | ||
}), null), | ||
{"true": 3, "false": 2} | ||
[ | ||
{baz: false, count: 2}, | ||
{baz: true, count: 3} | ||
] | ||
) | ||
return t.end() | ||
t.deepEqual( | ||
@@ -83,15 +86,15 @@ objs.reduce(R({ | ||
{ | ||
name: 'pfraze', country: 'US', house: 'apartment' | ||
name: 'pfraze', country: 'US', dwelling: 'apartment' | ||
}, | ||
{ | ||
name: 'substack', country: 'US', house: 'house' | ||
name: 'substack', country: 'US', dwelling: 'house' | ||
}, | ||
{ | ||
name: 'mix', country: 'NZ', house: 'house' | ||
name: 'mix', country: 'NZ', dwelling: 'house' | ||
}, | ||
{ | ||
name: 'du5t', country: 'US', house: 'apartment' | ||
name: 'du5t', country: 'US', dwelling: 'apartment' | ||
}, | ||
{ | ||
name: 'dominic', country: 'NZ', house: 'sailboat' | ||
name: 'dominic', country: 'NZ', dwelling: 'sailboat' | ||
} | ||
@@ -102,31 +105,20 @@ ] | ||
t.deepEqual(groups.reduce(R({ | ||
$group: ['country', 'house'], | ||
$collect: 'name' | ||
}), null), | ||
country: 'country', dwelling: 'dwelling', people: {$collect: 'name'} | ||
}), null), [ | ||
{ | ||
US: { | ||
apartment: ['pfraze', 'du5t'], | ||
house: ['substack'] | ||
}, | ||
NZ: { | ||
house: ['mix'], | ||
sailboat: ['dominic'] | ||
} | ||
country: 'NZ', dwelling: 'house', people: ['mix'] | ||
}, | ||
{ | ||
country: 'NZ', dwelling: 'sailboat', people: ['dominic'] | ||
}, | ||
{ | ||
country: 'US', dwelling: 'apartment', people: ['pfraze', 'du5t'] | ||
}, | ||
{ | ||
country: 'US', dwelling: 'house', people: ['substack'] | ||
} | ||
) | ||
]) | ||
t.end() | ||
}) | ||
tape('nested groups', function (t) { | ||
t.deepEqual( | ||
groups.reduce(R({ | ||
$group: 'country', | ||
population: {$count: true}, | ||
housing: {$group: 'house', $count: true} | ||
}), null), | ||
{ US: { population: 3, housing: { apartment: 2, house: 1 } }, | ||
NZ: { population: 2, housing: { house: 1, sailboat: 1 } } } | ||
) | ||
t.end() | ||
}) | ||
53
util.js
@@ -47,4 +47,4 @@ 'use strict' | ||
if(isString(v.$prefix)) return v.$prefix | ||
if(has(v, '$lt')) return v.$lt | ||
if(has(v, '$lte')) return v.$lte | ||
if(has(v, '$gt')) return v.$gt | ||
if(has(v, '$gte')) return v.$gte | ||
} | ||
@@ -59,4 +59,4 @@ if(isArray(v)) return v.map(lower) | ||
if(isString(v.$prefix)) return v.$prefix+'\uffff' | ||
if(has(v, '$gt')) return v.$gt | ||
if(has(v, '$gte')) return v.$gte | ||
if(has(v, '$le')) return v.$lt | ||
if(has(v, '$lte')) return v.$lte | ||
} | ||
@@ -89,2 +89,37 @@ if(isArray(v)) return v.map(upper) | ||
function each(obj, iter) { | ||
if(Array.isArray(obj)) return obj.forEach(iter) | ||
for(var k in obj) iter(obj[k], k, obj) | ||
} | ||
function project (value, map, isObj) { | ||
isObj = isObj || isObject | ||
if(!isObj(value)) | ||
return map(value) | ||
else { | ||
var o | ||
for(var k in value) { | ||
var v = project(value[k], map, isObj) | ||
if(v !== undefined) | ||
(o = o || {})[k] = v | ||
} | ||
return o | ||
} | ||
} | ||
//get all paths within an object | ||
//this can probably be optimized to create less arrays! | ||
function paths (object, test) { | ||
var p = [] | ||
for(var key in object) { | ||
var value = object[key] | ||
if(test(value)) p.push(key) | ||
else if(isObject(value)) | ||
p = p.concat(paths(value, test).map(function (path) { | ||
return [key].concat(path) | ||
})) | ||
} | ||
return p | ||
} | ||
exports.isString = isString | ||
@@ -99,5 +134,8 @@ exports.isNumber = isNumber | ||
exports.has = has | ||
exports.get = get | ||
exports.map = map | ||
exports.has = has | ||
exports.get = get | ||
exports.map = map | ||
exports.project = project | ||
exports.paths = paths | ||
exports.each = each | ||
@@ -109,1 +147,2 @@ exports.upper = upper | ||
exports.LO = null | ||
24675
18
715
240