Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
remo - npm Package Compare versions
Comparing version 0.0.11 to 0.0.12
@@ -7,8 +7,6 @@ var remo = require('../index'); | ||
| conf.defaults.db = false; | ||
| module.exports = function(message, cb){ | ||
| connect(conf.defaults, function(err, db) { | ||
| conf.err = err; | ||
| if(!conf.defaults.db){ conf.defaults.db = db; } | ||
| conf.defaults.db = conf.defaults.db || db; | ||
| attempt(db, merge(conf.defaults, message), function(err, res) { | ||
@@ -15,0 +13,0 @@ cb(err, res); |
@@ -5,5 +5,6 @@ var gulp = require('gulp'); | ||
| var scripts = [ | ||
| './*.js', | ||
| './api/*.js', | ||
| './lib/*.js' | ||
| './*.js', | ||
| './api/*.js', | ||
| './lib/*.js', | ||
| './test/*.js' | ||
| ]; | ||
@@ -10,0 +11,0 @@ |
@@ -30,3 +30,5 @@ var remo = {}; | ||
| remo.parseOId = require('./api/parseOId'); | ||
| //--- db pool connection ---// | ||
| remo.pool = require('./api/pool'); | ||
| module.exports = remo; |
| { | ||
| "name": "remo", | ||
| "version": "0.0.11", | ||
| "version": "0.0.12", | ||
| "description": "Re[dis]Mo[ngo] queue manager", | ||
@@ -5,0 +5,0 @@ "main": "index.js", |
132
README.md
| # ReMo | ||
| Redis and Mongo for async tasks or slow servers. | ||
| Save data from memory to disk for async/background/slow tasks. | ||
@@ -9,10 +9,12 @@ ## Metrics | ||
| ## Queue Query List | ||
| Create a list on memory, save on disk. | ||
| ## Standalone ReMo | ||
| Use ReMo to manage input queries. | ||
| Message must contain the 'action'. See [attempt actions](lib/attempt.js). | ||
| ````js | ||
| var remo = require('remo'); | ||
| var message = {collection: 'mine',doc: {qty: 1},action: 'inc'}; | ||
| remo.pool(message, function(err, res){ /* stuff here*/ }); | ||
| ```` | ||
| ## ReMo | ||
| Configure your queues: | ||
| Configure your queues and defaults: | ||
| ````sh | ||
@@ -27,102 +29,9 @@ # conf.json | ||
| Configure your defaults | ||
| ````sh | ||
| # conf.json | ||
| { | ||
| "defaults": { | ||
| ... | ||
| "url": "mongodb://127.0.0.1/remo" | ||
| ... | ||
| } | ||
| } | ||
| ```` | ||
| ## Sample ReMo Work | ||
| ## Learn ReMo | ||
| Follow the [step by step guide](steps.md) for massive queries. | ||
| ````js | ||
| var remo = require('remo'); | ||
| var subR = redis.createClient(); | ||
| var mongo = require('mongodb').MongoClient; | ||
| mongo.connect(conf.defaults.url, function(err, db) { | ||
| subR.lpop('db:mongo:input', function(err, message) { | ||
| remo.insert(db, message, function(err, results) { | ||
| subR.lpush(['db:mongo:errors', JSON.stringify(err)], empty); | ||
| subR.lpush('db:mongo:results', JSON.stringify(results)], empty); | ||
| }); | ||
| }); | ||
| }); | ||
| ```` | ||
| ## Learn By Example | ||
| Run your mongod v3 service | ||
| ````sh | ||
| mongod --directoryperdb --storageEngine wiredTiger | ||
| ```` | ||
| Clean redis | ||
| ````sh | ||
| redis-cli | ||
| 127.0.0.1:6379>FLUSHALL | ||
| ```` | ||
| Flood redis | ||
| ````sh | ||
| node lib/fill.js | ||
| #done! | ||
| ```` | ||
| Count input queries | ||
| ````sh | ||
| redis-cli | ||
| 127.0.0.1:6379>LLEN "db:mongo:input" | ||
| (integer) 100000 | ||
| ```` | ||
| Open your monitor | ||
| ````sh | ||
| redis-cli monitor | ||
| ```` | ||
| From redis to mongodb listening for changes | ||
| ````sh | ||
| node lib/init.js | ||
| ```` | ||
| Play with times | ||
| ````js | ||
| #conf.json | ||
| { | ||
| "sleep": 1000, # Listen for changes every second | ||
| "queue":{ | ||
| "size": 1000, # Slice whole process into blocks of 1000 queries | ||
| "wait": 0, # Time between each query | ||
| ... | ||
| } | ||
| ```` | ||
| Flood redis again and see monitor | ||
| ````sh | ||
| node lib/fill.js | ||
| #done! | ||
| ```` | ||
| ````sh | ||
| { rss: 81006592, heapTotal: 61790464, heapUsed: 38060592 } | ||
| Attempt with 1000 queries on 1430927005463 | ||
| ```` | ||
| Mesure of processed queries | ||
| ````sh | ||
| 127.0.0.1:6379> LLEN "db:mongo:results" | ||
| (integer) 100000 | ||
| 127.0.0.1:6379> LRANGE "db:mongo:results" 0 1 | ||
| 1) "{\"ok\":1,\"n\":1}" | ||
| 2) "{\"ok\":1,\"n\":1}" | ||
| ```` | ||
| ## Redis List Expected Message | ||
| Save your messages on redis | ||
| ````js | ||
| var redis = require('redis'); | ||
| var conf = require('../conf.json'); | ||
| var pub = redis.createClient(); | ||
@@ -136,18 +45,11 @@ | ||
| ## Workflow | ||
| * Send JSON http request. | ||
| * Save JSON in redis. | ||
| * Listening for changes node/ruby | ||
| * Count pending queries | ||
| * Open a connection with mongo | ||
| * Process a block of queries | ||
| * Save on disk | ||
| * Close the mongo connection | ||
| ## Code Quality | ||
| ## Code Quality | ||
| ````sh | ||
| npm install gulp | ||
| npm install gulp-jshint | ||
| gulp | ||
| # gulp and jshint | ||
| npm install gulp gulp-jshint -g; gulp; | ||
| # mocha for tests | ||
| npm install mocha -g; mocha; | ||
| # coverage | ||
| make cov | ||
| ```` |
New alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Fixed alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Improved metrics
- Number of package files
- increased by1.69%
60
- Lines of code
- increased by0.11%
890
Worsened metrics
- Total package byte prevSize
- decreased by-1.2%
26659
- Number of lines in readme file
- decreased by-64.9%
53
No dependency changes