Socket
Socket
Sign inDemoInstall

leo-connector-common

Package Overview
Dependencies
Maintainers
4
Versions
108
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

leo-connector-common - npm Package Compare versions

Comparing version 1.1.0-alpha2 to 1.1.0-beta1

61

botHelper.js

@@ -299,63 +299,2 @@ "use strict";

};
this.checksum = function (params) {
if (!params.source_connector || !params.slave_connector || !params.checksum_connector) {
console.log(params);
throw new Error('Missing one of the following parameter values: source_connector, source_connection, slave_connector, slave_connection, or checksum_connector');
}
let sourceParams = [];
let slaveParams = [];
let checksum = params.checksum_connector;
if (!params.ls) {
params.ls = sdk.streams;
}
return {
sourceQuery: function(query) {
sourceParams.query = query;
return this;
},
slaveQuery: function(query) {
slaveParams.query = query;
return this;
},
run: function(callback) {
let system = event.botId;
let source = checksum.lambdaConnector(event.botId, "SqlserverConnector", sourceParams);
let slave = checksum.lambdaConnector(event.botId, "PostgresConnector", slaveParams);
checksum.checksum(system, system, source, slave, {
stopOnStreak: 1750000,
stop_at: moment().add({
minutes: 4
}),
maxLimit: 2500,
loadSize: 50000,
limit: 2500,
reverse: true,
sample: true,
version: 2,
queue: {
name: "test-checksum.output",
transform: params.ls.through((obj, done) => {
done(null, {
orders: obj.missing.concat(obj.incorrect)
});
})
},
skipBatch: true,
showOutput: true
})
.then((result) => {
console.log(result);
}, (err) => {
console.log(err);
});
}
}
};
};

33

checksum/index.js

@@ -20,2 +20,3 @@ let leo = require("leo-sdk");

const tableName = leo.configuration.resources.LeoCron;
let logger = require("leo-sdk/lib/logger")("leo-checksum");

@@ -88,3 +89,3 @@ function saveProgress(systemId, botId, data) {

return new Promise((resolve, reject) => {
console.log("Getting Session", systemId, botId);
logger.log("Getting Session", systemId, botId);
dynamodb.get(tableName, botId, function(err, result) {

@@ -109,7 +110,4 @@ if (err) {

checksum: function(system, botId, master, slave, opts) {
console.log('in checksum');
return new Promise((resolve, reject) => {
function logError(err) {
console.log('logging error');
saveProgress(system, botId, {

@@ -121,3 +119,3 @@ status: "error",

this.getSession(system, botId, opts).then((session) => {
console.log("Session:", session);
logger.log("Session:", session);
let tasks = [];

@@ -128,3 +126,2 @@ master.setSession(session.master);

if (session.status === 'initializing') {
console.log('initializing');
tasks.push(Promise.all([

@@ -143,3 +140,3 @@ master.init({}),

master.range = (opts) => {
console.log("Using Cached Range Value");
logger.log("Using Cached Range Value");
return Promise.resolve({

@@ -173,3 +170,2 @@ min: opts.min,

opts.stats = function(nibble, result, total, done) {
console.log('in opts stats');
let percent = (nibble.progress / nibble.total) * 100;

@@ -221,3 +217,2 @@ let fixed = percent.toFixed(2);

});
console.log('saving progress 214');
// logger.log(JSON.stringify(data, null, 2));

@@ -228,3 +223,2 @@ saveProgress(system, botId, data).then(result => done(null, result), done);

opts.onSample = opts.sample && function(type, diff, done) {
console.log('ops onsample');
session.sample[type] = diff.concat(session.sample[type]).slice(0, 4);

@@ -238,3 +232,2 @@ if (done) {

if (opts.queue) {
console.log('opts queue');
let load = leo.load(botId, opts.queue.name, {

@@ -261,3 +254,2 @@ useS3: true,

}).sync(opts, function(result, done) {
console.log('in sync');
if (stream) {

@@ -275,3 +267,3 @@ if (!stream.write(result)) {

let tasks = [];
console.log('stopReason', stopReason)
logger.log('stopReason', stopReason);

@@ -294,3 +286,2 @@ if (status === "complete") {

stream.end((err) => {
console.log('saving progress 282');
saveProgress(system, botId,

@@ -468,3 +459,3 @@ Object.assign(session, {

req.on('error', function(e) {
console.error('problem with request: ' + e.message);
logger.error('problem with request: ' + e.message);
reject(e);

@@ -525,3 +516,3 @@ });

getChecksum: function(data, callback) {
console.log(" BATCH", settings.name, data, rand.batch)
logger.log(" BATCH", settings.name, data, rand.batch)
callback(null, {

@@ -548,7 +539,7 @@ qty: data.end - data.start + 1,

}
console.log(" INDIVIDUAL", settings.name, data)
logger.log(" INDIVIDUAL", settings.name, data)
callback(null, result)
},
sample: function(data, callback) {
console.log(" SAMPLE", settings.name, data);
logger.log(" SAMPLE", settings.name, data);
let result = {

@@ -576,3 +567,3 @@ qty: 0,

};
console.log(" RANGE", settings.name, data, result)
logger.log(" RANGE", settings.name, data, result)
callback(null, result);

@@ -582,3 +573,3 @@ },

setTimeout(function() {
console.log(" NIBBLE", settings.name, data);
logger.log(" NIBBLE", settings.name, data);
data.end = Math.min(data.start + data.limit - 1, settings.mock.max);

@@ -610,3 +601,3 @@ data.next = data.start + data.limit < settings.mock.max ? data.start + data.limit : null;

} catch (err) {
console.log(`${id} ${method} Error: ${err}`);
logger.log(`${id} ${method} Error: ${err}`);
reject(err);

@@ -613,0 +604,0 @@ }

24

checksum/lib/checksumNibbler.js

@@ -18,3 +18,3 @@ "use strict";

}, (opts || {}).totals);
console.log("START VALUES", data);
// logger.log("START VALUES", data);

@@ -24,7 +24,5 @@ // Add extra param to this sync;

nibbler.sync = function(opts, resultCallback, callback) {
console.log('in nibbler sync');
let asyncMethod = opts.inSeries ? "mapSeries" : "map";
let fieldNames = opts.fieldNames || [];
let update = function(nibble, obj, results) {
console.log('in update');
obj.progress += results.qty;

@@ -44,7 +42,5 @@ obj.totalCorrect += results.correct;

nibbler.log(`Correct: ${results.correct}, Incorrect: ${results.incorrect}, Missing: ${results.missing}, Extra: ${results.extra}, Start: ${nibble.start}, End: ${nibble.end}`);
console.log(`Correct: ${results.correct}, Incorrect: ${results.incorrect}, Missing: ${results.missing}, Extra: ${results.extra}, Start: ${nibble.start}, End: ${nibble.end}`);
};
let compare = function(start, end, callback) {
console.log('in compare');
if (opts.skipBatch) {

@@ -64,3 +60,2 @@ return callback(true, {

nibbler.timeLog("Running Master & Slave Batch Checksum");
console.log("Running Master & Slave Batch Checksum");
async [asyncMethod]([local, remote], (connector, done) => {

@@ -71,3 +66,2 @@ connector.getChecksum({

}).then(result => {
console.log('getChecksum result: ', result);
done(null, result)

@@ -80,3 +74,2 @@ }, callback);

}
console.log('async batchResults: ', batchResults);
let localData = batchResults[0];

@@ -92,7 +85,5 @@ let remoteData = batchResults[1];

result.errors.push(remote.name + " has too few");
console.log(remote.name + " has too few");
}
if (localData.qty < remoteData.qty) {
result.errors.push(remote.name + " has too many");
console.log(remote.name + " has too many");
}

@@ -106,3 +97,2 @@ if (

result.errors.push("Hashes do not match");
console.log("Hashes do not match");
}

@@ -113,3 +103,2 @@

console.log('result', result);
if (result.errors.length == 0) {

@@ -126,3 +115,2 @@ callback(null, result);

nibbler.timeLog("Running Master & Slave Individual Checksum");
console.log("Running Master & Slave Individual Checksum");
async [asyncMethod]([local, remote], (connector, done) => {

@@ -175,3 +163,2 @@ connector.getIndividualChecksums({

});
console.log('162: results', results);
callback(null, results);

@@ -183,3 +170,2 @@ });

nibbler.timeLog("Running Master & Slave Sample");
console.log("Running Master & Slave Sample");

@@ -261,12 +247,10 @@ async [asyncMethod]([local, remote], (connector, done) => {

onInit: function(nibble) {
console.log('Init');
nibble.progress = data.progress
},
onEnd: function(err, nibble, callback) {
console.log(`Summary`);
console.log(`Correct: ${data.totalCorrect}, Incorrect:${data.totalIncorrect}, Missing:${data.totalMissing}, Extra:${data.totalExtra}`);
logger.log(`Summary`);
logger.log(`Correct: ${data.totalCorrect}, Incorrect:${data.totalIncorrect}, Missing:${data.totalMissing}, Extra:${data.totalExtra}`);
callback();
},
onError: function(err, result, nibble, done) {
console.log('error');
if (err && !result) {

@@ -277,3 +261,2 @@ return done(err)

compareIndividual(nibble.start, nibble.end, (err, dataResult) => {
console.log('comparing individual');
if (err) {

@@ -349,3 +332,2 @@ done(err);

onBite: function(nibble, done) {
console.log('biting');
compare(nibble.start, nibble.end, (err, result) => {

@@ -352,0 +334,0 @@ if (err) {

8

checksum/lib/nibbler.js

@@ -88,6 +88,2 @@ var async = require("async");

//Now let's nibble our way through it.
console.log('opts:', opts.start, opts.end);
console.log('range:', range.min, range.max);
console.log('bool 1:', opts.start && opts.start > range.min);
console.log('bool 2:', opts.end && opts.end < range.max);
nibble = {

@@ -111,3 +107,2 @@ start: opts.start && opts.start > range.min ? opts.start : range.min,

};
console.log('nibble 110: ', nibble.start, nibble.end, nibble.reverse);

@@ -125,5 +120,2 @@ if (opts.onInit) {

nibble.end = forward ? nibble.max : nibble.end;
console.log('nibble 123: ', nibble.start, nibble.end, forward);
console.log('nibble 123 strings: ', typeof nibble.start, typeof nibble.end);
process.exit();

@@ -130,0 +122,0 @@ connector.nibble(nibble).then((n) => {

15

datawarehouse/combine.js

@@ -22,5 +22,12 @@ const exec = require('child_process').exec;

return ls.through((obj, done) => {
count++;
if (count % 10000 == 0) {
console.log(count);
}
let payload = obj.payload;
let table = transform.parseTable(payload);
if (table == undefined) {
return done(null);
}
let table = transform.parseTable(payload);
let values = transform.parseValues(payload.data, dateFormat);

@@ -40,6 +47,2 @@

Object.keys(values).forEach(f => stream.fields[f] = 1);
count++;
if (count % 10000 == 0) {
console.log(count);
}
let id = crypto.createHash('md5');

@@ -144,2 +147,2 @@ id.update(tableIds[table].map(f => values[f]).join(','));

return pass;
}
}

2

datawarehouse/load.js

@@ -126,2 +126,2 @@ "use strict";

});
};
};

15

datawarehouse/transform.js

@@ -8,6 +8,13 @@ "use strict";

} else if (obj.type) {
if (obj.type == "fact") {
return "f_" + obj.entity.toLowerCase().replace(/\s/g, '_');
} else {
return "d_" + obj.entity.toLowerCase().replace(/\s/g, '_');
try {
if (obj.type == "fact" && obj.entity) {
return "f_" + obj.entity.toLowerCase().replace(/\s/g, '_');
} else if (obj.entity) {
return "d_" + obj.entity.toLowerCase().replace(/\s/g, '_');
} else {
return null;
}
} catch (e) {
console.log("Invalid Table to Parse", obj);
return null;
}

@@ -14,0 +21,0 @@ } else {

4

package.json
{
"name": "leo-connector-common",
"version": "1.1.0-alpha2",
"version": "1.1.0-beta1",
"description": "",

@@ -28,5 +28,5 @@ "main": "index.js",

"async": "^2.6.0",
"leo-sdk": "^1.0.67",
"leo-sdk": "^1.0.68",
"moment": "^2.22.1"
}
}

9

sql/loader.js

@@ -24,3 +24,3 @@ "use strict";

if (buildIds) {
if (buildIds.length) {
buildEntities(buildIds, push, done);

@@ -109,2 +109,3 @@ }

function buildEntities(ids, push, callback) {
let r = domainObj(ids, builder.createLoader);

@@ -211,3 +212,6 @@ if (typeof r.get == "function") {

return done(err);
} else if (!results.length) {
return done();
}
mapResults(results, fields, row => {

@@ -229,3 +233,6 @@ if (t.transform) {

return done(err);
} else if (!results.length) {
return done();
}
mapResults(results, fields, row => {

@@ -232,0 +239,0 @@ if (row.length) {

2

sql/loader/lib.js

@@ -55,3 +55,3 @@ const async = require("async");

sqlClient.query(idthing, (err, results, fields) => {
if (!err) {
if (!err && results.length) {
let firstColumn = fields[0].name;

@@ -58,0 +58,0 @@ ids = ids.concat(results.map(row => row[firstColumn]));

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc