Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

@apla/clickhouse

Package Overview
Dependencies
Maintainers
1
Versions
16
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@apla/clickhouse - npm Package Compare versions

Comparing version 1.2.3 to 1.3.0

src/legacy-support.js
test/03-errors.js
test/04-select.js
test/90-torture.js

7

package.json
{
"name": "@apla/clickhouse",
"version": "1.2.3",
"version": "1.3.0",
"description": "Yandex ClickHouse database interface",

@@ -29,6 +29,7 @@ "main": "src/clickhouse.js",

"dependencies": {
"object-assign": "^4.1.0",
"buffer-indexof-polyfill": "^1.0.1"
"buffer-indexof-polyfill": "^1.0.1",
"object-assign": "^4.1.0"
},
"devDependencies": {
"bluebird": "^3.5.0",
"codecov": "^2.2.0",

@@ -35,0 +36,0 @@ "memwatch-next": "^0.3.0",

59

README.md

@@ -15,5 +15,15 @@ Database interface for http://clickhouse.yandex

```javascript
var ch = new ClickHouse ({host: clickhouse.host});
var ch = new ClickHouse ({host: clickhouse.host, port: 8123, auth: "user:password"});
// or
var ch = new ClickHouse (clickhouse.host);
// stream is an object stream. you can pipe it
// do the query, callback interface, not recommended for selects
ch.query ("CREATE DATABASE clickhouse_test", function (err, data) {
});
// promise interface (requires 'util.promisify' for node < 8, Promise shim for node < 4)
ch.querying ("CREATE DATABASE clickhouse_test").then (…);
// it is better to use stream interface to fetch select results
var stream = ch.query ("SELECT 1");

@@ -73,4 +83,4 @@

`queryOptions` object can contain any option from Settings (docs:
[en](https://clickhouse.yandex/reference_en.html#Settings)
[ru](https://clickhouse.yandex/reference_ru.html#Настройки)
[en](https://clickhouse.yandex/docs/en/operations/settings/index.html)
[ru](https://clickhouse.yandex/docs/ru/operations/settings/index.html)
)

@@ -119,4 +129,43 @@

### clickHouse.ping ()
You should have at least one error handler listening. Via callbacks or via stream errors.
If you have callback and stream listener, you'll have error notification in both listeners.
### clickHouse.querying (statement, [options]).then (…)
Promise interface. Similar to the callback one.
### clickHouse.ping (function (err, response) {})
Sends an empty query and check if it "Ok.\n"
### clickHouse.pinging ().then (…)
Promise interface for `ping`
Notes
-----
## Memory size
You can read all the records into memory in single call like this:
```javascript
var ch = new ClickHouse ({host: host, port: port});
ch.query ("SELECT number FROM system.numbers LIMIT 10", {syncParser: true}, function (err, result) {
// result will contain all the data you need
});
```
In this case whole JSON response from the server will be read into memory,
then parsed into memory hogging your CPU. Default parser will parse server response
line by line and emits events. This is slower, but much more memory and CPU efficient
for larger datasets.
## Promise interface
Promise interface have some restrictions. It is not recommended to use this interface
for `INSERT` and `SELECT` queries. For the `INSERT` you cannot bulk load data via stream,
`SELECT` will collect all the records in the memory. For simple usage where data size
is controlled it is ok.

82

src/clickhouse.js

@@ -10,6 +10,4 @@ var http = require ('http');

Object.assign = require ('object-assign');
require ('./legacy-support');
require ('buffer-indexof-polyfill');
var JSONStream = require ('./json-stream');

@@ -34,3 +32,31 @@

function parseError (e) {
var fields = new Error (e.toString ('utf8'));
e.toString ('utf8')
.split (/\,\s+(?=e\.)/gm)
.map (function (f) {
f = f.trim ().split (/\n/gm).join ('');
var m;
if (m = f.match (/^(?:Error: )?Code: (\d+)$/)) {
fields.code = parseInt (m[1]);
} else if (m = f.match (/^e\.displayText\(\) = ([A-Za-z0-9\:]+:) ([^]+)/m)) {
// e.displayText() = DB::Exception: Syntax error: failed at position 0: SEL
fields.scope = m[1];
fields.message = m[2];
if (m = fields.message.match (/Syntax error: (?:failed at position (\d+)(?:\s*\(line\s*(\d+)\,\s+col\s*(\d+)\))?)/)) {
// console.log ('!!! syntax error: pos %s line %s col %s', m[1], m[2], m[3]);
fields.lineno = parseInt (m[2] || 1, 10);
fields.colno = parseInt (m[3] || m[1], 10);
}
} else if (m = f.match (/^e\.what\(\) = (.*)/)) {
fields.type = m[1];
} else {
console.warn ('Unknown error field:', f)
}
});
return fields;
}
function httpRequest (reqParams, reqData, cb) {

@@ -40,2 +66,6 @@

if (reqParams.query) {
reqParams.path = (reqParams.pathname || reqParams.path) + '?' + qs.stringify (reqParams.query);
}
var onResponse = function(response) {

@@ -46,10 +76,14 @@ var str;

if (response.statusCode === 200) {
str = new Buffer (0);
str = Buffer.alloc ? Buffer.alloc (0) : new Buffer (0);
} else {
error = new Buffer (0);
error = Buffer.alloc ? Buffer.alloc (0) : new Buffer (0);
}
function errorHandler (e) {
stream.emit ('error', e);
return cb && cb (e);
var err = parseError (e);
// user should define callback or add event listener for the error event
if (!cb || (cb && stream.listeners ('error').length))
stream.emit ('error', err);
return cb && cb (err);
}

@@ -81,2 +115,3 @@

&& chunk.lastIndexOf ("\n") !== -1
&& str
) {

@@ -117,3 +152,3 @@

if (error) {
return errorHandler (new Error (error.toString ('utf8')))
return errorHandler (error);
}

@@ -138,2 +173,3 @@

// we already pushed all the data
if (jsonParser.columns.length) {

@@ -223,2 +259,4 @@ try {

urlObject.port = urlObject.port || 8123;
return urlObject;

@@ -285,3 +323,3 @@ }

reqParams.path += '?' + qs.stringify (queryObject);
reqParams.query = queryObject;

@@ -293,2 +331,13 @@ var stream = httpRequest (reqParams, reqData, cb);

ClickHouse.prototype.querying = function (chQuery, options) {
return new Promise (function (resolve, reject) {
var stream = this.query (chQuery, options, function (err, data) {
if (err)
return reject (err);
resolve (data);
});
}.bind (this));
}
ClickHouse.prototype.ping = function (cb) {

@@ -305,2 +354,17 @@

ClickHouse.prototype.pinging = function () {
return new Promise (function (resolve, reject) {
var reqParams = this.getReqParams ();
reqParams.method = 'GET';
httpRequest (reqParams, {finalized: true}, function (err, data) {
if (err)
return reject (err);
resolve (data);
});
}.bind (this));
}
module.exports = ClickHouse;

164

test/02-real-server.js

@@ -9,3 +9,3 @@ var ClickHouse = require ("../src/clickhouse");

describe ("real server queries", function () {
describe ("real server", function () {

@@ -20,3 +20,3 @@ var server,

ch.ping (function (err, ok) {
assert (!err);
assert.ifError (err);
assert.equal (ok, "Ok.\n", "ping response should be 'Ok.\\n'");

@@ -27,8 +27,21 @@ done ();

it ("selects using callback", function (done) {
var ch = new ClickHouse ({host: host, port: port, useQueryString: true});
ch.query ("SELECT 1", {syncParser: true}, function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `data` key to represent rows");
assert (result.data, "result should be Object with `meta` key to represent column info");
it ("pinging using promise interface", function () {
var ch = new ClickHouse ({host: host, port: port});
return ch.pinging ();
});
it ("pinging using promise interface with bad connection option", function () {
var ch = new ClickHouse ();
return ch.pinging ().then (function () {
return Promise.reject (new Error ("Driver should throw without host name"))
}, function (e) {
return Promise.resolve ();
});
});
it ("pings with options as host", function (done) {
var ch = new ClickHouse (host);
ch.ping (function (err, ok) {
assert.ifError (err);
assert.equal (ok, "Ok.\n", "ping response should be 'Ok.\\n'");
done ();

@@ -38,2 +51,7 @@ });

it ("nothing to ping", function () {
var ch = new ClickHouse ();
assert (ch);
});
it ("returns error", function (done) {

@@ -48,3 +66,3 @@ var ch = new ClickHouse ({host: host, port: port, useQueryString: true});

assert (err);
console.log (err);
// console.log (err);
done();

@@ -54,25 +72,7 @@ });

it ("selects using callback and query submitted in the POST body", function (done) {
it ("selects from system columns", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.query ("SELECT 1", {syncParser: true}, function (err, result) {
ch.query ("SELECT * FROM system.columns", function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `data` key to represent rows");
assert (result.data, "result should be Object with `meta` key to represent column info");
done ();
});
});
it ("selects numbers using callback", function (done) {
var ch = new ClickHouse ({host: host, port: port, useQueryString: true});
ch.query ("SELECT number FROM system.numbers LIMIT 10", {syncParser: true}, function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `data` key to represent rows");
assert (result.data, "result should be Object with `meta` key to represent column info");
assert (result.meta.constructor === Array, "metadata is an array with column descriptions");
assert (result.meta[0].name === "number");
assert (result.data.constructor === Array, "data is a row set");
assert (result.data[0].constructor === Array, "each row contains list of values (using FORMAT JSONCompact)");
assert (result.data[9][0] === "9"); // this should be corrected at database side
assert (result.rows === 10);
assert (result.rows_before_limit_at_least === 10);
done ();

@@ -82,67 +82,6 @@ });

it ("selects numbers using callback and query submitted in the POST body", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.query ("SELECT number FROM system.numbers LIMIT 10", {syncParser: true}, function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `meta` key to represent rows");
assert (result.data, "result should be Object with `data` key to represent column info");
assert (result.meta.constructor === Array, "metadata is an array with column descriptions");
assert (result.meta[0].name === "number");
assert (result.data.constructor === Array, "data is a row set");
assert (result.data[0].constructor === Array, "each row contains list of values (using FORMAT JSONCompact)");
assert (result.data[9][0] === "9"); // this should be corrected at database side
assert (result.rows === 10);
assert (result.rows_before_limit_at_least === 10);
done ();
});
});
it ("selects numbers asynchronously using events and query submitted in the POST body", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var rows = [];
var stream = ch.query ("SELECT number FROM system.numbers LIMIT 10", function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `meta` key to represent rows");
assert (rows, "result should be Object with `data` key to represent column info");
assert (result.meta.constructor === Array, "metadata is an array with column descriptions");
assert (result.meta[0].name === "number");
assert (rows.length === 10, "total 10 rows");
assert (rows[0].constructor === Array, "each row contains list of values (using FORMAT JSONCompact)");
assert (rows[9][0] === "9"); // this should be corrected at database side
assert (result.rows === 10);
assert (result.rows_before_limit_at_least === 10);
done ();
});
stream.on ('data', function (row) {
rows.push (row);
})
});
it ("selects numbers asynchronously using stream and query submitted in the POST body", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var metadata;
var rows = [];
var stream = ch.query ("SELECT number FROM system.numbers LIMIT 10");
stream.on ('metadata', function (_meta) {
metadata = _meta;
});
stream.on ('data', function (row) {
rows.push (row);
});
stream.on ('error', function (err) {
it ("selects from system columns no more than 10 rows throws exception", function (done) {
var ch = new ClickHouse ({host: host, port: port, queryOptions: {max_rows_to_read: 10}});
ch.query ("SELECT * FROM system.columns", function (err, result) {
assert (err);
});
stream.on ('end', function () {
assert (metadata, "result should be Object with `meta` key to represent rows");
assert (rows, "result should be Object with `data` key to represent column info");
assert (metadata.constructor === Array, "metadata is an array with column descriptions");
assert (metadata[0].name === "number");
assert (rows.length === 10, "total 10 rows");
assert (rows[0].constructor === Array, "each row contains list of values (using FORMAT JSONCompact)");
assert (rows[9][0] === "9"); // this should be corrected at database side
assert (stream.supplemental.rows === 10);
assert (stream.supplemental.rows_before_limit_at_least === 10);

@@ -153,41 +92,2 @@ done ();

it ("selects number objects asynchronously using stream and query submitted in the POST body", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var metadata;
var rows = [];
var stream = ch.query ("SELECT number FROM system.numbers LIMIT 10", {dataObjects: true});
stream.on ('metadata', function (_meta) {
metadata = _meta;
});
stream.on ('data', function (row) {
rows.push (row);
});
stream.on ('error', function (err) {
assert (err);
});
stream.on ('end', function () {
assert (metadata, "result should be Object with `meta` key to represent rows");
assert (rows, "result should be Object with `data` key to represent column info");
assert (metadata.constructor === Array, "metadata is an array with column descriptions");
assert (metadata[0].name === "number");
assert (rows.length === 10, "total 10 rows");
assert ('number' in rows[0], "each row contains fields (using FORMAT JSON)");
assert (rows[9].number === "9"); // this should be corrected at database side
assert (stream.supplemental.rows === 10);
assert (stream.supplemental.rows_before_limit_at_least === 10);
done ();
});
});
it ("selects from system columns", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.query ("SELECT * FROM system.columns", function (err, result) {
assert (!err);
done ();
});
});
it ("creates a database", function (done) {

@@ -194,0 +94,0 @@ var ch = new ClickHouse ({host: host, port: port});

test/03-torture.js
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc