elasticsearch-helper - npm Package Compare versions

Comparing version 1.0.1 to 1.0.2

example.js

@@ -8,36 +8,67 @@ const ES = require("./index")
  */
-ES.AddClient("client name","cliend ip:port");
+ES.AddClient("127.0.0.1:9200");
 
 
-//ID functions
+// Aggregation
+ES.Query("user")
+.must(
+  ES.type.term("name","jacques"),
+  ES.type.range("age",{gt:20,lte:40}),
+  ES.filter.should(
+    ES.type.term("color","blue"),
+    ES.type.term("vehicle","car")
+  )
+)
+.aggs(
+  ES.agg.date_histogram("created_date")("date_created","1d")
+    // Child aggregation to the "created_date" aggregation
+    .aggs(
+      ES.agg.terms("first_name")("data.first_name.raw")
+    )
+)
+.run()
+.then(function(response){
+  console.log(response)
+  // retrieve the "created_date" aggregation
+  var arrayAggList = response.agg("created_date")
+  var arrayValues = arrayAggList.values() // return an array of values objects. array types values will depend on the aggregation type
 
-//Retrieve a specific ID
-ES.query("index","type")
-.id("123")
-.run().then(function(hit){
-  //hit object or false
-}).catch(function(err){
-  //error
-})
+  var firstValue = arrayValues[0];
+  var valueID = firstValue.id(); // key of the value. If it is a date_histogram type it will be a moment object
+  var valueData = firstValue.data(); // value of the aggregation for this key.
 
-//Overwrite a specific ID
-ES.query("index","type")
-.id("123")
-.body({...}) //Contains the data
-.run().then(function(body){
-  //body object
-}).catch(function(err){
-  //error
-})
 
-//Update a specific ID
-ES.query("index","type")
-.id("123")
-.update({...}) //Contains the data to be updated
-.run().then(function(body){
-  //body object
+  // To retrieve a child aggregation:
+  // Note: Each parent aggregation value has its own aggregation so you will have to loop through to get the child aggregation
+  var arrayChildAggList = arrayAggList.agg("first_name");
+  for(var parentKeyvalue in arrayChildAggList){
+    arrayChildAggList[parentKeyvalue].values().forEach(function(value){
+      console.log(parentKeyvalue, value.id(),value.data());
+    })
+  }
 }).catch(function(err){
+  console.log(err)
   //error
 })
 
+// //Overwrite a specific ID
+// ES.query("index","type")
+// .id("123")
+// .body({...}) //Contains the data
+// .run().then(function(body){
+//   //body object
+// }).catch(function(err){
+//   //error
+// })
+//
+// //Update a specific ID
+// ES.query("index","type")
+// .id("123")
+// .update({...}) //Contains the data to be updated
+// .run().then(function(body){
+//   //body object
+// }).catch(function(err){
+//   //error
+// })
+
 //Query functions

index.js

 'use strict';
 
-var promise = require("bluebird");
 var ES = require("elasticsearch");
+var ElasticsearchScrollStream = require('elasticsearch-scroll-stream');
 
+var Response = require("./class/response")
+var QueryBuilder = require("./class/query-builder")
+var ConditionBuilder = require("./class/query-builder")
+var AggregationBuilder = require("./class/aggregation-builder")
+var SearchType = require("./class/search-type")
+var AggregationType = require("./class/aggregation-type")
+
 // var oESClient = false;
 var oESClientList = {};
 
+var sDefaultName = "default";
+
+/**
+ * Create a client
+ * @method AddClient with 3 arguments
+ * @param {String} Name - name of the host (ex: 'myES')
+ * @param {String} Host - host (ex: '127.0.0.1:9200')
+ * @param {Boolean} Default - set this client as default for queries
+ * @return {Void}
+ *
+ * @method AddClient with 1 arguments
+ * @param {String} Host - host (ex: '127.0.0.1:9200')
+ * @return {Void}
+ */
 var AddClient = function(){
@@ -25,2 +46,5 @@ var sName = "default";
     sHost = arguments[1];
+
+    if(arguments[2] && arguments[2] == true)
+      sDefaultName = sName;
   }
@@ -34,98 +58,23 @@
 
+/**
+ * Return a client
+ * @method getClient
+ * @param {String} Name - host (ex: 'myES')
+ * @return {Object} Client or false if not found
+ */
 var getClient = function(){
-  var sName = "default";
-
-  if(arguments.length == 0){
-    if(oESClientList["default"])
-      return oESClientList["default"];
-  }else{
-    if(oESClientList[arguments[0]])
+  if(arguments.length == 0 && oESClientList[sDefaultName]){
+      return oESClientList[sDefaultName];
+  }else if(oESClientList[arguments[0]]){
       return oESClientList[arguments[0]]
+  }else {
+      return false;
   }
 }
 
-var Response = function(p_oResponse){
-  this.oResponse = p_oResponse;
-}
 
-Response.prototype = {
-  count : function(){
-    if(!this.oResponse.hits)
-      return promise.resolve(this.oResponse.found?1:0);
 
-    return promise.resolve(this.oResponse.hits.total);
-  },
-  result: function(){
-    if(this.oResponse.found){
-      return promise.resolve(new Hit(this.oResponse))
-    }else{
-      return promise.resolve(false)
-    }
-  },
-  results: function(){
-    return promise.resolve(this.oResponse.hits.hits.map(function(o){return new Hit(o)}))
-  }
-}
 
-var Hit = function(p_oHit){
-  this.oHit = p_oHit
-}
 
-Hit.prototype = {
-  id : function(){
-    return this.oHit._id;
-  },
-  data: function(){
-    return this.oHit._source;
-  },
-  index: function(){
-    return this.oHit._index;
-  },
-  type: function(){
-    return this.oHit._type;
-  }
-}
 
-var SearchType = function(){
-  this.sKey = false;
-  this.uValue = false;
-  this.sType = false;
-}
-
-SearchType.prototype = {
-  _setValues : function(p_sType,p_sKey,p_sValue){
-    this.sKey = p_sKey;
-    this.uValue = p_sValue;
-    this.sType = p_sType;
-  },
-  term : function(p_sKey,p_sValue){
-    if(p_sValue.constructor === Array)
-      throw "cannot to be an array, use terms";
-
-    this._setValues("term",p_sKey,p_sValue)
-    return this;
-  },
-  exists: function(p_sKey){
-    this._setValues("exists","field",p_sKey)
-    return this;
-  },
-  range: function(p_sKey,p_oRange){
-    this._setValues("range",p_sKey,p_oRange)
-    return this;
-  },
-  terms : function(p_sKey,p_arrsValue){
-    if(p_arrsValue.constructor !== Array)
-      throw "needs to be an array";
-
-    this._setValues("terms",p_sKey,p_sValue)
-    return this;
-  },
-  render: function(){
-    var oObj = {}
-    oObj[this.sType] = {};
-    oObj[this.sType][this.sKey] = this.uValue;
-    return oObj;
-  }
-}
-
 var Elasticsearch = function(p_sIndex,p_sType){
@@ -139,5 +88,3 @@ this.oESClient = false;
   this.oQuery = {
-    "query": {
-      "bool": {}
-    }
+    "query": {}
   }
@@ -149,9 +96,32 @@
   this.bHasCondition = false;
-}
 
-var addType =  function(){
-  return new SearchType();
+  this.oQB = new QueryBuilder();
+  this.oAB = new AggregationBuilder();
 }
 
 Elasticsearch.prototype = {
+    must: function(){
+      this.oQB.must.apply(this.oQB,arguments)
+      return this;
+    },
+    should: function(){
+      this.oQB.should.apply(this.oQB,arguments)
+      return this;
+    },
+    filter: function(){
+      this.oQB.filter.apply(this.oQB,arguments)
+      return this;
+    },
+    must_not: function(){
+      this.oQB.must_not.apply(this.oQB,arguments)
+      return this;
+    },
+    log: function(){
+      console.log(JSON.stringify({
+        query: this.oQB.render(),
+        aggs: this.oAB.render()
+      },null,2));
+
+      return this;
+    },
     use: function(p_sClientName){
@@ -184,37 +154,6 @@ if(!oESClientList[p_sClientName])
     },
-    must: function(){
-      for(var i  = 0 ; i < arguments.length; i++){
-        this._addCondition("must",arguments[i]);
-      }
+    aggs: function(){
+      this.oAB.add.apply(this.oAB,arguments)
       return this;
     },
-    should: function(){
-      for(var i  = 0 ; i < arguments.length; i++){
-        this._addCondition("should",arguments[i]);
-      }
-      return this;
-    },
-    filter: function(){
-      for(var i  = 0 ; i < arguments.length; i++){
-        this._addCondition("filter",arguments[i]);
-      }
-      return this;
-    },
-    must_not: function(){
-      for(var i  = 0 ; i < arguments.length; i++){
-        this._addCondition("must_not",arguments[i]);
-      }
-
-     return this;
-    },
-    _addCondition: function(p_sType,p_oType){
-      if(!this.oQuery.query.bool[p_sType]) this.oQuery.query.bool[p_sType] = [];
-      this.oQuery.query.bool[p_sType].push(p_oType.render());
-      this.bHasCondition = true;
-    },
-    agg: function(p_oType,p_sParent){
-      if(!this.oQuery.aggs) this.oQuery.aggs = {};
-      this.oQuery.aggs[p_oType.getName()] = p_oType.render();
-      return this;
-    },
     delete:function(){
@@ -257,3 +196,6 @@ this.bDelete = true;
 
-           oQuery.body = self.oQuery;
+            oQuery.body = {
+              query: self.oQB.render()
+            };
+
            sType = "deleteByQuery";
@@ -277,4 +219,6 @@ }
         }else if(!self.sID){
-          oQuery.body = {};
-          oQuery.body.query = self.oQuery.query;
+          oQuery.body = {
+            query: self.oQB.render()
+          };
+
           if(self.oQuery.size) oQuery.body.size = self.oQuery.size;
@@ -284,16 +228,78 @@ if(self.oQuery._source) oQuery.body._source = self.oQuery._source;
 
-        self.oESClient[sType](oQuery,function(err,response){
-          if(err){
-            if(err.status == 404){
-              return resolve(false);
-            }else{
-              return reject(err)
+
+
+        //if size is more than 500 we do an automatic scroll
+        if(self.oQuery.size && self.oQuery.size > 500 && !self.oAB.count()){
+          var arroData = [];
+
+          oQuery.scroll = "1m";
+
+          var es_stream = new ElasticsearchScrollStream(self.oESClient, oQuery, ['_id', '_index','_type']);
+
+          es_stream.on('data',function(data){
+            var current_doc = JSON.parse(data.toString());
+
+            var _id = current_doc._id;
+            var _index = current_doc._index;
+            var _type = current_doc._type;
+
+            delete current_doc._id;
+            delete current_doc._index;
+            delete current_doc._type;
+
+            arroData.push({
+              _id: _id,
+              _index: _index,
+              _type: _type,
+              _source: current_doc
+            });
+          });
+
+          es_stream.on('end', function() {
+            var response = {
+                hits: {
+                   total : arroData.length,
+                   hits: arroData
+               }
             }
+
+            resolve((new Response(response)).results());
+          });
+
+          es_stream.on('error', function(err) {
+            reject(err);
+          });
+        }else{
+          if(sType=="search" && self.oAB.count()){
+            oQuery.body.size = 0 ;
+            oQuery.body.aggs = self.oAB.render();
           }
 
-          if(sType=="search" || sType=="get")
-            return resolve((new Response(response))[sType=="get"?"result":"results"]())
-          else
-            return resolve(self.oBody || self.oDoc || self.bDelete);
-        })
+          // console.log(JSON.stringify(oQuery,null,2))
+
+          self.oESClient[sType](oQuery,function(err,response){
+            if(err){
+              if(err.status == 404){
+                return resolve(false);
+              }else{
+                return reject(err)
+              }
+            }
+
+
+
+            if(sType=="search"){
+              if(response.aggregations){
+                response.aggregations.pattern = self.oAB;
+                return resolve(new Response(response))
+              }
+
+              return resolve((new Response(response)).results())
+            }else if(sType=="get"){
+              return resolve((new Response(response)).result())
+            }else{
+              return resolve(self.oBody || self.oDoc || self.bDelete);
+            }
+          })
+        }
       });
@@ -309,3 +315,58 @@ }
   },
-  addType:addType
+  addType: function(){
+    return new SearchType();
+  },
+  addFilter: function(){
+    return new QueryBuilder();
+  },
+  filter: {
+    should: function(){
+      var oQB = new QueryBuilder();
+      return oQB.should.apply(oQB,arguments);
+    },
+    must: function(){
+      var oQB = new QueryBuilder();
+      return oQB.must.apply(oQB,arguments);
+    },
+    filter: function(){
+      var oQB = new QueryBuilder();
+      return oQB.filter.apply(oQB,arguments);
+    },
+    must_not: function(){
+      var oQB = new QueryBuilder();
+      return oQB.must_not.apply(oQB,arguments);
+    }
+  },
+  type:{
+    term: function(){
+      var oST = new SearchType();
+      return oST.term.apply(oST,arguments);
+    },
+    terms: function(){
+      var oST = new SearchType();
+      return oST.terms.apply(oST,arguments);
+    },
+    exists: function(){
+      var oST = new SearchType();
+      return oST.exists.apply(oST,arguments);
+    },
+    range: function(){
+      var oST = new SearchType();
+      return oST.range.apply(oST,arguments);
+    }
+  },
+  agg:{
+    terms: function(p_sName){
+      return function(){
+        var oST = new AggregationType(p_sName);
+        return oST.terms.apply(oST,arguments);
+      }
+    },
+    date_histogram: function(p_sName){
+      return function(){
+        var oST = new AggregationType(p_sName);
+        return oST.date_histogram.apply(oST,arguments);
+      }
+    }
+  }
 }

package.json

 {
   "name": "elasticsearch-helper",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "A nodejs module that simplify elasticsearch querying",
@@ -25,5 +25,6 @@ "main": "index.js",
   "dependencies": {
-    "bluebird": "^3.5.1",
-    "elasticsearch": "^13.3.1"
+    "elasticsearch": "^13.3.1",
+    "elasticsearch-scroll-stream": "^1.1.3",
+    "moment": "^2.19.1"
   }
 }

README.md

 # elasticsearch-helper
+
 A nodejs module to do elasticsearch queries easily
 
+# disclaimer
+
+I experienced a lot of issues in the past due to the way Elasticsearch handles the queries. I decided to create helper that we currently use on production level at https://headhunterportal.com and some other projects and so far it had helped us to drastically reduce the complexity of readability of our code.
+
+With this helper you will be able to query your elasticsearch clusters very easily. Everything is chainable and the query always returns a promise.
+
+NOTE: Even if we use this on production level still find bugs and add improvements to the module codebase. Feel free to for it and modify it for your own needs.
+
 # installation
-run `npm install --save elasticsearch-helper`
 
+`npm install --save elasticsearch-helper`
+
 # usage
 
 ## add client
+
 ```javascript
@@ -18,2 +29,5 @@ let esH = require("elasticsearch-helper")
 esH.AddClient("client1","127.0.0.1:9200");
+
+// Will create a client with name "client1" and will be used as default
+esH.AddClient("client1","127.0.0.1:9200",true);
 ```
@@ -23,6 +37,6 @@
 
-The client is chainable which means that you can call functions one after the other until you execute the query.
-The query is then returning a promise.
+The client is chainable which means that you can call functions one after the other until you execute the query. The query is then returning a promise.
 
 Initialise a query:
+
 ```javascript
@@ -34,2 +48,5 @@ const esH = require("elasticsearch-helper")
 
+// Querying on all indexes starting with "Index"
+esH.query("Index*");
+
 // Querying on index "Index1" and type "Type1"
@@ -45,4 +62,5 @@ esH.query("Index1","Type1");
 For those example we will use the query variable 'q':
+
 ```javascript
-//initialise query
+// initialise query
 var q = esH.query("Index1","Type1");
@@ -53,3 +71,4 @@ ```
 
-#### GET
+#### Retrieve
+
 ```javascript
@@ -59,7 +78,12 @@ q.id("ID")
   .then(function(hit){
-  //return hit object or false if not found
+  // return hit object or false if not found
+  console.log(hit.id()) // get Document ID
+  console.log(hit.index()) // get Document index
+  console.log(hit.type()) // get Document type
+  console.log(hit.data()) // get Document source
 })
 ```
 
-#### DELETE
+#### Delete
+
 ```javascript
@@ -69,23 +93,25 @@ q.id("ID")
   .then(function(hit){
-  //return true
+  // return true
 })
 ```
 
-#### CREATE/OVERWRITE
+#### Create/Overwrite
+
 ```javascript
 q.id("ID")
-q.body({...}) //Data object to store
+q.body({...}) // Data object to store
 q.run()
   .then(function(hit){
-  //return the data object
+  // return the data object
 })
 ```
 
-#### UPDATE
+#### Update
+
 ```javascript
 q.id("ID")
-q.update({...}) //Data object to update 
+q.update({...}) // Data object to update
 q.run()
   .then(function(hit){
-  //return the data object
+  // return the data object
 })
@@ -100,45 +126,56 @@ ```
 
-GETs and DELETEs are using the same methodology for querying building. 
-Example:
+GETs and DELETEs are using the same methodology for querying building. Example:
+
 ```javascript
 q.must(
-  //Term type
-  esH.addType().term("fieldname","fieldvalue"),
-  //Terms type
-  esH.addType().terms("fieldname","fieldvalues"),
-  //Exists Type
-  esH.addType().exists("fieldname"),
-  //Range Type
-  esH.addType().range({
+  // Term type
+  esH.type.term("fieldname","fieldvalue"),
+  // Terms type
+  esH.type.terms("fieldname","fieldvalues"),
+  // Exists Type
+  esH.type.exists("fieldname"),
+  // Range Type
+  esH.type.range({
     gte:1,
     lte:10
-  })
+  }),
+  // Add a sub filter in the query
+  esH.filter.should(
+    esH.type.terms("fieldname2","fieldvalues")
+  )
 )
 
-//Other query methods include
-//It is not possible to do nested booleans
+// Other query methods include
+// It is not possible to do nested booleans
 
 q.must_not(
-  //Types
+  // Types
 )
 
 q.should(
-  //Types
+  // Types
 )
 
 q.filter(
-  //Types
+  // Types
 )
 ```
 
-#### GET
+#### Retrieve
+
 ```javascript
 q.must(
-  //Types
+  // Types
 ).run().then(function(hits){
-  //return array of hits objects
+  // return array of hits objects
+  var hit = hits[0];
+  console.log(hit.id()) // get Document ID
+  console.log(hit.index()) // get Document index
+  console.log(hit.type()) // get Document type
+  console.log(hit.data()) // get Document source
 })
 ```
 
-#### DELETE
+#### Delete
+
 Delete by query is only avalaible on Elasticsearch 5.X
@@ -148,19 +185,62 @@
 q.must(
-  //Types
+  // Types
 ).delete().then(function(hits){
-  //return array of hits objects
+  // return array of hits objects
 })
 ```
 
+#### aggregations // BETA
+
+Elasticsearch has a very powerful aggregation system but the way to handle it can be tricky. I tried to solve this issue by wrapping it in what I think is the simplest way.
+
+NOTE: Right now I only handle 2 types of aggregation, `terms` and `date_histogram`
+
+```javascript
+q.aggs(
+  ES.agg.date_histogram("created_date")("date_created","1d")
+    // Child aggregation to the "created_date" aggregation
+    .aggs(
+      ES.agg.terms("first_name")("data.first_name")
+    )
+  // Add more aggregations
+).run()
+.then(function(response){
+  // retrieve the "created_date" aggregation
+  var arrayAggList = response.agg("created_date")
+  var arrayValues = arrayAggList.values() // return an array of values objects. array types values will depend on the aggregation type
+
+  var firstValue = arrayValues[0];
+  var valueID = firstValue.id(); // key of the value. If it is a date_histogram type it will be a moment object
+  var valueData = firstValue.data(); // value of the aggregation for this key.
+
+
+  // To retrieve a child aggregation:
+  // Note: Each parent aggregation value has its own aggregation so you will have to loop through to get the child aggregation
+
+  var arrayChildAggList = arrayAggList.agg("first_name");
+  for(var parentKeyvalue in arrayChildAggList){
+    arrayChildAggList[parentKeyvalue].values().forEach(function(value){
+      console.log(parentKeyvalue, value.id(),value.data());
+    })
+  }
+
+})
+```
+
 #### Other options
+
 ```javascript
 
-//will retrieve 1000 results maximum
+// will retrieve 1000 results maximum
+// all queries with a size over 500 will be converted into a scroll.
 q.size(1000)
 
-//will retrieve the documents values for specific keys
+// will retrieve the documents values for specific keys
 q.fields("name","id")
 ```
 
-## Full example
+## Examples
+
+
+#### Query
 ```javascript
@@ -190,1 +270,50 @@ let esH = require("elasticsearch-helper")
 
+#### Query with aggregation
+
+```javascript
+let esH = require("elasticsearch-helper")
+
+esH.AddClient("client1","127.0.0.1:9200");
+
+esH.Query("user")
+.size(1001) // when an aggregation is set, size is set to 0.
+.must(
+  esH.type.term("name","jacques"),
+  esH.type.range("age",{gt:20,lte:40}),
+  esH.filter.should(
+    esH.type.term("color","blue"),
+    esH.type.term("vehicle","car")
+  )
+)
+.aggs(
+  esH.agg.date_histogram("created_date")("date_created","1d")
+    // Child aggregation to the "created_date" aggregation
+    .aggs(
+      esH.agg.terms("first_name")("data.first_name.raw")
+    )
+)
+.run()
+.then(function(response){
+  // retrieve the "created_date" aggregation
+  var arrayAggList = response.agg("created_date")
+  var arrayValues = arrayAggList.values() // return an array of values objects. array types values will depend on the aggregation type
+
+  var firstValue = arrayValues[0];
+  var valueID = firstValue.id(); // key of the value. If it is a date_histogram type it will be a moment object
+  var valueData = firstValue.data(); // value of the aggregation for this key.
+
+
+  // To retrieve a child aggregation:
+  // Note: Each parent aggregation value has its own aggregation so you will have to loop through to get the child aggregation
+  var arrayChildAggList = arrayAggList.agg("first_name");
+  for(var parentKeyvalue in arrayChildAggList){
+    arrayChildAggList[parentKeyvalue].values().forEach(function(value){
+      console.log(parentKeyvalue, value.id(),value.data());
+    })
+  }
+}).catch(function(err){
+  // error
+  console.log(err)
+})
+
+```

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

26022

increased by127.01%

Number of package files

13

increased by160%

Lines of code

648

increased by118.92%

Number of lines in readme file

310

increased by71.27%

Worsened metrics

Dependency count

3

increased by50%

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Addedelasticsearch-scroll-stream@^1.1.3

• + Addedmoment@^2.19.1

• + Addedelasticsearch-scroll-stream@1.3.5(transitive)

• + Addedmoment@2.30.1(transitive)

• - Removedbluebird@^3.5.1

• - Removedbluebird@3.7.2(transitive)