Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

flexsearch

Package Overview
Dependencies
Maintainers
1
Versions
87
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

flexsearch - npm Package Compare versions

Comparing version 0.3.21 to 0.3.51

dist/flexsearch.compact.js
dist/flexsearch.es5.js
dist/flexsearch.light.js
dist/flexsearch.min.js
dist/flexsearch.node.js
test/benchmark-presets.html
test/matching-presets.html

5

compile.js

@@ -75,2 +75,3 @@ var child_process = require('child_process');

language_out: language_out || "ECMASCRIPT6_STRICT",
//rewrite_polyfills: false,
process_closure_primitives: true,

@@ -116,3 +117,3 @@ summary_detail_level: 3,

exec("java -jar node_modules/google-closure-compiler-java/compiler.jar" + parameter + "' --js='flexsearch.js' --js='lang/**.js' --js='!lang/**.min.js'" + flag_str + " --js_output_file='flexsearch." + (options["RELEASE"] || "custom") + ".js' && exit 0", function(){
exec("java -jar node_modules/google-closure-compiler-java/compiler.jar" + parameter + "' --js='flexsearch.js' --js='lang/**.js' --js='!lang/**.min.js'" + flag_str + " --js_output_file='dist/flexsearch." + (options["RELEASE"] || "custom") + ".js' && exit 0", function(){

@@ -123,2 +124,3 @@ var filename = "flexsearch." + (options["RELEASE"] || "custom") + ".js";

/*
if(release === "es5"){

@@ -133,2 +135,3 @@

}
*/
});

@@ -135,0 +138,0 @@ }

25

package.json
{
"name": "flexsearch",
"version": "0.3.21",
"version": "0.3.51",
"description": "Next-Generation full text search library with zero dependencies.",

@@ -10,6 +10,7 @@ "homepage": "https://github.com/nextapps-de/flexsearch/",

"keywords": [
"full text search",
"fulltext search",
"elastic search",
"fastest search",
"contextual search"
"contextual search",
"fuzzy search"
],

@@ -20,3 +21,4 @@ "bugs": {

},
"main": "flexsearch.min.js",
"main": "dist/flexsearch.node.js",
"browser": "dist/flexsearch.min.js",
"preferGlobal": false,

@@ -29,9 +31,10 @@ "bin": {},

"scripts": {
"build": "node compile RELEASE=min DEBUG=false PROFILER=false SUPPORT_WORKER=true SUPPORT_ENCODER=true SUPPORT_CACHE=true SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false",
"build-light": "node compile RELEASE=light DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=false SUPPORT_CACHE=false SUPPORT_ASYNC=false SUPPORT_PRESETS=false SUPPORT_SUGGESTIONS=false SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false",
"build-compact": "node compile RELEASE=compact DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=true SUPPORT_CACHE=false SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=false SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false",
"build-custom": "node compile RELEASE=custom DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=false SUPPORT_CACHE=false SUPPORT_ASYNC=false SUPPORT_PRESETS=false SUPPORT_SUGGESTIONS=false SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false",
"build-es5": "node compile RELEASE=es5 DEBUG=false PROFILER=false SUPPORT_WORKER=true SUPPORT_ENCODER=true SUPPORT_CACHE=true SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false LANGUAGE_OUT=ECMASCRIPT5_STRICT",
"build": "node compile RELEASE=min DEBUG=false PROFILER=false SUPPORT_WORKER=true SUPPORT_ENCODER=true SUPPORT_CACHE=true SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=true SUPPORT_SERIALIZE=true SUPPORT_INFO=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false",
"build-light": "node compile RELEASE=light DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=false SUPPORT_CACHE=false SUPPORT_ASYNC=false SUPPORT_PRESETS=false SUPPORT_SUGGESTIONS=false SUPPORT_SERIALIZE=false SUPPORT_INFO=false SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false",
"build-compact": "node compile RELEASE=compact DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=true SUPPORT_CACHE=false SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=false SUPPORT_SERIALIZE=false SUPPORT_INFO=false SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false",
"build-custom": "node compile RELEASE=custom DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=false SUPPORT_CACHE=false SUPPORT_ASYNC=false SUPPORT_PRESETS=false SUPPORT_SUGGESTIONS=false SUPPORT_SERIALIZE=false SUPPORT_INFO=false SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false",
"build-es5": "node compile RELEASE=es5 DEBUG=true PROFILER=false SUPPORT_WORKER=true SUPPORT_ENCODER=true SUPPORT_CACHE=true SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=true SUPPORT_SERIALIZE=true SUPPORT_INFO=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false LANGUAGE_OUT=ECMASCRIPT5_STRICT",
"build-node": "node compile RELEASE=node DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=true SUPPORT_CACHE=true SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=true SUPPORT_SERIALIZE=true SUPPORT_INFO=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false",
"build-lang": "node compile RELEASE=lang",
"build-all": "npm run build && npm run build-light && npm run build-compact && npm run build-es5 && npm run build-lang",
"build-all": "npm run build && npm run build-light && npm run build-compact && npm run build-es5 && npm run build-node && npm run build-lang",
"test-production": "nyc --reporter=html --reporter=text mocha --timeout=3000 test --exit",

@@ -47,3 +50,3 @@ "test-light": "nyc --reporter=html --reporter=text mocha --timeout=3000 test/ --exit",

"flexsearch.js",
"flexsearch.min.js",
"dist/",
"lang/",

@@ -50,0 +53,0 @@ "test/",

556

README.md

@@ -10,15 +10,15 @@ <p align="center">

<a target="_blank" href="https://www.codacy.com/app/ts-thomas/FlexSearch?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=nextapps-de/flexsearch&amp;utm_campaign=Badge_Grade"><img src="https://api.codacy.com/project/badge/Grade/a896e010f6b4429aa7bc9a89550320a7"/></a>
<a target="_blank" href="https://github.com/nextapps-de/flexsearch/issues"><img src="https://img.shields.io/github/issues/nextapps-de/xone.svg"></a>
<a target="_blank" href="https://github.com/nextapps-de/flexsearch/issues"><img src="https://img.shields.io/github/issues/nextapps-de/flexsearch.svg"></a>
<!--<img src="https://badges.greenkeeper.io/nextapps-de/flexsearch.svg">-->
<a target="_blank" href="https://github.com/nextapps-de/flexsearch/blob/master/LICENSE.md"><img src="https://img.shields.io/npm/l/xone.svg"></a>
<a target="_blank" href="https://github.com/nextapps-de/flexsearch/blob/master/LICENSE.md"><img src="https://img.shields.io/npm/l/flexsearch.svg"></a>
</p>
<h1></h1>
<h3>World's fastest and most memory efficient full text search library with zero dependencies.</h3>
<h3>Web's fastest and most memory-flexible full-text search library with zero dependencies.</h3>
When it comes to raw search speed <a href="https://rawgit.com/nextapps-de/flexsearch/master/test/benchmark.html" target="_blank">FlexSearch outperforms every single searching library out there</a> and also provides flexible search capabilities like multi-word matching, phonetic transformations or partial matching.
It also has the <a href="#memory">most memory-efficient index</a>. Keep in mind that updating and/or removing existing items from the index has a significant cost. When your index needs to be updated continuously then <a href="bulksearch/" target="_blank">BulkSearch</a> may be a better choice.
Depending on the used options it also providing the <a href="#memory">most memory-efficient index</a>. Keep in mind that updating and/or removing existing items from the index has a significant cost. When your index needs to be updated very often then <a href="https://github.com/nextapps-de/bulksearch" target="_blank">BulkSearch</a> may be a better choice.
FlexSearch also provides you a non-blocking asynchronous processing model as well as web workers to perform any updates or queries on the index in parallel through dedicated balanced threads.
<a href="#installation">Installation Guide</a> &ensp;&bull;&ensp; <a href="#api">API Reference</a> &ensp;&bull;&ensp; <a href="#profiles">Example Options</a> &ensp;&bull;&ensp; <a href="#builds">Custom Builds</a>
<a href="#installation">Installation Guide</a> &ensp;&bull;&ensp; <a href="#api">API Reference</a> &ensp;&bull;&ensp; <a href="#profiles">Example Options</a> &ensp;&bull;&ensp; <a href="#builds">Custom Builds</a> &ensp;&bull;&ensp; <a target="_blank" href="https://github.com/nextapps-de/flexsearch-server">Flexsearch Server</a>

@@ -29,2 +29,4 @@ Supported Platforms:

> FlexSearch Server is also available here: <a target="_blank" href="https://github.com/nextapps-de/flexsearch-server">https://github.com/nextapps-de/flexsearch-server</a>
Library Comparison:

@@ -46,4 +48,4 @@ - <a href="https://rawgit.com/nextapps-de/flexsearch/master/test/benchmark.html" target="_blank">Benchmark "Gulliver's Travels"</a>

<td>flexsearch.min.js</td>
<td><a href="https://github.com/nextapps-de/flexsearch/raw/master/flexsearch.min.js" target="_blank">Download</a></td>
<td><a href="https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/flexsearch.min.js" target="_blank">https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/flexsearch.min.js</a></td>
<td><a href="https://github.com/nextapps-de/flexsearch/raw/master/dist/flexsearch.min.js" target="_blank">Download</a></td>
<td><a href="https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/dist/flexsearch.min.js" target="_blank">https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/dist/flexsearch.min.js</a></td>
</tr>

@@ -53,4 +55,4 @@ <tr></tr>

<td>flexsearch.light.js</td>
<td><a href="https://github.com/nextapps-de/flexsearch/raw/master/flexsearch.light.js" target="_blank">Download</a></td>
<td><a href="https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/flexsearch.light.js" target="_blank">https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/flexsearch.light.js</a></td>
<td><a href="https://github.com/nextapps-de/flexsearch/raw/master/dist/flexsearch.light.js" target="_blank">Download</a></td>
<td><a href="https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/dist/flexsearch.light.js" target="_blank">https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/dist/flexsearch.light.js</a></td>
</tr>

@@ -60,4 +62,4 @@ <tr></tr>

<td>flexsearch.compact.js</td>
<td><a href="https://github.com/nextapps-de/flexsearch/raw/master/flexsearch.compact.js" target="_blank">Download</a></td>
<td><a href="https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/flexsearch.compact.js" target="_blank">https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/flexsearch.compact.js</a></td>
<td><a href="https://github.com/nextapps-de/flexsearch/raw/master/dist/flexsearch.compact.js" target="_blank">Download</a></td>
<td><a href="https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/dist/flexsearch.compact.js" target="_blank">https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@master/dist/flexsearch.compact.js</a></td>
</tr>

@@ -92,3 +94,3 @@ <tr></tr>

<td>
<a href="#async">Async Processing</a>
<a href="#async_search">Async Search</a>
</td>

@@ -182,5 +184,5 @@ <td>x</td>

<td>File Size (gzip)</td>
<td>6.7 kb</td>
<td>4.5 kb</td>
<td>4.1 kb</td>
<td>5.0 kb</td>
<td>3.9 kb</td>
<td>2.7 kb</td>
</tr>

@@ -204,11 +206,13 @@ </table>

<td align="left">Library Version</td>
<td align="left">Library Size</td>
<td align="left">Operations per second</td>
<td align="left">Single Phrase (op/s)</td>
<td align="left">Multi Phrase (op/s)</td>
<td align="left">Not Found (op/s)</td>
</tr>
<tr>
<td>1</td>
<td>FlexSearch</td>
<td>0.3.2</td>
<td>4.1 kb</td>
<td><b>333738</b></td>
<td>FlexSearch <a href="#notes">***</a></td>
<td>0.3.3</td>
<td><b>363757</b></td>
<td><b>182603</b></td>
<td><b>1627219</b></td>
</tr>

@@ -220,4 +224,5 @@ <tr></tr>

<td>0.3.3</td>
<td>1.6 kb</td>
<td><b>1587</b></td>
<td><b>899</b></td>
<td><b>6098</b></td>
<td><b>214286</b></td>
</tr>

@@ -229,4 +234,5 @@ <tr></tr>

<td>1.4.2</td>
<td>3.8 kb</td>
<td><b>771</b></td>
<td><b>735</b></td>
<td><b>8889</b></td>
<td><b>800000</b></td>
</tr>

@@ -238,4 +244,5 @@ <tr></tr>

<td>1.0</td>
<td>3.9 kb</td>
<td><b>584</b></td>
<td><b>551</b></td>
<td><b>9970</b></td>
<td><b>75000</b></td>
</tr>

@@ -247,4 +254,5 @@ <tr></tr>

<td>2.3.5</td>
<td>8.8 kb</td>
<td><b>322</b></td>
<td><b>355</b></td>
<td><b>1051</b></td>
<td><b>25000</b></td>
</tr>

@@ -256,4 +264,5 @@ <tr></tr>

<td>0.9.6</td>
<td>5.6 kb</td>
<td><b>319</b></td>
<td><b>327</b></td>
<td><b>781</b></td>
<td><b>6667</b></td>
</tr>

@@ -265,4 +274,5 @@ <tr></tr>

<td>0.1.3</td>
<td>3.1 kb</td>
<td><b>265</b></td>
<td><b>535</b></td>
<td><b>2778</b></td>
</tr>

@@ -274,4 +284,5 @@ <tr></tr>

<td>0.2</td>
<td>3.5 kb</td>
<td><b>107</b></td>
<td><b>71</b></td>
<td><b>116</b></td>
<td><b>2065</b></td>
</tr>

@@ -283,4 +294,5 @@ <tr></tr>

<td>3.3.0</td>
<td>3.7 kb</td>
<td><b>1</b></td>
<td><b>0.5</b></td>
<td><b>0.4</b></td>
<td><b>0.7</b></td>
</tr>

@@ -302,3 +314,3 @@ </table>

<td>1</td>
<td>FlexSearch</td>
<td>FlexSearch <a href="#notes">****</a></td>
<td>0.3.1</td>

@@ -376,3 +388,5 @@ <td>1.33 Mb</td>

_* Index Size: The size of memory the index requires_<br>
_** Memory Allocation: The amount of memory which was additionally allocated during a row of 10 queries_
_** Memory Allocation: The amount of memory which was additionally allocated during a row of 10 queries_<br>
_*** The preset "fastest" was used for this test_ <br>
_**** The preset "memory" was used for this test_

@@ -382,4 +396,6 @@ Library Comparison: <a href="https://rawgit.com/nextapps-de/flexsearch/master/test/benchmark.html" target="_blank">Benchmark "Gulliver's Travels"</a>

<a name="contextual"></a>
#### Contextual Search
## Contextual Search
> "TF-IDF and all kinds of variations (like BM25) is a big mistake in searching algorithms today. They don't provide neither: a meaningful relevance of a term nor the importance of it! Like many pseudo-intelligent algorithms this is also just an example of mathematical stupidity." — Thomas Wilkerling, _Contextual-based Scoring_, 2018
FlexSearch introduce a new scoring mechanism called __Contextual Search__ which was invented by Thomas Wilkerling, the author of this library. A Contextual Search <a href="https://rawgit.com/nextapps-de/flexsearch/master/test/benchmark.html" target="_blank">incredibly boost up queries to a complete new level</a> but also requires a lot of additionally memory.

@@ -394,15 +410,4 @@ The basic idea of this concept is to limit relevance by its context instead of calculating relevance through the whole (unlimited) distance.

__Note:__ This feature is actually not enabled by default.
__Note:__ This feature is actually not enabled by default. Read <a href="#contextual_enable">here</a> how to enable.
###### TF-IDF / BM25?
> "TF-IDF and all kinds of variations (like BM25) is a big mistake in searching algorithms today. They don't provide neither: a meaningful relevance of a term nor the importance of it! Like many pseudo-intelligent algorithms this is also just an example of mathematical stupidity." — Thomas Wilkerling, _Contextual-based Scoring_, 2018
<a name="webworker"></a>
#### Web-Worker Support
Workers get its own dedicated memory. Especially for larger indexes, web worker improves speed and available memory a lot. FlexSearch index was tested with a 250 Mb text file including 10 Million words. The indexing was done silently in background by multiple parallel running workers in about 7 minutes. The final index reserves ~ 8.2 Mb memory/space. The search result took ~ 0.25 ms.
__Note:__ It is slightly faster to use no web worker when the index or query isn't too big (index < 500,000 words, query < 25 words).
#### Compare BulkSearch vs. FlexSearch

@@ -413,5 +418,5 @@

<tr>
<td align="left">Description</th>
<td align="left">BulkSearch</th>
<td align="left">FlexSearch</th>
<td align="left"></th>
<td align="left"><b>BulkSearch</b></th>
<td align="left"><b>FlexSearch</b></th>
</tr>

@@ -438,3 +443,3 @@ <tr>

<td>Strength</td>
<td><ul><li>fast adds / fast updates / fast removals</li></ul></td>
<td><ul><li>fast adds</li><li>fast updates</li><li>fast removals</li></ul></td>
<td><ul><li>fast queries</li><li>memory-efficient index</li></ul></td>

@@ -446,3 +451,3 @@ </tr>

<td><ul><li>less powerful contextual search</li><li>less memory efficient (has to be defragmented from time to time)</li></ul></td>
<td><ul><li>updating / deleting extisting items from index is slow</li><li>adding items to the index optimized for super partial matching (tokenize: "full") is slow</li></ul></td>
<td><ul><li>updating / deleting existing items from index is slow</li><li>adding items to the index optimized for super partial matching (tokenize: "full") is slow</li></ul></td>
</tr>

@@ -498,4 +503,6 @@ <tr></tr>

##### HTML / Javascript
#### HTML / Javascript
> Use _flexsearch.min.js_ for production and _flexsearch.js_ for development.
```html

@@ -508,11 +515,21 @@ <html>

```
__Note:__ Use _flexsearch.min.js_ for production and _flexsearch.js_ for development.
Use latest from CDN:
```html
<script src="https://cdn.rawgit.com/nextapps-de/flexsearch/master/flexsearch.min.js"></script>
<script src="https://cdn.rawgit.com/nextapps-de/flexsearch/master/dist/flexsearch.min.js"></script>
```
##### Node.js
Or a specific version:
```html
<script src="https://cdn.rawgit.com/nextapps-de/flexsearch/0.3.51/dist/flexsearch.min.js"></script>
```
__AMD__
```javascript
var FlexSearch = require("./flexsearch.js");
```
#### Node.js
```npm

@@ -534,8 +551,2 @@ npm install flexsearch

__AMD__
```javascript
var FlexSearch = require("./flexsearch.js");
```
<a name="api"></a>

@@ -563,2 +574,4 @@ ## API Overview

- <a href="#index.encode">Index.__encode__(string)</a>
- <a href="#index.export">Index.__export__()</a>
- <a href="#index.import">Index.__import__(string)</a>

@@ -581,3 +594,3 @@ ## Usage

##### Create a new index and choosing one of the built-in profiles
Create a new index and choosing one of the built-in profiles:

@@ -588,3 +601,3 @@ ```js

##### Create a new index with custom options
Create a new index with custom options:

@@ -596,6 +609,6 @@ ```js

profile: "balance",
encode: "icase",
tokenize: "ngram",
encode: "balance",
tokenize: "forward",
async: false,
worker: false,
cache: false

@@ -610,3 +623,3 @@ });

> Index.__add___(id, string)
> Index.__add(id, string)__

@@ -619,3 +632,3 @@ ```js

> Index.__search(string|options, \<limit\>, \<callback\>)__
> Index.__search(string | options, \<limit\>, \<callback\>)__

@@ -632,2 +645,4 @@ ```js

<a name="async_search"></a>
#### Async Search
Perform queries asynchronously:

@@ -642,2 +657,4 @@

> Passing a callback always will perform as asynchronous even if the "async" option was not set.
Perform queries asynchronously (Promise-based):

@@ -658,8 +675,8 @@

async function search(query){
return await index.search(query);
const result = await index.search(query);
}
const result = search("John");
```
#### Custom Search
Pass custom options for each query:

@@ -693,2 +710,3 @@

#### Suggestions
Get also suggestions for a query:

@@ -707,5 +725,5 @@

<a name="index.update"></a>
#### Update item to the index
#### Update item of an index
> Index.__update__(id, string)
> Index.__update(id, string)__

@@ -719,3 +737,3 @@ ```js

> Index.__remove__(id)
> Index.__remove(id)__

@@ -740,8 +758,6 @@ ```js

<a name="index.init"></a>
#### Re-Initialize index
#### Re-Initialize the index
> Index.__init(\<options\>)__
__Note:__ Re-initialization will also destroy the old index!
Initialize (with same options):

@@ -759,2 +775,5 @@ ```js

```
> Re-initialization will also destroy the old index.
<a name="flexsearch.addmatcher"></a>

@@ -774,2 +793,3 @@ #### Add custom matcher

```
<a name="index.addmatcher"></a>

@@ -785,6 +805,7 @@ Add private matchers for a specific instance:

```
<a name="flexsearch.encoder"></a>
#### Add custom encoder
Define a private custom encoder during creation/initialization:
Assign a custom encoder by passing a function during index creation/initialization:
```js

@@ -801,32 +822,33 @@ var index = new FlexSearch({

```
Call a custom encoder directly:
```js
var encoded = index.encode("sample text");
```
<a name="flexsearch.register"></a>
##### Register a global encoder to be used by all instances
#### Register a global encoder
> FlexSearch.__registerEncoder(name, encoder)__
Global encoders can be shared/used by all instances.
```js
FlexSearch.registerEncoder('whitespace', function(str){
FlexSearch.registerEncoder("whitespace", function(str){
return str.replace(/ /g, '');
return str.replace(/\s/g, "");
});
```
Use global encoders:
Initialize index and assign a global encoder:
```js
var index = new FlexSearch({ encode: 'whitespace' });
var index = new FlexSearch({ encode: "whitespace" });
```
<a name="index.encode"></a>
##### Call encoders directly
Private encoder:
Call a global encoder directly:
```js
var encoded = index.encode("sample text");
```
<a name="flexsearch.encode"></a>
Global encoder:
```js
var encoded = FlexSearch.encode("whitespace", "sample text");
```
##### Mixup/Extend multiple encoders
#### Mix/Extend multiple encoders

@@ -836,21 +858,16 @@ ```js

str = this.encode("icase", str); // built-in
str = this.encode("icase", str); // built-in
str = this.encode("whitespace", str); // custom
// do something additional with str ...
return str;
});
```
```js
FlexSearch.registerEncoder('extended', function(str){
str = this.encode("custom", str);
// do something additional with str ...
return str;
});
```
<a name="flexsearch.tokenizer"></a>
#### Add custom tokenizer
> A tokenizer split words into components or chunks.
Define a private custom tokenizer during creation/initialization:

@@ -861,9 +878,8 @@ ```js

tokenize: function(str){
// split string into components, e.g.:
return str.split(/ -\//g);
return str.split(/\s-\//g);
}
});
```
<a name="flexsearch.language"></a>

@@ -876,3 +892,3 @@ #### Add language-specific stemmer and/or filter

Define a private custom stemmer or filter during creation/initialization:
Assign a private custom stemmer or filter during creation/initialization:
```js

@@ -903,11 +919,14 @@ var index = new FlexSearch({

Or assign stemmer/filters globally to a language:
> Stemmer are passed as a object (key-value-pair), filter as an array.
```js
FlexSearch.registerLanguage('us', {
FlexSearch.registerLanguage("us", {
stemmer: {/* ... */},
filter: [/* ... */]
stemmer: { /* ... */ },
filter: [ /* ... */ ]
});
```
Or use built-in stemmer or filter of your preferred languages:
Or use some pre-defined stemmer or filter of your preferred languages:
```html

@@ -925,11 +944,19 @@ <html>

```js
var index_en = new FlexSearch({stemmer: 'en', filter: 'en'});
var index_de = new FlexSearch({stemmer: 'de', filter: [/* custom */]});
var index_en = new FlexSearch({
stemmer: "en",
filter: "en"
});
var index_de = new FlexSearch({
stemmer: "de",
filter: [ /* custom */ ]
});
```
In Node.js you just need require the language pack files to make them available:
In Node.js you just have to require the language pack files to make them available:
```js
require('lang/en.js');
require('lang/de.js');
require("flexsearch.js");
require("lang/en.js");
require("lang/de.js");
```

@@ -944,4 +971,6 @@

<a name="index.info"></a>
#### Get info about an index
### Get info about an index
> This feature is available in _DEBUG_ mode.
```js

@@ -955,11 +984,18 @@ index.info();

{
"bytes": 64000,
"id": 0,
"memory": 10000,
"items": 500,
"sequences": 3000,
"matchers": 0,
"size": 10000,
"status": false
"chars": 3500,
"cache": false,
"matcher": 0,
"worker": false,
"threshold": 7,
"depth": 3,
"contextual": true
}
```
<a name="chaining"></a>
#### Chaining
### Chaining

@@ -979,5 +1015,6 @@ Simply chain methods like:

#### Enable Contextual Index
<a name="contextual_enable"></a>
### Enable Contextual Scoring
Create index and just set the limit of relevance ("depth"):
Create an index and just set the limit of relevance as "depth":
```js

@@ -988,2 +1025,3 @@ var index = new FlexSearch({

tokenize: "strict",
threshold: 7,
depth: 3

@@ -993,4 +1031,10 @@ });

> Only the tokenizer "strict" is actually supported by the contextual index.
> The contextual index requires <a href="#memory">additional amount of memory</a> depending on depth.
> Try to use the __lowest depth__ and __highest threshold__ which fits your needs.
<a name="cache"></a>
#### Enable Auto-Balanced Cache
### Enable Auto-Balanced Cache

@@ -1006,4 +1050,13 @@ Create index and just set a limit of cache entries:

#### Use WebWorker Sharding (Browser only)
> When passing a number as a limit the cache automatically balance stored entries related to their popularity.
> When just using "true" the cache is unbounded and perform actually 2-3 times faster (because the balancer do not have to run).
<a name="webworker"></a>
### WebWorker Sharding (Browser only)
Worker get its own dedicated memory and also run in their own dedicated thread without blocking the UI while processing. Especially for larger indexes, web worker improves speed and available memory a lot. FlexSearch index was tested with a 250 Mb text file including 10 Million words. <!--The indexing was done silently in background by multiple parallel running workers in about 7 minutes. The final index reserves ~ 8.2 Mb memory/space. The search result took ~ 0.25 ms.-->
> When the index isn't big enough it is faster to use no web worker.
Create index and just set the count of parallel threads:

@@ -1033,2 +1086,10 @@ ```js

Or use promises accordingly:
```js
index.search("John Doe").then(function(results){
// do something with array of results
});
```
<a name="options"></a>

@@ -1062,8 +1123,8 @@ ## Options

<tr>
<td align="top">tokenize<br><br><br><br><br><br></td>
<td align="top">tokenize<br><br><br><br><br><!--<br>--></td>
<td vertical="top" vertical-align="top">
"strict"<br>
"foward"<br>
"forward"<br>
"reverse"<br>
"ngram"<br>
<!--"ngram"<br>-->
"full"<br>

@@ -1198,2 +1259,3 @@ function()

<tr></tr>
<!--
<tr>

@@ -1206,4 +1268,5 @@ <td><b>"ngram"</b> (default)</td>

<tr></tr>
-->
<tr>
<td><b>"foward"</b></td>
<td><b>"forward"</b></td>
<td>incrementally index words in forward direction</td>

@@ -1295,6 +1358,6 @@ <td><b>fo</b>obar<br><b>foob</b>ar<br></td>

<td align="left">Query</td>
<td align="left">iCase</td>
<td align="left">Simple</td>
<td align="left">Advanced</td>
<td align="left">Extra</td>
<td align="left">icase</td>
<td align="left">simple</td>
<td align="left">advanced</td>
<td align="left">extra</td>
</tr>

@@ -1440,2 +1503,3 @@ <tr>

</tr>
<!--
<tr></tr>

@@ -1446,2 +1510,3 @@ <tr>

</tr>
-->
<tr></tr>

@@ -1481,7 +1546,7 @@ <tr>

<a name="profiles"></a>
## Built-in Profiles
## Presets
You can pass a built-in profile during creation/initialization. They have these following settings:
You can pass a preset during creation/initialization. They represents these following settings:
Standard profile: __"default"__
__"default"__: Standard profile
```js

@@ -1494,3 +1559,3 @@ {

Memory-optimized profile: __"memory"__
__"memory"__: Memory-optimized profile
```js

@@ -1504,3 +1569,3 @@ {

Speed-optimized profile: __"speed"__
__"speed"__: Speed-optimized profile

@@ -1516,3 +1581,3 @@ ```js

Matching-tolerant profile: __"match"__
__"match"__: Matching-tolerant profile

@@ -1526,3 +1591,3 @@ ```js

Relevance-optimized profile: __"score"__
__"score"__: Relevance-optimized profile

@@ -1538,8 +1603,8 @@ ```js

Most-balanced profile: __"balanced"__
__"balance"__: Most-balanced profile
```js
{
encode: "balanced",
tokenize: "ngram",
encode: "balance",
tokenize: "strict",
threshold: 6,

@@ -1550,3 +1615,3 @@ depth: 3

Absolute fastest profile: __"fastest"__
__"fastest"__: Absolute fastest profile

@@ -1561,8 +1626,186 @@ ```js

<!--
Compare these options above:
- <a href="https://rawgit.com/nextapps-de/flexsearch/master/test/benchmark.html" target="_blank">Benchmarks</a>
- <a href="https://rawgit.com/nextapps-de/flexsearch/master/test/matching-flexsearch.html" target="_blank">Relevance Scoring</a>
-->
Compare these presets:
- <a href="https://rawgit.com/nextapps-de/flexsearch/master/test/matching-presets.html" target="_blank">Relevance Scoring</a><br>
- <a href="https://rawgit.com/nextapps-de/flexsearch/master/test/benchmark-presets.html" target="_blank">Benchmarks</a>
## Best Practices
__Split Complexity__
Whenever you can, try to divide content by categories and add them to its own index, e.g.:
```js
var action = new FlexSearch();
var adventure = new FlexSearch();
var comedy = new FlexSearch();
```
This way you can also provide different settings for each category.
To make this workaround more extendable you can define a tiny helper:
```js
var settings = {};
var index = {};
function add(id, cat, content){
(index[cat] || (
index[cat] = new FlexSearch(settings[cat])
)).add(id, content);
}
function search(cat, query){
return index[cat] ? index[cat].search(query) : [];
}
```
Provide settings optionally (or skip and use defaults):
```js
settings = {
action: "score",
adventure: "match",
comedy: {
encode: "advanced",
tokenize: "forward",
threshold: 5
}
};
```
Add content to the index:
```js
add(1, "action", "Movie Title");
add(2, "adventure", "Movie Title");
add(3, "comedy", "Movie Title");
```
Perform queries:
```js
var results = search("action", "movie title"); // --> [1]
```
Filter queries by categories will hugely improve performance.
__Use numeric IDs__
It is recommended to use numeric id values as reference when adding content to the index. The byte length of passed ids influences the memory consumption significantly. If this is not possible you should consider to use a index table and map the ids with indexes, this becomes important especially when using contextual indexes on a large amount of content.
e.g. instead of this:
```js
index.add("fdf12cad-8779-47ab-b614-4dbbd649178b", "content");
```
you should probably use this:
```js
var index_table = {
"fdf12cad-8779-47ab-b614-4dbbd649178b": 0,
"48b3041c-a243-4a52-b1ed-225041847366": 1,
"7236c8b5-86e1-451a-842f-d9aba9642e4d": 2,
// ....
};
index.add(index_table["fdf12cad-8779-47ab-b614-4dbbd649178b"], "content");
```
It is planned to provide a built-in feature which should replace this workaround.
<a name="export"></a>
## Export/Import Index
> index.export() returns a serialized dump as a string.
> index.import(string) takes a serialized dump as a string and load it to the index.
Assuming you have one or several indexes:
```js
var feeds_2017 = new FlexSearch();
var feeds_2018 = new FlexSearch();
var feeds_2019 = new FlexSearch();
```
Export indexes, e.g. to the local storage:
```js
localStorage.setItem("feeds_2017", feeds_2017.export());
localStorage.setItem("feeds_2018", feeds_2018.export());
localStorage.setItem("feeds_2019", feeds_2019.export());
```
Import indexes, e.g. from the local storage:
```js
feeds_2017.import(localStorage.getItem("feeds_2017"));
feeds_2018.import(localStorage.getItem("feeds_2018"));
feeds_2019.import(localStorage.getItem("feeds_2019"));
```
<a name="debug"></a>
## Debug
> Do not use DEBUG in production builds.
If you get issues, you can temporary set the _DEBUG_ flag to _true_ on top of _flexsearch.js_:
```js
DEBUG = true;
```
This enables console logging of several processes. Just open the browsers console to make this information visible.
<a name="profile"></a>
## Profiler Stats
> Do not use PROFILER in production builds.
To collect some performance statistics of your indexes you need to temporary set the _PROFILER_ flag to _true_ on top of _flexsearch.js_:
```js
PROFILER = true;
```
This enables profiling of several processes.
An array of all profiles is available on:
```js
window.stats;
```
You can also just open the browsers console and enter this line to get stats.
> The index of the array corresponds to the _index.id_.
Get stats from a specific index:
```js
index.stats;
```
The returning stats payload is divided into several categories. Each of these category provides its own statistic values.
__Profiler Stats Properties__
<table>
<tr></tr>
<tr>
<td>Property</td>
<td>Description</td>
</tr>
<tr>
<td>time</td>
<td>The sum of time (ms) the process takes (lower is better)</td>
</tr>
<tr></tr>
<tr>
<td>count</td>
<td>How often the process was called</td>
</tr>
<tr></tr>
<tr>
<td>ops</td>
<td>Average operations per seconds (higher is better)</td>
</tr>
<tr></tr>
<tr>
<td>nano</td>
<td>Average cost (ns) per operation/call (lower is better)</td>
</tr>
</table>
<a name="builds"></a>

@@ -1645,3 +1888,8 @@ ## Custom Builds

</tr>
<tr></tr>
<tr>
<td>SUPPORT_SERIALIZE</td>
<td>true, false</td>
</tr>
<tr>
<td><br><b>Language Flags </b>(includes stemmer and filter)</td>

@@ -1648,0 +1896,0 @@ <td></td>

344

test/test.js

@@ -206,3 +206,3 @@ if(typeof module !== "undefined"){

expect(flexsearch_sync.index).to.have.keys([0, 1, 2]);
expect(flexsearch_sync.index).to.have.keys(["@0", "@1", "@2"]);
expect(flexsearch_sync.length).to.equal(3);

@@ -400,2 +400,4 @@ });

expect(flexsearch_async.length).to.equal(0);
flexsearch_async.add(0, "foo");

@@ -410,3 +412,3 @@ flexsearch_async.add(2, "bar");

expect(flexsearch_async.length).to.equal(3);
expect(flexsearch_async.index).to.have.keys([0, 1, 2]);
expect(flexsearch_async.index).to.have.keys(["@0", "@1", "@2"]);

@@ -428,4 +430,3 @@ done();

flexsearch_async.add(3, {});
flexsearch_async.add(3, function(){
});
flexsearch_async.add(3, function(){});

@@ -435,3 +436,3 @@ setTimeout(function(){

expect(flexsearch_async.length).to.equal(3);
expect(flexsearch_async.index).to.have.keys([0, 1, 2]);
expect(flexsearch_async.index).to.have.keys(["@0", "@1", "@2"]);

@@ -461,9 +462,5 @@ done();

expect(result).to.include(1);
});
setTimeout(function(){
done();
}, 25);
});
});

@@ -477,9 +474,5 @@

expect(result).to.not.include(1);
});
setTimeout(function(){
done();
}, 25);
});
});

@@ -507,9 +500,5 @@

expect(result).to.have.lengthOf(0);
});
setTimeout(function(){
done();
}, 25);
});
});

@@ -526,20 +515,2 @@ });

expect(flexsearch_async.length).to.equal(3);
flexsearch_async.search("foo").then(function(result){
expect(result).to.not.have.members([2, 1]);
});
flexsearch_async.search("bar").then(function(result){
expect(result).to.not.include(0);
});
flexsearch_async.search("bar").then(function(result){
expect(result).to.include(2);
});
flexsearch_async.search("foobar").then(function(result){
expect(result).to.not.include(2);
});
setTimeout(function(){

@@ -563,6 +534,6 @@

expect(result).to.include(2);
done();
});
done();
}, 25);

@@ -601,6 +572,6 @@ });

expect(result).to.include(2);
done();
});
done();
}, 25);

@@ -616,25 +587,21 @@ });

flexsearch_async.remove(2);
flexsearch_async.remove(1);
flexsearch_async.remove(1).then(function(){
expect(flexsearch_async.length).to.equal(0);
});
expect(flexsearch_async.length).to.equal(3);
setTimeout(function(){
flexsearch_async.search("foo", function(result){
expect(result).to.have.lengthOf(0);
});
expect(flexsearch_async.length).to.equal(0);
flexsearch_async.search("bar", function(result){
expect(result).to.have.lengthOf(0);
});
flexsearch_async.search("foo", function(result){
expect(result).to.have.lengthOf(0);
});
flexsearch_async.search("foobar", function(result){
expect(result).to.have.lengthOf(0);
flexsearch_async.search("bar", function(result){
expect(result).to.have.lengthOf(0);
});
flexsearch_async.search("foobar", function(result){
expect(result).to.have.lengthOf(0);
});
done();
}, 25);
});
});

@@ -647,193 +614,166 @@ });

describe("Add (Worker)", function(){
if(typeof Worker !== "undefined" && !this._phantom){
it("Should support worker", function(){
describe("Add (Worker)", function(){
if(typeof Worker === "undefined"){
it("Should support worker", function(){
Worker = function(){};
flexsearch_worker = new FlexSearch({
Worker.prototype.postMessage = function(val){
this.onmessage(val);
};
Worker.prototype.onmessage = function(val){
return val;
};
}
flexsearch_worker = new FlexSearch({
encode: "icase",
tokenize: "strict",
async: false,
worker: 4
encode: "icase",
tokenize: "reverse",
async: false,
worker: 4
});
});
});
it("Should have been added to the index", function(done){
it("Should have been added to the index", function(done){
flexsearch_worker.add(0, "foo");
flexsearch_worker.add(2, "bar");
flexsearch_worker.add(1, "foobar");
flexsearch_worker.add(0, "foo");
flexsearch_worker.add(2, "bar");
flexsearch_worker.add(1, "foobar");
expect(flexsearch_worker.length).to.equal(3);
expect(flexsearch_worker.index).to.have.keys([0, 1, 2]);
setTimeout(function(){
flexsearch_worker.search("foo", function(result){
expect(flexsearch_worker.length).to.equal(3);
expect(flexsearch_worker.index).to.have.keys(["@0", "@1", "@2"]);
expect(result).to.have.length(0);
});
flexsearch_worker.search("foo", function(result){
setTimeout(function(){
expect(result).to.have.length(2);
expect(result).to.have.members([0, 1]);
done();
done();
});
}, 25);
});
}, 25);
});
it("Should not have been added to the index", function(done){
it("Should not have been added to the index", function(done){
flexsearch_worker.add("foo");
flexsearch_worker.add(3);
flexsearch_worker.add(null, "foobar");
flexsearch_worker.add(void 0, "foobar");
flexsearch_worker.add(4, null);
flexsearch_worker.add(5, false);
flexsearch_worker.add(6, []);
flexsearch_worker.add(7, {});
flexsearch_worker.add(8, function(){});
flexsearch_worker.add("foo");
flexsearch_worker.add(3);
flexsearch_worker.add(null, "foobar");
flexsearch_worker.add(void 0, "foobar");
flexsearch_worker.add(4, null);
flexsearch_worker.add(5, false);
flexsearch_worker.add(6, []);
flexsearch_worker.add(7, {});
flexsearch_worker.add(8, function(){
});
setTimeout(function(){
setTimeout(function(){
expect(flexsearch_worker.length).to.equal(3);
expect(flexsearch_worker.index).to.have.keys([0, 1, 2]);
expect(flexsearch_worker.length).to.equal(3);
expect(flexsearch_worker.index).to.have.keys(["@0", "@1", "@2"]);
done();
done();
}, 25);
}, 25);
});
});
});
describe("Search (Worker)", function(){
describe("Search (Worker)", function(){
it("Should have been matched from index", function(done){
it("Should have been matched from index", function(done){
flexsearch_worker.search("foo", function(result){
flexsearch_worker.search("foo", function(result){
expect(result).to.have.lengthOf(2);
});
expect(result).to.have.lengthOf(2);
flexsearch_worker.search("bar", function(result){
flexsearch_worker.search("bar", function(result){
expect(result).to.have.lengthOf(2);
});
expect(result).to.have.lengthOf(2);
flexsearch_worker.search("foobar", function(result){
flexsearch_worker.search("foobar", function(result){
expect(result).to.have.lengthOf(1);
expect(result).to.have.lengthOf(1);
done();
});
});
});
});
setTimeout(function(){
it("Should have been limited", function(done){
done();
flexsearch_worker.search("foo", 1, function(result){
}, 25);
});
expect(result).to.have.lengthOf(1);
it("Should have been limited", function(done){
flexsearch_worker.search("foo", 1, function(result){
expect(result).to.include(0);
expect(result).to.not.include(1);
done();
});
});
setTimeout(function(){
it("Should not have been matched from index", function(done){
done();
flexsearch_worker.search("barfoo", function(result){
}, 25);
});
expect(result).to.have.lengthOf(0);
it("Should not have been matched from index", function(done){
flexsearch_worker.search("", function(result){
flexsearch_worker.search("barfoo", function(result){
expect(result).to.have.lengthOf(0);
expect(result).to.have.lengthOf(0);
});
flexsearch_worker.search(" ", function(result){
flexsearch_worker.search("", function(result){
expect(result).to.have.lengthOf(0);
expect(result).to.have.lengthOf(0);
});
flexsearch_worker.search(" o ", function(result){
flexsearch_worker.search(" ", function(result){
expect(result).to.have.lengthOf(1);
expect(result).to.have.lengthOf(0);
});
flexsearch_worker.search(" fob ", function(result){
flexsearch_worker.search(" o ", function(result){
expect(result).to.have.lengthOf(0);
expect(result).to.have.lengthOf(0);
done();
});
});
});
});
});
});
setTimeout(function(){
done();
}, 25);
});
});
// TODO:
/*
describe("Update (Worker)", function(){
describe("Update (Worker)", function(){
it("Should have been updated to the index", function(done){
it("Should have been updated to the index", function(done){
flexsearch_worker.update(0, "bar");
flexsearch_worker.update(2, "foobar");
flexsearch_worker.update(1, "foo");
flexsearch_worker.update(0, "bar");
flexsearch_worker.update(2, "foobar");
flexsearch_worker.update(1, "foo", function(){
setTimeout(function(){
expect(flexsearch_worker.length).to.equal(3);
expect(flexsearch_worker.length).to.equal(3);
flexsearch_worker.search("foo", function(results){
flexsearch_worker.search("foo", function(results){
expect(results).to.have.members([2, 1]);
expect(results).to.have.members([2, 1]);
});
flexsearch_worker.search("bar", function(results){
flexsearch_worker.search("bar", function(results){
expect(results).to.have.members([0, 2]);
expect(results).to.have.members([0, 2]);
});
flexsearch_worker.search("foobar", function(results){
flexsearch_worker.search("foobar", function(results){
expect(results).to.have.members([2]);
expect(results).to.have.members([2]);
done();
});
});
});
});
setTimeout(function(){
done();
}, 25);
}, 25);
});
});
});
*/
describe("Remove (Worker)", function(){
describe("Remove (Worker)", function(){
it("Should have been removed from the index", function(done){
it("Should have been removed from the index", function(done){
expect(flexsearch_worker.length).to.equal(3);
expect(flexsearch_worker.length).to.equal(3);
flexsearch_worker.remove(0);
flexsearch_worker.remove(2);
flexsearch_worker.remove(1);
flexsearch_worker.remove(0);
flexsearch_worker.remove(2);
flexsearch_worker.remove(1);
setTimeout(function(){
expect(flexsearch_worker.length).to.equal(0);

@@ -845,32 +785,28 @@

expect(results).to.not.include(2);
});
flexsearch_worker.search("bar", function(results){
flexsearch_worker.search("bar", function(results){
expect(results).to.not.include(0);
expect(results).to.not.include(2);
});
expect(results).to.not.include(0);
expect(results).to.not.include(2);
flexsearch_worker.search("foobar", function(results){
flexsearch_worker.search("foobar", function(results){
expect(results).to.not.include(2);
expect(results).to.not.include(2);
done();
});
});
});
});
setTimeout(function(){
if(env !== "light" && env !== "min"){
done();
it("Should have been debug mode activated", function(){
}, 25);
}, 25);
flexsearch_worker.info();
});
}
});
}
if(env !== "light" && env !== "min"){
it("Should have been debug mode activated", function(){
flexsearch_worker.info();
});
}
});
describe("Worker Not Supported", function(){

@@ -888,3 +824,3 @@

encode: false,
async: true,
async: false,
worker: 4

@@ -891,0 +827,0 @@ });

flexsearch.min.js
test/flexsearch.es5.js
test/matching-flexsearch.html
flexsearch.js

Sorry, the diff of this file is too big to display

test/benchmark.html

Sorry, the diff of this file is not supported yet

test/index.html

Sorry, the diff of this file is not supported yet

test/matching.html

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc