falcon-vis - npm Package Compare versions

Comparing version 0.17.1 to 0.17.2

build/core/bitset.d.ts

@@ -9,2 +9,3 @@ export declare class BitSet {
     union(other: BitSet): void;
+    [Symbol.iterator](): Generator<boolean, void, unknown>;
 }
@@ -11,0 +12,0 @@ /**

build/core/bitset.js

@@ -31,2 +31,7 @@ // from https://github.com/mikolalysenko/minimal-bit-array/blob/master/bitarray.js with modifications
     }
+    *[Symbol.iterator]() {
+        for (let i = 0; i < this.length; i++) {
+            yield this.get(i);
+        }
+    }
 }
@@ -33,0 +38,0 @@ /**

build/core/db/arrow.d.ts

@@ -8,3 +8,4 @@ import { Table } from "apache-arrow";
 import type { Filters, FalconCounts, FalconIndex, FalconCube } from "./db";
-import type { CategoricalRange, ContinuousRange, Dimension } from "../dimension";
+import type { CategoricalRange, ContinuousDimension, ContinuousRange, Dimension } from "../dimension";
+import type { BinNumberFunction } from "../util";
 import type { View } from "../views";
@@ -26,3 +27,3 @@ type DimensionFilterHash = string;
      */
-    constructor(data: Table);
+    constructor(data: Table, filterMaskCacheSize?: number);
     /**
@@ -35,3 +36,12 @@ * Easy helper method to create a new ArrowDB from an arrow file
     static fromArrowFile(url: string): Promise<ArrowDB>;
-    length(): number;
+    /**
+     * compute the best number of bins for a histogram
+     * given the data
+     *
+     * @resource [plot](https://github.com/observablehq/plot/blob/97924e7682e49d35a34da794ca98bf0c7e8a3c28/src/transforms/bin.js#L320)
+     * @resource [lord and savior](https://twitter.com/mbostock/status/1429281697854464002)
+     * @resource [numpy](https://numpy.org/doc/stable/reference/generated/numpy.histogram_bin_edges.html)
+     */
+    estimateNumBins(dimension: ContinuousDimension, maxThreshold?: number, noKnowledgeEstimate?: number): number;
+    length(filters?: Filters): number;
     private categoricalRange;
@@ -42,3 +52,3 @@ private continuousRange;
     tableExists(): boolean;
-    entries(offset?: number, length?: number, filters?: Filters | undefined): Iterable<Row | null>;
+    entries(offset?: number, length?: number, filters?: Filters | undefined): Promise<Iterable<Row | null>>;
     histogramView1D(view: View1D, filters?: Filters): FalconCounts;
@@ -53,3 +63,3 @@ falconIndexView1D(activeView: View1D, passiveViews: View[], filters: Filters): FalconIndex;
      */
-    cubeSlice1DCategorical(view: View, activeCol: Vector, filterMasks: FilterMasks<Dimension>, binActive: (x: number) => number, binCountActive: number): FalconCube;
+    cubeSlice1DCategorical(view: View, activeCol: Vector, filterMasks: FilterMasks<Dimension>, binActive: BinNumberFunction, binCountActive: number): FalconCube;
     /**
@@ -62,3 +72,3 @@ * Takes a view and computes the falcon cube for that passive view
      */
-    cubeSlice1DContinuous(view: View, activeCol: Vector, filterMasks: FilterMasks<Dimension>, numPixels: number, binActive: (x: number) => number): FalconCube;
+    cubeSlice1DContinuous(view: View, activeCol: Vector, filterMasks: FilterMasks<Dimension>, numPixels: number, binActive: BinNumberFunction): FalconCube;
     /**
@@ -65,0 +75,0 @@ * given the dimension and filters

build/core/db/arrow.js

 import { tableFromIPC } from "apache-arrow";
 import { BitSet, union } from "../bitset";
+import { greatScott } from "../bins";
 import { FalconArray } from "../falconArray";
 import { RowIterator } from "../iterator";
-import { binNumberFunctionContinuous, binNumberFunctionBinsContinuous, numBinsContinuous, numBinsCategorical, binNumberFunctionCategorical, } from "../util";
+import { binNumberFunctionContinuous, binNumberFunctionPixels, numBinsContinuous, numBinsCategorical, binNumberFunctionCategorical, } from "../util";
 import { View0D, View1D } from "../views";
@@ -17,6 +18,6 @@ export class ArrowDB {
      */
-    constructor(data) {
+    constructor(data, filterMaskCacheSize = 64) {
         this.blocking = true;
         // bitmask to determine what rows filter out or not
-        this.filterMaskIndex = new Map();
+        this.filterMaskIndex = new LRUMap(filterMaskCacheSize); // only save a few recent filter masks in memory
         this.data = data;
@@ -36,7 +37,43 @@ }
     }
-    length() {
-        return this.data.numRows;
+    /**
+     * compute the best number of bins for a histogram
+     * given the data
+     *
+     * @resource [plot](https://github.com/observablehq/plot/blob/97924e7682e49d35a34da794ca98bf0c7e8a3c28/src/transforms/bin.js#L320)
+     * @resource [lord and savior](https://twitter.com/mbostock/status/1429281697854464002)
+     * @resource [numpy](https://numpy.org/doc/stable/reference/generated/numpy.histogram_bin_edges.html)
+     */
+    estimateNumBins(dimension, maxThreshold = 200, noKnowledgeEstimate = 15) {
+        const arrowColumn = this.data.getChild(dimension.name);
+        if (arrowColumn.length <= 1) {
+            // can't do much with one data point
+            return 1;
+        }
+        if (dimension.range) {
+            const standardDeviation = Math.sqrt(sampleVariance(arrowColumn)); // \sqrt{\sigma^2}
+            const [min, max] = dimension.range;
+            const optimalBins = greatScott(min, max, standardDeviation);
+            return Math.min(optimalBins, maxThreshold);
+        }
+        // if we don't have a min max range, just return the no knowledge estimate
+        return noKnowledgeEstimate;
     }
+    length(filters) {
+        if (filters) {
+            const filterMask = union(...this.getFilterMasks(filters).values());
+            let total = 0;
+            for (const bit of filterMask) {
+                // if the bit is not set (aka false) then add 1 to the total
+                if (bit === false) {
+                    total++;
+                }
+            }
+            return total;
+        }
+        else {
+            return this.data.numRows;
+        }
+    }
     categoricalRange(arrowColumn) {
-        return arrowColumnUnique(arrowColumn);
+        return arrowColumnUnique(arrowColumn).filter((item) => item !== null);
     }
@@ -70,3 +107,3 @@ continuousRange(arrowColumn) {
     }
-    entries(offset = 0, length = Infinity, filters) {
+    async entries(offset = 0, length = Infinity, filters) {
         const filterMask = union(...this.getFilterMasks(filters ?? new Map()).values());
@@ -103,3 +140,3 @@ return new RowIterator(this.data.numRows, (i) => this.data.get(i), filterMask, offset, length);
             // increment the specific bin
-            if (0 <= binLocation && binLocation < binCount) {
+            if (0 <= binLocation && binLocation < binCount && isNotNull(value)) {
                 noFilter.increment([binLocation]);
@@ -124,3 +161,3 @@ if (filterMask && !filterMask.get(i)) {
             const activeDim = activeView.dimension;
-            const binActive = binNumberFunctionBinsContinuous(activeDim.binConfig, pixels);
+            const binActive = binNumberFunctionPixels(activeDim.binConfig, pixels);
             const activeCol = this.data.getChild(activeDim.name);
@@ -187,3 +224,3 @@ const numPixels = pixels + 1; // extending by one pixel so we can compute the right diff later
         else if (view instanceof View1D) {
-            let bin;
+            let binPassive;
             let binCount;
@@ -193,3 +230,3 @@ if (view.dimension.type === "continuous") {
                 const binConfig = view.dimension.binConfig;
-                bin = binNumberFunctionContinuous(binConfig);
+                binPassive = binNumberFunctionContinuous(binConfig);
                 binCount = numBinsContinuous(binConfig);
@@ -199,3 +236,3 @@ }
                 // categorical bins for passive view that we accumulate across
-                bin = binNumberFunctionCategorical(view.dimension.range);
+                binPassive = binNumberFunctionCategorical(view.dimension.range);
                 binCount = numBinsCategorical(view.dimension.range);
@@ -208,3 +245,3 @@ }
             noFilter = FalconArray.allocCounts(binCount, [binCount]);
-            const column = this.data.getChild(view.dimension.name);
+            const passiveCol = this.data.getChild(view.dimension.name);
             // add data to aggregation matrix
@@ -216,9 +253,15 @@ for (let i = 0; i < this.data.numRows; i++) {
                 }
-                const key = bin(column.get(i));
-                const keyActive = binActive(activeCol.get(i));
-                if (0 <= key && key < binCount) {
-                    if (0 <= keyActive && keyActive < binCountActive) {
-                        filter.increment([keyActive, key]);
+                const valueActive = activeCol.get(i);
+                const valuePassive = passiveCol.get(i);
+                const keyPassive = binPassive(valuePassive);
+                const keyActive = binActive(valueActive);
+                if (0 <= keyPassive &&
+                    keyPassive < binCount &&
+                    isNotNull(valuePassive)) {
+                    if (0 <= keyActive &&
+                        keyActive < binCountActive &&
+                        isNotNull(valueActive)) {
+                        filter.increment([keyActive, keyPassive]);
                     }
-                    noFilter.increment([key]);
+                    noFilter.increment([keyPassive]);
                 }
@@ -262,4 +305,5 @@ }
                 }
-                const keyActive = binActive(activeCol.get(i)) + 1;
-                if (0 <= keyActive && keyActive < numPixels) {
+                const valueActive = activeCol.get(i);
+                const keyActive = binActive(valueActive) + 1;
+                if (0 <= keyActive && keyActive < numPixels && isNotNull(valueActive)) {
                     filter.increment([keyActive]);
@@ -273,3 +317,3 @@ }
         else if (view instanceof View1D) {
-            let bin;
+            let binPassive;
             let binCount;
@@ -279,3 +323,3 @@ if (view.dimension.type === "continuous") {
                 const binConfig = view.dimension.binConfig;
-                bin = binNumberFunctionContinuous(binConfig);
+                binPassive = binNumberFunctionContinuous(binConfig);
                 binCount = numBinsContinuous(binConfig);
@@ -285,3 +329,3 @@ }
                 // categorical bins for passive view that we accumulate across
-                bin = binNumberFunctionCategorical(view.dimension.range);
+                binPassive = binNumberFunctionCategorical(view.dimension.range);
                 binCount = numBinsCategorical(view.dimension.range);
@@ -294,3 +338,3 @@ }
             noFilter = FalconArray.allocCounts(binCount, [binCount]);
-            const column = this.data.getChild(view.dimension.name);
+            const passiveCol = this.data.getChild(view.dimension.name);
             // add data to aggregation matrix
@@ -302,9 +346,15 @@ for (let i = 0; i < this.data.numRows; i++) {
                 }
-                const key = bin(column.get(i));
-                const keyActive = binActive(activeCol.get(i)) + 1;
-                if (0 <= key && key < binCount) {
-                    if (0 <= keyActive && keyActive < numPixels) {
-                        filter.increment([keyActive, key]);
+                const valueActive = activeCol.get(i);
+                const valuePassive = passiveCol.get(i);
+                const keyActive = binActive(valueActive) + 1;
+                const keyPassive = binPassive(valuePassive);
+                if (0 <= keyPassive &&
+                    keyPassive < binCount &&
+                    isNotNull(valuePassive)) {
+                    if (0 <= keyActive &&
+                        keyActive < numPixels &&
+                        isNotNull(valueActive)) {
+                        filter.increment([keyActive, keyPassive]);
                     }
-                    noFilter.increment([key]);
+                    noFilter.increment([keyPassive]);
                 }
@@ -362,3 +412,3 @@ }
             const column = this.data.getChild(dimension.name);
-            const mask = arrowFilterMask(column, (value) => !filterSet.has(value));
+            const mask = arrowFilterMask(column, (rowValue) => filterSet.has(rowValue));
             // set the cache
@@ -382,3 +432,3 @@ this.filterMaskIndex.set(key, mask);
             const column = this.data.getChild(dimension.name);
-            const mask = arrowFilterMask(column, (value) => (value && value < filter[0]) || value >= filter[1]);
+            const mask = arrowFilterMask(column, (value) => value > filter[0] && value <= filter[1]);
             // set the cache
@@ -398,3 +448,3 @@ this.filterMaskIndex.set(key, mask);
  */
-function arrowFilterMask(column, shouldFilterOut) {
+function arrowFilterMask(column, shouldKeep) {
     const bitmask = new BitSet(column.length);
@@ -410,3 +460,3 @@ /**
         const rowValue = column.get(i);
-        if (shouldFilterOut(rowValue)) {
+        if (!shouldKeep(rowValue)) {
             bitmask.set(i, true);
@@ -450,2 +500,42 @@ }
 }
+function isNotNull(value) {
+    return value !== null;
+}
+class LRUMap extends Map {
+    /**
+     * A Map that only keeps the most recent `limit` number of entries.
+     * @param limit - The maximum size of the map.
+     */
+    constructor(limit) {
+        super();
+        this.limit = limit;
+    }
+    set(key, value) {
+        if (this.size >= this.limit) {
+            this.delete(this.keys().next().value);
+        }
+        return super.set(key, value);
+    }
+}
+/**
+ * sample defined by
+ * $$\sigma^2 = \frac{1}{n} \sum_{i=1}^n (x_i - \mu)^2$$
+ *
+ * this can probably be optimized faster to be like [the boss](https://github.com/d3/d3-array/blob/main/src/variance.js#L1)
+ */
+function sampleVariance(vector) {
+    let variance = 0, n = vector.length;
+    let mu = mean(vector);
+    for (const x_i of vector) {
+        variance += (x_i - mu) ** 2;
+    }
+    return n > 1 ? variance / (n - 1) : variance;
+}
+function mean(vector) {
+    let mean = 0, n = vector.length;
+    for (const x_i of vector) {
+        mean += x_i;
+    }
+    return mean / n;
+}
 //# sourceMappingURL=arrow.js.map

build/core/db/db.d.ts

 import type { Row } from "../iterator";
-import type { CategoricalRange, ContinuousRange, Dimension, DimensionFilter } from "../dimension";
+import type { CategoricalRange, ContinuousDimension, ContinuousRange, Dimension, DimensionFilter } from "../dimension";
 import type { FalconArray } from "../falconArray";
@@ -15,3 +15,3 @@ import type { View, View1D } from "../views";
 export type AsyncIndex = Map<View, Promise<FalconCube>>;
-export type FalconIndex = SyncIndex | AsyncIndex;
+export type FalconIndex = Map<View, Promise<FalconCube> | FalconCube>;
 export type AsyncOrSync<T> = Promise<T> | T;
@@ -24,2 +24,3 @@ export type Filter = DimensionFilter;
 export interface FalconDB {
+    estimateNumBins(dimension: ContinuousDimension, maxThreshold?: number, noKnowledgeEstimate?: number): AsyncOrSync<number>;
     /**
@@ -31,3 +32,3 @@ * loads the ENTIRE (not filtered) length of the data
      */
-    length(): AsyncOrSync<number>;
+    length(filters?: Filters): AsyncOrSync<number>;
     /**
@@ -34,0 +35,0 @@ * determines the min and max of a continuous dimensions

build/core/db/duckdb.d.ts

@@ -17,2 +17,3 @@ import { SQLDB, SQLQueryResult } from "./sql";
     protected query(q: SQLQuery): Promise<SQLQueryResult>;
+    protected castTime(name: string): string;
     /**
@@ -23,3 +24,3 @@ * creates new FalconDB instance from this parquet file
      */
-    static fromParquetFile(url: string, table?: string, nameMap?: SQLNameMap): Promise<DuckDB>;
+    static fromParquetFile(url: string, table?: string, nameMap?: SQLNameMap, useFullUrlFromWindow?: boolean): Promise<DuckDB>;
 }

build/core/db/duckdb.js

@@ -28,2 +28,6 @@ import { compactQuery } from "../util";
     }
+    castTime(name) {
+        // converts to seconds UTC, then to milliseconds
+        return `epoch(${name})*1000`;
+    }
     /**
@@ -34,4 +38,4 @@ * creates new FalconDB instance from this parquet file
      */
-    static async fromParquetFile(url, table = "data", nameMap) {
-        const db = await createNewTable(table, `CREATE VIEW '${table}' AS SELECT * FROM parquet_scan('${url}')`);
+    static async fromParquetFile(url, table = "data", nameMap, useFullUrlFromWindow = true) {
+        const db = await createNewTable(table, `CREATE TABLE '${table}' AS SELECT * FROM parquet_scan('${useFullUrlFromWindow ? fullUrl(url) : url}')`);
         return new DuckDB(db, table, nameMap);
@@ -54,2 +58,5 @@ }
 }
+function fullUrl(filename) {
+    return `${window.location.href}${filename}`;
+}
 //# sourceMappingURL=duckdb.js.map

build/core/db/sql.d.ts

@@ -1,7 +0,8 @@
-import { Dimension, ContinuousRange, CategoricalRange } from "../dimension";
+import { FalconDB, Filters, AsyncIndex } from "./db";
+import { Dimension, ContinuousRange, CategoricalRange, ContinuousDimension } from "../dimension";
 import { FalconArray } from "../falconArray";
+import { Row } from "../iterator";
 import { View1D } from "../views";
-import { FalconDB, Filters, AsyncIndex } from "./db";
+import type { BinNumberFunction } from "../util";
 import type { View } from "../views";
-import { Row } from "../iterator";
 export type SQLNameMap = Map<string, string>;
@@ -21,2 +22,3 @@ export type SQLQuery = string;
     protected castBins(input: number): string;
+    protected castTime(name: string): string;
     /**
@@ -39,3 +41,12 @@ * intermediary function incase we mapped the names to something else
     entries(offset?: number, length?: number, filters?: Filters): Promise<SQLQueryResult>;
-    length(): Promise<any>;
+    /**
+     * compute the best number of bins for a histogram
+     * given the data
+     *
+     * @resource [plot](https://github.com/observablehq/plot/blob/97924e7682e49d35a34da794ca98bf0c7e8a3c28/src/transforms/bin.js#L320)
+     * @resource [lord and savior](https://twitter.com/mbostock/status/1429281697854464002)
+     * @resource [numpy](https://numpy.org/doc/stable/reference/generated/numpy.histogram_bin_edges.html)
+     */
+    estimateNumBins(dimension: ContinuousDimension, maxThreshold?: number, noKnowledgeEstimate?: number): Promise<number>;
+    length(filters?: Filters): Promise<any>;
     range(dimension: Dimension): Promise<ContinuousRange | CategoricalRange[]>;
@@ -47,3 +58,3 @@ histogramView1D(view: View1D, filters?: Filters): Promise<{
     falconIndexView1D(activeView: View1D, passiveViews: View[], filters: Filters): AsyncIndex;
-    cubeSlice1DCategorical(view: View, sqlFilters: SQLFilters, binActive: SQLBin, binActiveIndexMap: (x: any) => number, binCountActive: number): Promise<{
+    cubeSlice1DCategorical(view: View, sqlFilters: SQLFilters, binActive: SQLBin, binActiveIndexMap: BinNumberFunction, binCountActive: number): Promise<{
         noFilter: FalconArray;
@@ -50,0 +61,0 @@ filter: FalconArray;

build/core/db/sql.js

@@ -0,3 +1,4 @@
+import { greatScott } from "../bins";
 import { FalconArray } from "../falconArray";
-import { binNumberFunctionCategorical, numBinsCategorical, numBinsContinuous, stepSize, } from "../util";
+import { binNumberFunctionContinuousSQL, binNumberFunctionCategorical, numBinsCategorical, numBinsContinuous, stepSize, } from "../util";
 import { View0D, View1D } from "../views";
@@ -12,2 +13,5 @@ export class SQLDB {
     }
+    castTime(name) {
+        return name;
+    }
     /**
@@ -20,7 +24,11 @@ * intermediary function incase we mapped the names to something else
     getName(dimension) {
-        return this.nameMap?.get(dimension.name) ?? dimension.name;
+        let name = this.nameMap?.get(dimension.name) ?? dimension.name;
+        if (dimension.type === "continuous" && dimension.time) {
+            name = this.castTime(name);
+        }
+        return name;
     }
     async dimensionExists(dimension) {
         const result = await this.query(`SELECT EXISTS 
-      (SELECT 0 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '${this.table}' AND COLUMN_NAME = '${this.getName(dimension)}') as _exists`);
+      (SELECT 0 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '${this.table}' AND COLUMN_NAME = '${dimension.name}') as _exists`);
         const { _exists } = this.getASValues(result);
@@ -46,5 +54,33 @@ return _exists;
     }
-    async length() {
+    /**
+     * compute the best number of bins for a histogram
+     * given the data
+     *
+     * @resource [plot](https://github.com/observablehq/plot/blob/97924e7682e49d35a34da794ca98bf0c7e8a3c28/src/transforms/bin.js#L320)
+     * @resource [lord and savior](https://twitter.com/mbostock/status/1429281697854464002)
+     * @resource [numpy](https://numpy.org/doc/stable/reference/generated/numpy.histogram_bin_edges.html)
+     */
+    async estimateNumBins(dimension, maxThreshold = 200, noKnowledgeEstimate = 15) {
+        const count = await this.length();
+        if (count <= 1) {
+            return 1;
+        }
+        if (dimension.range) {
+            const standardDeviationQuery = await this.query(`SELECT STDDEV(${this.getName(dimension)}) AS standardDeviation FROM ${this.table}`);
+            const { standardDeviation } = this.getASValues(standardDeviationQuery);
+            const [min, max] = dimension.range;
+            const optimalBins = greatScott(min, max, standardDeviation);
+            return Math.min(optimalBins, maxThreshold);
+        }
+        // if we don't have a min max range, just return the no knowledge estimate
+        return noKnowledgeEstimate;
+    }
+    async length(filters) {
+        let filterSQL = "";
+        if (filters) {
+            filterSQL = [...this.filtersToSQLWhereClauses(filters).values()].join(" AND ");
+        }
         const result = await this.query(`SELECT count(*) AS _count
-       FROM ${this.table}`);
+       FROM ${this.table}
+       ${filterSQL ? `WHERE ${filterSQL}` : ""}`);
         const { _count } = this.getASValues(result);
@@ -67,3 +103,3 @@ return _count;
             }
-            return range;
+            return range.filter((x) => x !== null);
         }
@@ -102,7 +138,9 @@ }
             const where = [...this.filtersToSQLWhereClauses(filters).values()].join(" AND ");
-            const result = await this.query(`SELECT ${bSql.select}
+            const queryText = `SELECT ${bSql.select}
          AS binIndex, count(*) AS binCount
          FROM ${this.table}
          WHERE ${bSql.where} AND ${where} 
-         GROUP BY key`);
+         GROUP BY binIndex`;
+            const result = await this.query(queryText);
+            console.log(view.dimension.name, queryText);
             for (const { binIndex, binCount } of result) {
@@ -120,4 +158,5 @@ filter.set(binIndex, binCount);
             // 1. active bin for each pixel
-            const binActive = this.binSQLPixel(activeView.dimension, activeView.dimension.binConfig, activeView.dimension.resolution);
-            const numPixels = activeView.dimension.resolution + 1; // extending by one pixel so we can compute the right diff later
+            const numPixelBins = activeView.dimension.resolution;
+            const binActive = this.binSQLPixel(activeView.dimension, activeView.dimension.binConfig, numPixelBins);
+            const numPixels = numPixelBins + 1; // for example 10 bins -> 11 total edges (pixels)
             // 2. iterate through passive views and compute cubes
@@ -255,3 +294,4 @@ const promises = [];
         let binPassiveIndexMap = (x) => x;
-        const select = `CASE WHEN ${binActive.where} 
+        const select = `CASE
+     WHEN ${binActive.where} 
      THEN ${binActive.select}
@@ -299,3 +339,3 @@ ELSE -1 END AS "keyActive",
                 if (keyActive >= 0) {
-                    filter.set(keyActive, cnt);
+                    filter.set(keyActive + 1, cnt);
                 }
@@ -310,3 +350,3 @@ noFilter.increment([0], cnt);
                 if (keyActive >= 0) {
-                    filter.set(keyActive, binPassiveIndex, cnt);
+                    filter.set(keyActive + 1, binPassiveIndex, cnt);
                 }
@@ -357,3 +397,3 @@ noFilter.increment([binPassiveIndex], cnt);
         const field = this.getName(dimension);
-        const select = `cast((${field} - ${binConfig.start}) / ${binConfig.step} as int)`;
+        const select = binNumberFunctionContinuousSQL(field, binConfig, this.castBins);
         const where = `${field} BETWEEN ${binConfig.start} AND ${binConfig.stop}`;
@@ -360,0 +400,0 @@ return {

build/core/dimension.d.ts

@@ -8,4 +8,4 @@ import type { BinConfig, Interval } from "./util";
     type: "continuous";
-    bins: number;
     resolution: number;
+    bins?: number;
     range?: ContinuousRange;
@@ -27,2 +27,4 @@ binConfig?: BinConfig;
 export type DimensionFilter = ContinuousRange | CategoricalRange;
+export type KeyRequired<T, K extends keyof T> = T & Required<Pick<T, K>>;
+export type KeyOptional<T, K extends keyof T> = T & Partial<Pick<T, K>>;
 export {};

build/core/falcon.d.ts

@@ -5,3 +5,3 @@ import { View0D, View1D, ViewSet, View1DState, View0DState } from "./views";
 import type { OnChange } from "./views/viewAbstract";
-export declare class Falcon {
+export declare class FalconVis {
     db: FalconDB;
@@ -19,13 +19,8 @@ views: ViewSet;
     /**
-     * verifies if we can continue to do stuff on the data
-     * throws an error if the data does not exist
-     */
-    assertDataExists(...dimensions: Dimension[]): Promise<void>;
-    /**
      * Creates a 0D view that counts the number of entries
+     * if this is called when there is already an active view, it will just compute this passive index
      *
      * @returns the view
      */
-    linkCount(onChange?: OnChange<View0DState>): Promise<View0D>;
-    private createView0D;
+    view0D(onChange?: OnChange<View0DState>): Promise<View0D>;
     /**
@@ -36,10 +31,13 @@ * Creates a 1D view and links with the other views under this falcon object
      */
-    linkView1D(dimension: Dimension, onChange?: OnChange<View1DState>): Promise<View1D>;
-    private createView1D;
+    view1D(dimension: Dimension, onChange?: OnChange<View1DState>): Promise<View1D>;
     /**
+     * creates an iterator over the filtered entries located at the offset and with a specified length
+     * for example, if we have 100 filtered entries, I can position myself halfway (offset = 50) and if I want
+     * just 10 entries I would use length = 10
+     *
      * @param offset the offset to start the iteration from (within filtered)
      * @param length the number of entries to return
-     * @returns an iterable that iterates over instances from the filter
+     * @returns iterator over the filtered entries basically a list of objects
      */
-    getEntries({ offset, length }?: {
+    entries({ offset, length }?: {
         offset?: number | undefined;
@@ -52,3 +50,4 @@ length?: number | undefined;
      */
-    initializeAllCounts(): Promise<void>;
+    all(): Promise<void>;
+    link(): Promise<void>;
     /**
@@ -58,2 +57,8 @@ * @returns the filters and excludes the active view dimension's filters
     get passiveFilters(): Filters;
+    otherFilters(view: View1D): Filters;
+    /**
+     * verifies if we can continue to do stuff on the data
+     * throws an error if the data does not exist
+     */
+    assertDataExists(...dimensions: Dimension[]): Promise<void>;
 }

build/core/falcon.js

 import { View0D, View1D, ViewSet } from "./views";
 import { excludeMap } from "./util";
-export class Falcon {
+export class FalconVis {
     /**
@@ -12,3 +12,3 @@ * Takes a data and creates the main driver of the Falcon library
         this.db = db;
-        this.views = new ViewSet();
+        this.views = new ViewSet(); // this may be a bs abstraction
         this.filters = new Map();
@@ -18,42 +18,29 @@ this.index = new Map();
     /**
-     * verifies if we can continue to do stuff on the data
-     * throws an error if the data does not exist
-     */
-    async assertDataExists(...dimensions) {
-        /**
-         * Check if the table exists
-         */
-        const tableExistsInDB = await this.db.tableExists();
-        if (!tableExistsInDB) {
-            throw new Error(`Table does not exists in the database`);
-        }
-        /**
-         * If provided, check if the dimensions exist
-         */
-        dimensions?.forEach(async (dimension) => {
-            if (dimension) {
-                const dimensionExistsInDB = await this.db.dimensionExists(dimension);
-                if (!dimensionExistsInDB) {
-                    throw new Error(`Dimension '${dimension.name}' does not exist in the data table`);
-                }
-            }
-        });
-    }
-    /**
      * Creates a 0D view that counts the number of entries
+     * if this is called when there is already an active view, it will just compute this passive index
      *
      * @returns the view
      */
-    async linkCount(onChange) {
+    async view0D(onChange) {
         await this.assertDataExists();
-        return this.createView0D(onChange);
-    }
-    createView0D(onChange) {
-        const view = new View0D(this);
-        this.views.add(view);
+        const view0D = new View0D(this);
+        this.views.add(view0D);
         if (onChange) {
-            view.addOnChangeListener(onChange);
+            view0D.addOnChangeListener(onChange);
         }
-        return view;
+        // // if we have an active view, just compute the passive index for this view0D only
+        // // not the whole thing!
+        // if (this.views.active) {
+        //   await this.addToIndexIfAddedLaterOn(view0D, this.views.active);
+        // }
+        return view0D;
     }
+    // private async addToIndexIfAddedLaterOn(view0D: View0D, av: View1D) {
+    //   // handle the case when there is already an active view
+    //   // aka, just compute the passive index for this view0D
+    //   const index = this.db.falconIndexView1D(av, [view0D], this.filters);
+    //   const view0DCube = await index.get(view0D)!;
+    //   this.index.set(view0D, view0DCube);
+    //   av.select(av.lastFilter, true);
+    // }
     /**
@@ -64,7 +51,4 @@ * Creates a 1D view and links with the other views under this falcon object
      */
-    async linkView1D(dimension, onChange) {
+    async view1D(dimension, onChange) {
         await this.assertDataExists(dimension);
-        return this.createView1D(dimension, onChange);
-    }
-    createView1D(dimension, onChange) {
         const view = new View1D(this, dimension);
@@ -78,7 +62,11 @@ this.views.add(view);
     /**
+     * creates an iterator over the filtered entries located at the offset and with a specified length
+     * for example, if we have 100 filtered entries, I can position myself halfway (offset = 50) and if I want
+     * just 10 entries I would use length = 10
+     *
      * @param offset the offset to start the iteration from (within filtered)
      * @param length the number of entries to return
-     * @returns an iterable that iterates over instances from the filter
+     * @returns iterator over the filtered entries basically a list of objects
      */
-    async getEntries({ offset = 0, length = Infinity } = {}) {
+    async entries({ offset = 0, length = Infinity } = {}) {
         return this.db.entries(offset, length, this.filters);
@@ -90,7 +78,23 @@ }
      */
-    async initializeAllCounts() {
-        this.views.forEach(async (view) => {
-            await view.initializeAllCounts();
-        });
+    async all() {
+        for (const view of this.views) {
+            await view.all();
+        }
     }
+    async link() {
+        const av = this.views.active;
+        if (!av) {
+            await this.all();
+        }
+        else {
+            /**
+             * @todo compute the passive index individually instead of the entire thing
+             * @todo this fails for the .update() on view1D
+             * setup a system that diffs from the current index and filters
+             */
+            const forceToCompute = true; // turns off optimizations
+            await av.computeIndex(forceToCompute);
+            await av.select(av.lastFilter, forceToCompute);
+        }
+    }
     /**
@@ -112,3 +116,30 @@ * @returns the filters and excludes the active view dimension's filters
     }
+    otherFilters(view) {
+        return excludeMap(this.filters, view.dimension);
+    }
+    /**
+     * verifies if we can continue to do stuff on the data
+     * throws an error if the data does not exist
+     */
+    async assertDataExists(...dimensions) {
+        /**
+         * Check if the table exists
+         */
+        const tableExistsInDB = await this.db.tableExists();
+        if (!tableExistsInDB) {
+            throw new Error(`Table does not exists in the database`);
+        }
+        /**
+         * If provided, check if the dimensions exist
+         */
+        dimensions?.forEach(async (dimension) => {
+            if (dimension) {
+                const dimensionExistsInDB = await this.db.dimensionExists(dimension);
+                if (!dimensionExistsInDB) {
+                    throw new Error(`Dimension '${dimension.name}' does not exist in the data table`);
+                }
+            }
+        });
+    }
 }
 //# sourceMappingURL=falcon.js.map

build/core/falconArray/arrayTypes.d.ts

 export type TypedArray = Int8Array | Int16Array | Int32Array | Uint8Array | Uint8ClampedArray | Uint16Array | Uint32Array | Float32Array | Float64Array;
 export type TypedArrayConstructor = Int8ArrayConstructor | Int16ArrayConstructor | Int32ArrayConstructor | Uint8ArrayConstructor | Uint8ClampedArrayConstructor | Uint16ArrayConstructor | Uint32ArrayConstructor | Float32ArrayConstructor | Float64ArrayConstructor;
-export declare const CumulativeCountsArray: Float32ArrayConstructor;
+export declare const CumulativeCountsArray: Uint32ArrayConstructor;
 export declare const CountsArray: Uint32ArrayConstructor;
-export type CumulativeCountsArrayType = Float32Array;
+export type CumulativeCountsArrayType = Uint32Array;
 export type CountsArrayType = Uint32Array;

build/core/falconArray/arrayTypes.js

@@ -1,3 +0,3 @@
-export const CumulativeCountsArray = Float32Array;
+export const CumulativeCountsArray = Uint32Array;
 export const CountsArray = Uint32Array;
 //# sourceMappingURL=arrayTypes.js.map

build/core/falconArray/falconArray.d.ts

@@ -77,2 +77,6 @@ import type { TypedArray, TypedArrayConstructor } from "./arrayTypes";
     static allocCounts(length: number, shape?: number[], stride?: number[], offset?: number): FalconArray;
+    toString2D(): string;
+    toString1D(): string;
+    toString(): string;
+    deepCopy(ArrayType: TypedArrayConstructor): FalconArray;
 }

build/core/falconArray/falconArray.js

@@ -129,4 +129,40 @@ import { CumulativeCountsArray, CountsArray } from "./arrayTypes";
     }
+    toString2D() {
+        let totalString = "";
+        for (let i = 0; i < this.shape[0]; i++) {
+            let row = "";
+            for (let j = 0; j < this.shape[1]; j++) {
+                row += this.get(i, j) + " ";
+            }
+            totalString += row + "\n";
+        }
+        return totalString;
+    }
+    toString1D() {
+        let totalString = "";
+        for (let i = 0; i < this.shape[0]; i++) {
+            totalString += this.get(i) + " ";
+        }
+        return totalString;
+    }
+    toString() {
+        if (this.shape.length === 1) {
+            return this.toString1D();
+        }
+        else if (this.shape.length === 2) {
+            return this.toString2D();
+        }
+        else {
+            return "not implemented yet";
+        }
+    }
+    deepCopy(ArrayType) {
+        const copy = FalconArray.typedArray(ArrayType, this.length, this.shape, this.strides, this.offset);
+        for (let i = 0; i < this.length; i++) {
+            copy.data[i] = this.data[i];
+        }
+        return copy;
+    }
 }
 FalconArray.ALL = null;
 //# sourceMappingURL=falconArray.js.map

build/core/util.d.ts

@@ -7,2 +7,3 @@ import type { CategoricalRange, ContinuousDimension } from "./dimension";
 export type Interval<T> = [T, T];
+export type BinNumberFunction = (key: any) => number | undefined;
 /**
@@ -19,7 +20,2 @@ * Binning configuration.
  */
-interface StartStepBinConfig {
-    start: number;
-    stop?: number;
-    step: number;
-}
 /**
@@ -45,3 +41,3 @@ * BinConfig that does not need to have a step.
  */
-export declare function binNumberFunctionCategorical(range: CategoricalRange): (item: any) => number;
+export declare function binNumberFunctionCategorical(range: CategoricalRange): (item: any) => number | undefined;
 /**
@@ -53,2 +49,5 @@ * Get the number of bins for a bin configuration.
 export declare function binTime(maxbins: number, extent: Interval<number>): BinConfig;
+/**
+ * This function requires dimension.bins to exist
+ */
 export declare function createBinConfigContinuous(dimension: ContinuousDimension, extent: Interval<number>): BinConfig;
@@ -69,3 +68,3 @@ export declare function readableBinsContinuous(binConfig: BinConfig): {
  */
-export declare function scaleFilterToResolution(extent: Interval<number>, resolution: number): (x: number) => number;
+export declare function scaleFilterToResolution(extent: Interval<number>, resolution: number, nearestPixelInt?: (x: number) => number): (x: number) => number;
 export declare function excludeMap<K, V>(map: Map<K, V>, ...exclude: K[]): Map<K, V>;
@@ -75,9 +74,13 @@ /**
  */
-export declare function binNumberFunctionContinuous({ start, step, }: StartStepBinConfig): (v: number) => number;
+export declare function binNumberFunctionContinuous({ start, step, stop }: BinConfig): (v: number) => number;
 /**
+ * Creates a string equivalent to binNumberFunctionContinuous operation in SQL
+ */
+export declare function binNumberFunctionContinuousSQL(field: string, { start, step, stop }: BinConfig, castString?: (x: number) => string): string;
+/**
  * Returns a function that returns the bin for a pixel. Starts one pixel before so that the brush contains the data.
  */
-export declare function binNumberFunctionBinsContinuous({ start, stop }: StartStopBinConfig, pixel: number): (v: number) => number;
+export declare function binNumberFunctionPixels({ start, stop }: StartStopBinConfig, pixel: number): (v: number) => number;
 export declare function stepSize({ start, stop }: StartStopBinConfig, bins: number): number;
 export declare function compactQuery(str: string): string;
 export {};

build/core/util.js

@@ -44,2 +44,5 @@ import { scaleLinear } from "d3";
 }
+/**
+ * This function requires dimension.bins to exist
+ */
 export function createBinConfigContinuous(dimension, extent) {
@@ -87,9 +90,10 @@ const binningFunc = dimension.time ? binTime : binContinuous;
  */
-export function scaleFilterToResolution(extent, resolution) {
-    const pixelSpace = [0, resolution];
+export function scaleFilterToResolution(extent, resolution, nearestPixelInt = Math.floor) {
+    const pixelSpace = [1, resolution];
     const valueSpace = extent;
     const toPixels = scaleLinear().domain(valueSpace).range(pixelSpace);
+    toPixels.clamp(true);
     return (x) => {
         const pixels = toPixels(x);
-        return Math.floor(pixels);
+        return nearestPixelInt(pixels);
     };
@@ -103,11 +107,43 @@ }
  */
-export function binNumberFunctionContinuous({ start, step, }) {
-    return (v) => Math.floor((v - start) / step);
+export function binNumberFunctionContinuous({ start, step, stop }) {
+    /**
+     * this used to be (v: number) => Math.floor((v - start) / step
+     * using floor seems to map to a non-existent bin sometimes
+     *
+     * for example when the v=stop
+     * suppose we have start = 0, step = 20, stop = 100
+     * this should be 5 bins in total, each bin corresponding to 0, 1, 2, 3, or 4. (5 bins in total)
+     * however when v=100 (aka the stop)
+     * Math.floor(100-0 / 20) = 5. ???? 5 should not be an index that is a non-existent bin
+     *
+     * Math.ceil(100-0 / 20) - 1 = 4 is correct
+     * frick but when its 0, the whole things goes to -1
+     * so we need to clamp it to 0
+     */
+    const numBins = numBinsContinuous({ start, step, stop });
+    const lastBinIndex = numBins - 1;
+    return (v) => {
+        const binIndexMapping = Math.floor((v - start) / step);
+        if (binIndexMapping >= numBins) {
+            return lastBinIndex;
+        }
+        return binIndexMapping;
+    };
 }
 /**
+ * Creates a string equivalent to binNumberFunctionContinuous operation in SQL
+ */
+export function binNumberFunctionContinuousSQL(field, { start, step, stop }, castString = (x) => `${x}`) {
+    const numBins = numBinsContinuous({ start, step, stop });
+    const binIndexMapping = `FLOOR((${field} - ${castString(start)}) / ${castString(step)})::INT`;
+    const lastBinIndex = castString(numBins - 1);
+    const clamp = `LEAST(${lastBinIndex}, ${binIndexMapping})::INT`;
+    return clamp;
+}
+/**
  * Returns a function that returns the bin for a pixel. Starts one pixel before so that the brush contains the data.
  */
-export function binNumberFunctionBinsContinuous({ start, stop }, pixel) {
+export function binNumberFunctionPixels({ start, stop }, pixel) {
     const step = stepSize({ start, stop }, pixel);
-    return binNumberFunctionContinuous({ start: start, step });
+    return binNumberFunctionContinuous({ start, step, stop });
 }
@@ -114,0 +150,0 @@ export function stepSize({ start, stop }, bins) {

build/core/views/view0D.d.ts

 import { ViewAbstract } from "./viewAbstract";
 import type { Interval } from "../util";
-import type { Falcon } from "../falcon";
+import type { FalconVis } from "../falcon";
 import type { CategoricalRange } from "../dimension";
@@ -12,7 +12,7 @@ export interface View0DState {
     state: View0DState;
-    constructor(falcon: Falcon);
+    constructor(falcon: FalconVis);
     /**
      * @returns all count from the db and signals the user
      */
-    initializeAllCounts(): Promise<this>;
+    all(): Promise<this>;
     /**
@@ -26,2 +26,10 @@ * Given an active 1D view, count for this passive view
     countFromActiveCategorical1D(selection?: CategoricalRange, totalRange?: CategoricalRange): Promise<void>;
+    /**
+     * attaches to the global falcon index
+     */
+    attach(): Promise<void>;
+    /**
+     * detaches from the global falcon index
+     */
+    detach(): Promise<void>;
 }

build/core/views/view0D.js

@@ -11,4 +11,4 @@ import { binNumberFunctionCategorical } from "../util";
      */
-    async initializeAllCounts() {
-        const total = await this.falcon.db.length();
+    async all() {
+        const total = await this.falcon.db.length(this.falcon.filters.size > 0 ? this.falcon.filters : undefined);
         this.state.total = total;
@@ -60,3 +60,5 @@ this.state.filter = total;
                 const binKey = bin(s);
-                total += index.filter.get(binKey);
+                if (binKey) {
+                    total += index.filter.get(binKey);
+                }
             }
@@ -68,3 +70,17 @@ this.state.filter = total;
     }
+    /**
+     * attaches to the global falcon index
+     */
+    async attach() {
+        this.falcon.views.add(this);
+        await this.falcon.link();
+    }
+    /**
+     * detaches from the global falcon index
+     */
+    async detach() {
+        this.falcon.views.remove(this);
+        this.falcon.index.delete(this);
+    }
 }
 //# sourceMappingURL=view0D.js.map

build/core/views/view1D.d.ts

 import { ViewAbstract } from "./viewAbstract";
-import type { Falcon } from "../falcon";
+import type { FalconVis } from "../falcon";
 import type { CategoricalRange, Dimension, DimensionFilter } from "../dimension";
@@ -25,4 +25,11 @@ import type { Interval } from "../util";
     lastFilter: DimensionFilter | undefined;
-    constructor(falcon: Falcon, dimension: Dimension);
+    constructor(falcon: FalconVis, dimension: Dimension);
     /**
+     * slowest way to update data
+     *
+     * @todo this breaks when an active view is on
+     * @todo replace this with targeted updates instead of just recomputing everything
+     */
+    update(dimension: Dimension): Promise<void>;
+    /**
      * populates the extent in the dimension if not already defined
@@ -36,7 +43,7 @@ */
      */
-    initializeAllCounts(): Promise<this>;
+    all(): Promise<this>;
     /**
      * prefetch the 1D falcon index
      */
-    prefetch(): Promise<void>;
+    computeIndex(force?: boolean): Promise<void>;
     /**
@@ -51,3 +58,3 @@ * activates this view: makes this view the active view
      */
-    select(filter?: DimensionFilter, convertToPixels?: boolean): Promise<void>;
+    select(filter?: DimensionFilter, force?: boolean): Promise<void>;
     /**
@@ -61,2 +68,12 @@ * Given an active 1D view, count for this passive view
     countFromActiveCategorical1D(selection?: CategoricalRange, totalRange?: CategoricalRange): Promise<void>;
+    /**
+     * attaches to the global falcon index
+     */
+    attach(): Promise<void>;
+    /**
+     * detaches from the global falcon index
+     *
+     * if I detach an active view, I need to relink
+     */
+    detach(): Promise<void>;
 }

build/core/views/view1D.js

@@ -12,2 +12,12 @@ import { ViewAbstract } from "./viewAbstract";
     /**
+     * slowest way to update data
+     *
+     * @todo this breaks when an active view is on
+     * @todo replace this with targeted updates instead of just recomputing everything
+     */
+    async update(dimension) {
+        this.dimension = dimension;
+        await this.falcon.link();
+    }
+    /**
      * populates the extent in the dimension if not already defined
@@ -20,4 +30,9 @@ */
         if (this.dimension.type === "continuous") {
-            this.toPixels = brushToPixelSpace(this.dimension.range, this.dimension.resolution);
+            // if the bins are specified, autocompute the best num of bins!
+            this.dimension.bins =
+                this.dimension.bins ??
+                    (await this.falcon.db.estimateNumBins(this.dimension, 200, 15));
             this.dimension.binConfig = createBinConfigContinuous(this.dimension, this.dimension.range);
+            const { start: firstBinStart, stop: veryLastBinEnd } = this.dimension.binConfig;
+            this.toPixels = brushToPixelSpace([firstBinStart, veryLastBinEnd], this.dimension.resolution);
         }
@@ -37,5 +52,5 @@ if (this.dimension.type === "continuous") {
      */
-    async initializeAllCounts() {
+    async all() {
         await this.createBins();
-        const counts = await this.falcon.db.histogramView1D(this);
+        const counts = await this.falcon.db.histogramView1D(this, this.falcon.filters.size > 0 ? this.falcon.otherFilters(this) : undefined);
         this.state.total = counts.noFilter.data;
@@ -49,4 +64,4 @@ this.state.filter = counts.filter.data;
      */
-    async prefetch() {
-        if (!this.isActive) {
+    async computeIndex(force = false) {
+        if (!this.isActive || force) {
             // make sure we have binConfigs computed for all views if this one is activated
@@ -60,3 +75,3 @@ await this.falcon.views.forEach(async (view) => {
                 if (rangeNotComputed) {
-                    await view.initializeAllCounts();
+                    await view.all();
                 }
@@ -77,3 +92,3 @@ });
     async activate() {
-        await this.prefetch();
+        await this.computeIndex();
     }
@@ -83,4 +98,3 @@ /**
      */
-    async select(filter, convertToPixels = true) {
-        await this.prefetch();
+    async select(filter, force = false) {
         if (filter) {
@@ -92,3 +106,3 @@ if (this.dimension.type === "continuous") {
                     this.lastFilter[1] === filter[1];
-                if (filterStayedTheSame) {
+                if (filterStayedTheSame && force === false) {
                     return;
@@ -99,25 +113,9 @@ }
                 // convert active selection into pixels if needed
-                let selectPixels = convertToPixels
-                    ? this.toPixels(filter)
-                    : filter;
-                /**
-                 * if they query something outside the possible resolution.
-                 * just do nothing!
-                 */
-                if (selectPixels[1] > this.dimension.resolution) {
-                    selectPixels[1] = this.dimension.resolution;
+                let selectPixels = this.toPixels(filter);
+                if (this.isActive) {
+                    // use the index to count for the passive views
+                    this.falcon.views.passive.forEach(async (passiveView) => {
+                        await passiveView.countFromActiveContinuous1D(selectPixels);
+                    });
                 }
-                if (selectPixels[0] > this.dimension.resolution) {
-                    selectPixels[0] = this.dimension.resolution;
-                }
-                if (selectPixels[1] < 0) {
-                    selectPixels[1] = 0;
-                }
-                if (selectPixels[0] < 0) {
-                    selectPixels[0] = 0;
-                }
-                // use the index to count for the passive views
-                this.falcon.views.passive.forEach(async (passiveView) => {
-                    await passiveView.countFromActiveContinuous1D(selectPixels);
-                });
                 this.lastFilter = filter;
@@ -128,6 +126,8 @@ }
                 this.falcon.filters.set(this.dimension, filter);
-                // use the index to count for the passive views
-                this.falcon.views.passive.forEach(async (passiveView) => {
-                    await passiveView.countFromActiveCategorical1D(filter, this.dimension.range);
-                });
+                if (this.isActive) {
+                    // use the index to count for the passive views
+                    this.falcon.views.passive.forEach(async (passiveView) => {
+                        await passiveView.countFromActiveCategorical1D(filter, this.dimension.range);
+                    });
+                }
                 this.lastFilter = filter;
@@ -137,25 +137,27 @@ }
         else {
-            if (this.dimension.type === "continuous") {
-                // just end now if the filter hasn't changed (still undefined)
-                const filterStayedTheSame = this.lastFilter === filter;
-                if (filterStayedTheSame) {
-                    return;
+            if (this.isActive) {
+                if (this.dimension.type === "continuous") {
+                    // just end now if the filter hasn't changed (still undefined)
+                    const filterStayedTheSame = this.lastFilter === filter;
+                    if (filterStayedTheSame) {
+                        return;
+                    }
+                    // remove filter
+                    this.falcon.filters.delete(this.dimension);
+                    // and revert back counts
+                    this.falcon.views.passive.forEach(async (passiveView) => {
+                        await passiveView.countFromActiveContinuous1D();
+                    });
+                    this.lastFilter = filter;
                 }
-                // remove filter
-                this.falcon.filters.delete(this.dimension);
-                // and revert back counts
-                this.falcon.views.passive.forEach(async (passiveView) => {
-                    await passiveView.countFromActiveContinuous1D();
-                });
-                this.lastFilter = filter;
+                else {
+                    // remove filter
+                    this.falcon.filters.delete(this.dimension);
+                    // and revert back counts
+                    this.falcon.views.passive.forEach(async (passiveView) => {
+                        await passiveView.countFromActiveCategorical1D();
+                    });
+                    this.lastFilter = filter;
+                }
             }
-            else {
-                // remove filter
-                this.falcon.filters.delete(this.dimension);
-                // and revert back counts
-                this.falcon.views.passive.forEach(async (passiveView) => {
-                    await passiveView.countFromActiveCategorical1D();
-                });
-                this.lastFilter = filter;
-            }
         }
@@ -211,4 +213,6 @@ }
                 const binKey = bin(s);
-                const counts = index.filter.slice(binKey, null);
-                binCounts.addToItself(counts);
+                if (binKey) {
+                    const counts = index.filter.slice(binKey, null);
+                    binCounts.addToItself(counts);
+                }
             }
@@ -220,3 +224,23 @@ this.state.filter = binCounts.data;
     }
+    /**
+     * attaches to the global falcon index
+     */
+    async attach() {
+        this.falcon.views.add(this);
+        await this.falcon.link();
+    }
+    /**
+     * detaches from the global falcon index
+     *
+     * if I detach an active view, I need to relink
+     */
+    async detach() {
+        this.falcon.views.remove(this);
+        this.falcon.index.delete(this);
+        // if we remove the active view, revert back
+        if (this.isActive) {
+            await this.falcon.link();
+        }
+    }
 }
 //# sourceMappingURL=view1D.js.map

build/core/views/viewAbstract.d.ts

 import type { Interval } from "../util";
-import type { Falcon } from "../falcon";
+import type { FalconVis } from "../falcon";
 import type { CategoricalRange } from "../dimension";
 export type OnChange<S> = (state: S) => void;
 export declare abstract class ViewAbstract<S extends object> {
-    falcon: Falcon;
+    falcon: FalconVis;
     onChangeListeners: Set<OnChange<S>>;
@@ -13,7 +13,7 @@ isActive: boolean;
      */
-    constructor(falcon: Falcon);
+    constructor(falcon: FalconVis);
     /**
      * fetches all the counts
      */
-    abstract initializeAllCounts(): Promise<this> | this;
+    abstract all(): Promise<this> | this;
     /**
@@ -43,2 +43,4 @@ * given the active view is continuous 1D, compute the counts for this view as passive
     protected markThisViewActive(): void;
+    abstract detach(): void | Promise<void>;
+    abstract attach(): void | Promise<void>;
 }

build/core/views/viewSet.d.ts

@@ -1,2 +0,2 @@
-import { View } from "./index";
+import { View, View1D } from "./index";
 /**
@@ -9,2 +9,3 @@ * Collections of views
     constructor();
+    remove(viewToRemove: View): void;
     /**
@@ -20,5 +21,5 @@ * adds view to collections
     /**
-     * @returns the only active view
+     * @returns the only active view, undefined if didn't find it
      */
-    get active(): View;
+    get active(): View1D | undefined;
     get size(): number;
@@ -25,0 +26,0 @@ forEach(eachView: (view: View, index: number) => void): void;

build/core/views/viewSet.js

@@ -9,2 +9,8 @@ /**
     }
+    remove(viewToRemove) {
+        const index = this.views.findIndex((view) => view == viewToRemove);
+        if (index !== -1) {
+            this.views.splice(index, 1);
+        }
+    }
     /**
@@ -27,10 +33,13 @@ * adds view to collections
     /**
-     * @returns the only active view
+     * @returns the only active view, undefined if didn't find it
      */
     get active() {
         const activeView = this.views.find((view) => view.isActive);
-        if (activeView === undefined) {
-            throw Error("No active view found, this should not happen");
+        // if found, must be a View1D TODO extend to View2D
+        if (activeView) {
+            return activeView;
         }
-        return activeView;
+        else {
+            return undefined;
+        }
     }
@@ -37,0 +46,0 @@ get size() {

package.json

 {
   "name": "falcon-vis",
-  "version": "0.17.1",
+  "version": "0.17.2",
   "main": "build",
@@ -16,7 +16,7 @@ "repository": "[e-mail]:cmudig/falcon.git",
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.20.0",
+    "@duckdb/duckdb-wasm": "^1.24.0",
     "@heavyai/connector": "^7.0.0",
     "apache-arrow": "^11.0.0",
     "cwise": "^1.0.10",
-    "d3": "^7.8.2",
+    "d3": "^7.8.4",
     "fast-kde": "^0.2.1",
@@ -28,3 +28,3 @@ "ndarray": "^1.0.19",
     "ndarray-prefix-sum": "^1.0.0",
-    "vega-statistics": "^1.8.0"
+    "vega-statistics": "^1.8.1"
   },
@@ -34,7 +34,7 @@ "devDependencies": {
     "@types/d3": "^7.4.0",
-    "@types/jest": "^29.4.0",
+    "@types/jest": "^29.5.0",
     "@types/ndarray": "1.0.11",
     "@types/ndarray-ops": "^1.2.4",
-    "jest": "^29.3.1",
-    "prettier": "^2.8.1",
+    "jest": "^29.5.0",
+    "prettier": "^2.8.7",
     "ts-jest": "^29.0.3",
@@ -41,0 +41,0 @@ "typescript": "^4.9.4"

README.md

 <p align="center">
-  <img src="logo/logo.png" width="200" style="transform: rotate(90deg);">
+  <img src="https://user-images.githubusercontent.com/65095341/224896033-afc8bd8e-d0e0-4031-a7b2-3857bef51327.svg" width="65%">
 </p>
@@ -11,131 +11,112 @@
 
-A javascript library to cross-filter millions of records in your own applications without latencies!
+```bash
+npm install falcon-vis
+```
 
-Try out [the live demo](http://dig.cmu.edu/falcon/) using FalconVis.
+A simple javascript API to link visualizations at scale. FalconVis is a successor to [`vega/falcon`](https://github.com/vega/falcon).
 
-Implements the original [Falcon](https://github.com/vega/falcon) under the hood.
+Finally, you can cross-filter all the data you've been hoarding without the large delays or slowdowns. Interactions are smooth, fast, and responsive.
 
-## Usage
+**Live Browser Examples**
 
-Preview of [`data/movies.json`](data/movies.json): an array of objects
+-   TODO: add json 10k
+-   TODO: add arrow 1million
+-   TODO: add duckdb-wasm 5 million
+-   [[30 Million Flights | DuckDB Python]](https://huggingface.co/spaces/donnyb/FalconVis)
 
-```json
-[
-	{
-		"Title": "Forrest Gump",
-		"US_Gross": 329694499,
-		"MPAA_Rating": "PG-13",
-		...
-	},
-	...
-	{
-		"Title": "Star Wars Ep. VI: Return of the Jedi",
-		"US_Gross": 309205079,
-		"MPAA_Rating": "PG",
-		...
-	}
-]
-```
+## Example
 
-### Getting Started
+Check out real examples in the [`examples/`](examples/) directory. If you want a quick sneak-peak, keep reading.
 
-Create a falcon object to get ready to cross-filter
+TODO make this section more clear.
 
-```typescript
-import * as falconVis from "falcon-vis";
+### Initialization
 
-const data = await fetch("data/movies.json").then((d) => d.json());
-const jsonDB = new falconVis.JsonDB(data);
+Create a the `falcon` instance and link it up to some data and some views.
 
-// this object will be used for cross-filtering
-const falcon = new falconVis.Falcon(jsonDB);
-```
+```ts
+import { FalconVis, JsonDB } from "falcon-vis";
 
-For larger data, or data on a server, use a different DB also provided by FalconVis:
+/**
+ * 1. create the falcon instance with a database
+ */
+const plainOldJavascriptObject = await fetch("flights-10k.json").then((d) =>
+	d.json()
+);
+const db = new JsonDB(plainOldJavascriptObject);
+falcon = new FalconVis(db);
 
--   ArrowDB
--   DuckDB
--   HttpDB
--   MapDDB
+/**
+ * 2. create the views (you interact with these directly to cross-filter)
+ */
+// 0D is total count (num rows)
+const countView = await falcon.view0D();
+countView.onChange((updatedTotalCount) => {
+	// called every time the count changes
+	console.log(updatedTotalCount); // you can do whatever in here
+});
 
-(todo link docs on these usages)
+// 1D is a histogram
+const distanceView = await falcon.view1D({
+	type: "continuous",
+	name: "Distance",
+	resolution: 400,
+	bins: 5,
+});
+distanceView.onChange((updatedHistogram) => {
+	console.log(updatedHistogram);
+});
 
-### Initializing Views
+/**
+ * 3. initialize falcon by linking everything together
+ */
+await falcon.link();
+```
 
-FalconVis operates by linking together views. You directly interact with views after linking them to the falcon object.
+### Cross-filtering
 
-```typescript
-const falcon = new falconVis.Falcon(jsonDB);
+The view you that you want to filter is the active view. Once you do filter an active view, the `onChange` callbacks will be called with the updated counts for all the other linked views.
 
-// link the US_Gross earnings for the movies
-const usGrossView = await falcon.linkView1D(
-	{
-		type: "continuous",
-		name: "US_Gross",
-		bins: 25,
-		resolution: 400, // my actual visualization is 400px wide
-	},
-	// onChange gets called every time new filters are applied to any linked view
-	(updatedBinCounts) => {
-		// for example, I could update the DOM with these counts
-		console.log(updatedBinCounts);
-	}
-);
+```ts
+/**
+ * 1. make the view you are interacting with active
+ *    this computes the fast falcon index in the background
+ */
+await distanceView.activate();
 
-// link the movie ratings
-const ratingView = await falcon.linkView1D(
-	{
-		type: "categorical",
-		name: "MPAA_Rating",
-	},
-	(updatedBinCounts) => {
-		console.log(updatedBinCounts);
-	}
-);
+/**
+ * 2. select/filter a range between 0 and 100
+ */
+await distanceView.select([0, 100]);
+```
 
-// link the total number of rows selected (number that remain after filter)
-const totalCount = await falcon.linkCount((updatedCount) => {
-	console.log(updateCount);
-});
+## API Reference
 
-// initially populate ALL the views' counts
-await falcon.initializeAllCounts();
+TODO fill in and add examples
 
-// OR you can manually call view.initializeAllCounts() for each one
-await usGrossView.initializeAllCounts();
-await ratingView.initializeAllCounts();
-await totalCount.initializeAllCounts();
-```
+### Core
 
-### Cross-filtering views
+<br><a href="#">#</a> index.<b>view0D</b>()
 
-FalconVis uses [Falcon](https://github.com/vega/falcon) to cross-filter views latency free for even very large amounts of data (many millions or billions).
+<br><a href="#">#</a> index.<b>view1D</b>()
 
-All you need to do is call `.activate()` (for the [Falcon](https://github.com/vega/falcon) magic) and `.select()` to cross-filter (you `.select()` what to keep).
+<br><a href="#">#</a> index.<b>link</b>()
 
-Each time `.select()` is called, the onChange functions defined in the previous section for all linked views will be called (since the counts get updated).
+<br><a href="#">#</a> index.<b>entries</b>()
 
-```typescript
-// after activation, subsequence .select() will be O(1) constant time calls
-await usGrossView.activate();
+<br><a href="#">#</a> view.<b>activate</b>()
 
-// cross-filter between $500,000 and $1_000_000
-await usGrossView.select([500_000, 1_000_000]);
+<br><a href="#">#</a> view.<b>select</b>()
 
-// if you want to deselect
-await usGrossView.select();
+<br><a href="#">#</a> view.<b>onChange</b>()
 
-// or activate a different view and add another filter
-await ratingView.activate();
-await ratingView.select(["PG-13", "G"]);
-```
+## Databases
 
-### Examples
+<br><a href="#">#</a> <b>JsonDB</b>
+<br><a href="#">#</a> <b>ArrowDB</b>
+<br><a href="#">#</a> <b>DuckDB</b>
+<br><a href="#">#</a> <b>HttpDB</b>
 
-To see real working examples, check out the self-contained examples in the [`examples/`](examples/) directory.
+## Development
 
-Check out the workspace [`package.json`](package.json) or the specific example's `package.json` for more information on how to run them.
-
-### Development
-
 Check out the [`CONTRIBUTING.md`](CONTRIBUTING.md) document to see how to run the development server.

No alert changes

Improved metrics

Total package byte prevSize

179767

increased by14.37%

Number of package files

78

increased by4%

Lines of code

3011

increased by14.18%

Worsened metrics

Number of lines in readme file

122

decreased by-13.48%

Dependency changes

• Updated@duckdb/duckdb-wasm@^1.24.0

• Updatedd3@^7.8.4

• Updatedvega-statistics@^1.8.1