@operational/frame - npm Package Compare versions

Comparing version 0.3.5 to 0.3.6

lib/DataFrame.d.ts

 import { PivotFrame } from "./PivotFrame";
-import { ColumnCursor, IterableFrame, Matrix, PivotProps, Schema, RawRow } from "./types";
+import { ColumnCursor, IterableFrame, Matrix, PivotProps, Schema, RowCursor } from "./types";
 import { getData } from "./secret";
+import { GroupFrame } from "./GroupFrame";
 export declare class DataFrame<Name extends string = string> implements IterableFrame<Name> {
@@ -8,2 +9,3 @@ private readonly data;
     private readonly cursorCache;
+    private readonly referentialCache;
     constructor(schema: Schema<Name>, data: Matrix<any>);
@@ -17,6 +19,6 @@ stats(): {
     getCursor(column: Name): ColumnCursor<Name>;
-    groupBy(columns: Array<Name | ColumnCursor<Name>>): Array<IterableFrame<Name>>;
+    pivot<Column extends Name, Row extends Name>(prop: PivotProps<Column, Row>): PivotFrame<Name>;
+    groupBy(columns: Array<Name | ColumnCursor<Name>>): GroupFrame<Name>;
     uniqueValues(columns: Array<Name | ColumnCursor<Name>>): string[][];
-    mapRows<A>(callback: (row: RawRow[], index: number) => A): A[];
-    pivot<Column extends Name, Row extends Name>(prop: PivotProps<Column, Row>): PivotFrame<Name>;
+    mapRows<A>(callback: (row: RowCursor[], index: number) => A): A[];
     [getData](): readonly [{
@@ -23,0 +25,0 @@ name: Name;

lib/DataFrame.js

 import { PivotFrame } from "./PivotFrame";
 import { getData } from "./secret";
-import { isCursor } from "./utils";
-import { uniqueValueCombinations } from "./stats";
-import { FragmentFrame } from "./FragmentFrame";
+import { isCursor, hashCursors } from "./utils";
+import { GroupFrame } from "./GroupFrame";
+import flru from "flru";
 export class DataFrame {
@@ -11,2 +11,4 @@ constructor(schema, data) {
         this.cursorCache = new Map();
+        // this one is small cache, because it is for rare operations
+        this.referentialCache = flru(5);
     }
@@ -31,3 +33,3 @@ stats() {
             }
-            const cursor = ((row) => row[index]);
+            const cursor = (row => row[index]);
             cursor.column = column;
@@ -39,22 +41,21 @@ cursor.index = index;
     }
+    pivot(prop) {
+        const columnCursors = prop.columns.map(c => (isCursor(c) ? c : this.getCursor(c)));
+        const rowCursors = prop.rows.map(r => (isCursor(r) ? r : this.getCursor(r)));
+        const hash = `${hashCursors(columnCursors)}x${hashCursors(rowCursors)}`;
+        if (!this.referentialCache.has(hash)) {
+            this.referentialCache.set(hash, new PivotFrame(this, prop));
+        }
+        return this.referentialCache.get(hash);
+    }
     groupBy(columns) {
-        // If no columns are provided, returns an array with the current frame as the sole entry.
-        if (columns.length === 0) {
-            return [this];
+        const columnCursors = columns.map(c => (isCursor(c) ? c : this.getCursor(c)));
+        const hash = hashCursors(columnCursors);
+        if (!this.referentialCache.has(hash)) {
+            this.referentialCache.set(hash, new GroupFrame(this, columnCursors));
         }
-        const columnCursors = columns.map(c => isCursor(c) ? c : this.getCursor(c));
-        // Returns a FragmentFrame for every unique combination of column values.
-        return uniqueValueCombinations(this, columnCursors).map(u => {
-            const indices = this.data.reduce((arr, row, i) => {
-                if (columnCursors.every((cursor, j) => cursor(row) === u[j])) {
-                    arr.push(i);
-                }
-                return arr;
-            }, []);
-            return new FragmentFrame(this, indices);
-        });
+        return this.referentialCache.get(hash);
     }
     uniqueValues(columns) {
-        const columnCursors = columns.map(c => isCursor(c) ? c : this.getCursor(c));
-        return uniqueValueCombinations(this, columnCursors);
+        return this.groupBy(columns).uniqueValues();
     }
@@ -64,5 +65,2 @@ mapRows(callback) {
     }
-    pivot(prop) {
-        return new PivotFrame(this, prop);
-    }
     // for internal use only
@@ -69,0 +67,0 @@ [getData]() {

lib/FragmentFrame.d.ts

 import { DataFrame } from "./DataFrame";
-import { IterableFrame, RawRow, ColumnCursor, Schema } from "./types";
+import { IterableFrame, RowCursor, ColumnCursor, Schema } from "./types";
+import { getData } from "./secret";
+import { GroupFrame } from "./GroupFrame";
 export declare class FragmentFrame<Name extends string = string> implements IterableFrame<Name> {
@@ -8,10 +10,15 @@ private readonly data;
     private readonly origin;
-    constructor(origin: DataFrame<Name>, index: number[]);
-    getCursor(column: Name): ColumnCursor<Name, any>;
-    groupBy(columns: Array<Name | ColumnCursor<Name>>): Array<IterableFrame<Name>>;
+    private readonly groupByCache;
+    constructor(origin: DataFrame<Name> | FragmentFrame<Name>, index: number[]);
+    getCursor(column: Name): ColumnCursor<Name>;
+    groupBy(columns: Array<Name | ColumnCursor<Name>>): GroupFrame<Name>;
     uniqueValues(columns: Array<Name | ColumnCursor<Name>>): string[][];
-    mapRows<A>(callback: (row: RawRow, index: number) => A): A[];
+    mapRows<A>(callback: (row: RowCursor, index: number) => A): A[];
     row(rowIndex: number): any[];
+    [getData](): readonly [{
+        name: Name;
+        type?: any;
+    }[], any[][], number[]];
     peak(column: Name | ColumnCursor<Name>): any;
 }
 //# sourceMappingURL=FragmentFrame.d.ts.map

lib/FragmentFrame.js

 import { getData } from "./secret";
-import { isCursor } from "./utils";
-import { uniqueValueCombinations } from "./stats";
-const isColumnCursor = (column) => {
-    return column.index !== undefined;
-};
+import { isCursor, hashCursors } from "./utils";
+import { GroupFrame } from "./GroupFrame";
 export class FragmentFrame {
@@ -14,2 +11,3 @@ constructor(origin, index) {
         this.index = index;
+        this.groupByCache = new Map();
     }
@@ -20,21 +18,11 @@ getCursor(column) {
     groupBy(columns) {
-        // If no columns are provided, returns an array with the current frame as a FragmentFrame as the sole entry.
-        if (columns.length === 0) {
-            return [this];
+        const columnCursors = columns.map(c => (isCursor(c) ? c : this.getCursor(c)));
+        const hash = hashCursors(columnCursors);
+        if (!this.groupByCache.has(hash)) {
+            this.groupByCache.set(hash, new GroupFrame(this, columnCursors));
         }
-        const columnCursors = columns.map(c => isCursor(c) ? c : this.getCursor(c));
-        // Returns a FragmentFrame for every unique combination of column values.
-        return uniqueValueCombinations(this, columnCursors).map(u => {
-            const indices = this.data.reduce((arr, row, i) => {
-                if (columnCursors.every((cursor, j) => cursor(row) === u[j]) && this.index.includes(i)) {
-                    arr.push(i);
-                }
-                return arr;
-            }, []);
-            return new FragmentFrame(this.origin, indices);
-        });
+        return this.groupByCache.get(hash);
     }
     uniqueValues(columns) {
-        const columnCursors = columns.map(c => isCursor(c) ? c : this.getCursor(c));
-        return uniqueValueCombinations(this, columnCursors);
+        return this.groupBy(columns).uniqueValues();
     }
@@ -47,5 +35,9 @@ mapRows(callback) {
     }
+    // for internal use only
+    [getData]() {
+        return [this.schema, this.data, this.index];
+    }
     // we need this function for table display
     peak(column) {
-        const columnIndex = isColumnCursor(column) ? column.index : this.schema.findIndex(x => x.name === column);
+        const columnIndex = isCursor(column) ? column.index : this.schema.findIndex(x => x.name === column);
         if (columnIndex < 0) {
@@ -52,0 +44,0 @@ throw new Error(`Unknown column ${column}`);

lib/index.d.ts

@@ -7,2 +7,3 @@ export * from "./types";
 export { FragmentFrame } from "./FragmentFrame";
+export { GroupFrame } from "./GroupFrame";
 //# sourceMappingURL=index.d.ts.map

lib/index.js

@@ -6,2 +6,3 @@ export * from "./stats";
 export { FragmentFrame } from "./FragmentFrame";
+export { GroupFrame } from "./GroupFrame";
 //# sourceMappingURL=index.js.map

lib/PivotFrame.d.ts

 import { DataFrame } from "./DataFrame";
 import { FragmentFrame } from "./FragmentFrame";
-import { PivotProps, WithCursor } from "./types";
-export declare type DimensionValue = string;
+import { PivotProps, WithCursor, DimensionValue } from "./types";
 export declare class PivotFrame<Name extends string = string> implements WithCursor<Name> {
-    private readonly data;
-    private readonly schema;
     private readonly prop;
@@ -14,5 +11,3 @@ private readonly origin;
     protected rowIndex: number[][];
-    protected frameWithoutPivoting: FragmentFrame<Name>;
-    private readonly rowCache;
-    private readonly columnCache;
+    private readonly cache;
     constructor(origin: DataFrame<Name>, prop: PivotProps<Name, Name>);
@@ -19,0 +14,0 @@ getCursor(column: Name): import("./types").ColumnCursor<Name, any>;

lib/PivotFrame.js

@@ -0,1 +1,2 @@
+import flru from "flru";
 import { FragmentFrame } from "./FragmentFrame";
@@ -7,8 +8,4 @@ import { getData } from "./secret";
         this.origin = origin;
-        const [schema, data] = origin[getData]();
-        this.schema = schema;
-        this.data = data;
         this.prop = prop;
-        this.rowCache = new Map();
-        this.columnCache = new Map();
+        this.cache = flru(1024);
     }
@@ -18,3 +15,3 @@ getCursor(column) {
     }
-    // this is reverse operation of pivot in the DataFrame
+    // reverse operation of pivot in the DataFrame
     // in Pandas this called melt
@@ -52,6 +49,7 @@ unpivot() {
         }
-        if (!this.rowCache.has(rowIdentifier)) {
-            this.rowCache.set(rowIdentifier, new FragmentFrame(this.origin, row));
+        const key = `${rowIdentifier}r`;
+        if (!this.cache.has(key)) {
+            this.cache.set(key, new FragmentFrame(this.origin, row));
         }
-        return this.rowCache.get(rowIdentifier);
+        return this.cache.get(key);
     }
@@ -66,6 +64,7 @@ column(columnIdentifier) {
         }
-        if (!this.columnCache.has(columnIdentifier)) {
-            this.columnCache.set(columnIdentifier, new FragmentFrame(this.origin, column));
+        const key = `${columnIdentifier}c`;
+        if (!this.cache.has(key)) {
+            this.cache.set(key, new FragmentFrame(this.origin, column));
         }
-        return this.columnCache.get(columnIdentifier);
+        return this.cache.get(key);
     }
@@ -83,6 +82,10 @@ cell(rowIdentifier, columnIdentifier) {
         this.buildIndex();
-        const row = this.rowIndex[rowIdentifier];
-        const column = this.columnIndex[columnIdentifier];
-        const cell = intersect(row, column);
-        return new FragmentFrame(this.origin, cell);
+        const key = `${rowIdentifier}x${columnIdentifier}`;
+        if (!this.cache.has(key)) {
+            const row = this.rowIndex[rowIdentifier];
+            const column = this.columnIndex[columnIdentifier];
+            const cell = intersect(row, column);
+            this.cache.set(key, new FragmentFrame(this.origin, cell));
+        }
+        return this.cache.get(key);
     }
@@ -93,6 +96,8 @@ // This is very specific case when we need PivotFrame, but without pivoting itself.
     getFrameWithoutPivoting() {
-        if (!this.frameWithoutPivoting) {
-            this.frameWithoutPivoting = new FragmentFrame(this.origin, this.data.map((_, i) => i));
+        const key = `0`;
+        if (!this.cache.has(key)) {
+            const [, data] = this.origin[getData]();
+            this.cache.set(key, new FragmentFrame(this.origin, data.map((_, i) => i)));
         }
-        return this.frameWithoutPivoting;
+        return this.cache.get(key);
     }
@@ -103,3 +108,4 @@ buildIndex() {
         }
-        const nameToIndex = this.schema.reduce((acc, columnDefinition, index) => {
+        const [schema, data] = this.origin[getData]();
+        const nameToIndex = schema.reduce((acc, columnDefinition, index) => {
             acc[columnDefinition.name] = index;
@@ -116,3 +122,3 @@ return acc;
          * In the end there is a list of indexes of rows from original data.
-         * If we need to find what rows (from the `this.data`) correspond to [valueFromRowA, valueFromRowB ...]
+         * If we need to find what rows (from the `data`) correspond to [valueFromRowA, valueFromRowB ...]
          * we can do following rowTreeIndex[valueFromRowA][valueFromRowB][...] and we will get array on indexes
@@ -129,3 +135,3 @@ */
          * matrix structure implemented as array of arrays
-         * [[<numbers of rows in this.data with valueFromRowA, valueFromRowB ...>]]
+         * [[<numbers of rows in data with valueFromRowA, valueFromRowB ...>]]
          * first row in rowHeaders corresponds to first row in rowHeaders etc.
@@ -142,3 +148,3 @@ */
         const columnIndex = [];
-        this.data.forEach((dataRow, rowNumber) => {
+        data.forEach((dataRow, rowNumber) => {
             const rowHeader = [];
@@ -145,0 +151,0 @@ let previousRow = rowTreeIndex;

lib/PivotFramePreindexed.d.ts

 import { DataFrame } from "./DataFrame";
-import { DimensionValue, PivotFrame } from "./PivotFrame";
-import { PivotProps } from "./types";
+import { PivotFrame } from "./PivotFrame";
+import { PivotProps, DimensionValue } from "./types";
 /**
@@ -5,0 +5,0 @@ * This class exposes internal implementation of PivotFrame,

lib/stats.d.ts

 import { IterableFrame, ColumnCursor } from "./types";
-export declare const maxValue: <Name extends string>(frame: IterableFrame<Name>, column: ColumnCursor<Name, any>) => number;
-export declare const uniqueValues: <Name extends string>(frame: IterableFrame<Name>, column: ColumnCursor<Name, any>) => string[];
-export declare const uniqueValueCombinations: <Name extends string>(frame: IterableFrame<Name>, columns: ColumnCursor<Name, any>[]) => string[][];
+import { GroupFrame } from "./GroupFrame";
+export declare const maxValue: <Name extends string>(frame: IterableFrame<Name> | GroupFrame<Name>, column: ColumnCursor<Name, any>) => number;
+export declare const total: <Name extends string>(frame: IterableFrame<Name> | GroupFrame<Name>, column: ColumnCursor<Name, any>) => number;
+export declare const uniqueValues: <Name extends string>(frame: IterableFrame<Name> | GroupFrame<Name>, column: ColumnCursor<Name, any>) => string[];
 //# sourceMappingURL=stats.d.ts.map

lib/stats.js

@@ -0,1 +1,2 @@
+import { isGroupFrame } from "./GroupFrame";
 const statsCache = new WeakMap();
@@ -12,2 +13,3 @@ const getStatsCacheItem = (frame, column) => {
 };
+// rename to max
 export const maxValue = (frame, column) => {
@@ -26,5 +28,21 @@ const cacheItem = getStatsCacheItem(frame, column);
         let max = undefined;
-        frame.mapRows(row => {
-            max = max === undefined ? row[column.index] : Math.max(max, row[column.index]);
-        });
+        if (isGroupFrame(frame)) {
+            // we use sum as aggregation function, but we can use average or mean
+            // we can pass information about which aggregation to use inside GroupFrame
+            const aggregation = total;
+            frame.map(row => {
+                const value = aggregation(row, column);
+                if (value != null) {
+                    max = max === undefined ? value : Math.max(max, value);
+                }
+            });
+        }
+        else {
+            frame.mapRows(row => {
+                const value = row[column.index];
+                if (value != null) {
+                    max = max === undefined ? value : Math.max(max, value);
+                }
+            });
+        }
         cacheItem.max = max;
@@ -34,3 +52,25 @@ }
 };
+export const total = (frame, column) => {
+    if (isGroupFrame(frame)) {
+        frame = frame.ungroup();
+    }
+    const cacheItem = getStatsCacheItem(frame, column);
+    if (cacheItem.total === undefined) {
+        // https://github.com/contiamo/operational-visualizations/issues/72
+        let total = undefined;
+        frame.mapRows(row => {
+            const value = row[column.index];
+            if (value) {
+                total = total === undefined ? parseFloat(value) : total + parseFloat(value);
+            }
+        });
+        cacheItem.total = total;
+    }
+    return cacheItem.total;
+};
+// rename to unique
 export const uniqueValues = (frame, column) => {
+    if (isGroupFrame(frame)) {
+        frame = frame.ungroup();
+    }
     const cacheItem = getStatsCacheItem(frame, column);
@@ -40,3 +80,5 @@ if (cacheItem.unique === undefined) {
         frame.mapRows(row => {
-            unique.add(row[column.index]);
+            if (row[column.index] != null) {
+                unique.add(row[column.index]);
+            }
         });
@@ -47,13 +89,2 @@ cacheItem.unique = [...unique];
 };
-export const uniqueValueCombinations = (frame, columns) => {
-    const columnValues = columns.map(c => uniqueValues(frame, c));
-    const combineValues = (i, values) => {
-        return i === columns.length - 1
-            ? columnValues[i].map(val => [...values, val])
-            : columnValues[i].reduce((arr, val) => {
-                return [...arr, ...combineValues(i + 1, [...values, val])];
-            }, []);
-    };
-    return combineValues(0, []);
-};
 //# sourceMappingURL=stats.js.map

lib/types.d.ts

@@ -0,1 +1,2 @@
+import { GroupFrame } from "./GroupFrame";
 /**
@@ -10,14 +11,32 @@ * Can represent array of arrays or list of tuples or tuple of lists
 /**
- * Tuple representing raw row in the frame.
+ * Initially it was tuple representing raw row in the frame and the implementation is still the same.
  *
- * At the moment it is array of any, but we can try to derive more precise type from the Schema.
+ * But conceptually, it's supposed to be `RowCursor`. So you take `RowCursor` and `ColumnCursor`
+ * and can get value at their crossing (like x and y cooordinate).
  *
- * We expose implementation details with the fact that row is an array `any[]`.
- * Instead we can use RowCursor, this way we would be able to change implementation to column oriented storage
- * without changing external code.
+ * The fact that this is an array exposes implementation detail - DataFrame is row oriented storage.
+ *
+ * If we will keep exposing this implementation detail and rely on it a lot
+ * at some point it would be hard to change implementation without breaking existing code.
  */
-export declare type RawRow = any[];
+export declare type RowCursor = ReadonlyArray<any>;
+/**
+ * Cursor is a function which gets rowCursor from the IterableFrame.map
+ * and returns value from the corresponding column in the given row.
+ *
+ * As well it contains `column` name and index, so you can do `rowCursor[column.index]`,
+ * I guess it is more performant, but probably we will regret about it if we will change
+ * implementation of `RowCursor`.
+ *
+ * `rowCursor[column.index]` exposes implementation details.
+ */
+export interface ColumnCursor<Name extends string, ValueInRawRow = any> {
+    (rowCursor: RowCursor): ValueInRawRow;
+    column: Name;
+    index: number;
+}
 export interface WithCursor<Name extends string> {
     getCursor(column: Name): ColumnCursor<Name>;
 }
+export declare type DimensionValue = string;
 export interface IterableFrame<Name extends string> extends WithCursor<Name> {
@@ -27,9 +46,9 @@ /** needed for stats module */
     /** needed for visualisations */
-    mapRows<Result>(callback: (row: RawRow, index: number) => Result): Result[];
+    mapRows<Result>(callback: (rowCursor: RowCursor, rowIndex: number) => Result): Result[];
     /** needed for visualizations */
-    groupBy(columns: Array<string | ColumnCursor<string>>): Array<IterableFrame<Name>>;
+    groupBy(columns: Array<Name | ColumnCursor<Name>>): GroupFrame<Name>;
     /** needed for visualizations */
-    uniqueValues(columns: Array<Name | ColumnCursor<Name>>): string[][];
+    uniqueValues(columns: Array<Name | ColumnCursor<Name>>): DimensionValue[][];
     /** needed for visualizations */
-    row(rowIndex: number): RawRow;
+    row(rowIndex: number): RowCursor;
 }
@@ -40,15 +59,3 @@ export interface PivotProps<Column extends string, Row extends string> {
 }
-/**
- * Cursor is a function which gets row from the IterableFrame.map
- * and returns value from the corresponding column in the given row.
- *
- * As well it contains `column` name and index, so you can do `row[accessor.index]`,
- * I guess it is more performant, but probably we will regret about it if we will change
- * implementation of `RawRow`.
- */
-export interface ColumnCursor<Name extends string, ValueInRawRow = any> {
-    (row: RawRow): ValueInRawRow;
-    column: Name;
-    index: number;
-}
+export declare type GroupProps<Name extends string> = ReadonlyArray<Name | ColumnCursor<Name>>;
 //# sourceMappingURL=types.d.ts.map

lib/utils.d.ts

 import { ColumnCursor } from "./types";
 export declare const isCursor: (x: any) => x is ColumnCursor<any, any>;
+export declare const hashCursors: <Name extends string = string>(columns: ColumnCursor<Name, any>[]) => string;
 //# sourceMappingURL=utils.d.ts.map

lib/utils.js

 export const isCursor = (x) => (!!x.index || x.index === 0) && x instanceof Function;
+export const hashCursors = (columns) => `${columns.map(column => column.index).join(",")}`;
 //# sourceMappingURL=utils.js.map

package.json

 {
   "name": "@operational/frame",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "Contiamo DataFrame.",
@@ -22,3 +22,6 @@ "main": "./lib/index.js",
     "prepare": "tsc"
+  },
+  "dependencies": {
+    "flru": "^1.0.2"
   }
 }

No alert changes

Improved metrics

Total package byte prevSize

62939

increased by28.42%

Number of package files

46

increased by21.05%

Lines of code

729

increased by34.25%

Worsened metrics

Dependency count

1

increased byInfinity%

Dependency changes

• + Addedflru@^1.0.2

• + Addedflru@1.0.2(transitive)