@apache-arrow/ts - npm Package Compare versions

Comparing version 7.0.0 to 8.0.0

Arrow.dom.ts

@@ -37,3 +37,3 @@ // Licensed to the Apache Software Foundation (ASF) under one
 export type {
-    TypeMap,
+    TypeMap, StructRowProxy,
     ReadableSource, WritableSink,
@@ -68,3 +68,3 @@ ArrowJSONLike, FileHandle, Readable, Writable, ReadableWritable, ReadableDOMStreamOptions,
     Visitor,
-    Vector, makeVector, vectorFromArray,
+    Vector, makeVector, vectorFromArray, tableFromJSON,
     ByteStream, AsyncByteStream, AsyncByteQueue,
@@ -71,0 +71,0 @@ RecordBatchReader, RecordBatchFileReader, RecordBatchStreamReader, AsyncRecordBatchFileReader, AsyncRecordBatchStreamReader,

Arrow.ts

@@ -62,5 +62,6 @@ // Licensed to the Apache Software Foundation (ASF) under one
 export { StructRow } from './row/struct.js';
+export type { StructRowProxy } from './row/struct.js';
 
 export { Builder } from './builder.js';
-export { makeBuilder, vectorFromArray, builderThroughIterable, builderThroughAsyncIterable } from './factories.js';
+export { makeBuilder, vectorFromArray, tableFromJSON, builderThroughIterable, builderThroughAsyncIterable } from './factories.js';
 export type { BuilderOptions } from './builder.js';
@@ -67,0 +68,0 @@ export { BoolBuilder } from './builder/bool.js';

builder/struct.ts

@@ -27,9 +27,18 @@ // Licensed to the Apache Software Foundation (ASF) under one
     public setValue(index: number, value: Struct<T>['TValue']) {
-        const children = this.children;
+        const { children, type } = this;
         switch (Array.isArray(value) || value.constructor) {
-            case true: return this.type.children.forEach((_, i) => children[i].set(index, value[i]));
-            case Map: return this.type.children.forEach((f, i) => children[i].set(index, value.get(f.name)));
-            default: return this.type.children.forEach((f, i) => children[i].set(index, value[f.name]));
+            case true: return type.children.forEach((_, i) => children[i].set(index, value[i]));
+            case Map: return type.children.forEach((f, i) => children[i].set(index, value.get(f.name)));
+            default: return type.children.forEach((f, i) => children[i].set(index, value[f.name]));
         }
     }
+
+    /** @inheritdoc */
+    public setValid(index: number, valid: boolean) {
+        if (!super.setValid(index, valid)) {
+            this.children.forEach((child) => child.setValid(index, valid));
+        }
+        return valid;
+    }
+
     public addChild(child: Builder, name = `${this.numChildren}`) {
@@ -36,0 +45,0 @@ const childIndex = this.children.push(child);

factories.ts

@@ -18,6 +18,6 @@ // Licensed to the Apache Software Foundation (ASF) under one
 
-import { Field } from './schema.js';
+import { Field, Schema } from './schema.js';
 import * as dtypes from './type.js';
 import { Data, DataProps } from './data.js';
-import { BuilderType } from './interfaces.js';
+import { BuilderType, JavaScriptDataType } from './interfaces.js';
 import { Vector, makeVector } from './vector.js';
@@ -27,2 +27,5 @@ import { Builder, BuilderOptions } from './builder.js';
 import { ArrayDataType, BigIntArray, JavaScriptArrayDataType, TypedArray, TypedArrayDataType } from './interfaces.js';
+import { Table } from './table.js';
+import { RecordBatch } from './recordbatch.js';
+import { compareTypes } from './visitor/typecomparator.js';
 
@@ -61,2 +64,3 @@ export function makeBuilder<T extends dtypes.DataType = any, TNull = any>(options: BuilderOptions<T, TNull>): BuilderType<T, TNull> {
  * const vdict = vectorFromArray(['foo', 'bar']);
+ * const vstruct = vectorFromArray([{a: 'foo', b: 42}, {a: 'bar', b: 12}]);
  * ```
@@ -94,9 +98,19 @@ */
 
+/**
+ * Creates a {@link Table} from an array of objects.
+ *
+ * @param array A table of objects.
+ */
+export function tableFromJSON<T extends Record<string, unknown>>(array: T[]): Table<{ [P in keyof T]: JavaScriptDataType<T[P]> }> {
+    const vector = vectorFromArray(array) as Vector<dtypes.Struct<any>>;
+    const batch = new RecordBatch(new Schema(vector.type.children), vector.data[0]);
+    return new Table(batch);
+}
+
 /** @ignore */
+function inferType<T extends readonly unknown[]>(values: T): JavaScriptArrayDataType<T>;
 function inferType(value: readonly unknown[]): dtypes.DataType {
     if (value.length === 0) { return new dtypes.Null; }
     let nullsCount = 0;
-    // @ts-ignore
     let arraysCount = 0;
-    // @ts-ignore
     let objectsCount = 0;
@@ -139,4 +153,20 @@ let numbersCount = 0;
         return new dtypes.DateMillisecond;
+    } else if (arraysCount + nullsCount === value.length) {
+        const array = value as Array<unknown>[];
+        const childType = inferType(array[array.findIndex((ary) => ary != null)]);
+        if (array.every((ary) => ary == null || compareTypes(childType, inferType(ary)))) {
+            return new dtypes.List(new Field('', childType, true));
+        }
+    } else if (objectsCount + nullsCount === value.length) {
+        const fields = new Map<string, Field>();
+        for (const row of value as Record<string, unknown>[]) {
+            for (const key of Object.keys(row)) {
+                if (!fields.has(key) && row[key] != null) {
+                    // use the type inferred for the first instance of a found key
+                    fields.set(key, new Field(key, inferType([row[key]]), true));
+                }
+            }
+        }
+        return new dtypes.Struct([...fields.values()]);
     }
-    // TODO: add more types to infererence
 
@@ -151,3 +181,2 @@ throw new TypeError('Unable to infer Vector type from input values, explicit type declaration expected');
  */
-
 export interface IterableBuilderOptions<T extends dtypes.DataType = any, TNull = any> extends BuilderOptions<T, TNull> {
@@ -154,0 +183,0 @@ highWaterMark?: number;

interfaces.ts

@@ -164,2 +164,5 @@ // Licensed to the Apache Software Foundation (ASF) under one
 /** @ignore */
+export type JavaScriptDataType<T> = JavaScriptArrayDataType<T[]>;
+
+/** @ignore */
 export type JavaScriptArrayDataType<T extends readonly unknown[]> =
@@ -172,2 +175,4 @@ T extends readonly (null | undefined)[] ? type.Null :
     T extends readonly (null | undefined | number)[] ? type.Float64 :
+    T extends readonly (null | undefined | readonly (infer U)[])[] ? type.List<JavaScriptDataType<U>> :
+    T extends readonly (null | undefined | Record<string, unknown>)[] ? T extends readonly (null | undefined | infer U)[] ? type.Struct<{ [P in keyof U]: JavaScriptDataType<U[P]> }> : never :
     never;
@@ -174,0 +179,0 @@

ipc/metadata/message.ts

@@ -300,2 +300,5 @@ // Licensed to the Apache Software Foundation (ASF) under one
 function decodeRecordBatch(batch: _RecordBatch, version = MetadataVersion.V4) {
+    if (batch.compression() !== null) {
+        throw new Error('Record batch compression not implemented');
+    }
     return new RecordBatch(batch.length(), decodeFieldNodes(batch), decodeBuffers(batch, version));
@@ -302,0 +305,0 @@ }

package.json

 {
-  "version": "7.0.0",
+  "version": "8.0.0",
   "name": "@apache-arrow/ts",
@@ -31,8 +31,8 @@ "browser": "Arrow.dom.ts",
     "@types/flatbuffers": "*",
-    "@types/node": "^17.0.8",
+    "@types/node": "^17.0.24",
     "@types/command-line-args": "5.2.0",
     "@types/command-line-usage": "5.0.2",
     "@types/pad-left": "2.1.1",
-    "command-line-args": "5.2.0",
-    "command-line-usage": "6.1.1",
+    "command-line-args": "5.2.1",
+    "command-line-usage": "6.1.3",
     "flatbuffers": "2.0.4",
@@ -39,0 +39,0 @@ "json-bignum": "^0.0.3",

recordbatch.ts

@@ -165,3 +165,3 @@ // Licensed to the Apache Software Foundation (ASF) under one
     public [Symbol.iterator]() {
-        return iteratorVisitor.visit(new Vector([this.data]));
+        return iteratorVisitor.visit(new Vector([this.data])) as IterableIterator<Struct<T>['TValue']>;
     }
@@ -168,0 +168,0 @@

row/struct.ts

@@ -30,2 +30,4 @@ // Licensed to the Apache Software Foundation (ASF) under one
     [P in keyof T]: T[P]['TValue'];
+} & {
+    [key: symbol]: any;
 };
@@ -153,3 +155,3 @@
             return Reflect.set(row, key, val);
-        } else if (Reflect.has(row, key)) {
+        } else if (Reflect.has(row, key) || typeof key === 'symbol') {
             return Reflect.set(row, key, val);
@@ -156,0 +158,0 @@ }

table.ts

@@ -227,3 +227,3 @@ // Licensed to the Apache Software Foundation (ASF) under one
     public [Symbol.iterator]() {
-        return iteratorVisitor.visit(new Vector(this.data));
+        return iteratorVisitor.visit(new Vector(this.data)) as IterableIterator<Struct<T>['TValue']>;
     }
@@ -243,3 +243,3 @@
      *
-     * @returns  A string representation of the Table rows.
+     * @returns A string representation of the Table rows.
      */
@@ -264,3 +264,3 @@ public toString() {
      *
-     * @param start The beginning of the specified portion of the Table.
+     * @param begin The beginning of the specified portion of the Table.
      * @param end The end of the specified portion of the Table. This is exclusive of the element at the index 'end'.
@@ -433,3 +433,3 @@ */
  *
- * @param Input an object of typed arrays or JavaScript arrays.
+ * @param input Input an object of typed arrays or JavaScript arrays.
  * @returns A new Table.
@@ -436,0 +436,0 @@ */

util/bn.ts

@@ -75,11 +75,15 @@ // Licensed to the Apache Software Foundation (ASF) under one
     const { buffer, byteOffset, length, 'signed': signed } = bn;
-    const words = new Int32Array(buffer, byteOffset, length);
-    let number = 0, i = 0;
-    const n = words.length;
-    let hi, lo;
-    while (i < n) {
-        lo = words[i++];
-        hi = words[i++];
-        signed || (hi = hi >>> 0);
-        number += (lo >>> 0) + (hi * (i ** 32));
+    const words = new BigUint64Array(buffer, byteOffset, length);
+    const negative = signed && words[words.length - 1] & (BigInt(1) << BigInt(63));
+    let number = negative ? BigInt(1) : BigInt(0);
+    let i = BigInt(0);
+    if (!negative) {
+        for (const word of words) {
+            number += word * (BigInt(1) << (BigInt(32) * i++));
+        }
+    } else {
+        for (const word of words) {
+            number += ~word * (BigInt(1) << (BigInt(32) * i++));
+        }
+        number *= BigInt(-1);
     }
@@ -86,0 +90,0 @@ return number;

vector.ts

@@ -40,2 +40,5 @@ // Licensed to the Apache Software Foundation (ASF) under one
 
+// @ts-ignore
+import type { vectorFromArray } from './factories.js';
+
 export interface Vector<T extends DataType = any> {
@@ -42,0 +45,0 @@ ///

visitor/bytelength.ts

@@ -51,15 +51,42 @@ /* istanbul ignore file */
 export class GetByteLengthVisitor extends Visitor {
-    public visitNull(____: Data<Null>, _: number) { return 0; }
-    public visitInt(data: Data<Int>, _: number) { return data.type.bitWidth / 8; }
-    public visitFloat(data: Data<Float>, _: number) { return data.type.ArrayType.BYTES_PER_ELEMENT; }
-    public visitBool(____: Data<Bool>, _: number) { return 1 / 8; }
-    public visitDecimal(____: Data<Decimal>, _: number) { return 16; }
-    public visitDate(data: Data<Date_>, _: number) { return (data.type.unit + 1) * 4; }
-    public visitTime(data: Data<Time>, _: number) { return data.type.bitWidth / 8; }
-    public visitTimestamp(data: Data<Timestamp>, _: number) { return data.type.unit === TimeUnit.SECOND ? 4 : 8; }
-    public visitInterval(data: Data<Interval>, _: number) { return (data.type.unit + 1) * 4; }
-    public visitStruct(data: Data<Struct>, _: number) { return this.visitMany(data.children, data.children.map(() => _)).reduce(sum, 0); }
-    public visitFixedSizeBinary(data: Data<FixedSizeBinary>, _: number) { return data.type.byteWidth; }
-    public visitMap(data: Data<Map_>, _: number) { return this.visitMany(data.children, data.children.map(() => _)).reduce(sum, 0); }
-    public visitDictionary(data: Data<Dictionary>, _: number) { return (data.type.indices.bitWidth / 8) + (data.dictionary?.getByteLength(data.values[_]) || 0); }
+    public visitNull(____: Data<Null>, _: number) {
+        return 0;
+    }
+    public visitInt(data: Data<Int>, _: number) {
+        return data.type.bitWidth / 8;
+    }
+    public visitFloat(data: Data<Float>, _: number) {
+        return data.type.ArrayType.BYTES_PER_ELEMENT;
+    }
+    public visitBool(____: Data<Bool>, _: number) {
+        return 1 / 8;
+    }
+    public visitDecimal(data: Data<Decimal>, _: number) {
+        return data.type.bitWidth / 8;
+    }
+    public visitDate(data: Data<Date_>, _: number) {
+        return (data.type.unit + 1) * 4;
+    }
+    public visitTime(data: Data<Time>, _: number) {
+        return data.type.bitWidth / 8;
+    }
+    public visitTimestamp(data: Data<Timestamp>, _: number) {
+        return data.type.unit === TimeUnit.SECOND ? 4 : 8;
+    }
+    public visitInterval(data: Data<Interval>, _: number) {
+        return (data.type.unit + 1) * 4;
+    }
+    public visitStruct(data: Data<Struct>, i: number) {
+        return data.children.reduce((total, child) => total + instance.visit(child, i), 0);
+    }
+    public visitFixedSizeBinary(data: Data<FixedSizeBinary>, _: number) {
+        return data.type.byteWidth;
+    }
+    public visitMap(data: Data<Map_>, i: number) {
+        // 4 + 4 for the indices
+        return 8 + data.children.reduce((total, child) => total + instance.visit(child, i), 0);
+    }
+    public visitDictionary(data: Data<Dictionary>, i: number) {
+        return (data.type.indices.bitWidth / 8) + (data.dictionary?.getByteLength(data.values[i]) || 0);
+    }
 }
@@ -66,0 +93,0 @@

visitor/get.ts

@@ -23,3 +23,3 @@ // Licensed to the Apache Software Foundation (ASF) under one
 import { MapRow } from '../row/map.js';
-import { StructRow } from '../row/struct.js';
+import { StructRow, StructRowProxy } from '../row/struct.js';
 import { decodeUtf8 } from '../util/utf8.js';
@@ -113,4 +113,8 @@ import { TypeToDataType } from '../interfaces.js';
 const getVariableWidthBytes = (values: Uint8Array, valueOffsets: Int32Array, index: number) => {
-    const { [index]: x, [index + 1]: y } = valueOffsets;
-    return x != null && y != null ? values.subarray(x, y) : null as any;
+    if (index + 1 >= valueOffsets.length) {
+        return null as any;
+    }
+    const x = valueOffsets[index];
+    const y = valueOffsets[index + 1];
+    return values.subarray(x, y);
 };
@@ -212,6 +216,5 @@
 const getList = <T extends List>(data: Data<T>, index: number): T['TValue'] => {
-    const { valueOffsets, stride } = data;
-    const { [index * stride]: begin } = valueOffsets;
-    const { [index * stride + 1]: end } = valueOffsets;
-    const child: Data<T['valueType']> = data.children[0];
+    const { valueOffsets, stride, children } = data;
+    const { [index * stride]: begin, [index * stride + 1]: end } = valueOffsets;
+    const child: Data<T['valueType']> = children[0];
     const slice = child.slice(begin, end - begin);
@@ -223,6 +226,5 @@ return new Vector([slice]) as T['TValue'];
 const getMap = <T extends Map_>(data: Data<T>, index: number): T['TValue'] => {
-    const { valueOffsets } = data;
-    const { [index]: begin } = valueOffsets;
-    const { [index + 1]: end } = valueOffsets;
-    const child = data.children[0] as Data<T['childType']>;
+    const { valueOffsets, children } = data;
+    const { [index]: begin, [index + 1]: end } = valueOffsets;
+    const child = children[0] as Data<T['childType']>;
     return new MapRow(child.slice(begin, end - begin));
@@ -233,3 +235,3 @@ };
 const getStruct = <T extends Struct>(data: Data<T>, index: number): T['TValue'] => {
-    return new StructRow(data, index);
+    return new StructRow(data, index) as StructRowProxy<T['TValue']>;
 };
@@ -287,4 +289,4 @@
 const getFixedSizeList = <T extends FixedSizeList>(data: Data<T>, index: number): T['TValue'] => {
-    const { stride } = data;
-    const child: Data<T['valueType']> = data.children[0];
+    const { stride, children } = data;
+    const child: Data<T['valueType']> = children[0];
     const slice = child.slice(index * stride, stride);
@@ -291,0 +293,0 @@ return new Vector([slice]);

visitor/set.ts

@@ -121,4 +121,4 @@ // Licensed to the Apache Software Foundation (ASF) under one
 export const setVariableWidthBytes = (values: Uint8Array, valueOffsets: Int32Array, index: number, value: Uint8Array) => {
-    const { [index]: x, [index + 1]: y } = valueOffsets;
-    if (x != null && y != null) {
+    if (index + 1 < valueOffsets.length) {
+        const { [index]: x, [index + 1]: y } = valueOffsets;
         values.set(value.subarray(0, y - x), x);
@@ -149,3 +149,2 @@ }
         case Precision.DOUBLE:
-        default:
             return setFloat(data as Data<Float32 | Float64>, index, value);
@@ -235,3 +234,3 @@ }
     const values = data.children[0];
-    const valueOffsets = data.valueOffsets;
+    const { valueOffsets } = data;
     const set = instance.getVisitFn(values);
@@ -238,0 +237,0 @@ let { [index]: idx, [index + 1]: end } = valueOffsets;

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

880334

increased by0.42%

Lines of code

16162

increased by0.5%

Number of medium supply chain risk alerts

0

decreased by-100%

Dependency changes

• + Addedcommand-line-args@5.2.1(transitive)

• + Addedcommand-line-usage@6.1.3(transitive)

• - Removedcommand-line-args@5.2.0(transitive)

• - Removedcommand-line-usage@6.1.1(transitive)

• Updated@types/node@^17.0.24

• Updatedcommand-line-args@5.2.1

• Updatedcommand-line-usage@6.1.3