Socket
Socket
Sign inDemoInstall

@apache-arrow/ts

Package Overview
Dependencies
Maintainers
5
Versions
37
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@apache-arrow/ts - npm Package Compare versions

Comparing version 0.14.1 to 0.15.0

util/math.ts

3

Arrow.ts

@@ -47,3 +47,2 @@ // Licensed to the Apache Software Foundation (ASF) under one

export {
Row,
Vector,

@@ -106,2 +105,3 @@ BaseVector,

import * as util_bit_ from './util/bit';
import * as util_math_ from './util/math';
import * as util_buffer_ from './util/buffer';

@@ -117,4 +117,5 @@ import * as util_vector_ from './util/vector';

...util_bit_,
...util_math_,
...util_buffer_,
...util_vector_
};

2

bin/arrow2csv.ts

@@ -180,3 +180,3 @@ #! /usr/bin/env node

}
this.push(`${formatRow([rowId, ...row].map(valueToString), maxColWidths, sep)}\n`);
this.push(`${formatRow([rowId, ...row.toArray()].map(valueToString), maxColWidths, sep)}\n`);
}

@@ -183,0 +183,0 @@ }

4

builder.ts

@@ -28,3 +28,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

Date_, Time, Timestamp, Interval,
Utf8, Binary, List,
Utf8, Binary, List, Map_
} from './type';

@@ -445,3 +445,3 @@

/** @ignore */
export abstract class VariableWidthBuilder<T extends Binary | Utf8 | List, TNull = any> extends Builder<T, TNull> {
export abstract class VariableWidthBuilder<T extends Binary | Utf8 | List | Map_, TNull = any> extends Builder<T, TNull> {
protected _pendingLength: number = 0;

@@ -448,0 +448,0 @@ protected _offsets: OffsetsBufferBuilder;

8

builder/float.ts

@@ -18,2 +18,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { float64ToUint16 } from '../util/math';
import { FixedWidthBuilder } from '../builder';

@@ -26,3 +27,8 @@ import { Float, Float16, Float32, Float64 } from '../type';

/** @ignore */
export class Float16Builder<TNull = any> extends FloatBuilder<Float16, TNull> {}
export class Float16Builder<TNull = any> extends FloatBuilder<Float16, TNull> {
public setValue(index: number, value: number) {
// convert JS float64 to a uint16
this._values.set(index, float64ToUint16(value));
}
}

@@ -29,0 +35,0 @@ /** @ignore */

50

builder/map.ts

@@ -19,13 +19,47 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Field } from '../schema';
import { Builder } from '../builder';
import { DataType, Map_ } from '../type';
import { DataType, Map_, Struct } from '../type';
import { Builder, VariableWidthBuilder } from '../builder';
/** @ignore */ type MapValue<K extends DataType = any, V extends DataType = any> = Map_<K, V>['TValue'];
/** @ignore */ type MapValues<K extends DataType = any, V extends DataType = any> = Map<number, MapValue<K, V> | undefined>;
/** @ignore */ type MapValueExt<K extends DataType = any, V extends DataType = any> = MapValue<K, V> | { [key: string]: V } | { [key: number]: V } ;
/** @ignore */
export class MapBuilder<T extends { [key: string]: DataType } = any, TNull = any> extends Builder<Map_<T>, TNull> {
public addChild(child: Builder, name = `${this.numChildren}`) {
const { children, keysSorted } = this.type;
const childIndex = this.children.push(child);
this.type = new Map_([...children, new Field(name, child.type, true)], keysSorted);
return childIndex;
export class MapBuilder<K extends DataType = any, V extends DataType = any, TNull = any> extends VariableWidthBuilder<Map_<K, V>, TNull> {
protected _pending: MapValues<K, V> | undefined;
public set(index: number, value: MapValueExt<K, V> | TNull) {
return super.set(index, value as MapValue<K, V> | TNull);
}
public setValue(index: number, value: MapValueExt<K, V>) {
value = value instanceof Map ? value : new Map(Object.entries(value));
const pending = this._pending || (this._pending = new Map() as MapValues<K, V>);
const current = pending.get(index);
current && (this._pendingLength -= current.size);
this._pendingLength += value.size;
pending.set(index, value);
}
public addChild(child: Builder<Struct<{ key: K, value: V }>>, name = `${this.numChildren}`) {
if (this.numChildren > 0) {
throw new Error('ListBuilder can only have one child.');
}
this.children[this.numChildren] = child;
this.type = new Map_<K, V>(new Field(name, child.type, true), this.type.keysSorted);
return this.numChildren - 1;
}
protected _flushPending(pending: MapValues<K, V>) {
const offsets = this._offsets;
const setValue = this._setValue;
pending.forEach((value, index) => {
if (value === undefined) {
offsets.set(index, 0);
} else {
offsets.set(index, value.size);
setValue(this, index, value);
}
});
}
}

4

builder/null.ts

@@ -25,2 +25,6 @@ // Licensed to the Apache Software Foundation (ASF) under one

public setValue(index: number, value: null) {}
public setValid(index: number, valid: boolean) {
this.length = Math.max(index + 1, this.length);
return valid;
}
}

30

compute/dataframe.ts

@@ -34,2 +34,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

Table.prototype.scan = function(this: Table, next: NextFunc, bind?: BindFunc) { return new DataFrame(this.chunks).scan(next, bind); };
Table.prototype.scanReverse = function(this: Table, next: NextFunc, bind?: BindFunc) { return new DataFrame(this.chunks).scanReverse(next, bind); };
Table.prototype.filter = function(this: Table, predicate: Predicate): FilteredDataFrame { return new DataFrame(this.chunks).filter(predicate); };

@@ -53,2 +54,14 @@

}
public scanReverse(next: NextFunc, bind?: BindFunc) {
const batches = this.chunks, numBatches = batches.length;
for (let batchIndex = numBatches; --batchIndex >= 0;) {
// load batches
const batch = batches[batchIndex];
if (bind) { bind(batch); }
// yield all indices
for (let index = batch.length; --index >= 0;) {
next(index, batch);
}
}
}
public countBy(name: Col | string) {

@@ -135,2 +148,19 @@ const batches = this.chunks, numBatches = batches.length;

}
public scanReverse(next: NextFunc, bind?: BindFunc) {
const batches = this._chunks;
const numBatches = batches.length;
for (let batchIndex = numBatches; --batchIndex >= 0;) {
// load batches
const batch = batches[batchIndex];
// TODO: bind batches lazily
// If predicate doesn't match anything in the batch we don't need
// to bind the callback
if (bind) { bind(batch); }
const predicate = this._predicate.bind(batch);
// yield all indices
for (let index = batch.length; --index >= 0;) {
if (predicate(index, batch)) { next(index, batch); }
}
}
}
public count(): number {

@@ -137,0 +167,0 @@ // inlined version of this:

17

data.ts

@@ -150,2 +150,5 @@ // Licensed to the Apache Software Foundation (ASF) under one

public _changeLengthAndBackfillNullBitmap(newLength: number): Data<T> {
if (this.typeId === Type.Null) {
return this.clone(this.type, 0, newLength, 0);
}
const { length, nullCount } = this;

@@ -187,3 +190,3 @@ // start initialized with 0s (nulls), then fill from 0 to length with 1s (not null)

switch (type.typeId) {
case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length, nullCount || 0, buffers[BufferType.VALIDITY]) as Data<T>;
case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length) as Data<T>;
case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;

@@ -204,3 +207,3 @@ case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary!) as Data<T>;

case Type.Struct: return <unknown> Data.Struct( <unknown> type as Struct, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], childData || []) as Data<T>;
case Type.Map: return <unknown> Data.Map( <unknown> type as Map_, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], childData || []) as Data<T>;
case Type.Map: return <unknown> Data.Map( <unknown> type as Map_, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.OFFSET] || [], (childData || [])[0]) as Data<T>;
case Type.Union: return <unknown> Data.Union( <unknown> type as Union, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.TYPE] || [], buffers[BufferType.OFFSET] || childData, childData) as Data<T>;

@@ -212,4 +215,4 @@ }

/** @nocollapse */
public static Null<T extends Null>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, _data?: NullBuffer) {
return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)]);
public static Null<T extends Null>(type: T, offset: number, length: number) {
return new Data(type, offset, length, 0);
}

@@ -269,3 +272,3 @@ /** @nocollapse */

/** @nocollapse */
public static FixedSizeList<T extends FixedSizeList>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, child: Data | Vector) {
public static FixedSizeList<T extends FixedSizeList>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, child: Data<T['valueType']> | Vector<T['valueType']>) {
return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)], [child]);

@@ -278,4 +281,4 @@ }

/** @nocollapse */
public static Map<T extends Map_>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, children: (Data | Vector)[]) {
return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)], children);
public static Map<T extends Map_>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, child: (Data | Vector)) {
return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), undefined, toUint8Array(nullBitmap)], [child]);
}

@@ -282,0 +285,0 @@ public static Union<T extends SparseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, children: (Data | Vector)[], _?: any): Data<T>;

17

fb/File.ts

@@ -38,11 +38,2 @@ // automatically generated by the FlatBuffers compiler, do not modify

/**
* @param flatbuffers.ByteBuffer bb
* @param Footer= obj
* @returns Footer
*/
static getSizePrefixedRootAsFooter(bb: flatbuffers.ByteBuffer, obj?: Footer): Footer {
return (obj || new Footer).__init(bb.readInt32(bb.position()) + bb.position(), bb);
}
/**
* @returns org.apache.arrow.flatbuf.MetadataVersion

@@ -172,10 +163,2 @@ */

/**
* @param flatbuffers.Builder builder
* @param flatbuffers.Offset offset
*/
static finishSizePrefixedFooterBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) {
builder.finish(offset, undefined);
}
static createFooter(builder: flatbuffers.Builder, version: NS7624605610262437867.org.apache.arrow.flatbuf.MetadataVersion, schemaOffset: flatbuffers.Offset, dictionariesOffset: flatbuffers.Offset, recordBatchesOffset: flatbuffers.Offset): flatbuffers.Offset {

@@ -182,0 +165,0 @@ Footer.startFooter(builder);

35

fb/Message.ts

@@ -129,11 +129,2 @@ // automatically generated by the FlatBuffers compiler, do not modify

/**
* @param flatbuffers.ByteBuffer bb
* @param RecordBatch= obj
* @returns RecordBatch
*/
static getSizePrefixedRootAsRecordBatch(bb: flatbuffers.ByteBuffer, obj?: RecordBatch): RecordBatch {
return (obj || new RecordBatch).__init(bb.readInt32(bb.position()) + bb.position(), bb);
}
/**
* number of records / rows. The arrays in the batch should all have this

@@ -295,11 +286,2 @@ * length

/**
* @param flatbuffers.ByteBuffer bb
* @param DictionaryBatch= obj
* @returns DictionaryBatch
*/
static getSizePrefixedRootAsDictionaryBatch(bb: flatbuffers.ByteBuffer, obj?: DictionaryBatch): DictionaryBatch {
return (obj || new DictionaryBatch).__init(bb.readInt32(bb.position()) + bb.position(), bb);
}
/**
* @returns flatbuffers.Long

@@ -410,11 +392,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Message= obj
* @returns Message
*/
static getSizePrefixedRootAsMessage(bb: flatbuffers.ByteBuffer, obj?: Message): Message {
return (obj || new Message).__init(bb.readInt32(bb.position()) + bb.position(), bb);
}
/**
* @returns org.apache.arrow.flatbuf.MetadataVersion

@@ -555,10 +528,2 @@ */

/**
* @param flatbuffers.Builder builder
* @param flatbuffers.Offset offset
*/
static finishSizePrefixedMessageBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) {
builder.finish(offset, undefined);
}
static createMessage(builder: flatbuffers.Builder, version: NS7624605610262437867.org.apache.arrow.flatbuf.MetadataVersion, headerType: org.apache.arrow.flatbuf.MessageHeader, headerOffset: flatbuffers.Offset, bodyLength: flatbuffers.Long, customMetadataOffset: flatbuffers.Offset): flatbuffers.Offset {

@@ -565,0 +530,0 @@ Message.startMessage(builder);

92

interfaces.ts

@@ -254,3 +254,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

[Type.IntervalYearMonth ]: T extends type.IntervalYearMonth ? vecs.IntervalYearMonthVector : vecs.BaseVector<T> ;
[Type.Map ]: T extends type.Map_ ? vecs.MapVector<T['dataTypes']> : vecs.BaseVector<T> ;
[Type.Map ]: T extends type.Map_ ? vecs.MapVector<T['keyType'], T['valueType']> : vecs.BaseVector<T> ;
[Type.List ]: T extends type.List ? vecs.ListVector<T['valueType']> : vecs.BaseVector<T> ;

@@ -351,3 +351,3 @@ [Type.Struct ]: T extends type.Struct ? vecs.StructVector<T['dataTypes']> : vecs.BaseVector<T> ;

[Type.IntervalYearMonth ]: builders.IntervalYearMonthBuilder<TNull> ;
[Type.Map ]: builders.MapBuilder<any, TNull> ;
[Type.Map ]: builders.MapBuilder<any, any, TNull> ;
[Type.List ]: builders.ListBuilder<any, TNull> ;

@@ -361,46 +361,46 @@ [Type.Struct ]: builders.StructBuilder<any, TNull> ;

type DataTypeToBuilder<T extends DataType = any, TNull = any> = {
[key: number ]: builders.Builder<any, TNull> ;
[Type.Null ]: T extends type.Null ? builders.NullBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Bool ]: T extends type.Bool ? builders.BoolBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Int8 ]: T extends type.Int8 ? builders.Int8Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Int16 ]: T extends type.Int16 ? builders.Int16Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Int32 ]: T extends type.Int32 ? builders.Int32Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Int64 ]: T extends type.Int64 ? builders.Int64Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Uint8 ]: T extends type.Uint8 ? builders.Uint8Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Uint16 ]: T extends type.Uint16 ? builders.Uint16Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Uint32 ]: T extends type.Uint32 ? builders.Uint32Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Uint64 ]: T extends type.Uint64 ? builders.Uint64Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Int ]: T extends type.Int ? builders.IntBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.Float16 ]: T extends type.Float16 ? builders.Float16Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Float32 ]: T extends type.Float32 ? builders.Float32Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Float64 ]: T extends type.Float64 ? builders.Float64Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Float ]: T extends type.Float ? builders.FloatBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.Utf8 ]: T extends type.Utf8 ? builders.Utf8Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Binary ]: T extends type.Binary ? builders.BinaryBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.FixedSizeBinary ]: T extends type.FixedSizeBinary ? builders.FixedSizeBinaryBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Date ]: T extends type.Date_ ? builders.DateBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.DateDay ]: T extends type.DateDay ? builders.DateDayBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.DateMillisecond ]: T extends type.DateMillisecond ? builders.DateMillisecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Timestamp ]: T extends type.Timestamp ? builders.TimestampBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.TimestampSecond ]: T extends type.TimestampSecond ? builders.TimestampSecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimestampMillisecond ]: T extends type.TimestampMillisecond ? builders.TimestampMillisecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimestampMicrosecond ]: T extends type.TimestampMicrosecond ? builders.TimestampMicrosecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimestampNanosecond ]: T extends type.TimestampNanosecond ? builders.TimestampNanosecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Time ]: T extends type.Time ? builders.TimeBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.TimeSecond ]: T extends type.TimeSecond ? builders.TimeSecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimeMillisecond ]: T extends type.TimeMillisecond ? builders.TimeMillisecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimeMicrosecond ]: T extends type.TimeMicrosecond ? builders.TimeMicrosecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimeNanosecond ]: T extends type.TimeNanosecond ? builders.TimeNanosecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Decimal ]: T extends type.Decimal ? builders.DecimalBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Union ]: T extends type.Union ? builders.UnionBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.DenseUnion ]: T extends type.DenseUnion ? builders.DenseUnionBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.SparseUnion ]: T extends type.SparseUnion ? builders.SparseUnionBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.Interval ]: T extends type.Interval ? builders.IntervalBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.IntervalDayTime ]: T extends type.IntervalDayTime ? builders.IntervalDayTimeBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.IntervalYearMonth ]: T extends type.IntervalYearMonth ? builders.IntervalYearMonthBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Map ]: T extends type.Map_ ? builders.MapBuilder<T['dataTypes'], TNull> : builders.Builder<any, TNull> ;
[Type.List ]: T extends type.List ? builders.ListBuilder<T['valueType'], TNull> : builders.Builder<any, TNull> ;
[Type.Struct ]: T extends type.Struct ? builders.StructBuilder<T['dataTypes'], TNull> : builders.Builder<any, TNull> ;
[Type.Dictionary ]: T extends type.Dictionary ? builders.DictionaryBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.FixedSizeList ]: T extends type.FixedSizeList ? builders.FixedSizeListBuilder<T['valueType'], TNull> : builders.Builder<any, TNull> ;
[key: number ]: builders.Builder<any, TNull> ;
[Type.Null ]: T extends type.Null ? builders.NullBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Bool ]: T extends type.Bool ? builders.BoolBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Int8 ]: T extends type.Int8 ? builders.Int8Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Int16 ]: T extends type.Int16 ? builders.Int16Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Int32 ]: T extends type.Int32 ? builders.Int32Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Int64 ]: T extends type.Int64 ? builders.Int64Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Uint8 ]: T extends type.Uint8 ? builders.Uint8Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Uint16 ]: T extends type.Uint16 ? builders.Uint16Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Uint32 ]: T extends type.Uint32 ? builders.Uint32Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Uint64 ]: T extends type.Uint64 ? builders.Uint64Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Int ]: T extends type.Int ? builders.IntBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.Float16 ]: T extends type.Float16 ? builders.Float16Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Float32 ]: T extends type.Float32 ? builders.Float32Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Float64 ]: T extends type.Float64 ? builders.Float64Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Float ]: T extends type.Float ? builders.FloatBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.Utf8 ]: T extends type.Utf8 ? builders.Utf8Builder<TNull> : builders.Builder<any, TNull> ;
[Type.Binary ]: T extends type.Binary ? builders.BinaryBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.FixedSizeBinary ]: T extends type.FixedSizeBinary ? builders.FixedSizeBinaryBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Date ]: T extends type.Date_ ? builders.DateBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.DateDay ]: T extends type.DateDay ? builders.DateDayBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.DateMillisecond ]: T extends type.DateMillisecond ? builders.DateMillisecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Timestamp ]: T extends type.Timestamp ? builders.TimestampBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.TimestampSecond ]: T extends type.TimestampSecond ? builders.TimestampSecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimestampMillisecond ]: T extends type.TimestampMillisecond ? builders.TimestampMillisecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimestampMicrosecond ]: T extends type.TimestampMicrosecond ? builders.TimestampMicrosecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimestampNanosecond ]: T extends type.TimestampNanosecond ? builders.TimestampNanosecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Time ]: T extends type.Time ? builders.TimeBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.TimeSecond ]: T extends type.TimeSecond ? builders.TimeSecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimeMillisecond ]: T extends type.TimeMillisecond ? builders.TimeMillisecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimeMicrosecond ]: T extends type.TimeMicrosecond ? builders.TimeMicrosecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.TimeNanosecond ]: T extends type.TimeNanosecond ? builders.TimeNanosecondBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Decimal ]: T extends type.Decimal ? builders.DecimalBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Union ]: T extends type.Union ? builders.UnionBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.DenseUnion ]: T extends type.DenseUnion ? builders.DenseUnionBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.SparseUnion ]: T extends type.SparseUnion ? builders.SparseUnionBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.Interval ]: T extends type.Interval ? builders.IntervalBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.IntervalDayTime ]: T extends type.IntervalDayTime ? builders.IntervalDayTimeBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.IntervalYearMonth ]: T extends type.IntervalYearMonth ? builders.IntervalYearMonthBuilder<TNull> : builders.Builder<any, TNull> ;
[Type.Map ]: T extends type.Map_ ? builders.MapBuilder<T['keyType'], T['valueType'], TNull> : builders.Builder<any, TNull> ;
[Type.List ]: T extends type.List ? builders.ListBuilder<T['valueType'], TNull> : builders.Builder<any, TNull> ;
[Type.Struct ]: T extends type.Struct ? builders.StructBuilder<T['dataTypes'], TNull> : builders.Builder<any, TNull> ;
[Type.Dictionary ]: T extends type.Dictionary ? builders.DictionaryBuilder<T, TNull> : builders.Builder<any, TNull> ;
[Type.FixedSizeList ]: T extends type.FixedSizeList ? builders.FixedSizeListBuilder<T['valueType'], TNull> : builders.Builder<any, TNull> ;
}[T['TType']];

18

ipc/message.ts

@@ -43,2 +43,7 @@ // Licensed to the Apache Software Foundation (ASF) under one

if ((r = this.readMetadataLength()).done) { return ITERATOR_DONE; }
// ARROW-6313: If the first 4 bytes are continuation indicator (-1), read
// the next 4 for the 32-bit metadata length. Otherwise, assume this is a
// pre-v0.15 message, where the first 4 bytes are the metadata length.
if ((r.value === -1) &&
(r = this.readMetadataLength()).done) { return ITERATOR_DONE; }
if ((r = this.readMetadata(r.value)).done) { return ITERATOR_DONE; }

@@ -80,4 +85,4 @@ return (<any> r) as IteratorResult<Message>;

const bb = buf && new ByteBuffer(buf);
const len = +(bb && bb.readInt32(0))!;
return { done: len <= 0, value: len };
const len = bb && bb.readInt32(0) || 0;
return { done: len === 0, value: len };
}

@@ -109,2 +114,7 @@ protected readMetadata(metadataLength: number): IteratorResult<Message> {

if ((r = await this.readMetadataLength()).done) { return ITERATOR_DONE; }
// ARROW-6313: If the first 4 bytes are continuation indicator (-1), read
// the next 4 for the 32-bit metadata length. Otherwise, assume this is a
// pre-v0.15 message, where the first 4 bytes are the metadata length.
if ((r.value === -1) &&
(r = await this.readMetadataLength()).done) { return ITERATOR_DONE; }
if ((r = await this.readMetadata(r.value)).done) { return ITERATOR_DONE; }

@@ -146,4 +156,4 @@ return (<any> r) as IteratorResult<Message>;

const bb = buf && new ByteBuffer(buf);
const len = +(bb && bb.readInt32(0))!;
return { done: len <= 0, value: len };
const len = bb && bb.readInt32(0) || 0;
return { done: len === 0, value: len };
}

@@ -150,0 +160,0 @@ protected async readMetadata(metadataLength: number): Promise<IteratorResult<Message>> {

2

ipc/metadata/json.ts

@@ -201,3 +201,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

const t = f['type'];
return new Map_(children || [], t['keysSorted']);
return new Map_((children || [])[0], t['keysSorted']);
}

@@ -204,0 +204,0 @@ }

2

ipc/metadata/message.ts

@@ -471,3 +471,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

const t = f.type(new Schema_.org.apache.arrow.flatbuf.Map())!;
return new Map_(children || [], t.keysSorted());
return new Map_((children || [])[0], t.keysSorted());
}

@@ -474,0 +474,0 @@ }

55

ipc/writer.ts

@@ -35,4 +35,18 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Writable, ReadableInterop, ReadableDOMStreamOptions } from '../io/interfaces';
import { isPromise, isAsyncIterable, isWritableDOMStream, isWritableNodeStream, isIterable } from '../util/compat';
import { isPromise, isAsyncIterable, isWritableDOMStream, isWritableNodeStream, isIterable, isObject } from '../util/compat';
export interface RecordBatchStreamWriterOptions {
/**
*
*/
autoDestroy?: boolean;
/**
* A flag indicating whether the RecordBatchWriter should construct pre-0.15.0
* encapsulated IPC Messages, which reserves 4 bytes for the Message metadata
* length instead of 8.
* @see https://issues.apache.org/jira/browse/ARROW-6313
*/
writeLegacyIpcFormat?: boolean;
}
export class RecordBatchWriter<T extends { [key: string]: DataType } = any> extends ReadableInterop<Uint8Array> implements Writable<RecordBatch<T>> {

@@ -55,5 +69,7 @@

constructor(options?: { autoDestroy: boolean }) {
constructor(options?: RecordBatchStreamWriterOptions) {
super();
this._autoDestroy = options && (typeof options.autoDestroy === 'boolean') ? options.autoDestroy : true;
isObject(options) || (options = { autoDestroy: true, writeLegacyIpcFormat: false });
this._autoDestroy = (typeof options.autoDestroy === 'boolean') ? options.autoDestroy : true;
this._writeLegacyIpcFormat = (typeof options.writeLegacyIpcFormat === 'boolean') ? options.writeLegacyIpcFormat : false;
}

@@ -64,2 +80,3 @@

protected _autoDestroy: boolean;
protected _writeLegacyIpcFormat: boolean;
// @ts-ignore

@@ -184,4 +201,5 @@ protected _sink = new AsyncByteQueue();

const flatbufferSize = buffer.byteLength;
const alignedSize = (flatbufferSize + 4 + a) & ~a;
const nPaddingBytes = alignedSize - flatbufferSize - 4;
const prefixSize = !this._writeLegacyIpcFormat ? 8 : 4;
const alignedSize = (flatbufferSize + prefixSize + a) & ~a;
const nPaddingBytes = alignedSize - flatbufferSize - prefixSize;

@@ -194,4 +212,8 @@ if (message.headerType === MessageHeader.RecordBatch) {

// If not in legacy pre-0.15.0 mode, write the stream continuation indicator
if (!this._writeLegacyIpcFormat) {
this._write(Int32Array.of(-1));
}
// Write the flatbuffer size prefix including padding
this._write(Int32Array.of(alignedSize - 4));
this._write(Int32Array.of(alignedSize - prefixSize));
// Write the flatbuffer

@@ -220,3 +242,6 @@ if (flatbufferSize > 0) { this._write(buffer); }

protected _writeFooter(schema: Schema<T>) {
return this._writePadding(4); // eos bytes
// eos bytes
return this._writeLegacyIpcFormat
? this._write(Int32Array.of(0))
: this._write(Int32Array.of(-1, 0));
}

@@ -284,8 +309,8 @@

export class RecordBatchStreamWriter<T extends { [key: string]: DataType } = any> extends RecordBatchWriter<T> {
public static writeAll<T extends { [key: string]: DataType } = any>(input: Table<T> | Iterable<RecordBatch<T>>, options?: { autoDestroy: true }): RecordBatchStreamWriter<T>;
public static writeAll<T extends { [key: string]: DataType } = any>(input: AsyncIterable<RecordBatch<T>>, options?: { autoDestroy: true }): Promise<RecordBatchStreamWriter<T>>;
public static writeAll<T extends { [key: string]: DataType } = any>(input: PromiseLike<AsyncIterable<RecordBatch<T>>>, options?: { autoDestroy: true }): Promise<RecordBatchStreamWriter<T>>;
public static writeAll<T extends { [key: string]: DataType } = any>(input: PromiseLike<Table<T> | Iterable<RecordBatch<T>>>, options?: { autoDestroy: true }): Promise<RecordBatchStreamWriter<T>>;
public static writeAll<T extends { [key: string]: DataType } = any>(input: Table<T> | Iterable<RecordBatch<T>>, options?: RecordBatchStreamWriterOptions): RecordBatchStreamWriter<T>;
public static writeAll<T extends { [key: string]: DataType } = any>(input: AsyncIterable<RecordBatch<T>>, options?: RecordBatchStreamWriterOptions): Promise<RecordBatchStreamWriter<T>>;
public static writeAll<T extends { [key: string]: DataType } = any>(input: PromiseLike<AsyncIterable<RecordBatch<T>>>, options?: RecordBatchStreamWriterOptions): Promise<RecordBatchStreamWriter<T>>;
public static writeAll<T extends { [key: string]: DataType } = any>(input: PromiseLike<Table<T> | Iterable<RecordBatch<T>>>, options?: RecordBatchStreamWriterOptions): Promise<RecordBatchStreamWriter<T>>;
/** @nocollapse */
public static writeAll<T extends { [key: string]: DataType } = any>(input: any, options?: { autoDestroy: true }) {
public static writeAll<T extends { [key: string]: DataType } = any>(input: any, options?: RecordBatchStreamWriterOptions) {
const writer = new RecordBatchStreamWriter<T>(options);

@@ -333,4 +358,4 @@ if (isPromise<any>(input)) {

));
return this
._writePadding(4) // EOS bytes for sequential readers
return super
._writeFooter(schema) // EOS bytes for sequential readers
._write(buffer) // Write the flatbuffer

@@ -366,2 +391,4 @@ ._write(Int32Array.of(buffer.byteLength)) // then the footer size suffix

protected _writeMessage() { return this; }
// @ts-ignore
protected _writeFooter(schema: Schema<T>) { return this; }
protected _writeSchema(schema: Schema<T>) {

@@ -368,0 +395,0 @@ return this._write(`{\n "schema": ${

2

package.json
{
"version": "0.14.1",
"version": "0.15.0",
"name": "@apache-arrow/ts",

@@ -4,0 +4,0 @@ "browser": "Arrow.dom.ts",

2

README.md

@@ -254,3 +254,3 @@ <!---

Full list of broader Apache Arrow [projects & organizations](https://github.com/apache/arrow/blob/master/site/powered_by.md).
Full list of broader Apache Arrow [projects & organizations](https://arrow.apache.org/powered_by/).

@@ -257,0 +257,0 @@ ## Open Source Projects

39

recordbatch.ts

@@ -25,8 +25,7 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Chunked } from './vector/chunked';
import { MapVector } from './vector/index';
import { selectFieldArgs } from './util/args';
import { DataType, Struct, Map_, Dictionary } from './type';
import { DataType, Struct, Dictionary } from './type';
import { ensureSameLengthData } from './util/recordbatch';
import { Clonable, Sliceable, Applicative } from './vector';
import { VectorBuilderOptions, VectorBuilderOptionsAsync } from './vector/index';
import { StructVector, VectorBuilderOptions, VectorBuilderOptionsAsync } from './vector/index';

@@ -38,21 +37,21 @@ type VectorMap = { [key: string]: Vector };

export interface RecordBatch<T extends { [key: string]: DataType } = any> {
concat(...others: Vector<Map_<T>>[]): Table<T>;
concat(...others: Vector<Struct<T>>[]): Table<T>;
slice(begin?: number, end?: number): RecordBatch<T>;
clone(data: Data<Map_<T>>, children?: Vector[]): RecordBatch<T>;
clone(data: Data<Struct<T>>, children?: Vector[]): RecordBatch<T>;
}
export class RecordBatch<T extends { [key: string]: DataType } = any>
extends MapVector<T>
extends StructVector<T>
implements Clonable<RecordBatch<T>>,
Sliceable<RecordBatch<T>>,
Applicative<Map_<T>, Table<T>> {
Applicative<Struct<T>, Table<T>> {
public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptions<Struct<T> | Map_<T>, TNull>): Table<T>;
public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptionsAsync<Struct<T> | Map_<T>, TNull>): Promise<Table<T>>;
public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptions<Struct<T>, TNull>): Table<T>;
public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptionsAsync<Struct<T>, TNull>): Promise<Table<T>>;
/** @nocollapse */
public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptions<Struct<T> | Map_<T>, TNull> | VectorBuilderOptionsAsync<Struct<T> | Map_<T>, TNull>) {
if (isIterable<(Struct<T> | Map_<T>)['TValue'] | TNull>(options['values'])) {
return Table.from(options as VectorBuilderOptions<Struct<T> | Map_<T>, TNull>);
public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptions<Struct<T>, TNull> | VectorBuilderOptionsAsync<Struct<T>, TNull>) {
if (isIterable<(Struct<T>)['TValue'] | TNull>(options['values'])) {
return Table.from(options as VectorBuilderOptions<Struct<T>, TNull>);
}
return Table.from(options as VectorBuilderOptionsAsync<Struct<T> | Map_<T>, TNull>);
return Table.from(options as VectorBuilderOptionsAsync<Struct<T>, TNull>);
}

@@ -73,13 +72,13 @@

constructor(schema: Schema<T>, length: number, children: (Data | Vector)[]);
constructor(schema: Schema<T>, data: Data<Map_<T>>, children?: Vector[]);
constructor(schema: Schema<T>, data: Data<Struct<T>>, children?: Vector[]);
constructor(...args: any[]) {
let data: Data<Map_<T>>;
let data: Data<Struct<T>>;
let schema = args[0] as Schema<T>;
let children: Vector[] | undefined;
if (args[1] instanceof Data) {
[, data, children] = (args as [any, Data<Map_<T>>, Vector<T[keyof T]>[]?]);
[, data, children] = (args as [any, Data<Struct<T>>, Vector<T[keyof T]>[]?]);
} else {
const fields = schema.fields as Field<T[keyof T]>[];
const [, length, childData] = args as [any, number, Data<T[keyof T]>[]];
data = Data.Map(new Map_<T>(fields), 0, length, 0, null, childData);
data = Data.Struct(new Struct<T>(fields), 0, length, 0, null, childData);
}

@@ -90,7 +89,7 @@ super(data, children);

public clone(data: Data<Map_<T>>, children = this._children) {
public clone(data: Data<Struct<T>>, children = this._children) {
return new RecordBatch<T>(this._schema, data, children);
}
public concat(...others: Vector<Map_<T>>[]): Table<T> {
public concat(...others: Vector<Struct<T>>[]): Table<T> {
const schema = this._schema, chunks = Chunked.flatten(this, ...others);

@@ -137,3 +136,3 @@ return new Table(schema, chunks.map(({ data }) => new RecordBatch(schema, data)));

return new DictionaryCollector().visit(
batch.data, new Map_(batch.schema.fields)
batch.data, new Struct(batch.schema.fields)
).dictionaries;

@@ -140,0 +139,0 @@ }

38

table.ts

@@ -24,11 +24,9 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { RecordBatchReader } from './ipc/reader';
import { DataType, RowLike, Struct, Map_ } from './type';
import { DataType, RowLike, Struct } from './type';
import { selectColumnArgs, selectArgs } from './util/args';
import { Clonable, Sliceable, Applicative } from './vector';
import { isPromise, isIterable, isAsyncIterable } from './util/compat';
import { distributeColumnsIntoRecordBatches } from './util/recordbatch';
import { distributeVectorsIntoRecordBatches } from './util/recordbatch';
import { Vector, Chunked, MapVector, StructVector } from './vector/index';
import { RecordBatchFileWriter, RecordBatchStreamWriter } from './ipc/writer';
import { VectorBuilderOptions, VectorBuilderOptionsAsync } from './vector/index';
import { distributeColumnsIntoRecordBatches, distributeVectorsIntoRecordBatches } from './util/recordbatch';
import { Vector, Chunked, StructVector, VectorBuilderOptions, VectorBuilderOptionsAsync } from './vector/index';

@@ -46,6 +44,7 @@ type VectorMap = { [key: string]: Vector };

slice(begin?: number, end?: number): Table<T>;
concat(...others: Vector<Map_<T>>[]): Table<T>;
concat(...others: Vector<Struct<T>>[]): Table<T>;
clone(chunks?: RecordBatch<T>[], offsets?: Uint32Array): Table<T>;
scan(next: import('./compute/dataframe').NextFunc, bind?: import('./compute/dataframe').BindFunc): void;
scanReverse(next: import('./compute/dataframe').NextFunc, bind?: import('./compute/dataframe').BindFunc): void;
countBy(name: import('./compute/predicate').Col | string): import('./compute/dataframe').CountByResult;

@@ -56,7 +55,7 @@ filter(predicate: import('./compute/predicate').Predicate): import('./compute/dataframe').FilteredDataFrame<T>;

export class Table<T extends { [key: string]: DataType } = any>
extends Chunked<Map_<T>>
extends Chunked<Struct<T>>
implements DataFrame<T>,
Clonable<Table<T>>,
Sliceable<Table<T>>,
Applicative<Map_<T>, Table<T>> {
Applicative<Struct<T>, Table<T>> {

@@ -75,4 +74,4 @@ /** @nocollapse */

public static from<T extends { [key: string]: DataType } = any>(source: PromiseLike<RecordBatchReader<T>>): Promise<Table<T>>;
public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptions<Struct<T> | Map_<T>, TNull>): Table<T>;
public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptionsAsync<Struct<T> | Map_<T>, TNull>): Promise<Table<T>>;
public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptions<Struct<T>, TNull>): Table<T>;
public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptionsAsync<Struct<T>, TNull>): Promise<Table<T>>;
/** @nocollapse */

@@ -118,7 +117,2 @@ public static from<T extends { [key: string]: DataType } = any, TNull = any>(input?: any) {

/** @nocollapse */
public static fromMap<T extends { [key: string]: DataType } = any>(vector: Vector<Map_<T>>) {
return Table.new<T>(vector.data.childData as Data<T[keyof T]>[], vector.type.children);
}
/** @nocollapse */
public static fromStruct<T extends { [key: string]: DataType } = any>(vector: Vector<Struct<T>>) {

@@ -204,3 +198,3 @@ return Table.new<T>(vector.data.childData as Data<T[keyof T]>[], vector.type.children);

super(new Map_(schema.fields), chunks);
super(new Struct(schema.fields), chunks);

@@ -292,7 +286,5 @@ this._schema = schema;

function tableFromIterable<T extends { [key: string]: DataType } = any, TNull = any>(input: VectorBuilderOptions<Struct<T> | Map_<T>, TNull>) {
function tableFromIterable<T extends { [key: string]: DataType } = any, TNull = any>(input: VectorBuilderOptions<Struct<T>, TNull>) {
const { type } = input;
if (type instanceof Map_) {
return Table.fromMap(MapVector.from(input as VectorBuilderOptions<Map_<T>, TNull>));
} else if (type instanceof Struct) {
if (type instanceof Struct) {
return Table.fromStruct(StructVector.from(input as VectorBuilderOptions<Struct<T>, TNull>));

@@ -303,7 +295,5 @@ }

function tableFromAsyncIterable<T extends { [key: string]: DataType } = any, TNull = any>(input: VectorBuilderOptionsAsync<Struct<T> | Map_<T>, TNull>) {
function tableFromAsyncIterable<T extends { [key: string]: DataType } = any, TNull = any>(input: VectorBuilderOptionsAsync<Struct<T>, TNull>) {
const { type } = input;
if (type instanceof Map_) {
return MapVector.from(input as VectorBuilderOptionsAsync<Map_<T>, TNull>).then((vector) => Table.fromMap(vector));
} else if (type instanceof Struct) {
if (type instanceof Struct) {
return StructVector.from(input as VectorBuilderOptionsAsync<Struct<T>, TNull>).then((vector) => Table.fromStruct(vector));

@@ -310,0 +300,0 @@ }

31

type.ts

@@ -41,7 +41,14 @@ // Licensed to the Apache Software Foundation (ASF) under one

export type RowLike<T extends { [key: string]: DataType }> =
( Iterable<T[keyof T]['TValue'] | null> )
( Iterable<[string, T[keyof T]['TValue'] | null]> )
& { [P in keyof T]: T[P]['TValue'] | null }
& { get<K extends keyof T>(key: K): T[K]['TValue'] | null; }
& { set<K extends keyof T>(key: K, val: T[K]['TValue'] | null): void; }
;
/** @ignore */
export type MapLike<K extends DataType = any, V extends DataType = any> =
{ [P in K['TValue']]: V['TValue'] | null }
& ( Map<K['TValue'], V['TValue'] | null> )
;
export interface DataType<TType extends Type = Type, TChildren extends { [key: string]: DataType } = any> {

@@ -430,3 +437,3 @@ readonly TType: TType;

public get typeId() { return Type.Struct as Type.Struct; }
public toString() { return `Struct<[${this.children.map((f) => f.type).join(`, `)}]>`; }
public toString() { return `Struct<{${this.children.map((f) => `${f.name}:${f.type}`).join(`, `)}}>`; }
protected static [Symbol.toStringTag] = ((proto: Struct) => {

@@ -527,11 +534,21 @@ (<any> proto).children = null;

/** @ignore */
export interface Map_<T extends { [key: string]: DataType } = any> extends DataType<Type.Map> { TArray: IterableArrayLike<RowLike<T>>; TValue: RowLike<T>; dataTypes: T; }
export interface Map_<TKey extends DataType = any, TValue extends DataType = any> extends DataType<Type.Map> {
TArray: IterableArrayLike<Map<TKey['TValue'], TValue['TValue'] | null>>;
TChild: Struct<{ key: TKey, value: TValue }>;
TValue: MapLike<TKey, TValue>;
}
/** @ignore */
export class Map_<T extends { [key: string]: DataType } = any> extends DataType<Type.Map, T> {
constructor(public readonly children: Field<T[keyof T]>[],
public readonly keysSorted: boolean = false) {
export class Map_<TKey extends DataType = any, TValue extends DataType = any> extends DataType<Type.Map> {
constructor(child: Field<Struct<{ key: TKey, value: TValue }>>, keysSorted = false) {
super();
this.children = [child];
this.keysSorted = keysSorted;
}
public readonly keysSorted: boolean;
public readonly children: Field<Struct<{ key: TKey, value: TValue }>>[];
public get typeId() { return Type.Map as Type.Map; }
public toString() { return `Map<{${this.children.map((f) => `${f.name}:${f.type}`).join(`, `)}}>`; }
public get keyType(): TKey { return this.children[0].type.children[0].type as TKey; }
public get valueType(): TValue { return this.children[0].type.children[1].type as TValue; }
public toString() { return `Map<{${this.children[0].type.children.map((f) => `${f.name}:${f.type}`).join(`, `)}}>`; }
protected static [Symbol.toStringTag] = ((proto: Map_) => {

@@ -538,0 +555,0 @@ (<any> proto).children = null;

21

util/buffer.ts

@@ -128,23 +128,2 @@ // Licensed to the Apache Software Foundation (ASF) under one

/** @ignore */
export const toFloat16Array = (input: ArrayBufferViewInput) => {
let floats: Float32Array | Float64Array | null = null;
if (ArrayBuffer.isView(input)) {
switch (input.constructor) {
case Float32Array: floats = input as Float32Array; break;
case Float64Array: floats = input as Float64Array; break;
}
} else if (isIterable(input)) {
floats = toFloat64Array(input);
}
if (floats) {
const u16s = new Uint16Array(floats.length);
for (let i = -1, n = u16s.length; ++i < n;) {
u16s[i] = (floats[i] * 32767) + 32767;
}
return u16s;
}
return toUint16Array(input);
};
/** @ignore */
type ArrayBufferViewIteratorInput = Iterable<ArrayBufferViewInput> | ArrayBufferViewInput;

@@ -151,0 +130,0 @@

168

util/vector.ts

@@ -19,3 +19,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Vector } from '../vector';
import { Row, kLength } from '../vector/row';
import { MapRow, StructRow } from '../vector/row';
import { compareArrayLike } from '../util/buffer';

@@ -85,108 +85,116 @@ import { BigInt, BigIntAvailable } from './compat';

}
// Compare TypedArrays
if (ArrayBuffer.isView(search)) {
return (value: any) => value ? compareArrayLike(search, value) : false;
}
// Compare Maps and Rows
if (search instanceof Map) { return creatMapComparator(search); }
// Compare Array-likes
if (Array.isArray(search)) {
return createArrayLikeComparator(search);
}
// Compare Rows
if (search instanceof Row) {
return createRowComparator(search);
}
if (Array.isArray(search)) { return createArrayLikeComparator(search); }
// Compare Vectors
if (search instanceof Vector) {
return createVectorComparator(search);
}
if (search instanceof Vector) { return createVectorComparator(search); }
// Compare non-empty Objects
const keys = Object.keys(search);
if (keys.length > 0) {
return createObjectKeysComparator(search, keys);
return createObjectComparator(search);
}
/** @ignore */
function createArrayLikeComparator(lhs: ArrayLike<any>) {
const comparitors = [] as ((x: any) => boolean)[];
for (let i = -1, n = lhs.length; ++i < n;) {
comparitors[i] = createElementComparator(lhs[i]);
}
// No valid comparator
return () => false;
return createSubElementsComparator(comparitors);
}
/** @ignore */
function createArrayLikeComparator(search: ArrayLike<any>) {
const n = search.length;
const fns = [] as ((x: any) => boolean)[];
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator((search as any)[i]);
function creatMapComparator(lhs: Map<any, any>) {
let i = -1;
const comparitors = [] as ((x: any) => boolean)[];
lhs.forEach((v) => comparitors[++i] = createElementComparator(v));
return createSubElementsComparator(comparitors);
}
/** @ignore */
function createVectorComparator(lhs: Vector<any>) {
const comparitors = [] as ((x: any) => boolean)[];
for (let i = -1, n = lhs.length; ++i < n;) {
comparitors[i] = createElementComparator(lhs.get(i));
}
return (value: any) => {
if (!value) { return false; }
// Handle the case where the search element is an Array, but the
// values are Rows or Vectors, e.g. list.indexOf(['foo', 'bar'])
if (value instanceof Row) {
if (value[kLength] !== n) { return false; }
for (let i = -1; ++i < n;) {
if (!(fns[i](value.get(i)))) { return false; }
}
return true;
}
if (value.length !== n) { return false; }
if (value instanceof Vector) {
for (let i = -1; ++i < n;) {
if (!(fns[i](value.get(i)))) { return false; }
}
return true;
}
for (let i = -1; ++i < n;) {
if (!(fns[i](value[i]))) { return false; }
}
return true;
};
return createSubElementsComparator(comparitors);
}
/** @ignore */
function createRowComparator(search: Row<any>) {
const n = search[kLength];
const C = search.constructor as any;
const fns = [] as ((x: any) => boolean)[];
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator(search.get(i));
function createObjectComparator(lhs: any) {
const keys = Object.keys(lhs);
// Only compare non-empty Objects
if (keys.length === 0) { return () => false; }
const comparitors = [] as ((x: any) => boolean)[];
for (let i = -1, n = keys.length; ++i < n;) {
comparitors[i] = createElementComparator(lhs[keys[i]]);
}
return (value: any) => {
if (!(value instanceof C)) { return false; }
if (!(value[kLength] === n)) { return false; }
for (let i = -1; ++i < n;) {
if (!(fns[i](value.get(i)))) { return false; }
return createSubElementsComparator(comparitors, keys);
}
function createSubElementsComparator(comparitors: ((x: any) => boolean)[], keys?: Iterable<string>) {
return (rhs: any) => {
if (!rhs || typeof rhs !== 'object') {
return false;
}
return true;
switch (rhs.constructor) {
case Array: return compareArray(comparitors, rhs);
case Map:
case MapRow:
case StructRow:
return compareObject(comparitors, rhs, rhs.keys());
case Object:
case undefined: // support `Object.create(null)` objects
return compareObject(comparitors, rhs, keys || Object.keys(rhs));
}
return rhs instanceof Vector ? compareVector(comparitors, rhs) : false;
};
}
/** @ignore */
function createVectorComparator(search: Vector<any>) {
const n = search.length;
const C = search.constructor as any;
const fns = [] as ((x: any) => boolean)[];
function compareArray(comparitors: ((x: any) => boolean)[], arr: any[]) {
const n = comparitors.length;
if (arr.length !== n) { return false; }
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator((search as any).get(i));
if (!(comparitors[i](arr[i]))) { return false; }
}
return (value: any) => {
if (!(value instanceof C)) { return false; }
if (!(value.length === n)) { return false; }
for (let i = -1; ++i < n;) {
if (!(fns[i](value.get(i)))) { return false; }
}
return true;
};
return true;
}
/** @ignore */
function createObjectKeysComparator(search: any, keys: string[]) {
const n = keys.length;
const fns = [] as ((x: any) => boolean)[];
function compareVector(comparitors: ((x: any) => boolean)[], vec: Vector) {
const n = comparitors.length;
if (vec.length !== n) { return false; }
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator(search[keys[i]]);
if (!(comparitors[i](vec.get(i)))) { return false; }
}
return (value: any) => {
if (!value || typeof value !== 'object') { return false; }
for (let i = -1; ++i < n;) {
if (!(fns[i](value[keys[i]]))) { return false; }
return true;
}
function compareObject(comparitors: ((x: any) => boolean)[], obj: Map<any, any>, keys: Iterable<string>) {
const lKeyItr = keys[Symbol.iterator]();
const rKeyItr = obj instanceof Map ? obj.keys() : Object.keys(obj)[Symbol.iterator]();
const rValItr = obj instanceof Map ? obj.values() : Object.values(obj)[Symbol.iterator]();
let i = 0;
let n = comparitors.length;
let rVal = rValItr.next();
let lKey = lKeyItr.next();
let rKey = rKeyItr.next();
for (; i < n && !lKey.done && !rKey.done && !rVal.done;
++i, lKey = lKeyItr.next(), rKey = rKeyItr.next(), rVal = rValItr.next()) {
if (lKey.value !== rKey.value || !comparitors[i](rVal.value)) {
break;
}
}
if (i === n && lKey.done && rKey.done && rVal.done) {
return true;
};
}
lKeyItr.return && lKeyItr.return();
rKeyItr.return && rKeyItr.return();
rValItr.return && rValItr.return();
return false;
}

2

vector/base.ts

@@ -102,3 +102,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

protected _sliceInternal(self: this, begin: number, end: number) {
return self.clone(self.data.slice(begin, end - begin));
return self.clone(self.data.slice(begin, end - begin), null!);
}

@@ -105,0 +105,0 @@

4

vector/chunked.ts

@@ -270,5 +270,5 @@ // Licensed to the Apache Software Foundation (ASF) under one

const arraySet = (src: any[], dst: any[], offset: number) => {
let idx = offset - 1;
let idx = offset;
for (let i = -1, n = src.length; ++i < n;) {
dst[++idx] = src[i];
dst[idx++] = src[i];
}

@@ -275,0 +275,0 @@ return idx;

120

vector/float.ts

@@ -20,34 +20,84 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Vector } from '../vector';
import { Chunked } from './chunked';
import { BaseVector } from './base';
import { VectorType as V } from '../interfaces';
import { Float, Float16, Float32, Float64 } from '../type';
import { toFloat16Array, toFloat32Array, toFloat64Array } from '../util/buffer';
import { VectorBuilderOptions } from './index';
import { vectorFromValuesWithType } from './index';
import { VectorBuilderOptionsAsync } from './index';
import { Float, Float16, Float32, Float64, FloatArray } from '../type';
import { VectorType as V, TypedArrayConstructor } from '../interfaces';
/** @ignore */
type FloatVectorConstructors =
typeof FloatVector |
typeof Float16Vector |
typeof Float32Vector |
typeof Float64Vector ;
/** @ignore */
type FromInput<T extends Float, TNull = any> =
FloatArray |
Iterable<T['TValue'] | TNull> |
AsyncIterable<T['TValue'] | TNull> |
VectorBuilderOptions<T, TNull> |
VectorBuilderOptionsAsync<T, TNull> ;
/** @ignore */
type FloatArrayCtor = TypedArrayConstructor<FloatArray>;
/** @ignore */
export class FloatVector<T extends Float = Float> extends BaseVector<T> {
public static from(this: typeof FloatVector, data: Float16['TArray']): Float16Vector;
public static from(this: typeof FloatVector, data: Float32['TArray']): Float32Vector;
public static from(this: typeof FloatVector, data: Float64['TArray']): Float64Vector;
public static from<T extends Float>(this: typeof FloatVector, data: T['TArray']): V<T>;
// Guaranteed zero-copy variants
public static from(this: typeof FloatVector, input: Uint16Array): Float16Vector;
public static from(this: typeof FloatVector, input: Float32Array): Float32Vector;
public static from(this: typeof FloatVector, input: Float64Array): Float64Vector;
public static from(this: typeof Float16Vector, data: Float16['TArray'] | Iterable<number>): Float16Vector;
public static from(this: typeof Float32Vector, data: Float32['TArray'] | Iterable<number>): Float32Vector;
public static from(this: typeof Float64Vector, data: Float64['TArray'] | Iterable<number>): Float64Vector;
// Zero-copy if input is a TypedArray of the same type as the
// Vector that from is called on, otherwise uses the Builders
public static from<TNull = any>(this: typeof Float16Vector, input: FromInput<Float16, TNull>): Float16Vector;
public static from<TNull = any>(this: typeof Float32Vector, input: FromInput<Float32, TNull>): Float32Vector;
public static from<TNull = any>(this: typeof Float64Vector, input: FromInput<Float64, TNull>): Float64Vector;
// Not zero-copy
public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: Iterable<T['TValue'] | TNull>): V<T>;
public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: AsyncIterable<T['TValue'] | TNull>): Promise<V<T>>;
public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: VectorBuilderOptions<T, TNull>): Chunked<T>;
public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: VectorBuilderOptionsAsync<T, TNull>): Promise<Chunked<T>>;
/** @nocollapse */
public static from<T extends Float>(data: T['TArray']) {
let type: Float | null = null;
switch (this) {
case Float16Vector: data = toFloat16Array(data); break;
case Float32Vector: data = toFloat32Array(data); break;
case Float64Vector: data = toFloat64Array(data); break;
public static from<T extends Float, TNull = any>(this: FloatVectorConstructors, input: FromInput<T, TNull>) {
let ArrowType = vectorTypeToDataType(this);
if ((input instanceof ArrayBuffer) || ArrayBuffer.isView(input)) {
let InputType = arrayTypeToDataType(input.constructor as FloatArrayCtor) || ArrowType;
// Special case, infer the Arrow DataType from the input if calling the base
// FloatVector.from with a TypedArray, e.g. `FloatVector.from(new Float32Array())`
if (ArrowType === null) {
ArrowType = InputType;
}
// If the DataType inferred from the Vector constructor matches the
// DataType inferred from the input arguments, return zero-copy view
if (ArrowType && ArrowType === InputType) {
let type = new ArrowType();
let length = input.byteLength / type.ArrayType.BYTES_PER_ELEMENT;
// If the ArrowType is Float16 but the input type isn't a Uint16Array,
// let the Float16Builder handle casting the input values to Uint16s.
if (!convertTo16Bit(ArrowType, input.constructor)) {
return Vector.new(Data.Float(type, 0, length, 0, null, input as FloatArray));
}
}
}
switch (data.constructor) {
case Uint16Array: type = new Float16(); break;
case Float32Array: type = new Float32(); break;
case Float64Array: type = new Float64(); break;
if (ArrowType) {
// If the DataType inferred from the Vector constructor is different than
// the DataType inferred from the input TypedArray, or if input isn't a
// TypedArray, use the Builders to construct the result Vector
return vectorFromValuesWithType(() => new ArrowType!() as T, input);
}
return type !== null
? Vector.new(Data.Float(type, 0, data.length, 0, null, data))
: (() => { throw new TypeError('Unrecognized FloatVector input'); })();
if ((input instanceof DataView) || (input instanceof ArrayBuffer)) {
throw new TypeError(`Cannot infer float type from instance of ${input.constructor.name}`);
}
throw new TypeError('Unrecognized FloatVector input');
}

@@ -72,1 +122,25 @@ }

export class Float64Vector extends FloatVector<Float64> {}
const convertTo16Bit = (typeCtor: any, dataCtor: any) => {
return (typeCtor === Float16) && (dataCtor !== Uint16Array);
};
/** @ignore */
const arrayTypeToDataType = (ctor: FloatArrayCtor) => {
switch (ctor) {
case Uint16Array: return Float16;
case Float32Array: return Float32;
case Float64Array: return Float64;
default: return null;
}
};
/** @ignore */
const vectorTypeToDataType = (ctor: FloatVectorConstructors) => {
switch (ctor) {
case Float16Vector: return Float16;
case Float32Vector: return Float32;
case Float64Vector: return Float64;
default: return null;
}
};

2

vector/index.ts

@@ -18,3 +18,2 @@ // Licensed to the Apache Software Foundation (ASF) under one

export { Row } from './row';
export { Vector } from '../vector';

@@ -41,2 +40,3 @@ export { BaseVector } from './base';

export { Utf8Vector } from './utf8';
export { MapRow, StructRow } from './row';

@@ -43,0 +43,0 @@ import * as fn from '../util/fn';

184

vector/int.ts

@@ -20,66 +20,107 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Vector } from '../vector';
import { Chunked } from './chunked';
import { BaseVector } from './base';
import { VectorType as V } from '../interfaces';
import { Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64 } from '../type';
import {
toInt8Array, toInt16Array, toInt32Array,
toUint8Array, toUint16Array, toUint32Array,
toBigInt64Array, toBigUint64Array
} from '../util/buffer';
import { VectorBuilderOptions } from './index';
import { vectorFromValuesWithType } from './index';
import { VectorBuilderOptionsAsync } from './index';
import { BigInt64Array, BigUint64Array } from '../util/compat';
import { toBigInt64Array, toBigUint64Array } from '../util/buffer';
import { Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, IntArray } from '../type';
import { VectorType as V, TypedArrayConstructor, BigIntArrayConstructor, BigIntArray } from '../interfaces';
/** @ignore */
type IntVectorConstructors =
typeof IntVector |
typeof Int8Vector |
typeof Int16Vector |
typeof Int32Vector |
typeof Uint8Vector |
typeof Uint16Vector |
typeof Uint32Vector |
typeof Int64Vector |
typeof Uint64Vector ;
/** @ignore */
type FromInput<T extends Int, TNull = any> =
IntArray | BigIntArray |
Iterable<T['TValue'] | TNull> |
AsyncIterable<T['TValue'] | TNull> |
VectorBuilderOptions<T, TNull> |
VectorBuilderOptionsAsync<T, TNull> ;
/** @ignore */
type FromArgs<T extends Int, TNull = any> = [FromInput<T, TNull>, boolean?];
/** @ignore */
type IntArrayCtor = TypedArrayConstructor<IntArray> | BigIntArrayConstructor<BigIntArray>;
/** @ignore */
export class IntVector<T extends Int = Int> extends BaseVector<T> {
public static from(this: typeof IntVector, data: Int8Array): Int8Vector;
public static from(this: typeof IntVector, data: Int16Array): Int16Vector;
public static from(this: typeof IntVector, data: Int32Array): Int32Vector;
public static from(this: typeof IntVector, data: Uint8Array): Uint8Vector;
public static from(this: typeof IntVector, data: Uint16Array): Uint16Vector;
public static from(this: typeof IntVector, data: Uint32Array): Uint32Vector;
// Guaranteed zero-copy variants
public static from(this: typeof IntVector, input: Int8Array): Int8Vector;
public static from(this: typeof IntVector, input: Int16Array): Int16Vector;
public static from(this: typeof IntVector, input: Int32Array): Int32Vector;
public static from(this: typeof IntVector, input: BigInt64Array): Int64Vector;
public static from(this: typeof IntVector, input: Int32Array, is64bit: true): Int64Vector;
public static from(this: typeof IntVector, input: Uint8Array): Uint8Vector;
public static from(this: typeof IntVector, input: Uint16Array): Uint16Vector;
public static from(this: typeof IntVector, input: Uint32Array): Uint32Vector;
public static from(this: typeof IntVector, input: BigUint64Array): Uint64Vector;
public static from(this: typeof IntVector, input: Uint32Array, is64bit: true): Uint64Vector;
// @ts-ignore
public static from(this: typeof IntVector, data: Int32Array, is64: true): Int64Vector;
public static from(this: typeof IntVector, data: Uint32Array, is64: true): Uint64Vector;
public static from<T extends Int>(this: typeof IntVector, data: T['TArray']): V<T>;
// Zero-copy if input is a TypedArray of the same type as the
// Vector that from is called on, otherwise uses the Builders
public static from<TNull = any>(this: typeof Int8Vector, input: FromInput<Int8, TNull>): Int8Vector;
public static from<TNull = any>(this: typeof Int16Vector, input: FromInput<Int16, TNull>): Int16Vector;
public static from<TNull = any>(this: typeof Int32Vector, input: FromInput<Int32, TNull>): Int32Vector;
public static from<TNull = any>(this: typeof Int64Vector, input: FromInput<Int64, TNull>): Int64Vector;
public static from<TNull = any>(this: typeof Uint8Vector, input: FromInput<Uint8, TNull>): Uint8Vector;
public static from<TNull = any>(this: typeof Uint16Vector, input: FromInput<Uint16, TNull>): Uint16Vector;
public static from<TNull = any>(this: typeof Uint32Vector, input: FromInput<Uint32, TNull>): Uint32Vector;
public static from<TNull = any>(this: typeof Uint64Vector, input: FromInput<Uint64, TNull>): Uint64Vector;
public static from(this: typeof Int8Vector, data: Int8['TArray'] | Iterable<number>): Int8Vector;
public static from(this: typeof Int16Vector, data: Int16['TArray'] | Iterable<number>): Int16Vector;
public static from(this: typeof Int32Vector, data: Int32['TArray'] | Iterable<number>): Int32Vector;
public static from(this: typeof Int64Vector, data: Int32['TArray'] | Iterable<number>): Int64Vector;
public static from(this: typeof Uint8Vector, data: Uint8['TArray'] | Iterable<number>): Uint8Vector;
public static from(this: typeof Uint16Vector, data: Uint16['TArray'] | Iterable<number>): Uint16Vector;
public static from(this: typeof Uint32Vector, data: Uint32['TArray'] | Iterable<number>): Uint32Vector;
public static from(this: typeof Uint64Vector, data: Uint32['TArray'] | Iterable<number>): Uint64Vector;
// Not zero-copy
public static from<T extends Int, TNull = any>(this: typeof IntVector, input: Iterable<T['TValue'] | TNull>): V<T>;
public static from<T extends Int, TNull = any>(this: typeof IntVector, input: AsyncIterable<T['TValue'] | TNull>): Promise<V<T>>;
public static from<T extends Int, TNull = any>(this: typeof IntVector, input: VectorBuilderOptions<T, TNull>): Chunked<T>;
public static from<T extends Int, TNull = any>(this: typeof IntVector, input: VectorBuilderOptionsAsync<T, TNull>): Promise<Chunked<T>>;
/** @nocollapse */
public static from<T extends Int, TNull = any>(this: IntVectorConstructors, ...args: FromArgs<T, TNull>) {
/** @nocollapse */
public static from<T extends Int>(data: T['TArray'], is64?: boolean) {
let length: number = 0;
let type: Int | null = null;
switch (this) {
case Int8Vector: data = toInt8Array(data); is64 = false; break;
case Int16Vector: data = toInt16Array(data); is64 = false; break;
case Int32Vector: data = toInt32Array(data); is64 = false; break;
case Int64Vector: data = toInt32Array(data); is64 = true; break;
case Uint8Vector: data = toUint8Array(data); is64 = false; break;
case Uint16Vector: data = toUint16Array(data); is64 = false; break;
case Uint32Vector: data = toUint32Array(data); is64 = false; break;
case Uint64Vector: data = toUint32Array(data); is64 = true; break;
}
if (is64 === true) {
length = data.length * 0.5;
type = data instanceof Int32Array ? new Int64() : new Uint64();
} else {
length = data.length;
switch (data.constructor) {
case Int8Array: type = new Int8(); break;
case Int16Array: type = new Int16(); break;
case Int32Array: type = new Int32(); break;
case Uint8Array: type = new Uint8(); break;
case Uint16Array: type = new Uint16(); break;
case Uint32Array: type = new Uint32(); break;
let [input, is64bit = false] = args;
let ArrowType = vectorTypeToDataType(this, is64bit);
if ((input instanceof ArrayBuffer) || ArrayBuffer.isView(input)) {
let InputType = arrayTypeToDataType(input.constructor as IntArrayCtor, is64bit) || ArrowType;
// Special case, infer the Arrow DataType from the input if calling the base
// IntVector.from with a TypedArray, e.g. `IntVector.from(new Int32Array())`
if (ArrowType === null) {
ArrowType = InputType;
}
// If the DataType inferred from the Vector constructor matches the
// DataType inferred from the input arguments, return zero-copy view
if (ArrowType && ArrowType === InputType) {
let type = new ArrowType();
let length = input.byteLength / type.ArrayType.BYTES_PER_ELEMENT;
// If the ArrowType is 64bit but the input type is 32bit pairs, update the logical length
if (convert32To64Bit(ArrowType, input.constructor)) {
length *= 0.5;
}
return Vector.new(Data.Int(type, 0, length, 0, null, input as IntArray));
}
}
return type !== null
? Vector.new(Data.Int(type, 0, length, 0, null, data))
: (() => { throw new TypeError('Unrecognized IntVector input'); })();
if (ArrowType) {
// If the DataType inferred from the Vector constructor is different than
// the DataType inferred from the input TypedArray, or if input isn't a
// TypedArray, use the Builders to construct the result Vector
return vectorFromValuesWithType(() => new ArrowType!() as T, input);
}
if ((input instanceof DataView) || (input instanceof ArrayBuffer)) {
throw new TypeError(`Cannot infer integer type from instance of ${input.constructor.name}`);
}
throw new TypeError('Unrecognized IntVector input');
}

@@ -123,1 +164,36 @@ }

}
const convert32To64Bit = (typeCtor: any, dataCtor: any) => {
return (typeCtor === Int64 || typeCtor === Uint64) &&
(dataCtor === Int32Array || dataCtor === Uint32Array);
};
/** @ignore */
const arrayTypeToDataType = (ctor: IntArrayCtor, is64bit: boolean) => {
switch (ctor) {
case Int8Array: return Int8;
case Int16Array: return Int16;
case Int32Array: return is64bit ? Int64 : Int32;
case BigInt64Array: return Int64;
case Uint8Array: return Uint8;
case Uint16Array: return Uint16;
case Uint32Array: return is64bit ? Uint64 : Uint32;
case BigUint64Array: return Uint64;
default: return null;
}
};
/** @ignore */
const vectorTypeToDataType = (ctor: IntVectorConstructors, is64bit: boolean) => {
switch (ctor) {
case Int8Vector: return Int8;
case Int16Vector: return Int16;
case Int32Vector: return is64bit ? Int64 : Int32;
case Int64Vector: return Int64;
case Uint8Vector: return Uint8;
case Uint16Vector: return Uint16;
case Uint32Vector: return is64bit ? Uint64 : Uint32;
case Uint64Vector: return Uint64;
default: return null;
}
};

19

vector/map.ts

@@ -18,18 +18,19 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { MapRow } from './row';
import { Field } from '../schema';
import { Vector } from '../vector';
import { BaseVector } from './base';
import { RowProxyGenerator } from './row';
import { DataType, Map_, Struct } from '../type';
import { DataType, Map_, Struct, List } from '../type';
/** @ignore */
export class MapVector<T extends { [key: string]: DataType } = any> extends BaseVector<Map_<T>> {
public asStruct() {
return Vector.new(this.data.clone(new Struct<T>(this.type.children as Field<T[keyof T]>[])));
export class MapVector<K extends DataType = any, V extends DataType = any> extends BaseVector<Map_<K, V>> {
public asList() {
const child = this.type.children[0] as Field<Struct<{ key: K, value: V }>>;
return Vector.new(this.data.clone(new List<Struct<{ key: K, value: V }>>(child)));
}
// @ts-ignore
private _rowProxy: RowProxyGenerator<T>;
public get rowProxy(): RowProxyGenerator<T> {
return this._rowProxy || (this._rowProxy = RowProxyGenerator.new<T>(this, this.type.children || [], true));
public bind(index: number): Map_<K, V>['TValue'] {
const child = this.getChildAt<Struct<{ key: K, value: V }>>(0);
const { [index]: begin, [index + 1]: end } = this.valueOffsets;
return new MapRow(child!.slice(begin, end));
}
}

341

vector/row.ts

@@ -18,104 +18,279 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Field } from '../schema';
import { MapVector } from '../vector/map';
import { DataType } from '../type';
import { Vector } from '../vector';
import { StructVector } from './struct';
import { valueToString } from '../util/pretty';
import { StructVector } from '../vector/struct';
import { DataType, Struct, RowLike } from '../type';
/** @ignore */ export const kLength = Symbol.for('length');
/** @ignore */ export const kParent = Symbol.for('parent');
/** @ignore */ export const kRowIndex = Symbol.for('rowIndex');
/** @ignore */ const columnDescriptor = { enumerable: true, configurable: false, get: null as any };
/** @ignore */ const rowLengthDescriptor = { writable: false, enumerable: false, configurable: false, value: -1 };
/** @ignore */ const rowParentDescriptor = { writable: false, enumerable: false, configurable: false, value: null as any };
/** @ignore */ const kParent = Symbol.for('parent');
/** @ignore */ const kRowIndex = Symbol.for('rowIndex');
/** @ignore */ const kKeyToIdx = Symbol.for('keyToIdx');
/** @ignore */ const kIdxToVal = Symbol.for('idxToVal');
/** @ignore */ const kCustomInspect = Symbol.for('nodejs.util.inspect.custom');
/** @ignore */
export class Row<T extends { [key: string]: DataType }> implements Iterable<T[keyof T]['TValue']> {
[key: string]: T[keyof T]['TValue'];
// @ts-ignore
public [kParent]: MapVector<T> | StructVector<T>;
// @ts-ignore
public [kRowIndex]: number;
// @ts-ignore
public readonly [kLength]: number;
*[Symbol.iterator]() {
for (let i = -1, n = this[kLength]; ++i < n;) {
yield this[i];
abstract class Row<K extends PropertyKey = any, V = any> implements Map<K, V> {
public readonly size: number;
public readonly [Symbol.toStringTag]: string;
protected [kRowIndex]: number;
protected [kParent]: Vector<Struct>;
protected [kKeyToIdx]: Map<K, number>;
protected [kIdxToVal]: V[];
constructor(parent: Vector<Struct>, numKeys: number) {
this[kParent] = parent;
this.size = numKeys;
}
public abstract keys(): IterableIterator<K>;
public abstract values(): IterableIterator<V>;
public abstract getKey(idx: number): K;
public abstract getIndex(key: K): number;
public abstract getValue(idx: number): V;
public abstract setValue(idx: number, val: V): void;
public entries() { return this[Symbol.iterator](); }
public has(key: K) { return this.get(key) !== undefined; }
public get(key: K) {
let val = undefined;
if (key !== null && key !== undefined) {
const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map());
let idx = ktoi.get(key);
if (idx !== undefined) {
const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size));
((val = itov[idx]) !== undefined) || (itov[idx] = val = this.getValue(idx));
} else if ((idx = this.getIndex(key)) > -1) {
ktoi.set(key, idx);
const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size));
((val = itov[idx]) !== undefined) || (itov[idx] = val = this.getValue(idx));
}
}
return val;
}
public get<K extends keyof T>(key: K) {
return (this as any)[key] as T[K]['TValue'];
public set(key: K, val: V) {
if (key !== null && key !== undefined) {
const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map());
let idx = ktoi.get(key);
if (idx === undefined) {
ktoi.set(key, idx = this.getIndex(key));
}
if (idx > -1) {
const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size));
itov[idx] = <any> this.setValue(idx, val);
}
}
return this;
}
public toJSON(): any {
return DataType.isStruct(this[kParent].type) ? [...this] :
this[kParent].type.children.reduce((props: any, { name }: Field<T[keyof T]>) => {
return (props[name] = (this as any)[name]) && props || props;
}, {});
public clear(): void { throw new Error(`Clearing ${this[Symbol.toStringTag]} not supported.`); }
public delete(_: K): boolean { throw new Error(`Deleting ${this[Symbol.toStringTag]} values not supported.`); }
public *[Symbol.iterator](): IterableIterator<[K, V]> {
const ki = this.keys();
const vi = this.values();
const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map());
const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size));
for (let k: K, v: V, i = 0, kr: IteratorResult<K>, vr: IteratorResult<V>;
!((kr = ki.next()).done || (vr = vi.next()).done);
++i
) {
k = kr.value;
v = vr.value;
itov[i] = v;
ktoi.has(k) || ktoi.set(k, i);
yield [k, v];
}
}
public inspect() { return this.toString(); }
public [Symbol.for('nodejs.util.inspect.custom')]() { return this.toString(); }
public toString() {
return DataType.isStruct(this[kParent].type) ?
`[ ${[...this].map((x) => valueToString(x)).join(', ')} ]` :
`{ ${
this[kParent].type.children.reduce((xs: string[], { name }: Field<T[keyof T]>) => {
return [...xs, `"${name}": ${valueToString((this as any)[name])}`];
}, []).join(', ')
} }`
;
}
}
/** @ignore */
export class RowProxyGenerator<T extends { [key: string]: DataType }> {
/** @nocollapse */
public static new<T extends { [key: string]: DataType }>(parent: MapVector<T> | StructVector<T>, schemaOrFields: T | Field[], fieldsAreEnumerable = false): RowProxyGenerator<T> {
let schema: T, fields: Field[];
if (Array.isArray(schemaOrFields)) {
fields = schemaOrFields;
} else {
schema = schemaOrFields;
fieldsAreEnumerable = true;
fields = Object.keys(schema).map((x) => new Field(x, schema[x]));
public forEach(callbackfn: (value: V, key: K, map: Map<K, V>) => void, thisArg?: any): void {
const ki = this.keys();
const vi = this.values();
const callback = thisArg === undefined ? callbackfn :
(v: V, k: K, m: Map<K, V>) => callbackfn.call(thisArg, v, k, m);
const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map());
const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size));
for (let k: K, v: V, i = 0, kr: IteratorResult<K>, vr: IteratorResult<V>;
!((kr = ki.next()).done || (vr = vi.next()).done);
++i
) {
k = kr.value;
v = vr.value;
itov[i] = v;
ktoi.has(k) || ktoi.set(k, i);
callback(v, k, this);
}
return new RowProxyGenerator<T>(parent, fields, fieldsAreEnumerable);
}
private rowPrototype: Row<T>;
public toArray() { return [...this.values()]; }
public toJSON() {
const obj = {} as any;
this.forEach((val, key) => obj[key] = val);
return obj;
}
private constructor(parent: MapVector<T> | StructVector<T>, fields: Field[], fieldsAreEnumerable: boolean) {
const proto = Object.create(Row.prototype);
public inspect() { return this.toString(); }
public [kCustomInspect]() { return this.toString(); }
public toString() {
const str: string[] = [];
this.forEach((val, key) => {
key = valueToString(key);
val = valueToString(val);
str.push(`${key}: ${val}`);
});
return `{ ${str.join(', ')} }`;
}
rowParentDescriptor.value = parent;
rowLengthDescriptor.value = fields.length;
Object.defineProperty(proto, kParent, rowParentDescriptor);
Object.defineProperty(proto, kLength, rowLengthDescriptor);
fields.forEach((field, columnIndex) => {
if (!proto.hasOwnProperty(field.name)) {
columnDescriptor.enumerable = fieldsAreEnumerable;
columnDescriptor.get || (columnDescriptor.get = this._bindGetter(columnIndex));
Object.defineProperty(proto, field.name, columnDescriptor);
}
if (!proto.hasOwnProperty(columnIndex)) {
columnDescriptor.enumerable = !fieldsAreEnumerable;
columnDescriptor.get || (columnDescriptor.get = this._bindGetter(columnIndex));
Object.defineProperty(proto, columnIndex, columnDescriptor);
}
columnDescriptor.get = null as any;
protected static [Symbol.toStringTag] = ((proto: Row) => {
Object.defineProperties(proto, {
'size': { writable: true, enumerable: false, configurable: false, value: 0 },
[kParent]: { writable: true, enumerable: false, configurable: false, value: null },
[kRowIndex]: { writable: true, enumerable: false, configurable: false, value: -1 },
});
return (proto as any)[Symbol.toStringTag] = 'Row';
})(Row.prototype);
}
this.rowPrototype = proto;
export class MapRow<K extends DataType = any, V extends DataType = any> extends Row<K['TValue'], V['TValue'] | null> {
constructor(slice: Vector<Struct<{ key: K, value: V }>>) {
super(slice, slice.length);
return createRowProxy(this);
}
public keys() {
return this[kParent].getChildAt(0)![Symbol.iterator]();
}
public values() {
return this[kParent].getChildAt(1)![Symbol.iterator]();
}
public getKey(idx: number): K['TValue'] {
return this[kParent].getChildAt(0)!.get(idx);
}
public getIndex(key: K['TValue']): number {
return this[kParent].getChildAt(0)!.indexOf(key);
}
public getValue(index: number): V['TValue'] | null {
return this[kParent].getChildAt(1)!.get(index);
}
public setValue(index: number, value: V['TValue'] | null): void {
this[kParent].getChildAt(1)!.set(index, value);
}
}
private _bindGetter(columnIndex: number) {
return function(this: Row<T>) {
const child = this[kParent].getChildAt(columnIndex);
return child ? child.get(this[kRowIndex]) : null;
};
export class StructRow<T extends { [key: string]: DataType } = any> extends Row<keyof T, T[keyof T]['TValue'] | null> {
constructor(parent: StructVector<T>) {
super(parent, parent.type.children.length);
return defineRowProxyProperties(this);
}
public bind(rowIndex: number) {
const bound: Row<T> = Object.create(this.rowPrototype);
bound[kRowIndex] = rowIndex;
return bound;
public *keys() {
for (const field of this[kParent].type.children) {
yield field.name as keyof T;
}
}
public *values() {
for (const field of this[kParent].type.children) {
yield (this as RowLike<T>)[field.name];
}
}
public getKey(idx: number): keyof T {
return this[kParent].type.children[idx].name as keyof T;
}
public getIndex(key: keyof T): number {
return this[kParent].type.children.findIndex((f) => f.name === key);
}
public getValue(index: number): T[keyof T]['TValue'] | null {
return this[kParent].getChildAt(index)!.get(this[kRowIndex]);
}
public setValue(index: number, value: T[keyof T]['TValue'] | null): void {
return this[kParent].getChildAt(index)!.set(this[kRowIndex], value);
}
}
Object.setPrototypeOf(Row.prototype, Map.prototype);
/** @ignore */
const defineRowProxyProperties = (() => {
const desc = { enumerable: true, configurable: false, get: null as any, set: null as any };
return <T extends Row>(row: T) => {
let idx = -1, ktoi = row[kKeyToIdx] || (row[kKeyToIdx] = new Map());
const getter = (key: any) => function(this: T) { return this.get(key); };
const setter = (key: any) => function(this: T, val: any) { return this.set(key, val); };
for (const key of row.keys()) {
ktoi.set(key, ++idx);
desc.get = getter(key);
desc.set = setter(key);
row.hasOwnProperty(key) || (desc.enumerable = true, Object.defineProperty(row, key, desc));
row.hasOwnProperty(idx) || (desc.enumerable = false, Object.defineProperty(row, idx, desc));
}
desc.get = desc.set = null;
return row;
};
})();
/** @ignore */
const createRowProxy = (() => {
if (typeof Proxy === 'undefined') {
return defineRowProxyProperties;
}
const has = Row.prototype.has;
const get = Row.prototype.get;
const set = Row.prototype.set;
const getKey = Row.prototype.getKey;
const RowProxyHandler: ProxyHandler<Row> = {
isExtensible() { return false; },
deleteProperty() { return false; },
preventExtensions() { return true; },
ownKeys(row: Row) { return [...row.keys()].map((x) => `${x}`); },
has(row: Row, key: PropertyKey) {
switch (key) {
case 'getKey': case 'getIndex': case 'getValue': case 'setValue': case 'toArray': case 'toJSON': case 'inspect':
case 'constructor': case 'isPrototypeOf': case 'propertyIsEnumerable': case 'toString': case 'toLocaleString': case 'valueOf':
case 'size': case 'has': case 'get': case 'set': case 'clear': case 'delete': case 'keys': case 'values': case 'entries': case 'forEach':
case '__proto__': case '__defineGetter__': case '__defineSetter__': case 'hasOwnProperty': case '__lookupGetter__': case '__lookupSetter__':
case Symbol.iterator: case Symbol.toStringTag: case kParent: case kRowIndex: case kIdxToVal: case kKeyToIdx: case kCustomInspect:
return true;
}
if (typeof key === 'number' && !row.has(key)) {
key = row.getKey(key);
}
return row.has(key);
},
get(row: Row, key: PropertyKey, receiver: any) {
switch (key) {
case 'getKey': case 'getIndex': case 'getValue': case 'setValue': case 'toArray': case 'toJSON': case 'inspect':
case 'constructor': case 'isPrototypeOf': case 'propertyIsEnumerable': case 'toString': case 'toLocaleString': case 'valueOf':
case 'size': case 'has': case 'get': case 'set': case 'clear': case 'delete': case 'keys': case 'values': case 'entries': case 'forEach':
case '__proto__': case '__defineGetter__': case '__defineSetter__': case 'hasOwnProperty': case '__lookupGetter__': case '__lookupSetter__':
case Symbol.iterator: case Symbol.toStringTag: case kParent: case kRowIndex: case kIdxToVal: case kKeyToIdx: case kCustomInspect:
return Reflect.get(row, key, receiver);
}
if (typeof key === 'number' && !has.call(receiver, key)) {
key = getKey.call(receiver, key);
}
return get.call(receiver, key);
},
set(row: Row, key: PropertyKey, val: any, receiver: any) {
switch (key) {
case kParent: case kRowIndex: case kIdxToVal: case kKeyToIdx:
return Reflect.set(row, key, val, receiver);
case 'getKey': case 'getIndex': case 'getValue': case 'setValue': case 'toArray': case 'toJSON': case 'inspect':
case 'constructor': case 'isPrototypeOf': case 'propertyIsEnumerable': case 'toString': case 'toLocaleString': case 'valueOf':
case 'size': case 'has': case 'get': case 'set': case 'clear': case 'delete': case 'keys': case 'values': case 'entries': case 'forEach':
case '__proto__': case '__defineGetter__': case '__defineSetter__': case 'hasOwnProperty': case '__lookupGetter__': case '__lookupSetter__':
case Symbol.iterator: case Symbol.toStringTag:
return false;
}
if (typeof key === 'number' && !has.call(receiver, key)) {
key = getKey.call(receiver, key);
}
return has.call(receiver, key) ? !!set.call(receiver, key, val) : false;
},
};
return <T extends Row>(row: T) => new Proxy(row, RowProxyHandler) as T;
})();

19

vector/struct.ts

@@ -18,18 +18,17 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Field } from '../schema';
import { Vector } from '../vector';
import { StructRow } from './row';
import { BaseVector } from './base';
import { RowProxyGenerator } from './row';
import { DataType, Map_, Struct } from '../type';
import { DataType, Struct } from '../type';
/** @ignore */ const kRowIndex = Symbol.for('rowIndex');
/** @ignore */
export class StructVector<T extends { [key: string]: DataType } = any> extends BaseVector<Struct<T>> {
public asMap(keysSorted: boolean = false) {
return Vector.new(this.data.clone(new Map_<T>(this.type.children as Field<T[keyof T]>[], keysSorted)));
}
// @ts-ignore
private _rowProxy: RowProxyGenerator<T>;
public get rowProxy(): RowProxyGenerator<T> {
return this._rowProxy || (this._rowProxy = RowProxyGenerator.new<T>(this, this.type.children || [], false));
private _row: StructRow<T>;
public bind(index: number): Struct<T>['TValue'] {
const proto = this._row || (this._row = new StructRow<T>(this));
const bound = Object.create(proto);
bound[kRowIndex] = index;
return bound;
}
}

21

visitor/get.ts

@@ -21,4 +21,5 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Visitor } from '../visitor';
import { decodeUtf8 } from '../util/utf8';
import { VectorType } from '../interfaces';
import { decodeUtf8 } from '../util/utf8';
import { uint16ToFloat64 } from '../util/math';
import { Type, UnionMode, Precision, DateUnit, TimeUnit, IntervalUnit } from '../enum';

@@ -127,3 +128,3 @@ import {

/** @ignore */
const getFloat16 = <T extends Float16> ({ stride, values }: VectorType<T>, index: number): T['TValue'] => (values[stride * index] - 32767) / 32767;
const getFloat16 = <T extends Float16> ({ stride, values }: VectorType<T>, index: number): T['TValue'] => uint16ToFloat64(values[stride * index]);
/** @ignore */

@@ -214,9 +215,11 @@ const getBigInts = <T extends Numeric2X>({ stride, values, type }: VectorType<T>, index: number): T['TValue'] => <any> BN.new(values.subarray(stride * index, stride * (index + 1)), type.isSigned);

/** @ignore */
const getNested = <
S extends { [key: string]: DataType },
V extends VectorType<Map_<S>> | VectorType<Struct<S>>
>(vector: V, index: number): V['TValue'] => {
return vector.rowProxy.bind(index) as V['TValue'];
const getMap = <T extends Map_>(vector: VectorType<T>, index: number): T['TValue'] => {
return vector.bind(index) as T['TValue'];
};
/** @ignore */
const getStruct = <T extends Struct>(vector: VectorType<T>, index: number): T['TValue'] => {
return vector.bind(index) as T['TValue'];
};
/* istanbul ignore next */

@@ -309,3 +312,3 @@ /** @ignore */

GetVisitor.prototype.visitList = getList;
GetVisitor.prototype.visitStruct = getNested;
GetVisitor.prototype.visitStruct = getStruct;
GetVisitor.prototype.visitUnion = getUnion;

@@ -319,5 +322,5 @@ GetVisitor.prototype.visitDenseUnion = getDenseUnion;

GetVisitor.prototype.visitFixedSizeList = getFixedSizeList;
GetVisitor.prototype.visitMap = getNested;
GetVisitor.prototype.visitMap = getMap;
/** @ignore */
export const instance = new GetVisitor();

7

visitor/jsonvectorassembler.ts

@@ -76,4 +76,4 @@ // Licensed to the Apache Software Foundation (ASF) under one

'count': length,
'VALIDITY': nullCount <= 0
? Array.from({ length }, () => 1)
'VALIDITY': DataType.isNull(type) ? undefined
: nullCount <= 0 ? Array.from({ length }, () => 1)
: [...iterateBits(nullBitmap, offset, length, null, getBit)],

@@ -83,3 +83,3 @@ ...super.visit(Vector.new(data.clone(type, offset, length, 0, buffers)))

}
public visitNull() { return { 'DATA': [] }; }
public visitNull() { return {}; }
public visitBool<T extends Bool>({ values, offset, length }: V<T>) {

@@ -158,2 +158,3 @@ return { 'DATA': [...iterateBits(values, offset, length, null, getBool)] };

return {
'OFFSET': [...vector.valueOffsets],
'children': vector.type.children.map((f, i) =>

@@ -160,0 +161,0 @@ this.visit(new Column(f, [vector.getChildAt(i)!])))

50

visitor/set.ts

@@ -19,5 +19,8 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Data } from '../data';
import { Field } from '../schema';
import { Vector } from '../vector';
import { Visitor } from '../visitor';
import { encodeUtf8 } from '../util/utf8';
import { VectorType } from '../interfaces';
import { encodeUtf8 } from '../util/utf8';
import { float64ToUint16 } from '../util/math';
import { toArrayBufferView } from '../util/buffer';

@@ -135,3 +138,3 @@ import { Type, UnionMode, Precision, DateUnit, TimeUnit, IntervalUnit } from '../enum';

/** @ignore */
const setFloat16 = <T extends Float16> ({ stride, values }: VectorType<T>, index: number, value: T['TValue']): void => { values[stride * index] = (value * 32767) + 32767; };
const setFloat16 = <T extends Float16> ({ stride, values }: VectorType<T>, index: number, value: T['TValue']): void => { values[stride * index] = float64ToUint16(value); };
/** @ignore */

@@ -225,8 +228,5 @@ const setNumericX2 = <T extends Numeric2X> (vector: VectorType<T>, index: number, value: T['TValue']): void => {

const setList = <T extends List>(vector: VectorType<T>, index: number, value: T['TValue']): void => {
const values = vector.getChildAt(0)!;
const { valueOffsets } = vector;
let idx = -1, offset = valueOffsets[index];
let end = Math.min(offset + value.length, valueOffsets[index + 1]);
while (offset < end) {
values.set(offset++, value.get(++idx));
const values = vector.getChildAt(0)!, valueOffsets = vector.valueOffsets;
for (let idx = -1, itr = valueOffsets[index], end = valueOffsets[index + 1]; itr < end;) {
values.set(itr++, value.get(++idx));
}

@@ -236,21 +236,23 @@ };

/** @ignore */
const setStruct = <
S extends { [key: string]: DataType },
V extends VectorType<Map_<S>> | VectorType<Struct<S>>
>(vector: V, index: number, value: V['TValue']) => {
vector.type.children.forEach((_field, idx) => {
const child = vector.getChildAt(idx);
child && child.set(index, value[idx]);
});
const setMap = <T extends Map_>(vector: VectorType<T>, index: number, value: T['TValue']) => {
const values = vector.getChildAt(0)!, valueOffsets = vector.valueOffsets;
const entries = value instanceof Map ? [...value] : Object.entries(value);
for (let idx = -1, itr = valueOffsets[index], end = valueOffsets[index + 1]; itr < end;) {
values.set(itr++, entries[++idx]);
}
};
/** @ignore */ const _setStructArrayValue = (o: number, v: any[]) => (c: Vector | null, _: Field, i: number) => c && c.set(o, v[i]);
/** @ignore */ const _setStructVectorValue = (o: number, v: Vector) => (c: Vector | null, _: Field, i: number) => c && c.set(o, v.get(i));
/** @ignore */ const _setStructMapValue = (o: number, v: Map<string, any>) => (c: Vector | null, f: Field, _: number) => c && c.set(o, v.get(f.name));
/** @ignore */ const _setStructObjectValue = (o: number, v: { [key: string]: any }) => (c: Vector | null, f: Field, _: number) => c && c.set(o, v[f.name]);
/** @ignore */
const setMap = <
S extends { [key: string]: DataType },
V extends VectorType<Map_<S>> | VectorType<Struct<S>>
>(vector: V, index: number, value: V['TValue']) => {
vector.type.children.forEach(({ name }, idx) => {
const child = vector.getChildAt(idx);
child && child.set(index, value[name]);
});
const setStruct = <T extends Struct>(vector: VectorType<T>, index: number, value: T['TValue']) => {
const setValue = value instanceof Map ? _setStructMapValue(index, value) :
value instanceof Vector ? _setStructVectorValue(index, value) :
Array.isArray(value) ? _setStructArrayValue(index, value) :
_setStructObjectValue(index, value) ;
vector.type.children.forEach((f: Field, i: number) => setValue(vector.getChildAt(i), f, i));
};

@@ -257,0 +259,0 @@

21

visitor/vectorassembler.ts

@@ -79,6 +79,9 @@ // Licensed to the Apache Software Foundation (ASF) under one

}
addBuffer.call(this, nullCount <= 0
? new Uint8Array(0) // placeholder validity buffer
: truncateBitmap(data.offset, length, data.nullBitmap)
).nodes.push(new FieldNode(length, nullCount));
if (!DataType.isNull(vector.type)) {
addBuffer.call(this, nullCount <= 0
? new Uint8Array(0) // placeholder validity buffer
: truncateBitmap(data.offset, length, data.nullBitmap)
);
}
this.nodes.push(new FieldNode(length, nullCount));
}

@@ -89,3 +92,3 @@ return super.visit(vector);

public visitNull<T extends Null>(_nullV: V<T>) {
return addBuffer.call(this, new Uint8Array(0));
return this;
}

@@ -203,5 +206,5 @@ public visitDictionary<T extends Dictionary>(vector: V<T>) {

/** @ignore */
function assembleListVector<T extends List | FixedSizeList>(this: VectorAssembler, vector: V<T>) {
function assembleListVector<T extends Map_ | List | FixedSizeList>(this: VectorAssembler, vector: V<T>) {
const { length, valueOffsets } = vector;
// If we have valueOffsets (ListVector), push that buffer first
// If we have valueOffsets (MapVector, ListVector), push that buffer first
if (valueOffsets) {

@@ -215,3 +218,3 @@ addBuffer.call(this, rebaseValueOffsets(valueOffsets[0], length, valueOffsets));

/** @ignore */
function assembleNestedVector<T extends Struct | Map_ | Union>(this: VectorAssembler, vector: V<T>) {
function assembleNestedVector<T extends Struct | Union>(this: VectorAssembler, vector: V<T>) {
return this.visitMany(vector.type.children.map((_, i) => vector.getChildAt(i)!).filter(Boolean))[0];

@@ -235,2 +238,2 @@ }

VectorAssembler.prototype.visitFixedSizeList = assembleListVector;
VectorAssembler.prototype.visitMap = assembleNestedVector;
VectorAssembler.prototype.visitMap = assembleListVector;

4

visitor/vectorloader.ts

@@ -57,3 +57,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

public visitNull <T extends type.Null> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Null(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); }
public visitNull <T extends type.Null> (type: T, { length, } = this.nextFieldNode()) { return Data.Null(type, 0, length); }
public visitBool <T extends type.Bool> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Bool(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); }

@@ -77,3 +77,3 @@ public visitInt <T extends type.Int> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Int(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); }

public visitFixedSizeList <T extends type.FixedSizeList> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.FixedSizeList(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.visit(type.children[0])); }
public visitMap <T extends type.Map_> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Map(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.visitMany(type.children)); }
public visitMap <T extends type.Map_> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Map(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.visit(type.children[0])); }

@@ -80,0 +80,0 @@ protected nextFieldNode() { return this.nodes[++this.nodesIndex]; }

fb/Schema.ts

Sorry, the diff of this file is too big to display

LICENSE.txt

Sorry, the diff of this file is too big to display

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc