Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

@apache-arrow/esnext-esm

Package Overview
Dependencies
Maintainers
5
Versions
45
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@apache-arrow/esnext-esm - npm Package Compare versions

Comparing version 0.14.1 to 0.15.0

util/math.d.ts
util/math.js
util/math.mjs.map

5

Arrow.d.ts

@@ -9,3 +9,3 @@ /// <reference types="flatbuffers" />

export { Schema, Field } from './schema';
export { Row, Vector, BaseVector, BinaryVector, BoolVector, Chunked, DateVector, DateDayVector, DateMillisecondVector, DecimalVector, DictionaryVector, FixedSizeBinaryVector, FixedSizeListVector, FloatVector, Float16Vector, Float32Vector, Float64Vector, IntervalVector, IntervalDayTimeVector, IntervalYearMonthVector, IntVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, ListVector, MapVector, NullVector, StructVector, TimestampVector, TimestampSecondVector, TimestampMillisecondVector, TimestampMicrosecondVector, TimestampNanosecondVector, TimeVector, TimeSecondVector, TimeMillisecondVector, TimeMicrosecondVector, TimeNanosecondVector, UnionVector, DenseUnionVector, SparseUnionVector, Utf8Vector, } from './vector/index';
export { Vector, BaseVector, BinaryVector, BoolVector, Chunked, DateVector, DateDayVector, DateMillisecondVector, DecimalVector, DictionaryVector, FixedSizeBinaryVector, FixedSizeListVector, FloatVector, Float16Vector, Float32Vector, Float64Vector, IntervalVector, IntervalDayTimeVector, IntervalYearMonthVector, IntVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, ListVector, MapVector, NullVector, StructVector, TimestampVector, TimestampSecondVector, TimestampMillisecondVector, TimestampMicrosecondVector, TimestampNanosecondVector, TimeVector, TimeSecondVector, TimeMillisecondVector, TimeMicrosecondVector, TimeNanosecondVector, UnionVector, DenseUnionVector, SparseUnionVector, Utf8Vector, } from './vector/index';
export { Builder, BinaryBuilder, BoolBuilder, DateBuilder, DateDayBuilder, DateMillisecondBuilder, DecimalBuilder, DictionaryBuilder, FixedSizeBinaryBuilder, FixedSizeListBuilder, FloatBuilder, Float16Builder, Float32Builder, Float64Builder, IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder, IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder, ListBuilder, MapBuilder, NullBuilder, StructBuilder, TimestampBuilder, TimestampSecondBuilder, TimestampMillisecondBuilder, TimestampMicrosecondBuilder, TimestampNanosecondBuilder, TimeBuilder, TimeSecondBuilder, TimeMillisecondBuilder, TimeMicrosecondBuilder, TimeNanosecondBuilder, UnionBuilder, DenseUnionBuilder, SparseUnionBuilder, Utf8Builder, } from './builder/index';

@@ -63,3 +63,2 @@ export { ByteStream, AsyncByteStream, AsyncByteQueue, ReadableSource, WritableSink } from './io/stream';

toUint8ClampedArray: (input: util_buffer_.ArrayBufferViewInput) => Uint8ClampedArray;
toFloat16Array: (input: util_buffer_.ArrayBufferViewInput) => Uint16Array;
toInt8ArrayIterator: (input: string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView | ArrayLike<number> | Iterable<number> | flatbuffers.ByteBuffer | IteratorResult<string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView | ArrayLike<number> | Iterable<number> | flatbuffers.ByteBuffer | null | undefined> | ReadableStreamReadResult<string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView | ArrayLike<number> | Iterable<number> | flatbuffers.ByteBuffer | null | undefined> | Iterable<util_buffer_.ArrayBufferViewInput> | null | undefined) => IterableIterator<Int8Array>;

@@ -83,2 +82,4 @@ toInt16ArrayIterator: (input: string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView | ArrayLike<number> | Iterable<number> | flatbuffers.ByteBuffer | IteratorResult<string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView | ArrayLike<number> | Iterable<number> | flatbuffers.ByteBuffer | null | undefined> | ReadableStreamReadResult<string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView | ArrayLike<number> | Iterable<number> | flatbuffers.ByteBuffer | null | undefined> | Iterable<util_buffer_.ArrayBufferViewInput> | null | undefined) => IterableIterator<Int16Array>;

toUint8ClampedArrayAsyncIterator: (input: string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView | ArrayLike<number> | Iterable<number> | flatbuffers.ByteBuffer | IteratorResult<string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView | ArrayLike<number> | Iterable<number> | flatbuffers.ByteBuffer | null | undefined> | ReadableStreamReadResult<string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView | ArrayLike<number> | Iterable<number> | flatbuffers.ByteBuffer | null | undefined> | Iterable<util_buffer_.ArrayBufferViewInput> | PromiseLike<util_buffer_.ArrayBufferViewInput> | AsyncIterable<util_buffer_.ArrayBufferViewInput> | null | undefined) => AsyncIterableIterator<Uint8ClampedArray>;
uint16ToFloat64(h: number): number;
float64ToUint16(d: number): number;
getBool(_data: any, _index: number, byte: number, bit: number): boolean;

@@ -85,0 +86,0 @@ getBit(_data: any, _index: number, byte: number, bit: number): 0 | 1;

4

Arrow.js

@@ -24,3 +24,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

export { Schema, Field } from './schema';
export { Row, Vector, BaseVector, BinaryVector, BoolVector, Chunked, DateVector, DateDayVector, DateMillisecondVector, DecimalVector, DictionaryVector, FixedSizeBinaryVector, FixedSizeListVector, FloatVector, Float16Vector, Float32Vector, Float64Vector, IntervalVector, IntervalDayTimeVector, IntervalYearMonthVector, IntVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, ListVector, MapVector, NullVector, StructVector, TimestampVector, TimestampSecondVector, TimestampMillisecondVector, TimestampMicrosecondVector, TimestampNanosecondVector, TimeVector, TimeSecondVector, TimeMillisecondVector, TimeMicrosecondVector, TimeNanosecondVector, UnionVector, DenseUnionVector, SparseUnionVector, Utf8Vector, } from './vector/index';
export { Vector, BaseVector, BinaryVector, BoolVector, Chunked, DateVector, DateDayVector, DateMillisecondVector, DecimalVector, DictionaryVector, FixedSizeBinaryVector, FixedSizeListVector, FloatVector, Float16Vector, Float32Vector, Float64Vector, IntervalVector, IntervalDayTimeVector, IntervalYearMonthVector, IntVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, ListVector, MapVector, NullVector, StructVector, TimestampVector, TimestampSecondVector, TimestampMillisecondVector, TimestampMicrosecondVector, TimestampNanosecondVector, TimeVector, TimeSecondVector, TimeMillisecondVector, TimeMicrosecondVector, TimeNanosecondVector, UnionVector, DenseUnionVector, SparseUnionVector, Utf8Vector, } from './vector/index';
export { Builder, BinaryBuilder, BoolBuilder, DateBuilder, DateDayBuilder, DateMillisecondBuilder, DecimalBuilder, DictionaryBuilder, FixedSizeBinaryBuilder, FixedSizeListBuilder, FloatBuilder, Float16Builder, Float32Builder, Float64Builder, IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder, IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder, ListBuilder, MapBuilder, NullBuilder, StructBuilder, TimestampBuilder, TimestampSecondBuilder, TimestampMillisecondBuilder, TimestampMicrosecondBuilder, TimestampNanosecondBuilder, TimeBuilder, TimeSecondBuilder, TimeMillisecondBuilder, TimeMicrosecondBuilder, TimeNanosecondBuilder, UnionBuilder, DenseUnionBuilder, SparseUnionBuilder, Utf8Builder, } from './builder/index';

@@ -37,2 +37,3 @@ export { ByteStream, AsyncByteStream, AsyncByteQueue } from './io/stream';

import * as util_bit_ from './util/bit';
import * as util_math_ from './util/math';
import * as util_buffer_ from './util/buffer';

@@ -47,2 +48,3 @@ import * as util_vector_ from './util/vector';

...util_bit_,
...util_math_,
...util_buffer_,

@@ -49,0 +51,0 @@ ...util_vector_

2

bin/arrow2csv.js

@@ -161,3 +161,3 @@ #! /usr/bin/env node

}
this.push(`${formatRow([rowId, ...row].map(valueToString), maxColWidths, sep)}\n`);
this.push(`${formatRow([rowId, ...row.toArray()].map(valueToString), maxColWidths, sep)}\n`);
}

@@ -164,0 +164,0 @@ }

4

builder.d.ts
import { Data } from './data';
import { BuilderType as B, VectorType as V } from './interfaces';
import { BufferBuilder, BitmapBufferBuilder, DataBufferBuilder, OffsetsBufferBuilder } from './builder/buffer';
import { DataType, Float, Int, Decimal, FixedSizeBinary, Date_, Time, Timestamp, Interval, Utf8, Binary, List } from './type';
import { DataType, Float, Int, Decimal, FixedSizeBinary, Date_, Time, Timestamp, Interval, Utf8, Binary, List, Map_ } from './type';
/**

@@ -279,3 +279,3 @@ * A set of options required to create a `Builder` instance for a given `DataType`.

/** @ignore */
export declare abstract class VariableWidthBuilder<T extends Binary | Utf8 | List, TNull = any> extends Builder<T, TNull> {
export declare abstract class VariableWidthBuilder<T extends Binary | Utf8 | List | Map_, TNull = any> extends Builder<T, TNull> {
protected _pendingLength: number;

@@ -282,0 +282,0 @@ protected _offsets: OffsetsBufferBuilder;

2

builder.js

@@ -22,3 +22,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { BitmapBufferBuilder, DataBufferBuilder, OffsetsBufferBuilder } from './builder/buffer';
import { strideForType, } from './type';
import { strideForType } from './type';
/**

@@ -25,0 +25,0 @@ * An abstract base class for types that construct Arrow Vectors from arbitrary JavaScript values.

1

builder/float.d.ts

@@ -8,2 +8,3 @@ import { FixedWidthBuilder } from '../builder';

export declare class Float16Builder<TNull = any> extends FloatBuilder<Float16, TNull> {
setValue(index: number, value: number): void;
}

@@ -10,0 +11,0 @@ /** @ignore */

5

builder/float.js

@@ -17,2 +17,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

// under the License.
import { float64ToUint16 } from '../util/math';
import { FixedWidthBuilder } from '../builder';

@@ -24,2 +25,6 @@ /** @ignore */

export class Float16Builder extends FloatBuilder {
setValue(index, value) {
// convert JS float64 to a uint16
this._values.set(index, float64ToUint16(value));
}
}

@@ -26,0 +31,0 @@ /** @ignore */

25

builder/map.d.ts

@@ -1,8 +0,21 @@

import { Builder } from '../builder';
import { DataType, Map_ } from '../type';
import { DataType, Map_, Struct } from '../type';
import { Builder, VariableWidthBuilder } from '../builder';
/** @ignore */ declare type MapValue<K extends DataType = any, V extends DataType = any> = Map_<K, V>['TValue'];
/** @ignore */ declare type MapValues<K extends DataType = any, V extends DataType = any> = Map<number, MapValue<K, V> | undefined>;
/** @ignore */ declare type MapValueExt<K extends DataType = any, V extends DataType = any> = MapValue<K, V> | {
[key: string]: V;
} | {
[key: number]: V;
};
/** @ignore */
export declare class MapBuilder<T extends {
[key: string]: DataType;
} = any, TNull = any> extends Builder<Map_<T>, TNull> {
addChild(child: Builder, name?: string): number;
export declare class MapBuilder<K extends DataType = any, V extends DataType = any, TNull = any> extends VariableWidthBuilder<Map_<K, V>, TNull> {
protected _pending: MapValues<K, V> | undefined;
set(index: number, value: MapValueExt<K, V> | TNull): this;
setValue(index: number, value: MapValueExt<K, V>): void;
addChild(child: Builder<Struct<{
key: K;
value: V;
}>>, name?: string): number;
protected _flushPending(pending: MapValues<K, V>): void;
}
export {};

38

builder/map.js

@@ -18,14 +18,40 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Field } from '../schema';
import { Builder } from '../builder';
import { Map_ } from '../type';
import { VariableWidthBuilder } from '../builder';
/** @ignore */
export class MapBuilder extends Builder {
export class MapBuilder extends VariableWidthBuilder {
set(index, value) {
return super.set(index, value);
}
setValue(index, value) {
value = value instanceof Map ? value : new Map(Object.entries(value));
const pending = this._pending || (this._pending = new Map());
const current = pending.get(index);
current && (this._pendingLength -= current.size);
this._pendingLength += value.size;
pending.set(index, value);
}
addChild(child, name = `${this.numChildren}`) {
const { children, keysSorted } = this.type;
const childIndex = this.children.push(child);
this.type = new Map_([...children, new Field(name, child.type, true)], keysSorted);
return childIndex;
if (this.numChildren > 0) {
throw new Error('ListBuilder can only have one child.');
}
this.children[this.numChildren] = child;
this.type = new Map_(new Field(name, child.type, true), this.type.keysSorted);
return this.numChildren - 1;
}
_flushPending(pending) {
const offsets = this._offsets;
const setValue = this._setValue;
pending.forEach((value, index) => {
if (value === undefined) {
offsets.set(index, 0);
}
else {
offsets.set(index, value.size);
setValue(this, index, value);
}
});
}
}
//# sourceMappingURL=map.mjs.map

1

builder/null.d.ts

@@ -6,2 +6,3 @@ import { Null } from '../type';

setValue(index: number, value: null): void;
setValid(index: number, valid: boolean): boolean;
}

4

builder/null.js

@@ -22,4 +22,8 @@ // Licensed to the Apache Software Foundation (ASF) under one

setValue(index, value) { }
setValid(index, valid) {
this.length = Math.max(index + 1, this.length);
return valid;
}
}
//# sourceMappingURL=null.mjs.map

2

compute/dataframe.d.ts

@@ -16,2 +16,3 @@ import { Table } from '../table';

scan(next: NextFunc, bind?: BindFunc): void;
scanReverse(next: NextFunc, bind?: BindFunc): void;
countBy(name: Col | string): CountByResult<any, Int<import("../enum").Type.Int | import("../enum").Type.Int8 | import("../enum").Type.Int16 | import("../enum").Type.Int32 | import("../enum").Type.Int64 | import("../enum").Type.Uint8 | import("../enum").Type.Uint16 | import("../enum").Type.Uint32 | import("../enum").Type.Uint64>>;

@@ -34,2 +35,3 @@ }

scan(next: NextFunc, bind?: BindFunc): void;
scanReverse(next: NextFunc, bind?: BindFunc): void;
count(): number;

@@ -36,0 +38,0 @@ [Symbol.iterator](): IterableIterator<Struct<T>['TValue']>;

36

compute/dataframe.js

@@ -25,2 +25,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

Table.prototype.scan = function (next, bind) { return new DataFrame(this.chunks).scan(next, bind); };
Table.prototype.scanReverse = function (next, bind) { return new DataFrame(this.chunks).scanReverse(next, bind); };
Table.prototype.filter = function (predicate) { return new DataFrame(this.chunks).filter(predicate); };

@@ -45,2 +46,16 @@ export class DataFrame extends Table {

}
scanReverse(next, bind) {
const batches = this.chunks, numBatches = batches.length;
for (let batchIndex = numBatches; --batchIndex >= 0;) {
// load batches
const batch = batches[batchIndex];
if (bind) {
bind(batch);
}
// yield all indices
for (let index = batch.length; --index >= 0;) {
next(index, batch);
}
}
}
countBy(name) {

@@ -127,2 +142,23 @@ const batches = this.chunks, numBatches = batches.length;

}
scanReverse(next, bind) {
const batches = this._chunks;
const numBatches = batches.length;
for (let batchIndex = numBatches; --batchIndex >= 0;) {
// load batches
const batch = batches[batchIndex];
// TODO: bind batches lazily
// If predicate doesn't match anything in the batch we don't need
// to bind the callback
if (bind) {
bind(batch);
}
const predicate = this._predicate.bind(batch);
// yield all indices
for (let index = batch.length; --index >= 0;) {
if (predicate(index, batch)) {
next(index, batch);
}
}
}
}
count() {

@@ -129,0 +165,0 @@ // inlined version of this:

6

data.d.ts

@@ -54,3 +54,3 @@ import { Vector } from './vector';

/** @nocollapse */
static Null<T extends Null>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, _data?: NullBuffer): Data<T>;
static Null<T extends Null>(type: T, offset: number, length: number): Data<T>;
/** @nocollapse */

@@ -83,7 +83,7 @@ static Int<T extends Int>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>): Data<T>;

/** @nocollapse */
static FixedSizeList<T extends FixedSizeList>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, child: Data | Vector): Data<T>;
static FixedSizeList<T extends FixedSizeList>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, child: Data<T['valueType']> | Vector<T['valueType']>): Data<T>;
/** @nocollapse */
static Struct<T extends Struct>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, children: (Data | Vector)[]): Data<T>;
/** @nocollapse */
static Map<T extends Map_>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, children: (Data | Vector)[]): Data<T>;
static Map<T extends Map_>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, child: (Data | Vector)): Data<T>;
static Union<T extends SparseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, children: (Data | Vector)[], _?: any): Data<T>;

@@ -90,0 +90,0 @@ static Union<T extends DenseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, valueOffsets: ValueOffsetsBuffer, children: (Data | Vector)[]): Data<T>;

15

data.js

@@ -88,2 +88,5 @@ // Licensed to the Apache Software Foundation (ASF) under one

_changeLengthAndBackfillNullBitmap(newLength) {
if (this.typeId === Type.Null) {
return this.clone(this.type, 0, newLength, 0);
}
const { length, nullCount } = this;

@@ -127,3 +130,3 @@ // start initialized with 0s (nulls), then fill from 0 to length with 1s (not null)

switch (type.typeId) {
case Type.Null: return Data.Null(type, offset, length, nullCount || 0, buffers[BufferType.VALIDITY]);
case Type.Null: return Data.Null(type, offset, length);
case Type.Int: return Data.Int(type, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []);

@@ -144,3 +147,3 @@ case Type.Dictionary: return Data.Dictionary(type, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary);

case Type.Struct: return Data.Struct(type, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], childData || []);
case Type.Map: return Data.Map(type, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], childData || []);
case Type.Map: return Data.Map(type, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.OFFSET] || [], (childData || [])[0]);
case Type.Union: return Data.Union(type, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.TYPE] || [], buffers[BufferType.OFFSET] || childData, childData);

@@ -151,4 +154,4 @@ }

/** @nocollapse */
static Null(type, offset, length, nullCount, nullBitmap, _data) {
return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)]);
static Null(type, offset, length) {
return new Data(type, offset, length, 0);
}

@@ -216,4 +219,4 @@ /** @nocollapse */

/** @nocollapse */
static Map(type, offset, length, nullCount, nullBitmap, children) {
return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)], children);
static Map(type, offset, length, nullCount, nullBitmap, valueOffsets, child) {
return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), undefined, toUint8Array(nullBitmap)], [child]);
}

@@ -220,0 +223,0 @@ /** @nocollapse */

11

fb/File.d.ts

@@ -27,8 +27,2 @@ import { flatbuffers } from 'flatbuffers';

/**
* @param flatbuffers.ByteBuffer bb
* @param Footer= obj
* @returns Footer
*/
static getSizePrefixedRootAsFooter(bb: flatbuffers.ByteBuffer, obj?: Footer): Footer;
/**
* @returns org.apache.arrow.flatbuf.MetadataVersion

@@ -106,7 +100,2 @@ */

static finishFooterBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset): void;
/**
* @param flatbuffers.Builder builder
* @param flatbuffers.Offset offset
*/
static finishSizePrefixedFooterBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset): void;
static createFooter(builder: flatbuffers.Builder, version: NS7624605610262437867.org.apache.arrow.flatbuf.MetadataVersion, schemaOffset: flatbuffers.Offset, dictionariesOffset: flatbuffers.Offset, recordBatchesOffset: flatbuffers.Offset): flatbuffers.Offset;

@@ -113,0 +102,0 @@ }

15

fb/File.js

@@ -42,10 +42,2 @@ // automatically generated by the FlatBuffers compiler, do not modify

/**
* @param flatbuffers.ByteBuffer bb
* @param Footer= obj
* @returns Footer
*/
static getSizePrefixedRootAsFooter(bb, obj) {
return (obj || new Footer).__init(bb.readInt32(bb.position()) + bb.position(), bb);
}
/**
* @returns org.apache.arrow.flatbuf.MetadataVersion

@@ -160,9 +152,2 @@ */

}
/**
* @param flatbuffers.Builder builder
* @param flatbuffers.Offset offset
*/
static finishSizePrefixedFooterBuffer(builder, offset) {
builder.finish(offset, undefined);
}
static createFooter(builder, version, schemaOffset, dictionariesOffset, recordBatchesOffset) {

@@ -169,0 +154,0 @@ Footer.startFooter(builder);

23

fb/Message.d.ts

@@ -99,8 +99,2 @@ import { flatbuffers } from 'flatbuffers';

/**
* @param flatbuffers.ByteBuffer bb
* @param RecordBatch= obj
* @returns RecordBatch
*/
static getSizePrefixedRootAsRecordBatch(bb: flatbuffers.ByteBuffer, obj?: RecordBatch): RecordBatch;
/**
* number of records / rows. The arrays in the batch should all have this

@@ -205,8 +199,2 @@ * length

/**
* @param flatbuffers.ByteBuffer bb
* @param DictionaryBatch= obj
* @returns DictionaryBatch
*/
static getSizePrefixedRootAsDictionaryBatch(bb: flatbuffers.ByteBuffer, obj?: DictionaryBatch): DictionaryBatch;
/**
* @returns flatbuffers.Long

@@ -274,8 +262,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Message= obj
* @returns Message
*/
static getSizePrefixedRootAsMessage(bb: flatbuffers.ByteBuffer, obj?: Message): Message;
/**
* @returns org.apache.arrow.flatbuf.MetadataVersion

@@ -357,9 +339,4 @@ */

static finishMessageBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset): void;
/**
* @param flatbuffers.Builder builder
* @param flatbuffers.Offset offset
*/
static finishSizePrefixedMessageBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset): void;
static createMessage(builder: flatbuffers.Builder, version: NS7624605610262437867.org.apache.arrow.flatbuf.MetadataVersion, headerType: org.apache.arrow.flatbuf.MessageHeader, headerOffset: flatbuffers.Offset, bodyLength: flatbuffers.Long, customMetadataOffset: flatbuffers.Offset): flatbuffers.Offset;
}
}

31

fb/Message.js

@@ -158,10 +158,2 @@ // automatically generated by the FlatBuffers compiler, do not modify

/**
* @param flatbuffers.ByteBuffer bb
* @param RecordBatch= obj
* @returns RecordBatch
*/
static getSizePrefixedRootAsRecordBatch(bb, obj) {
return (obj || new RecordBatch).__init(bb.readInt32(bb.position()) + bb.position(), bb);
}
/**
* number of records / rows. The arrays in the batch should all have this

@@ -320,10 +312,2 @@ * length

/**
* @param flatbuffers.ByteBuffer bb
* @param DictionaryBatch= obj
* @returns DictionaryBatch
*/
static getSizePrefixedRootAsDictionaryBatch(bb, obj) {
return (obj || new DictionaryBatch).__init(bb.readInt32(bb.position()) + bb.position(), bb);
}
/**
* @returns flatbuffers.Long

@@ -435,10 +419,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Message= obj
* @returns Message
*/
static getSizePrefixedRootAsMessage(bb, obj) {
return (obj || new Message).__init(bb.readInt32(bb.position()) + bb.position(), bb);
}
/**
* @returns org.apache.arrow.flatbuf.MetadataVersion

@@ -563,9 +539,2 @@ */

}
/**
* @param flatbuffers.Builder builder
* @param flatbuffers.Offset offset
*/
static finishSizePrefixedMessageBuffer(builder, offset) {
builder.finish(offset, undefined);
}
static createMessage(builder, version, headerType, headerOffset, bodyLength, customMetadataOffset) {

@@ -572,0 +541,0 @@ Message.startMessage(builder);

246

fb/Schema.d.ts

@@ -100,3 +100,6 @@ import { flatbuffers } from 'flatbuffers';

Map = 17,
Duration = 18
Duration = 18,
LargeBinary = 19,
LargeUtf8 = 20,
LargeList = 21
}

@@ -138,8 +141,2 @@ }

/**
* @param flatbuffers.ByteBuffer bb
* @param Null= obj
* @returns Null
*/
static getSizePrefixedRootAsNull(bb: flatbuffers.ByteBuffer, obj?: Null): Null;
/**
* @param flatbuffers.Builder builder

@@ -180,8 +177,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Struct_= obj
* @returns Struct_
*/
static getSizePrefixedRootAsStruct_(bb: flatbuffers.ByteBuffer, obj?: Struct_): Struct_;
/**
* @param flatbuffers.Builder builder

@@ -218,8 +209,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param List= obj
* @returns List
*/
static getSizePrefixedRootAsList(bb: flatbuffers.ByteBuffer, obj?: List): List;
/**
* @param flatbuffers.Builder builder

@@ -237,5 +222,39 @@ */

/**
* Same as List, but with 64-bit offsets, allowing to represent
* extremely large data values.
*
* @constructor
*/
export declare namespace org.apache.arrow.flatbuf {
class LargeList {
bb: flatbuffers.ByteBuffer | null;
bb_pos: number;
/**
* @param number i
* @param flatbuffers.ByteBuffer bb
* @returns LargeList
*/
__init(i: number, bb: flatbuffers.ByteBuffer): LargeList;
/**
* @param flatbuffers.ByteBuffer bb
* @param LargeList= obj
* @returns LargeList
*/
static getRootAsLargeList(bb: flatbuffers.ByteBuffer, obj?: LargeList): LargeList;
/**
* @param flatbuffers.Builder builder
*/
static startLargeList(builder: flatbuffers.Builder): void;
/**
* @param flatbuffers.Builder builder
* @returns flatbuffers.Offset
*/
static endLargeList(builder: flatbuffers.Builder): flatbuffers.Offset;
static createLargeList(builder: flatbuffers.Builder): flatbuffers.Offset;
}
}
/**
* @constructor
*/
export declare namespace org.apache.arrow.flatbuf {
class FixedSizeList {

@@ -257,8 +276,2 @@ bb: flatbuffers.ByteBuffer | null;

/**
* @param flatbuffers.ByteBuffer bb
* @param FixedSizeList= obj
* @returns FixedSizeList
*/
static getSizePrefixedRootAsFixedSizeList(bb: flatbuffers.ByteBuffer, obj?: FixedSizeList): FixedSizeList;
/**
* Number of list items per value

@@ -331,8 +344,2 @@ *

/**
* @param flatbuffers.ByteBuffer bb
* @param Map= obj
* @returns Map
*/
static getSizePrefixedRootAsMap(bb: flatbuffers.ByteBuffer, obj?: Map): Map;
/**
* Set to true if the keys within each value are sorted

@@ -385,8 +392,2 @@ *

/**
* @param flatbuffers.ByteBuffer bb
* @param Union= obj
* @returns Union
*/
static getSizePrefixedRootAsUnion(bb: flatbuffers.ByteBuffer, obj?: Union): Union;
/**
* @returns org.apache.arrow.flatbuf.UnionMode

@@ -461,8 +462,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Int= obj
* @returns Int
*/
static getSizePrefixedRootAsInt(bb: flatbuffers.ByteBuffer, obj?: Int): Int;
/**
* @returns number

@@ -517,8 +512,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param FloatingPoint= obj
* @returns FloatingPoint
*/
static getSizePrefixedRootAsFloatingPoint(bb: flatbuffers.ByteBuffer, obj?: FloatingPoint): FloatingPoint;
/**
* @returns org.apache.arrow.flatbuf.Precision

@@ -566,8 +555,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Utf8= obj
* @returns Utf8
*/
static getSizePrefixedRootAsUtf8(bb: flatbuffers.ByteBuffer, obj?: Utf8): Utf8;
/**
* @param flatbuffers.Builder builder

@@ -585,2 +568,4 @@ */

/**
* Opaque binary data
*
* @constructor

@@ -605,8 +590,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Binary= obj
* @returns Binary
*/
static getSizePrefixedRootAsBinary(bb: flatbuffers.ByteBuffer, obj?: Binary): Binary;
/**
* @param flatbuffers.Builder builder

@@ -624,5 +603,73 @@ */

/**
* Same as Utf8, but with 64-bit offsets, allowing to represent
* extremely large data values.
*
* @constructor
*/
export declare namespace org.apache.arrow.flatbuf {
class LargeUtf8 {
bb: flatbuffers.ByteBuffer | null;
bb_pos: number;
/**
* @param number i
* @param flatbuffers.ByteBuffer bb
* @returns LargeUtf8
*/
__init(i: number, bb: flatbuffers.ByteBuffer): LargeUtf8;
/**
* @param flatbuffers.ByteBuffer bb
* @param LargeUtf8= obj
* @returns LargeUtf8
*/
static getRootAsLargeUtf8(bb: flatbuffers.ByteBuffer, obj?: LargeUtf8): LargeUtf8;
/**
* @param flatbuffers.Builder builder
*/
static startLargeUtf8(builder: flatbuffers.Builder): void;
/**
* @param flatbuffers.Builder builder
* @returns flatbuffers.Offset
*/
static endLargeUtf8(builder: flatbuffers.Builder): flatbuffers.Offset;
static createLargeUtf8(builder: flatbuffers.Builder): flatbuffers.Offset;
}
}
/**
* Same as Binary, but with 64-bit offsets, allowing to represent
* extremely large data values.
*
* @constructor
*/
export declare namespace org.apache.arrow.flatbuf {
class LargeBinary {
bb: flatbuffers.ByteBuffer | null;
bb_pos: number;
/**
* @param number i
* @param flatbuffers.ByteBuffer bb
* @returns LargeBinary
*/
__init(i: number, bb: flatbuffers.ByteBuffer): LargeBinary;
/**
* @param flatbuffers.ByteBuffer bb
* @param LargeBinary= obj
* @returns LargeBinary
*/
static getRootAsLargeBinary(bb: flatbuffers.ByteBuffer, obj?: LargeBinary): LargeBinary;
/**
* @param flatbuffers.Builder builder
*/
static startLargeBinary(builder: flatbuffers.Builder): void;
/**
* @param flatbuffers.Builder builder
* @returns flatbuffers.Offset
*/
static endLargeBinary(builder: flatbuffers.Builder): flatbuffers.Offset;
static createLargeBinary(builder: flatbuffers.Builder): flatbuffers.Offset;
}
}
/**
* @constructor
*/
export declare namespace org.apache.arrow.flatbuf {
class FixedSizeBinary {

@@ -644,8 +691,2 @@ bb: flatbuffers.ByteBuffer | null;

/**
* @param flatbuffers.ByteBuffer bb
* @param FixedSizeBinary= obj
* @returns FixedSizeBinary
*/
static getSizePrefixedRootAsFixedSizeBinary(bb: flatbuffers.ByteBuffer, obj?: FixedSizeBinary): FixedSizeBinary;
/**
* Number of bytes per value

@@ -693,8 +734,2 @@ *

/**
* @param flatbuffers.ByteBuffer bb
* @param Bool= obj
* @returns Bool
*/
static getSizePrefixedRootAsBool(bb: flatbuffers.ByteBuffer, obj?: Bool): Bool;
/**
* @param flatbuffers.Builder builder

@@ -731,8 +766,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Decimal= obj
* @returns Decimal
*/
static getSizePrefixedRootAsDecimal(bb: flatbuffers.ByteBuffer, obj?: Decimal): Decimal;
/**
* Total number of decimal digits

@@ -798,8 +827,2 @@ *

/**
* @param flatbuffers.ByteBuffer bb
* @param Date= obj
* @returns Date
*/
static getSizePrefixedRootAsDate(bb: flatbuffers.ByteBuffer, obj?: Date): Date;
/**
* @returns org.apache.arrow.flatbuf.DateUnit

@@ -849,8 +872,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Time= obj
* @returns Time
*/
static getSizePrefixedRootAsTime(bb: flatbuffers.ByteBuffer, obj?: Time): Time;
/**
* @returns org.apache.arrow.flatbuf.TimeUnit

@@ -912,8 +929,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Timestamp= obj
* @returns Timestamp
*/
static getSizePrefixedRootAsTimestamp(bb: flatbuffers.ByteBuffer, obj?: Timestamp): Timestamp;
/**
* @returns org.apache.arrow.flatbuf.TimeUnit

@@ -991,8 +1002,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Interval= obj
* @returns Interval
*/
static getSizePrefixedRootAsInterval(bb: flatbuffers.ByteBuffer, obj?: Interval): Interval;
/**
* @returns org.apache.arrow.flatbuf.IntervalUnit

@@ -1038,8 +1043,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param Duration= obj
* @returns Duration
*/
static getSizePrefixedRootAsDuration(bb: flatbuffers.ByteBuffer, obj?: Duration): Duration;
/**
* @returns org.apache.arrow.flatbuf.TimeUnit

@@ -1089,8 +1088,2 @@ */

/**
* @param flatbuffers.ByteBuffer bb
* @param KeyValue= obj
* @returns KeyValue
*/
static getSizePrefixedRootAsKeyValue(bb: flatbuffers.ByteBuffer, obj?: KeyValue): KeyValue;
/**
* @param flatbuffers.Encoding= optionalEncoding

@@ -1152,8 +1145,2 @@ * @returns string|Uint8Array|null

/**
* @param flatbuffers.ByteBuffer bb
* @param DictionaryEncoding= obj
* @returns DictionaryEncoding
*/
static getSizePrefixedRootAsDictionaryEncoding(bb: flatbuffers.ByteBuffer, obj?: DictionaryEncoding): DictionaryEncoding;
/**
* The known dictionary id in the application where this data is used. In

@@ -1234,8 +1221,2 @@ * the file or streaming formats, the dictionary ids are found in the

/**
* @param flatbuffers.ByteBuffer bb
* @param Field= obj
* @returns Field
*/
static getSizePrefixedRootAsField(bb: flatbuffers.ByteBuffer, obj?: Field): Field;
/**
* Name is not required, in i.e. a List

@@ -1428,8 +1409,2 @@ *

/**
* @param flatbuffers.ByteBuffer bb
* @param Schema= obj
* @returns Schema
*/
static getSizePrefixedRootAsSchema(bb: flatbuffers.ByteBuffer, obj?: Schema): Schema;
/**
* endianness of the buffer

@@ -1513,9 +1488,4 @@ * it is Little Endian by default

static finishSchemaBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset): void;
/**
* @param flatbuffers.Builder builder
* @param flatbuffers.Offset offset
*/
static finishSizePrefixedSchemaBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset): void;
static createSchema(builder: flatbuffers.Builder, endianness: org.apache.arrow.flatbuf.Endianness, fieldsOffset: flatbuffers.Offset, customMetadataOffset: flatbuffers.Offset): flatbuffers.Offset;
}
}

6

interfaces.d.ts

@@ -174,3 +174,3 @@ import { Data } from './data';

[Type.IntervalYearMonth]: T extends type.IntervalYearMonth ? vecs.IntervalYearMonthVector : vecs.BaseVector<T>;
[Type.Map]: T extends type.Map_ ? vecs.MapVector<T['dataTypes']> : vecs.BaseVector<T>;
[Type.Map]: T extends type.Map_ ? vecs.MapVector<T['keyType'], T['valueType']> : vecs.BaseVector<T>;
[Type.List]: T extends type.List ? vecs.ListVector<T['valueType']> : vecs.BaseVector<T>;

@@ -269,3 +269,3 @@ [Type.Struct]: T extends type.Struct ? vecs.StructVector<T['dataTypes']> : vecs.BaseVector<T>;

[Type.IntervalYearMonth]: builders.IntervalYearMonthBuilder<TNull>;
[Type.Map]: builders.MapBuilder<any, TNull>;
[Type.Map]: builders.MapBuilder<any, any, TNull>;
[Type.List]: builders.ListBuilder<any, TNull>;

@@ -317,3 +317,3 @@ [Type.Struct]: builders.StructBuilder<any, TNull>;

[Type.IntervalYearMonth]: T extends type.IntervalYearMonth ? builders.IntervalYearMonthBuilder<TNull> : builders.Builder<any, TNull>;
[Type.Map]: T extends type.Map_ ? builders.MapBuilder<T['dataTypes'], TNull> : builders.Builder<any, TNull>;
[Type.Map]: T extends type.Map_ ? builders.MapBuilder<T['keyType'], T['valueType'], TNull> : builders.Builder<any, TNull>;
[Type.List]: T extends type.List ? builders.ListBuilder<T['valueType'], TNull> : builders.Builder<any, TNull>;

@@ -320,0 +320,0 @@ [Type.Struct]: T extends type.Struct ? builders.StructBuilder<T['dataTypes'], TNull> : builders.Builder<any, TNull>;

22

ipc/message.js

@@ -41,2 +41,9 @@ // Licensed to the Apache Software Foundation (ASF) under one

}
// ARROW-6313: If the first 4 bytes are continuation indicator (-1), read
// the next 4 for the 32-bit metadata length. Otherwise, assume this is a
// pre-v0.15 message, where the first 4 bytes are the metadata length.
if ((r.value === -1) &&
(r = this.readMetadataLength()).done) {
return ITERATOR_DONE;
}
if ((r = this.readMetadata(r.value)).done) {

@@ -84,4 +91,4 @@ return ITERATOR_DONE;

const bb = buf && new ByteBuffer(buf);
const len = +(bb && bb.readInt32(0));
return { done: len <= 0, value: len };
const len = bb && bb.readInt32(0) || 0;
return { done: len === 0, value: len };
}

@@ -113,2 +120,9 @@ readMetadata(metadataLength) {

}
// ARROW-6313: If the first 4 bytes are continuation indicator (-1), read
// the next 4 for the 32-bit metadata length. Otherwise, assume this is a
// pre-v0.15 message, where the first 4 bytes are the metadata length.
if ((r.value === -1) &&
(r = await this.readMetadataLength()).done) {
return ITERATOR_DONE;
}
if ((r = await this.readMetadata(r.value)).done) {

@@ -156,4 +170,4 @@ return ITERATOR_DONE;

const bb = buf && new ByteBuffer(buf);
const len = +(bb && bb.readInt32(0));
return { done: len <= 0, value: len };
const len = bb && bb.readInt32(0) || 0;
return { done: len === 0, value: len };
}

@@ -160,0 +174,0 @@ async readMetadata(metadataLength) {

2

ipc/metadata/json.js

@@ -163,3 +163,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

const t = f['type'];
return new Map_(children || [], t['keysSorted']);
return new Map_((children || [])[0], t['keysSorted']);
}

@@ -166,0 +166,0 @@ }

2

ipc/metadata/message.js

@@ -371,3 +371,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

const t = f.type(new Schema_.org.apache.arrow.flatbuf.Map());
return new Map_(children || [], t.keysSorted());
return new Map_((children || [])[0], t.keysSorted());
}

@@ -374,0 +374,0 @@ }

35

ipc/writer.d.ts

@@ -13,2 +13,15 @@ /// <reference types="node" />

import { Writable, ReadableInterop, ReadableDOMStreamOptions } from '../io/interfaces';
export interface RecordBatchStreamWriterOptions {
/**
*
*/
autoDestroy?: boolean;
/**
* A flag indicating whether the RecordBatchWriter should construct pre-0.15.0
* encapsulated IPC Messages, which reserves 4 bytes for the Message metadata
* length instead of 8.
* @see https://issues.apache.org/jira/browse/ARROW-6313
*/
writeLegacyIpcFormat?: boolean;
}
export declare class RecordBatchWriter<T extends {

@@ -33,8 +46,7 @@ [key: string]: DataType;

};
constructor(options?: {
autoDestroy: boolean;
});
constructor(options?: RecordBatchStreamWriterOptions);
protected _position: number;
protected _started: boolean;
protected _autoDestroy: boolean;
protected _writeLegacyIpcFormat: boolean;
protected _sink: AsyncByteQueue<Uint8Array>;

@@ -79,20 +91,12 @@ protected _schema: Schema | null;

[key: string]: DataType;
} = any>(input: Table<T> | Iterable<RecordBatch<T>>, options?: {
autoDestroy: true;
}): RecordBatchStreamWriter<T>;
} = any>(input: Table<T> | Iterable<RecordBatch<T>>, options?: RecordBatchStreamWriterOptions): RecordBatchStreamWriter<T>;
static writeAll<T extends {
[key: string]: DataType;
} = any>(input: AsyncIterable<RecordBatch<T>>, options?: {
autoDestroy: true;
}): Promise<RecordBatchStreamWriter<T>>;
} = any>(input: AsyncIterable<RecordBatch<T>>, options?: RecordBatchStreamWriterOptions): Promise<RecordBatchStreamWriter<T>>;
static writeAll<T extends {
[key: string]: DataType;
} = any>(input: PromiseLike<AsyncIterable<RecordBatch<T>>>, options?: {
autoDestroy: true;
}): Promise<RecordBatchStreamWriter<T>>;
} = any>(input: PromiseLike<AsyncIterable<RecordBatch<T>>>, options?: RecordBatchStreamWriterOptions): Promise<RecordBatchStreamWriter<T>>;
static writeAll<T extends {
[key: string]: DataType;
} = any>(input: PromiseLike<Table<T> | Iterable<RecordBatch<T>>>, options?: {
autoDestroy: true;
}): Promise<RecordBatchStreamWriter<T>>;
} = any>(input: PromiseLike<Table<T> | Iterable<RecordBatch<T>>>, options?: RecordBatchStreamWriterOptions): Promise<RecordBatchStreamWriter<T>>;
}

@@ -139,2 +143,3 @@ /** @ignore */

protected _writeMessage(): this;
protected _writeFooter(schema: Schema<T>): this;
protected _writeSchema(schema: Schema<T>): this;

@@ -141,0 +146,0 @@ protected _writeDictionaries(batch: RecordBatch<T>): this;

28

ipc/writer.js

@@ -33,3 +33,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { ReadableInterop } from '../io/interfaces';
import { isPromise, isAsyncIterable, isWritableDOMStream, isWritableNodeStream, isIterable } from '../util/compat';
import { isPromise, isAsyncIterable, isWritableDOMStream, isWritableNodeStream, isIterable, isObject } from '../util/compat';
export class RecordBatchWriter extends ReadableInterop {

@@ -46,3 +46,5 @@ constructor(options) {

this._dictionaryDeltaOffsets = new Map();
this._autoDestroy = options && (typeof options.autoDestroy === 'boolean') ? options.autoDestroy : true;
isObject(options) || (options = { autoDestroy: true, writeLegacyIpcFormat: false });
this._autoDestroy = (typeof options.autoDestroy === 'boolean') ? options.autoDestroy : true;
this._writeLegacyIpcFormat = (typeof options.writeLegacyIpcFormat === 'boolean') ? options.writeLegacyIpcFormat : false;
}

@@ -160,4 +162,5 @@ /** @nocollapse */

const flatbufferSize = buffer.byteLength;
const alignedSize = (flatbufferSize + 4 + a) & ~a;
const nPaddingBytes = alignedSize - flatbufferSize - 4;
const prefixSize = !this._writeLegacyIpcFormat ? 8 : 4;
const alignedSize = (flatbufferSize + prefixSize + a) & ~a;
const nPaddingBytes = alignedSize - flatbufferSize - prefixSize;
if (message.headerType === MessageHeader.RecordBatch) {

@@ -169,4 +172,8 @@ this._recordBatchBlocks.push(new FileBlock(alignedSize, message.bodyLength, this._position));

}
// If not in legacy pre-0.15.0 mode, write the stream continuation indicator
if (!this._writeLegacyIpcFormat) {
this._write(Int32Array.of(-1));
}
// Write the flatbuffer size prefix including padding
this._write(Int32Array.of(alignedSize - 4));
this._write(Int32Array.of(alignedSize - prefixSize));
// Write the flatbuffer

@@ -194,3 +201,6 @@ if (flatbufferSize > 0) {

_writeFooter(schema) {
return this._writePadding(4); // eos bytes
// eos bytes
return this._writeLegacyIpcFormat
? this._write(Int32Array.of(0))
: this._write(Int32Array.of(-1, 0));
}

@@ -286,4 +296,4 @@ _writeMagic() {

const buffer = Footer.encode(new Footer(schema, MetadataVersion.V4, this._recordBatchBlocks, this._dictionaryBlocks));
return this
._writePadding(4) // EOS bytes for sequential readers
return super
._writeFooter(schema) // EOS bytes for sequential readers
._write(buffer) // Write the flatbuffer

@@ -307,2 +317,4 @@ ._write(Int32Array.of(buffer.byteLength)) // then the footer size suffix

_writeMessage() { return this; }
// @ts-ignore
_writeFooter(schema) { return this; }
_writeSchema(schema) {

@@ -309,0 +321,0 @@ return this._write(`{\n "schema": ${JSON.stringify({ fields: schema.fields.map(fieldToJSON) }, null, 2)}`);

2

package.json
{
"version": "0.14.1",
"version": "0.15.0",
"name": "@apache-arrow/esnext-esm",

@@ -4,0 +4,0 @@ "browser": "Arrow.dom.js",

2

README.md

@@ -254,3 +254,3 @@ <!---

Full list of broader Apache Arrow [projects & organizations](https://github.com/apache/arrow/blob/master/site/powered_by.md).
Full list of broader Apache Arrow [projects & organizations](https://arrow.apache.org/powered_by/).

@@ -257,0 +257,0 @@ ## Open Source Projects

17

recordbatch.d.ts

@@ -5,6 +5,5 @@ import { Data } from './data';

import { Schema, Field } from './schema';
import { MapVector } from './vector/index';
import { DataType, Struct, Map_ } from './type';
import { DataType, Struct } from './type';
import { Clonable, Sliceable, Applicative } from './vector';
import { VectorBuilderOptions, VectorBuilderOptionsAsync } from './vector/index';
import { StructVector, VectorBuilderOptions, VectorBuilderOptionsAsync } from './vector/index';
declare type VectorMap = {

@@ -22,15 +21,15 @@ [key: string]: Vector;

} = any> {
concat(...others: Vector<Map_<T>>[]): Table<T>;
concat(...others: Vector<Struct<T>>[]): Table<T>;
slice(begin?: number, end?: number): RecordBatch<T>;
clone(data: Data<Map_<T>>, children?: Vector[]): RecordBatch<T>;
clone(data: Data<Struct<T>>, children?: Vector[]): RecordBatch<T>;
}
export declare class RecordBatch<T extends {
[key: string]: DataType;
} = any> extends MapVector<T> implements Clonable<RecordBatch<T>>, Sliceable<RecordBatch<T>>, Applicative<Map_<T>, Table<T>> {
} = any> extends StructVector<T> implements Clonable<RecordBatch<T>>, Sliceable<RecordBatch<T>>, Applicative<Struct<T>, Table<T>> {
static from<T extends {
[key: string]: DataType;
} = any, TNull = any>(options: VectorBuilderOptions<Struct<T> | Map_<T>, TNull>): Table<T>;
} = any, TNull = any>(options: VectorBuilderOptions<Struct<T>, TNull>): Table<T>;
static from<T extends {
[key: string]: DataType;
} = any, TNull = any>(options: VectorBuilderOptionsAsync<Struct<T> | Map_<T>, TNull>): Promise<Table<T>>;
} = any, TNull = any>(options: VectorBuilderOptionsAsync<Struct<T>, TNull>): Promise<Table<T>>;
static new<T extends VectorMap = any>(children: T): RecordBatch<{

@@ -45,3 +44,3 @@ [P in keyof T]: T[P]['type'];

constructor(schema: Schema<T>, length: number, children: (Data | Vector)[]);
constructor(schema: Schema<T>, data: Data<Map_<T>>, children?: Vector[]);
constructor(schema: Schema<T>, data: Data<Struct<T>>, children?: Vector[]);
readonly schema: Schema<any>;

@@ -48,0 +47,0 @@ readonly numCols: number;

10

recordbatch.js

@@ -24,7 +24,7 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Chunked } from './vector/chunked';
import { MapVector } from './vector/index';
import { selectFieldArgs } from './util/args';
import { DataType, Map_ } from './type';
import { DataType, Struct } from './type';
import { ensureSameLengthData } from './util/recordbatch';
export class RecordBatch extends MapVector {
import { StructVector } from './vector/index';
export class RecordBatch extends StructVector {
constructor(...args) {

@@ -40,3 +40,3 @@ let data;

const [, length, childData] = args;
data = Data.Map(new Map_(fields), 0, length, 0, null, childData);
data = Data.Struct(new Struct(fields), 0, length, 0, null, childData);
}

@@ -102,3 +102,3 @@ super(data, children);

static collect(batch) {
return new DictionaryCollector().visit(batch.data, new Map_(batch.schema.fields)).dictionaries;
return new DictionaryCollector().visit(batch.data, new Struct(batch.schema.fields)).dictionaries;
}

@@ -105,0 +105,0 @@ visit(data, type) {

18

table.d.ts

@@ -7,6 +7,5 @@ import { Data } from './data';

import { RecordBatchReader } from './ipc/reader';
import { DataType, RowLike, Struct, Map_ } from './type';
import { DataType, RowLike, Struct } from './type';
import { Clonable, Sliceable, Applicative } from './vector';
import { Vector, Chunked } from './vector/index';
import { VectorBuilderOptions, VectorBuilderOptionsAsync } from './vector/index';
import { Vector, Chunked, VectorBuilderOptions, VectorBuilderOptionsAsync } from './vector/index';
declare type VectorMap = {

@@ -30,5 +29,6 @@ [key: string]: Vector;

slice(begin?: number, end?: number): Table<T>;
concat(...others: Vector<Map_<T>>[]): Table<T>;
concat(...others: Vector<Struct<T>>[]): Table<T>;
clone(chunks?: RecordBatch<T>[], offsets?: Uint32Array): Table<T>;
scan(next: import('./compute/dataframe').NextFunc, bind?: import('./compute/dataframe').BindFunc): void;
scanReverse(next: import('./compute/dataframe').NextFunc, bind?: import('./compute/dataframe').BindFunc): void;
countBy(name: import('./compute/predicate').Col | string): import('./compute/dataframe').CountByResult;

@@ -39,3 +39,3 @@ filter(predicate: import('./compute/predicate').Predicate): import('./compute/dataframe').FilteredDataFrame<T>;

[key: string]: DataType;
} = any> extends Chunked<Map_<T>> implements DataFrame<T>, Clonable<Table<T>>, Sliceable<Table<T>>, Applicative<Map_<T>, Table<T>> {
} = any> extends Chunked<Struct<T>> implements DataFrame<T>, Clonable<Table<T>>, Sliceable<Table<T>>, Applicative<Struct<T>, Table<T>> {
/** @nocollapse */

@@ -72,6 +72,6 @@ static empty<T extends {

[key: string]: DataType;
} = any, TNull = any>(options: VectorBuilderOptions<Struct<T> | Map_<T>, TNull>): Table<T>;
} = any, TNull = any>(options: VectorBuilderOptions<Struct<T>, TNull>): Table<T>;
static from<T extends {
[key: string]: DataType;
} = any, TNull = any>(options: VectorBuilderOptionsAsync<Struct<T> | Map_<T>, TNull>): Promise<Table<T>>;
} = any, TNull = any>(options: VectorBuilderOptionsAsync<Struct<T>, TNull>): Promise<Table<T>>;
/** @nocollapse */

@@ -82,6 +82,2 @@ static fromAsync<T extends {

/** @nocollapse */
static fromMap<T extends {
[key: string]: DataType;
} = any>(vector: Vector<Map_<T>>): Table<T>;
/** @nocollapse */
static fromStruct<T extends {

@@ -88,0 +84,0 @@ [key: string]: DataType;

23

table.js

@@ -21,9 +21,8 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { RecordBatchReader } from './ipc/reader';
import { Struct, Map_ } from './type';
import { Struct } from './type';
import { selectColumnArgs, selectArgs } from './util/args';
import { isPromise, isIterable, isAsyncIterable } from './util/compat';
import { distributeColumnsIntoRecordBatches } from './util/recordbatch';
import { distributeVectorsIntoRecordBatches } from './util/recordbatch';
import { Chunked, MapVector, StructVector } from './vector/index';
import { RecordBatchFileWriter, RecordBatchStreamWriter } from './ipc/writer';
import { distributeColumnsIntoRecordBatches, distributeVectorsIntoRecordBatches } from './util/recordbatch';
import { Chunked, StructVector } from './vector/index';
export class Table extends Chunked {

@@ -40,3 +39,3 @@ constructor(...args) {

chunks[0] || (chunks[0] = new _InternalEmptyPlaceholderRecordBatch(schema));
super(new Map_(schema.fields), chunks);
super(new Struct(schema.fields), chunks);
this._schema = schema;

@@ -85,6 +84,2 @@ this._chunks = chunks;

/** @nocollapse */
static fromMap(vector) {
return Table.new(vector.data.childData, vector.type.children);
}
/** @nocollapse */
static fromStruct(vector) {

@@ -171,6 +166,3 @@ return Table.new(vector.data.childData, vector.type.children);

const { type } = input;
if (type instanceof Map_) {
return Table.fromMap(MapVector.from(input));
}
else if (type instanceof Struct) {
if (type instanceof Struct) {
return Table.fromStruct(StructVector.from(input));

@@ -182,6 +174,3 @@ }

const { type } = input;
if (type instanceof Map_) {
return MapVector.from(input).then((vector) => Table.fromMap(vector));
}
else if (type instanceof Struct) {
if (type instanceof Struct) {
return StructVector.from(input).then((vector) => Table.fromStruct(vector));

@@ -188,0 +177,0 @@ }

37

type.d.ts

@@ -19,7 +19,13 @@ import { Field } from './schema';

[key: string]: DataType;
}> = (Iterable<T[keyof T]['TValue'] | null>) & {
}> = (Iterable<[string, T[keyof T]['TValue'] | null]>) & {
[P in keyof T]: T[P]['TValue'] | null;
} & {
get<K extends keyof T>(key: K): T[K]['TValue'] | null;
} & {
set<K extends keyof T>(key: K, val: T[K]['TValue'] | null): void;
};
/** @ignore */
export declare type MapLike<K extends DataType = any, V extends DataType = any> = {
[P in K['TValue']]: V['TValue'] | null;
} & (Map<K['TValue'], V['TValue'] | null>);
export interface DataType<TType extends Type = Type, TChildren extends {

@@ -529,17 +535,24 @@ [key: string]: DataType;

/** @ignore */
export interface Map_<T extends {
[key: string]: DataType;
} = any> extends DataType<Type.Map> {
TArray: IterableArrayLike<RowLike<T>>;
TValue: RowLike<T>;
dataTypes: T;
export interface Map_<TKey extends DataType = any, TValue extends DataType = any> extends DataType<Type.Map> {
TArray: IterableArrayLike<Map<TKey['TValue'], TValue['TValue'] | null>>;
TChild: Struct<{
key: TKey;
value: TValue;
}>;
TValue: MapLike<TKey, TValue>;
}
/** @ignore */
export declare class Map_<T extends {
[key: string]: DataType;
} = any> extends DataType<Type.Map, T> {
readonly children: Field<T[keyof T]>[];
export declare class Map_<TKey extends DataType = any, TValue extends DataType = any> extends DataType<Type.Map> {
constructor(child: Field<Struct<{
key: TKey;
value: TValue;
}>>, keysSorted?: boolean);
readonly keysSorted: boolean;
constructor(children: Field<T[keyof T]>[], keysSorted?: boolean);
readonly children: Field<Struct<{
key: TKey;
value: TValue;
}>>[];
readonly typeId: Type.Map;
readonly keyType: TKey;
readonly valueType: TValue;
toString(): string;

@@ -546,0 +559,0 @@ protected static [Symbol.toStringTag]: string;

10

type.js

@@ -348,3 +348,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

get typeId() { return Type.Struct; }
toString() { return `Struct<[${this.children.map((f) => f.type).join(`, `)}]>`; }
toString() { return `Struct<{${this.children.map((f) => `${f.name}:${f.type}`).join(`, `)}}>`; }
}

@@ -426,9 +426,11 @@ Struct[Symbol.toStringTag] = ((proto) => {

export class Map_ extends DataType {
constructor(children, keysSorted = false) {
constructor(child, keysSorted = false) {
super();
this.children = children;
this.children = [child];
this.keysSorted = keysSorted;
}
get typeId() { return Type.Map; }
toString() { return `Map<{${this.children.map((f) => `${f.name}:${f.type}`).join(`, `)}}>`; }
get keyType() { return this.children[0].type.children[0].type; }
get valueType() { return this.children[0].type.children[1].type; }
toString() { return `Map<{${this.children[0].type.children.map((f) => `${f.name}:${f.type}`).join(`, `)}}>`; }
}

@@ -435,0 +437,0 @@ Map_[Symbol.toStringTag] = ((proto) => {

2

util/buffer.d.ts

@@ -26,4 +26,2 @@ import { flatbuffers } from 'flatbuffers';

/** @ignore */
export declare const toFloat16Array: (input: ArrayBufferViewInput) => Uint16Array;
/** @ignore */
declare type ArrayBufferViewIteratorInput = Iterable<ArrayBufferViewInput> | ArrayBufferViewInput;

@@ -30,0 +28,0 @@ /** @ignore */

25

util/buffer.js

@@ -121,27 +121,2 @@ // Licensed to the Apache Software Foundation (ASF) under one

/** @ignore */
export const toFloat16Array = (input) => {
let floats = null;
if (ArrayBuffer.isView(input)) {
switch (input.constructor) {
case Float32Array:
floats = input;
break;
case Float64Array:
floats = input;
break;
}
}
else if (isIterable(input)) {
floats = toFloat64Array(input);
}
if (floats) {
const u16s = new Uint16Array(floats.length);
for (let i = -1, n = u16s.length; ++i < n;) {
u16s[i] = (floats[i] * 32767) + 32767;
}
return u16s;
}
return toUint16Array(input);
};
/** @ignore */
const pump = (iterator) => { iterator.next(); return iterator; };

@@ -148,0 +123,0 @@ /** @ignore */

184

util/vector.js

@@ -18,3 +18,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { Vector } from '../vector';
import { Row, kLength } from '../vector/row';
import { MapRow, StructRow } from '../vector/row';
import { compareArrayLike } from '../util/buffer';

@@ -67,5 +67,10 @@ import { BigInt, BigIntAvailable } from './compat';

}
// Compare TypedArrays
if (ArrayBuffer.isView(search)) {
return (value) => value ? compareArrayLike(search, value) : false;
}
// Compare Maps and Rows
if (search instanceof Map) {
return creatMapComparator(search);
}
// Compare Array-likes

@@ -75,6 +80,2 @@ if (Array.isArray(search)) {

}
// Compare Rows
if (search instanceof Row) {
return createRowComparator(search);
}
// Compare Vectors

@@ -85,118 +86,105 @@ if (search instanceof Vector) {

// Compare non-empty Objects
const keys = Object.keys(search);
if (keys.length > 0) {
return createObjectKeysComparator(search, keys);
return createObjectComparator(search);
}
/** @ignore */
function createArrayLikeComparator(lhs) {
const comparitors = [];
for (let i = -1, n = lhs.length; ++i < n;) {
comparitors[i] = createElementComparator(lhs[i]);
}
// No valid comparator
return () => false;
return createSubElementsComparator(comparitors);
}
/** @ignore */
function createArrayLikeComparator(search) {
const n = search.length;
const fns = [];
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator(search[i]);
function creatMapComparator(lhs) {
let i = -1;
const comparitors = [];
lhs.forEach((v) => comparitors[++i] = createElementComparator(v));
return createSubElementsComparator(comparitors);
}
/** @ignore */
function createVectorComparator(lhs) {
const comparitors = [];
for (let i = -1, n = lhs.length; ++i < n;) {
comparitors[i] = createElementComparator(lhs.get(i));
}
return (value) => {
if (!value) {
return false;
}
// Handle the case where the search element is an Array, but the
// values are Rows or Vectors, e.g. list.indexOf(['foo', 'bar'])
if (value instanceof Row) {
if (value[kLength] !== n) {
return false;
}
for (let i = -1; ++i < n;) {
if (!(fns[i](value.get(i)))) {
return false;
}
}
return true;
}
if (value.length !== n) {
return false;
}
if (value instanceof Vector) {
for (let i = -1; ++i < n;) {
if (!(fns[i](value.get(i)))) {
return false;
}
}
return true;
}
for (let i = -1; ++i < n;) {
if (!(fns[i](value[i]))) {
return false;
}
}
return true;
};
return createSubElementsComparator(comparitors);
}
/** @ignore */
function createRowComparator(search) {
const n = search[kLength];
const C = search.constructor;
const fns = [];
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator(search.get(i));
function createObjectComparator(lhs) {
const keys = Object.keys(lhs);
// Only compare non-empty Objects
if (keys.length === 0) {
return () => false;
}
return (value) => {
if (!(value instanceof C)) {
const comparitors = [];
for (let i = -1, n = keys.length; ++i < n;) {
comparitors[i] = createElementComparator(lhs[keys[i]]);
}
return createSubElementsComparator(comparitors, keys);
}
function createSubElementsComparator(comparitors, keys) {
return (rhs) => {
if (!rhs || typeof rhs !== 'object') {
return false;
}
if (!(value[kLength] === n)) {
return false;
switch (rhs.constructor) {
case Array: return compareArray(comparitors, rhs);
case Map:
case MapRow:
case StructRow:
return compareObject(comparitors, rhs, rhs.keys());
case Object:
case undefined: // support `Object.create(null)` objects
return compareObject(comparitors, rhs, keys || Object.keys(rhs));
}
for (let i = -1; ++i < n;) {
if (!(fns[i](value.get(i)))) {
return false;
}
}
return true;
return rhs instanceof Vector ? compareVector(comparitors, rhs) : false;
};
}
/** @ignore */
function createVectorComparator(search) {
const n = search.length;
const C = search.constructor;
const fns = [];
function compareArray(comparitors, arr) {
const n = comparitors.length;
if (arr.length !== n) {
return false;
}
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator(search.get(i));
}
return (value) => {
if (!(value instanceof C)) {
if (!(comparitors[i](arr[i]))) {
return false;
}
if (!(value.length === n)) {
return false;
}
for (let i = -1; ++i < n;) {
if (!(fns[i](value.get(i)))) {
return false;
}
}
return true;
};
}
return true;
}
/** @ignore */
function createObjectKeysComparator(search, keys) {
const n = keys.length;
const fns = [];
function compareVector(comparitors, vec) {
const n = comparitors.length;
if (vec.length !== n) {
return false;
}
for (let i = -1; ++i < n;) {
fns[i] = createElementComparator(search[keys[i]]);
}
return (value) => {
if (!value || typeof value !== 'object') {
if (!(comparitors[i](vec.get(i)))) {
return false;
}
for (let i = -1; ++i < n;) {
if (!(fns[i](value[keys[i]]))) {
return false;
}
}
return true;
}
function compareObject(comparitors, obj, keys) {
const lKeyItr = keys[Symbol.iterator]();
const rKeyItr = obj instanceof Map ? obj.keys() : Object.keys(obj)[Symbol.iterator]();
const rValItr = obj instanceof Map ? obj.values() : Object.values(obj)[Symbol.iterator]();
let i = 0;
let n = comparitors.length;
let rVal = rValItr.next();
let lKey = lKeyItr.next();
let rKey = rKeyItr.next();
for (; i < n && !lKey.done && !rKey.done && !rVal.done; ++i, lKey = lKeyItr.next(), rKey = rKeyItr.next(), rVal = rValItr.next()) {
if (lKey.value !== rKey.value || !comparitors[i](rVal.value)) {
break;
}
}
if (i === n && lKey.done && rKey.done && rVal.done) {
return true;
};
}
lKeyItr.return && lKeyItr.return();
rKeyItr.return && rKeyItr.return();
rValItr.return && rValItr.return();
return false;
}
//# sourceMappingURL=vector.mjs.map

2

vector/base.js

@@ -70,3 +70,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

_sliceInternal(self, begin, end) {
return self.clone(self.data.slice(begin, end - begin));
return self.clone(self.data.slice(begin, end - begin), null);
}

@@ -73,0 +73,0 @@ // @ts-ignore

4

vector/chunked.js

@@ -234,5 +234,5 @@ // Licensed to the Apache Software Foundation (ASF) under one

const arraySet = (src, dst, offset) => {
let idx = offset - 1;
let idx = offset;
for (let i = -1, n = src.length; ++i < n;) {
dst[++idx] = src[i];
dst[idx++] = src[i];
}

@@ -239,0 +239,0 @@ return idx;

25

vector/float.d.ts

@@ -0,13 +1,21 @@

import { Chunked } from './chunked';
import { BaseVector } from './base';
import { VectorBuilderOptions } from './index';
import { VectorBuilderOptionsAsync } from './index';
import { Float, Float16, Float32, Float64, FloatArray } from '../type';
import { VectorType as V } from '../interfaces';
import { Float, Float16, Float32, Float64 } from '../type';
/** @ignore */
declare type FromInput<T extends Float, TNull = any> = FloatArray | Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull> | VectorBuilderOptions<T, TNull> | VectorBuilderOptionsAsync<T, TNull>;
/** @ignore */
export declare class FloatVector<T extends Float = Float> extends BaseVector<T> {
static from(this: typeof FloatVector, data: Float16['TArray']): Float16Vector;
static from(this: typeof FloatVector, data: Float32['TArray']): Float32Vector;
static from(this: typeof FloatVector, data: Float64['TArray']): Float64Vector;
static from<T extends Float>(this: typeof FloatVector, data: T['TArray']): V<T>;
static from(this: typeof Float16Vector, data: Float16['TArray'] | Iterable<number>): Float16Vector;
static from(this: typeof Float32Vector, data: Float32['TArray'] | Iterable<number>): Float32Vector;
static from(this: typeof Float64Vector, data: Float64['TArray'] | Iterable<number>): Float64Vector;
static from(this: typeof FloatVector, input: Uint16Array): Float16Vector;
static from(this: typeof FloatVector, input: Float32Array): Float32Vector;
static from(this: typeof FloatVector, input: Float64Array): Float64Vector;
static from<TNull = any>(this: typeof Float16Vector, input: FromInput<Float16, TNull>): Float16Vector;
static from<TNull = any>(this: typeof Float32Vector, input: FromInput<Float32, TNull>): Float32Vector;
static from<TNull = any>(this: typeof Float64Vector, input: FromInput<Float64, TNull>): Float64Vector;
static from<T extends Float, TNull = any>(this: typeof FloatVector, input: Iterable<T['TValue'] | TNull>): V<T>;
static from<T extends Float, TNull = any>(this: typeof FloatVector, input: AsyncIterable<T['TValue'] | TNull>): Promise<V<T>>;
static from<T extends Float, TNull = any>(this: typeof FloatVector, input: VectorBuilderOptions<T, TNull>): Chunked<T>;
static from<T extends Float, TNull = any>(this: typeof FloatVector, input: VectorBuilderOptionsAsync<T, TNull>): Promise<Chunked<T>>;
}

@@ -25,1 +33,2 @@ /** @ignore */

}
export {};

77

vector/float.js

@@ -20,34 +20,38 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { BaseVector } from './base';
import { vectorFromValuesWithType } from './index';
import { Float16, Float32, Float64 } from '../type';
import { toFloat16Array, toFloat32Array, toFloat64Array } from '../util/buffer';
/** @ignore */
export class FloatVector extends BaseVector {
/** @nocollapse */
static from(data) {
let type = null;
switch (this) {
case Float16Vector:
data = toFloat16Array(data);
break;
case Float32Vector:
data = toFloat32Array(data);
break;
case Float64Vector:
data = toFloat64Array(data);
break;
static from(input) {
let ArrowType = vectorTypeToDataType(this);
if ((input instanceof ArrayBuffer) || ArrayBuffer.isView(input)) {
let InputType = arrayTypeToDataType(input.constructor) || ArrowType;
// Special case, infer the Arrow DataType from the input if calling the base
// FloatVector.from with a TypedArray, e.g. `FloatVector.from(new Float32Array())`
if (ArrowType === null) {
ArrowType = InputType;
}
// If the DataType inferred from the Vector constructor matches the
// DataType inferred from the input arguments, return zero-copy view
if (ArrowType && ArrowType === InputType) {
let type = new ArrowType();
let length = input.byteLength / type.ArrayType.BYTES_PER_ELEMENT;
// If the ArrowType is Float16 but the input type isn't a Uint16Array,
// let the Float16Builder handle casting the input values to Uint16s.
if (!convertTo16Bit(ArrowType, input.constructor)) {
return Vector.new(Data.Float(type, 0, length, 0, null, input));
}
}
}
switch (data.constructor) {
case Uint16Array:
type = new Float16();
break;
case Float32Array:
type = new Float32();
break;
case Float64Array:
type = new Float64();
break;
if (ArrowType) {
// If the DataType inferred from the Vector constructor is different than
// the DataType inferred from the input TypedArray, or if input isn't a
// TypedArray, use the Builders to construct the result Vector
return vectorFromValuesWithType(() => new ArrowType(), input);
}
return type !== null
? Vector.new(Data.Float(type, 0, data.length, 0, null, data))
: (() => { throw new TypeError('Unrecognized FloatVector input'); })();
if ((input instanceof DataView) || (input instanceof ArrayBuffer)) {
throw new TypeError(`Cannot infer float type from instance of ${input.constructor.name}`);
}
throw new TypeError('Unrecognized FloatVector input');
}

@@ -72,3 +76,24 @@ }

}
const convertTo16Bit = (typeCtor, dataCtor) => {
return (typeCtor === Float16) && (dataCtor !== Uint16Array);
};
/** @ignore */
const arrayTypeToDataType = (ctor) => {
switch (ctor) {
case Uint16Array: return Float16;
case Float32Array: return Float32;
case Float64Array: return Float64;
default: return null;
}
};
/** @ignore */
const vectorTypeToDataType = (ctor) => {
switch (ctor) {
case Float16Vector: return Float16;
case Float32Vector: return Float32;
case Float64Vector: return Float64;
default: return null;
}
};
//# sourceMappingURL=float.mjs.map

2

vector/index.d.ts

@@ -1,2 +0,1 @@

export { Row } from './row';
export { Vector } from '../vector';

@@ -23,2 +22,3 @@ export { BaseVector } from './base';

export { Utf8Vector } from './utf8';
export { MapRow, StructRow } from './row';
import { Data } from '../data';

@@ -25,0 +25,0 @@ import { Vector } from '../vector';

2

vector/index.js

@@ -17,3 +17,2 @@ // Licensed to the Apache Software Foundation (ASF) under one

// under the License.
export { Row } from './row';
export { Vector } from '../vector';

@@ -40,2 +39,3 @@ export { BaseVector } from './base';

export { Utf8Vector } from './utf8';
export { MapRow, StructRow } from './row';
import * as fn from '../util/fn';

@@ -42,0 +42,0 @@ import { Type } from '../enum';

49

vector/int.d.ts

@@ -0,23 +1,33 @@

import { Chunked } from './chunked';
import { BaseVector } from './base';
import { VectorType as V } from '../interfaces';
import { Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64 } from '../type';
import { VectorBuilderOptions } from './index';
import { VectorBuilderOptionsAsync } from './index';
import { Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, IntArray } from '../type';
import { VectorType as V, BigIntArray } from '../interfaces';
/** @ignore */
declare type FromInput<T extends Int, TNull = any> = IntArray | BigIntArray | Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull> | VectorBuilderOptions<T, TNull> | VectorBuilderOptionsAsync<T, TNull>;
/** @ignore */
export declare class IntVector<T extends Int = Int> extends BaseVector<T> {
static from(this: typeof IntVector, data: Int8Array): Int8Vector;
static from(this: typeof IntVector, data: Int16Array): Int16Vector;
static from(this: typeof IntVector, data: Int32Array): Int32Vector;
static from(this: typeof IntVector, data: Uint8Array): Uint8Vector;
static from(this: typeof IntVector, data: Uint16Array): Uint16Vector;
static from(this: typeof IntVector, data: Uint32Array): Uint32Vector;
static from(this: typeof IntVector, data: Int32Array, is64: true): Int64Vector;
static from(this: typeof IntVector, data: Uint32Array, is64: true): Uint64Vector;
static from<T extends Int>(this: typeof IntVector, data: T['TArray']): V<T>;
static from(this: typeof Int8Vector, data: Int8['TArray'] | Iterable<number>): Int8Vector;
static from(this: typeof Int16Vector, data: Int16['TArray'] | Iterable<number>): Int16Vector;
static from(this: typeof Int32Vector, data: Int32['TArray'] | Iterable<number>): Int32Vector;
static from(this: typeof Int64Vector, data: Int32['TArray'] | Iterable<number>): Int64Vector;
static from(this: typeof Uint8Vector, data: Uint8['TArray'] | Iterable<number>): Uint8Vector;
static from(this: typeof Uint16Vector, data: Uint16['TArray'] | Iterable<number>): Uint16Vector;
static from(this: typeof Uint32Vector, data: Uint32['TArray'] | Iterable<number>): Uint32Vector;
static from(this: typeof Uint64Vector, data: Uint32['TArray'] | Iterable<number>): Uint64Vector;
static from(this: typeof IntVector, input: Int8Array): Int8Vector;
static from(this: typeof IntVector, input: Int16Array): Int16Vector;
static from(this: typeof IntVector, input: Int32Array): Int32Vector;
static from(this: typeof IntVector, input: BigInt64Array): Int64Vector;
static from(this: typeof IntVector, input: Int32Array, is64bit: true): Int64Vector;
static from(this: typeof IntVector, input: Uint8Array): Uint8Vector;
static from(this: typeof IntVector, input: Uint16Array): Uint16Vector;
static from(this: typeof IntVector, input: Uint32Array): Uint32Vector;
static from(this: typeof IntVector, input: BigUint64Array): Uint64Vector;
static from(this: typeof IntVector, input: Uint32Array, is64bit: true): Uint64Vector;
static from<TNull = any>(this: typeof Int8Vector, input: FromInput<Int8, TNull>): Int8Vector;
static from<TNull = any>(this: typeof Int16Vector, input: FromInput<Int16, TNull>): Int16Vector;
static from<TNull = any>(this: typeof Int32Vector, input: FromInput<Int32, TNull>): Int32Vector;
static from<TNull = any>(this: typeof Int64Vector, input: FromInput<Int64, TNull>): Int64Vector;
static from<TNull = any>(this: typeof Uint8Vector, input: FromInput<Uint8, TNull>): Uint8Vector;
static from<TNull = any>(this: typeof Uint16Vector, input: FromInput<Uint16, TNull>): Uint16Vector;
static from<TNull = any>(this: typeof Uint32Vector, input: FromInput<Uint32, TNull>): Uint32Vector;
static from<TNull = any>(this: typeof Uint64Vector, input: FromInput<Uint64, TNull>): Uint64Vector;
static from<T extends Int, TNull = any>(this: typeof IntVector, input: Iterable<T['TValue'] | TNull>): V<T>;
static from<T extends Int, TNull = any>(this: typeof IntVector, input: AsyncIterable<T['TValue'] | TNull>): Promise<V<T>>;
static from<T extends Int, TNull = any>(this: typeof IntVector, input: VectorBuilderOptions<T, TNull>): Chunked<T>;
static from<T extends Int, TNull = any>(this: typeof IntVector, input: VectorBuilderOptionsAsync<T, TNull>): Promise<Chunked<T>>;
}

@@ -54,1 +64,2 @@ /** @ignore */

}
export {};

129

vector/int.js

@@ -20,74 +20,41 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { BaseVector } from './base';
import { vectorFromValuesWithType } from './index';
import { BigInt64Array, BigUint64Array } from '../util/compat';
import { toBigInt64Array, toBigUint64Array } from '../util/buffer';
import { Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64 } from '../type';
import { toInt8Array, toInt16Array, toInt32Array, toUint8Array, toUint16Array, toUint32Array, toBigInt64Array, toBigUint64Array } from '../util/buffer';
/** @ignore */
export class IntVector extends BaseVector {
/** @nocollapse */
static from(data, is64) {
let length = 0;
let type = null;
switch (this) {
case Int8Vector:
data = toInt8Array(data);
is64 = false;
break;
case Int16Vector:
data = toInt16Array(data);
is64 = false;
break;
case Int32Vector:
data = toInt32Array(data);
is64 = false;
break;
case Int64Vector:
data = toInt32Array(data);
is64 = true;
break;
case Uint8Vector:
data = toUint8Array(data);
is64 = false;
break;
case Uint16Vector:
data = toUint16Array(data);
is64 = false;
break;
case Uint32Vector:
data = toUint32Array(data);
is64 = false;
break;
case Uint64Vector:
data = toUint32Array(data);
is64 = true;
break;
static from(...args) {
let [input, is64bit = false] = args;
let ArrowType = vectorTypeToDataType(this, is64bit);
if ((input instanceof ArrayBuffer) || ArrayBuffer.isView(input)) {
let InputType = arrayTypeToDataType(input.constructor, is64bit) || ArrowType;
// Special case, infer the Arrow DataType from the input if calling the base
// IntVector.from with a TypedArray, e.g. `IntVector.from(new Int32Array())`
if (ArrowType === null) {
ArrowType = InputType;
}
// If the DataType inferred from the Vector constructor matches the
// DataType inferred from the input arguments, return zero-copy view
if (ArrowType && ArrowType === InputType) {
let type = new ArrowType();
let length = input.byteLength / type.ArrayType.BYTES_PER_ELEMENT;
// If the ArrowType is 64bit but the input type is 32bit pairs, update the logical length
if (convert32To64Bit(ArrowType, input.constructor)) {
length *= 0.5;
}
return Vector.new(Data.Int(type, 0, length, 0, null, input));
}
}
if (is64 === true) {
length = data.length * 0.5;
type = data instanceof Int32Array ? new Int64() : new Uint64();
if (ArrowType) {
// If the DataType inferred from the Vector constructor is different than
// the DataType inferred from the input TypedArray, or if input isn't a
// TypedArray, use the Builders to construct the result Vector
return vectorFromValuesWithType(() => new ArrowType(), input);
}
else {
length = data.length;
switch (data.constructor) {
case Int8Array:
type = new Int8();
break;
case Int16Array:
type = new Int16();
break;
case Int32Array:
type = new Int32();
break;
case Uint8Array:
type = new Uint8();
break;
case Uint16Array:
type = new Uint16();
break;
case Uint32Array:
type = new Uint32();
break;
}
if ((input instanceof DataView) || (input instanceof ArrayBuffer)) {
throw new TypeError(`Cannot infer integer type from instance of ${input.constructor.name}`);
}
return type !== null
? Vector.new(Data.Int(type, 0, length, 0, null, data))
: (() => { throw new TypeError('Unrecognized IntVector input'); })();
throw new TypeError('Unrecognized IntVector input');
}

@@ -131,3 +98,35 @@ }

}
const convert32To64Bit = (typeCtor, dataCtor) => {
return (typeCtor === Int64 || typeCtor === Uint64) &&
(dataCtor === Int32Array || dataCtor === Uint32Array);
};
/** @ignore */
const arrayTypeToDataType = (ctor, is64bit) => {
switch (ctor) {
case Int8Array: return Int8;
case Int16Array: return Int16;
case Int32Array: return is64bit ? Int64 : Int32;
case BigInt64Array: return Int64;
case Uint8Array: return Uint8;
case Uint16Array: return Uint16;
case Uint32Array: return is64bit ? Uint64 : Uint32;
case BigUint64Array: return Uint64;
default: return null;
}
};
/** @ignore */
const vectorTypeToDataType = (ctor, is64bit) => {
switch (ctor) {
case Int8Vector: return Int8;
case Int16Vector: return Int16;
case Int32Vector: return is64bit ? Int64 : Int32;
case Int64Vector: return Int64;
case Uint8Vector: return Uint8;
case Uint16Vector: return Uint16;
case Uint32Vector: return is64bit ? Uint64 : Uint32;
case Uint64Vector: return Uint64;
default: return null;
}
};
//# sourceMappingURL=int.mjs.map

15

vector/map.d.ts
import { BaseVector } from './base';
import { RowProxyGenerator } from './row';
import { DataType, Map_ } from '../type';
import { DataType, Map_, Struct } from '../type';
/** @ignore */
export declare class MapVector<T extends {
[key: string]: DataType;
} = any> extends BaseVector<Map_<T>> {
asStruct(): import("./struct").StructVector<T>;
private _rowProxy;
readonly rowProxy: RowProxyGenerator<T>;
export declare class MapVector<K extends DataType = any, V extends DataType = any> extends BaseVector<Map_<K, V>> {
asList(): import("./list").ListVector<Struct<{
key: K;
value: V;
}>>;
bind(index: number): Map_<K, V>['TValue'];
}

15

vector/map.js

@@ -17,13 +17,16 @@ // Licensed to the Apache Software Foundation (ASF) under one

// under the License.
import { MapRow } from './row';
import { Vector } from '../vector';
import { BaseVector } from './base';
import { RowProxyGenerator } from './row';
import { Struct } from '../type';
import { List } from '../type';
/** @ignore */
export class MapVector extends BaseVector {
asStruct() {
return Vector.new(this.data.clone(new Struct(this.type.children)));
asList() {
const child = this.type.children[0];
return Vector.new(this.data.clone(new List(child)));
}
get rowProxy() {
return this._rowProxy || (this._rowProxy = RowProxyGenerator.new(this, this.type.children || [], true));
bind(index) {
const child = this.getChildAt(0);
const { [index]: begin, [index + 1]: end } = this.valueOffsets;
return new MapRow(child.slice(begin, end));
}

@@ -30,0 +33,0 @@ }

83

vector/row.d.ts

@@ -1,34 +0,61 @@

import { Field } from '../schema';
import { MapVector } from '../vector/map';
import { DataType } from '../type';
import { StructVector } from '../vector/struct';
/** @ignore */ export declare const kLength: unique symbol;
/** @ignore */ export declare const kParent: unique symbol;
/** @ignore */ export declare const kRowIndex: unique symbol;
/** @ignore */
export declare class Row<T extends {
[key: string]: DataType;
}> implements Iterable<T[keyof T]['TValue']> {
[key: string]: T[keyof T]['TValue'];
[kParent]: MapVector<T> | StructVector<T>;
[kRowIndex]: number;
readonly [kLength]: number;
[Symbol.iterator](): IterableIterator<T[keyof T]["TValue"]>;
get<K extends keyof T>(key: K): T[K]["TValue"];
import { Vector } from '../vector';
import { StructVector } from './struct';
import { DataType, Struct } from '../type';
/** @ignore */ declare const kParent: unique symbol;
/** @ignore */ declare const kRowIndex: unique symbol;
/** @ignore */ declare const kKeyToIdx: unique symbol;
/** @ignore */ declare const kIdxToVal: unique symbol;
/** @ignore */ declare const kCustomInspect: unique symbol;
declare abstract class Row<K extends PropertyKey = any, V = any> implements Map<K, V> {
readonly size: number;
readonly [Symbol.toStringTag]: string;
protected [kRowIndex]: number;
protected [kParent]: Vector<Struct>;
protected [kKeyToIdx]: Map<K, number>;
protected [kIdxToVal]: V[];
constructor(parent: Vector<Struct>, numKeys: number);
abstract keys(): IterableIterator<K>;
abstract values(): IterableIterator<V>;
abstract getKey(idx: number): K;
abstract getIndex(key: K): number;
abstract getValue(idx: number): V;
abstract setValue(idx: number, val: V): void;
entries(): IterableIterator<[K, V]>;
has(key: K): boolean;
get(key: K): V | undefined;
set(key: K, val: V): this;
clear(): void;
delete(_: K): boolean;
[Symbol.iterator](): IterableIterator<[K, V]>;
forEach(callbackfn: (value: V, key: K, map: Map<K, V>) => void, thisArg?: any): void;
toArray(): V[];
toJSON(): any;
inspect(): string;
[kCustomInspect](): string;
toString(): string;
protected static [Symbol.toStringTag]: string;
}
/** @ignore */
export declare class RowProxyGenerator<T extends {
export declare class MapRow<K extends DataType = any, V extends DataType = any> extends Row<K['TValue'], V['TValue'] | null> {
constructor(slice: Vector<Struct<{
key: K;
value: V;
}>>);
keys(): IterableIterator<any>;
values(): IterableIterator<any>;
getKey(idx: number): K['TValue'];
getIndex(key: K['TValue']): number;
getValue(index: number): V['TValue'] | null;
setValue(index: number, value: V['TValue'] | null): void;
}
export declare class StructRow<T extends {
[key: string]: DataType;
}> {
/** @nocollapse */
static new<T extends {
[key: string]: DataType;
}>(parent: MapVector<T> | StructVector<T>, schemaOrFields: T | Field[], fieldsAreEnumerable?: boolean): RowProxyGenerator<T>;
private rowPrototype;
private constructor();
private _bindGetter;
bind(rowIndex: number): Row<T>;
} = any> extends Row<keyof T, T[keyof T]['TValue'] | null> {
constructor(parent: StructVector<T>);
keys(): IterableIterator<keyof T>;
values(): IterableIterator<T[string]["TValue"] | null>;
getKey(idx: number): keyof T;
getIndex(key: keyof T): number;
getValue(index: number): T[keyof T]['TValue'] | null;
setValue(index: number, value: T[keyof T]['TValue'] | null): void;
}
export {};

370

vector/row.js

@@ -17,86 +17,326 @@ // Licensed to the Apache Software Foundation (ASF) under one

// under the License.
import { Field } from '../schema';
import { DataType } from '../type';
import { valueToString } from '../util/pretty';
/** @ignore */ export const kLength = Symbol.for('length');
/** @ignore */ export const kParent = Symbol.for('parent');
/** @ignore */ export const kRowIndex = Symbol.for('rowIndex');
/** @ignore */ const columnDescriptor = { enumerable: true, configurable: false, get: null };
/** @ignore */ const rowLengthDescriptor = { writable: false, enumerable: false, configurable: false, value: -1 };
/** @ignore */ const rowParentDescriptor = { writable: false, enumerable: false, configurable: false, value: null };
/** @ignore */
export class Row {
/** @ignore */ const kParent = Symbol.for('parent');
/** @ignore */ const kRowIndex = Symbol.for('rowIndex');
/** @ignore */ const kKeyToIdx = Symbol.for('keyToIdx');
/** @ignore */ const kIdxToVal = Symbol.for('idxToVal');
/** @ignore */ const kCustomInspect = Symbol.for('nodejs.util.inspect.custom');
class Row {
constructor(parent, numKeys) {
this[kParent] = parent;
this.size = numKeys;
}
entries() { return this[Symbol.iterator](); }
has(key) { return this.get(key) !== undefined; }
get(key) {
let val = undefined;
if (key !== null && key !== undefined) {
const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map());
let idx = ktoi.get(key);
if (idx !== undefined) {
const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size));
((val = itov[idx]) !== undefined) || (itov[idx] = val = this.getValue(idx));
}
else if ((idx = this.getIndex(key)) > -1) {
ktoi.set(key, idx);
const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size));
((val = itov[idx]) !== undefined) || (itov[idx] = val = this.getValue(idx));
}
}
return val;
}
set(key, val) {
if (key !== null && key !== undefined) {
const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map());
let idx = ktoi.get(key);
if (idx === undefined) {
ktoi.set(key, idx = this.getIndex(key));
}
if (idx > -1) {
const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size));
itov[idx] = this.setValue(idx, val);
}
}
return this;
}
clear() { throw new Error(`Clearing ${this[Symbol.toStringTag]} not supported.`); }
delete(_) { throw new Error(`Deleting ${this[Symbol.toStringTag]} values not supported.`); }
*[Symbol.iterator]() {
for (let i = -1, n = this[kLength]; ++i < n;) {
yield this[i];
const ki = this.keys();
const vi = this.values();
const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map());
const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size));
for (let k, v, i = 0, kr, vr; !((kr = ki.next()).done || (vr = vi.next()).done); ++i) {
k = kr.value;
v = vr.value;
itov[i] = v;
ktoi.has(k) || ktoi.set(k, i);
yield [k, v];
}
}
get(key) {
return this[key];
forEach(callbackfn, thisArg) {
const ki = this.keys();
const vi = this.values();
const callback = thisArg === undefined ? callbackfn :
(v, k, m) => callbackfn.call(thisArg, v, k, m);
const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map());
const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size));
for (let k, v, i = 0, kr, vr; !((kr = ki.next()).done || (vr = vi.next()).done); ++i) {
k = kr.value;
v = vr.value;
itov[i] = v;
ktoi.has(k) || ktoi.set(k, i);
callback(v, k, this);
}
}
toArray() { return [...this.values()]; }
toJSON() {
return DataType.isStruct(this[kParent].type) ? [...this] :
this[kParent].type.children.reduce((props, { name }) => {
return (props[name] = this[name]) && props || props;
}, {});
const obj = {};
this.forEach((val, key) => obj[key] = val);
return obj;
}
inspect() { return this.toString(); }
[Symbol.for('nodejs.util.inspect.custom')]() { return this.toString(); }
[kCustomInspect]() { return this.toString(); }
toString() {
return DataType.isStruct(this[kParent].type) ?
`[ ${[...this].map((x) => valueToString(x)).join(', ')} ]` :
`{ ${this[kParent].type.children.reduce((xs, { name }) => {
return [...xs, `"${name}": ${valueToString(this[name])}`];
}, []).join(', ')} }`;
const str = [];
this.forEach((val, key) => {
key = valueToString(key);
val = valueToString(val);
str.push(`${key}: ${val}`);
});
return `{ ${str.join(', ')} }`;
}
}
/** @ignore */
export class RowProxyGenerator {
constructor(parent, fields, fieldsAreEnumerable) {
const proto = Object.create(Row.prototype);
rowParentDescriptor.value = parent;
rowLengthDescriptor.value = fields.length;
Object.defineProperty(proto, kParent, rowParentDescriptor);
Object.defineProperty(proto, kLength, rowLengthDescriptor);
fields.forEach((field, columnIndex) => {
if (!proto.hasOwnProperty(field.name)) {
columnDescriptor.enumerable = fieldsAreEnumerable;
columnDescriptor.get || (columnDescriptor.get = this._bindGetter(columnIndex));
Object.defineProperty(proto, field.name, columnDescriptor);
}
if (!proto.hasOwnProperty(columnIndex)) {
columnDescriptor.enumerable = !fieldsAreEnumerable;
columnDescriptor.get || (columnDescriptor.get = this._bindGetter(columnIndex));
Object.defineProperty(proto, columnIndex, columnDescriptor);
}
columnDescriptor.get = null;
});
this.rowPrototype = proto;
Row[Symbol.toStringTag] = ((proto) => {
Object.defineProperties(proto, {
'size': { writable: true, enumerable: false, configurable: false, value: 0 },
[kParent]: { writable: true, enumerable: false, configurable: false, value: null },
[kRowIndex]: { writable: true, enumerable: false, configurable: false, value: -1 },
});
return proto[Symbol.toStringTag] = 'Row';
})(Row.prototype);
export class MapRow extends Row {
constructor(slice) {
super(slice, slice.length);
return createRowProxy(this);
}
/** @nocollapse */
static new(parent, schemaOrFields, fieldsAreEnumerable = false) {
let schema, fields;
if (Array.isArray(schemaOrFields)) {
fields = schemaOrFields;
keys() {
return this[kParent].getChildAt(0)[Symbol.iterator]();
}
values() {
return this[kParent].getChildAt(1)[Symbol.iterator]();
}
getKey(idx) {
return this[kParent].getChildAt(0).get(idx);
}
getIndex(key) {
return this[kParent].getChildAt(0).indexOf(key);
}
getValue(index) {
return this[kParent].getChildAt(1).get(index);
}
setValue(index, value) {
this[kParent].getChildAt(1).set(index, value);
}
}
export class StructRow extends Row {
constructor(parent) {
super(parent, parent.type.children.length);
return defineRowProxyProperties(this);
}
*keys() {
for (const field of this[kParent].type.children) {
yield field.name;
}
else {
schema = schemaOrFields;
fieldsAreEnumerable = true;
fields = Object.keys(schema).map((x) => new Field(x, schema[x]));
}
*values() {
for (const field of this[kParent].type.children) {
yield this[field.name];
}
return new RowProxyGenerator(parent, fields, fieldsAreEnumerable);
}
_bindGetter(columnIndex) {
return function () {
const child = this[kParent].getChildAt(columnIndex);
return child ? child.get(this[kRowIndex]) : null;
};
getKey(idx) {
return this[kParent].type.children[idx].name;
}
bind(rowIndex) {
const bound = Object.create(this.rowPrototype);
bound[kRowIndex] = rowIndex;
return bound;
getIndex(key) {
return this[kParent].type.children.findIndex((f) => f.name === key);
}
getValue(index) {
return this[kParent].getChildAt(index).get(this[kRowIndex]);
}
setValue(index, value) {
return this[kParent].getChildAt(index).set(this[kRowIndex], value);
}
}
Object.setPrototypeOf(Row.prototype, Map.prototype);
/** @ignore */
const defineRowProxyProperties = (() => {
const desc = { enumerable: true, configurable: false, get: null, set: null };
return (row) => {
let idx = -1, ktoi = row[kKeyToIdx] || (row[kKeyToIdx] = new Map());
const getter = (key) => function () { return this.get(key); };
const setter = (key) => function (val) { return this.set(key, val); };
for (const key of row.keys()) {
ktoi.set(key, ++idx);
desc.get = getter(key);
desc.set = setter(key);
row.hasOwnProperty(key) || (desc.enumerable = true, Object.defineProperty(row, key, desc));
row.hasOwnProperty(idx) || (desc.enumerable = false, Object.defineProperty(row, idx, desc));
}
desc.get = desc.set = null;
return row;
};
})();
/** @ignore */
const createRowProxy = (() => {
if (typeof Proxy === 'undefined') {
return defineRowProxyProperties;
}
const has = Row.prototype.has;
const get = Row.prototype.get;
const set = Row.prototype.set;
const getKey = Row.prototype.getKey;
const RowProxyHandler = {
isExtensible() { return false; },
deleteProperty() { return false; },
preventExtensions() { return true; },
ownKeys(row) { return [...row.keys()].map((x) => `${x}`); },
has(row, key) {
switch (key) {
case 'getKey':
case 'getIndex':
case 'getValue':
case 'setValue':
case 'toArray':
case 'toJSON':
case 'inspect':
case 'constructor':
case 'isPrototypeOf':
case 'propertyIsEnumerable':
case 'toString':
case 'toLocaleString':
case 'valueOf':
case 'size':
case 'has':
case 'get':
case 'set':
case 'clear':
case 'delete':
case 'keys':
case 'values':
case 'entries':
case 'forEach':
case '__proto__':
case '__defineGetter__':
case '__defineSetter__':
case 'hasOwnProperty':
case '__lookupGetter__':
case '__lookupSetter__':
case Symbol.iterator:
case Symbol.toStringTag:
case kParent:
case kRowIndex:
case kIdxToVal:
case kKeyToIdx:
case kCustomInspect:
return true;
}
if (typeof key === 'number' && !row.has(key)) {
key = row.getKey(key);
}
return row.has(key);
},
get(row, key, receiver) {
switch (key) {
case 'getKey':
case 'getIndex':
case 'getValue':
case 'setValue':
case 'toArray':
case 'toJSON':
case 'inspect':
case 'constructor':
case 'isPrototypeOf':
case 'propertyIsEnumerable':
case 'toString':
case 'toLocaleString':
case 'valueOf':
case 'size':
case 'has':
case 'get':
case 'set':
case 'clear':
case 'delete':
case 'keys':
case 'values':
case 'entries':
case 'forEach':
case '__proto__':
case '__defineGetter__':
case '__defineSetter__':
case 'hasOwnProperty':
case '__lookupGetter__':
case '__lookupSetter__':
case Symbol.iterator:
case Symbol.toStringTag:
case kParent:
case kRowIndex:
case kIdxToVal:
case kKeyToIdx:
case kCustomInspect:
return Reflect.get(row, key, receiver);
}
if (typeof key === 'number' && !has.call(receiver, key)) {
key = getKey.call(receiver, key);
}
return get.call(receiver, key);
},
set(row, key, val, receiver) {
switch (key) {
case kParent:
case kRowIndex:
case kIdxToVal:
case kKeyToIdx:
return Reflect.set(row, key, val, receiver);
case 'getKey':
case 'getIndex':
case 'getValue':
case 'setValue':
case 'toArray':
case 'toJSON':
case 'inspect':
case 'constructor':
case 'isPrototypeOf':
case 'propertyIsEnumerable':
case 'toString':
case 'toLocaleString':
case 'valueOf':
case 'size':
case 'has':
case 'get':
case 'set':
case 'clear':
case 'delete':
case 'keys':
case 'values':
case 'entries':
case 'forEach':
case '__proto__':
case '__defineGetter__':
case '__defineSetter__':
case 'hasOwnProperty':
case '__lookupGetter__':
case '__lookupSetter__':
case Symbol.iterator:
case Symbol.toStringTag:
return false;
}
if (typeof key === 'number' && !has.call(receiver, key)) {
key = getKey.call(receiver, key);
}
return has.call(receiver, key) ? !!set.call(receiver, key, val) : false;
},
};
return (row) => new Proxy(row, RowProxyHandler);
})();
//# sourceMappingURL=row.mjs.map

6

vector/struct.d.ts
import { BaseVector } from './base';
import { RowProxyGenerator } from './row';
import { DataType, Struct } from '../type';

@@ -8,5 +7,4 @@ /** @ignore */

} = any> extends BaseVector<Struct<T>> {
asMap(keysSorted?: boolean): import("./map").MapVector<T>;
private _rowProxy;
readonly rowProxy: RowProxyGenerator<T>;
private _row;
bind(index: number): Struct<T>['TValue'];
}

15

vector/struct.js

@@ -17,16 +17,15 @@ // Licensed to the Apache Software Foundation (ASF) under one

// under the License.
import { Vector } from '../vector';
import { StructRow } from './row';
import { BaseVector } from './base';
import { RowProxyGenerator } from './row';
import { Map_ } from '../type';
/** @ignore */ const kRowIndex = Symbol.for('rowIndex');
/** @ignore */
export class StructVector extends BaseVector {
asMap(keysSorted = false) {
return Vector.new(this.data.clone(new Map_(this.type.children, keysSorted)));
bind(index) {
const proto = this._row || (this._row = new StructRow(this));
const bound = Object.create(proto);
bound[kRowIndex] = index;
return bound;
}
get rowProxy() {
return this._rowProxy || (this._rowProxy = RowProxyGenerator.new(this, this.type.children || [], false));
}
}
//# sourceMappingURL=struct.mjs.map

15

visitor/get.js

@@ -20,2 +20,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

import { decodeUtf8 } from '../util/utf8';
import { uint16ToFloat64 } from '../util/math';
import { UnionMode, Precision, DateUnit, TimeUnit, IntervalUnit } from '../enum';

@@ -52,3 +53,3 @@ /** @ignore */

/** @ignore */
const getFloat16 = ({ stride, values }, index) => (values[stride * index] - 32767) / 32767;
const getFloat16 = ({ stride, values }, index) => uint16ToFloat64(values[stride * index]);
/** @ignore */

@@ -124,5 +125,9 @@ const getBigInts = ({ stride, values, type }, index) => BN.new(values.subarray(stride * index, stride * (index + 1)), type.isSigned);

/** @ignore */
const getNested = (vector, index) => {
return vector.rowProxy.bind(index);
const getMap = (vector, index) => {
return vector.bind(index);
};
/** @ignore */
const getStruct = (vector, index) => {
return vector.bind(index);
};
/* istanbul ignore next */

@@ -204,3 +209,3 @@ /** @ignore */

GetVisitor.prototype.visitList = getList;
GetVisitor.prototype.visitStruct = getNested;
GetVisitor.prototype.visitStruct = getStruct;
GetVisitor.prototype.visitUnion = getUnion;

@@ -214,3 +219,3 @@ GetVisitor.prototype.visitDenseUnion = getDenseUnion;

GetVisitor.prototype.visitFixedSizeList = getFixedSizeList;
GetVisitor.prototype.visitMap = getNested;
GetVisitor.prototype.visitMap = getMap;
/** @ignore */

@@ -217,0 +222,0 @@ export const instance = new GetVisitor();

9

visitor/jsonvectorassembler.js

@@ -41,9 +41,9 @@ // Licensed to the Apache Software Foundation (ASF) under one

'count': length,
'VALIDITY': nullCount <= 0
? Array.from({ length }, () => 1)
: [...iterateBits(nullBitmap, offset, length, null, getBit)],
'VALIDITY': DataType.isNull(type) ? undefined
: nullCount <= 0 ? Array.from({ length }, () => 1)
: [...iterateBits(nullBitmap, offset, length, null, getBit)],
...super.visit(Vector.new(data.clone(type, offset, length, 0, buffers)))
};
}
visitNull() { return { 'DATA': [] }; }
visitNull() { return {}; }
visitBool({ values, offset, length }) {

@@ -119,2 +119,3 @@ return { 'DATA': [...iterateBits(values, offset, length, null, getBool)] };

return {
'OFFSET': [...vector.valueOffsets],
'children': vector.type.children.map((f, i) => this.visit(new Column(f, [vector.getChildAt(i)])))

@@ -121,0 +122,0 @@ };

39

visitor/set.js

@@ -17,4 +17,6 @@ // Licensed to the Apache Software Foundation (ASF) under one

// under the License.
import { Vector } from '../vector';
import { Visitor } from '../visitor';
import { encodeUtf8 } from '../util/utf8';
import { float64ToUint16 } from '../util/math';
import { toArrayBufferView } from '../util/buffer';

@@ -62,3 +64,3 @@ import { UnionMode, Precision, DateUnit, TimeUnit, IntervalUnit } from '../enum';

/** @ignore */
const setFloat16 = ({ stride, values }, index, value) => { values[stride * index] = (value * 32767) + 32767; };
const setFloat16 = ({ stride, values }, index, value) => { values[stride * index] = float64ToUint16(value); };
/** @ignore */

@@ -148,23 +150,26 @@ const setNumericX2 = (vector, index, value) => {

const setList = (vector, index, value) => {
const values = vector.getChildAt(0);
const { valueOffsets } = vector;
let idx = -1, offset = valueOffsets[index];
let end = Math.min(offset + value.length, valueOffsets[index + 1]);
while (offset < end) {
values.set(offset++, value.get(++idx));
const values = vector.getChildAt(0), valueOffsets = vector.valueOffsets;
for (let idx = -1, itr = valueOffsets[index], end = valueOffsets[index + 1]; itr < end;) {
values.set(itr++, value.get(++idx));
}
};
/** @ignore */
const setStruct = (vector, index, value) => {
vector.type.children.forEach((_field, idx) => {
const child = vector.getChildAt(idx);
child && child.set(index, value[idx]);
});
const setMap = (vector, index, value) => {
const values = vector.getChildAt(0), valueOffsets = vector.valueOffsets;
const entries = value instanceof Map ? [...value] : Object.entries(value);
for (let idx = -1, itr = valueOffsets[index], end = valueOffsets[index + 1]; itr < end;) {
values.set(itr++, entries[++idx]);
}
};
/** @ignore */ const _setStructArrayValue = (o, v) => (c, _, i) => c && c.set(o, v[i]);
/** @ignore */ const _setStructVectorValue = (o, v) => (c, _, i) => c && c.set(o, v.get(i));
/** @ignore */ const _setStructMapValue = (o, v) => (c, f, _) => c && c.set(o, v.get(f.name));
/** @ignore */ const _setStructObjectValue = (o, v) => (c, f, _) => c && c.set(o, v[f.name]);
/** @ignore */
const setMap = (vector, index, value) => {
vector.type.children.forEach(({ name }, idx) => {
const child = vector.getChildAt(idx);
child && child.set(index, value[name]);
});
const setStruct = (vector, index, value) => {
const setValue = value instanceof Map ? _setStructMapValue(index, value) :
value instanceof Vector ? _setStructVectorValue(index, value) :
Array.isArray(value) ? _setStructArrayValue(index, value) :
_setStructObjectValue(index, value);
vector.type.children.forEach((f, i) => setValue(vector.getChildAt(i), f, i));
};

@@ -171,0 +176,0 @@ /* istanbul ignore next */

2

visitor/vectorassembler.d.ts

@@ -37,3 +37,3 @@ import { Data } from '../data';

private constructor();
visitNull<T extends Null>(_nullV: V<T>): VectorAssembler;
visitNull<T extends Null>(_nullV: V<T>): this;
visitDictionary<T extends Dictionary>(vector: V<T>): this;

@@ -40,0 +40,0 @@ readonly nodes: FieldNode[];

15

visitor/vectorassembler.js

@@ -48,5 +48,8 @@ // Licensed to the Apache Software Foundation (ASF) under one

}
addBuffer.call(this, nullCount <= 0
? new Uint8Array(0) // placeholder validity buffer
: truncateBitmap(data.offset, length, data.nullBitmap)).nodes.push(new FieldNode(length, nullCount));
if (!DataType.isNull(vector.type)) {
addBuffer.call(this, nullCount <= 0
? new Uint8Array(0) // placeholder validity buffer
: truncateBitmap(data.offset, length, data.nullBitmap));
}
this.nodes.push(new FieldNode(length, nullCount));
}

@@ -56,3 +59,3 @@ return super.visit(vector);

visitNull(_nullV) {
return addBuffer.call(this, new Uint8Array(0));
return this;
}

@@ -163,3 +166,3 @@ visitDictionary(vector) {

const { length, valueOffsets } = vector;
// If we have valueOffsets (ListVector), push that buffer first
// If we have valueOffsets (MapVector, ListVector), push that buffer first
if (valueOffsets) {

@@ -190,4 +193,4 @@ addBuffer.call(this, rebaseValueOffsets(valueOffsets[0], length, valueOffsets));

VectorAssembler.prototype.visitFixedSizeList = assembleListVector;
VectorAssembler.prototype.visitMap = assembleNestedVector;
VectorAssembler.prototype.visitMap = assembleListVector;
//# sourceMappingURL=vectorassembler.mjs.map

2

visitor/vectorloader.d.ts

@@ -22,3 +22,3 @@ import { Data } from '../data';

constructor(bytes: Uint8Array, nodes: FieldNode[], buffers: BufferRegion[], dictionaries: Map<number, Vector<any>>);
visitNull<T extends type.Null>(type: T, { length, nullCount }?: FieldNode): Data<T>;
visitNull<T extends type.Null>(type: T, { length, }?: FieldNode): Data<T>;
visitBool<T extends type.Bool>(type: T, { length, nullCount }?: FieldNode): Data<T>;

@@ -25,0 +25,0 @@ visitInt<T extends type.Int>(type: T, { length, nullCount }?: FieldNode): Data<T>;

4

visitor/vectorloader.js

@@ -40,3 +40,3 @@ // Licensed to the Apache Software Foundation (ASF) under one

}
visitNull(type, { length, nullCount } = this.nextFieldNode()) { return Data.Null(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); }
visitNull(type, { length, } = this.nextFieldNode()) { return Data.Null(type, 0, length); }
visitBool(type, { length, nullCount } = this.nextFieldNode()) { return Data.Bool(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); }

@@ -60,3 +60,3 @@ visitInt(type, { length, nullCount } = this.nextFieldNode()) { return Data.Int(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); }

visitFixedSizeList(type, { length, nullCount } = this.nextFieldNode()) { return Data.FixedSizeList(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.visit(type.children[0])); }
visitMap(type, { length, nullCount } = this.nextFieldNode()) { return Data.Map(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.visitMany(type.children)); }
visitMap(type, { length, nullCount } = this.nextFieldNode()) { return Data.Map(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.visit(type.children[0])); }
nextFieldNode() { return this.nodes[++this.nodesIndex]; }

@@ -63,0 +63,0 @@ nextBufferRange() { return this.buffers[++this.buffersIndex]; }

Arrow.mjs.map

Sorry, the diff of this file is not supported yet

bin/arrow2csv.js.map

Sorry, the diff of this file is not supported yet

bin/arrow2csv.mjs.map

Sorry, the diff of this file is not supported yet

builder.mjs.map

Sorry, the diff of this file is not supported yet

builder/float.mjs.map

Sorry, the diff of this file is not supported yet

builder/map.mjs.map

Sorry, the diff of this file is not supported yet

builder/null.mjs.map

Sorry, the diff of this file is not supported yet

compute/dataframe.mjs.map

Sorry, the diff of this file is not supported yet

data.mjs.map

Sorry, the diff of this file is not supported yet

fb/File.mjs.map

Sorry, the diff of this file is not supported yet

fb/Message.mjs.map

Sorry, the diff of this file is not supported yet

fb/Schema.js

Sorry, the diff of this file is too big to display

fb/Schema.mjs.map

Sorry, the diff of this file is not supported yet

interfaces.mjs.map

Sorry, the diff of this file is not supported yet

ipc/message.mjs.map

Sorry, the diff of this file is not supported yet

ipc/metadata/json.mjs.map

Sorry, the diff of this file is not supported yet

ipc/metadata/message.mjs.map

Sorry, the diff of this file is not supported yet

ipc/writer.mjs.map

Sorry, the diff of this file is not supported yet

LICENSE.txt

Sorry, the diff of this file is too big to display

recordbatch.mjs.map

Sorry, the diff of this file is not supported yet

table.mjs.map

Sorry, the diff of this file is not supported yet

type.mjs.map

Sorry, the diff of this file is not supported yet

util/buffer.mjs.map

Sorry, the diff of this file is not supported yet

util/vector.mjs.map

Sorry, the diff of this file is not supported yet

vector/base.mjs.map

Sorry, the diff of this file is not supported yet

vector/chunked.mjs.map

Sorry, the diff of this file is not supported yet

vector/float.mjs.map

Sorry, the diff of this file is not supported yet

vector/index.mjs.map

Sorry, the diff of this file is not supported yet

vector/int.mjs.map

Sorry, the diff of this file is not supported yet

vector/map.mjs.map

Sorry, the diff of this file is not supported yet

vector/row.mjs.map

Sorry, the diff of this file is not supported yet

vector/struct.mjs.map

Sorry, the diff of this file is not supported yet

visitor/get.mjs.map

Sorry, the diff of this file is not supported yet

visitor/jsonvectorassembler.mjs.map

Sorry, the diff of this file is not supported yet

visitor/set.mjs.map

Sorry, the diff of this file is not supported yet

visitor/vectorassembler.mjs.map

Sorry, the diff of this file is not supported yet

visitor/vectorloader.mjs.map

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc