@dsnp/parquetjs - npm Package Compare versions

Comparing version 0.0.0-91ac19 to 0.0.0-91fc71

dist/lib/bloom/sbbf.js

@@ -7,3 +7,3 @@ "use strict";
 const parquet_types_1 = __importDefault(require("../../gen-nodejs/parquet_types"));
-const Long = require("long");
+const long_1 = __importDefault(require("long"));
 const xxhasher_1 = __importDefault(require("./xxhasher"));
@@ -106,4 +106,4 @@ /**
     static getBlockIndex(h, z) {
-        const zLong = Long.fromNumber(z, true);
-        const hTopBits = Long.fromNumber(h.getHighBitsUnsigned(), true);
+        const zLong = long_1.default.fromNumber(z, true);
+        const hTopBits = long_1.default.fromNumber(h.getHighBitsUnsigned(), true);
         return hTopBits.mul(zLong).shiftRightUnsigned(32).getLowBitsUnsigned();
@@ -330,3 +330,3 @@ }
         const hashed = await this.hasher.hash64(value);
-        return Long.fromString(hashed, true, 16);
+        return long_1.default.fromString(hashed, true, 16);
     }
@@ -374,2 +374,1 @@ insertHash(hashValue) {
 exports.default = SplitBlockBloomFilter;
-//# sourceMappingURL=sbbf.js.map

dist/lib/bloom/xxhasher.js

@@ -19,4 +19,4 @@ "use strict";
 class XxHasher {
-    static h64 = (0, xxhash_wasm_1.default)().then(x => x.h64);
-    async hashit(value) {
+    static h64 = (0, xxhash_wasm_1.default)().then(x => x.h64ToString);
+    async hashIt(value) {
         return (await XxHasher.h64)(value);
@@ -26,9 +26,10 @@ }
      * @function hash64
-     * @description creates a hash for certain data types.
-     * @return the 64 big XXHash as a string
-     * @param value one of n, throw an error.
+     * @description creates a hash for certain data types. All data is converted using toString()
+     * prior to hashing.
+     * @return the 64 big XXHash as a hex-encoded string.
+     * @param value, must be of type string, Buffer, Uint8Array, Long, boolean, number, or bigint
      */
     async hash64(value) {
         if (typeof value === 'string')
-            return this.hashit(value);
+            return this.hashIt(value);
         if (value instanceof Buffer ||
@@ -40,3 +41,3 @@ value instanceof Uint8Array ||
             typeof value === 'bigint') {
-            return this.hashit(value.toString());
+            return this.hashIt(value.toString());
         }
@@ -47,2 +48,1 @@ throw new Error("unsupported type: " + value);
 exports.default = XxHasher;
-//# sourceMappingURL=xxhasher.js.map

dist/lib/bloomFilterIO/bloomFilterReader.js

 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
@@ -31,4 +35,3 @@ if (k2 === undefined) k2 = k;
     return columnChunkDataCollection.filter((columnChunk) => {
-        const { column: { meta_data: { bloom_filter_offset: { buffer: bloomFilterOffsetBuffer }, }, }, } = columnChunk;
-        return bloomFilterOffsetBuffer;
+        return columnChunk.column.meta_data?.bloom_filter_offset;
     });
@@ -44,6 +47,6 @@ };
 const parseBloomFilterOffsets = (ColumnChunkDataCollection) => {
-    return ColumnChunkDataCollection.map((columnChunkData) => {
-        const { column: { meta_data: { bloom_filter_offset: { buffer: bloomFilterOffsetBuffer }, path_in_schema: pathInSchema, }, }, rowGroupIndex, } = columnChunkData;
+    return ColumnChunkDataCollection.map(({ rowGroupIndex, column }) => {
+        const { bloom_filter_offset: bloomOffset, path_in_schema: pathInSchema, } = column.meta_data || {};
         return {
-            offsetBytes: toInteger(bloomFilterOffsetBuffer),
+            offsetBytes: toInteger(bloomOffset.buffer),
             columnName: pathInSchema.join(","),
@@ -96,4 +99,4 @@ rowGroupIndex,
 exports.siftAllByteOffsets = siftAllByteOffsets;
-const getBloomFiltersFor = async (columnNames, envelopeReader) => {
-    const columnChunkDataCollection = envelopeReader.getAllColumnChunkDataFor(columnNames);
+const getBloomFiltersFor = async (paths, envelopeReader) => {
+    const columnChunkDataCollection = envelopeReader.getAllColumnChunkDataFor(paths);
     const bloomFilterOffsetData = (0, exports.siftAllByteOffsets)(columnChunkDataCollection);
@@ -112,2 +115,1 @@ const offsetByteValues = bloomFilterOffsetData.map(({ offsetBytes }) => offsetBytes);
 exports.getBloomFiltersFor = getBloomFiltersFor;
-//# sourceMappingURL=bloomFilterReader.js.map

dist/lib/bloomFilterIO/bloomFilterWriter.js

 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
@@ -65,3 +69,3 @@ if (k2 === undefined) k2 = k;
 const setFilterOffset = (column, offset) => {
-    column.meta_data.bloom_filter_offset = offset;
+    column.meta_data.bloom_filter_offset = parquet_util.cloneInteger(offset);
 };
@@ -75,2 +79,1 @@ exports.setFilterOffset = setFilterOffset;
 exports.getSerializedBloomFilterData = getSerializedBloomFilterData;
-//# sourceMappingURL=bloomFilterWriter.js.map

dist/lib/browser/compression.js

@@ -28,3 +28,3 @@ 'use strict';
 function deflate_identity(value) {
-    return value;
+    return buffer_from_result(value);
 }
@@ -35,3 +35,4 @@ function deflate_gzip(value) {
 function deflate_snappy(value) {
-    return snappy.compress(value);
+    const compressedValue = snappy.compress(value);
+    return buffer_from_result(compressedValue);
 }
@@ -48,3 +49,3 @@ /**
 function inflate_identity(value) {
-    return value;
+    return buffer_from_result(value);
 }
@@ -55,7 +56,15 @@ function inflate_gzip(value) {
 function inflate_snappy(value) {
-    return snappy.uncompress(value);
+    const uncompressedValue = snappy.uncompress(value);
+    return buffer_from_result(uncompressedValue);
 }
+function buffer_from_result(result) {
+    if (Buffer.isBuffer(result)) {
+        return result;
+    }
+    else {
+        return Buffer.from(result);
+    }
+}
 exports.PARQUET_COMPRESSION_METHODS = PARQUET_COMPRESSION_METHODS;
 exports.deflate = deflate;
 exports.inflate = inflate;
-//# sourceMappingURL=compression.js.map

dist/lib/bufferReader.js

@@ -19,3 +19,3 @@ "use strict";
     }
-    async read(offset, length) {
+    read(offset, length) {
         if (!this.scheduled) {
@@ -50,3 +50,3 @@ this.scheduled = true;
             processQueue.forEach(async (d) => {
-                d.resolve(buffer.slice(d.offset - start, d.offset + d.length - start));
+                d.resolve(buffer.subarray(d.offset - start, d.offset + d.length - start));
             });
@@ -72,2 +72,1 @@ };
 ;
-//# sourceMappingURL=bufferReader.js.map

dist/lib/codec/index.js

 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
@@ -22,6 +26,6 @@ if (k2 === undefined) k2 = k;
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.PLAIN_DICTIONARY = exports.RLE = exports.PLAIN = void 0;
+exports.RLE_DICTIONARY = exports.PLAIN_DICTIONARY = exports.RLE = exports.PLAIN = void 0;
 exports.PLAIN = __importStar(require("./plain"));
 exports.RLE = __importStar(require("./rle"));
 exports.PLAIN_DICTIONARY = __importStar(require("./plain_dictionary"));
-//# sourceMappingURL=index.js.map
+exports.RLE_DICTIONARY = __importStar(require("./plain_dictionary"));

dist/lib/codec/plain_dictionary.js

 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
@@ -25,7 +29,6 @@ if (k2 === undefined) k2 = k;
 const decodeValues = function (type, cursor, count, opts) {
-    opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
+    const bitWidth = cursor.buffer.subarray(cursor.offset, cursor.offset + 1).readInt8(0);
     cursor.offset += 1;
-    return rle.decodeValues(type, cursor, count, Object.assign({}, opts, { disableEnvelope: true }));
+    return rle.decodeValues(type, cursor, count, Object.assign({}, opts, { disableEnvelope: true, bitWidth }));
 };
 exports.decodeValues = decodeValues;
-//# sourceMappingURL=plain_dictionary.js.map

dist/lib/codec/plain.js

@@ -27,29 +27,96 @@ "use strict";
 }
-function encodeValues_INT32(values) {
+function encodeValues_INT32(values, opts) {
+    const isDecimal = opts?.originalType === 'DECIMAL' || opts?.column?.originalType === 'DECIMAL';
+    const scale = opts?.scale || 0;
     let buf = Buffer.alloc(4 * values.length);
     for (let i = 0; i < values.length; i++) {
-        buf.writeInt32LE(values[i], i * 4);
+        if (isDecimal) {
+            buf.writeInt32LE(values[i] * Math.pow(10, scale), i * 4);
+        }
+        else {
+            buf.writeInt32LE(values[i], i * 4);
+        }
     }
     return buf;
 }
-function decodeValues_INT32(cursor, count) {
+function decodeValues_INT32(cursor, count, opts) {
     let values = [];
-    for (let i = 0; i < count; ++i) {
-        values.push(cursor.buffer.readInt32LE(cursor.offset));
-        cursor.offset += 4;
+    const name = opts.name || opts.column?.name || undefined;
+    try {
+        if (opts.originalType === 'DECIMAL') {
+            values = decodeValues_DECIMAL(cursor, count, opts);
+        }
+        else {
+            for (let i = 0; i < count; ++i) {
+                values.push(cursor.buffer.readInt32LE(cursor.offset));
+                cursor.offset += 4;
+            }
+        }
     }
+    catch (e) {
+        console.log(`Error thrown for column: ${name}`);
+        throw e;
+    }
     return values;
 }
-function encodeValues_INT64(values) {
+function encodeValues_INT64(values, opts) {
+    const isDecimal = opts?.originalType === 'DECIMAL' || opts?.column?.originalType === 'DECIMAL';
+    const scale = opts?.scale || 0;
     let buf = Buffer.alloc(8 * values.length);
     for (let i = 0; i < values.length; i++) {
-        buf.writeBigInt64LE(BigInt(values[i]), i * 8);
+        if (isDecimal) {
+            buf.writeBigInt64LE(BigInt(Math.floor(values[i] * Math.pow(10, scale))), i * 8);
+        }
+        else {
+            buf.writeBigInt64LE(BigInt(values[i]), i * 8);
+        }
     }
     return buf;
 }
-function decodeValues_INT64(cursor, count) {
+function decodeValues_INT64(cursor, count, opts) {
     let values = [];
+    const name = opts.name || opts.column?.name || undefined;
+    try {
+        if (opts.originalType === 'DECIMAL' || opts.column?.originalType === 'DECIMAL') {
+            let columnOptions = opts.column?.originalType ? opts.column : opts;
+            values = decodeValues_DECIMAL(cursor, count, columnOptions);
+        }
+        else {
+            for (let i = 0; i < count; ++i) {
+                values.push(cursor.buffer.readBigInt64LE(cursor.offset));
+                cursor.offset += 8;
+            }
+        }
+    }
+    catch (e) {
+        console.log(`Error thrown for column: ${name}`);
+        throw e;
+    }
+    return values;
+}
+function decodeValues_DECIMAL(cursor, count, opts) {
+    const precision = opts.precision;
+    // Default scale to 0 per spec
+    const scale = opts.scale || 0;
+    const name = opts.name || undefined;
+    if (!precision) {
+        throw `missing option: precision (required for DECIMAL) for column: ${name}`;
+    }
+    let values = [];
+    // by default we prepare the offset and bufferFunction to work with 32bit integers
+    let offset = 4;
+    let bufferFunction = (offset) => cursor.buffer.readInt32LE(offset);
+    if (precision > 9) {
+        // if the precision is over 9 digits, then we are dealing with a 64bit integer
+        offset = 8;
+        bufferFunction = (offset) => cursor.buffer.readBigInt64LE(offset);
+    }
     for (let i = 0; i < count; ++i) {
-        values.push(cursor.buffer.readBigInt64LE(cursor.offset));
-        cursor.offset += 8;
+        const bufferSize = cursor.size || 0;
+        if (bufferSize === 0 || cursor.offset < bufferSize) {
+            const fullValue = bufferFunction(cursor.offset);
+            const valueWithDecimalApplied = Number(fullValue) / Math.pow(10, scale);
+            values.push(valueWithDecimalApplied);
+            cursor.offset += offset;
+        }
     }
@@ -117,3 +184,2 @@ return values;
 }
-// Waylands reminder to check again
 function encodeValues_BYTE_ARRAY(values) {
@@ -140,3 +206,3 @@ let buf_len = 0;
         cursor.offset += 4;
-        values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
+        values.push(cursor.buffer.subarray(cursor.offset, cursor.offset + len));
         cursor.offset += len;
@@ -161,8 +227,9 @@ }
     let values = [];
-    if (!opts.typeLength) {
+    const typeLength = opts.typeLength ?? (opts.column ? opts.column.typeLength : undefined);
+    if (!typeLength) {
         throw "missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)";
     }
     for (let i = 0; i < count; ++i) {
-        values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));
-        cursor.offset += opts.typeLength;
+        values.push(cursor.buffer.subarray(cursor.offset, cursor.offset + typeLength));
+        cursor.offset += typeLength;
     }
@@ -176,5 +243,5 @@ return values;
         case "INT32":
-            return encodeValues_INT32(values);
+            return encodeValues_INT32(values, opts);
         case "INT64":
-            return encodeValues_INT64(values);
+            return encodeValues_INT64(values, opts);
         case "INT96":
@@ -200,5 +267,5 @@ return encodeValues_INT96(values);
         case "INT32":
-            return decodeValues_INT32(cursor, count);
+            return decodeValues_INT32(cursor, count, opts);
         case "INT64":
-            return decodeValues_INT64(cursor, count);
+            return decodeValues_INT64(cursor, count, opts);
         case "INT96":
@@ -219,2 +286,1 @@ return decodeValues_INT96(cursor, count);
 exports.decodeValues = decodeValues;
-//# sourceMappingURL=plain.js.map

dist/lib/codec/rle.js

 "use strict";
+// For questions about RLE encoding, see the spec:
+//
+// https://github.com/apache/parquet-format/blob/master/Encodings.md
 var __importDefault = (this && this.__importDefault) || function (mod) {
@@ -25,5 +28,8 @@ return (mod && mod.__esModule) ? mod : { "default": mod };
     let buf = Buffer.alloc(Math.ceil(opts.bitWidth / 8));
+    let remainingValue = value;
+    // This is encoded LSB to MSB, so we pick off the least
+    // significant byte and shift to get the next one.
     for (let i = 0; i < buf.length; ++i) {
-        buf.writeUInt8(value & 0xff, i);
-        value >> 8;
+        buf.writeUInt8(remainingValue & 0xff, i);
+        remainingValue = remainingValue >> 8;
     }
@@ -111,6 +117,9 @@ return Buffer.concat([
 function decodeRunRepeated(cursor, count, opts) {
+    var bytesNeededForFixedBitWidth = Math.ceil(opts.bitWidth / 8);
     let value = 0;
-    for (let i = 0; i < Math.ceil(opts.bitWidth / 8); ++i) {
-        value << 8;
-        value += cursor.buffer[cursor.offset];
+    for (let i = 0; i < bytesNeededForFixedBitWidth; ++i) {
+        const byte = cursor.buffer[cursor.offset];
+        // Bytes are stored LSB to MSB, so we need to shift
+        // each new byte appropriately.
+        value += byte << (i * 8);
         cursor.offset += 1;
@@ -149,2 +158,1 @@ }
 exports.decodeValues = decodeValues;
-//# sourceMappingURL=rle.js.map

dist/lib/codec/types.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-//# sourceMappingURL=types.js.map

dist/lib/compression.js

@@ -40,3 +40,3 @@ "use strict";
 function deflate_identity(value) {
-    return value;
+    return buffer_from_result(value);
 }
@@ -47,3 +47,4 @@ function deflate_gzip(value) {
 function deflate_snappy(value) {
-    return snappyjs_1.default.compress(value);
+    const compressedValue = snappyjs_1.default.compress(value);
+    return buffer_from_result(compressedValue);
 }
@@ -69,10 +70,11 @@ async function deflate_brotli(value) {
 exports.inflate = inflate;
-function inflate_identity(value) {
-    return value;
+async function inflate_identity(value) {
+    return buffer_from_result(value);
 }
-function inflate_gzip(value) {
+async function inflate_gzip(value) {
     return zlib_1.default.gunzipSync(value);
 }
 function inflate_snappy(value) {
-    return snappyjs_1.default.uncompress(value);
+    const uncompressedValue = snappyjs_1.default.uncompress(value);
+    return buffer_from_result(uncompressedValue);
 }
@@ -83,2 +85,9 @@ async function inflate_brotli(value) {
 }
-//# sourceMappingURL=compression.js.map
+function buffer_from_result(result) {
+    if (Buffer.isBuffer(result)) {
+        return result;
+    }
+    else {
+        return Buffer.from(result);
+    }
+}

dist/lib/reader.js

 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
@@ -25,3 +29,3 @@ if (k2 === undefined) k2 = k;
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParquetEnvelopeReader = void 0;
+exports.ParquetEnvelopeReader = exports.ParquetReader = void 0;
 const node_int64_1 = __importDefault(require("node-int64"));
@@ -38,3 +42,4 @@ const parquet_types_1 = __importDefault(require("../gen-nodejs/parquet_types"));
 const cross_fetch_1 = __importDefault(require("cross-fetch"));
-const types_1 = require("./types/types");
+const declare_1 = require("./declare");
+const client_s3_1 = require("@aws-sdk/client-s3");
 const { getBloomFiltersFor, } = bloomFilterReader;
@@ -46,5 +51,5 @@ /**
 /**
- * Parquet File Format Version
+ * Supported Parquet File Format Version for reading
  */
-const PARQUET_VERSION = 1;
+const PARQUET_VERSIONS = [1, 2];
 /**
@@ -65,2 +70,3 @@ * Internal type used for repetition/definition levels
     rowGroupIndex;
+    cursorIndex;
     /**
@@ -79,2 +85,3 @@ * Create a new parquet reader from the file metadata and an envelope reader.
         this.rowGroupIndex = 0;
+        this.cursorIndex = 0;
     }
@@ -86,3 +93,3 @@ /**
     async next() {
-        if (this.rowGroup.length === 0) {
+        if (this.cursorIndex >= this.rowGroup.length) {
             if (this.rowGroupIndex >= this.metadata.row_groups.length) {
@@ -94,7 +101,8 @@ return null;
             this.rowGroupIndex++;
+            this.cursorIndex = 0;
         }
-        return this.rowGroup.shift();
+        return this.rowGroup[this.cursorIndex++];
     }
     /**
-     * Rewind the cursor the the beginning of the file
+     * Rewind the cursor to the beginning of the file
      */
@@ -104,2 +112,3 @@ rewind() {
         this.rowGroupIndex = 0;
+        this.cursorIndex = 0;
     }
@@ -132,9 +141,19 @@ }
     /**
-     * Open the parquet file from S3 using the supplied aws client and params
-     * The params have to include `Bucket` and `Key` to the file requested
-     * This function returns a new parquet reader
+     * Open the parquet file from S3 using the supplied aws client [, commands] and params
+     * The params have to include `Bucket` and `Key` to the file requested,
+     * If using v3 of the AWS SDK, combine the client and commands into an object wiht keys matching
+     * the original module names, and do not instantiate the commands; pass them as classes/modules.
+     *
+     * This function returns a new parquet reader [ or throws an Error.]
      */
     static async openS3(client, params, options) {
-        let envelopeReader = await ParquetEnvelopeReader.openS3(client, params, options);
-        return this.openEnvelopeReader(envelopeReader, options);
+        try {
+            let envelopeReader = 'function' === typeof client['headObject'] ?
+                await ParquetEnvelopeReader.openS3(client, params, options) : // S3 client v2
+                await ParquetEnvelopeReader.openS3v3(client, params, options); // S3 client v3
+            return this.openEnvelopeReader(envelopeReader, options);
+        }
+        catch (e) {
+            throw new Error(`Error accessing S3 Bucket ${params.Bucket}. Message: ${e.message}`);
+        }
     }
@@ -152,3 +171,3 @@ /**
     static async openEnvelopeReader(envelopeReader, opts) {
-        if (opts && opts.metadata) {
+        if (opts?.metadata) {
             return new ParquetReader(opts.metadata, envelopeReader, opts);
@@ -174,3 +193,3 @@ }
         opts = opts || {};
-        if (metadata.version != PARQUET_VERSION) {
+        if (!PARQUET_VERSIONS.includes(metadata.version)) {
             throw 'invalid parquet version';
@@ -196,7 +215,7 @@ }
                     if (column.offsetIndex) {
-                        column.offsetIndex.page_locations.forEach(d => {
+                        Promise.resolve(column.offsetIndex).then(offset => (offset.page_locations.forEach(d => {
                             if (Array.isArray(d)) {
                                 Object.setPrototypeOf(d, parquet_types_1.default.PageLocation.prototype);
                             }
-                        });
+                        })));
                     }
@@ -225,2 +244,19 @@ });
     /**
+     * Support `for await` iterators on the reader object
+     * Uses `ParquetCursor` still under the hood.
+     *
+     * ```js
+     *  for await (const record of reader) {
+     *    console.log(record);
+     *  }
+     * ```
+     */
+    async *[Symbol.asyncIterator]() {
+        const cursor = this.getCursor();
+        let record = null;
+        while (record = await cursor.next()) {
+            yield record;
+        }
+    }
+    /**
      * Return a cursor to the file. You may open more than one cursor and use
@@ -253,3 +289,3 @@ * them concurrently. All cursors become invalid once close() is called on
      * Return the number of rows in this file. Note that the number of rows is
-     * not neccessarily equal to the number of rows in each column.
+     * not necessarily equal to the number of rows in each column.
      */
@@ -275,3 +311,3 @@ getRowCount() {
     }
-    exportMetadata(indent) {
+    async exportMetadata(indent) {
         function replacer(_key, value) {
@@ -310,2 +346,14 @@ if (value instanceof parquet_types_1.default.PageLocation) {
         const metadata = Object.assign({}, this.metadata, { json: true });
+        for (let i = 0; i < metadata.row_groups.length; i++) {
+            const rowGroup = metadata.row_groups[i];
+            for (let j = 0; j < rowGroup.columns.length; j++) {
+                const column = rowGroup.columns[j];
+                if (column.offsetIndex instanceof Promise) {
+                    column.offsetIndex = await column.offsetIndex;
+                }
+                if (column.columnIndex instanceof Promise) {
+                    column.columnIndex = await column.columnIndex;
+                }
+            }
+        }
         return JSON.stringify(metadata, replacer, indent);
@@ -326,2 +374,3 @@ }
 }
+exports.ParquetReader = ParquetReader;
 /**
@@ -359,3 +408,3 @@ * The parquet envelope reader allows direct, unbuffered access to the individual
             }
-            return Promise.resolve(buffer.slice(offset, offset + length));
+            return Promise.resolve(buffer.subarray(offset, offset + length));
         };
@@ -378,5 +427,53 @@ let closeFn = () => ({});
     }
-    static async openUrl(params, options) {
-        if (typeof params === 'string')
-            params = { url: params };
+    static async openS3v3(client, params, options) {
+        const fileStat = async () => {
+            try {
+                let headObjectCommand = await client.send(new client_s3_1.HeadObjectCommand(params));
+                return Promise.resolve(headObjectCommand.ContentLength);
+            }
+            catch (e) {
+                // having params match command names makes e.message clear to user
+                return Promise.reject("rejected headObjectCommand: " + e.message);
+            }
+        };
+        const readFn = async (offset, length, file) => {
+            if (file) {
+                return Promise.reject("external references are not supported");
+            }
+            const Range = `bytes=${offset}-${offset + length - 1}`;
+            const input = { ...{ Range }, ...params };
+            const response = await client.send(new client_s3_1.GetObjectCommand(input));
+            const body = response.Body;
+            if (body) {
+                return ParquetEnvelopeReader.streamToBuffer(body);
+            }
+            return Buffer.of();
+        };
+        let closeFn = () => ({});
+        return new ParquetEnvelopeReader(readFn, closeFn, fileStat, options);
+    }
+    static async streamToBuffer(body) {
+        const blob = body;
+        if (blob.arrayBuffer !== undefined) {
+            const arrayBuffer = await blob.arrayBuffer();
+            const uint8Array = new Uint8Array(arrayBuffer);
+            return Buffer.from(uint8Array);
+        }
+        //Assumed to be a Readable like object
+        const readable = body;
+        return await new Promise((resolve, reject) => {
+            const chunks = [];
+            readable.on("data", (chunk) => chunks.push(chunk));
+            readable.on("error", reject);
+            readable.on("end", () => resolve(Buffer.concat(chunks)));
+        });
+    }
+    static async openUrl(url, options) {
+        let params;
+        if (typeof url === 'string')
+            params = { url };
+        else if (url instanceof URL)
+            params = { url: url.toString() };
+        else
+            params = url;
         if (!params.url)
@@ -429,10 +526,14 @@ throw new Error('URL missing');
         let column;
-        if (!isNaN(row_group)) {
-            row_group = this.metadata.row_groups[row_group];
+        let parsedRowGroup;
+        if (!isNaN(Number(row_group))) {
+            parsedRowGroup = this.metadata?.row_groups[Number(row_group)];
         }
+        else if (row_group instanceof parquet_types_1.default.RowGroup) {
+            parsedRowGroup = row_group;
+        }
         if (typeof path === 'string') {
-            if (!row_group) {
+            if (!parsedRowGroup) {
                 throw `Missing RowGroup ${row_group}`;
             }
-            column = row_group.columns.find(d => d.meta_data.path_in_schema.join(',') === path);
+            column = parsedRowGroup.columns.find(d => d.meta_data.path_in_schema.join(',') === path);
             if (!column) {
@@ -468,9 +569,5 @@ throw `Column ${path} Not Found`;
             Object.defineProperty(offset_index, 'column', { value: column, enumerable: false });
-            if (opts && opts.cache) {
-                column.offsetIndex = offset_index;
-            }
             return offset_index;
         });
-        if (opts && opts.cache) {
-            //@ts-ignore
+        if (opts?.cache) {
             column.offsetIndex = data;
@@ -488,5 +585,5 @@ }
         }
-        const data = this.read(+column.column_index_offset, column.column_index_length).then((data) => {
+        const data = this.read(+column.column_index_offset, column.column_index_length).then((buf) => {
             let column_index = new parquet_types_1.default.ColumnIndex();
-            parquet_util.decodeThrift(column_index, data);
+            parquet_util.decodeThrift(column_index, buf);
             Object.defineProperty(column_index, 'column', { value: column });
@@ -501,9 +598,5 @@ // decode the statistics values
             }
-            if (opts && opts.cache) {
-                column.columnIndex = column_index;
-            }
             return column_index;
         });
-        if (opts && opts.cache) {
-            //@ts-ignore
+        if (opts?.cache) {
             column.columnIndex = data;
@@ -520,9 +613,9 @@ }
             }
-            column.meta_data.data_page_offset = page.offset;
-            column.meta_data.total_compressed_size = page.compressed_page_size;
+            column.meta_data.data_page_offset = parquet_util.cloneInteger(page.offset);
+            column.meta_data.total_compressed_size = new node_int64_1.default(page.compressed_page_size);
         }
         else {
-            const offsetIndex = column.offsetIndex || await this.readOffsetIndex(column, null, opts);
-            column.meta_data.data_page_offset = offsetIndex.page_locations[page].offset;
-            column.meta_data.total_compressed_size = offsetIndex.page_locations[page].compressed_page_size;
+            const offsetIndex = await this.readOffsetIndex(column, null, opts);
+            column.meta_data.data_page_offset = parquet_util.cloneInteger(offsetIndex.page_locations[page].offset);
+            column.meta_data.total_compressed_size = new node_int64_1.default(offsetIndex.page_locations[page].compressed_page_size);
         }
@@ -571,3 +664,4 @@ const chunk = await this.readColumnChunk(this.schema, column);
         });
-        if (metadata.dictionary_page_offset) {
+        // If this exists and is greater than zero then we need to have an offset
+        if (metadata.dictionary_page_offset && +metadata.dictionary_page_offset > 0) {
             const offset = +metadata.dictionary_page_offset;
@@ -588,4 +682,5 @@ const size = Math.min(+this.fileSize - offset, this.default_dictionary_size);
         let trailerLen = PARQUET_MAGIC.length + 4;
-        let trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);
-        if (trailerBuf.slice(4).toString() != PARQUET_MAGIC) {
+        let offset = this.fileSize - trailerLen;
+        let trailerBuf = await this.read(offset, trailerLen);
+        if (trailerBuf.subarray(4).toString() != PARQUET_MAGIC) {
             throw 'not a valid parquet file';
@@ -599,3 +694,3 @@ }
         let metadataBuf = await this.read(metadataOffset, metadataSize);
-        let metadata = new types_1.NewFileMetaData();
+        let metadata = new parquet_types_1.default.FileMetaData();
         parquet_util.decodeThrift(metadata, metadataBuf);
@@ -625,3 +720,3 @@ return metadata;
     if (column.originalType) {
-        value = parquet_types.fromPrimitive(column.originalType, value);
+        value = parquet_types.fromPrimitive(column.originalType, value, column);
     }
@@ -647,5 +742,5 @@ return value;
     let page;
-    const pageHeader = new types_1.NewPageHeader();
+    const pageHeader = new declare_1.NewPageHeader();
     const headerOffset = cursor.offset;
-    const headerSize = parquet_util.decodeThrift(pageHeader, cursor.buffer.slice(cursor.offset));
+    const headerSize = parquet_util.decodeThrift(pageHeader, cursor.buffer.subarray(cursor.offset));
     cursor.offset += headerSize;
@@ -700,3 +795,6 @@ const pageType = parquet_util.getThriftEnum(parquet_types_1.default.PageType, pageHeader.type);
         }
-        if (opts.dictionary) {
+        // It's possible to have a column chunk where some pages should use
+        // the dictionary (PLAIN_DICTIONARY for example) and others should
+        // not (PLAIN for example).
+        if (opts.dictionary && pageData.useDictionary) {
             pageData.values = pageData.values.map(d => opts.dictionary[d]);
@@ -722,3 +820,3 @@ }
         offset: 0,
-        buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
+        buffer: cursor.buffer.subarray(cursor.offset, cursorEnd),
         size: cursorEnd - cursor.offset
@@ -728,3 +826,3 @@ };
     if (opts.compression && opts.compression !== 'UNCOMPRESSED') {
-        let valuesBuf = await parquet_compression.inflate(opts.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd));
+        let valuesBuf = await parquet_compression.inflate(opts.compression, dictCursor.buffer.subarray(dictCursor.offset, cursorEnd));
         dictCursor = {
@@ -746,3 +844,3 @@ buffer: valuesBuf,
     if (opts.compression && opts.compression !== 'UNCOMPRESSED') {
-        let valuesBuf = await parquet_compression.inflate(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd));
+        let valuesBuf = await parquet_compression.inflate(opts.compression, cursor.buffer.subarray(cursor.offset, cursorEnd));
         valuesBufCursor = {
@@ -782,3 +880,7 @@ buffer: valuesBuf,
         bitWidth: opts.column.typeLength,
-        disableEnvelope: opts.column.disableEnvelope
+        disableEnvelope: opts.column.disableEnvelope,
+        originalType: opts.column.originalType,
+        precision: opts.column.precision,
+        scale: opts.column.scale,
+        name: opts.column.name
     });
@@ -790,3 +892,4 @@ cursor.offset = cursorEnd;
         values: values,
-        count: valueCount
+        count: valueCount,
+        useDictionary: valueEncoding === 'PLAIN_DICTIONARY' || valueEncoding === 'RLE_DICTIONARY'
     };
@@ -825,3 +928,3 @@ }
     if (dataPageHeaderV2.is_compressed) {
-        let valuesBuf = await parquet_compression.inflate(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd));
+        let valuesBuf = await parquet_compression.inflate(opts.compression, cursor.buffer.subarray(cursor.offset, cursorEnd));
         valuesBufCursor = {
@@ -835,4 +938,4 @@ buffer: valuesBuf,
     let values = decodeValues(opts.type, valueEncoding, valuesBufCursor, valueCountNonNull, {
-        typeLength: opts.column.typeLength,
-        bitWidth: opts.column.typeLength
+        bitWidth: opts.column.typeLength,
+        ...opts.column
     });
@@ -843,3 +946,4 @@ return {
         values: values,
-        count: valueCount
+        count: valueCount,
+        useDictionary: valueEncoding === 'PLAIN_DICTIONARY' || valueEncoding === 'RLE_DICTIONARY'
     };
@@ -892,3 +996,5 @@ }
                 optional: optional,
-                repeated: repeated
+                repeated: repeated,
+                scale: schemaElement.scale,
+                precision: schemaElement.precision
             };
@@ -903,6 +1009,1 @@ }
 }
-module.exports = {
-    ParquetEnvelopeReader,
-    ParquetReader,
-};
-//# sourceMappingURL=reader.js.map

dist/lib/schema.js

 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
@@ -26,2 +30,3 @@ if (k2 === undefined) k2 = k;
 const parquet_types = __importStar(require("./types"));
+const jsonSchema_1 = require("./jsonSchema");
 const PARQUET_COLUMN_KEY_SEPARATOR = '.';
@@ -36,2 +41,9 @@ /**
     /**
+     * Create a new schema from JSON Schema (json-schema.org)
+     */
+    static fromJsonSchema(jsonSchema) {
+        const schema = (0, jsonSchema_1.fromJsonSchema)(jsonSchema);
+        return new ParquetSchema(schema);
+    }
+    /**
      * Create a new schema from a JSON schema definition
@@ -95,2 +107,3 @@ */
     let fieldList = {};
+    let fieldErrors = [];
     for (let name in schema) {
@@ -132,5 +145,10 @@ const opts = schema[name];
         }
-        const typeDef = opts.type ? parquet_types.PARQUET_LOGICAL_TYPES[opts.type] : undefined;
+        let nameWithPath = (`${name}` || 'missing name');
+        if (path && path.length > 0) {
+            nameWithPath = `${path}.${nameWithPath}`;
+        }
+        const typeDef = opts.type ? parquet_types.getParquetTypeDataObject(opts.type, opts) : undefined;
         if (!typeDef) {
-            throw 'invalid parquet type: ' + (opts.type || "missing type");
+            fieldErrors.push(`Invalid parquet type: ${(opts.type || "missing type")}, for Column: ${nameWithPath}`);
+            continue;
         }
@@ -142,3 +160,3 @@ /* field encoding */
         if (!(opts.encoding in parquet_codec)) {
-            throw 'unsupported parquet encoding: ' + opts.encoding;
+            fieldErrors.push(`Unsupported parquet encoding: ${opts.encoding}, for Column: ${nameWithPath}`);
         }
@@ -149,4 +167,10 @@ if (!opts.compression) {
         if (!(opts.compression in parquet_compression.PARQUET_COMPRESSION_METHODS)) {
-            throw 'unsupported compression method: ' + opts.compression;
+            fieldErrors.push(`Unsupported compression method: ${opts.compression}, for Column: ${nameWithPath}`);
         }
+        if (typeDef.originalType === 'DECIMAL') {
+            // Default scale to 0 per https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal
+            if (typeof opts.scale === "undefined")
+                opts.scale = 0;
+            fieldErrors = fieldErrors.concat(errorsForDecimalOpts(typeDef.originalType, typeDef.primitiveType, opts, nameWithPath));
+        }
         /* add to schema */
@@ -162,2 +186,4 @@ fieldList[name] = {
             compression: opts.compression,
+            precision: opts.precision,
+            scale: opts.scale,
             typeLength: opts.typeLength || typeDef.typeLength,
@@ -168,2 +194,5 @@ rLevelMax: rLevelMax,
     }
+    if (fieldErrors.length > 0) {
+        throw fieldErrors.reduce((accumulator, currentVal) => accumulator + '\n' + currentVal);
+    }
     return fieldList;
@@ -185,2 +214,23 @@ }
 }
-//# sourceMappingURL=schema.js.map
+function errorsForDecimalOpts(type, primitiveType, opts, columnName) {
+    const fieldErrors = [];
+    if (opts.precision === undefined || opts.precision < 1) {
+        fieldErrors.push(`invalid schema for type: ${type}, for Column: ${columnName}, precision is required and must be be greater than 0`);
+    }
+    else if (!Number.isInteger(opts.precision)) {
+        fieldErrors.push(`invalid schema for type: ${type}, for Column: ${columnName}, precision must be an integer`);
+    }
+    else if (primitiveType === "INT64" && opts.precision > 18) {
+        fieldErrors.push(`invalid schema for type: ${type} and primitive type: ${primitiveType} for Column: ${columnName}, can not handle precision over 18`);
+    }
+    if (typeof opts.scale === "undefined" || opts.scale < 0) {
+        fieldErrors.push(`invalid schema for type: ${type}, for Column: ${columnName}, scale is required to be 0 or greater`);
+    }
+    else if (!Number.isInteger(opts.scale)) {
+        fieldErrors.push(`invalid schema for type: ${type}, for Column: ${columnName}, scale must be an integer`);
+    }
+    else if (opts.precision !== undefined && opts.scale > opts.precision) {
+        fieldErrors.push(`invalid schema or precision for type: ${type}, for Column: ${columnName}, precision must be greater than or equal to scale`);
+    }
+    return fieldErrors;
+}

dist/lib/shred.js

 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
@@ -124,3 +128,3 @@ if (k2 === undefined) k2 = k;
                 data[path].distinct_values.add(values[i]);
-                data[path].values.push(parquet_types.toPrimitive(fieldType, values[i]));
+                data[path].values.push(parquet_types.toPrimitive(fieldType, values[i], field));
                 data[path].rlevels.push(rlvl_i);
@@ -170,3 +174,3 @@ data[path].dlevels.push(field.dLevelMax);
             if (dLevel === field.dLevelMax) {
-                value = parquet_types.fromPrimitive(field.originalType || field.primitiveType, values.next().value);
+                value = parquet_types.fromPrimitive(field.originalType || field.primitiveType, values.next().value, field);
             }
@@ -183,2 +187,4 @@ records[rLevels[0] - 1] = records[rLevels[0] - 1] || {};
     if (dLevel < node.dLevelMax) {
+        // This ensures that nulls are correctly processed
+        record[node.name] = value;
         return;
@@ -222,2 +228,1 @@ }
 }
-//# sourceMappingURL=shred.js.map

dist/lib/types.js

 'use strict';
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
@@ -22,5 +26,68 @@ if (k2 === undefined) k2 = k;
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.fromPrimitive = exports.toPrimitive = exports.PARQUET_LOGICAL_TYPES = void 0;
+exports.fromPrimitive = exports.toPrimitive = exports.getParquetTypeDataObject = void 0;
+// Thanks to https://github.com/kbajalc/parquets for some of the code.
 const BSON = __importStar(require("bson"));
-exports.PARQUET_LOGICAL_TYPES = {
+function getParquetTypeDataObject(type, field) {
+    if (type === 'DECIMAL') {
+        if (field?.typeLength !== undefined) {
+            return {
+                primitiveType: 'FIXED_LEN_BYTE_ARRAY',
+                originalType: 'DECIMAL',
+                typeLength: field.typeLength,
+                toPrimitive: toPrimitive_FIXED_LEN_BYTE_ARRAY_DECIMAL
+            };
+        }
+        else if (field?.precision !== undefined && field.precision > 18) {
+            return {
+                primitiveType: 'BYTE_ARRAY',
+                originalType: 'DECIMAL',
+                typeLength: field.typeLength,
+                toPrimitive: toPrimitive_BYTE_ARRAY_DECIMAL
+            };
+        }
+        else {
+            return {
+                primitiveType: 'INT64',
+                originalType: 'DECIMAL',
+                toPrimitive: toPrimitive_INT64
+            };
+        }
+    }
+    else {
+        return PARQUET_LOGICAL_TYPE_DATA[type];
+    }
+}
+exports.getParquetTypeDataObject = getParquetTypeDataObject;
+const PARQUET_LOGICAL_TYPES = new Set([
+    'BOOLEAN',
+    'INT32',
+    'INT64',
+    'INT96',
+    'FLOAT',
+    'DOUBLE',
+    'BYTE_ARRAY',
+    'FIXED_LEN_BYTE_ARRAY',
+    'UTF8',
+    'ENUM',
+    'TIME_MILLIS',
+    'TIME_MICROS',
+    'DATE',
+    'TIMESTAMP_MILLIS',
+    'TIMESTAMP_MICROS',
+    'UINT_8',
+    'UINT_16',
+    'UINT_32',
+    'UINT_64',
+    'INT_8',
+    'INT_16',
+    'INT_32',
+    'INT_64',
+    'DECIMAL',
+    'JSON',
+    'BSON',
+    'INTERVAL',
+    'MAP',
+    'LIST'
+]);
+const PARQUET_LOGICAL_TYPE_DATA = {
     'BOOLEAN': {
@@ -157,13 +224,29 @@ primitiveType: 'BOOLEAN',
         fromPrimitive: fromPrimitive_INTERVAL
+    },
+    MAP: {
+        originalType: 'MAP',
+        toPrimitive: toPrimitive_MAP,
+    },
+    LIST: {
+        originalType: 'LIST',
+        toPrimitive: toPrimitive_LIST,
     }
 };
 /**
+ * Test if something is a valid Parquet Type
+ * @param type the string of the type
+ * @returns if type is a valid Parquet Type
+ */
+function isParquetType(type) {
+    return type !== undefined && PARQUET_LOGICAL_TYPES.has(type);
+}
+/**
  * Convert a value from it's native representation to the internal/underlying
  * primitive type
  */
-function toPrimitive(type, value) {
-    if (type === undefined || !(type in exports.PARQUET_LOGICAL_TYPES)) {
+function toPrimitive(type, value, field) {
+    if (!isParquetType(type)) {
         throw 'invalid type: ' + type || "undefined";
     }
-    return exports.PARQUET_LOGICAL_TYPES[type].toPrimitive(value);
+    return getParquetTypeDataObject(type, field).toPrimitive(value);
 }
@@ -175,7 +258,7 @@ exports.toPrimitive = toPrimitive;
  */
-function fromPrimitive(type, value) {
-    if (type === undefined || !(type in exports.PARQUET_LOGICAL_TYPES)) {
+function fromPrimitive(type, value, field) {
+    if (!isParquetType(type)) {
         throw 'invalid type: ' + type || "undefined";
     }
-    const typeFromPrimitive = exports.PARQUET_LOGICAL_TYPES[type].fromPrimitive;
+    const typeFromPrimitive = getParquetTypeDataObject(type, field).fromPrimitive;
     if (typeFromPrimitive !== undefined) {
@@ -287,2 +370,4 @@ return typeFromPrimitive(value);
 }
+const MIN_64 = BigInt('0x8000000000000000') * -1n;
+const MAX_64 = BigInt('0x7fffffffffffffff');
 function toPrimitive_INT64(value) {
@@ -293,3 +378,3 @@ try {
             v = BigInt(value);
-        checkValidValue(-0x8000000000000000, 0x7fffffffffffffff, v);
+        checkValidValue(MIN_64, MAX_64, v);
         return v;
@@ -301,2 +386,3 @@ }
 }
+const MAX_U64 = BigInt('0xffffffffffffffff');
 function toPrimitive_UINT64(value) {
@@ -307,3 +393,3 @@ try {
             v = BigInt(value);
-        checkValidValue(0, 0xffffffffffffffff, v);
+        checkValidValue(0, MAX_U64, v);
         return v;
@@ -315,2 +401,4 @@ }
 }
+const MIN_96 = BigInt('0x800000000000000000000000') * -1n;
+const MAX_96 = BigInt('0x7fffffffffffffffffffffff');
 function toPrimitive_INT96(value) {
@@ -321,3 +409,3 @@ try {
             v = BigInt(value);
-        checkValidValue(-0x800000000000000000000000, 0x7fffffffffffffffffffffff, v);
+        checkValidValue(MIN_96, MAX_96, v);
         return v;
@@ -329,2 +417,14 @@ }
 }
+function toPrimitive_FIXED_LEN_BYTE_ARRAY_DECIMAL(value) {
+    return Buffer.from(value);
+}
+function toPrimitive_BYTE_ARRAY_DECIMAL(value) {
+    return Buffer.from(value);
+}
+function toPrimitive_MAP(value) {
+    return value;
+}
+function toPrimitive_LIST(value) {
+    return value;
+}
 function toPrimitive_BYTE_ARRAY(value) {
@@ -351,16 +451,27 @@ return Buffer.from(value);
 }
-function toPrimitive_TIME_MILLIS(value) {
-    let v = value;
-    if (typeof value === `string`) {
-        v = parseInt(value, 10);
+function toNumberInternal(typeName, value) {
+    let numberValue = 0;
+    switch (typeof value) {
+        case "string":
+            numberValue = parseInt(value, 10);
+            break;
+        case "number":
+            numberValue = value;
+            break;
+        default:
+            throw `${typeName} has an invalid type: ${typeof value}`;
     }
-    if (v < 0 || v > 0xffffffffffffffff || typeof v !== 'number') {
-        throw 'invalid value for TIME_MILLIS: ' + value;
+    // Year 2255 bug. Should eventually switch to bigint
+    if (numberValue < 0 || numberValue >= Number.MAX_SAFE_INTEGER) {
+        throw `${typeName} value is out of bounds: ${numberValue}`;
     }
-    return v;
+    return numberValue;
 }
+function toPrimitive_TIME_MILLIS(value) {
+    return toNumberInternal("TIME_MILLIS", value);
+}
 function toPrimitive_TIME_MICROS(value) {
     const v = BigInt(value);
     if (v < 0n) {
-        throw 'invalid value for TIME_MICROS: ' + value;
+        throw 'TIME_MICROS value is out of bounds: ' + value;
     }
@@ -375,11 +486,3 @@ return v;
     }
-    /* convert from integer */
-    let v = value;
-    if (typeof value === 'string') {
-        v = parseInt(value, 10);
-    }
-    if (v < 0 || typeof v !== 'number') {
-        throw 'invalid value for DATE: ' + value;
-    }
-    return v;
+    return toNumberInternal("DATE", value);
 }
@@ -394,11 +497,3 @@ function fromPrimitive_DATE(value) {
     }
-    /* convert from integer */
-    let v = value;
-    if (typeof value === 'string') {
-        v = parseInt(value, 10);
-    }
-    if (v < 0 || typeof v !== 'number') {
-        throw 'invalid value for TIMESTAMP_MILLIS: ' + value;
-    }
-    return v;
+    return toNumberInternal("TIMESTAMP_MILLIS", value);
 }
@@ -414,12 +509,18 @@ function fromPrimitive_TIMESTAMP_MILLIS(value) {
     /* convert from integer */
-    {
+    try {
+        // Will throw if NaN
         const v = BigInt(value);
-        if (v < 0n /*|| isNaN(v)*/) {
-            throw 'invalid value for TIMESTAMP_MICROS: ' + value;
+        if (v < 0n) {
+            throw 'out of bounds';
         }
         return v;
     }
+    catch (e) {
+        throw 'TIMESTAMP_MICROS value is out of bounds: ' + value;
+    }
 }
 function fromPrimitive_TIMESTAMP_MICROS(value) {
-    return typeof value === 'bigint' ? new Date(Number(value / 1000n)) : new Date(value / 1000);
+    if (typeof value === 'bigint')
+        return new Date(Number(value / 1000n));
+    return new Date(value / 1000);
 }
@@ -448,2 +549,1 @@ function toPrimitive_INTERVAL(value) {
 }
-//# sourceMappingURL=types.js.map

dist/lib/util.js

 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
@@ -25,6 +29,7 @@ if (k2 === undefined) k2 = k;
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.fieldIndexOf = exports.osopen = exports.osend = exports.oswrite = exports.fclose = exports.fread = exports.fstat = exports.fopen = exports.getThriftEnum = exports.getBitWidth = exports.decodeThrift = exports.serializeThrift = void 0;
+exports.cloneInteger = exports.fieldIndexOf = exports.osopen = exports.osend = exports.oswrite = exports.fclose = exports.fread = exports.fstat = exports.fopen = exports.getThriftEnum = exports.getBitWidth = exports.decodeThrift = exports.serializeThrift = void 0;
 const thrift_1 = __importDefault(require("thrift"));
 const fs_1 = __importDefault(require("fs"));
 const parquet_thrift = __importStar(require("../gen-nodejs/parquet_types"));
+const thrift_2 = require("thrift");
 /**
@@ -44,2 +49,7 @@ * We need to patch Thrift's TFramedTransport class bc the TS type definitions
 }
+/** Patch PageLocation to be three element array that has getters/setters
+  * for each of the properties (offset, compressed_page_size, first_row_index)
+  * This saves space considerably as we do not need to store the full variable
+  * names for every PageLocation
+  */
 const getterSetter = (index) => ({
@@ -214,2 +224,5 @@ get: function () { return this[index]; },
 exports.fieldIndexOf = fieldIndexOf;
-//# sourceMappingURL=util.js.map
+const cloneInteger = (int) => {
+    return new thrift_2.Int64(int.valueOf());
+};
+exports.cloneInteger = cloneInteger;

dist/lib/writer.js

@@ -1,11 +0,39 @@
-'use strict';
-const stream = require('stream');
-const parquet_thrift = require('../gen-nodejs/parquet_types');
-const parquet_shredder = require('./shred');
-const parquet_util = require('./util');
-const parquet_codec = require('./codec');
-const parquet_compression = require('./compression');
-const parquet_types = require('./types');
-const bloomFilterWriter = require("./bloomFilterIO/bloomFilterWriter");
-const Long = require('long');
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetTransformer = exports.ParquetEnvelopeWriter = exports.ParquetWriter = void 0;
+const stream_1 = __importDefault(require("stream"));
+const parquet_types_1 = __importStar(require("../gen-nodejs/parquet_types"));
+const parquet_shredder = __importStar(require("./shred"));
+const parquet_util = __importStar(require("./util"));
+const parquet_codec = __importStar(require("./codec"));
+const parquet_compression = __importStar(require("./compression"));
+const parquet_types = __importStar(require("./types"));
+const bloomFilterWriter = __importStar(require("./bloomFilterIO/bloomFilterWriter"));
+const node_int64_1 = __importDefault(require("node-int64"));
 /**
@@ -35,2 +63,8 @@ * Parquet File Magic String
 class ParquetWriter {
+    schema;
+    envelopeWriter;
+    rowBuffer;
+    rowGroupSize;
+    closed;
+    userMetadata;
     /**
@@ -78,3 +112,3 @@ * Convenience method to create a new buffered parquet writer that writes to
     async appendRow(row) {
-        if (this.closed) {
+        if (this.closed || this.envelopeWriter === null) {
             throw 'writer was closed';
@@ -107,12 +141,14 @@ }
         this.closed = true;
-        if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {
-            await encodePages(this.schema, this.rowBuffer, { useDataPageV2: this.envelopeWriter.useDataPageV2, bloomFilters: this.envelopeWriter.bloomFilters });
-            await this.envelopeWriter.writeRowGroup(this.rowBuffer);
-            this.rowBuffer = {};
+        if (this.envelopeWriter) {
+            if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {
+                await encodePages(this.schema, this.rowBuffer, { useDataPageV2: this.envelopeWriter.useDataPageV2, bloomFilters: this.envelopeWriter.bloomFilters });
+                await this.envelopeWriter.writeRowGroup(this.rowBuffer);
+                this.rowBuffer = {};
+            }
+            await this.envelopeWriter.writeBloomFilters();
+            await this.envelopeWriter.writeIndex();
+            await this.envelopeWriter.writeFooter(this.userMetadata);
+            await this.envelopeWriter.close();
+            this.envelopeWriter = null;
         }
-        await this.envelopeWriter.writeBloomFilters();
-        await this.envelopeWriter.writeIndex();
-        await this.envelopeWriter.writeFooter(this.userMetadata);
-        await this.envelopeWriter.close();
-        this.envelopeWriter = null;
         if (callback) {
@@ -142,12 +178,23 @@ callback();
     setPageSize(cnt) {
-        this.writer.setPageSize(cnt);
+        this.envelopeWriter.setPageSize(cnt);
     }
 }
+exports.ParquetWriter = ParquetWriter;
 /**
  * Create a parquet file from a schema and a number of row groups. This class
  * performs direct, unbuffered writes to the underlying output stream and is
- * intendend for advanced and internal users; the writeXXX methods must be
+ * intended for advanced and internal users; the writeXXX methods must be
  * called in the correct order to produce a valid file.
  */
 class ParquetEnvelopeWriter {
+    schema;
+    write;
+    close;
+    offset;
+    rowCount;
+    rowGroups;
+    pageSize;
+    useDataPageV2;
+    pageIndex;
+    bloomFilters; // TODO: OR filterCollection
     /**
@@ -159,3 +206,3 @@ * Create a new parquet envelope writer that writes to the specified stream
         let closeFn = parquet_util.osend.bind(undefined, outputStream);
-        return new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts);
+        return new ParquetEnvelopeWriter(schema, writeFn, closeFn, new node_int64_1.default(0), opts);
     }
@@ -167,3 +214,3 @@ constructor(schema, writeFn, closeFn, fileOffset, opts) {
         this.offset = fileOffset;
-        this.rowCount = 0;
+        this.rowCount = new node_int64_1.default(0);
         this.rowGroups = [];
@@ -179,3 +226,3 @@ this.pageSize = opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE;
     writeSection(buf) {
-        this.offset += buf.length;
+        this.offset.setValue(this.offset.valueOf() + buf.length);
         return this.write(buf);
@@ -200,14 +247,17 @@ }
         });
-        this.rowCount += records.rowCount;
+        this.rowCount.setValue(this.rowCount.valueOf() + records.rowCount);
         this.rowGroups.push(rgroup.metadata);
         return this.writeSection(rgroup.body);
     }
-    writeBloomFilters(_rowGroups) {
-        let rowGroups = _rowGroups || this.rowGroups;
-        rowGroups.forEach(group => {
+    writeBloomFilters() {
+        this.rowGroups.forEach(group => {
             group.columns.forEach(column => {
-                const columnName = column.meta_data.path_in_schema[0];
-                if (columnName in this.bloomFilters === false)
+                if (!column.meta_data?.path_in_schema.length) {
                     return;
-                const serializedBloomFilterData = bloomFilterWriter.getSerializedBloomFilterData(this.bloomFilters[columnName]);
+                }
+                const filterName = column.meta_data?.path_in_schema.join(',');
+                if (!(filterName in this.bloomFilters)) {
+                    return;
+                }
+                const serializedBloomFilterData = bloomFilterWriter.getSerializedBloomFilterData(this.bloomFilters[filterName]);
                 bloomFilterWriter.setFilterOffset(column, this.offset);
@@ -221,20 +271,19 @@ this.writeSection(serializedBloomFilterData);
      */
-    writeIndex(_rowGroups) {
-        let rowGroups = _rowGroups || this.rowGroups;
+    writeIndex() {
         this.schema.fieldList.forEach((c, i) => {
-            rowGroups.forEach(group => {
+            this.rowGroups.forEach(group => {
                 let column = group.columns[i];
                 if (!column)
                     return;
-                if (column.meta_data.columnIndex) {
+                if (column.meta_data?.columnIndex) {
                     let columnBody = parquet_util.serializeThrift(column.meta_data.columnIndex);
                     delete column.meta_data.columnIndex;
-                    column.column_index_offset = this.offset;
+                    column.column_index_offset = parquet_util.cloneInteger(this.offset);
                     column.column_index_length = columnBody.length;
                     this.writeSection(columnBody);
                 }
-                if (column.meta_data.offsetIndex) {
+                if (column.meta_data?.offsetIndex) {
                     let offsetBody = parquet_util.serializeThrift(column.meta_data.offsetIndex);
                     delete column.meta_data.offsetIndex;
-                    column.offset_index_offset = this.offset;
+                    column.offset_index_offset = parquet_util.cloneInteger(this.offset);
                     column.offset_index_length = offsetBody.length;
@@ -249,3 +298,3 @@ this.writeSection(offsetBody);
      */
-    writeFooter(userMetadata, schema, rowCount, rowGroups) {
+    writeFooter(userMetadata) {
         if (!userMetadata) {
@@ -268,6 +317,8 @@ userMetadata = {};
 }
+exports.ParquetEnvelopeWriter = ParquetEnvelopeWriter;
 /**
  * Create a parquet transform stream
  */
-class ParquetTransformer extends stream.Transform {
+class ParquetTransformer extends stream_1.default.Transform {
+    writer;
     constructor(schema, opts = {}) {
@@ -280,9 +331,10 @@ super({ objectMode: true });
         })(this);
-        this.writer = new ParquetWriter(schema, new ParquetEnvelopeWriter(schema, writeProxy, function () { }, 0, opts), opts);
+        this.writer = new ParquetWriter(schema, new ParquetEnvelopeWriter(schema, writeProxy, function () {
+        }, new node_int64_1.default(0), opts), opts);
     }
-    _transform(row, encoding, callback) {
+    _transform(row, _encoding, callback) {
         if (row) {
             this.writer.appendRow(row).then(data => callback(null, data), err => {
                 const fullErr = new Error(`Error transforming to parquet: ${err.toString()} row:${row}`);
-                fullErr.origErr = err;
+                fullErr.message = err;
                 callback(fullErr);
@@ -300,2 +352,3 @@ });
 }
+exports.ParquetTransformer = ParquetTransformer;
 /**
@@ -315,3 +368,3 @@ * Encode a consecutive array of data using one of the parquet encodings
     if (column.originalType) {
-        value = parquet_types.toPrimitive(column.originalType, value);
+        value = parquet_types.toPrimitive(column.originalType, value, column);
     }
@@ -329,3 +382,3 @@ if (column.primitiveType !== 'BYTE_ARRAY') {
     statistics.min = statistics.min_value;
-    return new parquet_thrift.Statistics(statistics);
+    return new parquet_types_1.default.Statistics(statistics);
 }
@@ -341,8 +394,9 @@ async function encodePages(schema, rowBuffer, opts) {
         let page;
-        const values = rowBuffer.columnData[field.path];
-        if (opts.bloomFilters && (field.name in opts.bloomFilters)) {
-            const splitBlockBloomFilter = opts.bloomFilters[field.name];
+        const columnPath = field.path.join(',');
+        const values = rowBuffer.columnData[columnPath];
+        if (opts.bloomFilters && (columnPath in opts.bloomFilters)) {
+            const splitBlockBloomFilter = opts.bloomFilters[columnPath];
             values.values.forEach(v => splitBlockBloomFilter.insert(v));
         }
-        let statistics;
+        let statistics = {};
         if (field.statistics !== false) {
@@ -358,4 +412,4 @@ statistics = {};
             });
-            statistics.null_count = values.dlevels.length - values.values.length;
-            statistics.distinct_count = values.distinct_values.size;
+            statistics.null_count = new node_int64_1.default(values.dlevels.length - values.values.length);
+            statistics.distinct_count = new node_int64_1.default(values.distinct_values.size);
         }
@@ -366,5 +420,5 @@ if (opts.useDataPageV2) {
         else {
-            page = await encodeDataPage(field, values.values, values.rlevels, values.dlevels, statistics);
+            page = await encodeDataPage(field, values.values || [], values.rlevels || [], values.dlevels || [], statistics);
         }
-        let pages = rowBuffer.pages[field.path];
+        let pages = rowBuffer.pages[field.path.join(',')];
         let lastPage = pages[pages.length - 1];
@@ -393,4 +447,4 @@ let first_row_index = lastPage ? lastPage.first_row_index + lastPage.count : 0;
     let valuesBuf = encodeValues(column.primitiveType, column.encoding, values, {
-        typeLength: column.typeLength,
-        bitWidth: column.typeLength
+        bitWidth: column.typeLength,
+        ...column
     });
@@ -409,7 +463,7 @@ /* encode repetition and definition levels */
     pageBody = await parquet_compression.deflate(column.compression, pageBody);
-    let pageHeader = new parquet_thrift.PageHeader();
-    pageHeader.type = parquet_thrift.PageType['DATA_PAGE'];
+    let pageHeader = new parquet_types_1.default.PageHeader();
+    pageHeader.type = parquet_types_1.default.PageType['DATA_PAGE'];
     pageHeader.uncompressed_page_size = rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length;
     pageHeader.compressed_page_size = pageBody.length;
-    pageHeader.data_page_header = new parquet_thrift.DataPageHeader();
+    pageHeader.data_page_header = new parquet_types_1.default.DataPageHeader();
     pageHeader.data_page_header.num_values = dlevels.length;
@@ -419,7 +473,7 @@ if (column.statistics !== false) {
     }
-    pageHeader.data_page_header.encoding = parquet_thrift.Encoding[column.encoding];
+    pageHeader.data_page_header.encoding = parquet_types_1.default.Encoding[column.encoding];
     pageHeader.data_page_header.definition_level_encoding =
-        parquet_thrift.Encoding[PARQUET_RDLVL_ENCODING];
+        parquet_types_1.default.Encoding[PARQUET_RDLVL_ENCODING];
     pageHeader.data_page_header.repetition_level_encoding =
-        parquet_thrift.Encoding[PARQUET_RDLVL_ENCODING];
+        parquet_types_1.default.Encoding[PARQUET_RDLVL_ENCODING];
     /* concat page header, repetition and definition levels and values */
@@ -434,4 +488,4 @@ return Buffer.concat([parquet_util.serializeThrift(pageHeader), pageBody]);
     let valuesBuf = encodeValues(column.primitiveType, column.encoding, values, {
-        typeLength: column.typeLength,
-        bitWidth: column.typeLength
+        bitWidth: column.typeLength,
+        ...column,
     });
@@ -455,5 +509,5 @@ let valuesBufCompressed = await parquet_compression.deflate(column.compression, valuesBuf);
     /* build page header */
-    let pageHeader = new parquet_thrift.PageHeader();
-    pageHeader.type = parquet_thrift.PageType['DATA_PAGE_V2'];
-    pageHeader.data_page_header_v2 = new parquet_thrift.DataPageHeaderV2();
+    let pageHeader = new parquet_types_1.default.PageHeader();
+    pageHeader.type = parquet_types_1.default.PageType['DATA_PAGE_V2'];
+    pageHeader.data_page_header_v2 = new parquet_types_1.default.DataPageHeaderV2();
     pageHeader.data_page_header_v2.num_values = dlevels.length;
@@ -469,3 +523,3 @@ pageHeader.data_page_header_v2.num_nulls = dlevels.length - values.length;
         rLevelsBuf.length + dLevelsBuf.length + valuesBufCompressed.length;
-    pageHeader.data_page_header_v2.encoding = parquet_thrift.Encoding[column.encoding];
+    pageHeader.data_page_header_v2.encoding = parquet_types_1.default.Encoding[column.encoding];
     pageHeader.data_page_header_v2.definition_levels_byte_length = dLevelsBuf.length;
@@ -491,16 +545,19 @@ pageHeader.data_page_header_v2.repetition_levels_byte_length = rLevelsBuf.length;
     /* prepare metadata header */
-    let metadata = new parquet_thrift.ColumnMetaData();
+    let metadata = new parquet_types_1.default.ColumnMetaData();
     metadata.path_in_schema = opts.column.path;
-    metadata.num_values = num_values;
-    metadata.data_page_offset = opts.baseOffset;
+    metadata.num_values = new node_int64_1.default(num_values);
+    metadata.data_page_offset = new node_int64_1.default(opts.baseOffset);
     metadata.encodings = [];
-    metadata.total_uncompressed_size = pagesBuf.length;
-    metadata.total_compressed_size = pagesBuf.length;
-    metadata.type = parquet_thrift.Type[opts.column.primitiveType];
-    metadata.codec = await parquet_thrift.CompressionCodec[opts.column.compression];
+    metadata.total_uncompressed_size = new node_int64_1.default(pagesBuf.length);
+    metadata.total_compressed_size = new node_int64_1.default(pagesBuf.length);
+    metadata.type = parquet_types_1.default.Type[opts.column.primitiveType];
+    metadata.codec = await parquet_types_1.default.CompressionCodec[opts.column.compression];
     /* compile statistics ColumnIndex and OffsetIndex*/
-    let columnIndex = new parquet_thrift.ColumnIndex();
+    let columnIndex = new parquet_types_1.default.ColumnIndex();
+    columnIndex.null_pages = [];
     columnIndex.max_values = [];
     columnIndex.min_values = [];
-    let offsetIndex = new parquet_thrift.OffsetIndex();
+    // Default to unordered
+    columnIndex.boundary_order = 0;
+    let offsetIndex = new parquet_types_1.default.OffsetIndex();
     offsetIndex.page_locations = [];
@@ -510,7 +567,7 @@ /* prepare statistics */
     let distinct_values = new Set();
-    statistics.null_count = 0;
-    statistics.distinct_count = 0;
+    statistics.null_count = new node_int64_1.default(0);
+    statistics.distinct_count = new node_int64_1.default(0);
     /* loop through pages and update indices and statistics */
     for (let i = 0; i < pages.length; i++) {
-        let page = pages[i];
+        const page = pages[i];
         if (opts.column.statistics !== false) {
@@ -523,12 +580,14 @@ if (page.statistics.max_value > statistics.max_value || i == 0) {
             }
-            statistics.null_count += page.statistics.null_count;
-            page.distinct_values.forEach(value => distinct_values.add(value));
+            statistics.null_count.setValue(statistics.null_count.valueOf() + (page.statistics.null_count?.valueOf() || 0));
+            page.distinct_values.forEach((value) => distinct_values.add(value));
+            // If the number of values and the count of nulls are the same, this is a null page
+            columnIndex.null_pages.push(page.num_values === statistics.null_count.valueOf());
             columnIndex.max_values.push(encodeStatisticsValue(page.statistics.max_value, opts.column));
             columnIndex.min_values.push(encodeStatisticsValue(page.statistics.min_value, opts.column));
         }
-        let pageLocation = new parquet_thrift.PageLocation();
-        pageLocation.offset = offset;
+        let pageLocation = new parquet_types_1.default.PageLocation();
+        pageLocation.offset = new node_int64_1.default(offset);
         offset += page.page.length;
         pageLocation.compressed_page_size = page.page.length;
-        pageLocation.first_row_index = page.first_row_index;
+        pageLocation.first_row_index = new node_int64_1.default(page.first_row_index);
         offsetIndex.page_locations.push(pageLocation);
@@ -540,3 +599,3 @@ }
     if (opts.column.statistics !== false) {
-        statistics.distinct_count = distinct_values.size;
+        statistics.distinct_count = new node_int64_1.default(distinct_values.size);
         metadata.statistics = encodeStatistics(statistics, opts.column);
@@ -548,8 +607,4 @@ if (opts.pageIndex !== false) {
     /* list encodings */
-    let encodingsSet = {};
-    encodingsSet[PARQUET_RDLVL_ENCODING] = true;
-    encodingsSet[opts.column.encoding] = true;
-    for (let k in encodingsSet) {
-        metadata.encodings.push(parquet_thrift.Encoding[k]);
-    }
+    metadata.encodings.push(parquet_types_1.default.Encoding[PARQUET_RDLVL_ENCODING]);
+    metadata.encodings.push(parquet_types_1.default.Encoding[opts.column.encoding]);
     /* concat metadata header and data pages */
@@ -564,6 +619,6 @@ let metadataOffset = opts.baseOffset + pagesBuf.length;
 async function encodeRowGroup(schema, data, opts) {
-    let metadata = new parquet_thrift.RowGroup();
-    metadata.num_rows = data.rowCount;
+    let metadata = new parquet_types_1.default.RowGroup();
+    metadata.num_rows = new node_int64_1.default(data.rowCount);
     metadata.columns = [];
-    metadata.total_byte_size = 0;
+    metadata.total_byte_size = new node_int64_1.default(0);
     let body = Buffer.alloc(0);
@@ -574,16 +629,15 @@ for (let field of schema.fieldList) {
         }
-        let cchunkData = await encodeColumnChunk(data.pages[field.path], {
+        let cchunkData = await encodeColumnChunk(data.pages[field.path.join(',')], {
             column: field,
-            baseOffset: opts.baseOffset + body.length,
-            pageSize: opts.pageSize,
-            encoding: field.encoding,
-            rowCount: data.rowCount,
-            useDataPageV2: opts.useDataPageV2,
-            pageIndex: opts.pageIndex
+            baseOffset: opts.baseOffset.valueOf() + body.length,
+            pageSize: opts.pageSize || 0,
+            rowCount: data.rowCount || 0,
+            useDataPageV2: opts.useDataPageV2 ?? true,
+            pageIndex: opts.pageIndex ?? true
         });
-        let cchunk = new parquet_thrift.ColumnChunk();
-        cchunk.file_offset = cchunkData.metadataOffset;
+        let cchunk = new parquet_types_1.default.ColumnChunk();
+        cchunk.file_offset = new node_int64_1.default(cchunkData.metadataOffset);
         cchunk.meta_data = cchunkData.metadata;
         metadata.columns.push(cchunk);
-        metadata.total_byte_size += cchunkData.body.length;
+        metadata.total_byte_size = new node_int64_1.default(metadata.total_byte_size.valueOf() + (cchunkData.body.length));
         body = Buffer.concat([body, cchunkData.body]);
@@ -597,5 +651,5 @@ }
 function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
-    let metadata = new parquet_thrift.FileMetaData();
+    let metadata = new parquet_types_1.default.FileMetaData();
     metadata.version = PARQUET_VERSION;
-    metadata.created_by = 'parquet.js';
+    metadata.created_by = '@dsnp/parquetjs';
     metadata.num_rows = rowCount;
@@ -606,3 +660,3 @@ metadata.row_groups = rowGroups;
     for (let k in userMetadata) {
-        let kv = new parquet_thrift.KeyValue();
+        let kv = new parquet_types_1.default.KeyValue();
         kv.key = k;
@@ -613,3 +667,3 @@ kv.value = userMetadata[k];
     {
-        let schemaRoot = new parquet_thrift.SchemaElement();
+        let schemaRoot = new parquet_types_1.default.SchemaElement();
         schemaRoot.name = 'root';
@@ -620,5 +674,5 @@ schemaRoot.num_children = Object.keys(schema.fields).length;
     for (let field of schema.fieldList) {
-        let schemaElem = new parquet_thrift.SchemaElement();
+        let schemaElem = new parquet_types_1.default.SchemaElement();
         schemaElem.name = field.name;
-        schemaElem.repetition_type = parquet_thrift.FieldRepetitionType[field.repetitionType];
+        schemaElem.repetition_type = parquet_types_1.default.FieldRepetitionType[field.repetitionType];
         if (field.isNested) {
@@ -628,7 +682,14 @@ schemaElem.num_children = field.fieldCount;
         else {
-            schemaElem.type = parquet_thrift.Type[field.primitiveType];
+            schemaElem.type = parquet_types_1.default.Type[field.primitiveType];
         }
         if (field.originalType) {
-            schemaElem.converted_type = parquet_thrift.ConvertedType[field.originalType];
+            schemaElem.converted_type = parquet_types_1.default.ConvertedType[field.originalType];
         }
+        // Support Decimal
+        switch (schemaElem.converted_type) {
+            case (parquet_types_1.ConvertedType.DECIMAL):
+                schemaElem.precision = field.precision;
+                schemaElem.scale = field.scale || 0;
+                break;
+        }
         schemaElem.type_length = field.typeLength;
@@ -644,7 +705,1 @@ metadata.schema.push(schemaElem);
 }
-module.exports = {
-    ParquetEnvelopeWriter,
-    ParquetWriter,
-    ParquetTransformer
-};
-//# sourceMappingURL=writer.js.map

dist/parquet.js

 "use strict";
-const reader = require('./lib/reader');
-const writer = require('./lib/writer');
-const schema = require('./lib/schema');
-const shredder = require('./lib/shred');
-const util = require('./lib/util');
-module.exports = {
-    ParquetEnvelopeReader: reader.ParquetEnvelopeReader,
-    ParquetReader: reader.ParquetReader,
-    ParquetEnvelopeWriter: writer.ParquetEnvelopeWriter,
-    ParquetWriter: writer.ParquetWriter,
-    ParquetTransformer: writer.ParquetTransformer,
-    ParquetSchema: schema.ParquetSchema,
-    ParquetShredder: shredder,
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
 };
-//# sourceMappingURL=parquet.js.map
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetFieldBuilder = exports.ParquetShredder = exports.ParquetSchema = exports.ParquetTransformer = exports.ParquetWriter = exports.ParquetEnvelopeWriter = exports.ParquetReader = exports.ParquetEnvelopeReader = void 0;
+const reader = __importStar(require("./lib/reader"));
+const writer = __importStar(require("./lib/writer"));
+const schema = __importStar(require("./lib/schema"));
+const shredder = __importStar(require("./lib/shred"));
+const fields = __importStar(require("./lib/fields"));
+exports.ParquetEnvelopeReader = reader.ParquetEnvelopeReader;
+exports.ParquetReader = reader.ParquetReader;
+exports.ParquetEnvelopeWriter = writer.ParquetEnvelopeWriter;
+exports.ParquetWriter = writer.ParquetWriter;
+exports.ParquetTransformer = writer.ParquetTransformer;
+exports.ParquetSchema = schema.ParquetSchema;
+exports.ParquetShredder = shredder;
+exports.ParquetFieldBuilder = fields;
+exports.default = {
+    ParquetEnvelopeReader: exports.ParquetEnvelopeReader,
+    ParquetReader: exports.ParquetReader,
+    ParquetEnvelopeWriter: exports.ParquetEnvelopeWriter,
+    ParquetWriter: exports.ParquetWriter,
+    ParquetTransformer: exports.ParquetTransformer,
+    ParquetSchema: exports.ParquetSchema,
+    ParquetShredder: exports.ParquetShredder,
+    ParquetFieldBuilder: exports.ParquetFieldBuilder,
+};

package.json

@@ -5,3 +5,4 @@ {
   "main": "dist/parquet.js",
-  "version": "0.0.0-91ac19",
+  "types": "dist/parquet.d.ts",
+  "version": "0.0.0-91fc71",
   "homepage": "https://github.com/LibertyDSNP/parquetjs",
@@ -18,47 +19,47 @@ "license": "MIT",
   "dependencies": {
-    "@types/varint": "^6.0.0",
+    "@aws-sdk/client-s3": "^3.489.0",
+    "@types/long": "^4.0.2",
+    "@types/node-int64": "^0.4.29",
+    "@types/thrift": "^0.10.11",
     "browserify-zlib": "^0.2.0",
-    "bson": "4.4.0",
+    "bson": "4.6.3",
     "cross-fetch": "^3.1.4",
     "int53": "^0.2.4",
     "long": "^4.0.0",
-    "snappyjs": "^0.6.0",
-    "thrift": "0.14.1",
-    "varint": "^5.0.0",
+    "snappyjs": "^0.6.1",
+    "thrift": "0.16.0",
+    "varint": "^6.0.0",
     "wasm-brotli": "^2.0.2",
-    "xxhash-wasm": "^0.4.1"
+    "xxhash-wasm": "^1.0.2"
   },
   "devDependencies": {
-    "@babel/core": "^7.14.6",
-    "@babel/preset-env": "^7.14.7",
-    "@babel/preset-typescript": "^7.14.5",
-    "@types/bson": "^4.0.3",
-    "@types/chai": "^4.2.16",
-    "@types/long": "^4.0.1",
-    "@types/mocha": "^8.2.2",
-    "@types/node": "^14.14.35",
-    "@types/sinon": "^10.0.0",
-    "@types/thrift": "^0.10.10",
+    "@smithy/util-stream": "^2.0.24",
+    "@types/chai": "^4.3.5",
+    "@types/json-schema": "^7.0.11",
+    "@types/mocha": "^10.0.1",
+    "@types/node": "^18.18.2",
+    "@types/sinon": "^10.0.15",
+    "@types/varint": "^6.0.1",
     "assert": "^2.0.0",
-    "babel-loader": "^8.2.2",
-    "babel-plugin-add-module-exports": "^1.0.4",
+    "aws-sdk-client-mock": "^3.0.1",
     "browserfs": "^1.4.3",
     "buffer": "^6.0.3",
-    "chai": "4.3.4",
-    "core-js": "^3.15.1",
-    "esbuild": "^0.14.1",
-    "mocha": "8.3.2",
-    "msw": "^0.29.0",
-    "object-stream": "0.0.1",
+    "chai": "4.3.6",
+    "core-js": "^3.22.5",
+    "esbuild": "^0.19.2",
+    "mocha": "^10.2.0",
+    "msw": "^1.2.1",
+    "object-stream": "^0.0.1",
     "process": "^0.11.10",
-    "regenerator-runtime": "^0.13.7",
-    "sinon": "^10.0.0",
-    "source-map-loader": "^3.0.0",
+    "regenerator-runtime": "^0.13.11",
+    "sinon": "^15.1.0",
+    "sinon-chai": "^3.7.0",
+    "sinon-chai-in-order": "^0.1.0",
     "stream-browserify": "^3.0.0",
-    "ts-loader": "^9.2.3",
-    "ts-node": "^9.1.1",
-    "typescript": "^4.5.2"
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4"
   },
   "scripts": {
-    "build": "npm run build:node",
+    "build": "npm run build:node && npm run build:types",
+    "build:types": "tsc -p tsconfig.types.json && cp gen-nodejs/parquet_types.d.ts dist/gen-nodejs/parquet_types.d.ts",
     "build:node": "tsc -b",
@@ -68,5 +69,6 @@ "build:browser": "node esbuild.js",
     "lint": "echo 'Linting, it is on the TODO list...'",
-    "test": "mocha -r ts-node/register 'test/**/*.{js,ts}'",
+    "test": "mocha -r ts-node/register 'test/{,!(browser)/**}/*.{js,ts}'",
+    "test:only": "mocha -r ts-node/register",
     "clean": "rm -Rf ./dist",
-    "prepublishOnly": "npm run clean && npm run build:node && npm run build:browser",
+    "prepublishOnly": "npm run clean && npm run build:node && npm run build:types && npm run build:browser",
     "thrift": "thrift -out gen-nodejs --gen js:ts parquet.thrift && thrift -out gen-nodejs --gen js:node parquet.thrift",
@@ -86,3 +88,3 @@ "serve": "node esbuild-serve.js"
   "engines": {
-    "node": ">=14.16.0"
+    "node": ">=18.18.2"
   },
@@ -89,0 +91,0 @@ "files": [

README.md

@@ -24,3 +24,3 @@ # parquet.js
 ## Installation
-_parquet.js requires node.js >= 14.16.0_
+_parquet.js requires node.js >= 18.18.2_
 
@@ -31,4 +31,4 @@ ```
 
-### NodeJS 
-To use with nodejs: 
+### NodeJS
+To use with nodejs:
 ```javascript
@@ -38,13 +38,43 @@ import parquetjs from "@dsnp/parquetjs"
 
-### Browser
-To use in a browser, in your bundler, depending on your needs, write the appropriate plugin or resolver to point to:
+### Browser with Bundler
+To use in a browser with a bundler, depending on your needs, write the appropriate plugin or resolver to point to either the Common JS or ES Module version:
 ```javascript
-"node_modules/@dsnp/parquetjs/browser/parquetjs"
+// Common JS
+"node_modules/@dsnp/parquetjs/dist/browser/parquetjs.cjs"
+// ES Modules
+"node_modules/@dsnp/parquetjs/dist/browser/parquetjs.esm"
 ```
 or:
-
 ```javascript
-import parquetjs from "@dsnp/parquetjs/browser/parquetjs"
+// Common JS
+import parquetjs from "@dsnp/parquetjs/dist/browser/parquetjs.cjs"
+// ES Modules
+import parquetjs from "@dsnp/parquetjs/dist/browser/parquetjs.esm"
 ```
 
+### Browser Direct: ES Modules
+To use directly in the browser without a bundler using ES Modules:
+
+1. Build the package: `npm install && npm run build:browser`
+2. Copy to `dist/browser/parquetjs.esm.js` the server
+3. Use it in your html or other ES Modules:
+    ```html
+    <script type="module">
+      import parquetjs from '../parquet.esm.js';
+      // Use parquetjs
+    </script>
+    ```
+
+### Browser Direct: Plain Ol' JavaScript
+To use directly in the browser without a bundler or ES Modules:
+
+1. Build the package: `npm install && npm run build:browser`
+2. Copy to `dist/browser/parquetjs.js` the server
+2. Use the global `parquetjs` variable to access parquetjs functions
+   ```html
+   <script>
+    // console.log(parquetjs)
+    </script>
+    ```
+
 ## Usage: Writing files
@@ -63,2 +93,4 @@
 
+### Native Schema Definition
+
 ``` js
@@ -75,2 +107,43 @@ // declare a schema for the `fruits` table
 
+### Helper Functions
+
+```js
+var schema = new parquet.ParquetSchema({
+  name: parquet.ParquetFieldBuilder.createStringField(),
+  quantity: parquet.ParquetFieldBuilder.createIntField(64),
+  price: parquet.ParquetFieldBuilder.createDoubleField(),
+  date: parquet.ParquetFieldBuilder.createTimestampField(),
+  in_stock: parquet.ParquetFieldBuilder.createBooleanField()
+});
+```
+
+### JSON Schema
+
+``` js
+// declare a schema for the `fruits` JSON Schema
+var schema = new parquet.ParquetSchema.fromJsonSchema({
+  "type": "object",
+  "properties": {
+    "name": {
+      "type": "string"
+    },
+    "quantity": {
+      "type": "integer"
+    },
+    "price": {
+      "type": "number"
+    },
+    "date": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "in_stock": {
+      "type": "boolean"
+    }
+  },
+  "required": ["name", "quantity", "price", "date", "in_stock"]
+});
+```
+
+
 Note that the Parquet schema supports nesting, so you can store complex, arbitrarily
@@ -82,3 +155,3 @@ nested records into a single row (more on that later) while still maintaining good
 take input rows as JSON objects, convert them to the Parquet format and store
-them on disk. 
+them on disk.
 
@@ -97,3 +170,3 @@ ``` js
 
-### Adding bloom filters 
+### Adding bloom filters
 
@@ -217,3 +290,3 @@ Bloom filters can be added to multiple columns as demonstrated below:
 Parquet files can be read from an S3 object without having to download the whole file.
-You will have to supply the aws-sdk client as first argument and the bucket/key information 
+You will have to supply the aws-sdk client as first argument and the bucket/key information
 as second argument to the function `parquetReader.openS3`.
@@ -267,3 +340,3 @@
 combination with the `BOOLEAN`, `INT32` and `INT64` types. The RLE encoding
-requires an additional `bitWidth` parameter that contains the maximum number of
+requires an additional `typeLength` parameter that contains the maximum number of
 bits required to store the largest value of the field.
@@ -273,3 +346,3 @@
 var schema = new parquet.ParquetSchema({
-  age: { type: 'UINT_32', encoding: 'RLE', bitWidth: 7 },
+  age: { type: 'UINT_32', encoding: 'RLE', typeLength: 7 },
 });
@@ -304,3 +377,3 @@ ```
 Consider this example, which allows us to store a more advanced "fruits" table
-where each row contains a name, a list of colours and a list of "stock" objects. 
+where each row contains a name, a list of colours and a list of "stock" objects.
 
@@ -307,0 +380,0 @@ ``` js

New alerts

Uses eval

Supply chain risk

Package uses dynamic code execution (e.g., eval()), which is a dangerous practice. This can prevent the code from running in certain environments and increases the risk that the code may contain exploits or malicious behavior.

Found 2 instances in 1 package

Dynamic require

Supply chain risk

Dynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 2 instances in 1 package

Minified code

Quality

This package contains minified code. This may be harmless in some cases where minified code is included in packaged libraries, however packages on npm should not minify code.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

3032553

increased by1.24%

Dev dependency count

25

decreased by-13.79%

Number of package files

55

increased by7.84%

Lines of code

20674

increased by15.66%

Number of lines in readme file

493

increased by17.38%

Worsened metrics

Dependency count

14

increased by27.27%

Number of low quality alerts

3

increased by200%

Number of low supply chain risk alerts

13

increased by1200%

Number of medium supply chain risk alerts

21

increased byInfinity%

Dependency changes

• + Added@aws-sdk/client-s3@^3.489.0

• + Added@types/long@^4.0.2

• + Added@types/node-int64@^0.4.29

• + Added@types/thrift@^0.10.11

• + Added@aws-crypto/crc32@5.2.0(transitive)

• + Added@aws-crypto/crc32c@5.2.0(transitive)

• + Added@aws-crypto/sha256-browser@5.2.0(transitive)

• + Added@aws-crypto/sha256-js@5.2.0(transitive)

• + Added@aws-crypto/supports-web-crypto@5.2.0(transitive)

• + Added@aws-crypto/util@5.2.0(transitive)

• + Added@aws-sdk/client-s3@3.749.0(transitive)

• + Added@aws-sdk/client-sso@3.749.0(transitive)

• + Added@aws-sdk/core@3.749.0(transitive)

• + Added@aws-sdk/credential-provider-env@3.749.0(transitive)

• + Added@aws-sdk/credential-provider-http@3.749.0(transitive)

• + Added@aws-sdk/credential-provider-ini@3.749.0(transitive)

• + Added@aws-sdk/credential-provider-node@3.749.0(transitive)

• + Added@aws-sdk/credential-provider-process@3.749.0(transitive)

• + Added@aws-sdk/credential-provider-sso@3.749.0(transitive)

• + Added@aws-sdk/credential-provider-web-identity@3.749.0(transitive)

• + Added@aws-sdk/middleware-bucket-endpoint@3.734.0(transitive)

• + Added@aws-sdk/middleware-expect-continue@3.734.0(transitive)

• + Added@aws-sdk/middleware-flexible-checksums@3.749.0(transitive)

• + Added@aws-sdk/middleware-host-header@3.734.0(transitive)

• + Added@aws-sdk/middleware-location-constraint@3.734.0(transitive)

• + Added@aws-sdk/middleware-logger@3.734.0(transitive)

• + Added@aws-sdk/middleware-recursion-detection@3.734.0(transitive)

• + Added@aws-sdk/middleware-sdk-s3@3.749.0(transitive)

• + Added@aws-sdk/middleware-ssec@3.734.0(transitive)

• + Added@aws-sdk/middleware-user-agent@3.749.0(transitive)

• + Added@aws-sdk/nested-clients@3.749.0(transitive)

• + Added@aws-sdk/region-config-resolver@3.734.0(transitive)

• + Added@aws-sdk/signature-v4-multi-region@3.749.0(transitive)

• + Added@aws-sdk/token-providers@3.749.0(transitive)

• + Added@aws-sdk/types@3.734.0(transitive)

• + Added@aws-sdk/util-arn-parser@3.723.0(transitive)

• + Added@aws-sdk/util-endpoints@3.743.0(transitive)

• + Added@aws-sdk/util-locate-window@3.723.0(transitive)

• + Added@aws-sdk/util-user-agent-browser@3.734.0(transitive)

• + Added@aws-sdk/util-user-agent-node@3.749.0(transitive)

• + Added@aws-sdk/xml-builder@3.734.0(transitive)

• + Added@smithy/abort-controller@4.0.1(transitive)

• + Added@smithy/chunked-blob-reader@5.0.0(transitive)

• + Added@smithy/chunked-blob-reader-native@4.0.0(transitive)

• + Added@smithy/config-resolver@4.0.1(transitive)

• + Added@smithy/core@3.1.4(transitive)

• + Added@smithy/credential-provider-imds@4.0.1(transitive)

• + Added@smithy/eventstream-codec@4.0.1(transitive)

• + Added@smithy/eventstream-serde-browser@4.0.1(transitive)

• + Added@smithy/eventstream-serde-config-resolver@4.0.1(transitive)

• + Added@smithy/eventstream-serde-node@4.0.1(transitive)

• + Added@smithy/eventstream-serde-universal@4.0.1(transitive)

• + Added@smithy/fetch-http-handler@5.0.1(transitive)

• + Added@smithy/hash-blob-browser@4.0.1(transitive)

• + Added@smithy/hash-node@4.0.1(transitive)

• + Added@smithy/hash-stream-node@4.0.1(transitive)

• + Added@smithy/invalid-dependency@4.0.1(transitive)

• + Added@smithy/is-array-buffer@2.2.04.0.0(transitive)

• + Added@smithy/md5-js@4.0.1(transitive)

• + Added@smithy/middleware-content-length@4.0.1(transitive)

• + Added@smithy/middleware-endpoint@4.0.5(transitive)

• + Added@smithy/middleware-retry@4.0.6(transitive)

• + Added@smithy/middleware-serde@4.0.2(transitive)

• + Added@smithy/middleware-stack@4.0.1(transitive)

• + Added@smithy/node-config-provider@4.0.1(transitive)

• + Added@smithy/node-http-handler@4.0.2(transitive)

• + Added@smithy/property-provider@4.0.1(transitive)

• + Added@smithy/protocol-http@5.0.1(transitive)

• + Added@smithy/querystring-builder@4.0.1(transitive)

• + Added@smithy/querystring-parser@4.0.1(transitive)

• + Added@smithy/service-error-classification@4.0.1(transitive)

• + Added@smithy/shared-ini-file-loader@4.0.1(transitive)

• + Added@smithy/signature-v4@5.0.1(transitive)

• + Added@smithy/smithy-client@4.1.5(transitive)

• + Added@smithy/types@4.1.0(transitive)

• + Added@smithy/url-parser@4.0.1(transitive)

• + Added@smithy/util-base64@4.0.0(transitive)

• + Added@smithy/util-body-length-browser@4.0.0(transitive)

• + Added@smithy/util-body-length-node@4.0.0(transitive)

• + Added@smithy/util-buffer-from@2.2.04.0.0(transitive)

• + Added@smithy/util-config-provider@4.0.0(transitive)

• + Added@smithy/util-defaults-mode-browser@4.0.6(transitive)

• + Added@smithy/util-defaults-mode-node@4.0.6(transitive)

• + Added@smithy/util-endpoints@3.0.1(transitive)

• + Added@smithy/util-hex-encoding@4.0.0(transitive)

• + Added@smithy/util-middleware@4.0.1(transitive)

• + Added@smithy/util-retry@4.0.1(transitive)

• + Added@smithy/util-stream@4.1.1(transitive)

• + Added@smithy/util-uri-escape@4.0.0(transitive)

• + Added@smithy/util-utf8@2.3.04.0.0(transitive)

• + Added@smithy/util-waiter@4.0.2(transitive)

• + Added@types/long@4.0.2(transitive)

• + Added@types/node-int64@0.4.32(transitive)

• + Added@types/q@1.5.8(transitive)

• + Added@types/thrift@0.10.17(transitive)

• + Addedbowser@2.11.0(transitive)

• + Addedbson@4.6.3(transitive)

• + Addedfast-xml-parser@4.4.1(transitive)

• + Addedstrnum@1.0.5(transitive)

• + Addedthrift@0.16.0(transitive)

• + Addedtslib@2.8.1(transitive)

• + Addeduuid@9.0.1(transitive)

• + Addedvarint@6.0.0(transitive)

• + Addedxxhash-wasm@1.1.0(transitive)

• - Removed@types/varint@^6.0.0

• - Removed@types/varint@6.0.3(transitive)

• - Removedbson@4.4.0(transitive)

• - Removedthrift@0.14.1(transitive)

• - Removedvarint@5.0.2(transitive)

• - Removedxxhash-wasm@0.4.2(transitive)

• Updatedbson@4.6.3

• Updatedsnappyjs@^0.6.1

• Updatedthrift@0.16.0

• Updatedvarint@^6.0.0

• Updatedxxhash-wasm@^1.0.2