Socket
Socket
Sign inDemoInstall

@dsnp/parquetjs

Package Overview
Dependencies
189
Maintainers
3
Versions
77
Alerts
File Explorer

Advanced tools

npm Scripts

Install Socket

Detect and block malicious and high-risk dependencies

Install

Comparing version 1.5.0 to 1.6.0

2

dist/lib/bufferReader.js

@@ -49,3 +49,3 @@ "use strict";

processQueue.forEach(async (d) => {
d.resolve(buffer.slice(d.offset - start, d.offset + d.length - start));
d.resolve(buffer.subarray(d.offset - start, d.offset + d.length - start));
});

@@ -52,0 +52,0 @@ };

2

dist/lib/codec/plain_dictionary.js

@@ -29,3 +29,3 @@ "use strict";

const decodeValues = function (type, cursor, count, opts) {
const bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
const bitWidth = cursor.buffer.subarray(cursor.offset, cursor.offset + 1).readInt8(0);
cursor.offset += 1;

@@ -32,0 +32,0 @@ return rle.decodeValues(type, cursor, count, Object.assign({}, opts, { disableEnvelope: true, bitWidth }));

5

dist/lib/codec/plain.js

@@ -183,3 +183,2 @@ "use strict";

}
// Waylands reminder to check again
function encodeValues_BYTE_ARRAY(values) {

@@ -206,3 +205,3 @@ let buf_len = 0;

cursor.offset += 4;
values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
values.push(cursor.buffer.subarray(cursor.offset, cursor.offset + len));
cursor.offset += len;

@@ -232,3 +231,3 @@ }

for (let i = 0; i < count; ++i) {
values.push(cursor.buffer.slice(cursor.offset, cursor.offset + typeLength));
values.push(cursor.buffer.subarray(cursor.offset, cursor.offset + typeLength));
cursor.offset += typeLength;

@@ -235,0 +234,0 @@ }

14

dist/lib/reader.d.ts

@@ -9,2 +9,3 @@ /// <reference types="node" />

import { Options } from './codec/types';
import { S3Client } from "@aws-sdk/client-s3";
/**

@@ -56,7 +57,10 @@ * A parquet cursor is used to retrieve rows from a parquet file in order

/**
* Open the parquet file from S3 using the supplied aws client and params
* The params have to include `Bucket` and `Key` to the file requested
* This function returns a new parquet reader
* Open the parquet file from S3 using the supplied aws client [, commands] and params
* The params have to include `Bucket` and `Key` to the file requested,
* If using v3 of the AWS SDK, combine the client and commands into an object wiht keys matching
* the original module names, and do not instantiate the commands; pass them as classes/modules.
*
* This function returns a new parquet reader [ or throws an Error.]
*/
static openS3(client: ClientS3, params: ClientParameters, options?: BufferReaderOptions): Promise<ParquetReader>;
static openS3(client: any, params: ClientParameters, options?: BufferReaderOptions): Promise<ParquetReader>;
/**

@@ -135,2 +139,4 @@ * Open the parquet file from a url using the supplied request module

static openS3(client: ClientS3, params: ClientParameters, options?: BufferReaderOptions): Promise<ParquetEnvelopeReader>;
static openS3v3(client: S3Client, params: any, options: any): Promise<ParquetEnvelopeReader>;
static streamToBuffer(body: any): Promise<Buffer>;
static openUrl(url: Parameter | URL | string, options?: BufferReaderOptions): Promise<ParquetEnvelopeReader>;

@@ -137,0 +143,0 @@ constructor(readFn: (offset: number, length: number, file?: string) => Promise<Buffer>, closeFn: () => unknown, fileSize: Function | number, options?: BufferReaderOptions, metadata?: FileMetaDataExt);

81

dist/lib/reader.js

@@ -42,2 +42,3 @@ "use strict";

const declare_1 = require("./declare");
const client_s3_1 = require("@aws-sdk/client-s3");
const { getBloomFiltersFor, } = bloomFilterReader;

@@ -133,9 +134,19 @@ /**

/**
* Open the parquet file from S3 using the supplied aws client and params
* The params have to include `Bucket` and `Key` to the file requested
* This function returns a new parquet reader
* Open the parquet file from S3 using the supplied aws client [, commands] and params
* The params have to include `Bucket` and `Key` to the file requested,
* If using v3 of the AWS SDK, combine the client and commands into an object wiht keys matching
* the original module names, and do not instantiate the commands; pass them as classes/modules.
*
* This function returns a new parquet reader [ or throws an Error.]
*/
static async openS3(client, params, options) {
let envelopeReader = await ParquetEnvelopeReader.openS3(client, params, options);
return this.openEnvelopeReader(envelopeReader, options);
try {
let envelopeReader = 'function' === typeof client['headObject'] ?
await ParquetEnvelopeReader.openS3(client, params, options) : // S3 client v2
await ParquetEnvelopeReader.openS3v3(client, params, options); // S3 client v3
return this.openEnvelopeReader(envelopeReader, options);
}
catch (e) {
throw new Error(`Error accessing S3 Bucket ${params.Bucket}. Message: ${e.message}`);
}
}

@@ -382,3 +393,3 @@ /**

}
return Promise.resolve(buffer.slice(offset, offset + length));
return Promise.resolve(buffer.subarray(offset, offset + length));
};

@@ -401,2 +412,45 @@ let closeFn = () => ({});

}
static async openS3v3(client, params, options) {
const fileStat = async () => {
try {
let headObjectCommand = await client.send(new client_s3_1.HeadObjectCommand(params));
return Promise.resolve(headObjectCommand.ContentLength);
}
catch (e) {
// having params match command names makes e.message clear to user
return Promise.reject("rejected headObjectCommand: " + e.message);
}
};
const readFn = async (offset, length, file) => {
if (file) {
return Promise.reject("external references are not supported");
}
const Range = `bytes=${offset}-${offset + length - 1}`;
const input = { ...{ Range }, ...params };
const response = await client.send(new client_s3_1.GetObjectCommand(input));
const body = response.Body;
if (body) {
return ParquetEnvelopeReader.streamToBuffer(body);
}
return Buffer.of();
};
let closeFn = () => ({});
return new ParquetEnvelopeReader(readFn, closeFn, fileStat, options);
}
static async streamToBuffer(body) {
const blob = body;
if (blob.arrayBuffer !== undefined) {
const arrayBuffer = await blob.arrayBuffer();
const uint8Array = new Uint8Array(arrayBuffer);
return Buffer.from(uint8Array);
}
//Assumed to be a Readable like object
const readable = body;
return await new Promise((resolve, reject) => {
const chunks = [];
readable.on("data", (chunk) => chunks.push(chunk));
readable.on("error", reject);
readable.on("end", () => resolve(Buffer.concat(chunks)));
});
}
static async openUrl(url, options) {

@@ -607,4 +661,5 @@ let params;

let trailerLen = PARQUET_MAGIC.length + 4;
let trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);
if (trailerBuf.slice(4).toString() != PARQUET_MAGIC) {
let offset = this.fileSize - trailerLen;
let trailerBuf = await this.read(offset, trailerLen);
if (trailerBuf.subarray(4).toString() != PARQUET_MAGIC) {
throw 'not a valid parquet file';

@@ -666,3 +721,3 @@ }

const headerOffset = cursor.offset;
const headerSize = parquet_util.decodeThrift(pageHeader, cursor.buffer.slice(cursor.offset));
const headerSize = parquet_util.decodeThrift(pageHeader, cursor.buffer.subarray(cursor.offset));
cursor.offset += headerSize;

@@ -741,3 +796,3 @@ const pageType = parquet_util.getThriftEnum(parquet_types_1.default.PageType, pageHeader.type);

offset: 0,
buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
buffer: cursor.buffer.subarray(cursor.offset, cursorEnd),
size: cursorEnd - cursor.offset

@@ -747,3 +802,3 @@ };

if (opts.compression && opts.compression !== 'UNCOMPRESSED') {
let valuesBuf = await parquet_compression.inflate(opts.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd));
let valuesBuf = await parquet_compression.inflate(opts.compression, dictCursor.buffer.subarray(dictCursor.offset, cursorEnd));
dictCursor = {

@@ -765,3 +820,3 @@ buffer: valuesBuf,

if (opts.compression && opts.compression !== 'UNCOMPRESSED') {
let valuesBuf = await parquet_compression.inflate(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd));
let valuesBuf = await parquet_compression.inflate(opts.compression, cursor.buffer.subarray(cursor.offset, cursorEnd));
valuesBufCursor = {

@@ -847,3 +902,3 @@ buffer: valuesBuf,

if (dataPageHeaderV2.is_compressed) {
let valuesBuf = await parquet_compression.inflate(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd));
let valuesBuf = await parquet_compression.inflate(opts.compression, cursor.buffer.subarray(cursor.offset, cursorEnd));
valuesBufCursor = {

@@ -850,0 +905,0 @@ buffer: valuesBuf,

5

package.json

@@ -6,3 +6,3 @@ {

"types": "dist/parquet.d.ts",
"version": "1.5.0",
"version": "1.6.0",
"homepage": "https://github.com/LibertyDSNP/parquetjs",

@@ -19,2 +19,3 @@ "license": "MIT",

"dependencies": {
"@aws-sdk/client-s3": "^3.489.0",
"@types/long": "^4.0.2",

@@ -35,2 +36,3 @@ "@types/node-int64": "^0.4.29",

"devDependencies": {
"@smithy/util-stream": "^2.0.24",
"@types/chai": "^4.3.5",

@@ -43,2 +45,3 @@ "@types/json-schema": "^7.0.11",

"assert": "^2.0.0",
"aws-sdk-client-mock": "^3.0.1",
"browserfs": "^1.4.3",

@@ -45,0 +48,0 @@ "buffer": "^6.0.3",

dist/browser/parquet.cjs.js

Sorry, the diff of this file is too big to display

dist/browser/parquet.esm.js

Sorry, the diff of this file is too big to display

dist/browser/parquet.js

Sorry, the diff of this file is too big to display

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap

About

Packages

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc