parquets - npm Package Compare versions

Comparing version 0.8.11 to 0.8.12

package.json

 {
   "name": "parquets",
   "description": "TypeScript implementation of the Parquet file format, based on parquet.js",
-  "version": "0.8.11",
+  "version": "0.8.12",
   "homepage": "https://github.com/kbajalc/parquets",
@@ -6,0 +6,0 @@ "author": "[e-mail]",

README.md

@@ -15,2 +15,3 @@ # parquets
 
+**WARNING**: *There are compatibility issues with the reference implementation when using 'optional' columns! Only GZIP and SNAPPY compressions are working properly. Testing done with [Appache Drill](https://drill.apache.org)*.
 
@@ -17,0 +18,0 @@ **What is Parquet?**: Parquet is a column-oriented file format; it allows you to

src/codec/plain.ts

@@ -180,3 +180,3 @@ import { FieldDefinition, ParquetType, TODO } from '../declare';
   if (!opts.typeLength) {
-    throw 'missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)';
+    throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
   }
@@ -188,3 +188,3 @@
     if (values[i].length !== opts.typeLength) {
-      throw 'invalid value for FIXED_LEN_BYTE_ARRAY: ' + values[i];
+      throw new Error('invalid value for FIXED_LEN_BYTE_ARRAY: ' + values[i]);
     }
@@ -200,3 +200,3 @@ }
   if (!opts.typeLength) {
-    throw 'missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)';
+    throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
   }
@@ -231,3 +231,3 @@
     default:
-      throw 'unsupported type: ' + type;
+      throw new Error('unsupported type: ' + type);
   }
@@ -255,4 +255,4 @@ }
     default:
-      throw 'unsupported type: ' + type;
+      throw new Error('unsupported type: ' + type);
   }
 }

src/codec/rle.ts

@@ -6,3 +6,3 @@ import varint = require('varint');
   if (values.length % 8 !== 0) {
-    throw 'must be a multiple of 8';
+    throw new Error('must be a multiple of 8');
   }
@@ -39,3 +39,3 @@
   if (!('bitWidth' in opts)) {
-    throw 'bitWidth is required';
+    throw new Error('bitWidth is required');
   }
@@ -53,3 +53,3 @@
     default:
-      throw 'unsupported type: ' + type;
+      throw new Error('unsupported type: ' + type);
   }
@@ -106,3 +106,3 @@
   if (count % 8 !== 0) {
-    throw 'must be a multiple of 8';
+    throw new Error('must be a multiple of 8');
   }
@@ -136,3 +136,3 @@
   if (!('bitWidth' in opts)) {
-    throw 'bitWidth is required';
+    throw new Error('bitWidth is required');
   }
@@ -158,3 +158,3 @@
   if (values.length !== count) {
-    throw 'invalid RLE encoding';
+    throw new Error('invalid RLE encoding');
   }
@@ -161,0 +161,0 @@

src/compression.ts

@@ -44,3 +44,3 @@ import zlib = require('zlib');
   if (!(method in PARQUET_COMPRESSION_METHODS)) {
-    throw 'invalid compression method: ' + method;
+    throw new Error('invalid compression method: ' + method);
   }
@@ -97,3 +97,3 @@
   if (!(method in PARQUET_COMPRESSION_METHODS)) {
-    throw 'invalid compression method: ' + method;
+    throw new Error('invalid compression method: ' + method);
   }
@@ -100,0 +100,0 @@

src/index.ts

 export * from './declare';
-export { ParquetEnvelopeReader, ParquetReader } from './reader';
+export { ParquetCursor, ParquetEnvelopeReader, ParquetReader } from './reader';
 export { ParquetSchema } from './schema';
@@ -4,0 +4,0 @@ export { ParquetEnvelopeWriter, ParquetTransformer, ParquetWriter } from './writer';

src/reader.ts

@@ -122,3 +122,3 @@ import { PARQUET_CODEC } from './codec';
     if (metadata.version !== PARQUET_VERSION) {
-      throw 'invalid parquet version';
+      throw new Error('invalid parquet version');
     }
@@ -227,3 +227,3 @@
     if (buf.toString() !== PARQUET_MAGIC) {
-      throw 'not valid parquet file';
+      throw new Error('not valid parquet file');
     }
@@ -254,3 +254,3 @@ }
     if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
-      throw 'external references are not supported';
+      throw new Error('external references are not supported');
     }
@@ -382,3 +382,7 @@
       valueCount,
-      { bitWidth: Util.getBitWidth(opts.rLevelMax) });
+      {
+        bitWidth: Util.getBitWidth(opts.rLevelMax),
+        disableEnvelope: false
+      }
+    );
   } else {
@@ -402,3 +406,7 @@ rLevels.fill(0);
       valueCount,
-      { bitWidth: Util.getBitWidth(opts.dLevelMax) });
+      {
+        bitWidth: Util.getBitWidth(opts.dLevelMax),
+        disableEnvelope: false
+      }
+    );
   } else {
@@ -405,0 +413,0 @@ dLevels.fill(0);

src/schema.ts

@@ -139,3 +139,3 @@ import { PARQUET_CODEC } from './codec';
     if (!typeDef) {
-      throw 'invalid parquet type: ' + opts.type;
+      throw new Error('invalid parquet type: ' + opts.type);
     }
@@ -149,3 +149,3 @@
     if (!(opts.encoding in PARQUET_CODEC)) {
-      throw 'unsupported parquet encoding: ' + opts.encoding;
+      throw new Error('unsupported parquet encoding: ' + opts.encoding);
     }
@@ -158,3 +158,3 @@
     if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {
-      throw 'unsupported compression method: ' + opts.compression;
+      throw new Error('unsupported compression method: ' + opts.compression);
     }
@@ -161,0 +161,0 @@

src/shred.ts

@@ -93,7 +93,7 @@ import { ColumnData, FieldDefinition, ParquetRow, RecordBuffer, TODO } from './declare';
     if (values.length === 0 && !!record && field.repetitionType === 'REQUIRED') {
-      throw 'missing required field: ' + field.name;
+      throw new Error('missing required field: ' + field.name);
     }
 
     if (values.length > 1 && field.repetitionType !== 'REPEATED') {
-      throw 'too many values for field: ' + field.name;
+      throw new Error('too many values for field: ' + field.name);
     }
@@ -100,0 +100,0 @@

src/types.ts

@@ -139,3 +139,3 @@ import { BSON } from 'bson';
   if (!(type in PARQUET_LOGICAL_TYPES)) {
-    throw 'invalid type: ' + type;
+    throw new Error('invalid type: ' + type);
   }
@@ -152,3 +152,3 @@
   if (!(type in PARQUET_LOGICAL_TYPES)) {
-    throw 'invalid type: ' + type;
+    throw new Error('invalid type: ' + type);
   }
@@ -175,3 +175,3 @@
   if (isNaN(v)) {
-    throw 'invalid value for FLOAT: ' + value;
+    throw new Error('invalid value for FLOAT: ' + value);
   }
@@ -185,3 +185,3 @@
   if (isNaN(v)) {
-    throw 'invalid value for DOUBLE: ' + value;
+    throw new Error('invalid value for DOUBLE: ' + value);
   }
@@ -195,3 +195,3 @@
   if (v < -0x80 || v > 0x7f || isNaN(v)) {
-    throw 'invalid value for INT8: ' + value;
+    throw new Error('invalid value for INT8: ' + value);
   }
@@ -205,3 +205,3 @@
   if (v < 0 || v > 0xff || isNaN(v)) {
-    throw 'invalid value for UINT8: ' + value;
+    throw new Error('invalid value for UINT8: ' + value);
   }
@@ -215,3 +215,3 @@
   if (v < -0x8000 || v > 0x7fff || isNaN(v)) {
-    throw 'invalid value for INT16: ' + value;
+    throw new Error('invalid value for INT16: ' + value);
   }
@@ -225,3 +225,3 @@
   if (v < 0 || v > 0xffff || isNaN(v)) {
-    throw 'invalid value for UINT16: ' + value;
+    throw new Error('invalid value for UINT16: ' + value);
   }
@@ -235,3 +235,3 @@
   if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {
-    throw 'invalid value for INT32: ' + value;
+    throw new Error('invalid value for INT32: ' + value);
   }
@@ -245,3 +245,3 @@
   if (v < 0 || v > 0xffffffffffff || isNaN(v)) {
-    throw 'invalid value for UINT32: ' + value;
+    throw new Error('invalid value for UINT32: ' + value);
   }
@@ -255,3 +255,3 @@
   if (isNaN(v)) {
-    throw 'invalid value for INT64: ' + value;
+    throw new Error('invalid value for INT64: ' + value);
   }
@@ -265,3 +265,3 @@
   if (v < 0 || isNaN(v)) {
-    throw 'invalid value for UINT64: ' + value;
+    throw new Error('invalid value for UINT64: ' + value);
   }
@@ -275,3 +275,3 @@
   if (isNaN(v)) {
-    throw 'invalid value for INT96: ' + value;
+    throw new Error('invalid value for INT96: ' + value);
   }
@@ -315,3 +315,3 @@
   if (v < 0 || v > 0xffffffffffffffff || isNaN(v)) {
-    throw 'invalid value for TIME_MILLIS: ' + value;
+    throw new Error('invalid value for TIME_MILLIS: ' + value);
   }
@@ -325,3 +325,3 @@
   if (v < 0 || isNaN(v)) {
-    throw 'invalid value for TIME_MICROS: ' + value;
+    throw new Error('invalid value for TIME_MICROS: ' + value);
   }
@@ -344,3 +344,3 @@
     if (v < 0 || isNaN(v)) {
-      throw 'invalid value for DATE: ' + value;
+      throw new Error('invalid value for DATE: ' + value);
     }
@@ -366,3 +366,3 @@
     if (v < 0 || isNaN(v)) {
-      throw 'invalid value for TIMESTAMP_MILLIS: ' + value;
+      throw new Error('invalid value for TIMESTAMP_MILLIS: ' + value);
     }
@@ -388,3 +388,3 @@
     if (v < 0 || isNaN(v)) {
-      throw 'invalid value for TIMESTAMP_MICROS: ' + value;
+      throw new Error('invalid value for TIMESTAMP_MICROS: ' + value);
     }
@@ -402,3 +402,3 @@
   if (!value.months || !value.days || !value.milliseconds) {
-    throw 'value for INTERVAL must be object { months: ..., days: ..., milliseconds: ... }';
+    throw new Error('value for INTERVAL must be object { months: ..., days: ..., milliseconds: ... }');
   }
@@ -405,0 +405,0 @@

src/util.ts

@@ -92,3 +92,3 @@ import fs = require('fs');
   }
-  throw 'Invalid ENUM value';
+  throw new Error('Invalid ENUM value');
 }
@@ -95,0 +95,0 @@

src/writer.ts

@@ -117,3 +117,3 @@ import { WriteStream } from 'fs';
     if (this.closed) {
-      throw 'writer was closed';
+      throw new Error('writer was closed');
     }
@@ -268,7 +268,7 @@
     if (this.rowCount === 0) {
-      throw 'cannot write parquet file with zero rows';
+      throw new Error('cannot write parquet file with zero rows');
     }
 
     if (this.schema.fieldList.length === 0) {
-      throw 'cannot write parquet file with zero fieldList';
+      throw new Error('cannot write parquet file with zero fieldList');
     }
@@ -336,3 +336,3 @@
   if (!(encoding in PARQUET_CODEC)) {
-    throw 'invalid encoding: ' + encoding;
+    throw new Error('invalid encoding: ' + encoding);
   }
@@ -374,3 +374,7 @@
       rlevels,
-      { bitWidth: Util.getBitWidth(column.rLevelMax) });
+      {
+        bitWidth: Util.getBitWidth(column.rLevelMax)
+        // disableEnvelope: false
+      }
+    );
   }
@@ -384,3 +388,7 @@
       dlevels,
-      { bitWidth: Util.getBitWidth(column.dLevelMax) });
+      {
+        bitWidth: Util.getBitWidth(column.dLevelMax)
+        // disableEnvelope: false
+      }
+    );
   }
@@ -493,3 +501,4 @@
   /* encode data page(s) */
-  const pages: Buffer[] = [];
+  // const pages: Buffer[] = [];
+  let pageBuf: Buffer;
   // tslint:disable-next-line:variable-name
@@ -522,3 +531,4 @@ let total_uncompressed_size = 0;
     }
-    pages.push(result.page);
+    // pages.push(result.page);
+    pageBuf = result.page;
     total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
@@ -528,3 +538,3 @@ total_compressed_size += result.header.compressed_page_size + result.headerSize;
 
-  const pagesBuf = Buffer.concat(pages);
+  // const pagesBuf = Buffer.concat(pages);
 
@@ -554,4 +564,4 @@ const compression = opts.column.compression === 'UNCOMPRESSED' ? (opts.compression || 'UNCOMPRESSED') : opts.column.compression;
   /* concat metadata header and data pages */
-  const metadataOffset = opts.baseOffset + pagesBuf.length;
-  const body = Buffer.concat([pagesBuf, Util.serializeThrift(metadata)]);
+  const metadataOffset = opts.baseOffset + pageBuf.length;
+  const body = Buffer.concat([pageBuf, Util.serializeThrift(metadata)]);
   return { body, metadata, metadataOffset };
@@ -558,0 +568,0 @@ }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Improved metrics

Number of lines in readme file

327

increased by0.31%

Number of low supply chain risk alerts

1

decreased by-50%

Worsened metrics

Total package byte prevSize

342370

decreased by-55.7%

Number of package files

36

decreased by-52%

Lines of code

8461

decreased by-45.18%

No dependency changes