parquets - npm Package Compare versions

Comparing version 0.10.9 to 0.10.10

lib/reader.d.ts

@@ -8,3 +8,3 @@ /// <reference types="node" />
  */
-export declare class ParquetCursor<T> {
+export declare class ParquetCursor<T> implements AsyncIterable<T> {
     metadata: FileMetaData;
@@ -32,2 +32,6 @@ envelopeReader: ParquetEnvelopeReader;
     rewind(): void;
+    /**
+     * Implement AsyncIterable
+     */
+    [Symbol.asyncIterator](): AsyncIterator<T>;
 }
@@ -41,3 +45,3 @@ /**
  */
-export declare class ParquetReader<T> {
+export declare class ParquetReader<T> implements AsyncIterable<T> {
     /**
@@ -48,2 +52,3 @@ * Open the parquet file pointed to by the specified path and return a new
     static openFile<T>(filePath: string): Promise<ParquetReader<T>>;
+    static openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>>;
     metadata: FileMetaData;
@@ -69,2 +74,3 @@ envelopeReader: ParquetEnvelopeReader;
     getCursor(): ParquetCursor<T>;
+    getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;
     getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;
@@ -89,2 +95,6 @@ /**
     close(): Promise<void>;
+    /**
+     * Implement AsyncIterable
+     */
+    [Symbol.asyncIterator](): AsyncIterator<T>;
 }
@@ -102,2 +112,3 @@ /**
     static openFile(filePath: string): Promise<ParquetEnvelopeReader>;
+    static openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader>;
     constructor(read: (position: number, length: number) => Promise<Buffer>, close: () => Promise<void>, fileSize: number);
@@ -104,0 +115,0 @@ readHeader(): Promise<void>;

lib/reader.js

@@ -64,2 +64,29 @@ "use strict";
     }
+    /**
+     * Implement AsyncIterable
+     */
+    // tslint:disable-next-line:function-name
+    [Symbol.asyncIterator]() {
+        let done = false;
+        return {
+            next: async () => {
+                if (done) {
+                    return { done, value: null };
+                }
+                const value = await this.next();
+                if (value === null) {
+                    return { done: true, value };
+                }
+                return { done: false, value };
+            },
+            return: async () => {
+                done = true;
+                return { done, value: null };
+            },
+            throw: async () => {
+                done = true;
+                return { done: true, value: null };
+            }
+        };
+    }
 }
@@ -107,2 +134,14 @@ exports.ParquetCursor = ParquetCursor;
     }
+    static async openBuffer(buffer) {
+        const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);
+        try {
+            await envelopeReader.readHeader();
+            const metadata = await envelopeReader.readFooter();
+            return new ParquetReader(metadata, envelopeReader);
+        }
+        catch (err) {
+            await envelopeReader.close();
+            throw err;
+        }
+    }
     getCursor(columnList) {
@@ -149,2 +188,9 @@ if (!columnList) {
     }
+    /**
+     * Implement AsyncIterable
+     */
+    // tslint:disable-next-line:function-name
+    [Symbol.asyncIterator]() {
+        return this.getCursor()[Symbol.asyncIterator]();
+    }
 }
@@ -171,2 +217,7 @@ exports.ParquetReader = ParquetReader;
     }
+    static async openBuffer(buffer) {
+        const readFn = (position, length) => Promise.resolve(buffer.slice(position, position + length));
+        const closeFn = () => Promise.resolve();
+        return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);
+    }
     async readHeader() {
@@ -173,0 +224,0 @@ const buf = await this.read(0, PARQUET_MAGIC.length);

lib/writer.js

@@ -191,8 +191,2 @@ "use strict";
         }
-        if (this.rowCount === 0) {
-            throw new Error('cannot write parquet file with zero rows');
-        }
-        if (this.schema.fieldList.length === 0) {
-            throw new Error('cannot write parquet file with zero fieldList');
-        }
         return this.writeSection(encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata));
@@ -385,8 +379,4 @@ }
     /* list encodings */
-    const encodingsSet = {};
-    encodingsSet[PARQUET_RDLVL_ENCODING] = true;
-    encodingsSet[column.encoding] = true;
-    for (const k in encodingsSet) {
-        metadata.encodings.push(thrift_1.Encoding[k]);
-    }
+    metadata.encodings.push(thrift_1.Encoding[PARQUET_RDLVL_ENCODING]);
+    metadata.encodings.push(thrift_1.Encoding[column.encoding]);
     /* concat metadata header and data pages */
@@ -393,0 +383,0 @@ const metadataOffset = baseOffset + pageBuf.length;

package.json

 {
   "name": "parquets",
   "description": "TypeScript implementation of the Parquet file format, based on parquet.js",
-  "version": "0.10.9",
+  "version": "0.10.10",
   "upstream": "0.10.1",
@@ -9,2 +9,5 @@ "homepage": "https://github.com/kbajalc/parquets",
   "license": "MIT",
+  "browser": {
+    "fs": false
+  },
   "main": "./lib/index.js",
@@ -62,3 +65,2 @@ "types": "./lib/index.d.ts",
     "@types/node-int64": "^0.4.29",
-    "@types/snappy": "^6.0.0",
     "@types/thrift": "^0.10.8",
@@ -74,5 +76,5 @@ "@types/varint": "^5.0.0",
     "lzo": "^0.4.11",
-    "node-snappy": "^0.1.4",
     "object-stream": "0.0.1",
-    "snappy": "6.2.3",
+    "prettier": "^2.1.2",
+    "snappy": "^6.3.5",
     "ts-jest": "^24.0.2",
@@ -102,3 +104,7 @@ "ts-node": "^8.3.0",
     ]
+  },
+  "prettier": {
+    "arrowParens": "avoid",
+    "singleQuote": true
   }
 }

README.md

@@ -297,3 +297,3 @@ # parquets
 
-Depdendencies
+Dependencies
 -------------
@@ -300,0 +300,0 @@

src/reader.ts

@@ -30,3 +30,3 @@ import { CursorBuffer, ParquetCodecOptions, PARQUET_CODEC } from './codec';
  */
-export class ParquetCursor<T> {
+export class ParquetCursor<T> implements AsyncIterable<T> {
 
@@ -82,2 +82,30 @@ public metadata: FileMetaData;
   }
+
+  /**
+   * Implement AsyncIterable
+   */
+  // tslint:disable-next-line:function-name
+  [Symbol.asyncIterator](): AsyncIterator<T> {
+    let done = false;
+    return {
+      next: async () => {
+        if (done) {
+          return { done, value: null };
+        }
+        const value = await this.next();
+        if (value === null) {
+          return { done: true, value };
+        }
+        return { done: false, value };
+      },
+      return: async () => {
+        done = true;
+        return { done, value: null };
+      },
+      throw: async () => {
+        done = true;
+        return { done: true, value: null };
+      }
+    };
+  }
 }
@@ -92,3 +120,3 @@
  */
-export class ParquetReader<T> {
+export class ParquetReader<T> implements AsyncIterable<T> {
 
@@ -111,2 +139,14 @@ /**
 
+  static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {
+    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);
+    try {
+      await envelopeReader.readHeader();
+      const metadata = await envelopeReader.readFooter();
+      return new ParquetReader<T>(metadata, envelopeReader);
+    } catch (err) {
+      await envelopeReader.close();
+      throw err;
+    }
+  }
+
   public metadata: FileMetaData;
@@ -144,2 +184,3 @@ public envelopeReader: ParquetEnvelopeReader;
   getCursor(): ParquetCursor<T>;
+  getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;
   getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;
@@ -198,2 +239,10 @@ getCursor(columnList?: (string | string[])[]): ParquetCursor<Partial<T>> {
   }
+
+  /**
+   * Implement AsyncIterable
+   */
+  // tslint:disable-next-line:function-name
+  [Symbol.asyncIterator](): AsyncIterator<T> {
+    return this.getCursor()[Symbol.asyncIterator]();
+  }
 }
@@ -219,2 +268,9 @@
 
+  static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {
+    const readFn = (position: number, length: number) =>
+      Promise.resolve(buffer.slice(position, position + length));
+    const closeFn = () => Promise.resolve();
+    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);
+  }
+
   constructor(
@@ -221,0 +277,0 @@ public read: (position: number, length: number) => Promise<Buffer>,

src/writer.ts

@@ -257,10 +257,2 @@ import { Transform, Writable } from 'stream';
 
-    if (this.rowCount === 0) {
-      throw new Error('cannot write parquet file with zero rows');
-    }
-
-    if (this.schema.fieldList.length === 0) {
-      throw new Error('cannot write parquet file with zero fieldList');
-    }
-
     return this.writeSection(encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata));
@@ -535,8 +527,4 @@ }
   /* list encodings */
-  const encodingsSet: Record<string, boolean> = {};
-  encodingsSet[PARQUET_RDLVL_ENCODING] = true;
-  encodingsSet[column.encoding] = true;
-  for (const k in encodingsSet) {
-    metadata.encodings.push((Encoding as any)[k]);
-  }
+  metadata.encodings.push(Encoding[PARQUET_RDLVL_ENCODING]);
+  metadata.encodings.push(Encoding[column.encoding]);
 
@@ -543,0 +531,0 @@ /* concat metadata header and data pages */

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

758009

increased by0.59%

Dev dependency count

26

decreased by-3.7%

Lines of code

15564

increased by0.61%

No dependency changes