hyparquet - npm Package Compare versions

Comparing version 0.1.0 to 0.1.1

package.json

 {
   "name": "hyparquet",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "parquet file parser for javascript",
   "keywords": [
     "parquet",
-    "parser"
+    "parser",
+    "snappy",
+    "thrift"
   ],
@@ -12,3 +14,3 @@ "license": "MIT",
     "type": "git",
-    "url": "https://github.com/hyparam/hyparquet"
+    "url": "git+https://github.com/hyparam/hyparquet.git"
   },
@@ -31,3 +33,3 @@ "main": "src/hyparquet.js",
     "@typescript-eslint/eslint-plugin": "6.17.0",
-    "@vitest/coverage-v8": "1.1.2",
+    "@vitest/coverage-v8": "1.1.3",
     "eslint": "8.56.0",
@@ -38,4 +40,4 @@ "eslint-plugin-import": "2.29.1",
     "typescript": "5.3.3",
-    "vitest": "1.1.2"
+    "vitest": "1.1.3"
   }
 }

src/hyparquet.d.ts

@@ -18,1 +18,21 @@ /**
 export function parquetMetadata(arrayBuffer: ArrayBuffer): any
+
+/**
+ * Decompress snappy data.
+ * Accepts an output buffer to avoid allocating a new buffer for each call.
+ *
+ * @param {Uint8Array} inputArray compressed data
+ * @param {Uint8Array} outputArray output buffer
+ * @returns {boolean} true if successful
+ */
+export function snappyUncompress(inputArray: Uint8Array, outputArray: Uint8Array): boolean
+
+/**
+ * Replace bigints with numbers.
+ * When parsing parquet files, bigints are used to represent 64-bit integers.
+ * However, JSON does not support bigints, so it's helpful to convert to numbers.
+ *
+ * @param {any} obj object to convert
+ * @returns {unknown} converted object
+ */
+export function toJson(obj: any): unknown

src/hyparquet.js

 import { parquetMetadata } from './metadata.js'
 export { parquetMetadata }
 
+import { snappyUncompress } from './snappy.js'
+export { snappyUncompress }
+
+import { toJson } from './toJson.js'
+export { toJson }
+
 /**
- * Read parquet data rows from a file
+ * Read parquet data rows from a buffer.
  *
@@ -7,0 +13,0 @@ * @param {ArrayBuffer} arrayBuffer parquet file contents

src/metadata.js

@@ -100,26 +100,1 @@ import { deserializeTCompactProtocol } from './thrift.js'
 }
-
-/**
- * Replace bigints with numbers.
- * When parsing parquet files, bigints are used to represent 64-bit integers.
- * However, JSON does not support bigints, so it's helpful to convert to numbers.
- *
- * @param {any} obj object to convert
- * @returns {unknown} converted object
- */
-export function toJson(obj) {
-  if (typeof obj === 'bigint') {
-    return Number(obj)
-  } else if (Array.isArray(obj)) {
-    return obj.map(toJson)
-  } else if (obj instanceof Object) {
-    /** @type {Record<string, unknown>} */
-    const newObj = {}
-    for (const key of Object.keys(obj)) {
-      newObj[key] = toJson(obj[key])
-    }
-    return newObj
-  } else {
-    return obj
-  }
-}

src/thrift.js

@@ -137,3 +137,3 @@ // TCompactProtocol types
  */
-function readVarInt(view, index) {
+export function readVarInt(view, index) {
   let result = 0
@@ -140,0 +140,0 @@ let shift = 0

src/types.d.ts

@@ -36,3 +36,3 @@ /**
   INT64 = 2,
-  INT96 = 3,
+  INT96 = 3, // deprecated
   FLOAT = 4,
@@ -97,3 +97,3 @@ DOUBLE = 5,
   RLE = 3,
-  BIT_PACKED = 4,
+  BIT_PACKED = 4, // deprecated
   DELTA_BINARY_PACKED = 5,
@@ -147,1 +147,40 @@ DELTA_LENGTH_BYTE_ARRAY = 6,
 }
+
+// Parquet file header types
+export interface PageHeader {
+  type: PageType
+  uncompressed_page_size: number
+  compressed_page_size: number
+  crc?: number
+  data_page_header?: DataPageHeader
+  index_page_header?: IndexPageHeader
+  dictionary_page_header?: DictionaryPageHeader
+  data_page_header_v2?: DataPageHeaderV2
+}
+
+export interface DataPageHeader {
+  num_values: number
+  encoding: Encoding
+  definition_level_encoding: Encoding
+  repetition_level_encoding: Encoding
+  statistics?: Statistics
+}
+
+interface IndexPageHeader {}
+
+export interface DictionaryPageHeader {
+  num_values: number
+  encoding: Encoding
+  is_sorted?: boolean
+}
+
+interface DataPageHeaderV2 {
+  num_values: number
+  num_nulls: number
+  num_rows: number
+  encoding: Encoding
+  definition_levels_byte_length: number
+  repetition_levels_byte_length: number
+  is_compressed?: boolean
+  statistics?: Statistics
+}

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

37780

increased by74.64%

Number of package files

13

increased by44.44%

Lines of code

1140

increased by72.73%

No dependency changes