Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
hyparquet - npm Package Compare versions
Comparing version 0.7.0 to 0.7.1
| { | ||
| "name": "hyparquet", | ||
| "version": "0.7.0", | ||
| "version": "0.7.1", | ||
| "description": "parquet file parser for javascript", | ||
@@ -30,5 +30,5 @@ "keywords": [ | ||
| "devDependencies": { | ||
| "@types/node": "20.11.27", | ||
| "@typescript-eslint/eslint-plugin": "7.2.0", | ||
| "@vitest/coverage-v8": "1.3.1", | ||
| "@types/node": "20.11.29", | ||
| "@typescript-eslint/eslint-plugin": "7.3.1", | ||
| "@vitest/coverage-v8": "1.4.0", | ||
| "eslint": "8.57.0", | ||
@@ -40,4 +40,4 @@ "eslint-plugin-import": "2.29.1", | ||
| "typescript": "5.4.2", | ||
| "vitest": "1.3.1" | ||
| "vitest": "1.4.0" | ||
| } | ||
| } |
@@ -0,4 +1,5 @@ | ||
| import { assembleObjects } from './assemble.js' | ||
| import { PageType } from './constants.js' | ||
| import { convert } from './convert.js' | ||
| import { assembleObjects, readDataPage, readDictionaryPage } from './datapage.js' | ||
| import { readDataPage, readDictionaryPage } from './datapage.js' | ||
| import { readDataPageV2 } from './datapageV2.js' | ||
@@ -32,3 +33,2 @@ import { parquetHeader } from './header.js' | ||
| let byteOffset = 0 // byteOffset within the column | ||
| const rowIndex = [0] // map/list object index | ||
| const rowData = [] | ||
@@ -70,6 +70,5 @@ | ||
| const isNull = columnMetadata && !isRequired(schema, [columnMetadata.path_in_schema[0]]) | ||
| const nullValue = false // TODO: unused? | ||
| const maxDefinitionLevel = getMaxDefinitionLevel(schema, columnMetadata.path_in_schema) | ||
| values = assembleObjects( | ||
| definitionLevels, repetitionLevels, dataPage, isNull, nullValue, maxDefinitionLevel, rowIndex[0] | ||
| definitionLevels, repetitionLevels, dataPage, isNull, maxDefinitionLevel | ||
| ) | ||
@@ -120,3 +119,3 @@ } else if (definitionLevels?.length) { | ||
| rowData.push(...assembleObjects( | ||
| definitionLevels, repetitionLevels, dataPage, true, false, maxDefinitionLevel, rowIndex[0] | ||
| definitionLevels, repetitionLevels, dataPage, true, maxDefinitionLevel | ||
| )) | ||
@@ -123,0 +122,0 @@ } else if (daph2.num_nulls) { |
@@ -171,66 +171,1 @@ import { readData, readPlain, readRleBitPackedHybrid, widthFromMaxInt } from './encoding.js' | ||
| } | ||
| /** | ||
| * Dremel-assembly of arrays of values into lists | ||
| * | ||
| * @param {number[] | undefined} definitionLevels definition levels, max 3 | ||
| * @param {number[]} repetitionLevels repetition levels, max 1 | ||
| * @param {ArrayLike<any>} value values to process | ||
| * @param {boolean} isNull can an entry be null? | ||
| * @param {boolean} nullValue can list elements be null? | ||
| * @param {number} maxDefinitionLevel definition level that corresponds to non-null | ||
| * @param {number} prevIndex 1 + index where the last row in the previous page was inserted (0 if first page) | ||
| * @returns {any[]} array of values | ||
| */ | ||
| export function assembleObjects( | ||
| definitionLevels, repetitionLevels, value, isNull, nullValue, maxDefinitionLevel, prevIndex | ||
| ) { | ||
| let vali = 0 | ||
| let started = false | ||
| let haveNull = false | ||
| let i = prevIndex | ||
| let part = [] | ||
| /** @type {any[]} */ | ||
| const assign = [] | ||
| for (let counter = 0; counter < repetitionLevels.length; counter++) { | ||
| const def = definitionLevels ? definitionLevels[counter] : maxDefinitionLevel | ||
| const rep = repetitionLevels[counter] | ||
| if (!rep) { | ||
| // new row - save what we have | ||
| if (started) { | ||
| assign[i] = haveNull ? undefined : part | ||
| part = [] | ||
| i++ | ||
| } else { | ||
| // first time: no row to save yet, unless it's a row continued from previous page | ||
| if (vali > 0) { | ||
| assign[i - 1] = assign[i - 1]?.concat(part) // add items to previous row | ||
| part = [] | ||
| // don't increment i since we only filled i-1 | ||
| } | ||
| started = true | ||
| } | ||
| } | ||
| if (def === maxDefinitionLevel) { | ||
| // append real value to current item | ||
| part.push(value[vali]) | ||
| vali++ | ||
| } else if (def > 0) { | ||
| // append null to current item | ||
| part.push(undefined) | ||
| } | ||
| haveNull = def === 0 && isNull | ||
| } | ||
| if (started) { | ||
| assign[i] = haveNull ? undefined : part | ||
| } else if (vali > 0) { | ||
| assign[i - 1] = assign[i - 1]?.concat(part) | ||
| } | ||
| return assign | ||
| } |
New alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Fixed alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Improved metrics
- Number of package files
- increased by5.26%
20
Worsened metrics
- Total package byte prevSize
- decreased by-0.4%
92764
- Lines of code
- decreased by-0.2%
2480
No dependency changes