ascii-grid - npm Package Compare versions

Comparing version 0.2.0 to 1.0.1

src/index.js

+const isAsciiGrid = require("./is-ascii-grid");
+const parseAsciiGridData = require("./parse-ascii-grid-data");
+const parseAsciiGridMetaData = require("./parse-ascii-grid-meta");
+
+module.exports = {
+  isAsciiGrid,
+  parseAsciiGridData,
+  parseAsciiGridMetaData
+};

package.json

 {
   "name": "ascii-grid",
-  "version": "0.2.0",
+  "version": "1.0.1",
   "description": "Identify and Read an ARC/INFO ASCII Grid",
-  "main": "src",
+  "main": "src/index.js",
+  "files": [
+    "src/index.js",
+    "src/is-ascii-grid.js",
+    "src/parse-ascii-grid-data.js",
+    "src/parse-ascii-grid-meta.js"
+  ],
   "scripts": {
-    "format": "prettier --arrow-parens=avoid --trailing-comma=none --write test.js src/*.js",
-    "test": "ava --timeout=2m --verbose"
+    "format": "prettier --arrow-parens=avoid --print-width=160 --trailing-comma=none --write test.js src/*.js",
+    "test": "node test.js"
   },
@@ -32,5 +38,5 @@ "repository": {
   "devDependencies": {
-    "ava": "^3.14.0",
     "fast-max": "0.0.0",
     "fast-min": "0.0.0",
+    "flug": "^1.1.0",
     "prettier": "^2.2.1",
@@ -40,5 +46,4 @@ "toab": "^1.0.2"
   "dependencies": {
-    "get-byte": "0.0.0",
-    "to-typed-array": "0.0.1"
+    "get-byte": "0.0.0"
   }
 }

README.md

 # ascii-grid: beta
-Identify and Read an ARC/INFO ASCII Grid
+> Identify and Read an ARC/INFO ASCII Grid
 
+# motivation
+I do a lot of client-side geoprocessing (see [geoblaze](http://github.com/geotiff/geoblaze)) and wanted to add support for .asc files.
+When I encountered large .asc files, I quickly ran out of memory because I was trying to load the whole file into memory.
+This package was created to make it easy to read specific areas of an ASCII Grid in a memory-safe way and prevent my laptop from overheating.
+
 # usage
@@ -11,3 +16,3 @@ ## identify ascii grid files
 const buffer = readFileSync('./test_data/michigan_lld/michigan_lld.asc');
-isAsciiGrid(buffer, { debug: false });
+isAsciiGrid({ data: buffer, debug: false });
 // true
@@ -21,3 +26,3 @@ ```
 const buffer = readFileSync('./test_data/michigan_lld/michigan_lld.asc');
-const metadata = parseAsciiGridMeta(buffer, { debug: false });
+const metadata = parseAsciiGridMeta({ data: buffer, debug: false });
 /*
@@ -42,3 +47,50 @@ {
 const result = await parseAsciiGridData({ data: buffer, debug: true });
-// result is a two-dimensional array of rows with pixel values
+/*
+ result is an object with a values array that holds
+ two-dimensional array of rows with pixel values
+ {
+   values:
+    [
+      [55.874908, 57.874924, 58.874939, ...], // first row
+      [62.875015, 63.875031, 62.875046, ...],
+      [62.875122, 64.875137, 63.875168, ...],
+      .
+      .
+      .
+      [52.875671, 50.875702, 51.875717, ...], // last row
+    ]
+  }
+*/
+```
+
+## Reading Pixel Values within Bounding Box
+You can specify a bounding box to read from by specifying the zero-based index
+values of the first and last row, and first and last column for each row
+```javascript
+const parseAsciiGridData = require("ascii-grid/parse-ascii-grid-data");
+
+const result = await parseAsciiGridData({
+  data: buffer,
+  debug: true,
+  start_column: 2, // start reading from the third column
+  end_column: 10, // read through the eleventh column
+  start_row: 1, // skip the first row
+  end_row: undefined // read through the end
+});
+
+/*
+ result's values array is the size of the bbox.
+ each row has a length of end_column - start_column + 1
+ {
+   values:
+    [
+      [62.875046, ...], // first row starting with index of start_row
+      [63.875168, ...],
+      .
+      .
+      .
+      [51.875717, ...], // last row
+    ]
+  }
+*/
 ```

src/is-ascii-grid.js

@@ -38,6 +38,3 @@ const getByte = require("get-byte");
     if (debug) console.log("data is a string");
-    return (
-      Boolean(data.match(/.asc(.gz|.tar|.tar.gz|.tgz|.zip)?$/i)) ||
-      (data.includes("ncols") && data.includes("nrows"))
-    );
+    return Boolean(data.match(/.asc(.gz|.tar|.tar.gz|.tgz|.zip)?$/i)) || (data.includes("ncols") && data.includes("nrows"));
   } else {
@@ -44,0 +41,0 @@ return false;

src/parse-ascii-grid-data.js

@@ -9,12 +9,22 @@ const getByte = require("get-byte");
 const NINE_CHARCODE = "9".charCodeAt(0);
-// const NULL_CHARACTER = "\u0000".charCodeAt(0);
+const DOT_CHARCODE = ".".charCodeAt(0);
+const NULL_CHARCODE = 0;
 
-module.exports = async ({
+/*
+  Notes: .asc files can end with a newline, null byte, or number
+*/
+
+module.exports = ({
   assume_clean = true,
-  debug = false,
+  debug_level = 0,
   data,
   max_read_length = Infinity,
   start_of_data_byte,
+  start_column = 0,
+  end_column, // index of last column (using zero-based index)
+  start_row = 0,
+  end_row, // index of last row (using zero-based index)
   meta
 }) => {
+  if (debug_level >= 1) console.time("[asci-grid] parse-ascii-grid-data took");
   const result = {};
@@ -26,23 +36,60 @@ const table = [];
   const read_length = Math.min(data.length, max_read_length);
-  if (debug) console.log("[ascii-grid/get-values] read_length:", read_length);
+  if (debug_level >= 1) console.log("[ascii-grid/parse-ascii-grid-data] read_length:", read_length);
 
-  let i =
-    start_of_data_byte ||
-    (meta && meta.last_metadata_byte + 1) ||
-    (await parseAsciiGridMetaData({ data })).last_metadata_byte + 1;
+  if (!meta) meta = parseAsciiGridMetaData({ data });
+  if (debug_level >= 1) console.log("[ascii-grid/parse-ascii-grid-data] meta:", meta);
 
-  if (debug) console.log("[ascii-grid/get-values] i:", i);
+  if (!end_row) end_row = meta.nrows - 1;
+  if (debug_level >= 1) console.log("[ascii-grid/parse-ascii-grid-data] end_row:", end_row);
 
-  while (i < read_length - 1) {
-    i++;
-    const byte = getByte(data, i);
+  if (!end_column) end_column = meta.ncols - 1;
+  if (debug_level >= 1) console.log("[ascii-grid/parse-ascii-grid-data] end_column:", end_column);
 
-    if (byte === NEWLINE_CHARCODE) {
-      if (num !== "") row.push(parseFloat(num));
+  let i = start_of_data_byte !== undefined ? start_of_data_byte : meta.last_metadata_byte + 1;
+  if (debug_level >= 1) console.log("[ascii-grid/parse-ascii-grid-data] i:", i);
+
+  // index of current row
+  let r = 0;
+
+  // index of current column
+  let c = 0;
+
+  // previous character
+  let prev;
+
+  while (i <= read_length) {
+    // add phantom null byte to end, because of the processing algo
+    const byte = i === read_length ? NULL_CHARCODE : getByte(data, i);
+
+    if (debug_level >= 2) console.log("[ascii-grid/parse-ascii-grid-data] i, byte:", [i, String.fromCharCode(byte)]);
+
+    if (byte === SPACE_CHARCODE || byte === NEWLINE_CHARCODE || byte === NULL_CHARCODE) {
+      if (prev === SPACE_CHARCODE || prev === NEWLINE_CHARCODE || prev === NULL_CHARCODE) {
+        // don't do anything because have reached weird edge case
+        // where file has two white space characters in a row
+        // for example, a new line + space before the start of the next row's data
+        prev = byte;
+        i++;
+        continue;
+      }
+
+      if (num !== "" && c >= start_column && c <= end_column) {
+        row.push(parseFloat(num));
+      } else {
+        if (debug_level >= 2) console.log("[ascii-grid/parse-ascii-grid-data] skipping value at [", r, "][", c, "]");
+      }
+
       num = "";
-      table.push(row);
-      row = [];
-    } else if (byte === SPACE_CHARCODE) {
-      if (num !== "") row.push(parseFloat(num));
-      num = "";
+
+      // reached end of the row
+      if (c == meta.ncols - 1) {
+        if (r >= start_row && r <= end_row) {
+          table.push(row);
+        }
+        r++;
+        c = 0;
+        row = [];
+      } else {
+        c++;
+      }
     } else if (
@@ -53,24 +100,20 @@ assume_clean ||
       byte === NINE_CHARCODE ||
+      byte === DOT_CHARCODE ||
       (byte > ZERO_CHARCODE && byte < NINE_CHARCODE)
     ) {
       num += String.fromCharCode(byte);
+    } else if (debug_level >= 2) {
+      console.error("[ascii-grid/parse-ascii-grid-data]: unknown byte", [byte]);
     }
-  }
 
-  // special case is the last byte because sometimes it's a null-byte (0)
-  const lastByte = getByte(data, read_length - 1);
-  if (debug) console.log("[ascii-grid/get-values] lastByte:", [lastByte]);
-  if (lastByte >= ZERO_CHARCODE && lastByte <= NINE_CHARCODE) {
-    num += String.fromCharCode(lastByte);
+    prev = byte;
+    i++;
   }
 
-  if (num !== "") row.push(num) && table.push(row);
-
-  result.end_of_data_byte = i;
-
   result.values = table;
 
-  if (debug) console.log("[ascii-grid/get-values] finishing");
+  if (debug_level >= 1) console.log("[ascii-grid/parse-ascii-grid-data] finishing");
 
+  if (debug_level >= 1) console.timeEnd("[asci-grid] parse-ascii-grid-data took");
   return result;
 };

src/parse-ascii-grid-meta.js

 const getByte = require("get-byte");
 
-module.exports = async ({ data, debug = false, max_read_length = 500 }) => {
+module.exports = ({ data, debug = false, max_read_length = 500 }) => {
   const result = {};
@@ -22,5 +22,3 @@
     } else if (line === null) {
-      if (
-        ["-", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"].includes(char)
-      ) {
+      if (["-", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"].includes(char)) {
         break;
@@ -27,0 +25,0 @@ } else {

test_data/setup.sh

-#!/bin/sh -e
-
-# originally from https://www.ngdc.noaa.gov/mgg/greatlakes/michigan.html

test.js

-const { readFileSync } = require("fs");
-const test = require("ava");
-const toab = require("toab");
-const fastMin = require("fast-min");
-const fastMax = require("fast-max");
-
-const isAsciiGrid = require("./src/is-ascii-grid");
-const parseAsciiGridMetaData = require("./src/parse-ascii-grid-meta");
-const parseAsciiGridData = require("./src/parse-ascii-grid-data");
-
-test("identifying ascii grid file extensions", async t => {
-  t.true(await isAsciiGrid({ data: "michigan_lld.asc" }));
-  t.true(await isAsciiGrid({ data: "michigan_lld.asc.tar" }));
-  t.true(await isAsciiGrid({ data: "michigan_lld.asc.tar.gz" }));
-  t.true(await isAsciiGrid({ data: "michigan_lld.asc.zip" }));
-  t.false(await isAsciiGrid({ data: "michigan_lld.asc.json" }));
-});
-
-test("identifying ascii grid file from buffers", async t => {
-  const buffer = readFileSync("./test_data/michigan_lld/michigan_lld.asc");
-  const bufferIsAsciiGrid = await isAsciiGrid({ data: buffer, debug: false });
-  t.true(bufferIsAsciiGrid);
-  t.true(await isAsciiGrid({ data: Uint8Array.from(buffer) }));
-  t.true(await isAsciiGrid({ data: await toab(buffer) }));
-});
-
-test("reading ascii metadata", async t => {
-  const buffer = readFileSync("./test_data/michigan_lld/michigan_lld.asc");
-  const meta = await parseAsciiGridMetaData({ data: buffer, debug: false });
-
-  // check metadata
-  t.is(meta.ncols, 4201);
-  t.is(meta.nrows, 5365);
-  t.is(meta.xllcenter, -88);
-  t.is(meta.yllcenter, 41.62);
-  t.is(meta.cellsize, 0.0008333333333);
-  t.is(meta.nodata_value, -9999);
-  t.is(meta.last_metadata_line, 5);
-  t.is(meta.last_metadata_byte, 95);
-});
-
-test("reading ascii values", async t => {
-  const buffer = readFileSync("./test_data/michigan_lld/michigan_lld.asc");
-  const result = await parseAsciiGridData({ data: buffer, debug: true });
-
-  const lastRow = result.values[result.values.length - 1];
-  t.is(fastMin(lastRow, { debug: false }), -0.102997);
-  t.is(fastMax(lastRow, { debug: false }), 165.940948);
-  t.is(result.values.length, 5365);
-  t.deepEqual(Array.from(new Set(result.values[0])).sort(), [-9999, 9999]);
-  t.true(result.values.every(ln => ln.length === 4201));
-  t.deepEqual(new Set(result.values[result.values.length - 1]).size, 3615);
-});

Fixed alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

No v1

Quality

Package is not semver >=1. This means it is not stable and does not support ^ ranges.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

16943

8.1%

Dependency count

1

-50%

Number of low quality alerts

0

-100%

Number of lines in readme file

93

126.83%

Number of low supply chain risk alerts

0

-100%

Number of medium supply chain risk alerts

0

-100%

Worsened metrics

Number of package files

7

-12.5%

Lines of code

170

-4.49%

Dependency changes

• - Removed

  to-typed-array@0.0.1

• - Removed

  @extra-number/significant-digits@2.0.27
  (transitive)

• - Removed

  choose-typed-array@0.0.1
  (transitive)

• - Removed

  to-typed-array@0.0.1
  (transitive)