@stencila/dockter - npm Package Compare versions

Comparing version 0.2.5 to 0.2.6

dist/Doer.js

@@ -8,12 +8,9 @@ "use strict";
 const fast_glob_1 = __importDefault(require("fast-glob"));
-// @ts-ignore
-const cached_request_1 = __importDefault(require("cached-request"));
+const got_1 = __importDefault(require("got"));
+const node_persist_1 = __importDefault(require("node-persist"));
 const path_1 = __importDefault(require("path"));
-const rimraf_1 = __importDefault(require("rimraf"));
-const request_1 = __importDefault(require("request"));
 const tmp_1 = __importDefault(require("tmp"));
 const errors_1 = require("./errors");
 const REQUEST_CACHE_DIR = '/tmp/dockter-request-cache';
-const requestCache = cached_request_1.default(request_1.default);
-requestCache.setCacheDirectory(REQUEST_CACHE_DIR);
+let REQUEST_CACHE_INITIALISED = false;
 /**
@@ -52,28 +49,48 @@ * A utility base class for the `Parser` and `Generator` classes
     }
-    fetch(url) {
-        return new Promise((resolve, reject) => {
-            requestCache({
-                url,
-                json: true,
+    async fetch(url) {
+        if (!REQUEST_CACHE_INITIALISED) {
+            await node_persist_1.default.init({
+                dir: REQUEST_CACHE_DIR,
                 ttl: 60 * 60 * 1000 // Milliseconds to cache responses for
-            }, (error, response, body) => {
-                if (error) {
-                    if (['ENOTFOUND', 'EAI_AGAIN', 'DEPTH_ZERO_SELF_SIGNED_CERT'].includes(error.code)) {
-                        // These are usually connection errors
-                        error = new errors_1.NetworkError(`There was a problem fetching ${url} (${error.code}). Are you connected to the internet?`);
-                    }
-                    else if (error instanceof SyntaxError && error.message.includes(' JSON ')) {
-                        // We can get here if a previous attempt had a network error and resulted in corrupt
-                        // JSON being written to the cache. So clear the cache...
-                        rimraf_1.default.sync(REQUEST_CACHE_DIR);
-                        // Ask the user to try again
-                        error = new errors_1.ApplicationError(`There was a problem fetching ${url}. Please try again.`);
-                    }
-                    return reject(error);
+            });
+            REQUEST_CACHE_INITIALISED = true;
+        }
+        let value;
+        try {
+            value = await node_persist_1.default.getItem(url);
+        }
+        catch (error) {
+            if (error.message.includes('does not look like a valid storage file')) {
+                // It seems that `persist.setItem` is not atomic and that the storage file can
+                // have zero bytes when we make multiple requests, some of which are for the same
+                // url. So we ignore this error and continue with the duplicate request.
+            }
+            else {
+                throw error;
+            }
+        }
+        if (!value) {
+            try {
+                const response = await got_1.default(url, {
+                    json: true
+                });
+                value = response.body;
+            }
+            catch (error) {
+                if (error.statusCode === 404) {
+                    value = null;
                 }
-                resolve(body);
-            });
-        });
+                else if (['ENOTFOUND', 'EAI_AGAIN', 'DEPTH_ZERO_SELF_SIGNED_CERT'].includes(error.code)) {
+                    // These are usually connection errors
+                    throw new errors_1.NetworkError(`There was a problem fetching ${url} (${error.code}). Are you connected to the internet?`);
+                }
+                else {
+                    throw error;
+                }
+            }
+            await node_persist_1.default.setItem(url, value);
+        }
+        return value;
     }
 }
 exports.default = Doer;

dist/RParser.js

@@ -135,9 +135,6 @@ "use strict";
         // Fetch meta-data from CRANDB
+        // If null (i.e. 404) then return package as is
         const crandb = await this.fetch(`http://crandb.r-pkg.org/${name}`);
-        if (crandb.error) {
-            if (crandb.error === 'not_found')
-                return pkg;
-            else
-                throw new Error(crandb.error);
-        }
+        if (crandb === null)
+            return pkg;
         // schema:Thing
@@ -144,0 +141,0 @@ pkg.description = crandb.Description;

package.json

 {
   "name": "@stencila/dockter",
-  "version": "0.2.5",
+  "version": "0.2.6",
   "description": "A Docker image builder for researchers",
@@ -50,6 +50,7 @@ "main": "dist/index.js",
     "@types/express": "^4.16.0",
+    "@types/got": "^8.3.4",
     "@types/jest": "^23.3.3",
     "@types/js-yaml": "^3.11.2",
+    "@types/node-persist": "0.0.33",
     "@types/request": "^2.47.1",
-    "@types/rimraf": "^2.0.2",
     "@types/tar-fs": "^1.16.1",
@@ -74,3 +75,2 @@ "@types/tar-stream": "^1.6.0",
   "dependencies": {
-    "cached-request": "^1.1.4",
     "docker-file-parser": "^1.0.4",
@@ -80,3 +80,5 @@ "dockerode": "^2.5.7",
     "fast-glob": "^2.2.3",
+    "got": "^9.2.2",
     "js-yaml": "^3.12.0",
+    "node-persist": "^3.0.1",
     "request": "^2.88.0",
@@ -83,0 +85,0 @@ "rimraf": "^2.6.2",

src/Doer.ts

 import fs from 'fs'
 import glob from 'fast-glob'
-// @ts-ignore
-import cachedRequest from 'cached-request'
+import got from 'got'
+import persist from 'node-persist'
 import path from 'path'
-import rimraf from 'rimraf'
-import request from 'request'
 import tmp from 'tmp'
 
-import { PermissionError, NetworkError, ApplicationError } from './errors'
+import { PermissionError, NetworkError } from './errors'
 
 const REQUEST_CACHE_DIR = '/tmp/dockter-request-cache'
-const requestCache = cachedRequest(request)
-requestCache.setCacheDirectory(REQUEST_CACHE_DIR)
+let REQUEST_CACHE_INITIALISED = false
 
@@ -58,26 +55,44 @@ /**
 
-  fetch (url: string): Promise<any> {
-    return new Promise((resolve, reject) => {
-      requestCache({
-        url,
-        json: true,
+  async fetch (url: string): Promise<any> {
+    if (!REQUEST_CACHE_INITIALISED) {
+      await persist.init({
+        dir: REQUEST_CACHE_DIR,
         ttl: 60 * 60 * 1000 // Milliseconds to cache responses for
-      }, (error: any, response: any, body: any) => {
-        if (error) {
-          if (['ENOTFOUND', 'EAI_AGAIN', 'DEPTH_ZERO_SELF_SIGNED_CERT'].includes(error.code)) {
-            // These are usually connection errors
-            error = new NetworkError(`There was a problem fetching ${url} (${error.code}). Are you connected to the internet?`)
-          } else if (error instanceof SyntaxError && error.message.includes(' JSON ')) {
-            // We can get here if a previous attempt had a network error and resulted in corrupt
-            // JSON being written to the cache. So clear the cache...
-            rimraf.sync(REQUEST_CACHE_DIR)
-            // Ask the user to try again
-            error = new ApplicationError(`There was a problem fetching ${url}. Please try again.`)
-          }
-          return reject(error)
+      })
+      REQUEST_CACHE_INITIALISED = true
+    }
+
+    let value
+    try {
+      value = await persist.getItem(url)
+    } catch (error) {
+      if (error.message.includes('does not look like a valid storage file')) {
+        // It seems that `persist.setItem` is not atomic and that the storage file can
+        // have zero bytes when we make multiple requests, some of which are for the same
+        // url. So we ignore this error and continue with the duplicate request.
+      } else {
+        throw error
+      }
+    }
+
+    if (!value) {
+      try {
+        const response = await got(url, {
+          json: true
+        })
+        value = response.body
+      } catch (error) {
+        if (error.statusCode === 404) {
+          value = null
+        } else if (['ENOTFOUND', 'EAI_AGAIN', 'DEPTH_ZERO_SELF_SIGNED_CERT'].includes(error.code)) {
+          // These are usually connection errors
+          throw new NetworkError(`There was a problem fetching ${url} (${error.code}). Are you connected to the internet?`)
+        } else {
+          throw error
         }
-        resolve(body)
-      })
-    })
+      }
+      await persist.setItem(url, value)
+    }
+    return value
   }
 }

src/RParser.ts

@@ -137,7 +137,5 @@ import path from 'path'
     // Fetch meta-data from CRANDB
+    // If null (i.e. 404) then return package as is
     const crandb = await this.fetch(`http://crandb.r-pkg.org/${name}`)
-    if (crandb.error) {
-      if (crandb.error === 'not_found') return pkg
-      else throw new Error(crandb.error)
-    }
+    if (crandb === null) return pkg
 
@@ -144,0 +142,0 @@ // schema:Thing

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

226012

increased by0.16%

Lines of code

3878

increased by0.65%

Worsened metrics

Dependency count

14

increased by7.69%

Dev dependency count

29

increased by3.57%

Dependency changes

• + Addedgot@^9.2.2

• + Addednode-persist@^3.0.1

• + Added@sindresorhus/is@0.14.0(transitive)

• + Added@szmarczak/http-timer@1.1.2(transitive)

• + Addedcacheable-request@6.1.0(transitive)

• + Addedclone-response@1.0.3(transitive)

• + Addeddecompress-response@3.3.0(transitive)

• + Addeddefer-to-connect@1.1.3(transitive)

• + Addedduplexer3@0.1.5(transitive)

• + Addedget-stream@5.2.0(transitive)

• + Addedgot@9.6.0(transitive)

• + Addedhttp-cache-semantics@4.1.1(transitive)

• + Addedjson-buffer@3.0.0(transitive)

• + Addedkeyv@3.1.0(transitive)

• + Addedlowercase-keys@1.0.12.0.0(transitive)

• + Addedmimic-response@1.0.1(transitive)

• + Addednode-persist@3.1.3(transitive)

• + Addednormalize-url@4.5.1(transitive)

• + Addedp-cancelable@1.1.0(transitive)

• + Addedprepend-http@2.0.0(transitive)

• + Addedresponselike@1.0.2(transitive)

• + Addedto-readable-stream@1.0.0(transitive)

• + Addedurl-parse-lax@3.0.0(transitive)

• - Removedcached-request@^1.1.4

• - Removedcached-request@1.3.0(transitive)

• - Removedgraceful-fs@4.2.11(transitive)