@stencila/dockter - npm Package Compare versions

Comparing version 0.12.1 to 0.12.2

.vscode/launch.json

@@ -13,3 +13,3 @@ {
         "windows": {
-          "program": "${workspaceFolder}/node_modules/jest/bin/jest",
+          "program": "${workspaceFolder}/node_modules/jest/bin/jest"
         }
@@ -26,3 +26,3 @@ },
         "windows": {
-          "program": "${workspaceFolder}/node_modules/jest/bin/jest",
+          "program": "${workspaceFolder}/node_modules/jest/bin/jest"
         }
@@ -29,0 +29,0 @@ }

dist/cli.js

@@ -15,3 +15,4 @@ #!/usr/bin/env node
 const errors_1 = require("./errors");
-const compiler = new DockerCompiler_1.default();
+const CachingUrlFetcher_1 = __importDefault(require("./CachingUrlFetcher"));
+const compiler = new DockerCompiler_1.default(new CachingUrlFetcher_1.default());
 yargonaut_1.default
@@ -18,0 +19,0 @@ .style('blue')

dist/DockerCompiler.d.ts

 import { SoftwareEnvironment } from '@stencila/schema';
+import IUrlFetcher from './IUrlFetcher';
 /**
@@ -7,2 +8,7 @@ * Compiles a project into a Dockerfile, or Docker image
     /**
+     * The instance of IUrlFetcher to fetch URLs
+     */
+    private readonly urlFetcher;
+    constructor(urlFetcher: IUrlFetcher);
+    /**
      * Compile a project
@@ -9,0 +15,0 @@ *

dist/DockerCompiler.js

@@ -18,2 +18,5 @@ "use strict";
 class DockerCompiler {
+    constructor(urlFetcher) {
+        this.urlFetcher = urlFetcher;
+    }
     /**
@@ -40,3 +43,3 @@ * Compile a project
             dockerfile = 'Dockerfile';
-            environ = await new DockerParser_1.default(folder).parse();
+            environ = await new DockerParser_1.default(this.urlFetcher, folder).parse();
         }
@@ -56,3 +59,3 @@ else {
                 for (let ParserClass of parsers_1.default) {
-                    const parser = new ParserClass(folder);
+                    const parser = new ParserClass(this.urlFetcher, folder);
                     const pkg = await parser.parse();
@@ -68,3 +71,3 @@ if (pkg)
             dockerfile = '.Dockerfile';
-            new DockerGenerator_1.default(environ, folder).generate(comments, stencila);
+            new DockerGenerator_1.default(this.urlFetcher, environ, folder).generate(comments, stencila);
         }
@@ -71,0 +74,0 @@ if (build) {

dist/DockerGenerator.d.ts

 import { SoftwareEnvironment } from '@stencila/schema';
 import Generator from './Generator';
+import IUrlFetcher from './IUrlFetcher';
 /**
@@ -19,3 +20,3 @@ * A Dockerfile generator that collects instructions from
     protected generators: Array<Generator>;
-    constructor(environ: SoftwareEnvironment, folder?: string);
+    constructor(urlFetcher: IUrlFetcher, environ: SoftwareEnvironment, folder?: string);
     /**
@@ -22,0 +23,0 @@ * Collect arrays of string from each child generator

dist/DockerGenerator.js

@@ -45,4 +45,4 @@ "use strict";
 class DockerGenerator extends Generator_1.default {
-    constructor(environ, folder) {
-        super(folder);
+    constructor(urlFetcher, environ, folder) {
+        super(urlFetcher, folder);
         this.environ = environ;
@@ -56,3 +56,3 @@ // Each of the environment's `softwareRequirements` is
                 // @ts-ignore
-                const generator = new GeneratorClass(pkg, folder);
+                const generator = new GeneratorClass(urlFetcher, pkg, folder);
                 if (generator.applies()) {
@@ -59,0 +59,0 @@ this.generators.push(generator);

dist/Doer.d.ts

@@ -1,2 +0,2 @@
-export declare const REQUEST_CACHE_DIR = "/tmp/dockter-request-cache";
+import IUrlFetcher from './IUrlFetcher';
 /**
@@ -11,4 +11,8 @@ * A utility base class for the `Parser` and `Generator` classes
     folder: string;
-    constructor(folder: string | undefined);
     /**
+     * The instance of IUrlFetcher to fetch URLs
+     */
+    protected readonly urlFetcher: IUrlFetcher;
+    constructor(urlFetcher: IUrlFetcher, folder: string | undefined);
+    /**
      * Does a path exist within the project folder?
@@ -15,0 +19,0 @@ *

dist/Doer.js

@@ -8,9 +8,5 @@ "use strict";
 const fast_glob_1 = __importDefault(require("fast-glob"));
-const got_1 = __importDefault(require("got"));
-const node_persist_1 = __importDefault(require("node-persist"));
 const path_1 = __importDefault(require("path"));
 const tmp_1 = __importDefault(require("tmp"));
 const errors_1 = require("./errors");
-exports.REQUEST_CACHE_DIR = '/tmp/dockter-request-cache';
-let REQUEST_CACHE_INITIALISED = false;
 /**
@@ -21,6 +17,7 @@ * A utility base class for the `Parser` and `Generator` classes
 class Doer {
-    constructor(folder) {
+    constructor(urlFetcher, folder) {
         if (!folder)
             folder = tmp_1.default.dirSync().name;
         this.folder = folder;
+        this.urlFetcher = urlFetcher;
     }
@@ -78,45 +75,5 @@ /**
     async fetch(url, options = { json: true }) {
-        if (!REQUEST_CACHE_INITIALISED) {
-            await node_persist_1.default.init({
-                dir: exports.REQUEST_CACHE_DIR,
-                ttl: 60 * 60 * 1000 // Milliseconds to cache responses for
-            });
-            REQUEST_CACHE_INITIALISED = true;
-        }
-        let value;
-        try {
-            value = false; // await persist.getItem(url)
-        }
-        catch (error) {
-            if (error.message.includes('does not look like a valid storage file')) {
-                // It seems that `persist.setItem` is not atomic and that the storage file can
-                // have zero bytes when we make multiple requests, some of which are for the same
-                // url. So we ignore this error and continue with the duplicate request.
-            }
-            else {
-                throw error;
-            }
-        }
-        if (!value) {
-            try {
-                const response = await got_1.default(url, options);
-                value = response.body;
-            }
-            catch (error) {
-                if (error.statusCode === 404) {
-                    value = null;
-                }
-                else if (['ENOTFOUND', 'EAI_AGAIN', 'DEPTH_ZERO_SELF_SIGNED_CERT'].includes(error.code)) {
-                    // These are usually connection errors
-                    throw new errors_1.NetworkError(`There was a problem fetching ${url} (${error.code}). Are you connected to the internet?`);
-                }
-                else {
-                    throw error;
-                }
-            }
-            await node_persist_1.default.setItem(url, value);
-        }
-        return value;
+        return this.urlFetcher.fetchUrl(url, options);
     }
 }
 exports.default = Doer;

dist/Generator.js

@@ -88,3 +88,3 @@ "use strict";
 # It's good practice to run Docker images as a non-root user.
-# This section creates a new user and it's home directory as the default working directory.`;
+# This section creates a new user and its home directory as the default working directory.`;
         }
@@ -91,0 +91,0 @@ dockerfile += `

dist/JavascriptGenerator.d.ts

 import { SoftwarePackage } from '@stencila/schema';
 import PackageGenerator from './PackageGenerator';
+import IUrlFetcher from './IUrlFetcher';
 /**
@@ -13,3 +14,3 @@ * A Dockerfile generator for Javascript projects
     nodeMajorVersion: number;
-    constructor(pkg: SoftwarePackage, folder?: string, nodeMajorVersion?: number);
+    constructor(urlFetcher: IUrlFetcher, pkg: SoftwarePackage, folder?: string, nodeMajorVersion?: number);
     applies(): boolean;
@@ -16,0 +17,0 @@ aptKeysCommand(sysVersion: string): string;

dist/JavascriptGenerator.js

@@ -14,4 +14,4 @@ "use strict";
     // Methods that override those in `Generator`
-    constructor(pkg, folder, nodeMajorVersion = 10) {
-        super(pkg, folder);
+    constructor(urlFetcher, pkg, folder, nodeMajorVersion = 10) {
+        super(urlFetcher, pkg, folder);
         this.nodeMajorVersion = nodeMajorVersion;
@@ -18,0 +18,0 @@ }

dist/JavascriptParser.d.ts

@@ -0,3 +1,3 @@
+import Parser from './Parser';
 import { SoftwarePackage } from '@stencila/schema';
-import Parser from './Parser';
 /**
@@ -4,0 +4,0 @@ * Dockter `Parser` class for Node.js.

dist/JavascriptParser.js

@@ -6,3 +6,2 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const schema_1 = require("@stencila/schema");
 // @ts-ignore
@@ -15,2 +14,3 @@ const builtin_modules_1 = __importDefault(require("builtin-modules"));
 const Parser_1 = __importDefault(require("./Parser"));
+const schema_1 = require("@stencila/schema");
 /**
@@ -108,3 +108,3 @@ * Dockter `Parser` class for Node.js.
                 let version = 'latest';
-                if (versionRange !== 'latest' || versionRange !== '*') {
+                if (versionRange !== 'latest' && versionRange !== '*') {
                     const range = semver_1.default.validRange(versionRange);
@@ -111,0 +111,0 @@ if (range) {

dist/PackageGenerator.d.ts

 import { SoftwarePackage } from '@stencila/schema';
 import Generator from './Generator';
+import IUrlFetcher from './IUrlFetcher';
 /**
@@ -11,3 +12,3 @@ * Generates a Dockerfile for a `SoftwarePackage` instance
     package: SoftwarePackage;
-    constructor(pkg: SoftwarePackage, folder?: string);
+    constructor(urlFetcher: IUrlFetcher, pkg: SoftwarePackage, folder?: string);
     /**
@@ -14,0 +15,0 @@ * Get a list of packages in `this.package.softwareRequirements`

dist/PackageGenerator.js

@@ -11,4 +11,4 @@ "use strict";
 class PackageGenerator extends Generator_1.default {
-    constructor(pkg, folder) {
-        super(folder);
+    constructor(urlFetcher, pkg, folder) {
+        super(urlFetcher, folder);
         this.package = pkg;
@@ -15,0 +15,0 @@ }

dist/Parser.d.ts

@@ -6,6 +6,6 @@ import Doer from './Doer';
  *
- * A language `Parser` generates a JSON-LD `SoftwareApplication` instance based on the
+ * A language `Parser` generates a JSON-LD `SoftwarePackage` instance based on the
  * contents of a directory. It is responsible for determining which packages the application
  * needs, resolving the dependencies of those packages (both system and language packages) and
- * turning those into a JSON-LD `SoftwareApplication` instance.
+ * turning those into a JSON-LD `SoftwarePackage` instance.
  *
@@ -12,0 +12,0 @@ * If the `Parser` finds a corresponding requirements file for the language (e.g. `requirements.txt` for Python),

dist/Parser.js

@@ -10,6 +10,6 @@ "use strict";
  *
- * A language `Parser` generates a JSON-LD `SoftwareApplication` instance based on the
+ * A language `Parser` generates a JSON-LD `SoftwarePackage` instance based on the
  * contents of a directory. It is responsible for determining which packages the application
  * needs, resolving the dependencies of those packages (both system and language packages) and
- * turning those into a JSON-LD `SoftwareApplication` instance.
+ * turning those into a JSON-LD `SoftwarePackage` instance.
  *
@@ -16,0 +16,0 @@ * If the `Parser` finds a corresponding requirements file for the language (e.g. `requirements.txt` for Python),

dist/PythonGenerator.d.ts

 import { SoftwarePackage } from '@stencila/schema';
 import PackageGenerator from './PackageGenerator';
+import IUrlFetcher from './IUrlFetcher';
 /**
@@ -7,5 +8,11 @@ * A Dockerfile generator for Python packages
 export default class PythonGenerator extends PackageGenerator {
+    /**
+     * The Python Major version, i.e. 2 or 3
+     */
     private readonly pythonMajorVersion;
+    /**
+     * An instance of `PythonSystemPackageLookup` with which to look up system dependencies of Python packages
+     */
     private readonly systemPackageLookup;
-    constructor(pkg: SoftwarePackage, folder?: string, pythonMajorVersion?: number);
+    constructor(urlFetcher: IUrlFetcher, pkg: SoftwarePackage, folder?: string, pythonMajorVersion?: number);
     /**
@@ -16,7 +23,26 @@ * Return the `pythonMajorVersion` (as string) if it is not 2, otherwise return an empty string (if it is 2). This is
     pythonVersionSuffix(): string;
+    /**
+     * Check if this Generator's package applies (if it is Python).
+     */
     applies(): boolean;
+    /**
+     * Generate a list of system (apt) packages by looking up with `this.systemPackageLookup`.
+     */
     aptPackages(sysVersion: string): Array<string>;
+    /**
+     * Build the contents of a `requirements.txt` file by joining the Python package name to its version specifier.
+     */
     generateRequirementsContent(): string;
+    /**
+     * Get the pip command to install the Stencila package
+     */
     stencilaInstall(sysVersion: string): string | undefined;
+    /**
+     * Write out the generated requirements content to `GENERATED_REQUIREMENTS_FILE` or none exists, just instruct the
+     * copy of a `requirements.txt` file as part of the Dockerfile. If that does not exist, then no COPY should be done.
+     */
     installFiles(sysVersion: string): Array<[string, string]>;
+    /**
+     * Generate the right pip command to install the requirements, appends the correct Python major version to `pip`.
+     */
     installCommand(sysVersion: string): string | undefined;
@@ -23,0 +49,0 @@ /**

dist/PythonGenerator.js

 "use strict";
-/* tslint:disable: completed-docs */
 var __importDefault = (this && this.__importDefault) || function (mod) {
@@ -16,4 +15,4 @@ return (mod && mod.__esModule) ? mod : { "default": mod };
     // Methods that override those in `Generator`
-    constructor(pkg, folder, pythonMajorVersion = 3) {
-        super(pkg, folder);
+    constructor(urlFetcher, pkg, folder, pythonMajorVersion = 3) {
+        super(urlFetcher, pkg, folder);
         this.pythonMajorVersion = pythonMajorVersion;
@@ -29,5 +28,11 @@ this.systemPackageLookup = PythonSystemPackageLookup_1.default.fromFile(path_1.default.join(__dirname, 'PythonSystemDependencies.json'));
     }
+    /**
+     * Check if this Generator's package applies (if it is Python).
+     */
     applies() {
         return this.package.runtimePlatform === 'Python';
     }
+    /**
+     * Generate a list of system (apt) packages by looking up with `this.systemPackageLookup`.
+     */
     aptPackages(sysVersion) {
@@ -46,2 +51,5 @@ let aptRequirements = [];
     }
+    /**
+     * Build the contents of a `requirements.txt` file by joining the Python package name to its version specifier.
+     */
     generateRequirementsContent() {
@@ -53,5 +61,12 @@ if (!this.package.softwareRequirements) {
     }
+    /**
+     * Get the pip command to install the Stencila package
+     */
     stencilaInstall(sysVersion) {
         return `pip${this.pythonVersionSuffix()} install --no-cache-dir https://github.com/stencila/py/archive/91a05a139ac120a89fc001d9d267989f062ad374.zip`;
     }
+    /**
+     * Write out the generated requirements content to `GENERATED_REQUIREMENTS_FILE` or none exists, just instruct the
+     * copy of a `requirements.txt` file as part of the Dockerfile. If that does not exist, then no COPY should be done.
+     */
     installFiles(sysVersion) {
@@ -68,2 +83,5 @@ let requirementsContent = this.generateRequirementsContent();
     }
+    /**
+     * Generate the right pip command to install the requirements, appends the correct Python major version to `pip`.
+     */
     installCommand(sysVersion) {
@@ -70,0 +88,0 @@ return `pip${this.pythonVersionSuffix()} install --user --requirement requirements.txt`;

dist/PythonParser.d.ts

@@ -12,10 +12,29 @@ import { SoftwarePackage } from '@stencila/schema';
 }
+/**
+ * Parser to be used on a directory with Python source code and (optionally) a `requirements.txt` file.
+ * If no `requirements.txt` file exists then the Parser will attempt to read requirements from the Python source code.
+ */
 export default class PythonParser extends Parser {
     parse(): Promise<SoftwarePackage | null>;
-    private createApplication;
+    /**
+     * Convert a `PythonRequirement` into a `SoftwarePackage` by augmenting with metadata from PyPI
+     */
+    private createPackage;
+    /**
+     * Parse a `requirements.txt` file at `path` and return a list of `PythonRequirement`s
+     */
     parseRequirementsFile(path: string): Promise<Array<PythonRequirement>>;
+    /**
+     * Parse Python source files are find any non-system imports, return this as an array of `PythonRequirement`s.
+     */
     generateRequirementsFromSource(): Array<PythonRequirement>;
+    /**
+     * Parse Python source files are find all imports (including system imports).
+     */
     findImports(): Array<string>;
+    /**
+     * Parse Python a single Python source file for imports.
+     */
     readImportsInFile(path: string): Array<string>;
 }
 export {};

dist/PythonParser.js

 "use strict";
-/* tslint:disable: completed-docs */
 var __importDefault = (this && this.__importDefault) || function (mod) {
@@ -16,5 +15,11 @@ return (mod && mod.__esModule) ? mod : { "default": mod };
 const REQUIREMENTS_STANDARD_REGEX = /^\s*([^\s]+)/;
+/**
+ * Return true if the passed in line is a requirements.txt comment (starts with "#" which might be preceded by spaces).
+ */
 function lineIsComment(line) {
     return REQUIREMENTS_COMMENT_REGEX.exec(line) !== null;
 }
+/**
+ * Execute the given `regex` against the line and return the first match. If there is no match, return `null`.
+ */
 function applyRegex(line, regex) {
@@ -27,14 +32,30 @@ const result = regex.exec(line);
 }
+/**
+ * Execute the `REQUIREMENTS_EDITABLE_SOURCE_REGEX` against a line and return the first result (or null if no match).
+ * This is used to find a requirements.txt line of a URL source (e.g. including a package from github).
+ */
 function extractEditableSource(line) {
     return applyRegex(line, REQUIREMENTS_EDITABLE_SOURCE_REGEX);
 }
+/**
+ * Execute the `REQUIREMENTS_INCLUDE_PATH_REGEX` against a line and return the first result (or null if no match).
+ * This is used to find a requirements.txt line that includes another requirements file.
+ */
 function extractIncludedRequirementsPath(line) {
     return applyRegex(line, REQUIREMENTS_INCLUDE_PATH_REGEX);
 }
+/**
+ * Execute the `REQUIREMENTS_STANDARD_REGEX` against a line and return the first result (or null if no match).
+ * This is used to find "standard" requirements.txt lines.
+ */
 function extractStandardRequirements(line) {
     return applyRegex(line, REQUIREMENTS_STANDARD_REGEX);
 }
+/**
+ * Split a requirement line into name and then version. For example "package==1.0.1" => ["package", "==1.0.1"]
+ * The version specifier can be `==`, `<=`, `>=`, `~=`, `<` or `>`.
+ */
 function splitStandardRequirementVersion(requirement) {
     let firstSplitterIndex = -1;
-    for (let splitter of ['<=', '>=', '==', '~=', '<', '>']) {
+    for (let splitter of ['==', '<=', '>=', '~=', '<', '>']) {
         let splitterIndex = requirement.indexOf(splitter);
@@ -50,2 +71,6 @@ if (splitterIndex > -1 && (firstSplitterIndex === -1 || splitterIndex < firstSplitterIndex)) {
 }
+/**
+ * Convert a list of classifiers to a Map between main classification and sub classification(s).
+ * e.g: ['A :: B', 'A :: C', 'D :: E'] => {'A': ['B', 'C'], 'D': ['E']}
+ */
 function buildClassifierMap(classifiers) {
@@ -117,2 +142,6 @@ const classifierMap = new Map();
 })(RequirementType = exports.RequirementType || (exports.RequirementType = {}));
+/**
+ * Parser to be used on a directory with Python source code and (optionally) a `requirements.txt` file.
+ * If no `requirements.txt` file exists then the Parser will attempt to read requirements from the Python source code.
+ */
 class PythonParser extends Parser_1.default {
@@ -139,3 +168,3 @@ async parse() {
             if (rawRequirement.type === RequirementType.Named) {
-                pkg.softwareRequirements.push(await this.createApplication(rawRequirement));
+                pkg.softwareRequirements.push(await this.createPackage(rawRequirement));
             }
@@ -150,17 +179,20 @@ else if (rawRequirement.type === RequirementType.URL) {
     }
-    async createApplication(requirement) {
-        const softwareApplication = new schema_1.SoftwareApplication();
-        softwareApplication.name = requirement.value;
-        softwareApplication.runtimePlatform = 'Python';
-        softwareApplication.programmingLanguages = [schema_1.ComputerLanguage.py];
+    /**
+     * Convert a `PythonRequirement` into a `SoftwarePackage` by augmenting with metadata from PyPI
+     */
+    async createPackage(requirement) {
+        const softwarePackage = new schema_1.SoftwarePackage();
+        softwarePackage.name = requirement.value;
+        softwarePackage.runtimePlatform = 'Python';
+        softwarePackage.programmingLanguages = [schema_1.ComputerLanguage.py];
         if (requirement.version) {
-            softwareApplication.version = requirement.version;
+            softwarePackage.version = requirement.version;
         }
-        const pyPiMetadata = await this.fetch(`https://pypi.org/pypi/${softwareApplication.name}/json`);
+        const pyPiMetadata = await this.fetch(`https://pypi.org/pypi/${softwarePackage.name}/json`);
         if (pyPiMetadata.info) {
             if (pyPiMetadata.info.author) {
-                softwareApplication.authors.push(schema_1.Person.fromText(`${pyPiMetadata.info.author} <${pyPiMetadata.info.author_email}>`));
+                softwarePackage.authors.push(schema_1.Person.fromText(`${pyPiMetadata.info.author} <${pyPiMetadata.info.author_email}>`));
             }
             if (pyPiMetadata.info.project_url) {
-                softwareApplication.codeRepository = pyPiMetadata.info.project_url;
+                softwarePackage.codeRepository = pyPiMetadata.info.project_url;
             }
@@ -172,5 +204,5 @@ if (pyPiMetadata.info.classifiers) {
                     if (topics.length)
-                        softwareApplication.applicationCategories = topics;
+                        softwarePackage.applicationCategories = topics;
                     if (subTopics.length)
-                        softwareApplication.applicationSubCategories = subTopics;
+                        softwarePackage.applicationSubCategories = subTopics;
                 }
@@ -185,18 +217,21 @@ if (classifiers.has('Operating System')) {
                     }
-                    softwareApplication.operatingSystems = operatingSystems;
+                    softwarePackage.operatingSystems = operatingSystems;
                 }
             }
             if (pyPiMetadata.info.keywords)
-                softwareApplication.keywords = pyPiMetadata.info.keywords;
+                softwarePackage.keywords = pyPiMetadata.info.keywords;
             if (pyPiMetadata.info.license)
-                softwareApplication.license = pyPiMetadata.info.license;
+                softwarePackage.license = pyPiMetadata.info.license;
             if (pyPiMetadata.info.long_description) {
-                softwareApplication.description = pyPiMetadata.info.long_description;
+                softwarePackage.description = pyPiMetadata.info.long_description;
             }
             else if (pyPiMetadata.info.description) {
-                softwareApplication.description = pyPiMetadata.info.description;
+                softwarePackage.description = pyPiMetadata.info.description;
             }
         }
-        return softwareApplication;
+        return softwarePackage;
     }
+    /**
+     * Parse a `requirements.txt` file at `path` and return a list of `PythonRequirement`s
+     */
     async parseRequirementsFile(path) {
@@ -229,2 +264,5 @@ const requirementsContent = this.read(path);
     }
+    /**
+     * Parse Python source files are find any non-system imports, return this as an array of `PythonRequirement`s.
+     */
     generateRequirementsFromSource() {
@@ -238,2 +276,5 @@ const nonSystemImports = this.findImports().filter(pythonImport => !PythonBuiltins_1.default.includes(pythonImport));
     }
+    /**
+     * Parse Python source files are find all imports (including system imports).
+     */
     findImports() {
@@ -252,5 +293,8 @@ const files = this.glob(['**/*.py']);
     }
+    /**
+     * Parse Python a single Python source file for imports.
+     */
     readImportsInFile(path) {
         const fileContent = this.read(path);
-        const importRegex = /^from ([\w_]+)|^import ([\w_]+)/gm;
+        const importRegex = /^\s*from ([\w_]+)|^\s*import ([\w_]+)/gm;
         const imports = [];
@@ -257,0 +301,0 @@ const fileDirectory = path_1.dirname(path);

dist/PythonSystemPackageLookup.d.ts

 declare type PythonSystemPackageLookupMap = Map<string, Map<string, Map<string, Map<string, Array<string> | null>>>>;
+/**
+ * An object that looks up if any system packages are required for a Python package.
+ * The lookup is in the format {packageName: pythonVersion: systemPackageType: systemVersion: [sysPackage, sysPackage...]}
+ */
 export default class PythonSystemPackageLookup {
     private readonly packageLookup;
+    /**
+     * @param packageLookup: PythonSystemPackageLookupMap the Map
+     */
     constructor(packageLookup: PythonSystemPackageLookupMap);
+    /**
+     * Construct a `PythonSystemPackageLookup` by parsing a JSON representation of the package map from `path`
+     */
     static fromFile(path: string): PythonSystemPackageLookup;
+    /**
+     * Look up the system package required for a python package given python version, package type and system version.
+     * Will always return an Array, which will be empty if there are no packages to install.
+     */
     lookupSystemPackage(pythonPackage: string, pythonMajorVersion: number, systemPackageType: string, systemVersion: string): Array<string>;
 }
 export {};

dist/PythonSystemPackageLookup.js

 "use strict";
-/* tslint:disable: completed-docs */
 var __importDefault = (this && this.__importDefault) || function (mod) {
@@ -11,3 +10,2 @@ return (mod && mod.__esModule) ? mod : { "default": mod };
  * value
- * @param value
  */
@@ -27,6 +25,16 @@ function valueToMap(value) {
 }
+/**
+ * An object that looks up if any system packages are required for a Python package.
+ * The lookup is in the format {packageName: pythonVersion: systemPackageType: systemVersion: [sysPackage, sysPackage...]}
+ */
 class PythonSystemPackageLookup {
+    /**
+     * @param packageLookup: PythonSystemPackageLookupMap the Map
+     */
     constructor(packageLookup) {
         this.packageLookup = packageLookup;
     }
+    /**
+     * Construct a `PythonSystemPackageLookup` by parsing a JSON representation of the package map from `path`
+     */
     static fromFile(path) {
@@ -36,2 +44,6 @@ const dependencyLookupRaw = JSON.parse(fs_1.default.readFileSync(path, 'utf8'));
     }
+    /**
+     * Look up the system package required for a python package given python version, package type and system version.
+     * Will always return an Array, which will be empty if there are no packages to install.
+     */
     lookupSystemPackage(pythonPackage, pythonMajorVersion, systemPackageType, systemVersion) {
@@ -38,0 +50,0 @@ const pyPackageMap = this.packageLookup.get(pythonPackage);

dist/RGenerator.d.ts

 import { SoftwarePackage } from '@stencila/schema';
 import PackageGenerator from './PackageGenerator';
+import IUrlFetcher from './IUrlFetcher';
 /**
@@ -11,3 +12,3 @@ * A Dockerfile generator for R packages
     date: string;
-    constructor(pkg: SoftwarePackage, folder?: string);
+    constructor(urlFetcher: IUrlFetcher, pkg: SoftwarePackage, folder?: string);
     applies(): boolean;
@@ -14,0 +15,0 @@ envVars(sysVersion: string): Array<[string, string]>;

dist/RGenerator.js

@@ -11,4 +11,4 @@ "use strict";
 class RGenerator extends PackageGenerator_1.default {
-    constructor(pkg, folder) {
-        super(pkg, folder);
+    constructor(urlFetcher, pkg, folder) {
+        super(urlFetcher, pkg, folder);
         // Default to yesterday's date (to ensure MRAN is available for the date)
@@ -15,0 +15,0 @@ // Set here as it is required in two methods below

dist/router.js

@@ -9,3 +9,4 @@ "use strict";
 const DockerCompiler_1 = __importDefault(require("./DockerCompiler"));
-const compiler = new DockerCompiler_1.default();
+const CachingUrlFetcher_1 = __importDefault(require("./CachingUrlFetcher"));
+const compiler = new DockerCompiler_1.default(new CachingUrlFetcher_1.default());
 router.use(express_1.json());
@@ -12,0 +13,0 @@ /**

jest.config.js

@@ -8,4 +8,5 @@ module.exports = {
   coveragePathIgnorePatterns: [
-    "tests/fixture.ts"
+    "tests/MockUrlFetcher.ts",
+    "tests/test-functions.ts"
   ]
 };

package.json

 {
   "name": "@stencila/dockter",
-  "version": "0.12.1",
+  "version": "0.12.2",
   "description": "A Docker image builder for researchers",
@@ -77,3 +77,3 @@ "main": "dist/index.js",
   "dependencies": {
-    "@stencila/schema": "^0.2.2",
+    "@stencila/schema": "^0.2.4",
     "builtin-modules": "^3.0.0",
@@ -80,0 +80,0 @@ "detective": "^5.1.0",

README.md

@@ -27,4 +27,4 @@ # Dockter : a Docker image builder for researchers
     + [Jupyter](#jupyter)
-  * [Quicker re-installation of language packages](#quicker-re-installation-of-language-packages)
-    + [An example](#an-example)
+  * [Automatically determines system requirements](#automatically-determines-system-requirements)
+  * [Faster re-installation of language packages](#faster-re-installation-of-language-packages)
   * [Generates structured meta-data for your project](#generates-structured-meta-data-for-your-project)
@@ -73,4 +73,2 @@ * [Easy to pick up, easy to throw away](#easy-to-pick-up-easy-to-throw-away)
 
-Dockter checks if any of your dependencies (or dependencies of dependencies, or dependencies of...) requires system packages (e.g. `libxml-dev`) and installs those too. No more trial and error of build, fail, add dependency, repeat... cycles!
-
 #### Python
@@ -97,4 +95,34 @@
 
-### Quicker re-installation of language packages
+### Automatically determines system requirements
 
+One of the headaches researchers face when hand writing Dockerfiles is figuring out which system dependencies your project needs. Often this involves a lot of trial and error. 
+
+Dockter automatically checks if any of your dependencies (or dependencies of dependencies, or dependencies of...) requires system packages and installs those into the image. For example, let's say you have a project with an R script that requires the `rgdal` package for geospatial analyses,
+
+```R
+library(rgdal)
+```
+
+When you run `dockter compile` in this project, Dockter will generate a Dockerfile with the following section which installs R, plus the three system dependencies required `gdal-bin`, `libgdal-dev`, and `libproj-dev`:
+
+```Dockerfile
+# This section installs system packages required for your project
+# If you need extra system packages add them here.
+RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+      gdal-bin \
+      libgdal-dev \
+      libproj-dev \
+      r-base \
+ && apt-get autoremove -y \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+```
+
+For R, Dockter does this by querying the https://sysreqs.r-hub.io/ database. For Python, Dockter includes a mapping of system requirements of packages that users can [contribute to](CONTRIBUTING.md#python-system-dependencies).
+
+No more trial and error of build, fail, add dependency, repeat... cycles!
+
+### Faster re-installation of language packages
+
 If you have built a Docker image before, you'll know that it can be frustrating waiting for *all* your project's dependencies to reinstall when you simply add or remove one of them.
@@ -108,4 +136,2 @@
 
-#### An example
-
 Here's a simple motivating [example](fixtures/tests/py-pandas). It's a Python project with a `requirements.txt` file which specifies that the project depends upon `pandas` which, to ensure reproducibility, is pinned to version `0.23.0`,
@@ -350,3 +376,3 @@
 
-Dockter compiles a meta-data tree of all the packages that your project relies on. Use the `who` 🦄 [#54](https://github.com/stencila/dockter/issues/54) command to get a list of the authors of those packages:
+Dockter compiles a meta-data tree of all the packages that your project relies on. Use the `who` 🦄 [#55](https://github.com/stencila/dockter/issues/55) command to get a list of the authors of those packages:
 
@@ -439,3 +465,3 @@ ```bash
 
-Dockter was inspired by similar tools for researchers including [`binder`](https://github.com/binder-project/binder), [`repo2docker`](https://github.com/jupyter/repo2docker) and [`containerit`](https://github.com/o2r-project/containerit). It relies on many great open source projects, in particular:
+Dockter was inspired by, and combines ideas from, several similar tools including [`binder`](https://github.com/binder-project/binder), [`repo2docker`](https://github.com/jupyter/repo2docker), [`source-to-image`](https://github.com/openshift/source-to-image) and [`containerit`](https://github.com/o2r-project/containerit). It relies on many great open source projects, in particular:
 
@@ -442,0 +468,0 @@ - [CodeMeta](https://codemeta.github.io/)

src/cli.ts

@@ -13,4 +13,6 @@ #!/usr/bin/env node
 import { ApplicationError } from './errors'
-const compiler = new DockerCompiler()
+import CachingUrlFetcher from './CachingUrlFetcher'
 
+const compiler = new DockerCompiler(new CachingUrlFetcher())
+
 yargonaut
@@ -17,0 +19,0 @@ .style('blue')

src/DockerCompiler.ts

@@ -11,2 +11,3 @@ import fs from 'fs'
 import DockerExecutor from './DockerExecutor'
+import IUrlFetcher from './IUrlFetcher'
 
@@ -19,2 +20,11 @@ /**
   /**
+   * The instance of IUrlFetcher to fetch URLs
+   */
+  private readonly urlFetcher: IUrlFetcher
+
+  constructor (urlFetcher: IUrlFetcher) {
+    this.urlFetcher = urlFetcher
+  }
+
+  /**
    * Compile a project
@@ -41,3 +51,3 @@ *
       dockerfile = 'Dockerfile'
-      environ = await new DockerParser(folder).parse()
+      environ = await new DockerParser(this.urlFetcher, folder).parse()
     } else {
@@ -55,3 +65,3 @@ if (fs.existsSync(path.join(folder, 'environ.jsonld'))) {
         for (let ParserClass of parsers) {
-          const parser = new ParserClass(folder)
+          const parser = new ParserClass(this.urlFetcher, folder)
           const pkg = await parser.parse()
@@ -68,3 +78,3 @@ if (pkg) environ.softwareRequirements.push(pkg)
       dockerfile = '.Dockerfile'
-      new DockerGenerator(environ, folder).generate(comments, stencila)
+      new DockerGenerator(this.urlFetcher, environ, folder).generate(comments, stencila)
     }
@@ -71,0 +81,0 @@

src/DockerGenerator.ts

@@ -5,2 +5,3 @@ import { SoftwareEnvironment } from '@stencila/schema'
 import generators from './generators'
+import IUrlFetcher from './IUrlFetcher'
 
@@ -64,4 +65,4 @@ const PREFERRED_UBUNTU_VERSION = '18.04'
 
-  constructor (environ: SoftwareEnvironment, folder?: string) {
-    super(folder)
+  constructor (urlFetcher: IUrlFetcher, environ: SoftwareEnvironment, folder?: string) {
+    super(urlFetcher, folder)
     this.environ = environ
@@ -76,3 +77,3 @@
         // @ts-ignore
-        const generator = new GeneratorClass(pkg, folder)
+        const generator = new GeneratorClass(urlFetcher, pkg, folder)
         if (generator.applies()) {
@@ -79,0 +80,0 @@ this.generators.push(generator)

src/Doer.ts

 import fs from 'fs'
 import glob from 'fast-glob'
-import got from 'got'
-import persist from 'node-persist'
 import path from 'path'
 import tmp from 'tmp'
 
-import { PermissionError, NetworkError } from './errors'
+import { PermissionError } from './errors'
+import IUrlFetcher from './IUrlFetcher'
 
-export const REQUEST_CACHE_DIR = '/tmp/dockter-request-cache'
-let REQUEST_CACHE_INITIALISED = false
-
 /**
@@ -24,5 +20,11 @@ * A utility base class for the `Parser` and `Generator` classes
 
-  constructor (folder: string | undefined) {
+  /**
+   * The instance of IUrlFetcher to fetch URLs
+   */
+  protected readonly urlFetcher: IUrlFetcher
+
+  constructor (urlFetcher: IUrlFetcher, folder: string | undefined) {
     if (!folder) folder = tmp.dirSync().name
     this.folder = folder
+    this.urlFetcher = urlFetcher
   }
@@ -52,3 +54,3 @@
         throw new PermissionError(
-          `You do no have permission to access the whole of folder "${this.folder}". Are you sure you want Dockter to compile this folder?`
+            `You do no have permission to access the whole of folder "${this.folder}". Are you sure you want Dockter to compile this folder?`
         )
@@ -85,41 +87,4 @@ } else throw error
   async fetch (url: string, options: any = { json: true }): Promise<any> {
-    if (!REQUEST_CACHE_INITIALISED) {
-      await persist.init({
-        dir: REQUEST_CACHE_DIR,
-        ttl: 60 * 60 * 1000 // Milliseconds to cache responses for
-      })
-      REQUEST_CACHE_INITIALISED = true
-    }
-
-    let value
-    try {
-      value = false // await persist.getItem(url)
-    } catch (error) {
-      if (error.message.includes('does not look like a valid storage file')) {
-        // It seems that `persist.setItem` is not atomic and that the storage file can
-        // have zero bytes when we make multiple requests, some of which are for the same
-        // url. So we ignore this error and continue with the duplicate request.
-      } else {
-        throw error
-      }
-    }
-
-    if (!value) {
-      try {
-        const response = await got(url, options)
-        value = response.body
-      } catch (error) {
-        if (error.statusCode === 404) {
-          value = null
-        } else if (['ENOTFOUND', 'EAI_AGAIN', 'DEPTH_ZERO_SELF_SIGNED_CERT'].includes(error.code)) {
-          // These are usually connection errors
-          throw new NetworkError(`There was a problem fetching ${url} (${error.code}). Are you connected to the internet?`)
-        } else {
-          throw error
-        }
-      }
-      await persist.setItem(url, value)
-    }
-    return value
+    return this.urlFetcher.fetchUrl(url, options)
   }
 }

src/Generator.ts

@@ -91,3 +91,3 @@ import Doer from './Doer'
 # It's good practice to run Docker images as a non-root user.
-# This section creates a new user and it's home directory as the default working directory.`
+# This section creates a new user and its home directory as the default working directory.`
     }
@@ -94,0 +94,0 @@ dockerfile += `

src/JavascriptGenerator.ts

 import { SoftwarePackage } from '@stencila/schema'
 
 import PackageGenerator from './PackageGenerator'
+import IUrlFetcher from './IUrlFetcher'
 
@@ -22,4 +23,4 @@ const PACKAGE_JSON_GENERATED = '.package.json'
 
-  constructor (pkg: SoftwarePackage, folder?: string, nodeMajorVersion: number = 10) {
-    super(pkg, folder)
+  constructor (urlFetcher: IUrlFetcher, pkg: SoftwarePackage, folder?: string, nodeMajorVersion: number = 10) {
+    super(urlFetcher, pkg, folder)
 
@@ -26,0 +27,0 @@ this.nodeMajorVersion = nodeMajorVersion

src/JavascriptParser.ts

@@ -1,2 +0,1 @@
-import { SoftwarePackage, Person } from '@stencila/schema'
 // @ts-ignore
@@ -10,2 +9,3 @@ import builtins from 'builtin-modules'
 import Parser from './Parser'
+import { Person, SoftwarePackage } from '@stencila/schema'
 
@@ -36,3 +36,3 @@ /**
           for (let require of requires) {
-            if (! builtins.includes(require)) {
+            if (!builtins.includes(require)) {
               // @ts-ignore
@@ -109,3 +109,3 @@ data.dependencies[require] = 'latest'
             let version = 'latest'
-            if (versionRange !== 'latest' || versionRange !== '*') {
+            if (versionRange !== 'latest' && versionRange !== '*') {
               const range = semver.validRange(versionRange as string)
@@ -112,0 +112,0 @@ if (range) {

src/PackageGenerator.ts

 import { SoftwarePackage } from '@stencila/schema'
 
 import Generator from './Generator'
+import IUrlFetcher from './IUrlFetcher'
 
@@ -15,4 +16,4 @@ /**
 
-  constructor (pkg: SoftwarePackage, folder?: string) {
-    super(folder)
+  constructor (urlFetcher: IUrlFetcher, pkg: SoftwarePackage, folder?: string) {
+    super(urlFetcher, folder)
     this.package = pkg
@@ -19,0 +20,0 @@ }

src/Parser.ts

@@ -7,6 +7,6 @@ import Doer from './Doer'
  *
- * A language `Parser` generates a JSON-LD `SoftwareApplication` instance based on the
+ * A language `Parser` generates a JSON-LD `SoftwarePackage` instance based on the
  * contents of a directory. It is responsible for determining which packages the application
  * needs, resolving the dependencies of those packages (both system and language packages) and
- * turning those into a JSON-LD `SoftwareApplication` instance.
+ * turning those into a JSON-LD `SoftwarePackage` instance.
  *
@@ -13,0 +13,0 @@ * If the `Parser` finds a corresponding requirements file for the language (e.g. `requirements.txt` for Python),

src/PythonGenerator.ts

@@ -1,4 +0,2 @@
-/* tslint:disable: completed-docs */
-
-import { SoftwarePackage, SoftwareEnvironment } from '@stencila/schema'
+import { SoftwarePackage } from '@stencila/schema'
 import path from 'path'
@@ -8,2 +6,3 @@
 import PythonSystemPackageLookup from './PythonSystemPackageLookup'
+import IUrlFetcher from './IUrlFetcher'
 
@@ -16,3 +15,10 @@ const GENERATED_REQUIREMENTS_FILE = '.requirements.txt'
 export default class PythonGenerator extends PackageGenerator {
+  /**
+   * The Python Major version, i.e. 2 or 3
+   */
   private readonly pythonMajorVersion: number
+
+  /**
+   * An instance of `PythonSystemPackageLookup` with which to look up system dependencies of Python packages
+   */
   private readonly systemPackageLookup: PythonSystemPackageLookup
@@ -22,4 +28,4 @@
 
-  constructor (pkg: SoftwarePackage, folder?: string, pythonMajorVersion: number = 3) {
-    super(pkg, folder)
+  constructor (urlFetcher: IUrlFetcher, pkg: SoftwarePackage, folder?: string, pythonMajorVersion: number = 3) {
+    super(urlFetcher, pkg, folder)
 
@@ -38,2 +44,5 @@ this.pythonMajorVersion = pythonMajorVersion
 
+  /**
+   * Check if this Generator's package applies (if it is Python).
+   */
   applies (): boolean {
@@ -43,2 +52,5 @@ return this.package.runtimePlatform === 'Python'
 
+  /**
+   * Generate a list of system (apt) packages by looking up with `this.systemPackageLookup`.
+   */
   aptPackages (sysVersion: string): Array<string> {
@@ -66,2 +78,5 @@ let aptRequirements: Array<string> = []
 
+  /**
+   * Build the contents of a `requirements.txt` file by joining the Python package name to its version specifier.
+   */
   generateRequirementsContent (): string {
@@ -73,6 +88,9 @@ if (!this.package.softwareRequirements) {
     return this.filterPackages('Python').map(
-      requirement => `${requirement.name}${requirement.version}`
+        requirement => `${requirement.name}${requirement.version}`
     ).join('\n')
   }
 
+  /**
+   * Get the pip command to install the Stencila package
+   */
   stencilaInstall (sysVersion: string): string | undefined {
@@ -82,2 +100,6 @@ return `pip${this.pythonVersionSuffix()} install --no-cache-dir https://github.com/stencila/py/archive/91a05a139ac120a89fc001d9d267989f062ad374.zip`
 
+  /**
+   * Write out the generated requirements content to `GENERATED_REQUIREMENTS_FILE` or none exists, just instruct the
+   * copy of a `requirements.txt` file as part of the Dockerfile. If that does not exist, then no COPY should be done.
+   */
   installFiles (sysVersion: string): Array<[string, string]> {
@@ -98,2 +120,5 @@ let requirementsContent = this.generateRequirementsContent()
 
+  /**
+   * Generate the right pip command to install the requirements, appends the correct Python major version to `pip`.
+   */
   installCommand (sysVersion: string): string | undefined {
@@ -100,0 +125,0 @@ return `pip${this.pythonVersionSuffix()} install --user --requirement requirements.txt`

src/PythonParser.ts

@@ -1,3 +0,1 @@
-/* tslint:disable: completed-docs */
-
 import { dirname, basename } from 'path'
@@ -7,3 +5,2 @@ import {
   Person,
-  SoftwareApplication,
   SoftwarePackage,
@@ -22,2 +19,5 @@ SoftwareSourceCode
 
+/**
+ * Return true if the passed in line is a requirements.txt comment (starts with "#" which might be preceded by spaces).
+ */
 function lineIsComment (line: string): boolean {
@@ -27,2 +27,5 @@ return REQUIREMENTS_COMMENT_REGEX.exec(line) !== null
 
+/**
+ * Execute the given `regex` against the line and return the first match. If there is no match, return `null`.
+ */
 function applyRegex (line: string, regex: RegExp): string | null {
@@ -37,2 +40,6 @@ const result = regex.exec(line)
 
+/**
+ * Execute the `REQUIREMENTS_EDITABLE_SOURCE_REGEX` against a line and return the first result (or null if no match).
+ * This is used to find a requirements.txt line of a URL source (e.g. including a package from github).
+ */
 function extractEditableSource (line: string): string | null {
@@ -42,2 +49,6 @@ return applyRegex(line, REQUIREMENTS_EDITABLE_SOURCE_REGEX)
 
+/**
+ * Execute the `REQUIREMENTS_INCLUDE_PATH_REGEX` against a line and return the first result (or null if no match).
+ * This is used to find a requirements.txt line that includes another requirements file.
+ */
 function extractIncludedRequirementsPath (line: string): string | null {
@@ -47,2 +58,6 @@ return applyRegex(line, REQUIREMENTS_INCLUDE_PATH_REGEX)
 
+/**
+ * Execute the `REQUIREMENTS_STANDARD_REGEX` against a line and return the first result (or null if no match).
+ * This is used to find "standard" requirements.txt lines.
+ */
 function extractStandardRequirements (line: string): string | null {
@@ -52,6 +67,10 @@ return applyRegex(line, REQUIREMENTS_STANDARD_REGEX)
 
+/**
+ * Split a requirement line into name and then version. For example "package==1.0.1" => ["package", "==1.0.1"]
+ * The version specifier can be `==`, `<=`, `>=`, `~=`, `<` or `>`.
+ */
 function splitStandardRequirementVersion (requirement: string): [string, string | null] {
   let firstSplitterIndex = -1
 
-  for (let splitter of ['<=', '>=', '==', '~=', '<', '>']) {
+  for (let splitter of ['==', '<=', '>=', '~=', '<', '>']) {
     let splitterIndex = requirement.indexOf(splitter)
@@ -70,2 +89,6 @@ if (splitterIndex > -1 && (firstSplitterIndex === -1 || splitterIndex < firstSplitterIndex)) {
 
+/**
+ * Convert a list of classifiers to a Map between main classification and sub classification(s).
+ * e.g: ['A :: B', 'A :: C', 'D :: E'] => {'A': ['B', 'C'], 'D': ['E']}
+ */
 function buildClassifierMap (classifiers: Array<string>): Map<string, Array<string>> {
@@ -157,2 +180,6 @@ const classifierMap = new Map<string, Array<string>>()
 
+/**
+ * Parser to be used on a directory with Python source code and (optionally) a `requirements.txt` file.
+ * If no `requirements.txt` file exists then the Parser will attempt to read requirements from the Python source code.
+ */
 export default class PythonParser extends Parser {
@@ -185,3 +212,3 @@
       if (rawRequirement.type === RequirementType.Named) {
-        pkg.softwareRequirements.push(await this.createApplication(rawRequirement))
+        pkg.softwareRequirements.push(await this.createPackage(rawRequirement))
       } else if (rawRequirement.type === RequirementType.URL) {
@@ -197,21 +224,24 @@ let sourceRequirement = new SoftwareSourceCode()
 
-  private async createApplication (requirement: PythonRequirement): Promise<SoftwareApplication> {
-    const softwareApplication = new SoftwareApplication()
-    softwareApplication.name = requirement.value
-    softwareApplication.runtimePlatform = 'Python'
-    softwareApplication.programmingLanguages = [ComputerLanguage.py]
+  /**
+   * Convert a `PythonRequirement` into a `SoftwarePackage` by augmenting with metadata from PyPI
+   */
+  private async createPackage (requirement: PythonRequirement): Promise<SoftwarePackage> {
+    const softwarePackage = new SoftwarePackage()
+    softwarePackage.name = requirement.value
+    softwarePackage.runtimePlatform = 'Python'
+    softwarePackage.programmingLanguages = [ComputerLanguage.py]
 
     if (requirement.version) {
-      softwareApplication.version = requirement.version
+      softwarePackage.version = requirement.version
     }
 
-    const pyPiMetadata = await this.fetch(`https://pypi.org/pypi/${softwareApplication.name}/json`)
+    const pyPiMetadata = await this.fetch(`https://pypi.org/pypi/${softwarePackage.name}/json`)
 
     if (pyPiMetadata.info) {
       if (pyPiMetadata.info.author) {
-        softwareApplication.authors.push(Person.fromText(`${pyPiMetadata.info.author} <${pyPiMetadata.info.author_email}>`))
+        softwarePackage.authors.push(Person.fromText(`${pyPiMetadata.info.author} <${pyPiMetadata.info.author_email}>`))
       }
 
       if (pyPiMetadata.info.project_url) {
-        softwareApplication.codeRepository = pyPiMetadata.info.project_url
+        softwarePackage.codeRepository = pyPiMetadata.info.project_url
       }
@@ -225,4 +255,4 @@
 
-          if (topics.length) softwareApplication.applicationCategories = topics
-          if (subTopics.length) softwareApplication.applicationSubCategories = subTopics
+          if (topics.length) softwarePackage.applicationCategories = topics
+          if (subTopics.length) softwarePackage.applicationSubCategories = subTopics
         }
@@ -238,18 +268,21 @@
           }
-          softwareApplication.operatingSystems = operatingSystems
+          softwarePackage.operatingSystems = operatingSystems
         }
       }
-      if (pyPiMetadata.info.keywords) softwareApplication.keywords = pyPiMetadata.info.keywords
+      if (pyPiMetadata.info.keywords) softwarePackage.keywords = pyPiMetadata.info.keywords
 
-      if (pyPiMetadata.info.license) softwareApplication.license = pyPiMetadata.info.license
+      if (pyPiMetadata.info.license) softwarePackage.license = pyPiMetadata.info.license
 
       if (pyPiMetadata.info.long_description) {
-        softwareApplication.description = pyPiMetadata.info.long_description
+        softwarePackage.description = pyPiMetadata.info.long_description
       } else if (pyPiMetadata.info.description) {
-        softwareApplication.description = pyPiMetadata.info.description
+        softwarePackage.description = pyPiMetadata.info.description
       }
     }
-    return softwareApplication
+    return softwarePackage
   }
 
+  /**
+   * Parse a `requirements.txt` file at `path` and return a list of `PythonRequirement`s
+   */
   async parseRequirementsFile (path: string): Promise<Array<PythonRequirement>> {
@@ -291,2 +324,5 @@ const requirementsContent = this.read(path)
 
+  /**
+   * Parse Python source files are find any non-system imports, return this as an array of `PythonRequirement`s.
+   */
   generateRequirementsFromSource (): Array<PythonRequirement> {
@@ -302,2 +338,5 @@ const nonSystemImports = this.findImports().filter(pythonImport => !pythonSystemModules.includes(pythonImport))
 
+  /**
+   * Parse Python source files are find all imports (including system imports).
+   */
   findImports (): Array<string> {
@@ -318,5 +357,8 @@ const files = this.glob(['**/*.py'])
 
+  /**
+   * Parse Python a single Python source file for imports.
+   */
   readImportsInFile (path: string): Array<string> {
     const fileContent = this.read(path)
-    const importRegex = /^from ([\w_]+)|^import ([\w_]+)/gm
+    const importRegex = /^\s*from ([\w_]+)|^\s*import ([\w_]+)/gm
     const imports: Array<string> = []
@@ -323,0 +365,0 @@ const fileDirectory = dirname(path)

src/PythonSystemPackageLookup.ts

@@ -1,3 +0,1 @@
-/* tslint:disable: completed-docs */
-
 import fs from 'fs'
@@ -10,3 +8,2 @@
  * value
- * @param value
  */
@@ -28,6 +25,16 @@ function valueToMap (value: any): any {
 
+/**
+ * An object that looks up if any system packages are required for a Python package.
+ * The lookup is in the format {packageName: pythonVersion: systemPackageType: systemVersion: [sysPackage, sysPackage...]}
+ */
 export default class PythonSystemPackageLookup {
+  /**
+   * @param packageLookup: PythonSystemPackageLookupMap the Map
+   */
   constructor (private readonly packageLookup: PythonSystemPackageLookupMap) {
   }
 
+  /**
+   * Construct a `PythonSystemPackageLookup` by parsing a JSON representation of the package map from `path`
+   */
   static fromFile (path: string): PythonSystemPackageLookup {
@@ -38,2 +45,6 @@ const dependencyLookupRaw = JSON.parse(fs.readFileSync(path, 'utf8'))
 
+  /**
+   * Look up the system package required for a python package given python version, package type and system version.
+   * Will always return an Array, which will be empty if there are no packages to install.
+   */
   lookupSystemPackage (pythonPackage: string, pythonMajorVersion: number, systemPackageType: string, systemVersion: string): Array<string> {
@@ -40,0 +51,0 @@ const pyPackageMap = this.packageLookup.get(pythonPackage)

src/RGenerator.ts

 import { SoftwarePackage } from '@stencila/schema'
 
 import PackageGenerator from './PackageGenerator'
+import IUrlFetcher from './IUrlFetcher'
 
@@ -15,4 +16,4 @@ /**
 
-  constructor (pkg: SoftwarePackage, folder?: string) {
-    super(pkg, folder)
+  constructor (urlFetcher: IUrlFetcher, pkg: SoftwarePackage, folder?: string) {
+    super(urlFetcher, pkg, folder)
 
@@ -19,0 +20,0 @@ // Default to yesterday's date (to ensure MRAN is available for the date)

src/router.ts

 import { Router, Request, Response, json } from 'express'
+
 const router = Router()
 
 import DockerCompiler from './DockerCompiler'
-const compiler = new DockerCompiler()
+import CachingUrlFetcher from './CachingUrlFetcher'
 
+const compiler = new DockerCompiler(new CachingUrlFetcher())
+
 router.use(json())
@@ -8,0 +11,0 @@

src/RParser.ts

@@ -178,2 +178,3 @@ import path from 'path'
     const sysreqs = await this.fetch(`https://sysreqs.r-hub.io/pkg/${name}`)
+
     for (let sysreq of sysreqs) {
@@ -180,0 +181,0 @@ const keys = Object.keys(sysreq)

tests/DockerBuilder.test.ts

 import Docker from 'dockerode'
 
 import DockerBuilder from '../src/DockerBuilder'
-import fixture from './fixture'
+import { fixture } from './test-functions'
 
@@ -6,0 +6,0 @@ // This is intended to be the only test file where we do any actual Docker image builds

tests/DockerCompiler.test.ts

 import fs from 'fs'
 
 import DockerCompiler from '../src/DockerCompiler'
-import fixture from './fixture'
+import { fixture } from './test-functions'
 import { SoftwareEnvironment, Person } from '@stencila/schema'
+import MockUrlFetcher from './MockUrlFetcher'
 
+const urlFetcher = new MockUrlFetcher()
+
 jest.setTimeout(30 * 60 * 1000)
@@ -17,3 +20,3 @@
 test('compile:empty', async () => {
-  const compiler = new DockerCompiler()
+  const compiler = new DockerCompiler(urlFetcher)
   let environ = await compiler.compile('file://' + fixture('empty'), false)
@@ -27,3 +30,3 @@
 test('compile:dockerfile-date', async () => {
-  const compiler = new DockerCompiler()
+  const compiler = new DockerCompiler(urlFetcher)
   let environ = await compiler.compile('file://' + fixture('dockerfile-date'), false)
@@ -35,4 +38,4 @@
 
-test.skip('compile:multi-lang', async () => {
-  const compiler = new DockerCompiler()
+test('compile:multi-lang', async () => {
+  const compiler = new DockerCompiler(urlFetcher)
   let environ = await compiler.compile('file://' + fixture('multi-lang'), false, false)
@@ -39,0 +42,0 @@

tests/DockerGenerator.test.ts

 import { SoftwareEnvironment, SoftwarePackage } from '@stencila/schema'
 
-import fixture from './fixture'
 import DockerGenerator from '../src/DockerGenerator'
+import MockUrlFetcher from './MockUrlFetcher'
 
+const urlFetcher = new MockUrlFetcher()
+
 /**
@@ -12,3 +14,3 @@ * When applied to an empty environment, generate should return
   const environ = new SoftwareEnvironment()
-  const generator = new DockerGenerator(environ)
+  const generator = new DockerGenerator(urlFetcher, environ)
   expect(await generator.generate(false)).toEqual(`FROM ubuntu:18.04
@@ -40,2 +42,3 @@
   pkg2.runtimePlatform = 'Python'
+  pkg2.softwareRequirements = []
 
@@ -45,5 +48,5 @@ const environ = new SoftwareEnvironment()
   environ.softwareRequirements = [pkg1, pkg2]
-  
-  const generator = new DockerGenerator(environ)
+
+  const generator = new DockerGenerator(urlFetcher, environ)
   expect(generator.aptPackages('18.04')).toEqual(['libxml2-dev', 'python3', 'python3-pip', 'r-base'])
 })

tests/DockerParser.test.ts

 import DockerParser from '../src/DockerParser'
-import {Person, SoftwareEnvironment} from '@stencila/schema'
-import fixture from './fixture'
+import { Person, SoftwareEnvironment } from '@stencila/schema'
+import { fixture } from './test-functions'
+import MockUrlFetcher from './MockUrlFetcher'
 
+const urlFetcher = new MockUrlFetcher()
+
 /**
@@ -10,3 +13,3 @@ * When passed Dockerfile strings, parse should extract LABEL
 test('parse:strings', async () => {
-  const parser = new DockerParser('')
+  const parser = new DockerParser(urlFetcher, '')
   let environ
@@ -63,3 +66,3 @@
 test('parse:empty', async () => {
-  const parser = new DockerParser(fixture('empty'))
+  const parser = new DockerParser(urlFetcher, fixture('empty'))
   expect(await parser.parse()).toBeNull()
@@ -72,3 +75,3 @@ })
 test('parse:r-date', async () => {
-  const parser = new DockerParser(fixture('r-date'))
+  const parser = new DockerParser(urlFetcher, fixture('r-date'))
   expect(await parser.parse()).toBeNull()
@@ -82,3 +85,3 @@ })
 test('parse:dockerfile-date', async () => {
-  const parser = new DockerParser(fixture('dockerfile-date'))
+  const parser = new DockerParser(urlFetcher, fixture('dockerfile-date'))
   const environ = await parser.parse() as SoftwareEnvironment
@@ -85,0 +88,0 @@ expect(environ.description && environ.description.substring(0, 23)).toEqual('Prints the current date')

tests/JavascriptParser.test.ts

@@ -1,7 +0,9 @@
-import fixture from './fixture'
+import { fixture } from './test-functions'
 import JavascriptParser from '../src/JavascriptParser'
 import { Person, SoftwarePackage } from '@stencila/schema'
-import fs from 'fs'
-import { REQUEST_CACHE_DIR } from '../src/Doer'
 
+import MockUrlFetcher from './MockUrlFetcher'
+
+const urlFetcher = new MockUrlFetcher()
+
 // Increase timeout (in milliseconds) to allow for HTTP requests
@@ -11,14 +13,2 @@ // to get package meta data
 describe('JavascriptParser', () => {
-  beforeEach(() => {
-    if (fs.existsSync(REQUEST_CACHE_DIR)) {
-      for (let item of fs.readdirSync(REQUEST_CACHE_DIR)) {
-        try {
-          fs.unlinkSync(REQUEST_CACHE_DIR + '/' + item)
-        } catch (e) {
-          // Cleanups might execute in parallel in multiple test runs so don't worry if remove fails
-        }
-      }
-    }
-  })
-
   /**
@@ -28,3 +18,3 @@ * When applied to an empty folder, parse should return null.
   test('parse:empty', async () => {
-    const parser = new JavascriptParser(fixture('empty'))
+    const parser = new JavascriptParser(urlFetcher, fixture('empty'))
     expect(await parser.parse()).toBeNull()
@@ -36,4 +26,4 @@ })
    */
-  test('parse:r-date', async () => {
-    const parser = new JavascriptParser(fixture('empty'))
+  test('parse:non-js', async () => {
+    const parser = new JavascriptParser(urlFetcher, fixture('r-date'))
     expect(await parser.parse()).toBeNull()
@@ -47,4 +37,4 @@ })
    */
-  test('parse:js-package', async () => {
-    const parser = new JavascriptParser(fixture('js-package'))
+  test('parse:js-requirements', async () => {
+    const parser = new JavascriptParser(urlFetcher, fixture('js-requirements'))
     const pkg = await parser.parse() as SoftwarePackage
@@ -64,2 +54,3 @@
     ]
+
     for (let index in expecteds) {
@@ -78,3 +69,3 @@ let { name, version } = pkg.softwareRequirements[index]
   test('parse:js-sources', async () => {
-    const parser = new JavascriptParser(fixture('js-sources'))
+    const parser = new JavascriptParser(urlFetcher, fixture('js-sources'))
     const pkg = await parser.parse() as SoftwarePackage
@@ -87,2 +78,18 @@
   })
+
+  /**
+   * When applied to a folder with both `*.js` files and a `package.json` file, then parse should return a
+   * `SoftwarePackage` with `name`, `softwareRequirements` etc populated from the `package.json` in that directory.
+   */
+  test('parse:js-mixed', async () => {
+    const parser = new JavascriptParser(urlFetcher, fixture('js-mixed'))
+    const pkg = await parser.parse() as SoftwarePackage
+
+    expect(pkg.name).toEqual('js-mixed')
+    expect(pkg.description).toEqual('A test Node.js package for js-mixed')
+    expect(pkg.version).toEqual('1.2.3')
+    expect(pkg.softwareRequirements.length).toEqual(1)
+    expect(pkg.softwareRequirements[0].name).toEqual('is-even')
+    expect(pkg.softwareRequirements[0].version).toEqual('4.5.6')
+  })
 })

tests/PythonGenerator.test.ts

 import { SoftwarePackage } from '@stencila/schema'
 import PythonGenerator from '../src/PythonGenerator'
 import fs from 'fs'
-import fixture from './fixture'
+import { fixture, expectedFixture, cleanup } from './test-functions'
 
+import MockUrlFetcher from './MockUrlFetcher'
+
+const urlFetcher = new MockUrlFetcher()
+
 /**
@@ -12,3 +16,3 @@ * When applied to an empty package, generate should return
   const pkg = new SoftwarePackage()
-  const generator = new PythonGenerator(pkg)
+  const generator = new PythonGenerator(urlFetcher, pkg)
   expect(await generator.generate(false)).toEqual('FROM ubuntu:18.04\n')
@@ -21,3 +25,5 @@ })
  */
-test.skip('generate:pydate', async () => {
+test('generate:requirements', async () => {
+  cleanup(['py-generator-generated/.Dockerfile', 'py-generator-generated/.requirements.txt'])
+
   const arrowPackage = new SoftwarePackage()
@@ -32,5 +38,5 @@ arrowPackage.name = 'arrow'
 
-  const generator = new PythonGenerator(pkg, fixture('py-date'))
+  const generator = new PythonGenerator(urlFetcher, pkg, fixture('py-generator-generated'))
 
-  expect(await generator.generate(false)).toEqual(`FROM ubuntu:18.04
+  expect(await generator.generate(false, true)).toEqual(`FROM ubuntu:18.04
 
@@ -48,3 +54,2 @@ RUN apt-get update \\
 RUN useradd --create-home --uid 1001 -s /bin/bash dockteruser
-USER dockteruser
 WORKDIR /home/dockteruser
@@ -54,3 +59,3 @@
 
-COPY requirements.txt requirements.txt
+COPY .requirements.txt requirements.txt
 
@@ -61,6 +66,10 @@ RUN pip3 install --user --requirement requirements.txt
 
-CMD python3  cmd.py
+USER dockteruser
+
+CMD python3 cmd.py
 `)
-  
-  expect(fs.existsSync(fixture('py-date/.requirements.txt'))).toBe(false)
+
+  expectedFixture(fixture('py-generator-generated'), '.requirements.txt')
+
+  cleanup(['py-generator-generated/.Dockerfile', 'py-generator-generated/.requirements.txt'])
 })
@@ -72,7 +81,8 @@
  */
-test.skip('generate:requirements-file', async () => {
+test('generate:requirements-file', async () => {
+  cleanup(['py-generator-existing/.Dockerfile'])
   const pkg = new SoftwarePackage()
   pkg.runtimePlatform = 'Python'
 
-  const generator = new PythonGenerator(pkg, fixture('py-date'), 2)
+  const generator = new PythonGenerator(urlFetcher, pkg, fixture('py-generator-existing'), 2)
 
@@ -90,3 +100,2 @@ expect(await generator.generate(false)).toEqual(`FROM ubuntu:18.04
 RUN useradd --create-home --uid 1001 -s /bin/bash dockteruser
-USER dockteruser
 WORKDIR /home/dockteruser
@@ -102,4 +111,10 @@
 
+USER dockteruser
+
 CMD python cmd.py
 `)
+
+  // it should not generate a .requirements.txt
+  expect(fs.existsSync(fixture('py-generator-existing/.requirements.txt'))).toBeFalsy()
+  cleanup(['py-generator-existing/.Dockerfile'])
 })
@@ -111,3 +126,4 @@
  */
-test.skip('generate:requirements-file', async () => {
+test('generate:apt-packages', async () => {
+  cleanup(['py-generator-generated/.Dockerfile', 'py-generator-generated/.requirements.txt'])
   const pygit2 = new SoftwarePackage()
@@ -122,3 +138,3 @@ pygit2.name = 'pygit2'
 
-  const generator = new PythonGenerator(pkg, fixture('py-date'), 2)
+  const generator = new PythonGenerator(urlFetcher, pkg, fixture('py-generator-generated'), 2)
 
@@ -136,6 +152,3 @@ expect(await generator.generate(false)).toEqual(`FROM ubuntu:18.04
 
-RUN pip install --no-cache-dir https://github.com/stencila/py/archive/91a05a139ac120a89fc001d9d267989f062ad374.zip
-
 RUN useradd --create-home --uid 1001 -s /bin/bash dockteruser
-USER dockteruser
 WORKDIR /home/dockteruser
@@ -151,4 +164,7 @@
 
-CMD python  cmd.py
+USER dockteruser
+
+CMD python cmd.py
 `)
+  cleanup(['py-generator-generated/.Dockerfile', 'py-generator-generated/.requirements.txt'])
 })

tests/PythonParser.test.ts

@@ -1,21 +0,11 @@
-import fixture from './fixture'
-import fs from 'fs'
+import { fixture } from './test-functions'
 
 import PythonParser, { RequirementType } from '../src/PythonParser'
-import { ComputerLanguage, OperatingSystem, Person, SoftwareApplication, SoftwarePackage } from '@stencila/schema'
-import { REQUEST_CACHE_DIR } from '../src/Doer'
+import { ComputerLanguage, OperatingSystem, Person, SoftwarePackage } from '@stencila/schema'
 
+import MockUrlFetcher from './MockUrlFetcher'
+
+const urlFetcher = new MockUrlFetcher()
+
 describe('PythonParser', () => {
-  beforeEach(() => {
-    if (fs.existsSync(REQUEST_CACHE_DIR)) {
-      for (let item of fs.readdirSync(REQUEST_CACHE_DIR)) {
-        try {
-          fs.unlinkSync(REQUEST_CACHE_DIR + '/' + item)
-        } catch (e) {
-          // Cleanups might execute in parallel in multiple test runs so don't worry if remove fails
-        }
-      }
-    }
-  })
-
   /**
@@ -25,3 +15,3 @@ * When applied to an empty folder, parse should return null.
   test('parse:empty', async () => {
-    const parser = new PythonParser(fixture('empty'))
+    const parser = new PythonParser(urlFetcher, fixture('empty'))
     expect(await parser.parse()).toBeNull()
@@ -33,4 +23,4 @@ })
    */
-  test('parse:r-date', async () => {
-    const parser = new PythonParser(fixture('r-date'))
+  test('parse:non-py', async () => {
+    const parser = new PythonParser(urlFetcher, fixture('r-date'))
     expect(await parser.parse()).toBeNull()
@@ -43,4 +33,4 @@ })
    */
-  test('parse:example-requirements', async () => {
-    const parser = new PythonParser(fixture('py-requirements-example'))
+  test('parse:py-requirements', async () => {
+    const parser = new PythonParser(urlFetcher, fixture('py-requirements'))
 
@@ -108,6 +98,6 @@ const requirementsContent = await parser.parseRequirementsFile('requirements.txt')
    */
-  test.skip('parse:py-pandas', async () => {
-    const parser = new PythonParser(fixture('py-date'))
+  test('parse:py-requirements', async () => {
+    const parser = new PythonParser(urlFetcher, fixture('py-mixed'))
 
-    const arrowPackage = new SoftwareApplication()
+    const arrowPackage = new SoftwarePackage()
     arrowPackage.name = 'arrow'
@@ -130,4 +120,5 @@ arrowPackage.version = '==0.12.1'
     const environ = new SoftwarePackage()
-    environ.name = 'py-date'
+    environ.name = 'py-mixed'
     environ.softwareRequirements = [arrowPackage]
+    environ.runtimePlatform = 'Python'
 
@@ -141,7 +132,7 @@ expect(await parser.parse()).toEqual(environ)
    */
-  test('parse:py-generated-requirements', async () => {
-    const parser = new PythonParser(fixture('py-generated-requirements'))
-    let environ = await parser.parse()
+  test('parse:py-source', async () => {
+    const parser = new PythonParser(urlFetcher, fixture('py-source'))
+    const environ = await parser.parse()
     expect(environ).not.toBeNull()
-    let requirementNames = environ!.softwareRequirements.map(requirement => requirement.name)
+    const requirementNames = environ!.softwareRequirements.map(requirement => requirement.name)
     expect(requirementNames.length).toEqual(2)
@@ -151,2 +142,16 @@ expect(requirementNames).toContain('django')
   })
-})
+
+  /**
+   * If a directory has both a `requirements.txt` file and Python source files, the `PythonParser` should only read
+   * requirements from the `requirements.txt` and should not parse the source code.
+   */
+  test('parse:py-mixed', async () => {
+    const parser = new PythonParser(urlFetcher, fixture('py-mixed'))
+    const environ = await parser.parse()
+    expect(environ).not.toBeNull()
+
+    expect(environ!.softwareRequirements.length).toEqual(1)
+    expect(environ!.softwareRequirements[0].name).toEqual('arrow')
+    expect(environ!.softwareRequirements[0].version).toEqual('==0.12.1')
+  })
+})

tests/PythonSystemPackageLookup.test.ts

 import PythonSystemPackageLookup from '../src/PythonSystemPackageLookup'
-import fixture from './fixture'
+import { fixture } from './test-functions'
 
@@ -4,0 +4,0 @@ const packageLookup = PythonSystemPackageLookup.fromFile(fixture('py-system-package-lookup.json'))

tests/RGenerator.test.ts

@@ -1,6 +0,8 @@
-import fixture from './fixture'
-import RParser from '../src/RParser'
+import { fixture } from './test-functions'
 import RGenerator from '../src/RGenerator'
 import { SoftwarePackage } from '@stencila/schema'
+import MockUrlFetcher from './MockUrlFetcher'
 
+const urlFetcher = new MockUrlFetcher()
+
 /**
@@ -12,3 +14,3 @@ * When applied to an empty package, generate should return
   const pkg = new SoftwarePackage()
-  const generator = new RGenerator(pkg)
+  const generator = new RGenerator(urlFetcher, pkg)
   expect(await generator.generate(false)).toEqual('FROM ubuntu:18.04\n')
@@ -21,3 +23,3 @@ })
  */
-test.skip('generate:packages', async () => {
+test('generate:packages', async () => {
   const pkg1 = new SoftwarePackage()
@@ -31,9 +33,6 @@ pkg1.name = 'ggplot2'
   pkg2.softwareRequirements = [pkg1]
-  
-  const generator = new RGenerator(pkg2)
+
+  const generator = new RGenerator(urlFetcher, pkg2)
   expect(await generator.generate(false)).toEqual(`FROM ubuntu:18.04
 
-ENV TZ="Etc/UTC" \\
-    R_LIBS_USER="~/R"
-
 RUN apt-get update \\
@@ -49,2 +48,4 @@ && DEBIAN_FRONTEND=noninteractive apt-get install -y \\
 
+ENV TZ="Etc/UTC"
+
 RUN apt-get update \\
@@ -58,3 +59,2 @@ && DEBIAN_FRONTEND=noninteractive apt-get install -y \\
 RUN useradd --create-home --uid 1001 -s /bin/bash dockteruser
-USER dockteruser
 WORKDIR /home/dockteruser
@@ -66,3 +66,5 @@
 
-RUN mkdir -p ~/R && bash -c "Rscript <(curl -sL https://unpkg.com/@stencila/dockter/src/install.R)"
+RUN bash -c "Rscript <(curl -sL https://unpkg.com/@stencila/dockter/src/install.R)"
+
+USER dockteruser
 `)
@@ -75,11 +77,28 @@ })
  */
-test.skip('generate:r-xml2', async () => {
+test('generate:r-xml2', async () => {
   const folder = fixture('r-xml2')
-  const pkg = await new RParser(folder).parse() as SoftwarePackage
-  const dockerfile = await new RGenerator(pkg, folder).generate(false)
+  const pkg = new SoftwarePackage()
+  pkg.name = 'rxml2'
+  pkg.datePublished = '2018-11-11'
+  pkg.runtimePlatform = 'R'
+
+  const xml2DevPackage = new SoftwarePackage()
+  xml2DevPackage.name = 'libxml2-dev'
+  xml2DevPackage.runtimePlatform = 'deb'
+
+  const xml2Package = new SoftwarePackage()
+  xml2Package.name = 'xml2'
+  xml2Package.runtimePlatform = 'R'
+
+  xml2Package.softwareRequirements = [
+    xml2DevPackage
+  ]
+
+  pkg.softwareRequirements = [
+    xml2Package
+  ]
+
+  const dockerfile = await new RGenerator(urlFetcher, pkg, folder).generate(false)
   expect(dockerfile).toEqual(`FROM ubuntu:18.04
 
-ENV TZ="Etc/UTC" \\
-    R_LIBS_USER="~/R"
-
 RUN apt-get update \\
@@ -95,2 +114,4 @@ && DEBIAN_FRONTEND=noninteractive apt-get install -y \\
 
+ENV TZ="Etc/UTC"
+
 RUN apt-get update \\
@@ -105,3 +126,2 @@ && DEBIAN_FRONTEND=noninteractive apt-get install -y \\
 RUN useradd --create-home --uid 1001 -s /bin/bash dockteruser
-USER dockteruser
 WORKDIR /home/dockteruser
@@ -113,3 +133,3 @@
 
-RUN mkdir -p ~/R && bash -c "Rscript <(curl -sL https://unpkg.com/@stencila/dockter/src/install.R)"
+RUN bash -c "Rscript <(curl -sL https://unpkg.com/@stencila/dockter/src/install.R)"
 
@@ -119,4 +139,6 @@ COPY cmd.R cmd.R
 
+USER dockteruser
+
 CMD Rscript cmd.R
 `)
 })

tests/RParser.test.ts

@@ -1,4 +0,5 @@
-import fixture from './fixture'
+import { fixture } from './test-functions'
 import RParser from '../src/RParser'
-import { SoftwareEnvironment, SoftwarePackage } from '@stencila/schema'
+import { SoftwarePackage } from '@stencila/schema'
+import MockUrlFetcher from './MockUrlFetcher'
 
@@ -9,2 +10,4 @@ // Increase timeout (in milliseconds) to allow for HTTP requests
 
+const urlFetcher = new MockUrlFetcher()
+
 /**
@@ -14,3 +17,3 @@ * When applied to an empty folder, parse should return null.
 test('parse:empty', async () => {
-  const parser = new RParser(fixture('empty'))
+  const parser = new RParser(urlFetcher, fixture('empty'))
   expect(await parser.parse()).toBeNull()
@@ -22,4 +25,4 @@ })
  */
-test('parse:dockerfile-date', async () => {
-  const parser = new RParser(fixture('empty'))
+test('parse:non-r', async () => {
+  const parser = new RParser(urlFetcher, fixture('py-source'))
   expect(await parser.parse()).toBeNull()
@@ -29,48 +32,59 @@ })
 /**
- * When applied to a folder with a DESCRIPTION file, parse should return
- * a `SoftwareEnvironment` with `name`, `softwareRequirements` etc
- * populated correctly.
+ * When applied to a folder with only R source files, parse should return a `SoftwareEnvironment`
+ * with `name`, `softwareRequirements` etc populated correctly.
  */
-test('parse:r-date', async () => {
-  const parser = new RParser(fixture('r-date'))
+test('parse:r-source', async () => {
+  const parser = new RParser(urlFetcher, fixture('r-source'))
   const environ = await parser.parse() as SoftwarePackage
-  expect(environ.name).toEqual('rdate')
-  
+  expect(environ.name).toEqual('rsource')
+
   const reqs = environ.softwareRequirements
-  expect(reqs).toBeDefined()
-  expect(reqs && reqs.length).toEqual(1)
-  expect(reqs && reqs[0].name).toEqual('lubridate')
+  expect(reqs).not.toBeNull()
+  expect(reqs.map(req => req.name)).toEqual(['MASS', 'digest', 'dplyr', 'ggplot2', 'lubridate'])
 })
 
 /**
- * When applied to a folder with no DESCRIPTION file but with .R files, 
- * parse should generate a `.DESCRIPTION` file and 
- * return a `SoftwareEnvironment` with packages listed.
+ * When applied to a folder with a DESCRIPTION file,
+ * parse should return a `SoftwareEnvironment` with the packages listed.
  */
-test('parse:r-no-desc', async () => {
-  const parser = new RParser(fixture('r-no-desc'))
+test('parse:r-requirements', async () => {
+  const parser = new RParser(urlFetcher, fixture('r-requirements'))
   const environ = await parser.parse() as SoftwarePackage
-  
-  expect(environ.name).toEqual('rnodesc')
 
+  expect(environ.name).toEqual('rrequirements')
+
   const reqs = environ.softwareRequirements
-  expect(reqs).toBeDefined()
-  expect(reqs && reqs.map(req => req.name)).toEqual(['MASS', 'digest', 'dplyr', 'ggplot2', 'lubridate'])
+  expect(reqs).not.toBeNull()
+  expect(reqs.map(req => req.name)).toEqual(['packageone', 'packagetwo'])
 })
 
 /**
- * When applied to fixture with more system dependencies...
+ * When applied to a folder with a `DESCRIPTION` file, and R sources, only the `DESCRIPTION` file should be used to
+ * generate the requirements (i.e. R sources aren't parsed for requirements).
  */
-test('parse:r-elife', async () => {
-  const parser = new RParser(fixture('r-elife'))
+test('parse:r-mixed', async () => {
+  const parser = new RParser(urlFetcher, fixture('r-mixed'))
   const environ = await parser.parse() as SoftwarePackage
 
   const reqs = environ.softwareRequirements
-  expect(reqs).toBeDefined()
-  expect(reqs && reqs.map(req => req.name)).toEqual([
-    'car', 'coin', 'ggplot2', 'httr', 'lsmeans', 'MBESS',
-    'metafor', 'pander', 'psychometric', 'reshape2',
-    'rjson', 'tidyr'
-  ])
+  expect(reqs).not.toBeNull()
+  expect(reqs.map(req => req.name)).toEqual(['car', 'coin', 'ggplot2', 'httr', 'lsmeans'])
 })
 
+/**
+ * When applied to the `r-gsl` fixture, handles the sysreqs (which has an 
+ * array of Debian packages) correctly.
+ */
+test('parse:r-gsl', async () => {
+  const parser = new RParser(urlFetcher, fixture('r-gsl'))
+  const environ = await parser.parse() as SoftwarePackage
+
+  const reqs = environ.softwareRequirements
+  expect(reqs).not.toBeNull()
+  expect(reqs.map(req => req.name)).toEqual(['gsl'])
+
+  const gsl = reqs.filter(req => req.name === 'gsl')[0]
+  const gslReqs = gsl.softwareRequirements
+  expect(gslReqs[0].name).toEqual('libgsl-dev')
+  expect(gslReqs[1].name).toEqual('libgsl2')
+})

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

1596917

increased by1.73%

Number of package files

172

increased by12.42%

Lines of code

7043

increased by9.26%

Number of lines in readme file

470

increased by5.86%

Dependency changes

• Updated@stencila/schema@^0.2.4