@f5devcentral/f5-fast-core - npm Package Compare versions

Comparing version 0.18.0 to 0.19.0

CHANGELOG.md

@@ -0,1 +1,8 @@
+# v0.19.0
+## Added
+* template: Add dataFile parameter property for including arbitrary text from files
+
+## Fixed
+* Fix loading sub-templates with GitHubTemplateProvider
+
 # v0.18.0
@@ -2,0 +9,0 @@ ## Added

lib/github_provider.js

@@ -22,4 +22,5 @@ /* Copyright 2021 F5 Networks, Inc.
 
-const ResourceCache = require('./resource_cache').ResourceCache;
 const Template = require('./template').Template;
+const { BaseSchemaProvider } = require('./schema_provider');
+const { BaseDataProvider } = require('./data_provider');
 const { BaseTemplateProvider } = require('./template_provider');
@@ -79,4 +80,6 @@ const { stripExtension } = require('./utils');
  */
-class GitHubSchemaProvider {
+class GitHubSchemaProvider extends BaseSchemaProvider {
     constructor(repo, schemaRootPath, options) {
+        super();
+
         options = options || {};
@@ -88,19 +91,47 @@
         });
+    }
 
-        this.cache = new ResourceCache(schemaName => Promise.resolve()
-            .then(() => this._contentsApi.getContentsData(`${this._rootDir}/${schemaName}.json`)));
+    _loadSchema(schemaName) {
+        return Promise.resolve()
+            .then(() => this._contentsApi.getContentsData(`${this._rootDir}/${schemaName}.json`));
     }
 
     /**
-     * Get the schema associated with the supplied key
+     * List all schema known to the provider
      *
-     * @param {string} key
-     * @returns {object}
+     * @returns {string[]}
      */
-    fetch(key) {
-        return this.cache.fetch(key);
+    list() {
+        return Promise.resolve()
+            .then(() => this._contentsApi.getContentsByType(this._rootDir, 'file'))
+            .then(files => files
+                .filter(x => x.endsWith('.json'))
+                .map(x => stripExtension(x)));
     }
+}
 
+/**
+ * DataProvider that fetches data from a GitHub repository
+ */
+class GitHubDataProvider extends BaseDataProvider {
     /**
-     * List all schema known to the provider
+     * @param {string} dataRootPath - a path to a directory containing data files
+     */
+    constructor(repo, dataRootPath, options) {
+        super();
+        options = options || {};
+
+        this._rootDir = `/${dataRootPath}`;
+        this._contentsApi = new GitHubContentsApi(repo, {
+            apiToken: options.apiToken
+        });
+    }
+
+    _loadData(dataName) {
+        return Promise.resolve()
+            .then(() => this._contentsApi.getContentsData(`${this._rootDir}/${dataName}.data`));
+    }
+
+    /**
+     * List all data files known to the provider
      *
@@ -113,3 +144,3 @@ * @returns {string[]}
             .then(files => files
-                .filter(x => x.endsWith('.json'))
+                .filter(x => x.endsWith('.data'))
                 .map(x => stripExtension(x)));
@@ -134,2 +165,3 @@ }
         this._schemaProviders = {};
+        this._dataProviders = {};
 
@@ -147,2 +179,3 @@ this._contentsApi = new GitHubContentsApi(this.repo, {
         const schemaProvider = this._getSchemaProvider(tmplDir);
+        const dataProvider = this._getDataProvider(tmplDir);
         return Promise.resolve()
@@ -171,3 +204,5 @@ .then(() => this._contentsApi.getContentsByType(tmplDir, 'file'))
                         schemaProvider,
-                        templateProvider: this
+                        dataProvider,
+                        templateProvider: this,
+                        rootDir: tmplDir
                     }));
@@ -185,2 +220,10 @@ });
 
+    _getDataProvider(tsName) {
+        if (!this._dataProviders[tsName]) {
+            this._dataProviders[tsName] = new GitHubDataProvider(this.repo, tsName, { apiToken: this._apiToken });
+        }
+
+        return this._dataProviders[tsName];
+    }
+
     /**
@@ -261,2 +304,31 @@ * Get a list of set names known to the provider
     }
+
+    /**
+     * Get all data files known to the provider (optionally filtered by the supplied set name)
+     *
+     * @param {string} [filteredSetName] - only return data for this template set (instead of all template sets)
+     * @returns {Promise} Promise resolves to an object containing data files
+     */
+    getDataFiles(filteredSetName) {
+        const dataFiles = {};
+        return Promise.resolve()
+            .then(() => (filteredSetName ? [filteredSetName] : this.listSets()))
+            .then(setList => Promise.all(setList.map(
+                tsName => this._getDataProvider(tsName).list()
+                    .then(dataFileList => Promise.all(dataFileList.map(
+                        dataName => this._getDataProvider(tsName).fetch(dataName)
+                            .then((data) => {
+                                const name = `${tsName}/${dataName}`;
+                                const dataHash = crypto.createHash('sha256');
+                                dataHash.update(data);
+                                dataFiles[name] = {
+                                    name,
+                                    data,
+                                    hash: dataHash.digest('hex')
+                                };
+                            })
+                    )))
+            )))
+            .then(() => dataFiles);
+    }
 }
@@ -263,0 +335,0 @@

lib/schema_provider.js

@@ -23,16 +23,21 @@ /* Copyright 2021 F5 Networks, Inc.
 /**
- * SchemaProvider that fetches data from the file system
+ * Abstract base class for SchemaProvider classes
  */
-class FsSchemaProvider {
-    /**
-     * @param {string} schemaRootPath - a path to a directory containing schema files
-     */
-    constructor(schemaRootPath) {
-        this.schema_path = schemaRootPath;
-        this.cache = new ResourceCache(schemaName => new Promise((resolve, reject) => {
-            fs.readFile(`${schemaRootPath}/${schemaName}.json`, (err, data) => {
-                if (err) return reject(err);
-                return resolve(data.toString('utf8'));
-            });
-        }));
+class BaseSchemaProvider {
+    constructor() {
+        if (new.target === BaseSchemaProvider) {
+            throw new TypeError('Cannot instantiate Abstract BaseSchemaProvider');
+        }
+
+        const abstractMethods = [
+            '_loadSchema',
+            'list'
+        ];
+        abstractMethods.forEach((method) => {
+            if (this[method] === undefined) {
+                throw new TypeError(`Expected ${method} to be defined`);
+            }
+        });
+
+        this.cache = new ResourceCache(schemaName => this._loadSchema(schemaName));
     }
@@ -49,4 +54,35 @@
     }
+}
 
+/**
+ * SchemaProvider that fetches data from the file system
+ */
+class FsSchemaProvider extends BaseSchemaProvider {
     /**
+     * @param {string} schemaRootPath - a path to a directory containing schema files
+     */
+    constructor(schemaRootPath) {
+        super();
+
+        this.schemaPath = schemaRootPath;
+    }
+
+    get schema_path() {
+        return this.schemaPath;
+    }
+
+    set schema_path(value) {
+        this.schemaPath = value;
+    }
+
+    _loadSchema(schemaName) {
+        return new Promise((resolve, reject) => {
+            fs.readFile(`${this.schemaPath}/${schemaName}.json`, (err, data) => {
+                if (err) return reject(err);
+                return resolve(data.toString('utf8'));
+            });
+        });
+    }
+
+    /**
      * List all schema known to the provider
@@ -58,3 +94,3 @@ *
         return new Promise((resolve, reject) => {
-            fs.readdir(this.schema_path, (err, data) => {
+            fs.readdir(this.schemaPath, (err, data) => {
                 if (err) return reject(err);
@@ -73,3 +109,3 @@
  */
-class DataStoreSchemaProvider {
+class DataStoreSchemaProvider extends BaseSchemaProvider {
     /**
@@ -80,31 +116,24 @@ * @param {object} datastore - an atg-storage DataStore
     constructor(datastore, tsName) {
+        super();
+
         this.storage = datastore;
         this.tsName = tsName;
-        this.cache = new ResourceCache(
-            schemaName => this.storage.hasItem(this.tsName)
-                .then((result) => {
-                    if (result) {
-                        return Promise.resolve();
-                    }
-                    return Promise.reject(new Error(`Could not find template set "${this.tsName}" in data store`));
-                })
-                .then(() => this.storage.getItem(this.tsName))
-                .then(ts => ts.schemas[schemaName])
-                .then((schema) => {
-                    if (typeof schema === 'undefined') {
-                        return Promise.reject(new Error(`Failed to find schema named "${schemaName}"`));
-                    }
-                    return Promise.resolve(schema);
-                })
-        );
     }
 
-    /**
-     * Get the schema associated with the supplied key
-     *
-     * @param {string} key
-     * @returns {object}
-     */
-    fetch(key) {
-        return this.cache.fetch(key);
+    _loadSchema(schemaName) {
+        return this.storage.hasItem(this.tsName)
+            .then((result) => {
+                if (result) {
+                    return Promise.resolve();
+                }
+                return Promise.reject(new Error(`Could not find template set "${this.tsName}" in data store`));
+            })
+            .then(() => this.storage.getItem(this.tsName))
+            .then(ts => ts.schemas[schemaName])
+            .then((schema) => {
+                if (typeof schema === 'undefined') {
+                    return Promise.reject(new Error(`Failed to find schema named "${schemaName}"`));
+                }
+                return Promise.resolve(schema);
+            });
     }
@@ -131,4 +160,5 @@
 module.exports = {
+    BaseSchemaProvider,
     FsSchemaProvider,
     DataStoreSchemaProvider
 };

lib/template_provider.js

@@ -27,2 +27,3 @@ /* Copyright 2021 F5 Networks, Inc.
 const { stripExtension } = require('./utils');
+const { FsDataProvider } = require('./data_provider');
 
@@ -46,3 +47,4 @@ /**
             'list',
-            'getSchemas'
+            'getSchemas',
+            'getDataFiles'
         ];
@@ -151,5 +153,6 @@ abstractMethods.forEach((method) => {
             this.fetchSet(setName),
-            this.getSchemas(setName)
+            this.getSchemas(setName),
+            this.getDataFiles(setName)
         ])
-            .then(([templates, schemas]) => {
+            .then(([templates, schemas, dataFiles]) => {
                 const tsHash = crypto.createHash('sha256');
@@ -164,2 +167,6 @@ const tmplHashes = Object.values(templates).map(x => x.sourceHash).sort();
                 });
+                const dataHashes = Object.values(dataFiles).map(x => x.hash).sort();
+                dataHashes.forEach((hash) => {
+                    tsHash.update(hash);
+                });
 
@@ -192,2 +199,10 @@ const tsHashDigest = tsHash.digest('hex');
                         return acc;
+                    }, []),
+                    dataFiles: Object.keys(dataFiles).reduce((acc, curr) => {
+                        const data = dataFiles[curr];
+                        acc.push({
+                            name: data.name,
+                            hash: data.hash
+                        });
+                        return acc;
                     }, [])
@@ -224,2 +239,3 @@ });
         this.schemaProviders = {};
+        this.dataProviders = {};
         this.filteredSets = new Set(filteredSets || []);
@@ -231,3 +247,5 @@ }
         this._ensureSchemaProvider(tsName);
+        this._ensureDataProvider(tsName);
         const schemaProvider = this.schemaProviders[tsName];
+        const dataProvider = this.dataProviders[tsName];
         let useMst = 0;
@@ -256,2 +274,3 @@ let tmplpath = `${this.config_template_path}/${templateName}`;
                 schemaProvider,
+                dataProvider,
                 templateProvider: this,
@@ -271,2 +290,11 @@ rootDir: path.resolve(this.config_template_path, tsName)
 
+    _ensureDataProvider(tsName) {
+        if (!this.dataProviders[tsName]) {
+            this.dataProviders[tsName] = new FsDataProvider(path.resolve(
+                this.config_template_path,
+                tsName
+            ));
+        }
+    }
+
     /**
@@ -357,2 +385,40 @@ * Get a list of set names known to the provider
     /**
+     * Get all data files known to the provider (optionally filtered by the supplied set name)
+     *
+     * @param {string} [filteredSetName] - only return data for this template set (instead of all template sets)
+     * @returns {Promise} Promise resolves to an object containing data files
+     */
+    getDataFiles(filteredSetName) {
+        const dataFiles = {};
+        return Promise.resolve()
+            .then(() => {
+                if (filteredSetName) {
+                    return Promise.resolve([filteredSetName]);
+                }
+                return this.listSets();
+            })
+            .then((setList) => {
+                setList.forEach(tsName => this._ensureDataProvider(tsName));
+                return setList;
+            })
+            .then(setList => Promise.all(setList.map(
+                tsName => this.dataProviders[tsName].list()
+                    .then(dataList => Promise.all(dataList.map(
+                        dataName => this.dataProviders[tsName].fetch(dataName)
+                            .then((data) => {
+                                const name = `${tsName}/${dataName}`;
+                                const dataHash = crypto.createHash('sha256');
+                                dataHash.update(data);
+                                dataFiles[name] = {
+                                    name,
+                                    data,
+                                    hash: dataHash.digest('hex')
+                                };
+                            })
+                    )))
+            )))
+            .then(() => dataFiles);
+    }
+
+    /**
      * Delete the template set associated with the supplied set ID.
@@ -536,2 +602,35 @@ *
     /**
+     * Get all data files known to the provider (optionally filtered by the supplied set name)
+     *
+     * @param {string} [filteredSetName] - only return data for this template set (instead of all template sets)
+     * @returns {Promise} Promise resolves to an object containing data files
+     */
+    getDataFiles(filteredSetName) {
+        return Promise.resolve()
+            .then(() => {
+                if (filteredSetName) {
+                    return Promise.resolve([filteredSetName]);
+                }
+                return this.listSets();
+            })
+            .then(setNames => Promise.all(setNames.map(x => this.storage.getItem(x))))
+            .then(setData => setData.filter(x => x))
+            .then(setData => setData.reduce((acc, curr) => {
+                const tsName = curr.name;
+                Object.keys(curr.dataFiles || []).forEach((dataName) => {
+                    const dataFile = curr.dataFiles[dataName];
+                    const name = `${tsName}/${dataName}`;
+                    const dataHash = crypto.createHash('sha256');
+                    dataHash.update(dataFile);
+                    acc[name] = {
+                        name,
+                        data: dataFile,
+                        hash: dataHash.digest('hex')
+                    };
+                });
+                return acc;
+            }, {}));
+    }
+
+    /**
      * Create a new DataStoreTemplateProvider by searching the file system for template sets
@@ -552,5 +651,6 @@ *
                 fsprovider.fetchSet(tsName),
-                fsprovider.getSchemas(tsName)
+                fsprovider.getSchemas(tsName),
+                fsprovider.getDataFiles(tsName)
             ])
-                .then(([setTemplates, setSchemas]) => {
+                .then(([setTemplates, setSchemas, setDataFiles]) => {
                     const templates = Object.entries(setTemplates).reduce((acc, curr) => {
@@ -568,2 +668,8 @@ const [tmplPath, tmplData] = curr;
                     }, {});
+                    const dataFiles = Object.entries(setDataFiles).reduce((acc, curr) => {
+                        const [dataPath, data] = curr;
+                        const dataName = dataPath.split('/')[1];
+                        acc[dataName] = data.data;
+                        return acc;
+                    }, {});
 
@@ -573,3 +679,4 @@ const tsData = {
                         templates,
-                        schemas
+                        schemas,
+                        dataFiles
                     };
@@ -585,2 +692,3 @@
                         schemas: {},
+                        dataFiles: {},
                         error: e.message
@@ -587,0 +695,0 @@ };

lib/template.js

@@ -262,2 +262,18 @@ /* Copyright 2021 F5 Networks, Inc.
 
+    _loadDataFiles(dataProvider) {
+        if (!dataProvider) {
+            return Promise.resolve({});
+        }
+
+        return dataProvider.list()
+            .then(dataList => Promise.all(
+                dataList.map(x => Promise.all([Promise.resolve(x), dataProvider.fetch(x)]))
+            ))
+            .then(dataFiles => dataFiles.reduce((acc, curr) => {
+                const [dataName, data] = curr;
+                acc[dataName] = data;
+                return acc;
+            }, {}));
+    }
+
     _descriptionFromTemplate() {
@@ -301,2 +317,3 @@ const tokens = Mustache.parse(this.templateText);
             && !propDef.mathExpression
+            && !propDef.dataFile
             && typeof propDef.default === 'undefined'
@@ -306,3 +323,3 @@ );
 
-    _handleParsed(parsed, typeSchemas) {
+    _handleParsed(parsed, typeSchemas, dataFiles) {
         const primitives = {
@@ -394,2 +411,18 @@ boolean: false,
                 }
+
+                if (propDef.dataFile) {
+                    if (!propDef.format) {
+                        propDef.format = 'hidden';
+                    }
+
+                    const dataFile = dataFiles[propDef.dataFile];
+                    if (propDef.toBase64) {
+                        propDef.default = Buffer.from(dataFile, 'utf8').toString('base64');
+                    } else if (propDef.fromBase64) {
+                        propDef.default = Buffer.from(dataFile, 'base64').toString('utf8');
+                    } else {
+                        propDef.default = dataFile;
+                    }
+                    delete propDef.dataFile;
+                }
                 break;
@@ -409,3 +442,3 @@ }
             case '#': {
-                const items = this._handleParsed(curr[4], typeSchemas);
+                const items = this._handleParsed(curr[4], typeSchemas, dataFiles);
                 const schemaDef = deepmerge(
@@ -485,3 +518,3 @@ this._typeDefinitions[type] || {},
             case '^': {
-                const items = this._handleParsed(curr[4], typeSchemas);
+                const items = this._handleParsed(curr[4], typeSchemas, dataFiles);
                 const schemaDef = Object.assign(
@@ -591,3 +624,3 @@ this._typeDefinitions[type] || {},
 
-    _parametersSchemaFromTemplate(typeSchemas) {
+    _parametersSchemaFromTemplate(typeSchemas, dataFiles) {
         const mergedDefs = [];
@@ -611,3 +644,3 @@ ['oneOf', 'allOf', 'anyOf'].forEach((xOf) => {
             if (def.template) {
-                const newDef = this._handleParsed(Mustache.parse(def.template), typeSchemas);
+                const newDef = this._handleParsed(Mustache.parse(def.template), typeSchemas, dataFiles);
                 delete newDef.template;
@@ -620,3 +653,3 @@ this._typeDefinitions[name] = newDef;
         });
-        this._parametersSchema = this._handleParsed(Mustache.parse(this.templateText), typeSchemas);
+        this._parametersSchema = this._handleParsed(Mustache.parse(this.templateText), typeSchemas, dataFiles);
 
@@ -716,6 +749,7 @@ // If we just ended up with an empty string type, then we have no types and we
      * @param {SchemaProvider} [schemaProvider] - SchemaProvider to use to fetch schema referenced by the template
+     * @param {DataProvider} [dataProvider] - DataProvider to use to fetch data files referenced by the template
      *
      * @returns {Promise} Promise resolves to `Template`
      */
-    static loadMst(msttext, schemaProvider) {
+    static loadMst(msttext, schemaProvider, dataProvider) {
         if (schemaProvider && schemaProvider.schemaProvider) {
@@ -728,6 +762,9 @@ schemaProvider = schemaProvider.schemaProvider;
         tmpl.templateText = msttext;
-        return tmpl._loadTypeSchemas(schemaProvider)
-            .then((typeSchemas) => {
+        return Promise.all([
+            tmpl._loadTypeSchemas(schemaProvider),
+            tmpl._loadDataFiles(dataProvider)
+        ])
+            .then(([typeSchemas, dataFiles]) => {
                 tmpl._descriptionFromTemplate();
-                tmpl._parametersSchemaFromTemplate(typeSchemas);
+                tmpl._parametersSchemaFromTemplate(typeSchemas, dataFiles);
             })
@@ -757,2 +794,3 @@ .then(() => tmpl._createParametersValidator())
         let templateProvider;
+        let dataProvider;
         let filePath;
@@ -772,2 +810,3 @@ let rootDir;
             templateProvider = options.templateProvider;
+            dataProvider = options.dataProvider;
             filePath = options.filePath;
@@ -869,5 +908,8 @@ rootDir = options.rootDir;
             })
-            .then(() => tmpl._loadTypeSchemas(schemaProvider))
-            .then((typeSchemas) => {
-                tmpl._parametersSchemaFromTemplate(typeSchemas);
+            .then(() => Promise.all([
+                tmpl._loadTypeSchemas(schemaProvider),
+                tmpl._loadDataFiles(dataProvider)
+            ]))
+            .then(([typeSchemas, dataFiles]) => {
+                tmpl._parametersSchemaFromTemplate(typeSchemas, dataFiles);
             })
@@ -874,0 +916,0 @@ .then(() => {

package.json

 {
   "name": "@f5devcentral/f5-fast-core",
-  "version": "0.18.0",
+  "version": "0.19.0",
   "author": "F5 Networks",
@@ -26,8 +26,8 @@ "license": "Apache-2.0",
     "@f5devcentral/eslint-config-f5-atg": "^0.1.7",
-    "chai": "^4.3.4",
+    "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
-    "eslint": "^8.7.0",
+    "eslint": "^8.10.0",
     "eslint-plugin-import": "^2.25.4",
-    "mocha": "^9.2.0",
-    "nock": "^13.2.2",
+    "mocha": "^9.2.1",
+    "nock": "^13.2.4",
     "nyc": "^15.1.0"
@@ -70,3 +70,3 @@ },
     "jsonpath-plus": "^4.0.0",
-    "math-expression-evaluator": "^1.3.8",
+    "math-expression-evaluator": "^1.3.14",
     "merge-lite": "^1.0.2",
@@ -73,0 +73,0 @@ "mustache": "^4.2.0",

README.md

@@ -290,2 +290,49 @@ ![Pipeline](https://github.com/f5devcentral/f5-fast-core/workflows/Pipeline/badge.svg)
 
+### Template Data Files
+
+Sometimes it is desirable to keep a portion of a template in a separate file and include it into the template text.
+This can be done with parameters and the `dataFile` property:
+
+```javascript
+const fast = require('@f5devcentral/f5-fast-core');
+
+const templatesPath = '/path/to/templatesdir'; // directory containing example.data
+const dataProvider = new fast.FsDataProvider(templatesPath);
+const yamldata = `
+    definitions:
+        var:
+            dataFile: example
+    template: |
+        {{var}}
+`;
+
+fast.Template.loadYaml(yamldata, { dataProvider })
+    .then((template) => {
+        console.log(template.getParametersSchema());
+        console.log(template.render({virtual_port: 443});
+    });
+```
+The `FsDataProvider` will pick up on any files with the `.data` extension in the template set directory.
+When referencing the file in a template, use the filename (without the extension) as a key.
+
+Parameters with a `dataFile` property:
+
+* are removed from `required`
+* have their `default` set to the contents of the file
+* given a default `format` of `hidden`
+
+Additionally, the contents of the data file can be base64-encoded before being used as for `default` by setting the `toBase64` property to `true`:
+
+```yaml
+definitions:
+    var:
+        dataFile: example
+        toBase64: true
+    template: |
+        {{var}}
+```
+
+Similarly, if the data file is base64-encoded, it can be decoded using `fromBase64`.
+If both `toBase64` and `fromBase64` are set, then `toBase64` takes precedence.
+
 ## Development
@@ -292,0 +339,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

148696

increased by11.71%

Number of package files

17

increased by6.25%

Lines of code

3089

increased by13.32%

Number of lines in readme file

364

increased by14.83%

Worsened metrics

Number of low supply chain risk alerts

5

increased by25%

Dependency changes

• Updatedmath-expression-evaluator@^1.3.14