loadmill - npm Package Compare versions

Comparing version 0.5.5 to 1.0.0

lib/index.d.ts

 import './polyfills';
 export = Loadmill;
+declare function Loadmill(options: Loadmill.LoadmillOptions): {
+    run(config: any, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<string>;
+    runFolder(folderPath: string, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<Loadmill.TestResult[]>;
+    wait(testDefOrId: string | Loadmill.TestDef, callback?: Loadmill.Callback): Promise<Loadmill.TestResult>;
+    runFunctional(config: any, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<Loadmill.TestResult>;
+    runFunctionalFolder(folderPath: string, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<Loadmill.TestResult[]>;
+    runFunctionalLocally(config: any, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback, testArgs?: Loadmill.Args | undefined): Promise<Loadmill.TestResult>;
+    runFunctionalFolderLocally(folderPath: string, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<Loadmill.TestResult[]>;
+    runAsyncFunctional(config: any, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<Loadmill.TestResult>;
+    runTestSuite(suiteId: string, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<Loadmill.TestDef>;
+};
 declare namespace Loadmill {
@@ -11,2 +22,5 @@ interface LoadmillOptions {
     }
+    interface TestSuiteDef {
+        id: string;
+    }
     interface TestResult extends TestDef {
@@ -34,12 +48,8 @@ url: string;
     };
+    enum TYPES {
+        LOAD = "load",
+        FUNCTIONAL = "functional",
+        SUITE = "test-suite",
+        LOCAL = "local",
+    }
 }
-declare function Loadmill(options: Loadmill.LoadmillOptions): {
-    run(config: any, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<string>;
-    runFolder(folderPath: string, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<Loadmill.TestResult[]>;
-    wait(testDefOrId: string | Loadmill.TestDef, callback?: Loadmill.Callback): Promise<Loadmill.TestResult>;
-    runFunctional(config: any, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<Loadmill.TestResult>;
-    runFunctionalFolder(folderPath: string, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<Loadmill.TestResult[]>;
-    runFunctionalLocally(config: any, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback, testArgs?: Loadmill.Args | undefined): Promise<Loadmill.TestResult>;
-    runFunctionalFolderLocally(folderPath: string, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<Loadmill.TestResult[]>;
-    runAsyncFunctional(config: any, paramsOrCallback?: Loadmill.ParamsOrCallback, callback?: Loadmill.Callback): Promise<Loadmill.TestResult>;
-};

lib/index.js

@@ -8,7 +8,4 @@ "use strict";
 var loadmill_runner_1 = require("loadmill-runner");
-var TYPE_LOAD = 'load';
-var TYPE_FUNCTIONAL = 'functional';
-var LOCAL = 'local';
 function Loadmill(options) {
-    var _a = options, token = _a.token, _b = _a._testingServerHost, _testingServerHost = _b === void 0 ? "www.loadmill.com" : _b;
+    var _a = options, token = _a.token, _b = _a._testingServerHost, _testingServerHost = _b === void 0 ? process.env.LOADMILL_SERVER_HOST || "www.loadmill.com" : _b;
     var testingServer = "https://" + _testingServerHost;
@@ -36,3 +33,3 @@ function _runFolderSync(listOfFiles, execFunc) {
                         if (!(!utils_1.isString(res) && !res.id)) return [3 /*break*/, 3];
-                        testResult = { url: LOCAL, passed: res.passed };
+                        testResult = { url: Loadmill.TYPES.LOCAL, passed: res.passed };
                         return [3 /*break*/, 5];
@@ -63,20 +60,20 @@ case 3: return [4 /*yield*/, _wait(res)];
                     id: testDefOrId,
-                    type: TYPE_LOAD
+                    type: Loadmill.TYPES.LOAD
                 } : testDefOrId;
-                apiUrl = getTestUrl(testDef, testingServer + '/api/tests/', 'trials/', '');
-                webUrl = getTestUrl(testDef, testingServer + '/app/', 'functional/', 'test/');
+                apiUrl = getTestAPIUrl(testDef, testingServer);
+                webUrl = getTestWebUrl(testDef, testingServer);
                 intervalId = setInterval(function () { return tslib_1.__awaiter(_this, void 0, void 0, function () {
-                    var _a, trialResult, result, testResult, err_1;
-                    return tslib_1.__generator(this, function (_b) {
-                        switch (_b.label) {
+                    var body, trialResult, result, isRunning, testResult, err_1;
+                    return tslib_1.__generator(this, function (_a) {
+                        switch (_a.label) {
                             case 0:
-                                _b.trys.push([0, 2, , 3]);
+                                _a.trys.push([0, 2, , 3]);
                                 return [4 /*yield*/, superagent.get(apiUrl)
                                         .auth(token, '')];
                             case 1:
-                                _a = (_b.sent()).body, trialResult = _a.trialResult, result = _a.result;
-                                if (result || trialResult) {
+                                body = (_a.sent()).body;
+                                trialResult = body.trialResult, result = body.result, isRunning = body.isRunning;
+                                if (result || trialResult || isRunning === false) {
                                     clearInterval(intervalId);
-                                    testResult = tslib_1.__assign({}, testDef, { url: webUrl, passed: testDef.type === TYPE_LOAD ?
-                                            result === 'done' : isFunctionalPassed(trialResult) });
+                                    testResult = tslib_1.__assign({}, testDef, { url: webUrl, passed: isTestPassed(body, testDef.type) });
                                     if (callback) {
@@ -91,4 +88,4 @@ callback(null, testResult);
                             case 2:
-                                err_1 = _b.sent();
-                                if (testDef.type === TYPE_FUNCTIONAL && err_1.status === 404) {
+                                err_1 = _a.sent();
+                                if (testDef.type === Loadmill.TYPES.FUNCTIONAL && err_1.status === 404) {
                                     // 404 for functional could be fine when async - keep going:
@@ -138,3 +135,3 @@ return [2 /*return*/];
                                     return [2 /*return*/, {
-                                            type: TYPE_FUNCTIONAL,
+                                            type: Loadmill.TYPES.FUNCTIONAL,
                                             passed: isFunctionalPassed(trialRes),
@@ -172,3 +169,3 @@ description: description
                                                 id: id,
-                                                type: TYPE_FUNCTIONAL,
+                                                type: Loadmill.TYPES.FUNCTIONAL,
                                                 url: testingServer + "/app/functional/" + id,
@@ -186,2 +183,22 @@ passed: async ? null : isFunctionalPassed(trialResult),
     }
+    function _runTestSuite(suite, paramsOrCallback, callback) {
+        return tslib_1.__awaiter(this, void 0, void 0, function () {
+            var _this = this;
+            return tslib_1.__generator(this, function (_a) {
+                return [2 /*return*/, wrap(function () { return tslib_1.__awaiter(_this, void 0, void 0, function () {
+                        var testSuiteRunId;
+                        return tslib_1.__generator(this, function (_a) {
+                            switch (_a.label) {
+                                case 0: return [4 /*yield*/, superagent.post(testingServer + "/api/test-suites/" + suite.id + "/run")
+                                        .send({})
+                                        .auth(token, '')];
+                                case 1:
+                                    testSuiteRunId = (_a.sent()).body.testSuiteRunId;
+                                    return [2 /*return*/, { id: testSuiteRunId, type: Loadmill.TYPES.SUITE }];
+                            }
+                        });
+                    }); }, callback || paramsOrCallback)];
+            });
+        });
+    }
     return {
@@ -261,2 +278,6 @@ run: function (config, paramsOrCallback, callback) {
             return _runFunctional(config, true, paramsOrCallback, callback);
+        },
+        runTestSuite: function (suiteId, paramsOrCallback, callback) {
+            var suite = { id: suiteId };
+            return _runTestSuite(suite, paramsOrCallback, callback);
         }
@@ -268,7 +289,36 @@ };
 }
-function getTestUrl(_a, prefix, funcSuffix, loadSuffix) {
+var isTestPassed = function (body, type) {
+    switch (type) {
+        case Loadmill.TYPES.FUNCTIONAL:
+            return isFunctionalPassed(body.trialResult);
+        case Loadmill.TYPES.SUITE:
+            return body.isPassed;
+        default://load
+            return body.result === 'done';
+    }
+};
+function getTestAPIUrl(_a, server) {
     var id = _a.id, type = _a.type;
-    var suffix = type === TYPE_FUNCTIONAL ? funcSuffix : loadSuffix;
-    return "" + prefix + suffix + id;
+    var prefix = server + "/api";
+    switch (type) {
+        case Loadmill.TYPES.FUNCTIONAL:
+            return prefix + "/tests/trials/" + id;
+        case Loadmill.TYPES.SUITE:
+            return prefix + "/test-suites-runs/" + id;
+        default://load
+            return prefix + "/tests/" + id;
+    }
 }
+function getTestWebUrl(_a, server) {
+    var id = _a.id, type = _a.type;
+    var prefix = server + "/app";
+    switch (type) {
+        case Loadmill.TYPES.FUNCTIONAL:
+            return prefix + "/functional/" + id;
+        case Loadmill.TYPES.SUITE:
+            return prefix + "/api-tests/test-suite-runs/" + id;
+        default://load
+            return prefix + "/test/" + id;
+    }
+}
 function wrap(asyncFunction, paramsOrCallback) {
@@ -302,2 +352,12 @@ var promise = asyncFunction();
 }
+(function (Loadmill) {
+    var TYPES;
+    (function (TYPES) {
+        TYPES["LOAD"] = "load";
+        TYPES["FUNCTIONAL"] = "functional";
+        TYPES["SUITE"] = "test-suite";
+        TYPES["LOCAL"] = "local";
+    })(TYPES = Loadmill.TYPES || (Loadmill.TYPES = {}));
+    ;
+})(Loadmill || (Loadmill = {}));
 module.exports = Loadmill;

lib/loadmill.js

@@ -8,4 +8,4 @@ "use strict";
 program
-    .usage("<config-file> -t <token> [options] [parameter=value...]")
-    .description("Run a load test or a functional test on loadmill.com.\n  " +
+    .usage("<config-file-or-folder | testSuiteId> -t <token> [options] [parameter=value...]")
+    .description("Run a load test or a test suite on loadmill.com.\n  " +
     "You may set parameter values by passing space-separated 'name=value' pairs, e.g. 'host=www.myapp.com port=80'.\n\n  " +
@@ -15,2 +15,3 @@ "Learn more at https://www.npmjs.com/package/loadmill#cli")
     .option("-l, --load-test", "Launch a load test. If not set, a functional test will run instead.")
+    .option("-s, --test-suite", "Launch a test suite. If set then a test suite id must be provided instead of config file.")
     .option("-a, --async", "Run the test asynchronously - affects only functional tests. " +
@@ -32,11 +33,8 @@ "Use this if your test can take longer than 25 seconds (otherwise it will timeout).")
     return tslib_1.__awaiter(this, void 0, void 0, function () {
-        var wait, bail, async, quiet, token, verbose, colors, local, loadTest, _a, fileOrFolder, rawParams, logger, parameters, loadmill, listOfFiles, _i, listOfFiles_1, file, res, id, method;
+        var wait, bail, async, quiet, token, verbose, colors, local, loadTest, testSuite, _a, input, rawParams, logger, parameters, loadmill, res, running, testSuiteRunId, fileOrFolder, listOfFiles, _i, listOfFiles_1, file, res, id, method;
         return tslib_1.__generator(this, function (_b) {
             switch (_b.label) {
                 case 0:
-                    wait = program.wait, bail = program.bail, async = program.async, quiet = program.quiet, token = program.token, verbose = program.verbose, colors = program.colors, local = program.local, loadTest = program.loadTest, _a = program.args, fileOrFolder = _a[0], rawParams = _a.slice(1);
+                    wait = program.wait, bail = program.bail, async = program.async, quiet = program.quiet, token = program.token, verbose = program.verbose, colors = program.colors, local = program.local, loadTest = program.loadTest, testSuite = program.testSuite, _a = program.args, input = _a[0], rawParams = _a.slice(1);
                     logger = new utils_1.Logger(verbose, colors);
-                    if (!fileOrFolder) {
-                        validationFailed("No configuration file or folder were provided.");
-                    }
                     if (!token) {
@@ -49,4 +47,4 @@ validationFailed("No API token provided.");
                         quiet = false;
-                        logger.log("Input:", {
-                            fileOrFolder: fileOrFolder,
+                        logger.log("Inputs:", {
+                            input: input,
                             wait: wait,
@@ -63,2 +61,41 @@ bail: bail,
                     loadmill = Loadmill({ token: token });
+                    if (!testSuite) return [3 /*break*/, 6];
+                    if (!utils_1.isUUID(input)) {
+                        validationFailed("Test suite run flag is on but no valid test suite id was provided.");
+                    }
+                    res = void 0;
+                    return [4 /*yield*/, loadmill.runTestSuite(input, parameters)];
+                case 1:
+                    running = _b.sent();
+                    if (!(running && running.id)) return [3 /*break*/, 4];
+                    testSuiteRunId = running.id;
+                    if (!wait) return [3 /*break*/, 3];
+                    logger.verbose("Waiting for test suite:", testSuiteRunId);
+                    return [4 /*yield*/, loadmill.wait(running)];
+                case 2:
+                    res = _b.sent();
+                    _b.label = 3;
+                case 3:
+                    if (!quiet) {
+                        logger.log(res ? utils_1.getObjectAsString(res, colors) : testSuiteRunId);
+                    }
+                    if (res && res.passed != null && !res.passed) {
+                        logger.error("\u274C  Test suite with id " + input + " failed.");
+                        if (bail) {
+                            process.exit(1);
+                        }
+                    }
+                    return [3 /*break*/, 5];
+                case 4:
+                    logger.error("\u274C  Couldn't run test suite with id " + input + ".");
+                    if (bail) {
+                        process.exit(1);
+                    }
+                    _b.label = 5;
+                case 5: return [3 /*break*/, 17];
+                case 6:
+                    fileOrFolder = input;
+                    if (!fileOrFolder) {
+                        validationFailed("No configuration file or folder were provided.");
+                    }
                     listOfFiles = utils_1.getJSONFilesInFolderRecursively(fileOrFolder);
@@ -69,35 +106,35 @@ if (listOfFiles.length === 0) {
                     _i = 0, listOfFiles_1 = listOfFiles;
-                    _b.label = 1;
-                case 1:
-                    if (!(_i < listOfFiles_1.length)) return [3 /*break*/, 11];
+                    _b.label = 7;
+                case 7:
+                    if (!(_i < listOfFiles_1.length)) return [3 /*break*/, 17];
                     file = listOfFiles_1[_i];
                     res = void 0, id = void 0;
-                    if (!local) return [3 /*break*/, 3];
+                    if (!local) return [3 /*break*/, 9];
                     logger.verbose("Running " + file + " as functional test locally");
                     return [4 /*yield*/, loadmill.runFunctionalLocally(file, parameters, undefined, { verbose: verbose, colors: colors })];
-                case 2:
+                case 8:
                     res = _b.sent();
-                    return [3 /*break*/, 7];
-                case 3:
-                    if (!loadTest) return [3 /*break*/, 5];
+                    return [3 /*break*/, 13];
+                case 9:
+                    if (!loadTest) return [3 /*break*/, 11];
                     logger.verbose("Launching " + file + " as load test");
                     return [4 /*yield*/, loadmill.run(file, parameters)];
-                case 4:
+                case 10:
                     id = _b.sent();
-                    return [3 /*break*/, 7];
-                case 5:
+                    return [3 /*break*/, 13];
+                case 11:
                     logger.verbose("Running " + file + " as functional test");
                     method = async ? 'runAsyncFunctional' : 'runFunctional';
                     return [4 /*yield*/, loadmill[method](file, parameters)];
-                case 6:
+                case 12:
                     res = _b.sent();
-                    _b.label = 7;
-                case 7:
-                    if (!(wait && (loadTest || async))) return [3 /*break*/, 9];
+                    _b.label = 13;
+                case 13:
+                    if (!(wait && (loadTest || async))) return [3 /*break*/, 15];
                     logger.verbose("Waiting for test:", res ? res.id : id);
                     return [4 /*yield*/, loadmill.wait(res || id)];
-                case 8:
+                case 14:
                     res = _b.sent();
-                    _b.label = 9;
-                case 9:
+                    _b.label = 15;
+                case 15:
                     if (!quiet) {
@@ -112,7 +149,7 @@ logger.log(JSON.stringify(res, null, 4) || id);
                     }
-                    _b.label = 10;
-                case 10:
+                    _b.label = 16;
+                case 16:
                     _i++;
-                    return [3 /*break*/, 1];
-                case 11: return [2 /*return*/];
+                    return [3 /*break*/, 7];
+                case 17: return [2 /*return*/];
             }
@@ -119,0 +156,0 @@ });

lib/utils.js

@@ -32,3 +32,3 @@ "use strict";
 };
-var getObjectAsString = function (obj, colors) {
+exports.getObjectAsString = function (obj, colors) {
     // trim response body to length of 255
@@ -43,3 +43,3 @@ if (obj.response && obj.response.text && obj.response.text.length > 1024) {
         logger.error('Test failure response -');
-        logger.log(getObjectAsString(trialRes, testArgs.colors));
+        logger.log(exports.getObjectAsString(trialRes, testArgs.colors));
     }
@@ -49,3 +49,3 @@ else {
         for (var requestIndex in assertionErrorsPerRequest) {
-            logger.log(getObjectAsString(trialRes.resolvedRequests[requestIndex], testArgs && testArgs.colors));
+            logger.log(exports.getObjectAsString(trialRes.resolvedRequests[requestIndex], testArgs && testArgs.colors));
         }
@@ -117,2 +117,5 @@ }
 exports.isString = function (obj) { return isAString(obj); };
+exports.isUUID = function (s) {
+    return /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/.test(s);
+};
 var Logger = /** @class */ (function () {
@@ -119,0 +122,0 @@ function Logger(verbose, colors) {

package.json

 {
   "name": "loadmill",
-  "version": "0.5.5",
+  "version": "1.0.0",
   "description": "A node.js module for running load tests and functional tests on loadmill.com",
@@ -5,0 +5,0 @@ "keywords": [

README.md

@@ -124,2 +124,13 @@ # Loadmill
 
+### Test Suites
+
+You may also launch an existing test suite by supplying the suite id - this is usually useful for testing your API for regressions after every new deployment.
+Test suites are launched and not awaiting the results.
+You can explicitly wait for a test to finish using the `wait` function:
+```js
+loadmill.runTestSuite("test-suite-uuid")
+    // -> [{id: string}]
+    .then(result => console.log(result));
+```
+
 ### Parameters
@@ -138,2 +149,4 @@
 
+**NOTE** - currently `run test suite` option doesn't accept parameters.
+
 ## CLI
@@ -143,3 +156,3 @@
 ```
-loadmill <config-file-or-folder> -t <token> [options] [parameter=value...]
+loadmill <config-file-or-folder | test-suite-id> -t <token> [options] [parameter=value...]
 ```
@@ -179,2 +192,13 @@
 
+### Test suites
+
+You may launch a test suite by setting the `-s` or `--test-suite` option:
+```
+loadmill test-suite-id --test-suite -t DW2rTlkNmE6A3ax5LVTSDxv2Jfw4virjQpmbOaLG
+```
+
+The test suite will be launched and its unique identifier will be printed to the standard output. You may alternatively
+set the `-w` or `--wait` option in order to wait for the load test to finish, in which case only the result JSON will be
+printed out at the end
+
 ### Exit Status
@@ -199,2 +223,3 @@
 - `-l, --load-test` Launch a load test. If not set, a functional test will run instead.
+- `-s, --test-suite` Launch a test suite. If set then a test suite id must be provided instead of config file..
 - `-a, --async` Run the test asynchronously - affects only functional tests. Use this if your test can take longer than 25 seconds (otherwise it will timeout).
@@ -201,0 +226,0 @@ - `-w, --wait` Wait for the test to finish. Functional tests are automatically waited on unless async flag is turned on.

src/index.ts

 import './polyfills'
 import * as fs from 'fs';
 import * as superagent from 'superagent';
-import {getJSONFilesInFolderRecursively, isEmptyObj, isString, checkAndPrintErrors, Logger} from './utils';
-import {runFunctionalOnLocalhost} from 'loadmill-runner';
+import { getJSONFilesInFolderRecursively, isEmptyObj, isString, checkAndPrintErrors, Logger } from './utils';
+import { runFunctionalOnLocalhost } from 'loadmill-runner';
 
 export = Loadmill;
 
-namespace Loadmill {
-    export interface LoadmillOptions {
-        token: string;
-    }
-
-    export interface TestDef {
-        id: string;
-        type: string;
-    }
-
-    export interface TestResult extends TestDef {
-        url: string;
-        passed: boolean;
-        descrption: string
-    }
-
-    export type Configuration = object | string | any ; // todo: bad typescript
-    export type ParamsOrCallback = object | Callback;
-    export type Callback = {(err: Error | null, result: any): void} | undefined;
-    export type Histogram = {[reason: string]: number};
-    export type TestFailures = {[reason: string]: {[histogram: string]: Histogram}};
-    export type Args = {verbose: boolean, colors?: boolean};
-}
-
-const TYPE_LOAD = 'load';
-const TYPE_FUNCTIONAL = 'functional';
-const LOCAL = 'local';
-
 function Loadmill(options: Loadmill.LoadmillOptions) {
     const {
         token,
-        _testingServerHost = "www.loadmill.com"
+        _testingServerHost = process.env.LOADMILL_SERVER_HOST || "www.loadmill.com"
     } = options as any;
@@ -52,7 +24,7 @@
 
-        for (let file of listOfFiles)  {
+        for (let file of listOfFiles) {
             let res = await execFunc(file, ...funcArgs);
             let testResult;
             if (!isString(res) && !res.id) { // obj but without id -> local test
-                testResult = {url: LOCAL, passed: res.passed} as Loadmill.TestResult;
+                testResult = { url: Loadmill.TYPES.LOCAL, passed: res.passed } as Loadmill.TestResult;
             } else { // obj with id -> functional test. id as string -> load test
@@ -73,50 +45,49 @@ testResult = await _wait(res);
             id: testDefOrId,
-            type: TYPE_LOAD,
+            type: Loadmill.TYPES.LOAD,
         } : testDefOrId;
 
-        const apiUrl = getTestUrl(testDef,
-            testingServer + '/api/tests/', 'trials/', '');
+        const apiUrl = getTestAPIUrl(testDef, testingServer);
 
-        const webUrl = getTestUrl(testDef,
-            testingServer + '/app/', 'functional/', 'test/');
+        const webUrl = getTestWebUrl(testDef, testingServer);
 
         const intervalId = setInterval(async () => {
-                try {
-                    const {body: {trialResult, result}} = await superagent.get(apiUrl)
-                        .auth(token, '');
+            try {
+                const { body } = await superagent.get(apiUrl)
+                    .auth(token, '');
 
-                    if (result || trialResult) {
-                        clearInterval(intervalId);
+                const { trialResult, result, isRunning } = body;
 
-                        const testResult = {
-                            ...testDef,
-                            url: webUrl,
-                            passed: testDef.type === TYPE_LOAD ?
-                                result === 'done' : isFunctionalPassed(trialResult),
-                        };
+                if (result || trialResult || isRunning === false) {
+                    clearInterval(intervalId);
 
-                        if (callback) {
-                            callback(null, testResult);
-                        }
-                        else {
-                            resolve(testResult);
-                        }
-                    }
-                }
-                catch (err) {
-                    if (testDef.type === TYPE_FUNCTIONAL && err.status === 404) {
-                        // 404 for functional could be fine when async - keep going:
-                        return;
-                    }
+                    const testResult = {
+                        ...testDef,
+                        url: webUrl,
+                        passed: isTestPassed(body, testDef.type),
+                    };
 
-                    clearInterval(intervalId);
-
                     if (callback) {
-                        callback(err, null);
+                        callback(null, testResult);
                     }
                     else {
-                        reject(err);
+                        resolve(testResult);
                     }
                 }
-            },
+            }
+            catch (err) {
+                if (testDef.type === Loadmill.TYPES.FUNCTIONAL && err.status === 404) {
+                    // 404 for functional could be fine when async - keep going:
+                    return;
+                }
+
+                clearInterval(intervalId);
+
+                if (callback) {
+                    callback(err, null);
+                }
+                else {
+                    reject(err);
+                }
+            }
+        },
             10 * 1000);
@@ -149,7 +120,7 @@
                 if (!isEmptyObj(trialRes.failures)) {
-                  checkAndPrintErrors(trialRes, testArgs, logger, description);
+                    checkAndPrintErrors(trialRes, testArgs, logger, description);
                 }
 
                 return {
-                    type: TYPE_FUNCTIONAL,
+                    type: Loadmill.TYPES.FUNCTIONAL,
                     passed: isFunctionalPassed(trialRes),
@@ -193,3 +164,3 @@ description: description
                         id,
-                        type: TYPE_FUNCTIONAL,
+                        type: Loadmill.TYPES.FUNCTIONAL,
                         url: `${testingServer}/app/functional/${id}`,
@@ -205,2 +176,24 @@ passed: async ? null : isFunctionalPassed(trialResult),
 
+    async function _runTestSuite(
+        suite: Loadmill.TestSuiteDef,
+        paramsOrCallback: Loadmill.ParamsOrCallback,
+        callback: Loadmill.Callback) {
+
+        return wrap(
+            async () => {
+                const {
+                    body: {
+                        testSuiteRunId
+                    }
+                } = await superagent.post(`${testingServer}/api/test-suites/${suite.id}/run`)
+                    .send({})
+                    .auth(token, '');
+
+                return {id: testSuiteRunId, type: Loadmill.TYPES.SUITE};
+
+            },
+            callback || paramsOrCallback
+        );
+    }
+
     return {
@@ -216,3 +209,3 @@ run(
 
-                    const {body: {testId}} = await superagent.post(testingServer + "/api/tests")
+                    const { body: { testId } } = await superagent.post(testingServer + "/api/tests")
                         .send(config)
@@ -244,3 +237,3 @@ .auth(token, '');
         wait(testDefOrId: string | Loadmill.TestDef, callback?: Loadmill.Callback): Promise<Loadmill.TestResult> {
-           return _wait(testDefOrId, callback);
+            return _wait(testDefOrId, callback);
         },
@@ -270,5 +263,5 @@
         async runFunctionalLocally(config: Loadmill.Configuration,
-                                   paramsOrCallback?: Loadmill.ParamsOrCallback,
-                                   callback?: Loadmill.Callback,
-                                   testArgs?: Loadmill.Args): Promise<Loadmill.TestResult> {
+            paramsOrCallback?: Loadmill.ParamsOrCallback,
+            callback?: Loadmill.Callback,
+            testArgs?: Loadmill.Args): Promise<Loadmill.TestResult> {
             return _runFunctionalLocally(config, paramsOrCallback, callback, testArgs);
@@ -295,4 +288,13 @@ },
 
-            return _runFunctional(config,true, paramsOrCallback, callback);
+            return _runFunctional(config, true, paramsOrCallback, callback);
         },
+
+        runTestSuite(
+            suiteId: string,
+            paramsOrCallback?: Loadmill.ParamsOrCallback,
+            callback?: Loadmill.Callback): Promise<Loadmill.TestDef> {
+
+            const suite = { id: suiteId };
+            return _runTestSuite(suite, paramsOrCallback, callback);
+        },
     };
@@ -305,7 +307,37 @@ }
 
-function getTestUrl({id, type}: Loadmill.TestDef, prefix: string, funcSuffix: string, loadSuffix: string) {
-    const suffix = type === TYPE_FUNCTIONAL ? funcSuffix : loadSuffix;
-    return `${prefix}${suffix}${id}`
+const isTestPassed = (body, type) => {
+    switch (type) {
+        case Loadmill.TYPES.FUNCTIONAL:
+            return isFunctionalPassed(body.trialResult);
+        case Loadmill.TYPES.SUITE:
+            return body.isPassed;
+        default: //load
+            return body.result === 'done';
+    }
 }
 
+function getTestAPIUrl({ id, type }: Loadmill.TestDef, server: string) {
+    const prefix = `${server}/api`;
+    switch (type) {
+        case Loadmill.TYPES.FUNCTIONAL:
+            return `${prefix}/tests/trials/${id}`
+        case Loadmill.TYPES.SUITE:
+            return `${prefix}/test-suites-runs/${id}`
+        default: //load
+            return `${prefix}/tests/${id}`;
+    }
+}
+
+function getTestWebUrl({ id, type }: Loadmill.TestDef, server: string) {
+    const prefix = `${server}/app`;
+    switch (type) {
+        case Loadmill.TYPES.FUNCTIONAL:
+            return `${prefix}/functional/${id}`
+        case Loadmill.TYPES.SUITE:
+            return `${prefix}/api-tests/test-suite-runs/${id}`
+        default: //load
+            return `${prefix}/test/${id}`
+    }
+}
+
 function wrap(asyncFunction, paramsOrCallback?: Loadmill.ParamsOrCallback) {
@@ -345,1 +377,36 @@ const promise = asyncFunction();
 }
+
+namespace Loadmill {
+    export interface LoadmillOptions {
+        token: string;
+    }
+
+    export interface TestDef {
+        id: string;
+        type: string;
+    }
+
+    export interface TestSuiteDef {
+        id: string;
+    }
+
+    export interface TestResult extends TestDef {
+        url: string;
+        passed: boolean;
+        descrption: string
+    }
+
+    export type Configuration = object | string | any; // todo: bad typescript
+    export type ParamsOrCallback = object | Callback;
+    export type Callback = { (err: Error | null, result: any): void } | undefined;
+    export type Histogram = { [reason: string]: number };
+    export type TestFailures = { [reason: string]: { [histogram: string]: Histogram } };
+    export type Args = { verbose: boolean, colors?: boolean };
+
+    export enum TYPES {
+        LOAD = 'load',
+        FUNCTIONAL = 'functional',
+        SUITE = 'test-suite',
+        LOCAL = 'local'
+    };
+}

src/loadmill.ts

 import * as Loadmill from './index';
 import * as program from 'commander';
-import {getJSONFilesInFolderRecursively, Logger} from './utils';
+import { getJSONFilesInFolderRecursively, Logger, isUUID, getObjectAsString } from './utils';
 
 program
-    .usage("<config-file> -t <token> [options] [parameter=value...]")
+    .usage("<config-file-or-folder | testSuiteId> -t <token> [options] [parameter=value...]")
     .description(
-        "Run a load test or a functional test on loadmill.com.\n  " +
+        "Run a load test or a test suite on loadmill.com.\n  " +
         "You may set parameter values by passing space-separated 'name=value' pairs, e.g. 'host=www.myapp.com port=80'.\n\n  " +
@@ -14,2 +14,3 @@ "Learn more at https://www.npmjs.com/package/loadmill#cli"
     .option("-l, --load-test", "Launch a load test. If not set, a functional test will run instead.")
+    .option("-s, --test-suite", "Launch a test suite. If set then a test suite id must be provided instead of config file.")
     .option("-a, --async", "Run the test asynchronously - affects only functional tests. " +
@@ -44,3 +45,4 @@ "Use this if your test can take longer than 25 seconds (otherwise it will timeout).")
         loadTest,
-        args: [fileOrFolder, ...rawParams]
+        testSuite,
+        args: [input, ...rawParams]
     } = program;
@@ -50,6 +52,2 @@
 
-    if (!fileOrFolder) {
-        validationFailed("No configuration file or folder were provided.");
-    }
-
     if (!token) {
@@ -65,4 +63,4 @@ validationFailed("No API token provided.");
 
-        logger.log("Input:", {
-            fileOrFolder,
+        logger.log("Inputs:", {
+            input,
             wait,
@@ -79,40 +77,84 @@ bail,
 
-    const loadmill = Loadmill({token});
+    const loadmill = Loadmill({ token });
 
-    const listOfFiles = getJSONFilesInFolderRecursively(fileOrFolder);
-    if (listOfFiles.length === 0) {
-        logger.log(`No Loadmill test files were found at ${fileOrFolder} - exiting...`);
-    }
+    if (testSuite) {
+        if (!isUUID(input)) { //if test suite flag is on then the input should be uuid
+            validationFailed("Test suite run flag is on but no valid test suite id was provided.");
+        }
+        let res;
+        let running = await loadmill.runTestSuite(input, parameters);
 
-    for (let file of listOfFiles)  {
-        let res, id;
+        if (running && running.id) {
 
-        if(local) {
-            logger.verbose(`Running ${file} as functional test locally`);
-            res = await loadmill.runFunctionalLocally(file, parameters, undefined, {verbose, colors});
+            const testSuiteRunId = running.id;
+            
+            if (wait) {
+                logger.verbose("Waiting for test suite:", testSuiteRunId);
+                res = await loadmill.wait(running);
+            }
+            
+            if (!quiet) {
+                logger.log(res ? getObjectAsString(res, colors) : testSuiteRunId);
+            }
+
+            if (res && res.passed != null && !res.passed) {
+                logger.error(`❌  Test suite with id ${input} failed.`);
+
+                if (bail) {
+                    process.exit(1);
+                }
+            }
+
         } else {
-            if (loadTest) {
-                logger.verbose(`Launching ${file} as load test`);
-                id = await loadmill.run(file, parameters);
-            } else {
-                logger.verbose(`Running ${file} as functional test`);
-                const method = async ? 'runAsyncFunctional' : 'runFunctional';
-                res = await loadmill[method](file, parameters);
+            logger.error(`❌  Couldn't run test suite with id ${input}.`);
+
+            if (bail) {
+                process.exit(1);
             }
         }
-        if (wait && (loadTest || async)) {
-            logger.verbose("Waiting for test:", res ? res.id : id);
-            res = await loadmill.wait(res || id);
+
+    } else { // if test suite flag is off then the input should be fileOrFolder
+
+        const fileOrFolder = input;
+        if (!fileOrFolder) {
+            validationFailed("No configuration file or folder were provided.");
         }
 
-        if (!quiet) {
-            logger.log(JSON.stringify(res, null, 4) || id);
+        const listOfFiles = getJSONFilesInFolderRecursively(fileOrFolder);
+        if (listOfFiles.length === 0) {
+            logger.log(`No Loadmill test files were found at ${fileOrFolder} - exiting...`);
         }
 
-        if (res && res.passed != null && !res.passed) {
-            logger.error(`❌  Test ${file} failed.`);
+        for (let file of listOfFiles) {
+            let res, id;
 
-            if (bail) {
-                process.exit(1);
+            if (local) {
+                logger.verbose(`Running ${file} as functional test locally`);
+                res = await loadmill.runFunctionalLocally(file, parameters, undefined, { verbose, colors });
+            } else {
+                if (loadTest) {
+                    logger.verbose(`Launching ${file} as load test`);
+                    id = await loadmill.run(file, parameters);
+                } else {
+                    logger.verbose(`Running ${file} as functional test`);
+                    const method = async ? 'runAsyncFunctional' : 'runFunctional';
+                    res = await loadmill[method](file, parameters);
+                }
             }
+            if (wait && (loadTest || async)) {
+                logger.verbose("Waiting for test:", res ? res.id : id);
+                res = await loadmill.wait(res || id);
+            }
+
+            if (!quiet) {
+                logger.log(JSON.stringify(res, null, 4) || id);
+            }
+
+            if (res && res.passed != null && !res.passed) {
+                logger.error(`❌  Test ${file} failed.`);
+
+                if (bail) {
+                    process.exit(1);
+                }
+            }
         }
@@ -124,3 +166,3 @@ }
     console.log('');
-    console.error(... args);
+    console.error(...args);
     program.outputHelp();
@@ -127,0 +169,0 @@ process.exit(3);

src/utils.ts

@@ -38,3 +38,3 @@ import * as fs from "fs";
 
-const getObjectAsString = (obj, colors) => {
+export const getObjectAsString = (obj, colors) => {
     // trim response body to length of 255
@@ -134,3 +134,5 @@ if (obj.response && obj.response.text && obj.response.text.length > 1024) {
 export const isString = (obj) => isAString(obj);
-
+export const isUUID = s =>
+    /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/.test(s);
+  
 export class Logger {
@@ -137,0 +139,0 @@ private readonly verb: boolean = false;

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

No v1

Quality

Package is not semver >=1. This means it is not stable and does not support ^ ranges.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

107174

increased by10.25%

Number of package files

17

increased by6.25%

Lines of code

1417

increased by18.08%

Number of low quality alerts

1

decreased by-50%

Number of lines in readme file

227

increased by12.38%

Worsened metrics

Number of low supply chain risk alerts

4

increased by33.33%

No dependency changes