@qualweb/core - npm Package Compare versions

Comparing version 0.3.20 to 0.4.0

dist/index.js

@@ -6,2 +6,3 @@ 'use strict';
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateEarlReport = exports.evaluate = exports.close = exports.start = void 0;
 const system_1 = __importDefault(require("./system"));
@@ -8,0 +9,0 @@ const system = new system_1.default();

dist/lib/constants.js

 'use strict';
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_MOBILE_PAGE_VIEWPORT_HEIGHT = exports.DEFAULT_MOBILE_PAGE_VIEWPORT_WIDTH = exports.DEFAULT_DESKTOP_PAGE_VIEWPORT_HEIGHT = exports.DEFAULT_DESKTOP_PAGE_VIEWPORT_WIDTH = exports.DEFAULT_MOBILE_USER_AGENT = exports.DEFAULT_DESKTOP_USER_AGENT = void 0;
 exports.DEFAULT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firefox/22.0';
@@ -4,0 +5,0 @@ exports.DEFAULT_MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; U; Android 2.2; en-us; DROID2 GLOBAL Build/S273) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1';

dist/lib/managers/module.manager.js

@@ -6,8 +6,9 @@ 'use strict';
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluateHtml = exports.evaluateUrl = void 0;
 const crypto_1 = require("crypto");
-const act_rules_1 = require("@qualweb/act-rules");
-const html_techniques_1 = require("@qualweb/html-techniques");
 const css_techniques_1 = require("@qualweb/css-techniques");
-const best_practices_1 = require("@qualweb/best-practices");
+const node_fetch_1 = __importDefault(require("node-fetch"));
 const evaluation_object_1 = __importDefault(require("../data/evaluation.object"));
+const util_1 = require("@qualweb/util");
+let endpoint = 'http://194.117.20.242/validate/';
 function parseUrl(url, pageUrl) {
@@ -79,18 +80,48 @@ let inputUrl = url;
     const evaluation = new evaluation_object_1.default(evaluator);
-    const promises = new Array();
-    const act = new act_rules_1.ACTRules();
-    const html = new html_techniques_1.HTMLTechniques();
+    const reports = new Array();
     const css = new css_techniques_1.CSSTechniques();
-    const bp = new best_practices_1.BestPractices();
+    await page.addScriptTag({
+        path: require.resolve('../../../node_modules/@qualweb/qw-page/dist/qwPage.js')
+    });
     if (execute.act) {
-        if (options['act-rules']) {
-            act.configure(options['act-rules']);
-        }
-        promises.push(act.execute(sourceHtml, page, stylesheets));
+        await page.addScriptTag({
+            path: require.resolve('@qualweb/act-rules')
+        });
+        sourceHtml.html.parsed = [];
+        const actReport = await page.evaluate((sourceHtml, stylesheets, options) => {
+            const act = new ACTRules.ACTRules();
+            if (options)
+                act.configure(options);
+            const report = act.execute(sourceHtml, new QWPage.QWPage(document), stylesheets);
+            return report;
+        }, sourceHtml, stylesheets, options['act-rules']);
+        reports.push(actReport);
     }
     if (execute.html) {
-        if (options['html-techniques']) {
-            html.configure(options['html-techniques']);
+        await page.addScriptTag({
+            path: require.resolve('@qualweb/html-techniques')
+        });
+        const url = page.url();
+        const urlVal = await page.evaluate(() => {
+            return location.href;
+        });
+        const validationUrl = endpoint + encodeURIComponent(urlVal);
+        let response, validation;
+        try {
+            response = await node_fetch_1.default(validationUrl);
         }
-        promises.push(html.execute(page));
+        catch (err) {
+            console.log(err);
+        }
+        if (response && response.status === 200)
+            validation = JSON.parse(await response.json());
+        const newTabWasOpen = await util_1.BrowserUtils.detectIfUnwantedTabWasOpened(page.browser(), url);
+        const htmlReport = await page.evaluate((newTabWasOpen, validation, options) => {
+            const html = new HTMLTechniques.HTMLTechniques();
+            if (options)
+                html.configure(options);
+            const report = html.execute(new QWPage.QWPage(document), newTabWasOpen, validation);
+            return report;
+        }, newTabWasOpen, validation, options['html-techniques']);
+        reports.push(htmlReport);
     }
@@ -101,11 +132,17 @@ if (execute.css) {
         }
-        promises.push(css.execute(stylesheets, mappedDOM));
+        reports.push(await css.execute(stylesheets, mappedDOM));
     }
     if (execute.bp) {
-        if (options['best-practices']) {
-            bp.configure(options['best-practices']);
-        }
-        promises.push(bp.execute(page, stylesheets));
+        await page.addScriptTag({
+            path: require.resolve('@qualweb/best-practices')
+        });
+        const bpReport = await page.evaluate((options) => {
+            const bp = new BestPractices.BestPractices();
+            if (options)
+                bp.configure(options);
+            const report = bp.execute(new QWPage.QWPage(document));
+            return report;
+        }, options['best-practices']);
+        reports.push(bpReport);
     }
-    const reports = await Promise.all(promises);
     for (const report of reports || []) {
@@ -116,7 +153,5 @@ if (report.type === 'wappalyzer') {
         else if (report.type === 'act-rules') {
-            act.resetConfiguration();
             evaluation.addModuleEvaluation('act-rules', report);
         }
         else if (report.type === 'html-techniques') {
-            html.resetConfiguration();
             evaluation.addModuleEvaluation('html-techniques', report);
@@ -129,3 +164,2 @@ }
         else if (report.type === 'best-practices') {
-            bp.resetConfiguration();
             evaluation.addModuleEvaluation('best-practices', report);
@@ -179,18 +213,48 @@ }
     const evaluation = new evaluation_object_1.default(evaluator);
-    const promises = new Array();
-    const act = new act_rules_1.ACTRules();
-    const html = new html_techniques_1.HTMLTechniques();
+    const reports = new Array();
     const css = new css_techniques_1.CSSTechniques();
-    const bp = new best_practices_1.BestPractices();
+    await page.addScriptTag({
+        path: require.resolve('../../../node_modules/@qualweb/qw-page/dist/qwPage.js')
+    });
     if (execute.act) {
-        if (options['act-rules']) {
-            act.configure(options['act-rules']);
-        }
-        promises.push(act.execute(sourceHtml, page, stylesheets));
+        await page.addScriptTag({
+            path: require.resolve('@qualweb/act-rules')
+        });
+        sourceHtml.html.parsed = [];
+        const actReport = await page.evaluate((sourceHtml, stylesheets, options) => {
+            const act = new ACTRules.ACTRules();
+            if (options)
+                act.configure(options);
+            const report = act.execute(sourceHtml, new QWPage.QWPage(document), stylesheets);
+            return report;
+        }, sourceHtml, stylesheets, options['act-rules']);
+        reports.push(actReport);
     }
     if (execute.html) {
-        if (options['html-techniques']) {
-            html.configure(options['html-techniques']);
+        await page.addScriptTag({
+            path: require.resolve('@qualweb/html-techniques')
+        });
+        const url = page.url();
+        const urlVal = await page.evaluate(() => {
+            return location.href;
+        });
+        const validationUrl = endpoint + encodeURIComponent(urlVal);
+        let response, validation;
+        try {
+            response = await node_fetch_1.default(validationUrl);
         }
-        promises.push(html.execute(page));
+        catch (err) {
+            console.log(err);
+        }
+        if (response && response.status === 200)
+            validation = JSON.parse(await response.json());
+        const newTabWasOpen = await util_1.BrowserUtils.detectIfUnwantedTabWasOpened(page.browser(), url);
+        const htmlReport = await page.evaluate((newTabWasOpen, validation, options) => {
+            const html = new HTMLTechniques.HTMLTechniques();
+            if (options)
+                html.configure(options);
+            const report = html.execute(new QWPage.QWPage(document), newTabWasOpen, validation);
+            return report;
+        }, newTabWasOpen, validation, options['html-techniques']);
+        reports.push(htmlReport);
     }
@@ -201,11 +265,17 @@ if (execute.css) {
         }
-        promises.push(css.execute(stylesheets, mappedDOM));
+        reports.push(css.execute(stylesheets, mappedDOM));
     }
     if (execute.bp) {
-        if (options['best-practices']) {
-            bp.configure(options['best-practices']);
-        }
-        promises.push(bp.execute(page, stylesheets));
+        await page.addScriptTag({
+            path: require.resolve('@qualweb/best-practices')
+        });
+        const bpReport = await page.evaluate((options) => {
+            const bp = new BestPractices.BestPractices();
+            if (options)
+                bp.configure(options);
+            const report = bp.execute(new QWPage.QWPage(document));
+            return report;
+        }, options['best-practices']);
+        reports.push(bpReport);
     }
-    const reports = await Promise.all(promises);
     for (const report of reports || []) {
@@ -216,7 +286,5 @@ if (report.type === 'wappalyzer') {
         else if (report.type === 'act-rules') {
-            act.resetConfiguration();
             evaluation.addModuleEvaluation('act-rules', report);
         }
         else if (report.type === 'html-techniques') {
-            html.resetConfiguration();
             evaluation.addModuleEvaluation('html-techniques', report);
@@ -229,3 +297,2 @@ }
         else if (report.type === 'best-practices') {
-            bp.resetConfiguration();
             evaluation.addModuleEvaluation('best-practices', report);
@@ -232,0 +299,0 @@ }

dist/lib/managers/startup.manager.js

@@ -6,2 +6,3 @@ 'use strict';
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.crawlDomain = exports.getFileUrls = void 0;
 const crawler_1 = __importDefault(require("@qualweb/crawler"));
@@ -8,0 +9,0 @@ const fs_1 = __importDefault(require("fs"));

dist/system.js

 'use strict';
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
 var __importStar = (this && this.__importStar) || function (mod) {
     if (mod && mod.__esModule) return mod;
     var result = {};
-    if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
-    result["default"] = mod;
+    if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
     return result;
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 const puppeteer_1 = __importDefault(require("puppeteer"));
@@ -22,2 +34,3 @@ const htmlparser2_1 = require("htmlparser2");
 const css_1 = __importDefault(require("css"));
+const node_fetch_1 = __importDefault(require("node-fetch"));
 const constants_1 = require("./lib/constants");
@@ -58,5 +71,2 @@ class System {
         }
-        if (options.html) {
-            this.html = options.html;
-        }
         if (!this.html && this.urls.length === 0) {
@@ -136,8 +146,9 @@ throw new Error('Invalid input method');
                 });
-                const response = await page.goto(url, {
+                const _sourceHtml = await this.getSourceHtml(url);
+                await page.goto(url, {
                     timeout: 0,
                     waitUntil: ['networkidle2', 'domcontentloaded']
                 });
-                if (response) {
-                    const sourceHtml = await this.parseSourceHTML(await response.text());
+                if (_sourceHtml) {
+                    const sourceHtml = await this.parseSourceHTML(_sourceHtml);
                     const styles = css_select_1.default('style', sourceHtml.html.parsed);
@@ -272,2 +283,12 @@ let k = 0;
     }
+    async getSourceHtml(url, options) {
+        const fetchOptions = {
+            'headers': {
+                'User-Agent': options ? options.userAgent ? options.userAgent : options.mobile ? constants_1.DEFAULT_MOBILE_USER_AGENT : constants_1.DEFAULT_DESKTOP_USER_AGENT : constants_1.DEFAULT_DESKTOP_USER_AGENT
+            }
+        };
+        const response = await node_fetch_1.default(url, fetchOptions);
+        const sourceHTML = (await response.text()).trim();
+        return sourceHTML;
+    }
     async parseStylesheets(plainStylesheets) {
@@ -274,0 +295,0 @@ const stylesheets = new Array();

package.json

 {
   "name": "@qualweb/core",
-  "version": "0.3.20",
+  "version": "0.4.0",
   "description": "QualWeb evaluator core engine",
@@ -49,26 +49,26 @@ "main": "dist/index.js",
   "dependencies": {
-    "@qualweb/act-rules": "^0.3.36",
-    "@qualweb/best-practices": "^0.3.3",
+    "@qualweb/act-rules": "^0.4.1",
+    "@qualweb/best-practices": "^0.4.1",
     "@qualweb/crawler": "^0.1.1",
-    "@qualweb/css-techniques": "^0.3.6",
+    "@qualweb/css-techniques": "^0.3.8",
     "@qualweb/earl-reporter": "^0.2.5",
-    "@qualweb/html-techniques": "^0.3.4",
+    "@qualweb/html-techniques": "^0.4.1",
     "css": "^2.2.4",
     "css-select": "^2.1.0",
     "domhandler": "^3.0.0",
-    "domutils": "^2.0.0",
+    "domutils": "^2.1.0",
     "htmlparser2": "^4.1.0",
     "lodash.clone": "^4.5.0",
     "lodash.clonedeep": "^4.5.0",
-    "puppeteer": "^3.0.3"
+    "puppeteer": "^3.0.4"
   },
   "devDependencies": {
-    "@qualweb/types": "^0.3.30",
-    "@types/node": "^13.13.5",
-    "@types/puppeteer": "^2.0.1",
+    "@types/node": "^13.13.6",
+    "@types/puppeteer": "^2.1.0",
+    "@qualweb/types": "^0.4.0",
     "chai": "^4.2.0",
     "mocha": "^7.1.2",
     "node-fetch": "^2.6.0",
-    "typescript": "^3.8.3"
+    "typescript": "^3.9.2"
   }
 }

src/lib/managers/module.manager.ts

@@ -6,9 +6,10 @@ 'use strict';
 import { Url, QualwebOptions, ProcessedHtml, SourceHtml, CSSStylesheet } from '@qualweb/core';
-import { ACTRules } from '@qualweb/act-rules';
-import { HTMLTechniques } from '@qualweb/html-techniques';
 import { CSSTechniques } from '@qualweb/css-techniques';
-import { BestPractices } from '@qualweb/best-practices';
+import fetch from 'node-fetch';
 
 import Evaluation from '../data/evaluation.object';
+import { BrowserUtils } from '@qualweb/util';
+let endpoint = 'http://194.117.20.242/validate/';
 
+
 function parseUrl(url: string, pageUrl: string): Url {
@@ -26,3 +27,3 @@ let inputUrl = url;
   const tmp = domainName.split('.');
-  domain = tmp[tmp.length-1];
+  domain = tmp[tmp.length - 1];
   uri = completeUrl.split('.' + domain)[1];
@@ -43,3 +44,3 @@
 async function evaluateUrl(url: string, sourceHtml: SourceHtml, page: Page, stylesheets: CSSStylesheet[], mappedDOM: any, execute: any, options: QualwebOptions): Promise<Evaluation> {
-  
+
   const [pageUrl, plainHtml, pageTitle, elements, browserUserAgent] = await Promise.all([
@@ -93,21 +94,59 @@ page.url(),
 
-  const promises = new Array<any>();
-
-  const act = new ACTRules();
-  const html = new HTMLTechniques();
+  const reports = new Array<any>();
   const css = new CSSTechniques();
-  const bp = new BestPractices();
+  await page.addScriptTag({
+    path: require.resolve('../../../node_modules/@qualweb/qw-page/dist/qwPage.js')
+  })
 
   if (execute.act) {
-    if (options['act-rules']) {
-      act.configure(options['act-rules']);
-    }
-    promises.push(act.execute(sourceHtml, page, stylesheets));
+    await page.addScriptTag({
+      path: require.resolve('@qualweb/act-rules')
+    })
+    sourceHtml.html.parsed = [];
+    const actReport = await page.evaluate((sourceHtml, stylesheets, options) => {
+      // @ts-ignore 
+      const act = new ACTRules.ACTRules();
+      if (options)
+        act.configure(options);
+      // @ts-ignore 
+      const report = act.execute(sourceHtml, new QWPage.QWPage(document), stylesheets);
+      return report;
+      // @ts-ignore 
+    }, sourceHtml, stylesheets, options['act-rules']);
+
+    reports.push(actReport);
   }
 
   if (execute.html) {
-    if (options['html-techniques']) {
-      html.configure(options['html-techniques']);
+    await page.addScriptTag({
+      path: require.resolve('@qualweb/html-techniques')
+    })
+    const url = page.url();
+    const urlVal = await page.evaluate(() => {
+      return location.href;
+    });
+
+    const validationUrl = endpoint + encodeURIComponent(urlVal);
+
+    let response, validation;
+
+    try {
+      response = await fetch(validationUrl);
+    } catch (err) {
+      console.log(err);
     }
-    promises.push(html.execute(page));
+    if (response && response.status === 200)
+      validation = JSON.parse(await response.json());
+    const newTabWasOpen = await BrowserUtils.detectIfUnwantedTabWasOpened(page.browser(), url);
+    const htmlReport = await page.evaluate((newTabWasOpen, validation, options) => {
+      // @ts-ignore 
+      const html = new HTMLTechniques.HTMLTechniques();
+      if (options)
+        html.configure(options)
+      // @ts-ignore 
+      const report = html.execute(new QWPage.QWPage(document), newTabWasOpen, validation);
+      return report;
+      // @ts-ignore 
+    }, newTabWasOpen, validation, options['html-techniques']);
+    reports.push(htmlReport);
   }
@@ -119,13 +158,21 @@
     }
-    promises.push(css.execute(stylesheets, mappedDOM));
+    reports.push(await css.execute(stylesheets, mappedDOM));
   }
 
   if (execute.bp) {
-    if (options['best-practices']) {
-      bp.configure(options['best-practices']);
-    }
-    promises.push(bp.execute(page, stylesheets));
+    await page.addScriptTag({
+      path: require.resolve('@qualweb/best-practices')//'../../../node_modules/@qualweb/best-practices/dist/bp.js'
+    })
+    const bpReport = await page.evaluate((options) => {
+      // @ts-ignore 
+      const bp = new BestPractices.BestPractices();
+      if (options)
+        bp.configure(options)
+      // @ts-ignore 
+      const report = bp.execute(new QWPage.QWPage(document));
+      return report;
+    }, options['best-practices']);
+    reports.push(bpReport);
   }
 
-  const reports = await Promise.all(promises);
 
@@ -136,6 +183,4 @@ for (const report of reports || []) {
     } else if (report.type === 'act-rules') {
-      act.resetConfiguration();
       evaluation.addModuleEvaluation('act-rules', report);
     } else if (report.type === 'html-techniques') {
-      html.resetConfiguration();
       evaluation.addModuleEvaluation('html-techniques', report);
@@ -146,3 +191,2 @@ } else if (report.type === 'css-techniques') {
     } else if (report.type === 'best-practices') {
-      bp.resetConfiguration();
       evaluation.addModuleEvaluation('best-practices', report);
@@ -156,3 +200,3 @@ }
 async function evaluateHtml(sourceHtml: SourceHtml, page: Page, stylesheets: CSSStylesheet[], mappedDOM: any, execute: any, options: QualwebOptions): Promise<Evaluation> {
- 
+
   const [plainHtml, pageTitle, elements, browserUserAgent] = await Promise.all([
@@ -203,22 +247,60 @@ page.evaluate(() => {
   const evaluation = new Evaluation(evaluator);
+  const reports = new Array<any>();
+  const css = new CSSTechniques();
 
-  const promises = new Array<any>();
+  await page.addScriptTag({
+    path: require.resolve('../../../node_modules/@qualweb/qw-page/dist/qwPage.js')
+  })
 
-  const act = new ACTRules();
-  const html = new HTMLTechniques();
-  const css = new CSSTechniques();
-  const bp = new BestPractices();
+  if (execute.act) {
+    await page.addScriptTag({
+      path: require.resolve('@qualweb/act-rules')
+    })
+    sourceHtml.html.parsed = [];
+    const actReport = await page.evaluate((sourceHtml, stylesheets, options) => {
+      // @ts-ignore 
+      const act = new ACTRules.ACTRules();
+      if (options)
+        act.configure(options);
+      // @ts-ignore 
+      const report = act.execute(sourceHtml, new QWPage.QWPage(document), stylesheets);
+      return report;
+      // @ts-ignore 
+    }, sourceHtml, stylesheets, options['act-rules']);
 
-  if (execute.act) {
-    if (options['act-rules']) {
-      act.configure(options['act-rules']);
-    }
-    promises.push(act.execute(sourceHtml, page, stylesheets));
+    reports.push(actReport);
   }
 
   if (execute.html) {
-    if (options['html-techniques']) {
-      html.configure(options['html-techniques']);
+    await page.addScriptTag({
+      path: require.resolve('@qualweb/html-techniques')
+    })
+    const url = page.url();
+    const urlVal = await page.evaluate(() => {
+      return location.href;
+    });
+
+    const validationUrl = endpoint + encodeURIComponent(urlVal);
+
+    let response, validation;
+
+    try {
+      response = await fetch(validationUrl);
+    } catch (err) {
+      console.log(err);
     }
-    promises.push(html.execute(page));
+    if (response && response.status === 200)
+      validation = JSON.parse(await response.json());
+    const newTabWasOpen = await BrowserUtils.detectIfUnwantedTabWasOpened(page.browser(), url);
+    const htmlReport = await page.evaluate((newTabWasOpen, validation, options) => {
+      // @ts-ignore 
+      const html = new HTMLTechniques.HTMLTechniques();
+      if (options)
+        html.configure(options)
+      // @ts-ignore 
+      const report = html.execute(new QWPage.QWPage(document), newTabWasOpen, validation);
+      return report;
+      // @ts-ignore 
+    }, newTabWasOpen, validation, options['html-techniques']);
+    reports.push(htmlReport);
   }
@@ -230,13 +312,21 @@
     }
-    promises.push(css.execute(stylesheets, mappedDOM));
+    reports.push(css.execute(stylesheets, mappedDOM));
   }
 
   if (execute.bp) {
-    if (options['best-practices']) {
-      bp.configure(options['best-practices']);
-    }
-    promises.push(bp.execute(page, stylesheets));
+    await page.addScriptTag({
+      path: require.resolve('@qualweb/best-practices')//'../../../node_modules/@qualweb/best-practices/dist/bp.js'
+    })
+    const bpReport = await page.evaluate((options) => {
+      // @ts-ignore 
+      const bp = new BestPractices.BestPractices();
+      if (options)
+        bp.configure(options)
+      // @ts-ignore 
+      const report = bp.execute(new QWPage.QWPage(document));
+      return report;
+    }, options['best-practices']);
+    reports.push(bpReport);
   }
 
-  const reports = await Promise.all(promises);
 
@@ -247,6 +337,4 @@ for (const report of reports || []) {
     } else if (report.type === 'act-rules') {
-      act.resetConfiguration();
       evaluation.addModuleEvaluation('act-rules', report);
     } else if (report.type === 'html-techniques') {
-      html.resetConfiguration();
       evaluation.addModuleEvaluation('html-techniques', report);
@@ -257,3 +345,2 @@ } else if (report.type === 'css-techniques') {
     } else if (report.type === 'best-practices') {
-      bp.resetConfiguration();
       evaluation.addModuleEvaluation('best-practices', report);
@@ -260,0 +347,0 @@ }

src/system.ts

@@ -14,2 +14,3 @@ 'use strict';
 import css from 'css';
+import fetch from 'node-fetch';
 
@@ -71,5 +72,6 @@ import {
 
+    /*FIXME
     if (options.html) {
       this.html = options.html;
-    }
+    }*/
     
@@ -158,3 +160,5 @@ if (!this.html && this.urls.length === 0) {
 
-        const response = await page.goto(url, {
+        const _sourceHtml = await this.getSourceHtml(url);
+
+        await page.goto(url, {
           timeout: 0,
@@ -164,4 +168,4 @@ waitUntil: ['networkidle2', 'domcontentloaded']
 
-        if (response) {
-          const sourceHtml = await this.parseSourceHTML(await response.text());
+        if (_sourceHtml) {
+          const sourceHtml = await this.parseSourceHTML(_sourceHtml);
           const styles = CSSselect('style', sourceHtml.html.parsed);
@@ -190,3 +194,2 @@ let k = 0;
           const evaluation = await evaluateUrl(url, sourceHtml, page, stylesheets, mappedDOM, this.modulesToExecute, options);
-
           this.evaluations[url] = evaluation.getFinalReport();
@@ -305,2 +308,14 @@ } else {
 
+  private async getSourceHtml(url: string, options?: any): Promise<string> {
+    const fetchOptions = {
+      'headers': {
+          'User-Agent': options ? options.userAgent ? options.userAgent : options.mobile ? DEFAULT_MOBILE_USER_AGENT : DEFAULT_DESKTOP_USER_AGENT : DEFAULT_DESKTOP_USER_AGENT
+      }
+    };
+    const response = await fetch(url, fetchOptions);
+    const sourceHTML = (await response.text()).trim();
+    
+    return sourceHTML;
+  }
+
   private async parseStylesheets(plainStylesheets: any): Promise<any[]> {
@@ -307,0 +322,0 @@ const stylesheets = new Array<any>();

test/core.spec.js

@@ -7,3 +7,3 @@ const core = require('../dist/index');
 
-  it('Should evaluate', async function() {
+  it.only('Should evaluate', async function() {
     this.timeout(1000 * 1000);
@@ -13,5 +13,6 @@
     const reports = await core.evaluate({ url: 'http://ciencias.ulisboa.pt'});
-    await core.stop();
+    console.log(reports);
+    await core.close();
 
-    expect(reports[url].type).to.be.equal('evaluation');
+    expect(reports['http://ciencias.ulisboa.pt'].type).to.be.equal('evaluation');
   });
@@ -18,0 +19,0 @@

test/parallel.spec.js

@@ -10,4 +10,4 @@ const core = require('../dist/index');
     const response = await fetch('https://act-rules.github.io/testcases.json')
-    const testCases = JSON.parse(await response.json());
-    const rule = 'b33eff';
+    const testCases = await response.json();
+    const rule = '2779a5';
     const tcs = testCases.testcases.filter(tc => tc.ruleId === rule);
@@ -30,3 +30,3 @@ const urls = tcs.map(tc => tc.url);
     await core.evaluate(options);
-    const earlReport = await core.generateEarlReport({ aggregated: true, modules: { act: true }});
+    const earlReport = Object.values(await core.generateEarlReport({ aggregated: true, modules: { act: true }}));
     
@@ -38,5 +38,5 @@ await core.close();
       try {
-        const result = earlReport[0].graph.filter(r => r.source === tcs[i].url)[0];
-        console.warn(result.source + '   ' + tcs[i].url);
-        console.warn(result.assertions[0].result.outcome + '   earl:' + tcs[i].expected);
+        const result = earlReport[0]['@graph'].filter(r => r.source === tcs[i].url)[0];
+        //console.warn(result.source + '   ' + tcs[i].url);
+        //console.warn(result.assertions[0].result.outcome + '   earl:' + tcs[i].expected);
         
@@ -43,0 +43,0 @@ if (result.assertions[0].result.outcome !== 'earl:' + tcs[i].expected) {

No alert changes

Improved metrics

Total package byte prevSize

171966

increased by5.16%

Number of package files

27

increased by3.85%

Lines of code

1804

increased by11.98%

Dependency changes

• + Added@qualweb/act-rules@0.4.57(transitive)

• + Added@qualweb/best-practices@0.4.18(transitive)

• + Added@qualweb/html-techniques@0.4.11(transitive)

• + Added@qualweb/qw-element@0.1.33(transitive)

• + Added@qualweb/qw-page@0.1.39(transitive)

• + Added@qualweb/util@0.3.66(transitive)

• - Removed@qualweb/act-rules@0.3.39(transitive)

• - Removed@qualweb/best-practices@0.3.5(transitive)

• - Removed@qualweb/html-techniques@0.3.7(transitive)

• Updated@qualweb/act-rules@^0.4.1

• Updated@qualweb/best-practices@^0.4.1

• Updated@qualweb/css-techniques@^0.3.8

• Updated@qualweb/html-techniques@^0.4.1

• Updateddomutils@^2.1.0

• Updatedpuppeteer@^3.0.4