@qualweb/crawler
Advanced tools
Comparing version 0.3.14 to 0.3.15
@@ -60,3 +60,2 @@ "use strict"; | ||
const [firstPageUrls, relativePathsToTest] = await this.fetchPageLinks(this.startingUrl); | ||
console.log({ firstPageUrls, relativePathsToTest }); | ||
urlsByDepth[currentDepth] = [...firstPageUrls]; | ||
@@ -160,3 +159,2 @@ const newUrls = this.normalizeAndSort(await this.checkRelativePathsUrls(relativePathsToTest)); | ||
}); | ||
console.log(url); | ||
[urls, relativePathsToTest] = await page.evaluate((startingUrl, isDomain) => { | ||
@@ -180,3 +178,5 @@ function getUrlWithoutExtension(url) { | ||
if (link.hasAttribute('href')) { | ||
const href = (_a = link.getAttribute('href')) === null || _a === void 0 ? void 0 : _a.trim(); | ||
let href = (_a = link.getAttribute('href')) === null || _a === void 0 ? void 0 : _a.trim(); | ||
if (href === null || href === void 0 ? void 0 : href.startsWith('//')) | ||
href = href.replace('//', 'https://'); | ||
if (href && | ||
@@ -267,3 +267,2 @@ !isDomain && | ||
}, this.startingUrl, this.isDomain); | ||
await page.close(); | ||
} | ||
@@ -280,3 +279,2 @@ catch (err) { | ||
const page = await this.browser.newPage(); | ||
console.log(url); | ||
if (this.viewport) { | ||
@@ -283,0 +281,0 @@ await page.setViewport(this.viewport); |
{ | ||
"name": "@qualweb/crawler", | ||
"version": "0.3.14", | ||
"version": "0.3.15", | ||
"description": "Webpage crawler for qualweb", | ||
@@ -5,0 +5,0 @@ "main": "dist/index.js", |
Sorry, the diff of this file is not supported yet
30772
340