@qualweb/crawler
Advanced tools
Comparing version 0.3.13 to 0.3.14
@@ -60,2 +60,3 @@ "use strict"; | ||
const [firstPageUrls, relativePathsToTest] = await this.fetchPageLinks(this.startingUrl); | ||
console.log({ firstPageUrls, relativePathsToTest }); | ||
urlsByDepth[currentDepth] = [...firstPageUrls]; | ||
@@ -159,2 +160,3 @@ const newUrls = this.normalizeAndSort(await this.checkRelativePathsUrls(relativePathsToTest)); | ||
}); | ||
console.log(url); | ||
[urls, relativePathsToTest] = await page.evaluate((startingUrl, isDomain) => { | ||
@@ -269,9 +271,10 @@ function getUrlWithoutExtension(url) { | ||
} | ||
return [this.normalizeAndSort(urls), relativePathsToTest]; | ||
return [[], [...relativePathsToTest, ...this.normalizeAndSort(urls)]]; | ||
} | ||
async checkRelativePathsUrls(urls) { | ||
const newUrlsToValidate = new Array(); | ||
for (const url of urls !== null && urls !== void 0 ? urls : []) { | ||
await Promise.all(urls.map(async (url) => { | ||
try { | ||
const page = await this.browser.newPage(); | ||
console.log(url); | ||
if (this.viewport) { | ||
@@ -309,3 +312,3 @@ await page.setViewport(this.viewport); | ||
} | ||
} | ||
})); | ||
return newUrlsToValidate; | ||
@@ -312,0 +315,0 @@ } |
{ | ||
"name": "@qualweb/crawler", | ||
"version": "0.3.13", | ||
"version": "0.3.14", | ||
"description": "Webpage crawler for qualweb", | ||
@@ -5,0 +5,0 @@ "main": "dist/index.js", |
Sorry, the diff of this file is not supported yet
New author
Supply chain riskA new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.
Found 1 instance in 1 package
30829
342
0