crawler-url-parser
Advanced tools
An URL parser for crawling purpose
npm install crawler-url-parser
const cup = require('crawler-url-parser');
//// parse(current_url[,base_url])
let result = cup.parse("http://question.stackoverflow.com/aaa/bbb/ddd?q1=query1&q2=query2");
console.log(result.url);
// http://question.stackoverflow.com/aaa/bbb/ddd?q1=query1&q2=query2
console.log(result.baseurl);
// null
console.log(result.normalized);
// http://question.stackoverflow.com/aaa/bbb/ddd?q1=query1&q2=query2
console.log(result.host);
// question.stackoverflow.com
console.log(result.domain);
// stackoverflow.com
console.log(result.subdomain);
// question
console.log(result.protocol);
// http:
console.log(result.path);
// /aaa/bbb/ddd
console.log(result.search);
// q1=query1&q2=query2
console.log(result.querycount);
// 2
const cup = require('crawler-url-parser');
//// parse(current_url[,base_url])
let result = cup.parse("../ddd?q1=query1&q2=query2","http://question.stackoverflow.com/aaa/bbb/ccc/");
console.log(result.url);
// http://question.stackoverflow.com/aaa/bbb/ddd?q1=query1&q2=query2
console.log(result.baseurl);
// http://question.stackoverflow.com/aaa/bbb/ccc
console.log(result.normalized);
// http://question.stackoverflow.com/aaa/bbb/ddd?q1=query1&q2=query2
console.log(result.host);
// question.stackoverflow.com
console.log(result.domain);
// stackoverflow.com
console.log(result.subdomain);
// question
console.log(result.protocol);
// http:
console.log(result.path);
// /aaa/bbb/ddd
console.log(result.search);
// q1=query1&q2=query2
console.log(result.querycount);
// 2
const cup = require('crawler-url-parser');
//// extract(html_str,current_url);
let htmlStr='<html><body> \
<a href="http://best.question.stackoverflow.com">subdomain</a><br /> \
<a href="http://faq.stackoverflow.com">subdomain</a><br /> \
<a href="http://stackoverflow.com">updomain</a><br /> \
<a href="http://www.google.com">external</a><br /> \
<a href="http://www.facebook.com">external</a><br /> \
<a href="http://question.stackoverflow.com/aaa/bbb/ccc">sublevel</a><br /> \
<a href="http://question.stackoverflow.com/aaa/bbb/zzz">sublevel</a><br /> \
<a href="http://question.stackoverflow.com/aaa/">uplevel</a><br /> \
<a href="http://question.stackoverflow.com/aaa/ddd">samelevel</a><br /> \
<a href="http://question.stackoverflow.com/aaa/eee">samelevel</a><br /> \
<a href="http://question.stackoverflow.com/aaa/ddd/eee">internal</a><br /> \
<a href="http://question.stackoverflow.com/zzz">internal</a><br /> \
</body></html>';
let currentUrl= "http://question.stackoverflow.com/aaa/bbb";
let urls = cup.extract(htmlStr,currentUrl);
console.log(urls[0].type); //subdomain
console.log(urls[1].type); //subdomain
console.log(urls[2].type); //updomain
console.log(urls[3].type); //external
console.log(urls[4].type); //external
console.log(urls[5].type); //sublevel
console.log(urls[6].type); //sublevel
console.log(urls[7].type); //uplevel
console.log(urls[8].type); //samelevel
console.log(urls[9].type); //samelevel
console.log(urls[10].type); //internal
console.log(urls[11].type); //subdomain
const cup = require('crawler-url-parser');
//// gettype(current_url,base_url);
let level = cup.gettype("sub.domain.com/aaa/bbb/","sub.domain.com/aaa/bbb/ccc");
console.log(level); //sublevel
level = cup.gettype("sub.domain.com/aaa/bbb/ccc/ddd","sub.domain.com/aaa/bbb/ccc");
console.log(level); //uplevel
level = cup.gettype("sub.domain.com/aaa/bbb/eee","sub.domain.com/aaa/bbb/ccc");
console.log(level); //samelevel
level = cup.gettype("sub.domain.com/aaa/bbb/eee","sub.anotherdomain.com/aaa/bbb/ccc");
console.log(level); //external
mocha or npm test
more than 200 unit test cases. check test folder and QUICKSTART.js for extra usage.
FAQs
An `URL` parser for crawling purpose.
The npm package crawler-url-parser receives a total of 152 weekly downloads. As such, crawler-url-parser popularity was classified as not popular.
We found that crawler-url-parser demonstrated a not healthy version release cadence and project activity because the last version was released a year ago. It has 1 open source maintainer collaborating on the project.
Did you know?
Socket for GitHub automatically highlights issues in each pull request and monitors the health of all your open source dependencies. Discover the contents of your packages and block harmful activity before you install or update your dependencies.
Security News
NIST's new password guidelines remove periodic changes and special character requirements, focusing on longer, more secure passwords for better authentication practices.
Security News
A record 2,709 developers participated in the 2024 Ruby on Rails Community Survey, revealing key tools, practices, and trends shaping the Rails ecosystem.
Security News
In 2023, data breaches surged 78% from zero-day and supply chain attacks, but developers are still buried under alerts that are unable to prevent these threats.