crawler-url-parser
Advanced tools
Readme
An URL parser for crawling purpose
npm install crawler-url-parser
const cup = require('crawler-url-parser');
//// parse(current_url[,base_url])
let result = cup.parse("http://question.stackoverflow.com/aaa/bbb/ddd?q1=query1&q2=query2");
console.log(result.url);
// http://question.stackoverflow.com/aaa/bbb/ddd?q1=query1&q2=query2
console.log(result.baseurl);
// null
console.log(result.normalized);
// http://question.stackoverflow.com/aaa/bbb/ddd?q1=query1&q2=query2
console.log(result.host);
// question.stackoverflow.com
console.log(result.domain);
// stackoverflow.com
console.log(result.subdomain);
// question
console.log(result.protocol);
// http:
console.log(result.path);
// /aaa/bbb/ddd
console.log(result.search);
// q1=query1&q2=query2
console.log(result.querycount);
// 2
const cup = require('crawler-url-parser');
//// parse(current_url[,base_url])
let result = cup.parse("../ddd?q1=query1&q2=query2","http://question.stackoverflow.com/aaa/bbb/ccc/");
console.log(result.url);
// http://question.stackoverflow.com/aaa/bbb/ddd?q1=query1&q2=query2
console.log(result.baseurl);
// http://question.stackoverflow.com/aaa/bbb/ccc
console.log(result.normalized);
// http://question.stackoverflow.com/aaa/bbb/ddd?q1=query1&q2=query2
console.log(result.host);
// question.stackoverflow.com
console.log(result.domain);
// stackoverflow.com
console.log(result.subdomain);
// question
console.log(result.protocol);
// http:
console.log(result.path);
// /aaa/bbb/ddd
console.log(result.search);
// q1=query1&q2=query2
console.log(result.querycount);
// 2
const cup = require('crawler-url-parser');
//// extract(html_str,current_url);
let htmlStr='<html><body> \
<a href="http://best.question.stackoverflow.com">subdomain</a><br /> \
<a href="http://faq.stackoverflow.com">subdomain</a><br /> \
<a href="http://stackoverflow.com">updomain</a><br /> \
<a href="http://www.google.com">external</a><br /> \
<a href="http://www.facebook.com">external</a><br /> \
<a href="http://question.stackoverflow.com/aaa/bbb/ccc">sublevel</a><br /> \
<a href="http://question.stackoverflow.com/aaa/bbb/zzz">sublevel</a><br /> \
<a href="http://question.stackoverflow.com/aaa/">uplevel</a><br /> \
<a href="http://question.stackoverflow.com/aaa/ddd">samelevel</a><br /> \
<a href="http://question.stackoverflow.com/aaa/eee">samelevel</a><br /> \
<a href="http://question.stackoverflow.com/aaa/ddd/eee">internal</a><br /> \
<a href="http://question.stackoverflow.com/zzz">internal</a><br /> \
</body></html>';
let currentUrl= "http://question.stackoverflow.com/aaa/bbb";
let urls = cup.extract(htmlStr,currentUrl);
console.log(urls[0].type); //subdomain
console.log(urls[1].type); //subdomain
console.log(urls[2].type); //updomain
console.log(urls[3].type); //external
console.log(urls[4].type); //external
console.log(urls[5].type); //sublevel
console.log(urls[6].type); //sublevel
console.log(urls[7].type); //uplevel
console.log(urls[8].type); //samelevel
console.log(urls[9].type); //samelevel
console.log(urls[10].type); //internal
console.log(urls[11].type); //subdomain
const cup = require('crawler-url-parser');
//// gettype(current_url,base_url);
let level = cup.gettype("sub.domain.com/aaa/bbb/","sub.domain.com/aaa/bbb/ccc");
console.log(level); //sublevel
level = cup.gettype("sub.domain.com/aaa/bbb/ccc/ddd","sub.domain.com/aaa/bbb/ccc");
console.log(level); //uplevel
level = cup.gettype("sub.domain.com/aaa/bbb/eee","sub.domain.com/aaa/bbb/ccc");
console.log(level); //samelevel
level = cup.gettype("sub.domain.com/aaa/bbb/eee","sub.anotherdomain.com/aaa/bbb/ccc");
console.log(level); //external
mocha or npm test
more than 200 unit test cases. check test folder and QUICKSTART.js for extra usage.
FAQs
An `URL` parser for crawling purpose.
The npm package crawler-url-parser receives a total of 253 weekly downloads. As such, crawler-url-parser popularity was classified as not popular.
We found that crawler-url-parser demonstrated a not healthy version release cadence and project activity because the last version was released a year ago. It has 1 open source maintainer collaborating on the project.
Did you know?
Socket for GitHub automatically highlights issues in each pull request and monitors the health of all your open source dependencies. Discover the contents of your packages and block harmful activity before you install or update your dependencies.
Security News
The White House is addressing fragmented cybersecurity regulations as CISOs report spending up to 50% of their time on compliance, aiming to harmonize requirements and improve cybersecurity outcomes.
Security News
Research
The Socket Research Team has identified a malicious Python package that is typosquatting the popular crytic-compile utility, frequently used in popular toolkits and development environments for smart contracts and crypto applications.
Security News
NIST updates on the NVD backlog after media reports that over 50% of KEVs were unenriched since mid-February. They've contracted additional support and partnered with CISA to clear the backlog by fiscal year-end.