promise-crawler

promise-crawler

Promise support for node-crawler (Web Crawler/Spider for NodeJS + server-side jQuery)

Nodejs library for website crawling using node-crawler but on bluebird promises.

Install using npm:

npm i promise-crawler --save

Example:

const PromiseCrawler = require('promise-crawler');
//Initialize with node-crawler options
const crawler = new PromiseCrawler({
  maxConnections: 10,
  retries: 3
});

//perform setup and then use it
crawler.setup().then(() => {
  // makes request with node-crawler queue options
  crawler.request({
    url: 'http://example.com'
  }).then((res) => {
    //server side response parsing using cheerio
    let $ = res.$;
    console.log($("title").text());

    // destroy the instance
    process.nextTick(() => crawler.destroy())
  })
});