kickstarter-crawler

kickstarter-crawler

:pizza: Returns 30 + 8n data points - where n is the number of pledges

:pizza: Analyze 61,356 kickstarter projects using crapi

Installation

npm install kickstarter-crawler -g

Test

npm test

Getting Started

Examples
API

Examples

The following example crawls a project collecting general, funding, and location
related data

'use strict';


var kickstarterCrawler = require('kickstarter-crawler');


var config, project;


// Project configurations
config = {
  url: 'https://www.kickstarter.com/projects/maxtemkin/philosophy-posters',
  fields: ['general', 'funding', 'location']
};


// Initialize the crawler
project = new kickstarterCrawler.project(config);


// Make request (crawl)
project.request(function onRequest (err, data) {

  // Something broke
  if (err) {
    console.log(err);
    return;
  }

  // Log crawled data
  console.log(data);

});

** :pizza: MORE EXAMPLES COMING :pizza: **

API

kickstarterCrawler.project(config)

• {Object} project constructor
  
• Initializes the crawler and exposes its interface
  
  

config

• {Object} project configurations
  
• Configurations necessary to instatiate the project constructor
  
  

config.url

• {String} project profile url
  
  

config.fields

• {Array} array of project data-fields, which indicate what data points will be crawled
  
• If undefined, data from all fields will be returned.
  

VALID FIELD VALUES

• general
• time
• funding
• location
• other
• media
• pledges

project.request(callback)

• Makes a HTTP request to the respective project url
  
• @param callback(err, data) {Function}
  
  

project.parse(HTML)

• Parses the HTML corresponding to the respective project profile page
  
• Returns a nested JSON of parsed data
• @param {String} HTML
  
• @return {Object}

project.getTitle(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} generalTitle
  
  

project.getCreator(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} generalCreator
  
  

project.getCategory(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} generalCategory
  
  

project.getSubCategory(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} generalSubCategory
  
  

project.getAvatar(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} generalProjectAvatarURL
  
  

project.getProjectUrl(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} generalProjectURL
  
  

project.getCreatorUrl(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} generalCreatorURL
  
  

project.getProjectVideo(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} generalProjectVideoURL
  
  

project.getNumDays(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} timeNumDays
  
  

project.getStartTime(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} timeStart
  
  

project.getEndTime(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} timeEnd
  
  

project.getDollarsRaised(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} fundingDollarsRaised
  
  

project.getFundingGoal(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} fundingGoal
  
  

project.getPercentRaised(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} fundingPercentRaised
  
  

project.getCurrency(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} fundingCurrency
  
  

project.getSuccess(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Boolean} fundingSuccess
  
  

project.getBackers(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} fundingNumBackers
  
  

project.getCity(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} locationCity
  
  

project.getState(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} locationState
  
  

project.getCountry(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} locationCountry
  
  

project.getUpdates(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} otherUpdates
  
  

project.getComments(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} otherComments
  
  

project.getProjectsCreated(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} otherProjectsCreated
  
  

project.getProjectsBacked(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} otherProjectsBacked
  
  

project.getWebsiteUrl(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {String} otherWebsiteURL
  
  

project.getNumImages(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} mediaNumImages
  
  

project.getImages(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Array} mediaImages
  
  

project.getNumPledges(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} pledgesNumPledges
  
  

project.getNumLimitedPledges(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Number} pledgesNumLimitedPledges
  
  

project.getPledgeAmounts(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Array} pledgesAmounts
  
  

project.getPledgesData(callback)

• @param {Function} callback(err, data)
  
• data {Object} will consist of {Object} pledgesData