kickstarter-crawler - npm Package Compare versions

Comparing version 2.0.0 to 2.1.0

index.js

@@ -19,3 +19,15 @@ const cheerio = require('cheerio');
 function parseHTML(res) {
-  const data = {};
+  let data = {};
+  let title;
+  let creator;
+  let description;
+  let location;
+  let category;
+  let duration;
+  let funding;
+  let backers;
+  let goal;
+  let startdate;
+  let enddate;
+  let pledges;
 
@@ -27,65 +39,72 @@ // parse html ie "load"
 
-  const title =
-    $('.project-profile__title')
-      .eq(0)
-      .text()
-      .trim();
+  if (!was_cancelled) {
+    title =
+      $('.project-profile__title')
+        .eq(0)
+        .text()
+        .trim();
 
-  const creator =
-    $('.hero__link')
-      .eq(1)
-      .text()
-      .trim();
+    creator =
+      $('.hero__link')
+        .eq(1)
+        .text()
+        .trim();
 
-  const description =
-    $('.project-profile__blurb')
-      .text()
-      .trim();
+    description =
+      $('.project-profile__blurb')
+        .text()
+        .trim();
 
-  // Sometimes kickstarter has a "projects we love" section before category/location
-  const has_flag = $('.NS_projects__category_location').children().length === 3
-  const category_location_offset = has_flag ? 1 : 0;
+    // Sometimes kickstarter has a "projects we love" section before category/location
+    const has_flag = $('.NS_projects__category_location').children().length === 3
+    const category_location_offset = has_flag ? 1 : 0;
 
-  // leave unstructured (ie US vs international formatting)
-  const location =
-    $('.NS_projects__category_location a')
-      .eq(category_location_offset)
-      .text()
-      .trim()
-      .split(', ');
+    // leave unstructured (ie US vs international formatting)
+    location =
+      $('.NS_projects__category_location a')
+        .eq(category_location_offset)
+        .text()
+        .trim()
+        .split(', ');
 
-  const category =
-    $('.NS_projects__category_location a')
-      .eq(category_location_offset + 1)
-      .text()
-      .trim();
+    category =
+      $('.NS_projects__category_location a')
+        .eq(category_location_offset + 1)
+        .text()
+        .trim();
 
-  let duration =
-    $('.NS_campaigns__funding_period .f5')
-      .text()
-      .trim();
+    duration =
+      $('.NS_campaigns__funding_period .f5')
+        .text()
+        .trim();
 
-  duration = duration && duration.split('\n')[1];
-  duration = Number(duration.replace(/[^0-9]/g, ''));
+    duration = duration && duration.split('\n')[1];
+    duration = Number(duration.replace(/[^0-9]/g, ''));
 
-  const startdate =
-    $('.NS_campaigns__funding_period time')
-      .eq(0)
-      .text()
-      .trim();
-
-  const enddate =
-    $('.NS_campaigns__funding_period time')
+    funding =
+      $('.money')
       .eq(1)
-      .text()
-      .trim();
+      .text();
 
-  let funding =
-    $('.money')
-    .eq(1)
-    .text();
+    funding = funding && Number(funding.replace(/[^0-9]+/g, ''));
 
-  funding = funding && Number(funding.replace(/[^0-9]+/g, ''));
+    backers =
+      $('h3.mb0')
+        .eq(1)
+        .text();
 
-  let goal =
+    backers = backers && Number(backers.replace(/[^0-9]/g, ''));
+
+
+  } else {
+    title = $('.project-name').eq(0).text().trim();
+    creator = $('#experimental-creator-bio .mb6 .text-ellipsis').text();
+    description = $('.project-description').text();
+    category = $('.ml1').eq(1).text();
+    location = $('.ml1').eq(2).text().split(', ');
+    backers = Number($('.type-28-md span').eq(1).text());
+    funding = Number($('.type-28-md span').eq(0).text().replace(/[^0-9]+/g, ''));
+  }
+
+  goal =
     $('.money')
@@ -97,11 +116,16 @@ .eq(2)
 
-  let backers =
-    $('h3.mb0')
+  startdate =
+    $('.NS_campaigns__funding_period time')
+      .eq(0)
+      .text()
+      .trim();
+
+  enddate =
+    $('.NS_campaigns__funding_period time')
       .eq(1)
-      .text();
+      .text()
+      .trim();
 
-  backers = backers && Number(backers.replace(/[^0-9]/g, ''));
-
   const n = $('ol li .pledge__backer-stats').length;
-  let pledges = [];
+  pledges = [];
 
@@ -114,4 +138,5 @@ for (let i = 0; i < n; i++) {
 
-    amount = amount && amount.match(/[1-9]+[,]*[0-9]*/)[0];
-    amount = amount.replace(/,/, '');
+    amount = amount && amount.match(/[1-9]+[,]*[0-9]*/)
+    amount = amount && amount[0]
+    amount = amount && amount.replace(/,/, '');
 
@@ -118,0 +143,0 @@ let backers =

package.json

 {
   "name": "kickstarter-crawler",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Kickstarter crawler that does what you think it would",
@@ -13,3 +13,4 @@ "main": "index.js",
     "axios": "^0.19.2",
-    "cheerio": "^1.0.0-rc.3"
+    "cheerio": "^1.0.0-rc.3",
+    "mongodb": "^3.6.3"
   },
@@ -16,0 +17,0 @@ "devDependencies": {

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

5593

increased by14.82%

Lines of code

156

increased by17.29%

Number of medium supply chain risk alerts

0

decreased by-100%

Worsened metrics

Dependency count

3

increased by50%

Dependency changes

• + Addedmongodb@^3.6.3

• + Addedbl@2.2.1(transitive)

• + Addedbson@1.1.6(transitive)

• + Addedcore-util-is@1.0.3(transitive)

• + Addeddenque@1.5.1(transitive)

• + Addedinherits@2.0.4(transitive)

• + Addedisarray@1.0.0(transitive)

• + Addedmemory-pager@1.5.0(transitive)

• + Addedmongodb@3.7.4(transitive)

• + Addedoptional-require@1.1.8(transitive)

• + Addedprocess-nextick-args@2.0.1(transitive)

• + Addedreadable-stream@2.3.8(transitive)

• + Addedrequire-at@1.0.6(transitive)

• + Addedsafe-buffer@5.1.25.2.1(transitive)

• + Addedsaslprep@1.0.3(transitive)

• + Addedsparse-bitfield@3.0.3(transitive)

• + Addedstring_decoder@1.1.1(transitive)

• + Addedutil-deprecate@1.0.2(transitive)