app-store-scraper - npm Package Compare versions

Comparing version 0.17.0 to 0.18.0

lib/app.js

@@ -13,3 +13,3 @@ 'use strict';
     const idValue = opts.id || opts.appId;
-    resolve(common.lookup([idValue], idField, opts.country, opts.lang, opts.requestOptions));
+    resolve(common.lookup([idValue], idField, opts.country, opts.lang, opts.requestOptions, opts.throttle));
   })
@@ -16,0 +16,0 @@ .then((results) => {

lib/common.js

 'use strict';
 
 const request = require('request');
+const throttled = require('throttled-request')(request);
 const debug = require('debug')('app-store-scraper');
@@ -46,3 +47,3 @@ const c = require('./constants');
 // TODO add an optional parse function
-const doRequest = (url, headers, requestOptions) => new Promise(function (resolve, reject) {
+const doRequest = (url, headers, requestOptions, limit) => new Promise(function (resolve, reject) {
   debug('Making request: %s %j %o', url, headers, requestOptions);
@@ -52,3 +53,11 @@
 
-  request(Object.assign({ url, headers }, requestOptions), (error, response, body) => {
+  let req = request;
+  if (limit) {
+    throttled.configure({
+      requests: limit,
+      milliseconds: 1000
+    });
+    req = throttled;
+  }
+  req(Object.assign({ url, headers }, requestOptions), (error, response, body) => {
     if (error) {
@@ -68,3 +77,3 @@ debug('Request error', error);
 
-function lookup (ids, idField, country, lang, requestOptions) {
+function lookup (ids, idField, country, lang, requestOptions, limit) {
   idField = idField || 'id';
@@ -75,3 +84,3 @@ country = country || 'us';
   const url = `${LOOKUP_URL}?${idField}=${joinedIds}&country=${country}&entity=software${langParam}`;
-  return doRequest(url, {}, requestOptions)
+  return doRequest(url, {}, requestOptions, limit)
     .then(JSON.parse)
@@ -78,0 +87,0 @@ .then((res) => res.results.filter(function (app) {

lib/constants.js

@@ -162,2 +162,3 @@ 'use strict';
   KE: 143529,
+  KR: 143466,
   KW: 143493,
@@ -164,0 +165,0 @@ LV: 143519,

lib/developer.js

@@ -10,3 +10,3 @@ 'use strict';
     }
-    resolve(common.lookup([opts.devId], 'id', opts.country, opts.lang, opts.requestOptions));
+    resolve(common.lookup([opts.devId], 'id', opts.country, opts.lang, opts.requestOptions, opts.throttle));
   })
@@ -13,0 +13,0 @@ .then((results) => {

lib/list.js

@@ -54,3 +54,3 @@ 'use strict';
       const ids = apps.map((app) => app.id.attributes['im:id']);
-      return common.lookup(ids, 'id', opts.country, opts.lang, opts.requestOptions);
+      return common.lookup(ids, 'id', opts.country, opts.lang, opts.requestOptions, opts.throttle);
     }
@@ -63,3 +63,3 @@
 function validate (opts) {
-  if (opts.category && !R.contains(opts.category, R.values(c.category))) {
+  if (opts.category && !R.includes(opts.category, R.values(c.category))) {
     throw Error('Invalid category ' + opts.category);
@@ -69,3 +69,3 @@ }
   opts.collection = opts.collection || c.collection.TOP_FREE_IOS;
-  if (!R.contains(opts.collection, R.values(c.collection))) {
+  if (!R.includes(opts.collection, R.values(c.collection))) {
     throw Error(`Invalid collection ${opts.collection}`);
@@ -72,0 +72,0 @@ }

lib/privacy.js

@@ -6,2 +6,4 @@ 'use strict';
 function privacy (opts) {
+  opts.country = opts.country || 'US';
+
   return new Promise((resolve) => {
@@ -15,3 +17,3 @@ if (opts.id) {
     .then(() => {
-      const tokenUrl = `https://apps.apple.com/us/app/id${opts.id}`;
+      const tokenUrl = `https://apps.apple.com/${opts.country}/app/id${opts.id}`;
       return common.request(tokenUrl, {}, opts.requestOptions);
@@ -24,3 +26,3 @@ })
 
-      const url = `https://amp-api.apps.apple.com/v1/catalog/US/apps/${opts.id}?platform=web&fields=privacyDetails&l=en-us`;
+      const url = `https://amp-api.apps.apple.com/v1/catalog/${opts.country}/apps/${opts.id}?platform=web&fields=privacyDetails`;
       return common.request(url, {
@@ -39,1 +41,2 @@ 'Origin': 'https://apps.apple.com',
 module.exports = privacy;
+

lib/reviews.js

@@ -30,3 +30,4 @@ 'use strict';
     text: review.content.label,
-    url: review.link.attributes.href
+    url: review.link.attributes.href,
+    updated: review.updated.label
   }));
@@ -61,3 +62,3 @@ }
 
-  if (opts.sort && !R.contains(opts.sort, R.values(c.sort))) {
+  if (opts.sort && !R.includes(opts.sort, R.values(c.sort))) {
     throw new Error('Invalid sort ' + opts.sort);
@@ -64,0 +65,0 @@ }

lib/search.js

@@ -41,3 +41,3 @@ 'use strict';
         if (!opts.idsOnly) {
-          return common.lookup(ids, 'id', opts.country, opts.lang, opts.requestOptions);
+          return common.lookup(ids, 'id', opts.country, opts.lang, opts.requestOptions, opts.throttle);
         }
@@ -44,0 +44,0 @@ return ids;

lib/similar.js

@@ -33,3 +33,3 @@ 'use strict';
 
-      return common.lookup(ids, 'id', opts.country, opts.lang, opts.requestOptions);
+      return common.lookup(ids, 'id', opts.country, opts.lang, opts.requestOptions, opts.throttle);
     });
@@ -36,0 +36,0 @@ }

lib/suggest.js

@@ -6,3 +6,3 @@ 'use strict';
 
-const BASE_URL = 'https://search.itunes.apple.com/WebObjects/MZSearchHints.woa/wa/hints?media=software&q=';
+const BASE_URL = 'https://search.itunes.apple.com/WebObjects/MZSearchHints.woa/wa/hints?clientApplication=Software&term=';
 
@@ -23,4 +23,3 @@ function parseXML (string) {
   const toJSON = (item) => ({
-    term: item.string[0],
-    priority: item.integer[0]
+    term: item.string[0]
   });
@@ -42,3 +41,3 @@
   })
-    .then(url => common.request(url, {}, opts.requestOptions))
+    .then(url => common.request(url, {'X-Apple-Store-Front': `${common.storeId(opts.country)},29`}, opts.requestOptions))
     .then(parseXML)
@@ -45,0 +44,0 @@ .then(extractSuggestions);

package.json

 {
   "name": "app-store-scraper",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "description": "scrape data from the itunes app store",
   "main": "index.js",
   "scripts": {
-    "test": "mocha --timeout 8000"
+    "test": "mocha --timeout 8000",
+    "lint": "eslint ."
   },
@@ -29,10 +30,11 @@ "repository": {
     "debug": "^2.2.0",
-    "memoizee": "^0.3.10",
-    "ramda": "^0.21.0",
+    "memoizee": "^0.4.15",
+    "ramda": "^0.29.0",
     "request": "^2.87.0",
-    "xml2js": "^0.4.16"
+    "throttled-request": "^0.1.1",
+    "xml2js": "^0.6.2"
   },
   "devDependencies": {
-    "chai": "^4.1.2",
-    "eslint": "^4.18.2",
+    "chai": "^4.3.8",
+    "eslint": "^8.49.0",
     "eslint-config-semistandard": "^6.0.1",
@@ -42,5 +44,5 @@ "eslint-config-standard": "^5.1.0",
     "eslint-plugin-standard": "^4.0.1",
-    "mocha": "^6.2.2",
-    "validator": "^5.2.0"
+    "mocha": "^10.2.0",
+    "validator": "^13.11.0"
   }
 }

README.md

@@ -318,8 +318,8 @@ # app-store-scraper [![Build Status](https://secure.travis-ci.org/facundoolano/app-store-scraper.png)](http://travis-ci.org/facundoolano/app-store-scraper)
 [
-  { term: 'panda pop', priority: '7557' },
-  { term: 'panda pop free', priority: '5796' },
-  { term: 'panda', priority: '5512' },
-  { term: 'panda express', priority: '5174' },
-  { term: 'panda games', priority: '4773' },
-  { term: 'panda pop 2', priority: '4695' },
+  { term: 'panda pop' },
+  { term: 'panda pop free' },
+  { term: 'panda' },
+  { term: 'panda express' },
+  { term: 'panda games' },
+  { term: 'panda pop 2' },
   ...
@@ -397,2 +397,3 @@ ]
     text: 'I was a fan of Bejeweled many moons ago...',
+    updated: '2021-07-26T18:26:24-07:00',
     url: 'https://itunes.apple.com/us/review?id=553834731&type=Purple%20Software' },,
@@ -406,2 +407,3 @@ { id: '1472864708',
     text: 'STILL HAVING THE SAME ISSUE.  It\'s happening again...',
+    updated: '2021-07-26T18:04:41-07:00',
     url: 'https://itunes.apple.com/us/review?id=553834731&type=Purple%20Software' },
@@ -408,0 +410,0 @@ (...)

test/common.js

@@ -21,3 +21,3 @@ 'use strict';
     assert.isNumber(app.score);
-    assert(app.score > 0);
+    assert(app.score >= 0);
     assert(app.score <= 5);
@@ -24,0 +24,0 @@ }

test/lib.developer.js

@@ -16,3 +16,3 @@ 'use strict';
           assert.equal(app.developerId, FACEBOOK_ID);
-          assert.equal(app.developer, 'Facebook, Inc.');
+          assert.equal(app.developer, 'Meta Platforms, Inc.');
         });
@@ -19,0 +19,0 @@ });

test/lib.reviews.js

@@ -18,2 +18,5 @@ 'use strict';
   assertValidUrl(review.url);
+  assert.isNotNull(new Date(review.updated).toJSON());
+  assert.isString(review.updated);
+  assert(review.updated);
 }
@@ -20,0 +23,0 @@

test/lib.similar.js

@@ -13,6 +13,3 @@ 'use strict';
 
-  // FIXME improve code
-  // skipping as this functionality doesn't work anymore
-  // see https://github.com/facundoolano/app-store-scraper/issues/49
-  it.skip('should a different list in fr country', () => {
+  it('should a different list in fr country', () => {
     return store.similar({id: '553834731'})
@@ -19,0 +16,0 @@ .then((usApps) => {

test/lib.suggest.js

@@ -9,3 +9,3 @@ 'use strict';
     .then((results) => {
-      assert.equal(results.length, 50, `expected ${results} to have 50 elements`);
+      assert.equal(results.length, 10, `expected ${results} to have 10 elements`);
       results.map((r) => assert.include(r.term, 'p'));
@@ -12,0 +12,0 @@ }));

No alert changes

Improved metrics

Total package byte prevSize

54191

increased by1.73%

Number of package files

30

increased by7.14%

Lines of code

1193

increased by0.93%

Number of lines in readme file

464

increased by0.43%

Worsened metrics

Dependency count

7

increased by16.67%

Dependency changes

• + Addedthrottled-request@^0.1.1

• + Addedes6-weak-map@2.0.3(transitive)

• + Addedis-promise@2.2.2(transitive)

• + Addedmemoizee@0.4.17(transitive)

• + Addedramda@0.29.1(transitive)

• + Addedthrottled-request@0.1.1(transitive)

• + Addedxml2js@0.6.2(transitive)

• - Removedd@0.1.1(transitive)

• - Removedes6-iterator@0.1.3(transitive)

• - Removedes6-symbol@2.0.1(transitive)

• - Removedes6-weak-map@0.1.4(transitive)

• - Removedmemoizee@0.3.10(transitive)

• - Removednext-tick@0.2.2(transitive)

• - Removedramda@0.21.0(transitive)

• - Removedxml2js@0.4.23(transitive)

• Updatedmemoizee@^0.4.15

• Updatedramda@^0.29.0

• Updatedxml2js@^0.6.2