app-store-scraper - npm Package Compare versions

Comparing version 0.9.0 to 0.10.0

lib/app.js

 'use strict';
 
 const common = require('./common');
-const BASE_URL = 'https://itunes.apple.com/lookup';
 
@@ -13,15 +12,10 @@ function app (opts) {
     const idValue = opts.id || opts.appId;
-    const country = opts.country || 'us';
-    return resolve(`${BASE_URL}?${idField}=${idValue}&country=${country}`);
+    resolve(common.lookup([idValue], idField, opts.country));
   })
-  .then(common.request)
-  .then(JSON.parse)
-  .then((res) => {
-    return new Promise((resolve) => {
-      if (res.results.length === 0) {
-        throw Error('App not found (404)');
-      }
+  .then((results) => {
+    if (results.length === 0) {
+      throw Error('App not found (404)');
+    }
 
-      resolve(common.cleanApp(res.results[0]));
-    });
+    return results[0];
   });
@@ -28,0 +22,0 @@ }

lib/common.js

@@ -5,2 +5,3 @@ 'use strict';
 const debug = require('debug')('app-store-scraper');
+const c = require('./constants');
 
@@ -62,2 +63,20 @@ function cleanApp (app) {
 
-module.exports = {cleanApp, request: doRequest};
+const LOOKUP_URL = 'https://itunes.apple.com/lookup';
+
+function lookup (ids, idField, country) {
+  idField = idField || 'id';
+  country = country || 'us';
+  const joinedIds = ids.join(',');
+  const url = `${LOOKUP_URL}?${idField}=${joinedIds}&country=${country}`;
+  return doRequest(url)
+    .then(JSON.parse)
+    .then((res) => res.results.map(cleanApp));
+}
+
+function storeId (countryCode) {
+  const markets = c.markets;
+  const defaultStore = '143441';
+  return (countryCode && markets[countryCode.toUpperCase()]) || defaultStore;
+}
+
+module.exports = {cleanApp, lookup, request: doRequest, storeId};

lib/search.js

 'use strict';
 
+const R = require('ramda');
 const common = require('./common');
-const BASE_URL = 'https://itunes.apple.com/search';
-const c = require('./constants');
+const BASE_URL = 'https://itunes.apple.com/WebObjects/MZStore.woa/wa/search?clientApplication=Software&media=software&term=';
 
-// TODO allow override country & language
-// TODO handle not found
+// TODO find out if there's a way to filter by device
+// TODO refactor to allow memoization of the first request
 
+function paginate (num, page) {
+  num = num || 50;
+  page = page - 1 || 0;
+  const pageStart = num * page;
+  const pageEnd = pageStart + num;
+  return R.slice(pageStart, pageEnd);
+}
+
 function search (opts) {
@@ -15,16 +23,19 @@ return new Promise(function (resolve) {
     }
+    const url = BASE_URL + opts.term;
+    const storeId = common.storeId(opts.country);
+    const lang = opts.lang || 'en-us';
 
-    const entity = opts.device || c.device.ALL;
-    const num = opts.num || 50;
-    let url = `${BASE_URL}?media=software&entity=${entity}&term=${opts.term}&limit=${num}`;
-    if (opts.country) {
-      url = url + `&country=${opts.country}`;
-    }
-    return resolve(url);
-  })
-  .then(common.request)
-  .then(JSON.parse)
-  .then((res) => res.results.map(common.cleanApp));
+    common.request(url, {
+      'X-Apple-Store-Front': `${storeId},24 t:native`,
+      'Accept-Language': lang
+    })
+    .then(JSON.parse)
+    .then((response) => response.bubbles[0].results)
+    .then(paginate(opts.num, opts.page))
+    .then(R.pluck('id'))
+    .then((ids) => common.lookup(ids, 'id', opts.country))
+    .then(resolve);
+  });
 }
 
 module.exports = search;

lib/similar.js

@@ -5,12 +5,4 @@ 'use strict';
 const BASE_URL = 'https://itunes.apple.com/us/app/app/id';
-const LOOKUP_URL = 'https://itunes.apple.com/lookup?id=';
 const common = require('./common.js');
-const c = require('./constants.js');
 
-function storeId (countryCode) {
-  const markets = c.markets;
-  const defaultStore = '143441';
-  return (countryCode && markets[countryCode.toUpperCase()]) || defaultStore;
-}
-
 function similar (opts) {
@@ -27,3 +19,3 @@ return new Promise(function (resolve, reject) {
   .then((id) => common.request(`${BASE_URL}${id}`, {
-    'X-Apple-Store-Front': `${storeId(opts.country)},32`
+    'X-Apple-Store-Front': `${common.storeId(opts.country)},32`
   }))
@@ -38,5 +30,3 @@ .then(function (text) {
     const ids = JSON.parse(slice);
-    return common.request(LOOKUP_URL + ids.join(','))
-      .then(JSON.parse)
-      .then((res) => res.results.map(common.cleanApp));
+    return common.lookup(ids, 'id', opts.country);
   });
@@ -43,0 +33,0 @@ }

package.json

 {
   "name": "app-store-scraper",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "scrape data from the itunes app store",
@@ -5,0 +5,0 @@ "main": "index.js",

README.md

@@ -131,5 +131,7 @@ # app-store-scraper [![Build Status](https://secure.travis-ci.org/facundoolano/app-store-scraper.png)](http://travis-ci.org/facundoolano/app-store-scraper)
 * `term`: the term to search for (required).
-* `device`: the device to filter for. Defaults to `store.device.ALL`, available options are `store.device.ALL`, `store.device.MAC`, `store.device.IPAD`.
-* `num`: the amount of elements to retrieve. Defaults to `50`, maximum allowed is `200`.
-* `country`: the two letter country code to get the similar apps from. Defaults to `us`.
+* `num`: the amount of elements to retrieve. Defaults to `50`.
+* `page`: page of results to retrieve. Defaults to to `1`.
+* `country`: the two letter country code to get the similar apps
+  from. Defaults to `us`.
+* `lang`: language code for the result text. Defaults to `en-us`.
 
@@ -144,4 +146,5 @@ Example:
   num: 2,
-  device: store.device.IPAD,
-  country : 'us'
+  page: 3,
+  country : 'us',
+  lang: 'lang'
 })
@@ -148,0 +151,0 @@ .then(console.log)

test/lib.search.js

@@ -13,2 +13,15 @@ 'use strict';
 
+  it('should properly paginate results', () => {
+    const p1 = store.search({term: 'Panda', num: 10});
+    const p2 = store.search({term: 'Panda', num: 10, page: 2});
+    return Promise.all([p1, p2])
+      .then(([apps1, apps2]) => {
+        assert.equal(10, apps1.length);
+        assert.equal(10, apps2.length);
+        apps1.map(assertValidApp);
+        apps2.map(assertValidApp);
+        assert.notEqual(apps1[0].appId, apps2[0].appId);
+      });
+  });
+
   it('should fetch a valid application list in fr country', () => {
@@ -15,0 +28,0 @@ return store.search({country: 'fr', term: 'Panda vs Zombies'})

No alert changes

Improved metrics

Total package byte prevSize

33952

increased by2.45%

Lines of code

757

increased by3.13%

Number of lines in readme file

298

increased by1.02%

No dependency changes