app-store-scraper
Advanced tools
Comparing version 0.11.0 to 0.12.0
@@ -12,3 +12,3 @@ 'use strict'; | ||
const idValue = opts.id || opts.appId; | ||
resolve(common.lookup([idValue], idField, opts.country)); | ||
resolve(common.lookup([idValue], idField, opts.country, opts.requestOptions)); | ||
}) | ||
@@ -15,0 +15,0 @@ .then((results) => { |
@@ -46,6 +46,8 @@ 'use strict'; | ||
// TODO add an optional parse function | ||
const doRequest = (url, headers) => new Promise(function (resolve, reject) { | ||
debug('Making request: %s %j', url, headers); | ||
const doRequest = (url, headers, requestOptions) => new Promise(function (resolve, reject) { | ||
debug('Making request: %s %j %o', url, headers, requestOptions); | ||
request.get({url, headers}, (error, response, body) => { | ||
requestOptions = Object.assign({ method: 'GET' }, requestOptions); | ||
request(Object.assign({ url, headers }, requestOptions), (error, response, body) => { | ||
if (error) { | ||
@@ -56,3 +58,3 @@ debug('Request error', error); | ||
if (response.statusCode >= 400) { | ||
return reject({response}); | ||
return reject({ response }); | ||
} | ||
@@ -66,3 +68,3 @@ debug('Finished request'); | ||
function lookup (ids, idField, country) { | ||
function lookup (ids, idField, country, requestOptions) { | ||
idField = idField || 'id'; | ||
@@ -72,3 +74,3 @@ country = country || 'us'; | ||
const url = `${LOOKUP_URL}?${idField}=${joinedIds}&country=${country}`; | ||
return doRequest(url) | ||
return doRequest(url, {}, requestOptions) | ||
.then(JSON.parse) | ||
@@ -84,2 +86,2 @@ .then((res) => res.results.map(cleanApp)); | ||
module.exports = {cleanApp, lookup, request: doRequest, storeId}; | ||
module.exports = { cleanApp, lookup, request: doRequest, storeId }; |
@@ -34,3 +34,3 @@ 'use strict'; | ||
if (opts.fullDetail) { | ||
return common.lookup(R.pluck('id', apps), 'id', opts.country); | ||
return common.lookup(R.pluck('id', apps), 'id', opts.country, opts.requestOptions); | ||
} | ||
@@ -61,3 +61,3 @@ return apps.map(cleanApp); | ||
const url = `https://rss.itunes.apple.com/api/v1/${opts.country}/${opts.collection}/${opts.num}/explicit.json`; | ||
common.request(url) | ||
common.request(url, {}, opts.requestOptions) | ||
.then(JSON.parse) | ||
@@ -64,0 +64,0 @@ .then(processResults(opts)) |
@@ -31,13 +31,13 @@ 'use strict'; | ||
}) | ||
.then((id) => { | ||
opts = opts || {}; | ||
opts.sort = opts.sort || c.sort.RECENT; | ||
opts.page = opts.page || 1; | ||
opts.country = opts.country || 'us'; | ||
.then((id) => { | ||
opts = opts || {}; | ||
opts.sort = opts.sort || c.sort.RECENT; | ||
opts.page = opts.page || 1; | ||
opts.country = opts.country || 'us'; | ||
const url = `https://itunes.apple.com/${opts.country}/rss/customerreviews/page=${opts.page}/id=${id}/sortby=${opts.sort}/json`; | ||
return common.request(url); | ||
}) | ||
.then(JSON.parse) | ||
.then(cleanList); | ||
const url = `https://itunes.apple.com/${opts.country}/rss/customerreviews/page=${opts.page}/id=${id}/sortby=${opts.sort}/json`; | ||
return common.request(url, {}, opts.requestOptions); | ||
}) | ||
.then(JSON.parse) | ||
.then(cleanList); | ||
@@ -44,0 +44,0 @@ function validate (opts) { |
@@ -19,20 +19,25 @@ 'use strict'; | ||
function search (opts) { | ||
return new Promise(function (resolve) { | ||
return new Promise(function (resolve, reject) { | ||
if (!opts.term) { | ||
throw Error('term is required'); | ||
} | ||
const url = BASE_URL + opts.term; | ||
const url = BASE_URL + encodeURIComponent(opts.term); | ||
const storeId = common.storeId(opts.country); | ||
const lang = opts.lang || 'en-us'; | ||
common.request(url, { | ||
'X-Apple-Store-Front': `${storeId},24 t:native`, | ||
'Accept-Language': lang | ||
}) | ||
.then(JSON.parse) | ||
.then((response) => response.bubbles[0].results) | ||
.then(paginate(opts.num, opts.page)) | ||
.then(R.pluck('id')) | ||
.then((ids) => common.lookup(ids, 'id', opts.country)) | ||
.then(resolve); | ||
common.request( | ||
url, | ||
{ | ||
'X-Apple-Store-Front': `${storeId},24 t:native`, | ||
'Accept-Language': lang | ||
}, | ||
opts.requestOptions | ||
) | ||
.then(JSON.parse) | ||
.then((response) => (response.bubbles[0] && response.bubbles[0].results) || []) | ||
.then(paginate(opts.num, opts.page)) | ||
.then(R.pluck('id')) | ||
.then((ids) => common.lookup(ids, 'id', opts.country, opts.requestOptions)) | ||
.then(resolve) | ||
.catch(reject); | ||
}); | ||
@@ -39,0 +44,0 @@ } |
@@ -17,17 +17,21 @@ 'use strict'; | ||
}) | ||
.then((id) => common.request(`${BASE_URL}${id}`, { | ||
'X-Apple-Store-Front': `${common.storeId(opts.country)},32` | ||
})) | ||
.then(function (text) { | ||
const index = text.indexOf('customersAlsoBoughtApps'); | ||
if (index === -1) { | ||
return []; | ||
} | ||
.then((id) => common.request( | ||
`${BASE_URL}${id}`, | ||
{ | ||
'X-Apple-Store-Front': `${common.storeId(opts.country)},32` | ||
}, | ||
opts.requestOptions | ||
)) | ||
.then(function (text) { | ||
const index = text.indexOf('customersAlsoBoughtApps'); | ||
if (index === -1) { | ||
return []; | ||
} | ||
const slice = text.slice(index, index + 300).match(/\[(.*)\]/)[0]; | ||
const ids = JSON.parse(slice); | ||
return common.lookup(ids, 'id', opts.country); | ||
}); | ||
const slice = text.slice(index, index + 300).match(/\[(.*)\]/)[0]; | ||
const ids = JSON.parse(slice); | ||
return common.lookup(ids, 'id', opts.country, opts.requestOptions); | ||
}); | ||
} | ||
module.exports = similar; |
@@ -38,9 +38,9 @@ 'use strict'; | ||
return resolve(BASE_URL + opts.term); | ||
return resolve(BASE_URL + encodeURIComponent(opts.term)); | ||
}) | ||
.then(common.request) | ||
.then(parseXML) | ||
.then(extractSuggestions); | ||
.then(url => common.request(url, {}, opts.requestOptions)) | ||
.then(parseXML) | ||
.then(extractSuggestions); | ||
} | ||
module.exports = suggest; |
{ | ||
"name": "app-store-scraper", | ||
"version": "0.11.0", | ||
"version": "0.12.0", | ||
"description": "scrape data from the itunes app store", | ||
@@ -5,0 +5,0 @@ "main": "index.js", |
@@ -7,4 +7,4 @@ 'use strict'; | ||
function assertValidUrl (url) { | ||
return assert(validator.isURL(url, {allow_protocol_relative_urls: true}), | ||
`${url} is not a valid url`); | ||
return assert(validator.isURL(url, { allow_protocol_relative_urls: true }), | ||
`${url} is not a valid url`); | ||
} | ||
@@ -11,0 +11,0 @@ |
@@ -118,2 +118,17 @@ 'use strict'; | ||
}); | ||
it('should be able to set requestOptions', (done) => { | ||
store.app({ | ||
id: '553834731', | ||
requestOptions: { | ||
method: 'DELETE' | ||
} | ||
}) | ||
.then(() => done('should not resolve')) | ||
.catch((err) => { | ||
assert.equal(err.response.statusCode, 501); | ||
done(); | ||
}) | ||
.catch(done); | ||
}); | ||
}); |
@@ -63,2 +63,18 @@ 'use strict'; | ||
}); | ||
it('should be able to set requestOptions', (done) => { | ||
store.list({ | ||
collection: store.collection.TOP_FREE_GAMES_IOS, | ||
num: 5, | ||
requestOptions: { | ||
method: 'DELETE' | ||
} | ||
}) | ||
.then(() => done('should not resolve')) | ||
.catch((err) => { | ||
assert.equal(err.response.statusCode, 501); | ||
done(); | ||
}) | ||
.catch(done); | ||
}); | ||
}); |
@@ -45,2 +45,17 @@ 'use strict'; | ||
}); | ||
it('should be able to set requestOptions', (done) => { | ||
store.reviews({ | ||
id: '553834731', | ||
requestOptions: { | ||
method: 'DELETE' | ||
} | ||
}) | ||
.then(() => done('should not resolve')) | ||
.catch((err) => { | ||
assert.equal(err.response.statusCode, 501); | ||
done(); | ||
}) | ||
.catch(done); | ||
}); | ||
}); |
@@ -45,2 +45,17 @@ 'use strict'; | ||
}); | ||
it('should be able to set requestOptions', (done) => { | ||
store.search({ | ||
term: 'vr', | ||
requestOptions: { | ||
method: 'DELETE' | ||
} | ||
}) | ||
.then(() => done('should not resolve')) | ||
.catch((err) => { | ||
assert.equal(err.response.statusCode, 501); | ||
done(); | ||
}) | ||
.catch(done); | ||
}); | ||
}); |
@@ -37,2 +37,17 @@ 'use strict'; | ||
}); | ||
it('should be able to set requestOptions', (done) => { | ||
store.similar({ | ||
id: '553834731', | ||
requestOptions: { | ||
method: 'DELETE' | ||
} | ||
}) | ||
.then(() => done('should not resolve')) | ||
.catch((err) => { | ||
assert.equal(err.response.statusCode, 501); | ||
done(); | ||
}) | ||
.catch(done); | ||
}); | ||
}); |
@@ -12,2 +12,17 @@ 'use strict'; | ||
})); | ||
it('should be able to set requestOptions', (done) => { | ||
store.suggest({ | ||
term: 'p', | ||
requestOptions: { | ||
method: 'DELETE' | ||
} | ||
}) | ||
.then(() => done('should not resolve')) | ||
.catch((err) => { | ||
assert.equal(err.response.statusCode, 501); | ||
done(); | ||
}) | ||
.catch(done); | ||
}); | ||
}); |
36874
858