Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

climate-data-crawler

Package Overview
Dependencies
Maintainers
1
Versions
17
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

climate-data-crawler - npm Package Compare versions

Comparing version 1.0.3 to 1.1.0

cdoApiClientFactory.js

56

cdoApiClient.js

@@ -5,22 +5,31 @@ "use strict";

function CdoApiClient(httpClient, logger, eventEmitter, timer) {
var apiQueryPath;
function CdoApiClient(httpClient, logger, eventEmitter, timer,
locationid, dataset, datatypeid, startDate, endDate) {
var queryResults;
var resultDelegate;
function makeRequest(offset) {
if(offset === 1){
var queryPath =
'/cdo-web/api/v2/data?datasetid=' + dataset
+ '&locationid=' + locationid
+ '&startdate=' + startDate
+ '&enddate=' + endDate
+ '&datatypeid=' + datatypeid
+ '&limit=1000';
function queryNext(offset) {
if (!offset){
offset = 1;
queryResults = [];
}
var apiToken = readApiToken();
var options = {
host : 'www.ncdc.noaa.gov',
port : 80,
path : apiQueryPath + '&offset=' + offset,
path : queryPath + '&offset=' + offset,
method : 'GET',
headers: {'token': apiToken}
headers: {'token': readApiToken()}
};
logger.info(options.path);
httpClient.request(options, onRequestCompleted, function(error) {

@@ -50,6 +59,8 @@ logger.error(error);

eventEmitter.emit('done', queryResults);
resultDelegate(queryResults);
}
else {
timer.setTimeout(function () {
makeRequest(nextRequestOffset);
console.log(nextRequestOffset);
queryNext(nextRequestOffset);
}, 1500);

@@ -61,2 +72,3 @@ }

eventEmitter.emit('done', null);
resultDelegate(null);
}

@@ -67,12 +79,11 @@ }

// privileged functions
this.query = function(queryParams, offset) {
apiQueryPath =
'/cdo-web/api/v2/data?datasetid=' + queryParams.dataset
+ '&locationid=' + queryParams.locationId
+ '&startdate=' + queryParams.startDate
+ '&enddate=' + queryParams.endDate
+ '&datatypeid=' + queryParams.datatypeid
+ '&limit=1000';
this.query = function(resultCallback) {
if (resultCallback) {
resultDelegate = resultCallback;
}
else {
resultDelegate = function(){};
}
makeRequest(offset ? offset : 1);
queryNext();
};

@@ -85,3 +96,4 @@

CdoApiClient.createInstance = function(eventEmitter, timer){
CdoApiClient.createInstance = function(locationid, dataset, datatypeid, startDate, endDate,
httpClient, logger, eventEmitter, timer){
var events = require('events');

@@ -92,7 +104,9 @@ var HttpClient = require('./helpers/httpClient');

return new CdoApiClient(
new HttpClient(), new Logger(),
httpClient ? httpClient : new HttpClient(),
logger ? logger : new Logger(),
eventEmitter ? eventEmitter : new events.EventEmitter(),
timer ? timer : new Timer());
timer ? timer : new Timer(),
locationid, dataset, datatypeid, startDate, endDate);
};
module.exports = CdoApiClient;

85

cdoDataProbingQuery.js
"use strict";
var CdoApiClient = require('./cdoApiClient');
var CdoApiClientFactory = require('./cdoApiClientFactory');
var Timer = require('./helpers/timer');
function CdoDataProbingQuery(cdoApiClient, timer, params) {
var queryYear = params.startYear;
function CdoDataProbingQuery(
cdoApiClientFactory, timer,
locationId, dataset, datatypeid, startYear, endYear) {
var apiParameters = {
dataset: params.dataset,
datatypeid: params.datatypeid,
locationId: params.locationId,
startDate: queryYear + '-01-01',
endDate: queryYear + '-12-31'
var queryYear = startYear;
var onQueryComplete;
var onApiCallComplete = function(result){
if (result){
onQueryComplete(result);
}
else {
queryYear--;
if (queryYear >= endYear){
timer.setTimeout(function(){
queryNext()
}, 1000);
}
else {
onQueryComplete(null);
}
}
};
var queryNext = function(){
var cdoApiClient = cdoApiClientFactory.createInstance(
locationId, dataset, datatypeid,
queryYear + '-01-01', queryYear + '-12-31');
cdoApiClient.query(onApiCallComplete);
};
// privileged functions
this.run = function(onQueryCompleteCallback){
if (!onQueryCompleteCallback) onQueryCompleteCallback = function(){};
if (onQueryCompleteCallback){
onQueryComplete = onQueryCompleteCallback;
}
else {
onQueryComplete = function(){}
}
var onApiClientDone = function(result){
if (result){
onQueryCompleteCallback(result);
}
else {
queryYear--;
if (queryYear >= params.endYear){
timer.setTimeout(function(){
apiParameters.startDate = queryYear + '-01-01';
apiParameters.endDate = queryYear + '-12-31';
cdoApiClient.query(apiParameters);
}, 1000);
}
else {
onQueryCompleteCallback(null);
}
}
};
cdoApiClient.getEventEmitter().on('done', onApiClientDone);
cdoApiClient.query(apiParameters);
queryNext();
}
}
CdoDataProbingQuery.createInstance = function(
locationId, dataset, datatypeid, startYear, endYear){
var events = require('events');
var eventEmitter = new events.EventEmitter();
var timer = new Timer();
var apiClient = CdoApiClient.createInstance(eventEmitter, timer);
CdoDataProbingQuery.createInstance = function(locationId, dataset, datatypeid, startYear, endYear){
return new CdoDataProbingQuery(
apiClient, timer, {
locationId: locationId,
dataset: dataset,
datatypeid: datatypeid,
startYear: startYear,
endYear: endYear
});
CdoApiClientFactory, new Timer(),
locationId, dataset, datatypeid, startYear, endYear);
};
module.exports = CdoDataProbingQuery;

19

cdoDataProbingQueryFactory.js

@@ -5,22 +5,11 @@ "use strict";

var CdoApiClient = require('./cdoApiClient');
var CdoApiClientFactory = require('./cdoApiClientFactory');
exports.createInstance = function(
locationId, dataset, datatypeid, startYear, endYear){
var events = require('events');
var eventEmitter = new events.EventEmitter();
var HttpClient = require('./helpers/httpClient');
var Logger = require('./helpers/logger');
var timer = new Timer();
var apiClient =
new CdoApiClient(new HttpClient(), new Logger(), eventEmitter, timer);
return new CdoProbingDataQuery(
CdoApiClientFactory, new Timer(),
locationId, dataset, datatypeid, startYear, endYear);
return new CdoProbingDataQuery(
apiClient, timer, {
locationId: locationId,
dataset: dataset,
datatypeid: datatypeid,
startYear: startYear,
endYear: endYear
});
};

2

package.json
{
"name": "climate-data-crawler",
"version": "1.0.3",
"version": "1.1.0",
"description": "Data Crawler for CDO (Climate Data Online) web services",

@@ -5,0 +5,0 @@ "main": "app.js",

19

README.md

@@ -136,19 +136,8 @@ # Climate Data Crawler - CDO (Climate Data Online) data crawler

var CdoApiClient = require('./cdoApiClient');
var events = require('events');
var apiClient = CdoApiClient.createInstance();
var client = CdoApiClient.createInstance('CITY:AS000002', 'GHCNDMS', 'MNTM', '2014-01-01', '2014-12-31');
var parameters = {
dataset: 'GHCNDMS',
datatypeid: 'MNTM',
locationId: CITY:AS000002,
startDate: '2014-01-01',
endDate: '2014-12-31'
};
ngdcApiClient.getEventEmitter().on('done', function(result){
console.log(result);
client.query(function(result){
console.log(result);
});
ngdcApiClient.query(parameters);
```

@@ -179,2 +168,4 @@

[NCDC's (National Climatic Date Center) CDO (Climate Data Online) web services v2](https://www.ncdc.noaa.gov/cdo-web/webservices/v2)
[Wikipedia: Global Historical Climatology Network](http://en.wikipedia.org/wiki/Global_Historical_Climatology_Network)

47

tests/system/cdoApiclientTests.js

@@ -5,7 +5,3 @@ "use strict";

var sinon = require('sinon');
var cdoApiClient = require('../../cdoApiClient');
var events = require('events');
var Logger = require('../../helpers/logger');
var HttpClient = require('../../helpers/httpClient');
var Timer = require('../../helpers/timer');
var CdoApiClient = require('../../cdoApiClient');

@@ -15,15 +11,2 @@ describe('CdoApiClient', function(){

var getInstance = function() {
var httpClient = new HttpClient();
var logger = new Logger();
sinon.stub(logger, 'info');
var eventEmitter = new events.EventEmitter();
var timer = new Timer();
return new cdoApiClient(
httpClient, logger, eventEmitter, timer);
};
it('should raise done event', function(done){

@@ -33,12 +16,10 @@ // arrange

var api = getInstance();
var eventEmitter = api.getEventEmitter();
var client = CdoApiClient.createInstance('GHCNDMS',
'MNTM', 'CITY:BR000023', '1983-01-01', '1983-12-31');
var eventEmitter = client.getEventEmitter();
// act
api.query({
dataset: 'GHCNDMS',
datatypeid: 'MNTM',
locationId: 'CITY:BR000023',
startDate: '1983-01-01',
endDate: '1983-12-31'
client.query(function(result){
console.log(result);
});

@@ -58,13 +39,9 @@

var api = getInstance();
var eventEmitter = api.getEventEmitter();
var client = CdoApiClient.createInstance('GHCNDMS',
'MNTM', 'CITY:BR000023', '2000-01-01', '2000-12-31');
var eventEmitter = client.getEventEmitter();
// act
api.query({
dataset: 'GHCNDMS',
datatypeid: 'MNTM',
locationId: 'CITY:BR000023',
startDate: '2000-01-01',
endDate: '2000-12-31'
});
client.query();

@@ -71,0 +48,0 @@ // assert

15

tests/system/cdoDataProbingQueryTests.js
"use strict";
var assert = require('assert');
var should = require('should');
var sinon = require('sinon');

@@ -16,8 +15,9 @@ var cdoDataProbingQuery = require('../../cdoDataProbingQuery');

var brisbane = 'CITY:AS000002';
var crawler = cdoDataProbingQuery.createInstance(brisbane, 'GHCNDMS', 'MNTM', 2014);
var query = cdoDataProbingQuery.createInstance(
brisbane, 'GHCNDMS', 'MNTM', 2014, 2014);
// act + assert
crawler.run(function(result){
query.run(function(result){
console.log(result);
result.should.not.be.null;
assert.notEqual(result, null);
done();

@@ -33,8 +33,9 @@ });

var rioDeJaneiro = 'CITY:BR000023'; // 1988 is the most recent result in Rio de Janeiro
var crawler = cdoDataProbingQuery.createInstance(rioDeJaneiro, 'GHCNDMS', 'MNTM', 2014, 1980);
var query = cdoDataProbingQuery.createInstance(
rioDeJaneiro, 'GHCNDMS', 'MNTM', 2014, 1980);
// act + assert
crawler.run(function(result){
query.run(function(result){
console.log(result);
result.should.not.be.null;
assert.notEqual(result, null);
done();

@@ -41,0 +42,0 @@ });

471

tests/unit/cdoApiClientTests.js
"use strict";
var should = require('should');
var sinon = require('sinon');
var assert = require('assert');
var CdoApiClient = require('../../cdoApiClient');

@@ -14,3 +15,2 @@ var events = require('events');

var eventEmitter;
var queryParams;
var logger;

@@ -21,12 +21,16 @@ var httpClient;

var dataset;
var datatypeid;
var locationId;
var startDate;
var endDate;
beforeEach(function(){
eventEmitter = new events.EventEmitter();
queryParams = {
dataset: 'GHCND',
datatypeid: 'PRCP',
locationId: 'CITY:AS000002',
startDate: '2012-01-01',
endDate: '2012-06-01'
};
dataset = 'GHCND';
datatypeid = 'PRCP';
locationId = 'CITY:AS000002';
startDate = '2012-01-01';
endDate = '2012-06-01';

@@ -69,32 +73,11 @@ logger = new Logger();

var getInstance = function(){
var client = new CdoApiClient(
httpClient, logger, eventEmitter, timer);
return client;
return new CdoApiClient(
httpClient, logger, eventEmitter, timer,
locationId, dataset, datatypeid, startDate, endDate);
};
describe('#invoke', function() {
it('path should use supplied offset value', function(){
sinon.stub(httpClient, 'request');
var client = getInstance();
var expected1 = '/cdo-web/api/v2/data?datasetid=GHCND&locationid=CITY:AS000002&startdate=2012-01-01&enddate=2012-06-01&datatypeid=PRCP&limit=1000&offset=1';
var expected2 = '/cdo-web/api/v2/data?datasetid=GHCND&locationid=CITY:AS000002&startdate=2012-01-01&enddate=2012-06-01&datatypeid=PRCP&limit=1000&offset=7';
// act
client.query(queryParams, 1);
client.query(queryParams, 7);
// assert
httpClient.request.getCall(0).args[0].path
.should.equal(expected1);
httpClient.request.getCall(1).args[0].path
.should.equal(expected2);
});
it('path should use configuration\'s dataset value', function() {
it('should use dataset from constructor argument', function() {
// arrange
queryParams.dataset = "mydataset";
dataset = "mydataset";

@@ -114,3 +97,3 @@ sinon.stub(httpClient, 'request');

// act
client.query(queryParams);
client.query();

@@ -122,5 +105,5 @@ // assert

it('path should use configuration\'s locationid value', function() {
it('should use locationid from constructor argument', function() {
// arrange
queryParams.locationId = "AO:MYCITY";
locationId = "AO:MYCITY";

@@ -140,3 +123,3 @@ sinon.stub(httpClient, 'request');

// act
client.query(queryParams);
client.query();

@@ -148,5 +131,5 @@ // assert

it('should use configuration\'s enddate value', function() {
it('should use enddate from constructor parameter', function() {
// arrange
queryParams.endDate = "23-10-1983";
endDate = "23-10-1983";

@@ -166,3 +149,3 @@ sinon.stub(httpClient, 'request');

// act
client.query(queryParams);
client.query();

@@ -174,5 +157,5 @@ // assert

it('should use configuration\'s startdate value', function() {
it('should use startdate from constructor argument', function() {
// arrange
queryParams.startDate = "23-10-1983";
startDate = "23-10-1983";

@@ -192,3 +175,3 @@ sinon.stub(httpClient, 'request');

// act
client.query(queryParams);
client.query();

@@ -200,3 +183,3 @@ // assert

it('should make http request', function(){
it('should use httpClient to make http request', function(){
// arrange

@@ -207,3 +190,3 @@ sinon.stub(httpClient, 'request');

// act
client.query(queryParams);
client.query();

@@ -224,3 +207,3 @@ // assert

// act
client.query(queryParams);
client.query();

@@ -231,19 +214,6 @@ // assert

it('should make another api call if there is still data left to query', function(){
it('should use httpClient using correct options', function(){
// arrange
var apiResult = {
'metadata': {
'resultset': {
'limit': 25,
'count': 100,
'offset': 1
}
}
};
var counter = 0;
var stubRequest = function(options, successCallback, errorCallback) {
var resultset = apiResult.metadata.resultset;
resultset.offset = (counter * resultset.limit) + 1;
counter++;
successCallback(JSON.stringify(apiResult));
successCallback(JSON.stringify({}));
};

@@ -253,196 +223,241 @@ sinon.stub(httpClient, 'request', stubRequest);

var client = getInstance();
sinon.spy(client, 'query');
var expectedOptions = {
host : 'www.ncdc.noaa.gov',
port : 80,
path : '/cdo-web/api/v2/data?datasetid=GHCND&locationid=CITY:AS000002&startdate=2012-01-01&enddate=2012-06-01&datatypeid=PRCP&limit=1000&offset=1',
method : 'GET',
headers: {'token': apiKey}
};
// act
client.query(queryParams);
client.query();
// assert
var queryPath = '/cdo-web/api/v2/data?datasetid=GHCND&locationid=CITY:AS000002&startdate=2012-01-01&enddate=2012-06-01&datatypeid=PRCP&limit=1000';
httpClient.request.calledWith(expectedOptions)
.should.be.true;
});
httpClient.request.getCall(0).args[0].path
.should.equal(queryPath + '&offset=1');
describe("data paging functionality", function(){
it('should make another httpRequest against api if there is more data to query (data paging)', function(){
// arrange
var apiResult = {
'metadata': {
'resultset': {
'limit': 25,
'count': 100,
'offset': 1
}
}
};
var counter = 0;
var stubRequest = function(options, successCallback, errorCallback) {
var resultset = apiResult.metadata.resultset;
resultset.offset = (counter * resultset.limit) + 1;
counter++;
successCallback(JSON.stringify(apiResult));
};
sinon.stub(httpClient, 'request', stubRequest);
httpClient.request.getCall(1).args[0].path
.should.equal(queryPath + '&offset=26');
var client = getInstance();
sinon.spy(client, 'query');
httpClient.request.getCall(2).args[0].path
.should.equal(queryPath + '&offset=51');
// act
client.query();
httpClient.request.getCall(3).args[0].path
.should.equal(queryPath + '&offset=76');
});
// assert
var queryPath = '/cdo-web/api/v2/data?datasetid=GHCND&locationid=CITY:AS000002&startdate=2012-01-01&enddate=2012-06-01&datatypeid=PRCP&limit=1000';
it('should make delayed api calls in case there is more data to query', function(){
// arrange
var apiResult = {
'metadata': {
'resultset': {
'limit': 25,
'count': 50,
'offset': 1
httpClient.request.getCall(0).args[0].path
.should.equal(queryPath + '&offset=1');
httpClient.request.getCall(1).args[0].path
.should.equal(queryPath + '&offset=26');
httpClient.request.getCall(2).args[0].path
.should.equal(queryPath + '&offset=51');
httpClient.request.getCall(3).args[0].path
.should.equal(queryPath + '&offset=76');
});
it('should make delayed api calls in case there is more data to query', function(){
// arrange
var apiResult = {
'metadata': {
'resultset': {
'limit': 25,
'count': 50,
'offset': 1
}
}
}
};
var counter = 0;
var stubRequest = function(options, successCallback, errorCallback) {
var resultset = apiResult.metadata.resultset;
resultset.offset = (counter * resultset.limit) + 1;
counter++;
successCallback(JSON.stringify(apiResult));
};
};
var counter = 0;
var stubRequest = function(options, successCallback, errorCallback) {
var resultset = apiResult.metadata.resultset;
resultset.offset = (counter * resultset.limit) + 1;
counter++;
successCallback(JSON.stringify(apiResult));
};
sinon.stub(httpClient, 'request', stubRequest);
var client = getInstance();
sinon.stub(httpClient, 'request', stubRequest);
var client = getInstance();
// act
client.query(queryParams);
// act
client.query();
// assert
timer.setTimeout.getCall(0).args[1].should.be.greaterThan(1000);
// assert
timer.setTimeout.getCall(0).args[1].should.be.greaterThan(1000);
});
});
it('should emit done event when there is no more data to query', function(){
// arrange
var apiResult = {
'metadata': {
'resultset': {
'limit': 25,
'count': 15,
'offset': 1
describe('emit event', function(){
it('should emit done when there is no more data to query', function(){
// arrange
var apiResult = {
'metadata': {
'resultset': {
'limit': 25,
'count': 15,
'offset': 1
}
}
}
};
var stubRequest = function(options, successCallback, errorCallback) {
successCallback(JSON.stringify(apiResult));
};
};
var stubRequest = function(options, successCallback, errorCallback) {
successCallback(JSON.stringify(apiResult));
};
sinon.stub(httpClient, 'request', stubRequest);
sinon.spy(eventEmitter, 'emit');
sinon.stub(httpClient, 'request', stubRequest);
sinon.spy(eventEmitter, 'emit');
var client = getInstance();
var client = getInstance();
// act
client.query(queryParams);
// act
client.query();
// assert
eventEmitter.emit.calledWith('done').should.be.true;
});
// assert
assert.equal(eventEmitter.emit.calledWith('done'), true);
});
it('done event should contain result of query', function(){
// arrange
var apiResult = {
"results": [
{
"id": "CITY:AE000002",
"name": "Ajman, AE",
"datacoverage": 0.6862,
"mindate": "1944-03-01",
"maxdate": "2015-01-01"
},
{
"id": "CITY:AE000003",
"name": "Dubai, AE",
"datacoverage": 0.6862,
"mindate": "1944-03-01",
"maxdate": "2015-01-01"
it('event should contain api results', function(){
// arrange
var expectedResults = {
"results": [
{"example": [1,2,3] }, { "example": [4,5,6]}
],
'metadata': {
'resultset': {
'limit': 25,
'count': 2,
'offset': 1
}
}
],
'metadata': {
'resultset': {
'limit': 25,
'count': 2,
'offset': 1
}
}
};
var stubRequest = function(options, successCallback, errorCallback) {
successCallback(JSON.stringify(apiResult));
};
sinon.stub(httpClient, 'request', stubRequest);
var spy = sinon.spy(eventEmitter, 'emit');
};
var client = getInstance();
var stubRequest = function(options, successCallback, errorCallback) {
successCallback(JSON.stringify(expectedResults));
};
sinon.stub(httpClient, 'request', stubRequest);
var spy = sinon.spy(eventEmitter, 'emit');
// act
client.query(queryParams);
var client = getInstance();
// assert
spy.calledWith('done', apiResult.results)
.should.be.true;
});
// act
client.query();
it('should read api-token from file', function(){
// arrange
var client = getInstance();
// assert
assert.equal(spy.calledWith('done', expectedResults.results), true);
});
// act
client.query(queryParams);
it('should handle empty result {} from api and emit done', function(){
// arrange
var apiResult = {};
var stubRequest = function(options, successCallback, errorCallback) {
successCallback(JSON.stringify(apiResult));
};
sinon.stub(httpClient, 'request', stubRequest);
var spy = sinon.spy(eventEmitter, 'emit');
// assert
var call = fs.readFileSync.getCall(0);
call.args[0].should.equal('apitoken.txt');
call.args[1].should.equal('utf8');
});
var client = getInstance();
it('should write helpful message when there is no api-token file', function(){
// arrange
fs.readFileSync.restore();
sinon.stub(fs, 'readFileSync', function(filepath, encoding){
throw new Error('ENOENT, no such file or directory');
// act
client.query();
// assert
spy.calledWith('done', null).should.be.true;
});
var expectedMessage = "Please make sure there is an apitoken.txt file (containing your API token) in the executing directory.";
});
var client = getInstance();
describe('result callback', function(){
it('should return api result', function(){
// arrange
var expectedResults = {
"results": [
{"example": [1,2,3] }, { "example": [4,5,6]}
],
'metadata': {
'resultset': {
'limit': 25,
'count': 2,
'offset': 1
}
}
};
// act
client.query(queryParams);
var stubRequest = function(options, successCallback, errorCallback) {
successCallback(JSON.stringify(expectedResults));
};
sinon.stub(httpClient, 'request', stubRequest);
// assert
logger.info.called.should.be.true;
logger.info.calledWith(expectedMessage).should.be.true;
var actualResults = null;
var resultCallback = function(result){
actualResults = result;
};
var client = getInstance();
// act
client.query(resultCallback);
// assert
assert.equal(JSON.stringify(actualResults),
JSON.stringify(expectedResults.results));
});
});
it('should query http client using correct options', function(){
// arrange
var stubRequest = function(options, successCallback, errorCallback) {
successCallback(JSON.stringify({}));
};
sinon.stub(httpClient, 'request', stubRequest);
describe('api token', function(){
it('should read api-token from file', function(){
// arrange
var client = getInstance();
var client = getInstance();
// act
client.query();
var expectedOptions = {
host : 'www.ncdc.noaa.gov',
port : 80,
path : '/cdo-web/api/v2/data?datasetid=GHCND&locationid=CITY:AS000002&startdate=2012-01-01&enddate=2012-06-01&datatypeid=PRCP&limit=1000&offset=1',
method : 'GET',
headers: {'token': apiKey}
};
// assert
var call = fs.readFileSync.getCall(0);
call.args[0].should.equal('apitoken.txt');
call.args[1].should.equal('utf8');
});
// act
client.query(queryParams);
it('should write helpful message when there is no api-token file', function(){
// arrange
fs.readFileSync.restore();
sinon.stub(fs, 'readFileSync', function(filepath, encoding){
throw new Error('ENOENT, no such file or directory');
});
// assert
httpClient.request.calledWith(expectedOptions)
.should.be.true;
});
var expectedMessage = "Please make sure there is an apitoken.txt file (containing your API token) in the executing directory.";
it('should handle empty result {} from api and emit done', function(){
// arrange
var apiResult = {};
var stubRequest = function(options, successCallback, errorCallback) {
successCallback(JSON.stringify(apiResult));
};
sinon.stub(httpClient, 'request', stubRequest);
var spy = sinon.spy(eventEmitter, 'emit');
var client = getInstance();
var client = getInstance();
// act
client.query();
// act
client.query(queryParams);
// assert
spy.calledWith('done', null).should.be.true;
// assert
assert.equal(logger.info.called, true);
assert.equal(logger.info.calledWith(expectedMessage), true);
});
});

@@ -465,15 +480,35 @@

// arrange
var apiClient = CdoApiClient.createInstance();
var apiClient = CdoApiClient.createInstance(
dataset, datatypeid, locationId, startDate, endDate);
// assert
apiClient.should.not.be.null;
})
});
it('return not null (eventEmitter and timer given)', function(){
it('should use the factory arguments and pass to httpClient', function() {
// arrange
var apiClient = CdoApiClient.createInstance(new events.EventEmitter(), new Timer());
sinon.stub(httpClient, 'request');
var client = CdoApiClient.createInstance(
locationId, dataset, datatypeid, startDate, endDate,
httpClient);
var expected = '/cdo-web/api/v2/data?'
+ 'datasetid=GHCND'
+ '&locationid=CITY:AS000002'
+ '&startdate=2012-01-01'
+ '&enddate=2012-06-01'
+ '&datatypeid=PRCP'
+ '&limit=1000'
+ '&offset=1';
// act
client.query();
// assert
apiClient.should.not.be.null;
})
assert.equal(httpClient.request.called, true);
httpClient.request.getCall(0).args[0].path
.should.equal(expected);
});
});

@@ -480,0 +515,0 @@

249

tests/unit/cdoDataProbingQueryTests.js

@@ -20,3 +20,12 @@ "use strict";

var timer;
var apiClientFactory;
var dataset;
var datatypeid;
var locationId;
var startYear;
var endYear;
var startDate;
var endDate;
beforeEach(function(){

@@ -39,2 +48,16 @@ eventEmitter = new events.EventEmitter();

httpClient, logger, eventEmitter,timer);
apiClientFactory = {
createInstance: function(){
return api;
}
};
dataset = 'GHCNDMS';
datatypeid = 'MMNT';
locationId = 'CITY:BR000023';
startYear = 2014;
endYear = 2014;
startDate = startYear + '-01-01';
endDate = endYear + '-12-31';
});

@@ -54,140 +77,168 @@

var getInstance = function(locationId, dataset, datatypeid, startYear, endYear){
var getInstance = function(){
return new CdoDataProbingQuery(
api, timer, {
locationId: locationId,
dataset: dataset,
datatypeid: datatypeid,
startYear: startYear,
endYear: endYear
});
apiClientFactory, timer,
locationId, dataset, datatypeid, startYear, endYear);
};
describe('#run', function(){
it('should call apiClient\'s query method', function() {
it('should call apiClientFactory with expected parameters', function() {
// arrange
sinon.stub(api, 'query');
sinon.spy(apiClientFactory, 'createInstance');
var params = {
dataset: 'GHCNDMS',
datatypeid: 'MMNT',
locationId: 'CITY:BR000023',
startDate: '2014-01-01',
endDate: '2014-12-31'
};
var query = getInstance();
var crawler = getInstance(
params.locationId, params.dataset,
params.datatypeid, 2014);
// act
crawler.run();
query.run();
// assert
api.query.calledWith(params)
.should.be.true;
var isCalledWithParameters = apiClientFactory.createInstance.calledWith(
locationId, dataset, datatypeid, startDate, endDate);
assert.equal(isCalledWithParameters, true);
});
it('should call onRunCompleteCallback with results when receiving data from api client', function(){
it('should invoke apiClient\'s query method', function() {
// arrange
var dataset = fs.readFileSync('test-resources/dataset.json', {encoding: 'utf8'});
sinon.stub(api, 'query');
sinon.stub(api, 'query', function(){
eventEmitter.emit('done', dataset);
});
var query = getInstance();
var runResults;
var onRunComplete = function(results){
runResults = results;
};
// act
var query = getInstance('CITY:BR000023', 'GHCNDMS', 'MMNT', 2014);
query.run(onRunComplete);
query.run();
// assert
assert.notEqual(runResults, null);
runResults.should.be.equal(dataset);
assert.equal(api.query.called, true);
});
it('should be possible to call run without specifying onRunCompletecallback', function(){
// arrange
var dataset = fs.readFileSync('test-resources/dataset.json', {encoding: 'utf8'});
describe('#onQueryCompleteCallback', function(){
it('should call onQueryCompleteCallback with results when receiving data from api client', function(){
// arrange
var expectedRunResults = fs.readFileSync('test-resources/dataset.json', {encoding: 'utf8'});
var actualRunResults = null;
sinon.stub(api, 'query', function(){
eventEmitter.emit('done', dataset);
sinon.stub(api, 'query', function(onApiCallComplete){
onApiCallComplete(expectedRunResults);
});
var query = getInstance();
// act
query.run(function(results){
actualRunResults = results;
});
// assert
assert.notEqual(actualRunResults, null);
assert.equal(actualRunResults, expectedRunResults);
});
// act
var query = getInstance('CITY:BR000023', 'GHCNDMS', 'MMNT', 2014);
it('should be possible to call run without specifying onQueryCompleteCallback', function(){
// arrange
var runResults = fs.readFileSync('test-resources/dataset.json', {encoding: 'utf8'});
// assert
assert.doesNotThrow(function(){
query.run();
sinon.stub(api, 'query', function(onApiCallComplete){
onApiCallComplete(runResults);
});
var query = getInstance();
// act + assert
assert.doesNotThrow(function(){
query.run();
});
});
});
it('should try one year earlier (until endDate) if no data from server', function(){
// arrange
sinon.stub(api, 'query', function(){
eventEmitter.emit('done', null);
describe('api returns no data for the current year\'s query', function(){
it('should query each year until it reaches endYear to probe for data', function(){
// arrange
sinon.stub(api, 'query', function(onApiCallComplete){
onApiCallComplete(null);
});
sinon.spy(apiClientFactory, 'createInstance');
startYear = 2014;
endYear = 2000;
var expectedQueryCount = (startYear - endYear) + 1;
var query = getInstance();
// act
query.run();
// assert
api.query.callCount.should.be.equal(expectedQueryCount);
for (var i = 0; i < expectedQueryCount; i++){
var currentQueryYear = startYear - i;
var expectedStartDate = currentQueryYear + '-01-01';
var expectedEndDate = currentQueryYear + '-12-31';
var isCalledWithParameters = apiClientFactory.createInstance.calledWith(
locationId, dataset, datatypeid, expectedStartDate, expectedEndDate);
assert.equal(isCalledWithParameters, true);
}
});
var startYear = 2014;
var endYear = 2000;
var queryCount = (startYear - endYear) + 1;
describe('#onRunCompleteCallback', function(){
it('should invoke onRunCompleteCallback when no data until endYear', function(){
// arrange
sinon.stub(api, 'query', function(onApiCallComplete){
onApiCallComplete(null);
});
// act
var query = getInstance(
'CITY:BR000023', 'GHCNDMS', 'MMNT', startYear, endYear);
query.run();
var onRunCompleteCallbackCalled = false;
// assert
var finalCallParameters = {
dataset: 'GHCNDMS',
datatypeid: 'MMNT',
locationId: 'CITY:BR000023',
startDate: endYear + '-01-01',
endDate: endYear + '-12-31'
};
startYear = 2014;
endYear = 2000;
api.query.calledWith(finalCallParameters).should.be.true;
api.query.callCount.should.be.equal(queryCount);
});
var query = getInstance();
it('should still query onRunCompleteCallback even if there was no data returned from data until 1983', function(){
// arrange
sinon.stub(api, 'query', function(){
eventEmitter.emit('done', null);
// act
query.run(function(results){
onRunCompleteCallbackCalled = true;
});
// assert
assert.equal(onRunCompleteCallbackCalled, true);
});
});
var runResults;
var callbackCalled = false;
var onRunComplete = function(results){
runResults = results;
callbackCalled = true;
};
it('should pause shortly before probing next year', function() {
// arrange
sinon.stub(api, 'query', function(onApiCallComplete){
onApiCallComplete(null);
});
// act
var query = getInstance('CITY:BR000023', 'GHCNDMS', 'MMNT', 2014);
query.run(onRunComplete);
startYear = 2014;
endYear = 2000;
var expectedCallCount = startYear - endYear;
var actualCallCount = 0;
// assert
callbackCalled.should.be.true;
assert.equal(runResults, null);
var actualDelay = null;
});
timer.setTimeout.restore();
sinon.stub(timer, 'setTimeout', function(callback, delay){
actualDelay = delay;
actualCallCount++;
callback();
});
it('should pause shortly before doing next query if current year does not have results', function() {
// arrange
sinon.stub(api, 'query', function(){
eventEmitter.emit('done', null);
var query = getInstance();
// act
query.run();
// assert
assert.equal(actualDelay, 1000);
assert.equal(actualCallCount, expectedCallCount);
});
// act
var query = getInstance('CITY:BR000023', 'GHCNDMS', 'MMNT', 2014, 2000);
query.run();
// assert
timer.setTimeout.getCall(0).args[1].should.be.greaterThan(500);
});

@@ -198,3 +249,3 @@

describe('#createInstance', function(){
it('should not return null or undefined', function(){
it('should not return null', function(){
// arrange

@@ -204,8 +255,6 @@ var query = CdoDataProbingQuery.createInstance('CITY:BR000023', 'GHCNDMS', 'MMNT', 2014, 2000);

// assert
query.should.not.be.null;
query.should.not.be.undefined;
assert.notEqual(query, null);
});
});
});
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc