node-webhdfs
Advanced tools
Comparing version 0.0.2 to 0.0.3
@@ -5,3 +5,3 @@ { | ||
"description": "A WebHDFS module for Node.js.", | ||
"version": "0.0.2", | ||
"version": "0.0.3", | ||
"keywords": ["hdfs", "webhdfs", "http"], | ||
@@ -20,4 +20,4 @@ "maintainers": ["Ryan Cole <ryan@rycole.com> (http://rycole.com)"], | ||
"dependencies": { | ||
"underscore": "1.3.3", | ||
"request": "2.9.201" | ||
"underscore": ">=1.3.3", | ||
"request": ">=2.9.201" | ||
}, | ||
@@ -24,0 +24,0 @@ "engines": { |
@@ -5,10 +5,6 @@ A WebHDFS module for Node.js | ||
I am currently following the [WebHDFS REST API documentation](http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#Document+Conventions) for the `1.0.2` release, by Apache. | ||
I am currently following and testing against the [WebHDFS REST API documentation](http://hadoop.apache.org/docs/r1.2.1/webhdfs.html) for the `1.2.1` release, by Apache. Make sure you enable WebHDFS in the hdfs site configuration file. | ||
# Disclaimer | ||
I'm not an expert on HDFS. This module has been written as a requirement for a personal project. I am actively using the module and have written tests for it. Your mileage may vary. | ||
# Tests | ||
I use [Mocha](http://visionmedia.github.com/mocha/) and [should.js](https://github.com/visionmedia/should.js) for unit testing. They will be required if you want to run the unit tests. To execute the tests, simply `npm test`. | ||
I use [Mocha](http://visionmedia.github.com/mocha/) and [should.js](https://github.com/visionmedia/should.js) for unit testing. They will be required if you want to run the unit tests. To execute the tests, simply `npm test`, but install the requirements first. |
@@ -1,2 +0,1 @@ | ||
var querystring = require('querystring'), | ||
@@ -26,14 +25,29 @@ request = require('request'), | ||
// ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#DELETE | ||
WebHDFSClient.prototype.del = function (path, options, callback) { | ||
WebHDFSClient.prototype.del = function (path, hdfsoptions, requestoptions, callback) { | ||
// requestoptions may be omitted | ||
if (callback === undefined && typeof(requestoptions) === 'function') { | ||
callback = requestoptions; | ||
requestoptions = undefined; | ||
} | ||
// hdfsoptions may be omitted | ||
if (callback === undefined && typeof(hdfsoptions) === 'function') { | ||
callback = hdfsoptions; | ||
hdfsoptions = undefined; | ||
} | ||
// format request args | ||
var args = { | ||
var args = _.defaults({ | ||
json: true, | ||
uri: this.base_url + path, | ||
qs: { | ||
qs: _.defaults({ | ||
op: 'delete', | ||
recursive: options.recursive, | ||
'user.name': this.options.user | ||
} | ||
}; | ||
}, hdfsoptions || {}) | ||
}, requestoptions || {}); | ||
@@ -43,2 +57,5 @@ // send http request | ||
// forward request error | ||
if (error) return callback(error); | ||
// exception handling | ||
@@ -57,6 +74,27 @@ if ('RemoteException' in body) | ||
// ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#LISTSTATUS | ||
WebHDFSClient.prototype.listStatus = function (path, callback) { | ||
WebHDFSClient.prototype.listStatus = function (path, hdfsoptions, requestoptions, callback) { | ||
// requestoptions may be omitted | ||
if (callback === undefined && typeof(requestoptions) === 'function') { | ||
callback = requestoptions; | ||
requestoptions = undefined; | ||
} | ||
// hdfsoptions may be omitted | ||
if (callback === undefined && typeof(hdfsoptions) === 'function') { | ||
callback = hdfsoptions; | ||
hdfsoptions = undefined; | ||
} | ||
// format request args | ||
var args = { uri: this.base_url + path, qs: { op: 'liststatus' } } | ||
var args = _.defaults({ | ||
json:true, | ||
uri: this.base_url + path, | ||
qs: _.defaults({ | ||
op: 'liststatus' | ||
}, hdfsoptions || {}) | ||
}, requestoptions || {}); | ||
@@ -66,2 +104,5 @@ // send http request | ||
// forward request error | ||
if (error) return callback(error); | ||
// execute callback | ||
@@ -76,12 +117,28 @@ return callback(null, body.FileStatuses.FileStatus) | ||
// ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#GETFILESTATUS | ||
WebHDFSClient.prototype.getFileStatus = function (path, callback) { | ||
WebHDFSClient.prototype.getFileStatus = function (path, hdfsoptions, requestoptions, callback) { | ||
// requestoptions may be omitted | ||
if (callback === undefined && typeof(requestoptions) === 'function') { | ||
callback = requestoptions; | ||
requestoptions = undefined; | ||
} | ||
// hdfsoptions may be omitted | ||
if (callback === undefined && typeof(hdfsoptions) === 'function') { | ||
callback = hdfsoptions; | ||
hdfsoptions = undefined; | ||
} | ||
// format request args | ||
var args = { | ||
var args = _.defaults({ | ||
json: true, | ||
uri: this.base_url + path, | ||
qs: { | ||
qs: _.defaults({ | ||
op: 'getfilestatus' | ||
} | ||
}; | ||
}, hdfsoptions || {}) | ||
}, requestoptions || {}); | ||
@@ -91,2 +148,5 @@ // send http request | ||
// forward request error | ||
if (error) return callback(error); | ||
// exception handling | ||
@@ -105,12 +165,27 @@ if ('RemoteException' in body) | ||
// ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#GETCONTENTSUMMARY | ||
WebHDFSClient.prototype.getContentSummary = function (path, callback) { | ||
WebHDFSClient.prototype.getContentSummary = function (path, hdfsoptions, requestoptions, callback) { | ||
// requestoptions may be omitted | ||
if (callback === undefined && typeof(requestoptions) === 'function') { | ||
callback = requestoptions; | ||
requestoptions = undefined; | ||
} | ||
// hdfsoptions may be omitted | ||
if (callback===undefined && typeof(hdfsoptions) === 'function') { | ||
callback = hdfsoptions; | ||
hdfsoptions = undefined; | ||
} | ||
// format request args | ||
var args = { | ||
var args = _.defaults({ | ||
json: true, | ||
uri: this.base_url + path, | ||
qs: { | ||
qs: _.defaults({ | ||
op: 'getcontentsummary' | ||
} | ||
}; | ||
}, hdfsoptions || {}) | ||
}, requestoptions || {}); | ||
@@ -120,2 +195,5 @@ // send http request | ||
// forward request error | ||
if (error) return callback(error); | ||
// exception handling | ||
@@ -134,12 +212,28 @@ if ('RemoteException' in body) | ||
// ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#GETFILECHECKSUM | ||
WebHDFSClient.prototype.getFileChecksum = function (path, callback) { | ||
WebHDFSClient.prototype.getFileChecksum = function (path, hdfsoptions, requestoptions, callback) { | ||
// requestoptions may be omitted | ||
if (callback === undefined && typeof(requestoptions) === 'function') { | ||
callback = requestoptions; | ||
requestoptions = undefined; | ||
} | ||
// hdfsoptions may be omitted | ||
if (callback === undefined && typeof(hdfsoptions) === 'function') { | ||
callback = hdfsoptions; | ||
hdfsoptions = undefined; | ||
} | ||
// format request args | ||
var args = { | ||
var args = _.defaults({ | ||
json: true, | ||
uri: this.base_url + path, | ||
qs: { | ||
qs: _.defaults({ | ||
op: 'getfilechecksum' | ||
} | ||
}; | ||
}, hdfsoptions || {}) | ||
}, requestoptions || {}); | ||
@@ -149,2 +243,5 @@ // send http request | ||
// forward request error | ||
if (error) return callback(error); | ||
// exception handling | ||
@@ -162,6 +259,29 @@ if ('RemoteException' in body) | ||
// ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#GETHOMEDIRECTORY | ||
WebHDFSClient.prototype.getHomeDirectory = function (callback) { | ||
WebHDFSClient.prototype.getHomeDirectory = function (hdfsoptions, requestoptions, callback) { | ||
// requestoptions may be omitted | ||
if (callback === undefined && typeof(requestoptions) === 'function') { | ||
callback = requestoptions; | ||
requestoptions = undefined; | ||
} | ||
// hdfsoptions may be omitted | ||
if (callback===undefined && typeof(hdfsoptions) === 'function') { | ||
callback = hdfsoptions; | ||
hdfsoptions = undefined; | ||
} | ||
// format request args | ||
var args = { uri: this.base_url, qs: { op: 'gethomedirectory' } } | ||
var args = _.defaults({ | ||
json: true, | ||
uri: this.base_url, | ||
qs: _.defaults({ | ||
op: 'gethomedirectory', | ||
'user.name': this.options.user | ||
}, hdfsoptions || {}) | ||
}, requestoptions || {}); | ||
@@ -171,24 +291,46 @@ // send http request | ||
// forward request error | ||
if (error) return callback(error); | ||
// execute callback | ||
return callback(null, body.Path) | ||
return callback(null, body.Path); | ||
}) | ||
}); | ||
} | ||
}; | ||
// ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#OPEN | ||
WebHDFSClient.prototype.open = function (path, callback) { | ||
WebHDFSClient.prototype.open = function (path, hdfsoptions, requestoptions, callback) { | ||
// requestoptions may be omitted | ||
if (callback === undefined && typeof(requestoptions) === 'function') { | ||
callback = requestoptions; | ||
requestoptions = undefined; | ||
} | ||
// hdfsoptions may be omitted | ||
if (callback === undefined && typeof(hdfsoptions) === 'function') { | ||
callback = hdfsoptions; | ||
hdfsoptions = undefined; | ||
} | ||
// format request args | ||
var args = { | ||
var args = _.defaults({ | ||
uri: this.base_url + path, | ||
qs: { | ||
qs: _.defaults({ | ||
op: 'open' | ||
} | ||
}; | ||
}, hdfsoptions || {}) | ||
}, requestoptions || {}); | ||
// send http request | ||
request.get(args, function (error, response, body) { | ||
return request.get(args, function (error, response, body) { | ||
// forward request error | ||
if (error) return callback(error); | ||
// execute callback | ||
@@ -203,20 +345,37 @@ return callback(null, body); | ||
// ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#RENAME | ||
WebHDFSClient.prototype.rename = function (path, destination, callback) { | ||
WebHDFSClient.prototype.rename = function (path, destination, hdfsoptions, requestoptions, callback) { | ||
// requestoptions may be omitted | ||
if (callback === undefined && typeof(requestoptions) === 'function') { | ||
callback = requestoptions; | ||
requestoptions = undefined; | ||
} | ||
// hdfsoptions may be omitted | ||
if (callback === undefined && typeof(hdfsoptions) === 'function') { | ||
callback = hdfsoptions; | ||
hdfsoptions = undefined; | ||
} | ||
// format request args | ||
var args = { | ||
var args = _.defaults({ | ||
json: true, | ||
uri: this.base_url + path, | ||
qs: { | ||
qs: _.defaults({ | ||
op: 'rename', | ||
destination: destination, | ||
'user.name': this.options.user | ||
} | ||
}; | ||
}, hdfsoptions || {}) | ||
}, requestoptions || {}); | ||
// send http request | ||
request.put(args, function (err, res, body) { | ||
request.put(args, function (error, res, body) { | ||
// forward request error | ||
if (error) return callback(error); | ||
// exception handling | ||
@@ -235,13 +394,29 @@ if ('RemoteException' in body) | ||
// ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#MKDIRS | ||
WebHDFSClient.prototype.mkdirs = function (path, callback) { | ||
WebHDFSClient.prototype.mkdirs = function (path, hdfsoptions, requestoptions, callback) { | ||
// requestoptions may be omitted | ||
if (callback === undefined && typeof(requestoptions) === 'function') { | ||
callback = requestoptions; | ||
requestoptions = undefined; | ||
} | ||
// hdfsoptions may be omitted | ||
if (callback === undefined && typeof(hdfsoptions) === 'function') { | ||
callback = hdfsoptions; | ||
hdfsoptions = undefined; | ||
} | ||
// generate query string | ||
var args = { | ||
var args = _.defaults({ | ||
json: true, | ||
uri: this.base_url + path, | ||
qs: { | ||
qs: _.defaults({ | ||
op: 'mkdirs', | ||
'user.name': this.options.user | ||
} | ||
}; | ||
}, hdfsoptions || {}) | ||
}, requestoptions || {}); | ||
@@ -251,2 +426,5 @@ // send http request | ||
// forward request error | ||
if (error) return callback(error); | ||
// exception handling | ||
@@ -265,6 +443,22 @@ if ('RemoteException' in body) | ||
// ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#APPEND | ||
WebHDFSClient.prototype.append = function (path, data, callback) { | ||
WebHDFSClient.prototype.append = function (path, data, hdfsoptions, requestoptions, callback) { | ||
// requestoptions may be omitted | ||
if (callback === undefined && typeof(requestoptions) === 'function') { | ||
callback = requestoptions; | ||
requestoptions = undefined; | ||
} | ||
// hdfsoptions may be omitted | ||
if (callback === undefined && typeof(hdfsoptions) === 'function') { | ||
callback = hdfsoptions; | ||
hdfsoptions = undefined; | ||
} | ||
// format request args | ||
var args = { | ||
var args = _.defaults({ | ||
@@ -274,12 +468,15 @@ json: true, | ||
uri: this.base_url + path, | ||
qs: { | ||
qs: _.defaults({ | ||
op: 'append', | ||
'user.name': this.options.user | ||
} | ||
}, hdfsoptions || {}) | ||
}; | ||
}, requestoptions || {}); | ||
// send http request | ||
request.post(args, function (err, response, body) { | ||
request.post(args, function (error, response, body) { | ||
// forward request error | ||
if (error) return callback(error); | ||
// check for expected redirect | ||
@@ -289,3 +486,3 @@ if (response.statusCode == 307) { | ||
// format request args | ||
args = { | ||
args = _.defaults({ | ||
@@ -295,7 +492,10 @@ body: data, | ||
}; | ||
}, requestoptions || {}); | ||
// send http request | ||
request.post(args, function (err, response, body) { | ||
request.post(args, function (error, response, body) { | ||
// forward request error | ||
if (error) return callback(error); | ||
// check for expected response | ||
@@ -308,3 +508,3 @@ if (response.statusCode == 200) { | ||
return callback(new Error('expected http 200 ok')); | ||
return callback(new Error('expected http 200: ' + response.body)); | ||
@@ -327,14 +527,33 @@ } | ||
// ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#CREATE | ||
WebHDFSClient.prototype.create = function (path, data, callback) { | ||
WebHDFSClient.prototype.create = function (path, data, hdfsoptions, requestoptions, callback) { | ||
// requestoptions may be omitted | ||
if (callback === undefined && typeof(requestoptions) === 'function') { | ||
callback = requestoptions; | ||
requestoptions = undefined; | ||
} | ||
// hdfsoptions may be omitted | ||
if (callback === undefined && typeof(hdfsoptions) === 'function') { | ||
callback = hdfsoptions; | ||
hdfsoptions = undefined; | ||
} | ||
// generate query string | ||
var args = { | ||
var args = _.defaults({ | ||
json: true, | ||
followRedirect: false, | ||
uri: this.base_url + path, | ||
qs: { | ||
qs: _.defaults({ | ||
op: 'create', | ||
'user.name': this.options.user | ||
} | ||
}; | ||
}, hdfsoptions || {}) | ||
}, requestoptions || {}); | ||
@@ -344,2 +563,5 @@ // send http request | ||
// forward request error | ||
if (error) return callback(error); | ||
// check for expected redirect | ||
@@ -349,6 +571,6 @@ if (response.statusCode == 307) { | ||
// generate query string | ||
args = { | ||
args = _.defaults({ | ||
body: data, | ||
uri: response.headers.location | ||
}; | ||
}, requestoptions || {}); | ||
@@ -358,2 +580,5 @@ // send http request | ||
// forward request error | ||
if (error) return callback(error); | ||
// check for expected created response | ||
@@ -360,0 +585,0 @@ if (response.statusCode == 201) { |
@@ -51,3 +51,3 @@ | ||
client.create('/test/foo.txt', '{"foo":"bar",', function (err, path) { | ||
client.create('/test/foo.txt', '{"foo":"bar"}', function (err, path) { | ||
@@ -84,21 +84,2 @@ should.not.exist(err); | ||
describe('#append()', function () { | ||
it('should return `true` if the data was appended', function (done) { | ||
client.append('/test/bar.txt', '"bar": "baz"}', function (err, success) { | ||
should.not.exist(err); | ||
should.exist(success); | ||
success.should.be.true; | ||
return done(); | ||
}); | ||
}); | ||
}); | ||
describe('#getContentSummary()', function () { | ||
@@ -132,3 +113,3 @@ | ||
checksum.should.have.property('bytes', '000002000000000000000000dbdf72650428467285b0d32f1b12e8d500000000'); | ||
checksum.should.have.property('algorithm', 'MD5-of-0MD5-of-512CRC32'); | ||
@@ -152,3 +133,3 @@ return done(); | ||
JSON.parse(data).should.have.property('bar', 'baz'); | ||
JSON.parse(data).should.have.property('foo', 'bar'); | ||
@@ -155,0 +136,0 @@ return done(); |
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
21919
469
9
+ Addedajv@6.12.6(transitive)
+ Addedasn1@0.2.6(transitive)
+ Addedassert-plus@1.0.0(transitive)
+ Addedasynckit@0.4.0(transitive)
+ Addedaws-sign2@0.7.0(transitive)
+ Addedaws4@1.13.2(transitive)
+ Addedbcrypt-pbkdf@1.0.2(transitive)
+ Addedcaseless@0.12.0(transitive)
+ Addedcombined-stream@1.0.8(transitive)
+ Addedcore-util-is@1.0.2(transitive)
+ Addeddashdash@1.14.1(transitive)
+ Addeddelayed-stream@1.0.0(transitive)
+ Addedecc-jsbn@0.1.2(transitive)
+ Addedextend@3.0.2(transitive)
+ Addedextsprintf@1.3.0(transitive)
+ Addedfast-deep-equal@3.1.3(transitive)
+ Addedfast-json-stable-stringify@2.1.0(transitive)
+ Addedforever-agent@0.6.1(transitive)
+ Addedform-data@2.3.3(transitive)
+ Addedgetpass@0.1.7(transitive)
+ Addedhar-schema@2.0.0(transitive)
+ Addedhar-validator@5.1.5(transitive)
+ Addedhttp-signature@1.2.0(transitive)
+ Addedis-typedarray@1.0.0(transitive)
+ Addedisstream@0.1.2(transitive)
+ Addedjsbn@0.1.1(transitive)
+ Addedjson-schema@0.4.0(transitive)
+ Addedjson-schema-traverse@0.4.1(transitive)
+ Addedjson-stringify-safe@5.0.1(transitive)
+ Addedjsprim@1.4.2(transitive)
+ Addedmime-db@1.52.0(transitive)
+ Addedmime-types@2.1.35(transitive)
+ Addedoauth-sign@0.9.0(transitive)
+ Addedperformance-now@2.1.0(transitive)
+ Addedpsl@1.9.0(transitive)
+ Addedpunycode@2.3.1(transitive)
+ Addedqs@6.5.3(transitive)
+ Addedrequest@2.88.2(transitive)
+ Addedsafe-buffer@5.2.1(transitive)
+ Addedsafer-buffer@2.1.2(transitive)
+ Addedsshpk@1.18.0(transitive)
+ Addedtough-cookie@2.5.0(transitive)
+ Addedtunnel-agent@0.6.0(transitive)
+ Addedtweetnacl@0.14.5(transitive)
+ Addedunderscore@1.13.7(transitive)
+ Addeduri-js@4.4.1(transitive)
+ Addeduuid@3.4.0(transitive)
+ Addedverror@1.10.0(transitive)
- Removedrequest@2.9.201(transitive)
- Removedunderscore@1.3.3(transitive)
Updatedrequest@>=2.9.201
Updatedunderscore@>=1.3.3