node-webhdfs - npm Package Compare versions

Comparing version 0.0.2 to 0.0.3

package.json

@@ -5,3 +5,3 @@ {
     "description": "A WebHDFS module for Node.js.",
-    "version": "0.0.2",
+    "version": "0.0.3",
     "keywords": ["hdfs", "webhdfs", "http"],
@@ -20,4 +20,4 @@ "maintainers": ["Ryan Cole <[e-mail]> (http://rycole.com)"],
     "dependencies": {
-        "underscore": "1.3.3",
-        "request": "2.9.201"
+        "underscore": ">=1.3.3",
+        "request": ">=2.9.201"
     },
@@ -24,0 +24,0 @@ "engines": {

readme.md

@@ -5,10 +5,6 @@ A WebHDFS module for Node.js
 
-I am currently following the [WebHDFS REST API documentation](http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#Document+Conventions) for the `1.0.2` release, by Apache.
+I am currently following and testing against the [WebHDFS REST API documentation](http://hadoop.apache.org/docs/r1.2.1/webhdfs.html) for the `1.2.1` release, by Apache. Make sure you enable WebHDFS in the hdfs site configuration file.
 
-# Disclaimer
-
-I'm not an expert on HDFS. This module has been written as a requirement for a personal project. I am actively using the module and have written tests for it. Your mileage may vary.
-
 # Tests
 
-I use [Mocha](http://visionmedia.github.com/mocha/) and [should.js](https://github.com/visionmedia/should.js) for unit testing. They will be required if you want to run the unit tests. To execute the tests, simply `npm test`.
+I use [Mocha](http://visionmedia.github.com/mocha/) and [should.js](https://github.com/visionmedia/should.js) for unit testing. They will be required if you want to run the unit tests. To execute the tests, simply `npm test`, but install the requirements first.

src/webhdfs.js

@@ -1,2 +0,1 @@
-
 var querystring = require('querystring'),
@@ -26,14 +25,29 @@ request = require('request'),
 // ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#DELETE
-WebHDFSClient.prototype.del = function (path, options, callback) {
+WebHDFSClient.prototype.del = function (path, hdfsoptions, requestoptions, callback) {
     
+    // requestoptions may be omitted
+    if (callback === undefined && typeof(requestoptions) === 'function') {
+
+        callback = requestoptions;
+        requestoptions = undefined;
+
+    }
+
+    // hdfsoptions may be omitted
+    if (callback === undefined && typeof(hdfsoptions) === 'function') {
+
+        callback = hdfsoptions;
+        hdfsoptions = undefined;
+
+    }
+    
     // format request args
-    var args = {
+    var args = _.defaults({
         json: true,
         uri: this.base_url + path,
-        qs: {
+        qs: _.defaults({
             op: 'delete',
-            recursive: options.recursive,
             'user.name': this.options.user
-        }
-    };
+        }, hdfsoptions || {})
+    }, requestoptions || {});
     
@@ -43,2 +57,5 @@ // send http request
         
+        // forward request error
+        if (error) return callback(error);
+        
         // exception handling
@@ -57,6 +74,27 @@ if ('RemoteException' in body)
 // ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#LISTSTATUS
-WebHDFSClient.prototype.listStatus = function (path, callback) {
+WebHDFSClient.prototype.listStatus = function (path, hdfsoptions, requestoptions, callback) {
     
+    // requestoptions may be omitted
+    if (callback === undefined && typeof(requestoptions) === 'function') {
+
+        callback = requestoptions;
+        requestoptions = undefined;
+    }
+
+    // hdfsoptions may be omitted
+    if (callback === undefined && typeof(hdfsoptions) === 'function') {
+
+        callback = hdfsoptions;
+        hdfsoptions = undefined;
+
+    }
+    
     // format request args
-    var args = { uri: this.base_url + path, qs: { op: 'liststatus' } }
+    var args = _.defaults({
+        json:true,
+        uri: this.base_url + path,
+        qs: _.defaults({
+            op: 'liststatus'
+        }, hdfsoptions || {})
+    }, requestoptions || {});
     
@@ -66,2 +104,5 @@ // send http request
         
+        // forward request error
+        if (error) return callback(error);
+        
         // execute callback
@@ -76,12 +117,28 @@ return callback(null, body.FileStatuses.FileStatus)
 // ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#GETFILESTATUS
-WebHDFSClient.prototype.getFileStatus = function (path, callback) {
+WebHDFSClient.prototype.getFileStatus = function (path, hdfsoptions, requestoptions, callback) {
     
+    // requestoptions may be omitted
+    if (callback === undefined && typeof(requestoptions) === 'function') {
+
+        callback = requestoptions;
+        requestoptions = undefined;
+
+    }
+
+    // hdfsoptions may be omitted
+    if (callback === undefined && typeof(hdfsoptions) === 'function') {
+
+        callback = hdfsoptions;
+        hdfsoptions = undefined;
+
+    }
+    
     // format request args
-    var args = {
+    var args = _.defaults({
         json: true,
         uri: this.base_url + path,
-        qs: {
+        qs: _.defaults({
             op: 'getfilestatus'
-        }
-    };
+        }, hdfsoptions || {})
+    }, requestoptions || {});
     
@@ -91,2 +148,5 @@ // send http request
         
+        // forward request error
+        if (error) return callback(error);
+        
         // exception handling
@@ -105,12 +165,27 @@ if ('RemoteException' in body)
 // ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#GETCONTENTSUMMARY
-WebHDFSClient.prototype.getContentSummary = function (path, callback) {
+WebHDFSClient.prototype.getContentSummary = function (path, hdfsoptions, requestoptions, callback) {
     
+    // requestoptions may be omitted
+    if (callback === undefined && typeof(requestoptions) === 'function') {
+
+        callback = requestoptions;
+        requestoptions = undefined;
+    }
+
+    // hdfsoptions may be omitted
+    if (callback===undefined && typeof(hdfsoptions) === 'function') {
+
+        callback = hdfsoptions;
+        hdfsoptions = undefined;
+
+    }
+    
     // format request args
-    var args = {
+    var args = _.defaults({
         json: true,
         uri: this.base_url + path,
-        qs: {
+        qs: _.defaults({
             op: 'getcontentsummary'
-        }
-    };
+        }, hdfsoptions || {})
+    }, requestoptions || {});
     
@@ -120,2 +195,5 @@ // send http request
         
+        // forward request error
+        if (error) return callback(error);
+        
         // exception handling
@@ -134,12 +212,28 @@ if ('RemoteException' in body)
 // ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#GETFILECHECKSUM
-WebHDFSClient.prototype.getFileChecksum = function (path, callback) {
+WebHDFSClient.prototype.getFileChecksum = function (path, hdfsoptions, requestoptions, callback) {
     
+    // requestoptions may be omitted
+    if (callback === undefined && typeof(requestoptions) === 'function') {
+
+        callback = requestoptions;
+        requestoptions = undefined;
+
+    }
+
+    // hdfsoptions may be omitted
+    if (callback === undefined && typeof(hdfsoptions) === 'function') {
+
+        callback = hdfsoptions;
+        hdfsoptions = undefined;
+
+    }
+    
     // format request args
-    var args = {
+    var args = _.defaults({
         json: true,
         uri: this.base_url + path,
-        qs: {
+        qs: _.defaults({
             op: 'getfilechecksum'
-        }
-    };
+        }, hdfsoptions || {})
+    }, requestoptions || {});
     
@@ -149,2 +243,5 @@ // send http request
         
+        // forward request error
+        if (error) return callback(error);
+        
         // exception handling
@@ -162,6 +259,29 @@ if ('RemoteException' in body)
 // ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#GETHOMEDIRECTORY
-WebHDFSClient.prototype.getHomeDirectory = function (callback) {
+WebHDFSClient.prototype.getHomeDirectory = function (hdfsoptions, requestoptions, callback) {
     
+    // requestoptions may be omitted
+    if (callback === undefined && typeof(requestoptions) === 'function') {
+
+        callback = requestoptions;
+        requestoptions = undefined;
+
+    }
+
+    // hdfsoptions may be omitted
+    if (callback===undefined && typeof(hdfsoptions) === 'function') {
+
+        callback = hdfsoptions;
+        hdfsoptions = undefined;
+
+    }
+    
     // format request args
-    var args = { uri: this.base_url, qs: { op: 'gethomedirectory' } }
+    var args = _.defaults({
+        json: true,
+        uri: this.base_url,
+        qs: _.defaults({
+            op: 'gethomedirectory',
+            'user.name': this.options.user
+        }, hdfsoptions || {})
+    }, requestoptions || {});
     
@@ -171,24 +291,46 @@ // send http request
         
+        // forward request error
+        if (error) return callback(error);
+        
         // execute callback
-        return callback(null, body.Path)
+        return callback(null, body.Path);
         
-    })
+    });
     
-}
+};
 
 
 // ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#OPEN
-WebHDFSClient.prototype.open = function (path, callback) {
+WebHDFSClient.prototype.open = function (path, hdfsoptions, requestoptions, callback) {
     
+    // requestoptions may be omitted
+    if (callback === undefined && typeof(requestoptions) === 'function') {
+
+        callback = requestoptions;
+        requestoptions = undefined;
+
+    }
+
+    // hdfsoptions may be omitted
+    if (callback === undefined && typeof(hdfsoptions) === 'function') {
+
+        callback = hdfsoptions;
+        hdfsoptions = undefined;
+
+    }
+
     // format request args
-    var args = {
+    var args = _.defaults({
         uri: this.base_url + path,
-        qs: {
+        qs: _.defaults({
             op: 'open'
-        }
-    };
-    
+        }, hdfsoptions || {})
+    }, requestoptions || {});
+
     // send http request
-    request.get(args, function (error, response, body) {
+    return request.get(args, function (error, response, body) {
         
+        // forward request error
+        if (error) return callback(error);
+        
         // execute callback
@@ -203,20 +345,37 @@ return callback(null, body);
 // ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#RENAME
-WebHDFSClient.prototype.rename = function (path, destination, callback) {
+WebHDFSClient.prototype.rename = function (path, destination, hdfsoptions, requestoptions, callback) {
     
+    // requestoptions may be omitted
+    if (callback === undefined && typeof(requestoptions) === 'function') {
+
+        callback = requestoptions;
+        requestoptions = undefined;
+
+    }
+
+    // hdfsoptions may be omitted
+    if (callback === undefined && typeof(hdfsoptions) === 'function') {
+
+        callback = hdfsoptions;
+        hdfsoptions = undefined;
+
+    }
+    
     // format request args
-    var args = {
-        
+    var args = _.defaults({
         json: true,
         uri: this.base_url + path,
-        qs: {
+        qs: _.defaults({
             op: 'rename',
             destination: destination,
             'user.name': this.options.user
-        }
-        
-    };
+        }, hdfsoptions || {})
+    }, requestoptions || {});
     
     // send http request
-    request.put(args, function (err, res, body) {
+    request.put(args, function (error, res, body) {
         
+        // forward request error
+        if (error) return callback(error);
+        
         // exception handling
@@ -235,13 +394,29 @@ if ('RemoteException' in body)
 // ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#MKDIRS
-WebHDFSClient.prototype.mkdirs = function (path, callback) {
+WebHDFSClient.prototype.mkdirs = function (path, hdfsoptions, requestoptions, callback) {
     
+    // requestoptions may be omitted
+    if (callback === undefined && typeof(requestoptions) === 'function') {
+
+        callback = requestoptions;
+        requestoptions = undefined;
+
+    }
+
+    // hdfsoptions may be omitted
+    if (callback === undefined && typeof(hdfsoptions) === 'function') {
+
+        callback = hdfsoptions;
+        hdfsoptions = undefined;
+
+    }
+    
     // generate query string
-    var args = {
+    var args = _.defaults({
         json: true,
         uri: this.base_url + path,
-        qs: {
+        qs: _.defaults({
             op: 'mkdirs',
             'user.name': this.options.user
-        }
-    };
+        }, hdfsoptions || {})
+    }, requestoptions || {});
     
@@ -251,2 +426,5 @@ // send http request
         
+        // forward request error
+        if (error) return callback(error);
+        
         // exception handling
@@ -265,6 +443,22 @@ if ('RemoteException' in body)
 // ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#APPEND
-WebHDFSClient.prototype.append = function (path, data, callback) {
+WebHDFSClient.prototype.append = function (path, data, hdfsoptions, requestoptions, callback) {
     
+    // requestoptions may be omitted
+    if (callback === undefined && typeof(requestoptions) === 'function') {
+
+        callback = requestoptions;
+        requestoptions = undefined;
+
+    }
+
+    // hdfsoptions may be omitted
+    if (callback === undefined && typeof(hdfsoptions) === 'function') {
+
+        callback = hdfsoptions;
+        hdfsoptions = undefined;
+
+    }
+    
     // format request args
-    var args = {
+    var args = _.defaults({
         
@@ -274,12 +468,15 @@ json: true,
         uri: this.base_url + path,
-        qs: {
+        qs: _.defaults({
             op: 'append',
             'user.name': this.options.user
-        }
+        }, hdfsoptions || {})
         
-    };
+    }, requestoptions || {});
     
     // send http request
-    request.post(args, function (err, response, body) {
+    request.post(args, function (error, response, body) {
         
+        // forward request error
+        if (error) return callback(error);
+        
         // check for expected redirect
@@ -289,3 +486,3 @@ if (response.statusCode == 307) {
             // format request args
-            args = {
+            args = _.defaults({
                 
@@ -295,7 +492,10 @@ body: data,
                 
-            };
+            }, requestoptions || {});
             
             // send http request
-            request.post(args, function (err, response, body) {
+            request.post(args, function (error, response, body) {
                 
+                // forward request error
+                if (error) return callback(error);
+        
                 // check for expected response
@@ -308,3 +508,3 @@ if (response.statusCode == 200) {
                     
-                    return callback(new Error('expected http 200 ok'));
+                    return callback(new Error('expected http 200: ' + response.body));
                     
@@ -327,14 +527,33 @@ }
 // ref: http://hadoop.apache.org/common/docs/r1.0.2/webhdfs.html#CREATE
-WebHDFSClient.prototype.create = function (path, data, callback) {
+WebHDFSClient.prototype.create = function (path, data, hdfsoptions, requestoptions, callback) {
     
+    // requestoptions may be omitted
+    if (callback === undefined && typeof(requestoptions) === 'function') {
+
+        callback = requestoptions;
+        requestoptions = undefined;
+
+    }
+
+    // hdfsoptions may be omitted
+    if (callback === undefined && typeof(hdfsoptions) === 'function') {
+
+        callback = hdfsoptions;
+        hdfsoptions = undefined;
+
+    }
+    
     // generate query string
-    var args = {
+    var args = _.defaults({
+
         json: true,
         followRedirect: false,
         uri: this.base_url + path,
-        qs: {
+        qs: _.defaults({
+
             op: 'create',
             'user.name': this.options.user
-        }
-    };
+
+        }, hdfsoptions || {})
+    }, requestoptions || {});
     
@@ -344,2 +563,5 @@ // send http request
                 
+        // forward request error
+        if (error) return callback(error);
+        
         // check for expected redirect
@@ -349,6 +571,6 @@ if (response.statusCode == 307) {
             // generate query string
-            args = {
+            args = _.defaults({
                 body: data,
                 uri: response.headers.location
-            };
+            }, requestoptions || {});
             
@@ -358,2 +580,5 @@ // send http request
                 
+                // forward request error
+                if (error) return callback(error);
+        
                 // check for expected created response
@@ -360,0 +585,0 @@ if (response.statusCode == 201) {

test/webhdfs.js

@@ -51,3 +51,3 @@
             
-            client.create('/test/foo.txt', '{"foo":"bar",', function (err, path) {
+            client.create('/test/foo.txt', '{"foo":"bar"}', function (err, path) {
                 
@@ -84,21 +84,2 @@ should.not.exist(err);
     
-    describe('#append()', function () {
-        
-        it('should return `true` if the data was appended', function (done) {
-            
-            client.append('/test/bar.txt', '"bar": "baz"}', function (err, success) {
-                
-                should.not.exist(err);
-                should.exist(success);
-                
-                success.should.be.true;
-                
-                return done();
-                
-            });
-            
-        });
-        
-    });
-    
     describe('#getContentSummary()', function () {
@@ -132,3 +113,3 @@
                 
-                checksum.should.have.property('bytes', '000002000000000000000000dbdf72650428467285b0d32f1b12e8d500000000');
+                checksum.should.have.property('algorithm', 'MD5-of-0MD5-of-512CRC32');
                 
@@ -152,3 +133,3 @@ return done();
                 
-                JSON.parse(data).should.have.property('bar', 'baz');
+                JSON.parse(data).should.have.property('foo', 'bar');
                 
@@ -155,0 +136,0 @@ return done();

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

21919

increased by38.12%

Lines of code

469

increased by41.69%

Worsened metrics

Number of lines in readme file

9

decreased by-30.77%

Dependency changes

• + Addedajv@6.12.6(transitive)

• + Addedasn1@0.2.6(transitive)

• + Addedassert-plus@1.0.0(transitive)

• + Addedasynckit@0.4.0(transitive)

• + Addedaws-sign2@0.7.0(transitive)

• + Addedaws4@1.13.2(transitive)

• + Addedbcrypt-pbkdf@1.0.2(transitive)

• + Addedcaseless@0.12.0(transitive)

• + Addedcombined-stream@1.0.8(transitive)

• + Addedcore-util-is@1.0.2(transitive)

• + Addeddashdash@1.14.1(transitive)

• + Addeddelayed-stream@1.0.0(transitive)

• + Addedecc-jsbn@0.1.2(transitive)

• + Addedextend@3.0.2(transitive)

• + Addedextsprintf@1.3.0(transitive)

• + Addedfast-deep-equal@3.1.3(transitive)

• + Addedfast-json-stable-stringify@2.1.0(transitive)

• + Addedforever-agent@0.6.1(transitive)

• + Addedform-data@2.3.3(transitive)

• + Addedgetpass@0.1.7(transitive)

• + Addedhar-schema@2.0.0(transitive)

• + Addedhar-validator@5.1.5(transitive)

• + Addedhttp-signature@1.2.0(transitive)

• + Addedis-typedarray@1.0.0(transitive)

• + Addedisstream@0.1.2(transitive)

• + Addedjsbn@0.1.1(transitive)

• + Addedjson-schema@0.4.0(transitive)

• + Addedjson-schema-traverse@0.4.1(transitive)

• + Addedjson-stringify-safe@5.0.1(transitive)

• + Addedjsprim@1.4.2(transitive)

• + Addedmime-db@1.52.0(transitive)

• + Addedmime-types@2.1.35(transitive)

• + Addedoauth-sign@0.9.0(transitive)

• + Addedperformance-now@2.1.0(transitive)

• + Addedpsl@1.9.0(transitive)

• + Addedpunycode@2.3.1(transitive)

• + Addedqs@6.5.3(transitive)

• + Addedrequest@2.88.2(transitive)

• + Addedsafe-buffer@5.2.1(transitive)

• + Addedsafer-buffer@2.1.2(transitive)

• + Addedsshpk@1.18.0(transitive)

• + Addedtough-cookie@2.5.0(transitive)

• + Addedtunnel-agent@0.6.0(transitive)

• + Addedtweetnacl@0.14.5(transitive)

• + Addedunderscore@1.13.7(transitive)

• + Addeduri-js@4.4.1(transitive)

• + Addeduuid@3.4.0(transitive)

• + Addedverror@1.10.0(transitive)

• - Removedrequest@2.9.201(transitive)

• - Removedunderscore@1.3.3(transitive)

• Updatedrequest@>=2.9.201

• Updatedunderscore@>=1.3.3