@cloudant/couchbackup - npm Package Compare versions

Comparing version 2.3.2-SNAPSHOT.158 to 2.4.0

app.js

@@ -31,3 +31,3 @@ // Copyright © 2017, 2018 IBM Corp. All rights reserved.
 const fs = require('fs');
-const legacyUrl = require('url');
+const URL = require('url').URL;
 
@@ -99,3 +99,3 @@ /**
   try {
-    const urlObject = legacyUrl.parse(url);
+    const urlObject = new URL(url);
     // We require a protocol, host and path (for db), fail if any is missing.
@@ -110,7 +110,7 @@ if (urlObject.protocol !== 'https:' && urlObject.protocol !== 'http:') {
     }
-    if (!urlObject.path) {
+    if (!urlObject.pathname || urlObject.pathname === '/') {
       cb(new error.BackupError('InvalidOption', 'Invalid URL, missing path element (no database).'));
       return;
     }
-    if (opts && opts.iamApiKey && urlObject.auth) {
+    if (opts && opts.iamApiKey && (urlObject.username || url.password)) {
       cb(new error.BackupError('InvalidOption', 'URL user information must not be supplied when using IAM API key.'));
@@ -124,2 +124,14 @@ return;
 
+  // Perform validation of invalid options for shallow mode and WARN
+  // We don't error for backwards compatibility with scripts that may have been
+  // written passing complete sets of options through
+  if (opts && opts.mode === 'shallow') {
+    if (opts.log || opts.resume) {
+      console.warn('WARNING: the options "log" and "resume" are invalid when using shallow mode.');
+    }
+    if (opts.parallelism) {
+      console.warn('WARNING: the option "parallelism" has no effect when using shallow mode.');
+    }
+  }
+
   if (opts && opts.resume) {
@@ -126,0 +138,0 @@ if (!opts.log) {

CHANGES.md

@@ -1,6 +0,11 @@
-# 2.4.0
+# 2.4.0 (2019-03-15)
 
 - [NEW] Added request timeout option. Set via env var `COUCH_REQUEST_TIMEOUT`,
  as CLI option `--request-timeout`, or programmatically via
- `options.requestTimeout`
+ `options.requestTimeout`.
+- [IMPROVED] Replaced usages of Node.js legacy URL API. Note this changes some
+  URL validation error messages.
+- [IMPROVED] Documentation, help text and log warnings for invalid options in
+  "shallow" mode.
+- [UPGRADED] Moved nodejs-cloudant dependency to 4.x.x.
 
@@ -7,0 +12,0 @@ # 2.3.1 (2018-06-15)

includes/cliutils.js

@@ -1,2 +0,2 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
@@ -40,3 +40,3 @@ // Licensed under the Apache License, Version 2.0 (the "License");
     }
-    return url.resolve(root, encodeURIComponent(databaseName));
+    return new url.URL(encodeURIComponent(databaseName), root).toString();
   },
@@ -53,4 +53,4 @@
   getUsage: function getUsage(description, defaultValue) {
-    return description + ' (default: ' + defaultValue + ')';
+    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
   }
 };

includes/parser.js

@@ -25,5 +25,9 @@ // Copyright © 2017, 2018 IBM Corp. All rights reserved.
 
-  var defaults = config.cliDefaults();
-  config.applyEnvironmentVariables(defaults);
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
 
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
   program
@@ -35,36 +39,40 @@ .version(pkg.version)
       cliutils.getUsage('number of documents fetched at once', defaults.bufferSize),
-      Number, defaults.bufferSize)
+      Number)
     .option('-d, --db <db>',
-      cliutils.getUsage('name of the database to backup', defaults.db),
-      defaults.db)
+      cliutils.getUsage('name of the database to backup', defaults.db))
     .option('-k, --iam-api-key <API key>',
-      cliutils.getUsage('IAM API key to access the Cloudant server'),
-      defaults.iamApiKey)
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
     .option('-l, --log <file>',
-      cliutils.getUsage('file to store logging information during backup', 'a temporary file'),
-      path.normalize, defaults.log)
+      cliutils.getUsage('file to store logging information during backup; invalid in "shallow" mode', 'a temporary file'),
+      path.normalize)
     .option('-m, --mode <mode>',
       cliutils.getUsage('"shallow" if only a superficial backup is done (ignoring conflicts and revision tokens), else "full" for complete backup', defaults.mode),
-      (mode) => { return mode.toLowerCase(); }, defaults.mode)
+      (mode) => { return mode.toLowerCase(); })
     .option('-o, --output <file>',
       cliutils.getUsage('file name to store the backup data', 'stdout'),
-      path.normalize, defaults.output)
+      path.normalize)
     .option('-p, --parallelism <n>',
-      cliutils.getUsage('number of HTTP requests to perform in parallel when performing a backup', defaults.parallelism),
-      Number, defaults.parallelism)
+      cliutils.getUsage('number of HTTP requests to perform in parallel when performing a backup; ignored in "shallow" mode', defaults.parallelism),
+      Number)
     .option('-r, --resume',
-      cliutils.getUsage('continue a previous backup from its last known position', defaults.resume),
-      defaults.resume)
+      cliutils.getUsage('continue a previous backup from its last known position; invalid in "shallow" mode', defaults.resume))
     .option('-t, --request-timeout <n>',
       cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
-      Number, defaults.requestTimeout)
+      Number)
     .option('-u, --url <url>',
-      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url),
-      defaults.url)
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
     .parse(process.argv);
 
-  // Special case the iamTokenUrl which is only an env var
-  program.iamTokenUrl = defaults.iamTokenUrl;
+  // Remove defaults that don't apply when using shallow mode
+  if (program.mode === 'shallow' || envVarOptions.mode === 'shallow') {
+    delete defaults.parallelism;
+    delete defaults.log;
+    delete defaults.resume;
+  }
 
-  if (program.resume && (program.log === defaults.log)) {
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program);
+
+  if (opts.resume && (opts.log === defaults.log)) {
     // If resuming and the log file arg is the newly generated tmp name from defaults then we know that --log wasn't specified.
@@ -75,3 +83,3 @@ // We have to do this check here for the CLI case because of the default.
 
-  return program;
+  return opts;
 }
@@ -82,5 +90,9 @@
 
-  var defaults = config.cliDefaults();
-  config.applyEnvironmentVariables(defaults);
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
 
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
   program
@@ -92,24 +104,22 @@ .version(pkg.version)
       cliutils.getUsage('number of documents restored at once', defaults.bufferSize),
-      Number, defaults.bufferSize)
+      Number)
     .option('-d, --db <db>',
-      cliutils.getUsage('name of the new, existing database to restore to', defaults.db),
-      defaults.db)
+      cliutils.getUsage('name of the new, existing database to restore to', defaults.db))
     .option('-k, --iam-api-key <API key>',
-      cliutils.getUsage('IAM API key to access the Cloudant server'),
-      defaults.iamApiKey)
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
     .option('-p, --parallelism <n>',
       cliutils.getUsage('number of HTTP requests to perform in parallel when restoring a backup', defaults.parallelism),
-      Number, defaults.parallelism)
+      Number)
     .option('-t, --request-timeout <n>',
       cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
-      Number, defaults.requestTimeout)
+      Number)
     .option('-u, --url <url>',
-      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url),
-      defaults.url)
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
     .parse(process.argv);
 
-  // Special case the iamTokenUrl which is only an env var
-  program.iamTokenUrl = defaults.iamTokenUrl;
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program);
 
-  return program;
+  return opts;
 }
@@ -116,0 +126,0 @@

includes/spoolchanges.js

@@ -65,3 +65,3 @@ // Copyright © 2017, 2018 IBM Corp. All rights reserved.
   // stream the changes feed to disk
-  var changesRequest = db.changes({ seq_interval: 10000 })
+  var changesRequest = db.changesAsStream({ seq_interval: 10000 })
     .on('error', function(err) {
@@ -68,0 +68,0 @@ callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));

includes/writer.js

@@ -39,24 +39,53 @@ // Copyright © 2017, 2018 IBM Corp. All rights reserved.
     // Stream the payload through a zip stream to the server
-    var payloadStream = new stream.PassThrough();
+    const payloadStream = new stream.PassThrough();
     payloadStream.end(Buffer.from(JSON.stringify(payload), 'utf8'));
-    var zipstream = zlib.createGzip();
+    const zipstream = zlib.createGzip();
 
+    // Class for streaming _bulk_docs responses into
+    // In general the response is [] or a small error/reason JSON object
+    // so it is OK to have this in memory.
+    class ResponseWriteable extends stream.Writable {
+      constructor(options) {
+        super(options);
+        this.data = [];
+      }
+
+      _write(chunk, encoding, callback) {
+        this.data.push(chunk);
+        callback();
+      }
+
+      asJson() {
+        return JSON.parse(Buffer.concat(this.data).toString());
+      }
+    }
+
     if (!didError) {
-      var req = db.server.request({
+      var response;
+      const responseBody = new ResponseWriteable();
+      const req = db.server.request({
         db: db.config.db,
         path: '_bulk_docs',
         method: 'POST',
-        headers: { 'content-encoding': 'gzip' }
-      }, function(err) {
-        err = error.convertResponseError(err);
-        if (err) {
-          debug(`Error writing docs ${err.name} ${err.message}`);
-          cb(err, payload);
-        } else {
-          written += payload.docs.length;
-          writer.emit('restored', { documents: payload.docs.length, total: written });
-          cb();
-        }
-      });
+        headers: { 'content-encoding': 'gzip' },
+        stream: true
+      })
+        .on('response', function(resp) {
+          response = resp;
+        })
+        .on('end', function() {
+          if (response.statusCode >= 400) {
+            const err = error.convertResponseError(Object.assign({}, response, responseBody.asJson()));
+            debug(`Error writing docs ${err.name} ${err.message}`);
+            cb(err, payload);
+          } else {
+            written += payload.docs.length;
+            writer.emit('restored', { documents: payload.docs.length, total: written });
+            cb();
+          }
+        });
+      // Pipe the payload into the request object to POST to _bulk_docs
       payloadStream.pipe(zipstream).pipe(req);
+      // Pipe the request object's response into our bulkDocsResponse
+      req.pipe(responseBody);
     }
@@ -63,0 +92,0 @@ }, parallelism);

package.json

 {
   "name": "@cloudant/couchbackup",
-  "version": "2.3.2-SNAPSHOT.158",
+  "version": "2.4.0",
   "description": "CouchBackup - command-line backup utility for Cloudant/CouchDB",
@@ -26,3 +26,3 @@ "homepage": "https://github.com/cloudant/couchbackup",
     "debug": "~4.1.0",
-    "@cloudant/cloudant": "^2.2.0",
+    "@cloudant/cloudant": "^4.0.0",
     "tmp": "0.0.33"
@@ -36,19 +36,19 @@ },
   "devDependencies": {
-    "eslint": "^5.0.0",
+    "eslint": "^5.15.1",
+    "eslint-config-semistandard": "^13.0.0",
+    "eslint-config-standard": "^12.0.0",
+    "eslint-plugin-header": "^2.0.0",
     "eslint-plugin-import": "^2.8.0",
-    "eslint-plugin-node": "^7.0.1",
+    "eslint-plugin-node": "^8.0.0",
     "eslint-plugin-promise": "^4.0.0",
+    "eslint-plugin-react": "^7.7.0",
     "eslint-plugin-standard": "^4.0.0",
-    "eslint-plugin-react": "^7.7.0",
-    "eslint-config-standard": "^12.0.0",
-    "eslint-config-semistandard": "^12.0.1",
-    "eslint-plugin-header": "^2.0.0",
+    "http-proxy": "^1.16.2",
     "jsdoc": "^3.5.2",
-    "mocha": "^5.0.0",
-    "toxy": "^0.3.12",
-    "uuid": "^3.0.1",
+    "mocha": "^6.0.0",
+    "nock": "^10.0.0",
+    "rewire": "^4.0.0",
     "tail": "^2.0.0",
-    "rewire": "^4.0.0",
-    "nock": "^10.0.0",
-    "http-proxy": "^1.16.2"
+    "toxy": "^0.3.16",
+    "uuid": "^3.0.1"
   },
@@ -55,0 +55,0 @@ "scripts": {

README.md

@@ -175,2 +175,4 @@ # CouchBackup
 
+NOTE: Parallellism will not be in effect if `--mode shallow` is defined.
+
 ## Why use CouchBackup?
@@ -177,0 +179,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

No v1

Quality

Package is not semver >=1. This means it is not stable and does not support ^ ranges.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

97925

increased by2.35%

Lines of code

1725

increased by2.62%

Number of low quality alerts

0

decreased by-100%

Number of lines in readme file

422

increased by0.48%

Dependency changes

• + Added@cloudant/cloudant@4.5.1(transitive)

• + Added@types/node@22.7.6(transitive)

• + Addedcloudant-follow@0.18.2(transitive)

• + Addedcookie@0.4.2(transitive)

• + Addednano@8.2.3(transitive)

• - Removed@cloudant/cloudant@2.4.1(transitive)

• - Removed@types/events@3.0.3(transitive)

• - Removed@types/nano@6.4.6(transitive)

• - Removed@types/node@22.7.5(transitive)

• - Removedcloudant-follow@0.17.0(transitive)

• - Removeddebug@2.6.9(transitive)

• - Removedlodash.isempty@4.4.0(transitive)

• - Removedms@2.0.0(transitive)

• - Removednano@6.4.4(transitive)

• Updated@cloudant/cloudant@^4.0.0