@cloudant/couchbackup - npm Package Compare versions

Comparing version 2.10.3-SNAPSHOT-240 to 2.10.3-SNAPSHOT-241

app.js

@@ -100,3 +100,5 @@ // Copyright © 2017, 2024 IBM Corp. All rights reserved.
     { key: 'mode', type: 'enum', values: ['full', 'shallow'] },
-    { key: 'resume', type: 'boolean' }
+    { key: 'resume', type: 'boolean' },
+    { key: 'quiet', type: 'boolean' },
+    { key: 'attachments', type: 'boolean' }
   ];
@@ -193,2 +195,10 @@
 
+async function attachmentWarnings(opts) {
+  if (opts && opts.attachments) {
+    console.warn('WARNING: The "attachments" option is provided as-is and is not supported. ' +
+      'This option is for Apache CouchDB only and is experimental. ' +
+      'Do not use this option with IBM Cloudant.');
+  }
+}
+
 /**
@@ -206,3 +216,4 @@ * Validate arguments.
     validateURL(url, isIAM),
-    validateOptions(opts)
+    validateOptions(opts),
+    attachmentWarnings(opts)
   ];
@@ -332,3 +343,3 @@ if (isBackup) {
           debug('Will write backup file header.');
-          metadataToWrite = `${JSON.stringify({ name: pkg.name, version: pkg.version, mode: opts.mode })}\n`;
+          metadataToWrite = `${JSON.stringify({ name: pkg.name, version: pkg.version, mode: opts.mode, attachments: opts.attachments })}\n`;
         }
@@ -335,0 +346,0 @@ return new Promise((resolve, reject) => {

bin/couchbackup.bin.js

@@ -40,3 +40,4 @@ #!/usr/bin/env node
     iamApiKey: program.iamApiKey,
-    iamTokenUrl: program.iamTokenUrl
+    iamTokenUrl: program.iamTokenUrl,
+    attachments: program.attachments
   };
@@ -43,0 +44,0 @@

bin/couchrestore.bin.js

@@ -35,3 +35,4 @@ #!/usr/bin/env node
     iamApiKey: program.iamApiKey,
-    iamTokenUrl: program.iamTokenUrl
+    iamTokenUrl: program.iamTokenUrl,
+    attachments: program.attachments
   };
@@ -38,0 +39,0 @@

includes/allDocsGenerator.js

@@ -1,2 +0,2 @@
-// Copyright © 2023 IBM Corp. All rights reserved.
+// Copyright © 2023, 2024 IBM Corp. All rights reserved.
 //
@@ -31,2 +31,5 @@ // Licensed under the Apache License, Version 2.0 (the "License");
   const opts = { db: dbClient.dbName, limit: options.bufferSize, includeDocs: true };
+  if (options.attachments === true) {
+    opts.attachments = true;
+  }
   do {
@@ -33,0 +36,0 @@ if (startKey) opts.startKey = startKey;

includes/backup.js

@@ -18,2 +18,3 @@ // Copyright © 2017, 2024 IBM Corp. All rights reserved.
 const { pipeline } = require('node:stream/promises');
+const { Attachments } = require('./attachmentMappings.js');
 const { Backup } = require('./backupMappings.js');
@@ -91,3 +92,3 @@ const { BackupError } = require('./error.js');
     .then((srcStreams) => {
-      const backup = new Backup(dbClient);
+      const backup = new Backup(dbClient, options);
       const postWrite = (backupBatch) => {
@@ -99,2 +100,3 @@ total += backupBatch.docs.length;
 
+      const mappingStreams = [];
       const destinationStreams = [];
@@ -114,4 +116,6 @@ if (options.mode === 'shallow') {
         // full mode needs to fetch spooled changes and writes a backup file then finally a log file
+        mappingStreams.push(...[
+          new MappingStream(backup.pendingToFetched, options.parallelism) // fetch the batches at the configured concurrency
+        ]);
         destinationStreams.push(...[
-          new MappingStream(backup.pendingToFetched, options.parallelism), // fetch the batches at the configured concurrency
           new WritableWithPassThrough(
@@ -133,4 +137,11 @@ 'backup', // name for logging
 
+      if (options.attachments) {
+        mappingStreams.push(
+          new MappingStream(new Attachments().encode, options.parallelism)
+        );
+      }
+
       return pipeline(
         ...srcStreams, // the source streams from the previous block (all docs async generator for shallow or for full either spool changes or resumed log)
+        ...mappingStreams, // map from source to destination content
         ...destinationStreams // the appropriate destination streams for the mode
@@ -137,0 +148,0 @@ );

includes/backupMappings.js

@@ -1,2 +0,2 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2024 IBM Corp. All rights reserved.
 //
@@ -174,4 +174,5 @@ // Licensed under the Apache License, Version 2.0 (the "License");
 class Backup {
-  constructor(dbClient) {
+  constructor(dbClient, options) {
     this.dbClient = dbClient;
+    this.options = options;
   }
@@ -211,7 +212,11 @@
     try {
-      const response = await this.dbClient.service.postBulkGet({
+      const bulkGetOpts = {
         db: this.dbClient.dbName,
         revs: true,
         docs: backupBatch.docs
-      });
+      };
+      if (this.options.attachments) {
+        bulkGetOpts.attachments = true;
+      }
+      const response = await this.dbClient.service.postBulkGet(bulkGetOpts);
 
@@ -218,0 +223,0 @@ mappingDebug(`Good server response for batch ${backupBatch.batch}.`);

includes/config.js

@@ -1,2 +0,2 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2024 IBM Corp. All rights reserved.
 //
@@ -25,2 +25,3 @@ // Licensed under the Apache License, Version 2.0 (the "License");
   return {
+    attachments: false,
     parallelism: 5,
@@ -114,2 +115,7 @@ bufferSize: 500,
   }
+
+  // if we are instructed to be quiet
+  if (typeof process.env.COUCH_ATTACHMENTS !== 'undefined' && process.env.COUCH_ATTACHMENTS === 'true') {
+    opts.attachments = true;
+  }
 }
@@ -116,0 +122,0 @@

includes/error.js

@@ -1,2 +0,2 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2024 IBM Corp. All rights reserved.
 //
@@ -30,3 +30,5 @@ // Licensed under the Apache License, Version 2.0 (the "License");
   HTTPFatalError: 40,
-  BulkGetError: 50
+  BulkGetError: 50,
+  AttachmentsNotEnabledError: 60,
+  AttachmentsMetadataAbsent: 61
 };
@@ -33,0 +35,0 @@

includes/parser.js

@@ -36,2 +36,4 @@ // Copyright © 2017, 2024 IBM Corp. All rights reserved.
     .usage('[options...]')
+    .option('-a, --attachments',
+      cliutils.getUsage('*EXPERIMENTAL/UNSUPPORTED*: enable backup of attachments', defaults.attachments))
     .option('-b, --buffer-size <n>',
@@ -101,2 +103,4 @@ cliutils.getUsage('number of documents fetched at once', defaults.bufferSize),
     .usage('[options...]')
+    .option('-a, --attachments',
+      cliutils.getUsage('*EXPERIMENTAL/UNSUPPORTED*: enable restore of attachments', defaults.attachments))
     .option('-b, --buffer-size <n>',
@@ -103,0 +107,0 @@ cliutils.getUsage('number of documents restored at once', defaults.bufferSize),

includes/restore.js

@@ -17,4 +17,5 @@ // Copyright © 2017, 2024 IBM Corp. All rights reserved.
 const debug = require('debug')('couchbackup:restore');
-const { Liner } = require('../includes/liner.js');
-const { Restore } = require('../includes/restoreMappings.js');
+const { Attachments } = require('./attachmentMappings.js');
+const { Liner } = require('./liner.js');
+const { Restore } = require('./restoreMappings.js');
 const { BatchingStream, MappingStream } = require('./transforms.js');
@@ -34,3 +35,3 @@ const { Writable } = require('node:stream');
 module.exports = function(dbClient, options, readstream, ee) {
-  const restore = new Restore(dbClient);
+  const restore = new Restore(dbClient, options);
   const start = new Date().getTime(); // restore start time
@@ -53,3 +54,3 @@ let total = 0; // the total restored
 
-  return pipeline(
+  const batchPreparationStreams = [
     readstream, // the backup file
@@ -59,5 +60,20 @@ new Liner(), // line by line
     new BatchingStream(options.bufferSize, true), // make new arrays of the correct buffer size
-    new MappingStream(restore.docsToRestoreBatch), // make a restore batch
+    new MappingStream(restore.docsToRestoreBatch) // make a restore batch
+  ];
+  const mappingStreams = [];
+  const restoreStreams = [
     new MappingStream(restore.pendingToRestored, options.parallelism), // do the restore at the desired level of concurrency
     output // emit restored events
+  ];
+
+  if (options.attachments) {
+    mappingStreams.push(
+      new MappingStream(new Attachments().decode, options.parallelism)
+    );
+  }
+
+  return pipeline(
+    ...batchPreparationStreams,
+    ...mappingStreams,
+    ...restoreStreams
   ).then(() => {
@@ -64,0 +80,0 @@ return { total };

includes/restoreMappings.js

@@ -1,2 +0,2 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2024 IBM Corp. All rights reserved.
 //
@@ -31,4 +31,5 @@ // Licensed under the Apache License, Version 2.0 (the "License");
 
-  constructor(dbClient) {
+  constructor(dbClient, options) {
     this.dbClient = dbClient;
+    this.options = options;
     this.batchCounter = 0;
@@ -69,3 +70,3 @@ }
         // First line is metadata.
-        mappingDebug(`Parsed backup file metadata ${lineAsJson.name} ${lineAsJson.version} ${lineAsJson.mode}.`);
+        mappingDebug(`Parsed backup file metadata ${lineAsJson.name} ${lineAsJson.version} ${lineAsJson.mode} ${lineAsJson.attachments}.`);
         // This identifies a version of 2.10.0 or newer that wrote the backup file.
@@ -78,2 +79,12 @@ // Set the mode that was used for the backup file.
         // Later we may add other version/feature specific toggles here.
+        if (lineAsJson.attachments === true) {
+          if (!this.options.attachments) {
+            // Error out if trying to restore attachments without the option
+            throw new BackupError('AttachmentsNotEnabledError', 'To restore a backup file with attachments, enable the attachments option.');
+          }
+        } else {
+          if (this.options.attachments) {
+            throw new BackupError('AttachmentsMetadataAbsent', 'Cannot restore with attachments because the backup file was not created with the attachments option.');
+          }
+        }
       } else if (lineAsJson.marker && lineAsJson.marker === marker) {
@@ -80,0 +91,0 @@ mappingDebug(`Resume marker on line  ${backupLine.lineNumber} of backup file.`);

package.json

 {
   "name": "@cloudant/couchbackup",
-  "version": "2.10.3-SNAPSHOT-240",
+  "version": "2.10.3-SNAPSHOT-241",
   "description": "CouchBackup - command-line backup utility for Cloudant/CouchDB",
@@ -5,0 +5,0 @@ "homepage": "https://github.com/IBM/couchbackup",

README.md

@@ -25,3 +25,3 @@ # CouchBackup
 * **`couchbackup` does not do CouchDB replication as such, it simply streams through a database's `_changes` feed, and uses `POST /db/_bulk_get` to fetch the documents, storing the documents it finds on disk.**
-* **`couchbackup` does not support backing up or restoring databases containing documents with attachments. The recommendation is to store attachments directly in an object store. DO NOT USE THIS TOOL FOR DATABASES CONTAINING ATTACHMENTS.** [Note](#note-on-attachments)
+* **`couchbackup` does not support backing up or restoring databases containing documents with attachments. The recommendation is to store attachments directly in an object store. The "attachments" option is provided as-is and is not supported. This option is for Apache CouchDB only and is experimental. DO NOT USE THIS OPTION WITH IBM Cloudant backups.** [Note](#note-on-attachments)
 
@@ -232,2 +232,3 @@ ## Installation
  `https://iam.cloud.ibm.com/identity/token`, but can be overridden if necessary using the `CLOUDANT_IAM_TOKEN_URL` environment variable.
+* `COUCH_ATTACHMENTS` - _EXPERIMENTAL & UNSUPPORTED_ (see [Note](#note-on-attachments)) if `true` will include attachments as part of the backup or restore process.
 * `DEBUG` - if set to `couchbackup`, all debug messages print on `stderr` during a backup or restore process
@@ -251,2 +252,3 @@
 * `--quiet` - same as `COUCH_QUIET`
+* `--attachments` - _EXPERIMENTAL & UNSUPPORTED_ (see [Note](#note-on-attachments)) same as `COUCH_ATTACHMENTS`
 
@@ -295,2 +297,3 @@ ## Using programmatically
  retrieving IAM tokens.
+* `attachments`: _EXPERIMENTAL & UNSUPPORTED_ (see [Note](#note-on-attachments)), see `CLOUDANT_ATTACHMENTS`.
 
@@ -359,2 +362,3 @@ When the backup completes or fails the callback functions gets called with
  retrieving IAM tokens.
+* `attachments`: _EXPERIMENTAL & UNSUPPORTED_ (see [Note](#note-on-attachments)), see `CLOUDANT_ATTACHMENTS`.
 
@@ -444,8 +448,20 @@ When the restore completes or fails the callback functions gets called with
 * `13`: restore target database is not new and empty.
+* `60`: `attachments` option used for backup, but wasn't used for restore.
+* `61`: `attachments` option used for restore, but wasn't used for backup.
 
 ## Note on attachments
 
-TLDR; If you backup a database that has attachments `couchbackup` cannot restore it.
+TLDR; If you backup a database that has attachments without using the `attachments` option `couchbackup` can't restore it.
 
 As documented above `couchbackup` does not support backing up or restoring databases containing documents with attachments.
+
+The recommendation is to store attachments directly in an object store with a link in the JSON document instead of using the
+native attachment API.
+
+### With experimental `attachments` option
+
+The `attachments` option is provided as-is and is not supported. This option is for Apache CouchDB only and is experimental. Do not use this option with IBM Cloudant backups.
+
+### Without experimental `attachments` option
+
 Backing up a database that includes documents with attachments appears to complete successfully. However, the attachment
@@ -455,4 +471,1 @@ content is not downloaded and the backup file contains attachment metadata. So attempts to
 in the restored database.
-
-The recommendation is to store attachments directly in an object store with a link in the JSON document instead of using the
-native attachment API.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

128397

increased by4.88%

Number of package files

22

increased by4.76%

Lines of code

2450

increased by5.38%

Number of lines in readme file

465

increased by2.88%

Worsened metrics

Number of low supply chain risk alerts

27

increased by8%

No dependency changes