website-scraper - npm Package Compare versions

Comparing version 3.3.7 to 3.4.0

lib/config/defaults.js

@@ -62,5 +62,6 @@ 'use strict';
 	resourceSaver: null,
-	updateMissingSources: false
+	updateMissingSources: false,
+	updateSources: true,
 };
 
 module.exports = config;

lib/resource-handler/index.js

@@ -11,3 +11,3 @@ 'use strict';
 
-const supportedOptions = ['prettifyUrls', 'sources', 'recursiveSources', 'maxRecursiveDepth', 'defaultFilename', 'updateMissingSources'];
+const supportedOptions = ['prettifyUrls', 'sources', 'recursiveSources', 'maxRecursiveDepth', 'defaultFilename', 'updateMissingSources', 'updateSources'];
 
@@ -87,2 +87,5 @@ class ResourceHandler {
 		return utils.waitAllFulfilled(childrenPromises).then(function updateChildrenPaths () {
+			if (self.options.updateSources === false) {
+				return pathContainer.updateText([]);
+			}
 			return pathContainer.updateText(pathsToUpdate);
@@ -93,3 +96,3 @@ });
 	updateChildrenResources (pathContainer, parentResource, needToUpdate) {
-		if (!needToUpdate) {
+		if (!needToUpdate || this.options.updateSources === false) {
 			return Promise.resolve(pathContainer.updateText([]));
@@ -96,0 +99,0 @@ }

package.json

 {
   "name": "website-scraper",
-  "version": "3.3.7",
+  "version": "3.4.0",
   "description": "Download website to a local directory (including all css, images, js, etc.)",
@@ -5,0 +5,0 @@ "readmeFilename": "README.md",

README.md

@@ -67,2 +67,3 @@ ## Introduction
 * [requestConcurrency](#requestconcurrency) - set maximum concurrent requests
+* [updateSources](#updateSources) - set to false to keep all html content unmodified
  
@@ -300,2 +301,8 @@ Default options you can find in [lib/config/defaults.js](https://github.com/website-scraper/node-website-scraper/blob/master/lib/config/defaults.js) or get them using `scrape.defaults`.
 
+#### updateSources
+Boolean. Defaults to `true`. Use `false` when scraped site structure does not
+fit your custom filename generator or if you do not want html content to be
+modified in any way.
+
+
 ## callback 
@@ -302,0 +309,0 @@ Callback function, optional, includes following parameters:

No alert changes

Improved metrics

Total package byte prevSize

55153

increased by0.83%

Lines of code

1084

increased by0.37%

Number of lines in readme file

322

increased by2.22%

No dependency changes