website-scraper - npm Package Compare versions

Comparing version 1.2.0 to 1.2.1

lib/config/resource-extensions-by-type.js

@@ -5,5 +5,11 @@ var types = require('./resource-types');
 
-defaultExtensions[types.html] = '.html';
-defaultExtensions[types.css] = '.css';
+defaultExtensions[types.html] = {
+	defaultExtension: '.html',
+	possibleExtensions: [ '.html', '.htm' ]
+};
+defaultExtensions[types.css] = {
+	defaultExtension: '.css',
+	possibleExtensions: [ '.css' ]
+};
 
 module.exports = defaultExtensions;

lib/filename-generators/by-site-structure.js

 var _ = require('lodash');
 var path = require('path');
 var utils = require('../utils');
+var resourceTypes = require('../config/resource-types');
+var resourceTypeExtensions = require('../config/resource-extensions-by-type');
 
@@ -11,4 +13,9 @@ module.exports = function generateFilename (resource, options) {
 	// If we have HTML from 'http://example.com/path' => set 'path/index.html' as filepath
-	if (resource.isHtml() && !extension) {
-		filePath = path.join(filePath, options.defaultFilename);
+	if (resource.isHtml()) {
+		var htmlExtensions = resourceTypeExtensions[resourceTypes.html].possibleExtensions;
+		var resourceHasHtmlExtension = _.includes(htmlExtensions, extension);
+		// add index.html only if filepath has ext != html '/path/test.com' => '/path/test.com/index.html'
+		if (!resourceHasHtmlExtension) {
+			filePath = path.join(filePath, options.defaultFilename);
+		}
 	}
@@ -15,0 +22,0 @@

lib/filename-generators/by-type.js

 var _ = require('lodash');
 var path = require('path');
 var utils = require('../utils.js');
-var defaultExtensions = require('../config/resource-extensions-by-type');
+var typeExtensions = require('../config/resource-extensions-by-type');
 
@@ -35,4 +35,4 @@ module.exports = function generateFilename (resource, options, occupiedFileNames) {
 
-	if (!extension && defaultExtensions[resourceType]) {
-		extension = defaultExtensions[resourceType];
+	if (!extension && typeExtensions[resourceType]) {
+		extension = typeExtensions[resourceType].defaultExtension;
 		filename += extension;
@@ -39,0 +39,0 @@ }

lib/utils.js

@@ -52,3 +52,4 @@ var url = require('url');
 function getFilepathFromUrl (u) {
-	return url.parse(u).pathname.substring(1);
+	var normalizedUrl = normalizeUrl(u);
+	return url.parse(normalizedUrl).pathname.substring(1);
 }
@@ -60,4 +61,10 @@
 
-function getFilenameExtension (filename) {
-	return (typeof filename === 'string') ? path.extname(filename) : null;
+/**
+ * Returns extension for given filepath
+ * Example: some/path/file.js => .js
+ * @param {string} filepath
+ * @returns {string|null} - extension
+ */
+function getFilenameExtension (filepath) {
+	return (typeof filepath === 'string') ? path.extname(filepath) : null;
 }
@@ -64,0 +71,0 @@

package.json

 {
   "name": "website-scraper",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "description": "Download website to a local directory (including all css, images, js, etc.)",
@@ -5,0 +5,0 @@ "readmeFilename": "README.md",

No alert changes

Improved metrics

Total package byte prevSize

39203

increased by1.95%

Lines of code

778

increased by2.64%

No dependency changes