cartoon-catch - npm Package Compare versions

Comparing version 2.0.0 to 2.1.0

dist/download.js

@@ -24,3 +24,3 @@ "use strict";
     return __awaiter(this, void 0, void 0, function* () {
-        const { title = '名称', parallel = 5 } = options;
+        const { title = '名称', parallel = 5 } = options || {};
         const total = images.length;
@@ -54,4 +54,4 @@ bar = new progress_1.default(`${title}(:current/:total): [:bar] [:percent]`, { total: +total, width: 50 });
 function downloadFile(image, options, callback) {
-    const { timeout = 5000, gainInterval = 3000, againTimes = 0 } = options;
-    let { url, path = '/', fileName = helper_1.createFileName(), extract = 'jpg' } = image;
+    const { timeout = 5000, gainInterval = 3000, againTimes = 0, headers = {} } = options || {};
+    let { url, path = '/', fileName = helper_1.createFileName(), extract = 'jpg' } = image || {};
     const allPath = `${path}${fileName}.${extract}`;
@@ -99,5 +99,3 @@ if (!url)
         timeout: timeout + 300,
-        headers: {
-            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.3 Mobile/15E148 Safari/604.1",
-        }
+        headers: Object.assign({ "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.3 Mobile/15E148 Safari/604.1" }, headers)
     })
@@ -104,0 +102,0 @@ .then(res => res.body.pipe(writeStream))

dist/helper/index.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createFileName = void 0;
+exports.getHost = exports.isArray = exports.createFileName = void 0;
 function createFileName() {
@@ -8,2 +8,11 @@ return `${Date.now()}${Math.ceil(Math.random() * 5000 + 5000)}`;
 exports.createFileName = createFileName;
+function isArray(t) {
+    return Object.prototype.toString.call(t) == '[object Array]';
+}
+exports.isArray = isArray;
+function getHost(url) {
+    const urlArr = url.match(/^http(s)?:\/\/(.*?)\//);
+    return urlArr ? urlArr[0] : null;
+}
+exports.getHost = getHost;
 //# sourceMappingURL=index.js.map

dist/main.js

@@ -39,8 +39,10 @@ "use strict";
 const download_1 = __importDefault(require("./download"));
+const helper_1 = require("./helper");
 const log = debug_1.default.debug('debug');
-const grap = (url, options, transform) => __awaiter(void 0, void 0, void 0, function* () {
+const grap = (pageUrl, options, transform) => __awaiter(void 0, void 0, void 0, function* () {
+    const host = helper_1.getHost(pageUrl);
     options = Object.assign({ host: '', encoding: true }, options);
     if (!options.target)
         throw new Error('请输入target');
-    let html = yield node_fetch_1.default(url, {
+    let html = yield node_fetch_1.default(pageUrl, {
         method: 'get',
@@ -51,5 +53,5 @@ timeout: 5000,
         .then((html) => __awaiter(void 0, void 0, void 0, function* () { return (options.encoding ? yield html.text() : yield html.buffer()); }))
-        .then((html) => options.encoding ? html : Iconv.decode(html, 'gb2312'))
+        .then((html) => typeof html === 'string' ? html : Iconv.decode(html, 'gb2312'))
         .catch(e => {
-        log(`请求超时 ${url} ${e}`);
+        log(`请求超时 ${pageUrl} ${e}`);
         return null;
@@ -59,3 +61,6 @@ });
     const urls = $(options.target).toArray().map(item => {
-        let url = `${options.host}${$(item).attr('href')}`;
+        let url = $(item).attr('href');
+        if (!/^http(s)?:\/\//.test(url)) {
+            url = `${options.host || host}${url}`;
+        }
         let title = $(item).text().trim();
@@ -73,3 +78,3 @@ options.urlReplace && (url = url.replace(options.urlReplace[0], options.urlReplace[1]));
         let url, title, result;
-        if (Object.prototype.toString.call(item) == '[object Array]') {
+        if (helper_1.isArray(item)) {
             [url, title] = item;
@@ -84,3 +89,3 @@ }
         else {
-            // log('发起请求', url);
+            log('发起请求', url);
             result = yield node_fetch_1.default(url, {
@@ -87,0 +92,0 @@ method: 'get',

package.json

 {
   "name": "cartoon-catch",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "cartoon-catch",
@@ -5,0 +5,0 @@ "main": "dist/main.js",

README.md

@@ -27,5 +27,5 @@ # 漫画爬虫
 |:----|:----|:----|:----|:----|
-| host  | http://xxx.com | 是 | string | 网站域名 |
 | name  | ../鬼灭之刃 | 是 | string | 指定下载路径 |
 | target | .list a | 是 | string | 跳转到详情的dom |
+| host  | http://xxx.com | 否 | string | 设置详情页host, 默认使用页面host |
 | imageHost | http://cdn.xxx.com | 否 | string | 图片地址的host，如无就无需传 |
@@ -32,0 +32,0 @@ | encoding | true | 否 | boolean | 请求返回值乱码时传false |

src/download.ts

@@ -12,3 +12,3 @@ import Progress from 'progress'
 async function download(images: Array<ImageInterface>, options: OptionsInterface): Promise<string>{
-    const { title = '名称', parallel = 5 } = options;
+    const { title = '名称', parallel = 5 } = options || {};
     const total = images.length;
@@ -43,5 +43,5 @@ bar = new Progress(`${title}(:current/:total): [:bar] [:percent]`, { total: +total, width: 50 });
 
-function downloadFile(image: ImageInterface, options: OptionsInterface, callback): Function | void {
-    const { timeout = 5000, gainInterval = 3000, againTimes = 0 } = options;
-    let { url, path = '/', fileName = createFileName(), extract = 'jpg' } = image;
+function downloadFile(image: ImageInterface, options: OptionsInterface, callback: Function): Function | void {
+    const { timeout = 5000, gainInterval = 3000, againTimes = 0, headers = {} } = options || {};
+    let { url, path = '/', fileName = createFileName(), extract = 'jpg' } = image || {};
     const allPath = `${path}${fileName}.${extract}`;
@@ -93,11 +93,12 @@ if(!url) return callback(allPath, 2);
             "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.3 Mobile/15E148 Safari/604.1",
+            ...headers
 
         }
     })
-        .then(res => res.body.pipe(writeStream))
-        .catch((err) => {
-            log(`${url} 请求超时 ${err}`);
-            clearTimeout(tryAgain);
-            tryAgain = setTimeout(failCallBack, gainInterval);
-        })
+    .then(res => res.body.pipe(writeStream))
+    .catch((err) => {
+        log(`${url} 请求超时 ${err}`);
+        clearTimeout(tryAgain);
+        tryAgain = setTimeout(failCallBack, gainInterval);
+    })
 }
@@ -104,0 +105,0 @@

src/helper/index.ts

@@ -5,4 +5,15 @@ function createFileName(){
 
+function isArray (t) {
+	return Object.prototype.toString.call(t)== '[object Array]'
+}
+
+function getHost(url){
+	const urlArr = url.match(/^http(s)?:\/\/(.*?)\//);
+	return urlArr ? urlArr[0] : null;
+}
+
 export {
-	createFileName
+	createFileName,
+	isArray,
+	getHost
 }

src/interface/download.interface.ts

@@ -16,2 +16,3 @@ export interface ImageInterface {
 	againTimes: number
+	headers: any
 }

src/main.ts

@@ -8,9 +8,12 @@
 import { OptionsInterface } from './interface/main.interface';
+import { isArray, getHost } from './helper';
 
+
 const log = Debug.debug('debug')
 
-const grap = async (url: string, options: OptionsInterface, transform: Function) => {
+const grap = async (pageUrl: string, options: OptionsInterface, transform: Function) => {
+    const host = getHost(pageUrl);
     options = Object.assign({host: '', encoding: true }, options)
     if(!options.target) throw new Error('请输入target');
-    let html = await nodeFetch(url, {
+    let html = await nodeFetch(pageUrl, {
         method: 'get',
@@ -24,5 +27,5 @@ timeout: 5000,
     .then(async (html) => (options.encoding ? await html.text() : await html.buffer()))
-    .then((html: any) => options.encoding ? html : Iconv.decode(html, 'gb2312'))
+    .then((html: Buffer | string) => typeof html === 'string' ? html : Iconv.decode(html, 'gb2312'))
     .catch(e => {
-        log(`请求超时 ${url} ${e}`)
+        log(`请求超时 ${pageUrl} ${e}`)
         return null
@@ -32,3 +35,6 @@ });
     const urls = $(options.target).toArray().map(item => {
-        let url = `${options.host}${$(item).attr('href')}`;
+        let url = $(item).attr('href');
+        if(!/^http(s)?:\/\//.test(url)){
+            url = `${options.host || host}${url}`;
+        }
         let title = $(item).text().trim();
@@ -47,3 +53,3 @@ options.urlReplace && (url = url.replace(options.urlReplace[0], options.urlReplace[1]))
         let url, title, result;
-        if(Object.prototype.toString.call(item)== '[object Array]'){
+        if(isArray(item)){
             [url, title] = item;
@@ -57,3 +63,3 @@ }else{
         }else{
-            // log('发起请求', url);
+            log('发起请求', url);
             result = await nodeFetch(url, {
@@ -60,0 +66,0 @@ method: 'get',

Fixed alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

32137

increased by4.72%

Lines of code

508

increased by5.83%

Number of medium supply chain risk alerts

0

decreased by-100%

No dependency changes