4chan - npm Package Compare versions

Comparing version 0.0.9 to 0.0.10

4chan.js

@@ -69,3 +69,3 @@ var optimist = require('optimist');
     return movePics(tcb);
-    
+
 if(argv.d)
@@ -76,3 +76,3 @@ return deleteDirs(tcb);
     return splitPics(tcb);
-    
+
 if(argv._.length != 1) {
@@ -87,3 +87,3 @@ console.log(optimist.help());
     var m = /^(\d+)x(\d+)$/.exec(argv.r);
-    if(!m) 
+    if(!m)
         tcb('Invalid value for the -r (--min-resolution) parameter, try --help.');
@@ -114,3 +114,3 @@ minWidth = Number(m[1]);
     console.log('Only downloading GIFs.');
-    
+
 if(dirExists(landscapeDir) || dirExists(portraitDir)) {
@@ -140,11 +140,11 @@ argv.m = true;
 if(argv.f) {
-    
+
     console.log('Working in forum mode. Single-shot mode is >implied.');
     argv.s = true;
-    
-    var threads = { arr: [] };
-    
-    getIndex(url, _x(tcb, true, function(err, idx, threads) {
-        
-        var ff = _.map(idx, function(pageUrl) {
+
+    var threads = {};
+
+    getIndex(url, threads, _x(tcb, true, function(err, idx) {
+
+        var ff = _.map(idx, function(v, pageUrl) {
             return _x(null, false, function(cb) {
@@ -154,6 +154,6 @@ getPage(pageUrl, threads, cb);
         });
-        
+
         async.parallel(ff, _x(tcb, true, function(err) {
 
-            var f2 = _.map(threads.arr, function(threadUrl) {
+            var f2 = _.map(threads, function(v, threadUrl) {
                 return _x(null, false, function(cb) {
@@ -163,14 +163,14 @@ getThread(threadUrl, cb);
             });
-            
+
             // be kind to the server, work serially
             // (there's a lot of threads in a forum)
-            
+
             async.series(f2, _x(tcb, true, function(err) {
-                
+
                 tcb(null, 'Forum snapshot downloaded, exit.');
             }));
-            
+
         }));
     }));
-    
+
 }
@@ -188,8 +188,8 @@ else {
     fs.readdirSync('.').forEach(function(d) {
-    
+
         if(/^\./.test(d) || !fs.statSync(d).isDirectory())
             return;
-    
+
         fs.readdirSync(d).forEach(function(f) {
-            
+
             var t = f, m;
@@ -199,9 +199,9 @@ // undo the splitPics renaming if present
                 t = m[1];
-            
+
             fs.renameSync(d + '/' + f, t);
         });
-        
+
         fs.rmdirSync(d);
     });
-    
+
     cb();
@@ -215,7 +215,7 @@ }
     var cut, dir = 1, pfx = '';
-    
+
     var digits = Math.floor(log10(arr.length / num)) + 1;
     while(digits--)
         pfx += '0';
-    
+
     var cur = /\/([^\/]+)$/.exec(process.cwd())[1];
@@ -227,18 +227,18 @@
         pfx2 += '0';
-        
+
     var time = Math.floor(Date.now() / 1000) - arr.length;
-    
+
     while((cut = arr.splice(0, num)).length) {
-        
+
         var s = String(dir++);
         s = cur + '.' + pfx.slice(s.length) + s;
-        
+
         console.log('creating dir %s', s);
         fs.mkdirSync(s);
-        
+
         cut.forEach(function(v) {
-            
+
             var q = String(file++);
             q = pfx2.slice(q.length) + q;
-            
+
             fs.utimesSync(v, time, time++);
@@ -248,3 +248,3 @@ fs.renameSync(v, s + '/' + q + '-' + v);
     }
-    
+
     cb();
@@ -270,9 +270,9 @@ }
 function deleteDirs(cb) {
-    
+
     if(argv.y)
         console.log('Dry delete run, only print what would be done.');
-    
+
     var minNumber = typeof(url) === 'number' ? url : 0;
     var len;
-    
+
     fs.readdirSync('.').forEach(function(d) {
@@ -282,3 +282,3 @@
             return;
-        
+
         if(d.indexOf('-') !== -1) {
@@ -288,3 +288,3 @@ console.log('removing marked directory: ' + d);
                 fsutil.rm_rf(d);
-        } 
+        }
         else if((len = fs.readdirSync(d).length) <= minNumber) {
@@ -296,17 +296,14 @@ console.log('removing %s: %s', len === 0 ? 'empty directory' : ('directory with ' + len + ' items'), d);
     });
-    
+
     cb();
 }
 
-function getIndex(url, cb) { // cb(err, idx, threads)
-    
+function getIndex(url, threads, cb) { // cb(err, idx)
+
     console.log('Downloading page and thread index.');
-    
+
     request(url, _x(cb, true, function(err, res, body) {
-        
         var idx = extractPageUrls(body);
-        
         extractThreadUrls(body, threads);
-        
-        cb(null, idx, threads);
+        cb(null, idx);
     }));
@@ -316,7 +313,4 @@ }
 function getPage(url, threads, cb) {
-    
-    request(url, _x(cb, true, function(err, res, body) {
-    
+    request({ url: url }, _x(cb, true, function(err, res, body) {
         extractThreadUrls(body, threads);
-        
         cb(null);
@@ -329,13 +323,18 @@ }));
     var $ = cheerio.load(body);
-    threads.arr = _.union(threads.arr, $('div.thread').map(function() {
-        return url + 'res/' + $(this).attr('id').substr(1);
-    }));
+    $("a.button:contains('View Thread')").each(function(i, e) {
+        var a = url + $(this).attr('href');
+        threads[a] = true;
+    });
 }
 
-function extractPageUrls(body, idx) {
-    
+function extractPageUrls(body) {
+
     var $ = cheerio.load(body);
-    return $('div.desktop div.pages a').map(function() {
-        return url + $(this).attr('href');
+    var links = {};
+    $('div.desktop div.pages a').each(function() {
+        var lnk = $(this).attr('href');
+        if(lnk !== './catalog')
+            links[url + lnk] = true;
     });
+    return links;
 }
@@ -346,3 +345,3 @@
 function getPics(url, thread, cb) {
-    
+
     request(url, _x(cb, true, function(err, res, body) {
@@ -362,10 +361,10 @@
         var $ = cheerio.load(body);
-        
+
         $('a.fileThumb').each(function(i, elem) {
-            
+
             var el = $(this);
             var imgUrl = proto + el.attr('href'); // http://images.4chan.org/s/src/1347323616243.jpg
             var m = /\/([^\/\.]+)\.([^\/]+)$/.exec(imgUrl);
-            
-            var m2, m3, text = el.parent().find('div.fileInfo span.fileText').text();
+
+            var m2, m3, text = el.parent().find('div.fileText').text();
             if(text) {
@@ -375,7 +374,7 @@ m3 = /(\d+)x(\d+)/.exec(text);
             else {
-                // w/h for the index image (will slip through the w/h filter, no choice)
+                // w/h for the index image
                 var style = el.find('img').attr('style')
                 m2 = /height:\s+(\d+)px;\s+width:\s+(\d+)px/.exec(style); // 'height: 125px; width: 83px;'
             }
-            
+
             var entry = {
@@ -391,7 +390,7 @@ file: m[1] + '.' + m[2],
 
-            var skip = 
+            var skip =
                 (argv.r && (entry.width < minWidth || entry.height < minHeight))
                 || (argv.n && entry.ext === 'gif')
                 || (argv.g && entry.ext !== 'gif');
-            
+
             if(!skip) {
@@ -402,3 +401,3 @@ ret.index[entry.file] = entry;
         });
-        
+
         cb(null, ret);
@@ -420,3 +419,3 @@ }));
 function getThread(url, cb) { // cb(err, msg)
-    
+
     var thread = /([^\/]+)$/.exec(url)[1];
@@ -428,14 +427,14 @@ if(argv.t) {
     }
-    
+
     getPics(url, thread, _x(cb, true, function(err, ret) {
 
         downloadPics(ret, _x(cb, true, function(err) {
-        
+
             if(argv.s)
                 return cb(null, 'Thread snapshot downloaded, exiting.');
-        
+
             console.log("Initial download finished, \"I am monitoring this thread\" for new items now.");
-        
+
             setInterval(_x(cb, false, function() {
-            
+
                 getPics(url, thread, _x(cb, true, function(err, ret) {
@@ -445,6 +444,6 @@ downloadPics(ret, _x(cb, true, function(err) {
                 }));
-            
+
             }), 60000);
         }));
-    
+
     }));
@@ -487,3 +486,3 @@ }
 function downloadPics(ret, cb) {
-    
+
     if(current && !argv.f) {
@@ -500,3 +499,3 @@ _.each(current.order, function(entry) {
     current = ret;
-    
+
     var funcs = [];
@@ -508,10 +507,10 @@ _.each(ret.order, function(entry) {
             return;
-        
+
         funcs.push(function(cb) {
-            
+
             var opts = {
-                url: entry.url, 
+                url: entry.url,
                 encoding: null
             };
-            
+
             request(opts, _x(cb, true, function(err, res, body) {
@@ -532,7 +531,7 @@ if(res.statusCode == 404) {
             }));
-            
+
         });
     });
-    
+
     async.series(funcs, cb);
 }

package.json

@@ -6,3 +6,3 @@ {
 	"keywords": [ "4chan", "picture", "downloader", "utility", "command line" ],
-	"version": "0.0.9",
+	"version": "0.0.10",
 	"homepage": "http://github.com/ypocat/4chan",
@@ -9,0 +9,0 @@ "repository": {

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Lines of code

403

increased by1.26%

Worsened metrics

Total package byte prevSize

18144

decreased by-2.44%

No dependency changes