🎩 You're Invited:Meet the Socket team at Black Hat in Las Vegas, August 3-6.RSVP →
Sign In

bauer-plugin-scrape

Package Overview
Dependencies
Maintainers
1
Versions
4
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

bauer-plugin-scrape - npm Package Compare versions

Comparing version
0.2.1
 to
0.2.2
+43
.jscsrc
{
"disallowKeywords": ["with"],
"disallowKeywordsOnNewLine": ["else"],
"disallowMixedSpacesAndTabs": true,
"disallowMultipleVarDecl": "exceptUndefined",
"disallowNewlineBeforeBlockStatements": true,
"disallowQuotedKeysInObjects": true,
"disallowSpaceAfterObjectKeys": true,
"disallowSpaceAfterPrefixUnaryOperators": true,
"disallowTrailingWhitespace": true,
"disallowTrailingComma": true,
"maximumLineLength": 100,
"requireCamelCaseOrUpperCaseIdentifiers": true,
"requireCapitalizedConstructors": true,
"requireCurlyBraces": true,
"requireSpaceAfterKeywords": [
"if",
"else",
"for",
"while",
"do",
"switch",
"case",
"return",
"try",
"catch",
"typeof"
],
"requireSpaceAfterLineComment": true,
"requireSpaceAfterBinaryOperators": true,
"requireSpaceBeforeBinaryOperators": true,
"requireSpaceBeforeBlockStatements": true,
"requireSpaceBeforeObjectValues": true,
"requireSpaceBetweenArguments": true,
"requireSpacesInFunction": {
"beforeOpeningRoundBrace": true,
"beforeOpeningCurlyBrace": true
},
"requireEarlyReturn": true,
"validateIndentation": 2,
"validateLineBreaks": "LF",
"validateQuoteMarks": "'"
}
+10
-10
lib/index.js

@@ -9,8 +9,8 @@ /*!

"use strict";
'use strict';
module.exports = {
name: "scrape",
name: 'scrape',
config: {

@@ -22,13 +22,13 @@ workers: 1,

file: {
dir: ".",
ext: "json"
dir: '.',
ext: 'json'
}
}
},
worker: __dirname + "/worker.js",
promise: __dirname + "/promise.js"
worker: __dirname + '/worker.js',
promise: __dirname + '/promise.js'
};
// - -------------------------------------------------------------------- - //
+15
-12
lib/promise.js

@@ -9,3 +9,3 @@ /*!

"use strict";
'use strict';

@@ -15,16 +15,19 @@ // - -------------------------------------------------------------------- - //

module.exports = {
scrape: {
// .scrape(scrape Object) :Promise
o: function(scrape) {
return this.then(function(source) {
return this.promise().scrape(source,scrape);
o: function (scrape) {
return this.then(function (source) {
return this.requestWorker('scrape', {
source: source,
scrape: scrape
}).get('file');
});
},
// .scrape(source String, scrape Object) :Promise
so: function(source,scrape) {
return this.then(function() {
return this.requestWorker("scrape",{
so: function (source,scrape) {
return this.then(function () {
return this.requestWorker('scrape', {
source: source,

@@ -35,7 +38,7 @@ scrape: scrape

}
}
};
// - -------------------------------------------------------------------- - //
+22
-22
lib/worker.js

@@ -9,22 +9,22 @@ /*!

"use strict";
'use strict';
var path = require("path");
var dom = require("bauer-dom");
var Cache = require("bauer-cache");
var factory = require("bauer-factory");
var path = require('path');
var dom = require('bauer-dom');
var Cache = require('bauer-cache');
var factory = require('bauer-factory');
// - -------------------------------------------------------------------- - //
module.exports = function(worker,config) {
worker.on("request",function(options,response) {
module.exports = function (worker,config) {
worker.on('request', function (options,response) {
var input = new Cache({ file: options.source });
var outputFile = { file: { name: path.basename(input.getFile()) } };
var outputOptions = factory.merge(options.cache,outputFile,config.cache);
var outputOptions = factory.merge(options.cache, outputFile, config.cache);
var output = new Cache(outputOptions);
output.validate(function(error,valid) {
output.validate(function (error,valid) {
if (error) {

@@ -35,15 +35,15 @@ response.sendError(error);

} else {
input.exists(function(error,exists) {
input.exists(function (error,exists) {
if (error) {
response.sendError(error);
} else if (exists) {
input.read(function(error,content) {
input.read(function (error,content) {
if (error) {
response.sendError(error);
} else {
var data = dom(content).scrape(options.scrape);
output.write(data,function(error) {
output.write(data, function (error) {
if (error) {

@@ -58,3 +58,3 @@ response.sendError(error);

} else {
response.sendError(new Error("input not found"));
response.sendError(new Error('input not found'));
}

@@ -65,7 +65,7 @@ });

});
worker.sendReady();
};
// - -------------------------------------------------------------------- - //
+1
-1
package.json
{
"name": "bauer-plugin-scrape",
"version": "0.2.1",
"version": "0.2.2",
"description": "Plugin for bauer to scrape content.",

@@ -5,0 +5,0 @@ "keywords": [

+17
-17
test/sample/sample.js
// - -------------------------------------------------------------------- - //
"use strict";
'use strict';
var fs = require("fs");
var assert = require("assert");
var Crawler = require("bauer-crawler");
var fs = require('fs');
var assert = require('assert');
var Crawler = require('bauer-crawler');

@@ -21,16 +21,16 @@ var crawler = new Crawler({

crawler.loadPlugin(__dirname + "/../../");
crawler.loadPlugin(__dirname + '/../../');
crawler.start(function() {
crawler.start(function () {
return this.Promise
.scrape(__dirname + "/sample.html",{
"head > title": {
title: "text",
.scrape(__dirname + '/sample.html', {
'head > title': {
title: 'text'
},
"[id]": {
'[id]': {
ids: {
"a[href][title]": {
url: "attr:href",
title: "attr:title"
'a[href][title]': {
url: 'attr:href',
title: 'attr:title'
}

@@ -40,6 +40,6 @@ }

})
.then(function(file) {
.then(function (file) {
var output = JSON.parse(fs.readFileSync(file).toString());
var compare = JSON.parse(fs.readFileSync(__dirname + "/sample.json").toString());
assert.deepEqual(output,compare);
var compare = JSON.parse(fs.readFileSync(__dirname + '/sample.json').toString());
assert.deepEqual(output, compare);
fs.unlinkSync(file);

@@ -46,0 +46,0 @@ });

+20
-20
test/test-scrape.js
// - -------------------------------------------------------------------- - //
"use strict";
'use strict';
var cp = require("child_process");
var assert = require("assert");
var cp = require('child_process');
var assert = require('assert');
// - -------------------------------------------------------------------- - //
describe("scrape",function() {
describe('scrape', function () {
it("require",function() {
require(__dirname + "/../lib/promise.js");
require(__dirname + "/../lib/worker.js");
require(__dirname + "/../lib/index.js");
it('require', function () {
require(__dirname + '/../lib/promise.js');
require(__dirname + '/../lib/worker.js');
require(__dirname + '/../lib/index.js');
});
it("sample",function(done) {
var proc = cp.spawn("node",[__dirname + "/sample/sample.js"],{ stdio: "pipe" });
var output = "";
proc.stdout.on("data",function(data) {
output += data.toString("utf8");
it('sample', function (done) {
var proc = cp.spawn('node', [__dirname + '/sample/sample.js'], { stdio: 'pipe' });
var output = '';
proc.stdout.on('data', function (data) {
output += data.toString('utf8');
});
var error = "";
proc.stderr.on("data",function(data) {
error += data.toString("utf8");
var error = '';
proc.stderr.on('data', function (data) {
error += data.toString('utf8');
});
proc.on("exit",function() {
assert.equal(error,"");
proc.on('exit', function () {
assert.equal(error, '');
done();
});
});
});
// - -------------------------------------------------------------------- - //
-87
.jshintrc
{
// JSHint Default Configuration File (as on JSHint website)
// See http://jshint.com/docs/ for more details
"maxerr" : 50, // {int} Maximum error before stopping
// Enforcing
"bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.)
"camelcase" : true, // true: Identifiers must be in camelCase
"curly" : true, // true: Require {} for every new block or scope
"eqeqeq" : true, // true: Require triple equals (===) for comparison
"forin" : true, // true: Require filtering for..in loops with obj.hasOwnProperty()
"freeze" : true, // true: prohibits overwriting prototypes of native objects such as Array, Date etc.
"immed" : true, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
"indent" : 2, // {int} Number of spaces to use for indentation
"latedef" : true, // true: Require variables/functions to be defined before being used
"newcap" : true, // true: Require capitalization of all constructor functions e.g. `new F()`
"noarg" : true, // true: Prohibit use of `arguments.caller` and `arguments.callee`
"noempty" : true, // true: Prohibit use of empty blocks
"nonbsp" : true, // true: Prohibit "non-breaking whitespace" characters.
"nonew" : true, // true: Prohibit use of constructors for side-effects (without assignment)
"plusplus" : false, // true: Prohibit use of `++` & `--`
"quotmark" : "double", // Quotation mark consistency:
// false : do nothing (default)
// true : ensure whatever is used is consistent
// "single" : require single quotes
// "double" : require double quotes
"undef" : true, // true: Require all non-global variables to be declared (prevents global leaks)
"unused" : true, // true: Require all defined variables be used
"strict" : true, // true: Requires all functions run in ES5 Strict Mode
"maxparams" : false, // {int} Max number of formal params allowed per function
"maxdepth" : false, // {int} Max depth of nested blocks (within functions)
"maxstatements" : false, // {int} Max number statements per function
"maxcomplexity" : false, // {int} Max cyclomatic complexity per function
"maxlen" : false, // {int} Max number of characters per line
// Relaxing
"asi" : false, // true: Tolerate Automatic Semicolon Insertion (no semicolons)
"boss" : false, // true: Tolerate assignments where comparisons would be expected
"debug" : false, // true: Allow debugger statements e.g. browser breakpoints.
"eqnull" : false, // true: Tolerate use of `== null`
"es5" : false, // true: Allow ES5 syntax (ex: getters and setters)
"esnext" : false, // true: Allow ES.next (ES6) syntax (ex: `const`)
"moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features)
// (ex: `for each`, multiple try/catch, function expression…)
"evil" : false, // true: Tolerate use of `eval` and `new Function()`
"expr" : false, // true: Tolerate `ExpressionStatement` as Programs
"funcscope" : false, // true: Tolerate defining variables inside control statements
"globalstrict" : false, // true: Allow global "use strict" (also enables 'strict')
"iterator" : false, // true: Tolerate using the `__iterator__` property
"lastsemic" : false, // true: Tolerate omitting a semicolon for the last statement of a 1-line block
"laxbreak" : false, // true: Tolerate possibly unsafe line breakings
"laxcomma" : false, // true: Tolerate comma-first style coding
"loopfunc" : false, // true: Tolerate functions being defined in loops
"multistr" : false, // true: Tolerate multi-line strings
"noyield" : false, // true: Tolerate generator functions with no yield statement in them.
"notypeof" : false, // true: Tolerate invalid typeof operator values
"proto" : false, // true: Tolerate using the `__proto__` property
"scripturl" : false, // true: Tolerate script-targeted URLs
"shadow" : false, // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
"sub" : false, // true: Tolerate using `[]` notation when it can still be expressed in dot notation
"supernew" : false, // true: Tolerate `new function () { ... };` and `new Object;`
"validthis" : false, // true: Tolerate using this in a non-constructor function
// Environments
"browser" : false, // Web Browser (window, document, etc)
"browserify" : true, // Browserify (node.js code in the browser)
"couch" : false, // CouchDB
"devel" : true, // Development/debugging (alert, confirm, etc)
"dojo" : false, // Dojo Toolkit
"jasmine" : false, // Jasmine
"jquery" : false, // jQuery
"mocha" : true, // Mocha
"mootools" : false, // MooTools
"node" : true, // Node.js
"nonstandard" : false, // Widely adopted globals (escape, unescape, etc)
"prototypejs" : false, // Prototype and Scriptaculous
"qunit" : false, // QUnit
"rhino" : false, // Rhino
"shelljs" : false, // ShellJS
"worker" : false, // Web Workers
"wsh" : false, // Windows Scripting Host
"yui" : false, // Yahoo User Interface
// Custom Globals
"globals" : {} // additional predefined global variables
}