crawler-js - npm Package Compare versions

Comparing version 0.0.4 to 0.0.5

lib/csv_to_json.js

		@@ -26,4 +26,7 @@ readCsv = function(font,url,config){
		setTimeout(function(){
		line = line.replace('\r','');
		data = line.split(",");
		line = line.replace('\r','').replace(/^"(.+(?="$))"$/, '$1');
		data = line.split('"'+font[font['type']+'Attributes'][font[font['type']+'CurrentEntry']]['delimiter']+'"');
		if(data.length == 0){
		line.split(font[font['type']+'Attributes'][font[font['type']+'CurrentEntry']]['delimiter']);
		}
		toJson(font,url,data,config);
		@@ -36,10 +39,13 @@ cb();
		toJson = function(font,url,row,config){
		if(typeof row[0] !=='undefined' && row[0] !=''){
		if(typeof row[1] !=='undefined' && row[1] !=''){
		url.csvId = row[1];
		var data = font[font['type']+'Attributes'][font[font['type']+'CurrentEntry']]['data'];
		var csvId = font[font['type']+'Attributes'][font[font['type']+'CurrentEntry']]['csvId'];

		if(typeof row[data] !=='undefined' && row[data] !=''){
		if(typeof row[csvId] !=='undefined' && row[csvId] !=''){
		url.csvId = row[csvId];
		}
		if(typeof font['encodeGet'] != 'undefined'){
		row[0] = encodeURIComponent(row[0]);
		row[data] = encodeURIComponent(row[data]);
		}
		url[font['type']] = font[font['type']].replace(font[font['type']+'Attributes'][font[font['type']+'CurrentEntry']]['parameter'], row[0]);
		url[font['type']] = font[font['type']].replace(font[font['type']+'Attributes'][font[font['type']+'CurrentEntry']]['parameter'], row[data]);
		var fontTemp = clone(font);
		@@ -46,0 +52,0 @@ fontTemp[font['type']+'CurrentEntry']++;

lib/map_urls.js

		@@ -53,3 +53,3 @@ var RateLimiter = require('limiter').RateLimiter
		var letters = /\[letters:(.[0-9]{0,}?)\]/gim;
		var csv = /\[csv:(.*?)\]/gim;
		var csv = /\[csv:(.?){0,}:(.?){0,}:(.[0-9]{0,10}?):(.[0-9]{0,10}?)\]/gim;
		var mongodb = /\[mongodb:(.[\w]{0,}?):(.[\w]{0,}?):(.[\w]{0,}?)\]/gim;
		@@ -100,2 +100,5 @@ var words = /\[words:\[(.*?){0,}\]\]/gim;
		attributes['file'] = attribute[1];
		attributes['delimiter'] = attribute[2];
		attributes['data'] = attribute[3];
		attributes['csvId'] = attribute[4];
		attributes['position'] = 0;
		@@ -102,0 +105,0 @@ attributes['size'] = 1;

package.json

		{
		"name": "crawler-js",
		"description": "Opensource Framework Crawler in Node.js",
		"version": "0.0.4",
		"version": "0.0.5",
		"main": "index.js",
		@@ -6,0 +6,0 @@ "author": "Rodrigo Matheus <rodrigorizando@gmail.com>",

New alerts