open-graph-scraper - npm Package Compare versions

Comparing version 2.2.0 to 2.2.2

app.js

 var request = require('request'),
 	cheerio = require('cheerio'),
+	charset = require('charset'),
+	iconv = require('iconv-lite'),
 	_ = require('lodash');
@@ -337,5 +339,5 @@
 			options.timeout = inputTimeout;
-			options.headers = {
+			options.headers = Object.assign({
 				'user-agent': 'request.js'
-			};
+			}, options.headers);
 			options.gzip = true;
@@ -436,2 +438,9 @@ that.getOG(options, function (err, results) {
 		} else {
+			if (options.encoding === null) {
+				if (charset(response.headers)) {
+					body = iconv.decode(body, charset(response.headers));
+				} else {
+					body = body.toString();
+				}
+			}
 			var $ = cheerio.load(body),
@@ -442,2 +451,6 @@ meta = $('meta'),
 
+			if (options.withCharset) {
+				ogObject.charset = charset(response.headers, body);
+			}
+
 			keys.forEach(function (key) {
@@ -444,0 +457,0 @@ if (!(meta[key].attribs && (meta[key].attribs.property || meta[key].attribs.name))) {

package.json

 {
 	"name": "open-graph-scraper",
 	"description": "Node.js scraper service for Open Graph info",
-	"version": "2.2.0",
+	"version": "2.2.2",
 	"license": "MIT",
@@ -15,3 +15,5 @@ "main": "app.js",
 		"lodash": "4.2.1",
-		"cheerio": "0.20.0"
+		"cheerio": "0.20.0",
+		"charset": "1.0.0",
+		"iconv-lite": "0.4.13"
 	},
@@ -18,0 +20,0 @@ "devDependencies": {

README.md

@@ -21,2 +21,12 @@ openGraphScraper
 ```
+You can set custom headers. For example scraping data in a specific language:
+```
+var ogs = require('open-graph-scraper');
+var options = {'url': 'http://ogp.me/', 'headers': { 'accept-language': 'en' }};
+ogs(options, function (err, results) {
+	console.log('err:', err); // This is returns true or false. True if there was a error. The error it self is inside the results object.
+	console.log('results:', results);
+});
+```
+
 You can also set a timeout flag like... Example four seconds:
@@ -23,0 +33,0 @@ ```

tests/index.spec.js

@@ -111,2 +111,15 @@ var app = require('../app'),
 
+// test charset utf-8
+var optionCharset1 = {
+		'url': 'http://ogp.me/',
+		'withCharset': true
+	};
+
+// test charset windows-1251
+var optionCharset2 = {
+		'url': 'http://www.gazeta.ru/',
+		'encoding': null,
+		'withCharset': true
+	};
+
 describe('GET OG', function () {
@@ -317,3 +330,3 @@ this.timeout(10000); // should wait at least ten seconds before failing
 			expect(result.data.ogType).to.be('video.other');
-			expect(result.data.ogImage.url).to.be('https://static-cdn.jtvnw.net/v1/AUTH_system/vods_1138/warcraft_22339636096_485121236/thumb/thumb0-480x320.jpg');
+			expect(result.data.ogImage.url).to.be('https://static-cdn.jtvnw.net/s3_vods/294d4c5c42_warcraft_22339636096_485121236/thumb/thumb0-480x320.jpg');
 			expect(result.data.ogVideo.url).to.be('http://www-cdn.jtvnw.net/swflibs/TwitchPlayer.swf?videoId=v78039967&playerType=facebook');
@@ -358,2 +371,19 @@ expect(result.data.ogVideo.type).to.be('application/x-shockwave-flash');
 	});
+	it('Valid Call - Utf-8 charset - Should Return correct Open Graph Info + charset info', function (done) {
+		app(optionCharset1, function (err, result) {
+			expect(err).to.be(false);
+			expect(result.success).to.be(true);
+			expect(result.data.charset).to.be('utf8');
+			done();
+		});
+	});
+	it('Valid Call - windows-1251 charset - Should Return correct Open Graph Info + charset info', function (done) {
+		app(optionCharset2, function (err, result) {
+			expect(err).to.be(false);
+			expect(result.success).to.be(true);
+			expect(result.data.charset).to.be('windows-1251');
+			expect(result.data.ogTitle).to.be('Главные новости - Газета.Ru');
+			done();
+		});
+	});
 });

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

30913

increased by5.75%

Lines of code

899

increased by4.66%

Number of lines in readme file

82

increased by13.89%

Worsened metrics

Dependency count

5

increased by66.67%

Dependency changes

• + Addedcharset@1.0.0

• + Addediconv-lite@0.4.13

• + Addedcharset@1.0.0(transitive)

• + Addediconv-lite@0.4.13(transitive)