parliament-data-scraper - npm Package Compare versions

Comparing version 1.0.0 to 1.1.1

index.js

 const ukParliament = require('./scrapers/ukParliament.js');
+const sottishParliament = require('./scrapers/scottishParliament.js');
 
+module.exports.scrapeScottishParliamentData = sottishParliament.scrapeScottishParliamentData;
 module.exports.scrapeUkParliamentData = ukParliament.scrapeUkParliamentData;
@@ -8,3 +10,4 @@
     let scrapers = [
-        this.scrapeUkParliamentData( path )
+        this.scrapeScottishParliamentData( path ),
+        this.scrapeUkParliamentData ( path )
     ];
@@ -11,0 +14,0 @@

package.json

 {
   "name": "parliament-data-scraper",
-  "version": "1.0.0",
+  "version": "1.1.1",
   "description": "A parliament member information scraper. Scrapes member details, parties and constituencies.",
@@ -27,4 +27,5 @@ "main": "index.js",
   "dependencies": {
-    "axios": "^0.19.2"
+    "axios": "^0.19.2",
+    "memory-cache": "^0.2.0"
   }
 }

README.md

 # Parliamentary Data Scraper
 
-Scrapes MPs details, Constituencies and Parties of the House of Commons (United Kingdom Parliament) into useful a json format.
+Scrapes MPs details, Constituencies and Parties into useful a json format.
 
+Currently covers:
+- House of Commons (United Kingdom Parliament)
+- Scottish Parliament
+
 ## Usage Example
@@ -13,3 +17,3 @@
 (async () => {
-    await scraper.scrapeAllParliamentData('./data/');
+    await scraper.scrapeAllParliamentData(__dirname + '/data/');
 })();
@@ -25,2 +29,8 @@ ```
 uk-parliament/commons/constituencies.json
+
+scottish-parliament/msps-details.json
+scottish-parliament/msps.json
+scottish-parliament/parties.json
+scottish-parliament/constituencies.json
+scottish-parliament/government-roles.json
 ```
@@ -30,11 +40,10 @@
 
-- Extend UK Parliament House of Commons data scraping
-- Add House of Lords data to UK Parliament scraper `scrapers/ukParliament.js`
-- Add Scottish Parliament scraper
-- Add Northern Ireland Assembly scraper
-- Add Welsh Assembly scraper
-- Add other scrapers
+- Add House of Lords
+- Add Northern Ireland Assembly
+- Add Welsh Assembly
+- Extend scope of data
+- Add others
 
 ## Contributions
 
-- Contributions welcome just follow the general structure of the project and open a pull request
+- Contributions welcome just follow the general structure of the project and open a pull request

scrapers/helpers.js

@@ -7,15 +7,36 @@ const fs = require('fs');
 
-module.exports.write = async function (data, file) {
+module.exports.sortByKey = function (array, key) {
+    return array.sort(function(a, b) {
+        var x = a[key]; var y = b[key];
+        return ((x < y) ? -1 : ((x > y) ? 1 : 0));
+    });
+}
 
-    try {
+module.exports.removeDuplicateObjects = function (myArr, prop) {
+    return myArr.filter((obj, pos, arr) => {
+        return arr.map(mapObj => mapObj[prop]).indexOf(obj[prop]) === pos;
+    });
+}
 
-        if ( Array.isArray(data) ) {
+module.exports.removeDuplicateStrings = function ( data ) {
 
-            data = [...new Set(data)];
-            data = data.filter(function (el) {
-                return el != null;
-            });
+    if ( Array.isArray(data) ) {
 
-        }
+        data = [...new Set(data)];
+        data = data.filter(function (el) {
+            return el != null;
+        });
 
+        return data;
+
+    }
+
+    return data;
+
+}
+
+module.exports.write = async function (data, file) {
+
+    try {
+
         let path = file.split('/');
@@ -22,0 +43,0 @@ path.pop();

scrapers/ukParliament.js

@@ -87,5 +87,5 @@ const helpers = require('./helpers.js');
         writes.push(helpers.write(mps, path + 'uk-parliament/commons/mps-details.json'));
-        writes.push(helpers.write(mpsNames, path + 'uk-parliament/commons/mps.json'));
-        writes.push(helpers.write(parties, path + 'uk-parliament/commons/parties.json'));
-        writes.push(helpers.write(constituencies, path + 'uk-parliament/commons/constituencies.json'));
+        writes.push(helpers.write(helpers.removeDuplicateStrings(mpsNames), path + 'uk-parliament/commons/mps.json'));
+        writes.push(helpers.write(helpers.removeDuplicateStrings(parties), path + 'uk-parliament/commons/parties.json'));
+        writes.push(helpers.write(helpers.removeDuplicateStrings(constituencies), path + 'uk-parliament/commons/constituencies.json'));
 
@@ -92,0 +92,0 @@ await Promise.all(writes);

test.js

 const scraper = require('./index.js');
 
 (async () => {
-    await scraper.scrapeAllParliamentData(__dirname + '/data/');
+    await scraper.scrapeAllParliamentData('./data/');
 })();

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

17200

increased by156.95%

Number of package files

7

increased by16.67%

Lines of code

347

increased by221.3%

Number of lines in readme file

47

increased by27.03%

Worsened metrics

Dependency count

2

increased by100%

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Addedmemory-cache@^0.2.0

• + Addedmemory-cache@0.2.0(transitive)