mp-scraper - npm Package Compare versions

Comparing version 1.0.7 to 1.0.8

bin/index.js

@@ -47,3 +47,3 @@ "use strict";
             outputFilePath = path.join(process.cwd(), `mps.${format}`);
-            utils_1.logger.log(`No output file specified, using [${outputFilePath}]`);
+            utils_1.logger.log(`[--outFile] is not set, using [${outputFilePath}]`);
         }
@@ -50,0 +50,0 @@ else {

bin/index.js.map

@@ -1,1 +0,1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,oCAAoC;;;;;;;;;;;;;;;;;;;;AAEpC,2CAA6B;AAC7B,8CAAsB;AACtB,wDAAgC;AAChC,mCAAyE;AAEzE,yCAA6C;AAC7C,2EAGwC;AAExC,MAAM,IAAI,GAAG,kBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAE7C,SAAe,GAAG,CAAC,IAAS;;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,cAAM,CAAC,GAAG,CAAC,0CAA0C,OAAO,GAAG,CAAC,CAAC;SAClE;QAEA,OAAe,CAAC,OAAO,GAAG,OAAO,CAAC;QAEnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;YAClB,cAAM,CAAC,GAAG,CAAC,2CAA2C,QAAQ,GAAG,CAAC,CAAC;SACpE;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,cAAM,CAAC,GAAG,CACR,yCAAyC,MAAM,uBAAuB,CACvE,CAAC;SACH;QAED,IAAI,cAAc,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,MAAM,EAAE,CAAC,CAAC;YAC3D,cAAM,CAAC,GAAG,CAAC,oCAAoC,cAAc,GAAG,CAAC,CAAC;SACnE;aAAM;YACL,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACxD,cAAM,CAAC,GAAG,CAAC,oBAAoB,cAAc,GAAG,CAAC,CAAC;SACnD;QAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEpB,MAAM,aAAa,GAAG,aAAG,CACvB,IAAI,cAAM,CAAC,KAAK,EAAE,2BAA2B,CAC9C,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,UAAU,GAAG,MAAM,uBAAe,EAAE,CAAC;QAC3C,aAAa,CAAC,OAAO,CACnB,IAAI,cAAM,CAAC,KAAK,EAAE,YAAY,UAAU,CAAC,MAAM,mBAAmB,CACnE,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,+CAAsB,CAC3C,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,yCAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,yCAAgB,CAAC,GAAG,EAChE,cAAc,EACd,oBAAY,CACb,CAAC;QAEF,MAAM,eAAe,GAAG,aAAG,CACzB,IAAI,cAAM,CAAC,KAAK,EAAE,2BAA2B,UAAU,CAAC,MAAM,OAAO,CACtE,CAAC,KAAK,EAAE,CAAC;QACV,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,gBAAQ,CAAC,UAAU,EAAE,QAAQ,EAAE,CAAM,KAAK,EAAC,EAAE;YACjD,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,mBAAW,CAAC,KAAK,CAAC,CAAC;gBACxC,UAAU,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC7B,UAAU,IAAI,CAAC,CAAC;gBAChB,eAAe,CAAC,IAAI,GAAG,IAAI,cAAM,CAAC,KAAK,EAAE,sBAAsB,UAAU,IACvE,UAAU,CAAC,MACb,OAAO,CAAC;aACT;YAAC,OAAO,GAAG,EAAE;gBACZ,cAAM,CAAC,IAAI,CAAC,kBAAkB,KAAK,aAAa,CAAC,CAAC;aACnD;QACH,CAAC,CAAA,CAAC,CAAC;QACH,eAAe,CAAC,OAAO,CAAC,IAAI,cAAM,CAAC,KAAK,EAAE,UAAU,UAAU,cAAc,CAAC,CAAC;QAE9E,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEpB,UAAU,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,cAAc,CAAC;IACxB,CAAC;CAAA;AAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAC7B,GAAG,CAAC,IAAI,CAAC;KACN,IAAI,CAAC,cAAc,CAAC,EAAE;IACrB,cAAM,CAAC,GAAG,CACR,mCAAmC,cAAc,SAAS,IAAI,CAAC,GAAG,EAAE;QAClE,SAAS,KAAK,CACjB,CAAC;AACJ,CAAC,CAAC;KACD,KAAK,CAAC,GAAG,CAAC,EAAE;IACX,cAAM,CAAC,KAAK,CAAC,qCAAqC,SAAS,MAAM,EAAE,GAAG,CAAC,CAAC;AAC1E,CAAC,CAAC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,oCAAoC;;;;;;;;;;;;;;;;;;;;AAEpC,2CAA6B;AAC7B,8CAAsB;AACtB,wDAAgC;AAChC,mCAAyE;AACzE,yCAA6C;AAC7C,2EAGwC;AAExC,MAAM,IAAI,GAAG,kBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAE7C,SAAe,GAAG,CAAC,IAAS;;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,cAAM,CAAC,GAAG,CAAC,0CAA0C,OAAO,GAAG,CAAC,CAAC;SAClE;QAEA,OAAe,CAAC,OAAO,GAAG,OAAO,CAAC;QAEnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;YAClB,cAAM,CAAC,GAAG,CAAC,2CAA2C,QAAQ,GAAG,CAAC,CAAC;SACpE;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,cAAM,CAAC,GAAG,CACR,yCAAyC,MAAM,uBAAuB,CACvE,CAAC;SACH;QAED,IAAI,cAAc,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,MAAM,EAAE,CAAC,CAAC;YAC3D,cAAM,CAAC,GAAG,CAAC,kCAAkC,cAAc,GAAG,CAAC,CAAC;SACjE;aAAM;YACL,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACxD,cAAM,CAAC,GAAG,CAAC,oBAAoB,cAAc,GAAG,CAAC,CAAC;SACnD;QAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEpB,MAAM,aAAa,GAAG,aAAG,CACvB,IAAI,cAAM,CAAC,KAAK,EAAE,2BAA2B,CAC9C,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,UAAU,GAAG,MAAM,uBAAe,EAAE,CAAC;QAC3C,aAAa,CAAC,OAAO,CACnB,IAAI,cAAM,CAAC,KAAK,EAAE,YAAY,UAAU,CAAC,MAAM,mBAAmB,CACnE,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,+CAAsB,CAC3C,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,yCAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,yCAAgB,CAAC,GAAG,EAChE,cAAc,EACd,oBAAY,CACb,CAAC;QAEF,MAAM,eAAe,GAAG,aAAG,CACzB,IAAI,cAAM,CAAC,KAAK,EAAE,2BAA2B,UAAU,CAAC,MAAM,OAAO,CACtE,CAAC,KAAK,EAAE,CAAC;QACV,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,gBAAQ,CAAC,UAAU,EAAE,QAAQ,EAAE,CAAM,KAAK,EAAC,EAAE;YACjD,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,mBAAW,CAAC,KAAK,CAAC,CAAC;gBACxC,UAAU,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC7B,UAAU,IAAI,CAAC,CAAC;gBAChB,eAAe,CAAC,IAAI,GAAG,IAAI,cAAM,CAAC,KAAK,EAAE,sBAAsB,UAAU,IACvE,UAAU,CAAC,MACb,OAAO,CAAC;aACT;YAAC,OAAO,GAAG,EAAE;gBACZ,cAAM,CAAC,IAAI,CAAC,kBAAkB,KAAK,aAAa,CAAC,CAAC;aACnD;QACH,CAAC,CAAA,CAAC,CAAC;QACH,eAAe,CAAC,OAAO,CAAC,IAAI,cAAM,CAAC,KAAK,EAAE,UAAU,UAAU,cAAc,CAAC,CAAC;QAE9E,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEpB,UAAU,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,cAAc,CAAC;IACxB,CAAC;CAAA;AAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAC7B,GAAG,CAAC,IAAI,CAAC;KACN,IAAI,CAAC,cAAc,CAAC,EAAE;IACrB,cAAM,CAAC,GAAG,CACR,mCAAmC,cAAc,SAAS,IAAI,CAAC,GAAG,EAAE;QAClE,SAAS,KAAK,CACjB,CAAC;AACJ,CAAC,CAAC;KACD,KAAK,CAAC,GAAG,CAAC,EAAE;IACX,cAAM,CAAC,KAAK,CAAC,qCAAqC,SAAS,MAAM,EAAE,GAAG,CAAC,CAAC;AAC1E,CAAC,CAAC,CAAC"}

package.json

 {
   "name": "mp-scraper",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "",
@@ -9,3 +9,3 @@ "main": "bin/index.js",
     "start:dev": "ts-node-dev -r tsconfig-paths/register src/index.ts",
-    "lint": "npx tslint -c tslint.json \"test/**/*.ts\" \"src/**/*.ts\"",
+    "lint": "./node_modules/.bin/tslint -c tslint.json \"test/**/*.ts\" \"src/**/*.ts\"",
     "test": "nyc --extension .ts npm run test:no-spec | tap-spec && nyc report --reporter=text-lcov > coverage.lcov",
@@ -27,6 +27,6 @@ "test:full": "TEST_MODE=full npm run test:no-spec",
   "devDependencies": {
+    "@types/blue-tape": "^0.1.32",
     "@types/jsdom": "^12.2.3",
     "@types/log-symbols": "^2.0.0",
     "@types/minimist": "^1.2.0",
-    "@types/blue-tape": "^0.1.32",
     "@types/node": "^11.11.3",
@@ -39,2 +39,3 @@ "blue-tape": "^1.0.0",
     "tsconfig-paths": "^3.8.0",
+    "tslint": "^5.14.0",
     "typescript": "^3.3.3333",
@@ -50,3 +51,7 @@ "typescript-tslint-plugin": "^0.3.1"
     "ora": "^3.2.0"
+  },
+  "preferGlobal": true,
+  "bin": {
+    "mp-scraper": "./bin/mpscraper"
   }
 }

README.md

@@ -1,2 +0,2 @@
-# mp-scraper
+# MP Scraper
 ![npm](https://img.shields.io/npm/v/mp-scraper.svg)
@@ -8,34 +8,27 @@ [![Build Status](https://travis-ci.org/ISNIT0/mp-scraper.svg?branch=master)](https://travis-ci.org/ISNIT0/mp-scraper)
 
-## Example
-```typescript
-import { queryMw } from 'mp-scraper';
+> 🇬🇧 A simple and easy node scraper for [https://beta.parliament.uk/mps](https://beta.parliament.uk/mps)
 
-queryMw(
-    {
-        apiUrl: 'https://en.wikipedia.org/w/api.php',
-    }, {
-        coordinates: {
-            coprop: {
-                type: true,
-                dim: false,
-            },
-        },
-        categories: {
-            cllimit: 40,
-        },
-        redirects: {
-            rdlimit: 'max',
-        },
-    }, ['London', 'Paris'],
-).then((out) => {
-    console.log('Got Data:', out);
-});
+## Usage
+```bash
+# Install
+npm i -g mp-scraper
+
+# Run
+mp-scraper
 ```
 
-## Building
+### Options
+- `--format` (default: `csv`)
+- `--outFile` (default: `./mps.[format]`)
+- `--parallel` (default: `5`)
+- `--verbose` (default: `false`)
+
+
+## Contributing
+### Building
 ```bash
-> ./build.sh
+> npm run build
 ```
 
-## Testing
+### Testing
 ```bash
@@ -42,0 +35,0 @@ > npm t

src/index.ts

@@ -7,3 +7,2 @@ ///<reference path="./types.d.ts" />
 import { mapLimit, getMpDetailUrls, getMpDetail, logger } from './utils';
-import { createWriteStream } from 'fs';
 import { csvHeaderRow } from './utils/const';
@@ -36,7 +35,7 @@ import {
   }
-
+  
   let outputFilePath;
   if (!argv.outFile) {
     outputFilePath = path.join(process.cwd(), `mps.${format}`);
-    logger.log(`No output file specified, using [${outputFilePath}]`);
+    logger.log(`[--outFile] is not set, using [${outputFilePath}]`);
   } else {
@@ -43,0 +42,0 @@ outputFilePath = path.join(process.cwd(), argv.outFile);

Fixed alerts

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Improved metrics

Total package byte prevSize

42873

11.05%

Number of package files

37

2.78%

Number of low supply chain risk alerts

3

-25%

Worsened metrics

Dev dependency count

14

7.69%

Lines of code

672

-0.15%

Number of lines in readme file

38

-15.56%

No dependency changes