@pebble-finances/crawler-itau - npm Package Compare versions

Comparing version 0.2.7 to 0.2.8

package.json

 {
 	"name": "@pebble-finances/crawler-itau",
-	"version": "0.2.7",
+	"version": "0.2.8",
 	"description": "",
@@ -21,3 +21,3 @@ "main": "src/index.js",
 	"license": "MIT",
-	"gitHead": "c7a744d20672676c0d3c39416cb35507d08c9057"
+	"gitHead": "f83851b7f9a519016b7287d6da5cfb82fd892a30"
 }

src/cli/cli.js

@@ -15,2 +15,4 @@ #!/usr/bin/env node
 	.usage('Usage: $0 --credentials --endpoint')
+	.describe('credentials', 'JSON of job credentials')
+	.describe('endpoint', 'Http endpoint that will receive job data')
 	.demandOption(['credentials'])
@@ -17,0 +19,0 @@ .argv;

src/crawler/collectCreditCards.js

 const { fromReal } = require('../util/currency');
-const { getCorpoFrame, getMenuFrame } = require('../util/frame');
+const { getContext } = require('../util/frame');
 const moment = require('moment');
-const log = require('../util/log');
 
@@ -55,21 +54,11 @@ const clickToListCreditCards = () => {
 
-const getCurrentContext = (page, context = 'corpo') => {
-	let frame = getCorpoFrame(page);
-
-	if (context === 'menu') {
-		frame = getMenuFrame(page);
-	}
-
-	return frame || page;
-}
-
 const goToCreditCardList = async (page) => {
-	await getCurrentContext(page, 'menu').waitForSelector('a[title="Link para a Home"]', { timeout: 90000 });
-	await getCurrentContext(page, 'menu').click('a[title="Link para a Home"]');
+	await getContext(page, 'menu').waitForSelector('a[title="Link para a Home"]', { timeout: 90000 });
+	await getContext(page, 'menu').click('a[title="Link para a Home"]');
 	await page.waitForSelector('#saldo', { timeout: 90000 });
 	// Home has ads sometimes
-	await getCurrentContext(page).evaluate(closeAddIfVisible);
+	await getContext(page).evaluate(closeAddIfVisible);
 	// Click to enter list of credit cards
-	await getCurrentContext(page).evaluate(clickToListCreditCards);
-	await getCurrentContext(page).waitForSelector('#cartoes-paginacao', { timeout: 90000 });
+	await getContext(page).evaluate(clickToListCreditCards);
+	await getContext(page).waitForSelector('#cartoes-paginacao', { timeout: 90000 });
 }
@@ -103,7 +92,7 @@
 const extractTransactions = async (page, card) => {
-	await getCurrentContext(page).evaluate(clickCreditCard, card.name);
+	await getContext(page).evaluate(clickCreditCard, card.name);
 	await page.waitFor(5000);
-	await getCurrentContext(page).waitForSelector('#divBoxFatura', { timeout: 90000 });
+	await getContext(page).waitForSelector('#divBoxFatura', { timeout: 90000 });
 
-	return await getCurrentContext(page).evaluate(
+	return await getContext(page).evaluate(
 		async () => {
@@ -167,3 +156,3 @@ const transactions = [];
 	// Extracts available credit cards and balance
-	const creditCards = await getCurrentContext(page).evaluate(extractCreditCards);
+	const creditCards = await getContext(page).evaluate(extractCreditCards);
 
@@ -170,0 +159,0 @@ for (let i = 0; i < creditCards.length; i++) {

src/crawler/collectCurrentAccountBalance.js

 const { fromReal } = require('../util/currency');
 
 module.exports = async (page) => {
-	try {
-		await page.waitForSelector('#saldo', { timeout: 90000 });
-		await page.waitFor(5000);
+	await page.waitForSelector('#saldo', { timeout: 90000 });
 
-		return fromReal(await page.$eval('#saldo', el => el.innerText));
-	}
-	catch (error) {
-		const errorFileName = `error${(new Date()).getTime()}.png`;
-		const errorPath = `/tmp/${errorFileName}`;
-
-		await page.screenshot({ path: errorPath });
-
-		console.log('[Error]', errorPath);
-
-		throw error;
-	}
+	return fromReal(await page.$eval('#saldo', el => el.innerText));
 }

src/crawler/collectCurrentAccountTransactions.js

 const { fromReal } = require('../util/currency');
-const { getCorpoFrame } = require('../util/frame');
+const { getContext } = require('../util/frame');
 const moment = require('moment');
 
-const clickSalvarEmOutrosFormatos = async () => {
-    const start = new Date();
+const TIMEOUT = 90000;
 
-    return new Promise(
-        (resolve, reject) => {
-            const clickLink = () => {
-                const link = document.querySelector('a[title*="outros formatos"]');
-
-                if (link) {
-                    resolve();
-
-                    link.click();
-                }
-                else {
-                    const now = new Date();
-
-                    if (now - start < 10000) {
-                        setTimeout(() => clickLink(), 1000);
-                    }
-                    else {
-                        reject('Unable to find link after 10s');
-                    }
-                }
-            }
-
-            clickLink();
-        }
-    );
-}
-
 const inputDia = () => {
@@ -102,17 +74,20 @@ const input = document.querySelector('input[name="Dia"]');
     // Go to the transactions page
-    await page.waitForSelector('a[title*="ver extrato"]', { timeout: 5000 });
+    await page.waitForSelector('a[title*="ver extrato"]', { timeout: TIMEOUT });
     await page.click('a[title*="ver extrato"]');
-    await page.waitFor(15000);
-    // This page contains frames. Find the frame we want.
-    const corpoFrame = getCorpoFrame(page);
+    await page.waitForNavigation({ waitUntil: 'load' });
+
     // Go the export page
-    await corpoFrame.evaluate(clickSalvarEmOutrosFormatos);
-    await corpoFrame.waitFor(5000);
+    await getContext(page).waitForSelector('a[title="Salvar em outros formatos"]', { timeout: TIMEOUT });
+    await Promise.all([
+        getContext(page).click('a[title="Salvar em outros formatos"]'),
+        new Promise(resolve => page.once('framenavigated', resolve))
+    ]);
+    await getContext(page).waitForSelector('input[type="radio"][value="TXT"]', { timeout: TIMEOUT });
     // Fill out the form to export data for the current month
-    await corpoFrame.evaluate(inputDia);
-    await corpoFrame.evaluate(inputMes);
-    await corpoFrame.evaluate(inputAno);
-    await corpoFrame.evaluate(selectFormat);
+    await getContext(page).evaluate(inputDia);
+    await getContext(page).evaluate(inputMes);
+    await getContext(page).evaluate(inputAno);
+    await getContext(page).evaluate(selectFormat);
     // Download the result
-    const text = await corpoFrame.evaluate(downloadTransactions);
+    const text = await getContext(page).evaluate(downloadTransactions);
     // Format result
@@ -119,0 +94,0 @@ return text.split('\n')

src/util/frame.js

@@ -25,5 +25,16 @@ const getCorpoFrame = (page) => {
 
+const getContext = (page, context = 'corpo') => {
+	let frame = getCorpoFrame(page);
+
+	if (context === 'menu') {
+		frame = getMenuFrame(page);
+	}
+
+	return frame || page;
+}
+
 module.exports = {
+	getContext,
 	getCorpoFrame,
 	getMenuFrame
 }

No alert changes

Improved metrics

Number of package files

15

increased by7.14%

Worsened metrics

Total package byte prevSize

20449

decreased by-2.86%

Lines of code

623

decreased by-3.71%

No dependency changes