alis-request - npm Package Compare versions

Comparing version 1.0.1 to 1.0.2

122

index.js

		@@ -1,85 +0,75 @@
		const cheerio = require('cheerio');
		const request = require('request').defaults({ jar: true });
		import cheerio from 'cheerio';
		import request from 'request';
		import Stream from 'stream';

		function getPageUrl(body) {
		const $ = cheerio.load(body);
		const pageLink = $('a[id^=\'Agt\']');
		const pageUrl = `${$(pageLink).attr('href')}`;
		return pageUrl;
		export const ReadableStreamBooks = new Stream.Readable();
		ReadableStreamBooks._read = () => {};

		export function sendInitialQuery(query, callback) {
		if (!query.year) {
		return process.nextTick(callback, new Error('query.year is not provided'));
		}
		const j = request.jar();
		const INITIAL_URL = `http://${query.ip}/alis/EK/do_searh.php?radiodate=simple&valueINP=${query.year}&tema=1&tag=6`;
		request({ url: INITIAL_URL, jar: j }, (err, response, body) => {
		if (err) {
		callback(err);
		return;
		}
		callback(null, { page: body, jar: j });
		});
		}

		function run(urlsQueue, callback) {
		const books = [];
		function fn(q) {
		if (q[0] === 'undefined') {
		callback(null, books);
		export function getPage(url, jug, callback) {
		request({ url, jar: jug }, (err, response, body) => {
		if (err) {
		callback(err);
		return;
		}
		request(q[0], (err, res, body) => {
		if (err) {
		callback(err);
		}
		const $ = cheerio.load(body);
		const nextPageUrl = getPageUrl(body);
		const remainingQueue = q.slice(1);
		console.log(books.length);
		callback(null, body);
		});
		}

		$('.article').each(() => {
		books.push($(this).text());
		});
		export function getNextPageUrl($) {
		const pageLink = $('#Agt');
		const pageUrl = (`${$(pageLink).attr('href')}`);
		return pageUrl;
		}


		if (nextPageUrl === 'undefined') {
		callback(null, books);
		return;
		}

		if (q.length === 1) {
		remainingQueue.push(`http://86.57.174.45/alis/EK/${nextPageUrl}`);
		}
		fn(remainingQueue);
		});
		}
		fn(urlsQueue);
		export function getBooks($) {
		$('.article').each(function () {
		ReadableStreamBooks.push($(this).text());
		});
		}

		function getFirstTenPageUrls(html) {
		const $ = cheerio.load(html);
		export function getNumberedPageUrls(page, ip) {
		const $ = cheerio.load(page);
		const firstTenPageLinks = $('a[href^=\'do_other\']');
		const firstTenPageUrls = $(firstTenPageLinks).map((i, link) => `http://86.57.174.45/alis/EK/${$(link).attr('href')}`).toArray();
		const firstTenPageUrls = $(firstTenPageLinks).map((i, link) => `http://${ip}/alis/EK/${$(link).attr('href')}`).toArray();
		return firstTenPageUrls;
		}

		function sendInitialQuery(query, callback) {
		const url = `http://86.57.174.45/alis/EK/do_searh.php?radiodate=simple&valueINP=${query.year}&tema=1&tag=6`;
		request(url, (err, response, html) => {
		export function run(fn, q, ip, jar) {
		if (q.length === 0) {
		ReadableStreamBooks.push(null);
		return;
		}
		fn(q[0], jar, (err, page) => {
		if (err) {
		console.log(err);
		callback(err);
		return;
		}
		callback(null, html);
		});
		}

		function getAllBooksByYear(query, callback) {
		sendInitialQuery(query, (err, html) => {
		if (err) {
		callback(err);
		const $ = cheerio.load(page);
		getBooks($);
		const nextPageUrl = getNextPageUrl($);
		if (nextPageUrl === 'undefined') {
		ReadableStreamBooks.push(null);
		return;
		}
		const firstTenPageUrls = getFirstTenPageUrls(html);
		run(firstTenPageUrls, (error, books) => {
		if (error) {
		callback(error);
		}
		callback(null, books);
		});
		const remainingQueue = q.slice(1);
		if (q.length === 1) {
		remainingQueue.push(`http://${ip}/alis/EK/${nextPageUrl}`);
		}
		run(fn, remainingQueue, ip, jar);
		});
		}

		module.exports = {
		getAllBooksByYear,
		sendInitialQuery,
		getPageUrl,
		getFirstTenPageUrls,
		run,
		};

package.json

		{
		"name": "alis-request",
		"version": "1.0.1",
		"version": "1.0.2",
		"description": "Get books by year in alis library",
		@@ -10,13 +10,19 @@ "main": "index.js",
		"scripts": {
		"alis": "nodemon --exec babel-node -- ./index.js",
		"callback": "nodemon --exec babel-node -- ./callback.js"
		"start": "nodemon --exec babel-node -- ./consumer.js",
		"test": "mocha --compilers js:babel-core/register",
		"lint": "eslint index.js"
		},
		"dependencies": {
		"cheerio": "^0.22.0",
		"request": "^2.81.0"
		"cheerio": "0.22.0",
		"request": "2.81.0"
		},
		"devDependencies": {
		"babel-cli": "^6.24.0",
		"babel-preset-es2015": "^6.24.0"
		"babel-cli": "6.24.0",
		"babel-preset-es2015": "6.24.0",
		"chai": "3.5.0",
		"eslint": "3.19.0",
		"mocha": "3.2.0",
		"nock": "^9.0.11",
		"nodemon": "1.11.0"
		}
		}

README.md

		@@ -16,22 +16,45 @@ # alis-request

		### API
		## API

		* getAllBooksByYear - get all book by year
		* ReadableStreamBooks - read all book by year in stream
		* sendInitialQuery - make initial http query
		* getPageUrl - get page url
		* getFirstTenPageUrls - get ten page urls
		* getPage - get page body
		* getNumberedPageUrls - get ten page urls
		* run - recursive runner


		### Examples
		## Examples

		You can do what you want with books, for example save in database.
		Next example return all books 2016 year.
		```
		const AlisRequest = require('alis-request');
		Next example return all books 2015 year in stream.
		```js
		import Stream from 'stream';
		import { sendInitialQuery, getPage, getNumberedPageUrls, run, ReadableStreamBooks } from './index';

		AlisRequest.getAllBooksByYear({ year: 2016 }, (err, books) => {
		console.log(books.length);
		const WritableStreamBooks = new Stream.Writable();
		ReadableStreamBooks.pipe(WritableStreamBooks);

		const books = [];

		const initParams = {
		year: 2015,
		ip: '86.57.174.45',
		};

		sendInitialQuery(initParams, (err, res) => {
		if (err) {
		console.log(err);
		return;
		}
		const firstTenPageUrls = getNumberedPageUrls(res.page, initParams.ip);
		run(getPage, firstTenPageUrls, initParams.ip, res.jar);
		});

		WritableStreamBooks._write = (book, encoding, done) => {
		books.push(book);
		console.log(`STREAM: ${books.length}`);
		// ready to process the next chunk
		done();
		};

		```
		@@ -38,0 +61,0 @@

.babelrc