node-csfd-api - npm Package Compare versions

Comparing version 1.2.0 to 1.3.0-beta.1

fetchers/index.d.ts

@@ -1,2 +0,2 @@
-export declare const fetchUserRatings: (user: string | number) => Promise<string>;
+export declare const fetchUserRatings: (user: string | number, page?: number) => Promise<string>;
 export declare const fetchMovie: (movieId: number) => Promise<string>;

fetchers/index.js

@@ -18,4 +18,4 @@ "use strict";
 };
-const fetchUserRatings = async (user) => {
-    const url = vars_1.userRatingsUrl(user);
+const fetchUserRatings = async (user, page = 1) => {
+    const url = vars_1.userRatingsUrl(user, page);
     try {
@@ -22,0 +22,0 @@ const response = await cross_fetch_1.default(url, { headers });

helpers/user-ratings.helper.d.ts

@@ -13,1 +13,2 @@ import { HTMLElement } from 'node-html-parser';
 export declare const parseColor: (quality: Colors) => CSFDColorRating;
+export declare const sleep: (ms: number) => Promise<unknown>;

helpers/user-ratings.helper.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseColor = exports.getUrl = exports.getDate = exports.getColorRating = exports.getYear = exports.getTitle = exports.getType = exports.getUserRating = exports.getId = void 0;
+exports.sleep = exports.parseColor = exports.getUrl = exports.getDate = exports.getColorRating = exports.getYear = exports.getTitle = exports.getType = exports.getUserRating = exports.getId = void 0;
 const global_helper_1 = require("./global.helper");
@@ -58,1 +58,4 @@ const getId = (el) => {
 exports.parseColor = parseColor;
+// Sleep in loop
+const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
+exports.sleep = sleep;

index.d.ts

@@ -1,4 +0,3 @@
-import { CSFDFilmTypes } from './interfaces/global';
 import { CSFDMovie } from './interfaces/movie.interface';
-import { CSFDUserRatings } from './interfaces/user-ratings.interface';
+import { CSFDUserRatingConfig, CSFDUserRatings } from './interfaces/user-ratings.interface';
 import { MovieScraper } from './services/movie.service';
@@ -10,8 +9,5 @@ import { UserRatingsScraper } from './services/user-ratings.service';
     constructor(userRatingsService: UserRatingsScraper, movieService: MovieScraper);
-    userRatings(user: string | number, config?: {
-        includesOnly?: CSFDFilmTypes[];
-        excludes?: CSFDFilmTypes[];
-    }): Promise<CSFDUserRatings[]>;
+    userRatings(user: string | number, config?: CSFDUserRatingConfig): Promise<CSFDUserRatings[]>;
     movie(movie: number): Promise<CSFDMovie>;
 }
 export declare const csfd: Csfd;

interfaces/user-ratings.interface.d.ts

@@ -1,2 +0,2 @@
-import { CSFDScreening, CSFDStars } from './global';
+import { CSFDFilmTypes, CSFDScreening, CSFDStars } from './global';
 export interface CSFDUserRatings extends CSFDScreening {
@@ -6,2 +6,14 @@ userRating: CSFDStars;
 }
+export interface CSFDUserRatingConfig {
+    includesOnly?: CSFDFilmTypes[];
+    excludes?: CSFDFilmTypes[];
+    /**
+     * Fetch all ratings. (Warning: Use it wisely. Can be detected and banned. Consider using it together with `allPagesDelay` attribute)
+     */
+    allPages?: boolean;
+    /**
+     * Delay on each page request. In milliseconds
+     */
+    allPagesDelay?: number;
+}
 export declare type Colors = 'lightgrey' | 'blue' | 'red' | 'grey';

package.json

 {
   "name": "node-csfd-api",
-  "version": "1.2.0",
+  "version": "1.3.0-beta.1",
   "description": "Simple NPM library for scraping CSFD",
@@ -5,0 +5,0 @@ "main": "./index.js",

README.md

 [![npm version](https://badge.fury.io/js/node-csfd-api.svg)](https://badge.fury.io/js/node-csfd-api)
 [![Package License](https://img.shields.io/npm/l/node-csfd-api.svg)](https://www.npmjs.com/node-csfd-api)
-[![Build & Publish](https://github.com/bartholomej/node-csfd-api/workflows/Build%20&%20Publish/badge.svg)](https://github.com/bartholomej/node-csfd-api/actions)
+[![Build & Publish](https://github.com/bartholomej/node-csfd-api/workflows/Publish/badge.svg)](https://github.com/bartholomej/node-csfd-api/actions)
 [![codecov](https://codecov.io/gh/bartholomej/node-csfd-api/branch/master/graph/badge.svg?token=YQH9UoVrGP)](https://codecov.io/gh/bartholomej/node-csfd-api)
@@ -157,3 +157,5 @@
 
-- [ ] Search
+### Scraping more pages
+
+- [ ] Search ([branch](https://github.com/bartholomej/node-csfd-api/tree/search))
   - [ ] Movies
@@ -163,3 +165,6 @@ - [ ] TV Series
   - [ ] Creators
-- [ ] Creators...
+- [ ] Creators
+- [ ] User Ratings
+  - [x] Last ratings
+  - [ ] All pages ([branch](https://github.com/bartholomej/node-csfd-api/tree/user-rating-all))
 
@@ -166,0 +171,0 @@ ## Development

services/user-ratings.service.d.ts

@@ -1,10 +0,7 @@
-import { CSFDFilmTypes } from '../interfaces/global';
-import { CSFDUserRatings } from '../interfaces/user-ratings.interface';
+import { CSFDUserRatingConfig, CSFDUserRatings } from '../interfaces/user-ratings.interface';
 export declare class UserRatingsScraper {
     private films;
-    userRatings(user: string | number, config?: {
-        includesOnly?: CSFDFilmTypes[];
-        excludes?: CSFDFilmTypes[];
-    }): Promise<CSFDUserRatings[]>;
+    userRatings(user: string | number, config?: CSFDUserRatingConfig): Promise<CSFDUserRatings[]>;
+    private getPage;
     private buildUserRatings;
 }

services/user-ratings.service.js

@@ -12,6 +12,29 @@ "use strict";
     async userRatings(user, config) {
-        var _a, _b, _c, _d;
+        let allMovies = [];
         const response = await fetchers_1.fetchUserRatings(user);
         const items = node_html_parser_1.parse(response);
         const movies = items.querySelectorAll('.box-user-rating .table-container tbody tr');
+        // Get number of pages
+        const pagesNode = items.querySelector('.pagination');
+        const pages = +(pagesNode === null || pagesNode === void 0 ? void 0 : pagesNode.childNodes[pagesNode.childNodes.length - 4].rawText) || 1;
+        allMovies = this.getPage(config, movies);
+        if (config === null || config === void 0 ? void 0 : config.allPages) {
+            console.log('Fetching all pages', pages);
+            for (let i = 1; i <= pages; i++) {
+                console.log('Fetching page', i, 'out of', pages, '...');
+                const response = await fetchers_1.fetchUserRatings(user, i);
+                const items = node_html_parser_1.parse(response);
+                const movies = items.querySelectorAll('.box-user-rating .table-container tbody tr');
+                allMovies = [...allMovies, ...this.getPage(config, movies)];
+                // Sleep
+                if (config.allPagesDelay) {
+                    await user_ratings_helper_1.sleep(config.allPagesDelay);
+                }
+            }
+            return allMovies;
+        }
+        return allMovies;
+    }
+    getPage(config, movies) {
+        var _a, _b, _c, _d;
         if (config) {
@@ -18,0 +41,0 @@ if (((_a = config.includesOnly) === null || _a === void 0 ? void 0 : _a.length) && ((_b = config.excludes) === null || _b === void 0 ? void 0 : _b.length)) {

vars.d.ts

@@ -1,2 +0,2 @@
-export declare const userRatingsUrl: (user: string | number) => string;
+export declare const userRatingsUrl: (user: string | number, page?: number) => string;
 export declare const movieUrl: (movie: number) => string;

vars.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.movieUrl = exports.userRatingsUrl = void 0;
-const userRatingsUrl = (user) => `https://new.csfd.cz/uzivatel/${encodeURIComponent(user)}/hodnoceni/`;
+const userRatingsUrl = (user, page) => `https://new.csfd.cz/uzivatel/${encodeURIComponent(user)}/hodnoceni/${page ? '?page=' + page : ''}`;
 exports.userRatingsUrl = userRatingsUrl;
 const movieUrl = (movie) => `https://new.csfd.cz/film/${encodeURIComponent(movie)}`;
 exports.movieUrl = movieUrl;

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

No v1

Quality

Package is not semver >=1. This means it is not stable and does not support ^ ranges.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

32988

increased by6.07%

Lines of code

603

increased by5.6%

Number of lines in readme file

227

increased by2.25%

Worsened metrics

Number of low quality alerts

1

increased byInfinity%

No dependency changes