rss-url-finder
Advanced tools
Comparing version 0.0.5 to 0.0.6
"use strict"; | ||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { | ||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } | ||
return new (P || (P = Promise))(function (resolve, reject) { | ||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } | ||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } | ||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } | ||
step((generator = generator.apply(thisArg, _arguments || [])).next()); | ||
}); | ||
}; | ||
Object.defineProperty(exports, "__esModule", { value: true }); | ||
exports.getRssUrlsFromUrl = exports.getRssUrlsFromHtmlBody = void 0; | ||
const node_html_parser_1 = require("node-html-parser"); | ||
const types_1 = require("./types"); | ||
const utils_1 = require("./utils"); | ||
const node_html_parser_1 = require("node-html-parser"); | ||
function getRssUrlsFromHtmlBody(body) { | ||
@@ -21,11 +12,9 @@ const html = (0, node_html_parser_1.parse)(body); | ||
exports.getRssUrlsFromHtmlBody = getRssUrlsFromHtmlBody; | ||
function getRssUrlsFromUrl(url, options) { | ||
return __awaiter(this, void 0, void 0, function* () { | ||
let body = yield (0, utils_1.getHtmlBody)(url, options); | ||
const guessRssSources = yield (0, utils_1.guessRSSfromUrl)(url); | ||
return [ | ||
...getRssUrlsFromHtmlBody(body), | ||
...guessRssSources | ||
]; | ||
}); | ||
async function getRssUrlsFromUrl(url, options) { | ||
const body = await (0, utils_1.getHtmlBody)(url, options); | ||
const guessRssSources = await (0, utils_1.guessRSSfromUrl)(url); | ||
return [ | ||
...getRssUrlsFromHtmlBody(body), | ||
...guessRssSources | ||
]; | ||
} | ||
@@ -35,3 +24,3 @@ exports.getRssUrlsFromUrl = getRssUrlsFromUrl; | ||
const rssSources = []; | ||
for (const type of types_1.TYPES) { | ||
for (const type of types_1.RSS_MIME_TYPES) { | ||
const domain = (0, utils_1.getDomainName)(html); | ||
@@ -47,2 +36,3 @@ for (const search of html.querySelectorAll(`*[type="${type}"]`)) { | ||
catch (error) { | ||
console.log(error); | ||
url = new URL(href, domain).toString(); | ||
@@ -49,0 +39,0 @@ } |
@@ -5,2 +5,3 @@ export interface RssSource { | ||
} | ||
export declare const TYPES: string[]; | ||
export declare const RSS_MIME_TYPES: string[]; | ||
export declare const COMMON_URLS: string[]; |
"use strict"; | ||
Object.defineProperty(exports, "__esModule", { value: true }); | ||
exports.TYPES = void 0; | ||
exports.TYPES = [ | ||
exports.COMMON_URLS = exports.RSS_MIME_TYPES = void 0; | ||
exports.RSS_MIME_TYPES = [ | ||
'application/rss+xml', | ||
@@ -18,1 +18,4 @@ 'application/atom+xml', | ||
]; | ||
exports.COMMON_URLS = [ | ||
'/feed', '/rss', '/rss.xml', '/feed.xml' | ||
]; |
@@ -1,2 +0,2 @@ | ||
import { RssSource } from "./types"; | ||
import { RssSource } from './types'; | ||
import type { HTMLElement } from 'node-html-parser'; | ||
@@ -3,0 +3,0 @@ export declare function newRssSource({ name, url }: RssSource): RssSource; |
"use strict"; | ||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { | ||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } | ||
return new (P || (P = Promise))(function (resolve, reject) { | ||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } | ||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } | ||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } | ||
step((generator = generator.apply(thisArg, _arguments || [])).next()); | ||
}); | ||
}; | ||
Object.defineProperty(exports, "__esModule", { value: true }); | ||
@@ -20,2 +11,3 @@ exports.analyzeSitemap = exports.getDomainName = exports.guessRSSfromUrl = exports.generateGuesses = exports.getHtmlBody = exports.newRssSource = void 0; | ||
catch (error) { | ||
console.log(error); | ||
return undefined; | ||
@@ -26,12 +18,10 @@ } | ||
return { | ||
name: name === null || name === void 0 ? void 0 : name.trim(), | ||
url: url === null || url === void 0 ? void 0 : url.trim(), | ||
name: name?.trim(), | ||
url: url?.trim(), | ||
}; | ||
} | ||
exports.newRssSource = newRssSource; | ||
function getHtmlBody(url, options) { | ||
return __awaiter(this, void 0, void 0, function* () { | ||
const request = yield fetch(url, options); | ||
return yield request.text(); | ||
}); | ||
async function getHtmlBody(url, options) { | ||
const request = await fetch(url, options); | ||
return await request.text(); | ||
} | ||
@@ -41,5 +31,4 @@ exports.getHtmlBody = getHtmlBody; | ||
const urls = []; | ||
const commonUrls = ['/feed', '/rss', '/rss.xml', '/feed.xml']; | ||
const baseUrl = getBaseUrl(url); | ||
for (const url of commonUrls) { | ||
for (const url of types_1.COMMON_URLS) { | ||
urls.push(`${baseUrl}${url}`); | ||
@@ -50,51 +39,47 @@ } | ||
exports.generateGuesses = generateGuesses; | ||
function guessRSSfromUrl(url) { | ||
var _a; | ||
return __awaiter(this, void 0, void 0, function* () { | ||
const rssFeed = []; | ||
for (const guessUrl of generateGuesses(url)) { | ||
const request = yield fetch(guessUrl, { method: 'HEAD' }); | ||
if (request.status == 200) { | ||
const contentType = (_a = request.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.toLowerCase(); | ||
for (const type of types_1.TYPES) { | ||
if (contentType === null || contentType === void 0 ? void 0 : contentType.includes(type)) { | ||
rssFeed.push(newRssSource({ | ||
name: (new URL(guessUrl)).hostname, | ||
url: guessUrl | ||
})); | ||
} | ||
break; | ||
async function guessRSSfromUrl(url) { | ||
const rssFeed = []; | ||
for (const guessUrl of generateGuesses(url)) { | ||
const request = await fetch(guessUrl, { method: 'HEAD' }); | ||
if (request.status == 200) { | ||
const contentType = request.headers.get('content-type')?.toLowerCase(); | ||
for (const mimeType of types_1.RSS_MIME_TYPES) { | ||
if (contentType?.includes(mimeType)) { | ||
rssFeed.push(newRssSource({ | ||
name: (new URL(guessUrl)).hostname, | ||
url: guessUrl | ||
})); | ||
} | ||
break; | ||
} | ||
} | ||
return rssFeed; | ||
}); | ||
} | ||
return rssFeed; | ||
} | ||
exports.guessRSSfromUrl = guessRSSfromUrl; | ||
function getDomainName(html) { | ||
var _a, _b; | ||
let baseUrl = (_a = html.querySelector(`link[rel="canonical"]`)) === null || _a === void 0 ? void 0 : _a.attrs.href; | ||
if (baseUrl) | ||
let baseUrl = html.querySelector(`link[rel="canonical"]`)?.attrs.href; | ||
if (baseUrl) { | ||
return getBaseUrl(baseUrl); | ||
baseUrl = (_b = html.querySelector(`meta[property="og:url"]`)) === null || _b === void 0 ? void 0 : _b.attrs.content; | ||
if (baseUrl) | ||
} | ||
baseUrl = html.querySelector(`meta[property="og:url"]`)?.attrs.content; | ||
if (baseUrl) { | ||
return getBaseUrl(baseUrl); | ||
} | ||
return undefined; | ||
} | ||
exports.getDomainName = getDomainName; | ||
function analyzeSitemap(url) { | ||
return __awaiter(this, void 0, void 0, function* () { | ||
const baseUrl = getBaseUrl(url); | ||
const robotsUrl = (new URL('robots.txt', baseUrl)).toString(); | ||
const text = yield getHtmlBody(robotsUrl); | ||
const existsSitemap = /^Sitemap: (.+)$/s.exec(text); | ||
if (existsSitemap) { | ||
return { | ||
name: 'Sitemap', | ||
url: existsSitemap[1].trim(), | ||
}; | ||
} | ||
return undefined; | ||
}); | ||
async function analyzeSitemap(url) { | ||
const baseUrl = getBaseUrl(url); | ||
const robotsUrl = (new URL('robots.txt', baseUrl)).toString(); | ||
const text = await getHtmlBody(robotsUrl); | ||
const existsSitemap = /^Sitemap: (.+)$/s.exec(text); | ||
if (existsSitemap) { | ||
return { | ||
name: 'Sitemap', | ||
url: existsSitemap[1].trim(), | ||
}; | ||
} | ||
return undefined; | ||
} | ||
exports.analyzeSitemap = analyzeSitemap; |
{ | ||
"name": "rss-url-finder", | ||
"version": "0.0.5", | ||
"version": "0.0.6", | ||
"description": " A javascript/typescript library to search RSS feed via URL or HTML body ", | ||
@@ -8,6 +8,7 @@ "main": "dist/index.js", | ||
"scripts": { | ||
"start": "ts-node ./src/index.ts", | ||
"dev": "ts-node ./src/index.ts", | ||
"build": "tsc", | ||
"test": "jest", | ||
"publish": "npm run build && npm publish" | ||
"lint": "eslint .", | ||
"lint:fix": "eslint . --fix" | ||
}, | ||
@@ -36,9 +37,15 @@ "repository": { | ||
"devDependencies": { | ||
"@eslint/js": "^9.9.1", | ||
"@jest/globals": "^29.5.0", | ||
"@stylistic/eslint-plugin": "^2.6.4", | ||
"@types/eslint__js": "^8.42.3", | ||
"@types/jest": "^29.5.1", | ||
"@types/node": "^22.5.0", | ||
"eslint": "^9.9.1", | ||
"jest": "^29.5.0", | ||
"ts-jest": "^29.1.0", | ||
"ts-node": "^10.9.1", | ||
"typescript": "^5.0.4" | ||
}, | ||
"typescript": "^5.0.4", | ||
"typescript-eslint": "^8.3.0" | ||
}, | ||
"engines": { | ||
@@ -45,0 +52,0 @@ "node": ">= 18.0.0" |
@@ -14,3 +14,3 @@ { | ||
/* Language and Environment */ | ||
"target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ | ||
"target": "ESNext", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ | ||
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ | ||
@@ -104,3 +104,4 @@ // "jsx": "preserve", /* Specify what JSX code is generated. */ | ||
}, | ||
"include": ["./src/*.ts"] | ||
"include": ["./src/*.ts"], | ||
"exclude": ["./src/**/*.test.ts"] | ||
} |
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
22227
12
11
298
3