llmstxt - npm Package Compare versions

Comparing version

+2

-1

package.json

		{
		"version": "0.3.0",
		"version": "0.4.0",
		"name": "llmstxt",
		@@ -34,2 +34,3 @@ "description": "convert `sitemap.xml` to `llms.txt`",
		"picomatch": "^4.0.2",
		"replace-in-file": "^8.2.0",
		"sitemapper": "^3.2.18",
		@@ -36,0 +37,0 @@ "turndown": "^7.2.0",

+9

-0

README.md

		@@ -92,3 +92,12 @@ # llmstxt
		</details>
		* <details><summary>`gen --replace-title s/pattern/replacement/` - Replace string(s) from title</summary><br>

		Use `--replace-title` to remove redundant text from your page titles. For example, dotenvx's titles all end with `\| dotenvx`. I want to replace those with empty string.

		```sh
		$ llmstxt gen https://dotenvx.com/sitemap.xml --replace-title 's/\\| dotenvx//'
		```

		</details>

+33

-3

src/cli/actions/gen.js

		@@ -16,3 +16,2 @@ const cheerio = require('cheerio')
		} catch (_error) {
		// console.error(`Error fetching HTML for ${url}:`, error.message)
		return null
		@@ -54,2 +53,25 @@ }

		function parseSubstitutionCommand(command) {
		const match = command.match(/^s\/(.?)\/(.?)\/([gimsuy]*)$/) // Capture optional flags

		if (match) {
		const pattern = match[1] // The pattern to search for
		const replacement = match[2] // The replacement string
		const flags = match[3] \|\| '' // Extract flags (e.g., 'g', 'i')
		return { pattern: new RegExp(pattern, flags), replacement }
		} else {
		throw new Error("Invalid substitution command format")
		}
		}

		function substituteTitle(title, command) {
		if (!command \|\| command.length < 1 \|\| !command.startsWith('s/')) {
		return title
		}

		const { pattern, replacement } = parseSubstitutionCommand(command)

		return title.replace(pattern, replacement)
		}

		async function gen (sitemapUrl) {
		@@ -64,2 +86,5 @@ const options = this.opts()

		// replaceTitle logic
		const replaceTitle = options.replaceTitle

		let lines = []
		@@ -86,8 +111,13 @@

		const title = await getTitle(html)

		// title
		let title = await getTitle(html)
		if (!title) {
		continue
		}
		for (command of replaceTitle) {
		title = substituteTitle(title, command)
		}
		title = title.trim()

		// description
		const description = await getDescription(html)
		@@ -94,0 +124,0 @@

+1

-0

src/cli/llmstxt.js

		@@ -21,4 +21,5 @@ #!/usr/bin/env node
		.option('-ip, --include-path <includePath...>', 'path(s) to include from generation (default: all)')
		.option('-rt, --replace-title <replaceTitle...>', 'replace string(s) from title (default: none)')
		.action(genAction)

		program.parse()

Improved metrics