llmstxt - npm Package Compare versions

Comparing version 0.4.1 to 0.5.0

package.json

 {
-  "version": "0.4.1",
+  "version": "0.5.0",
   "name": "llmstxt",
@@ -26,4 +26,3 @@ "description": "convert `sitemap.xml` to `llms.txt`",
     "standard": "standard",
-    "standard:fix": "standard --fix",
-    "release": "standard-version"
+    "standard:fix": "standard --fix"
   },
@@ -39,3 +38,6 @@ "funding": "https://dotenvx.com",
     "undici": "^6.21.0"
+  },
+  "devDependencies": {
+    "standard": "^17.1.2"
   }
 }

README.md

@@ -98,3 +98,21 @@ # llmstxt
   </details>
+* <details><summary>`gen --title 'Your Heading'` - set title</summary><br>
 
+  Set your website's heading 1 title.
+
+  ```sh
+  $ llmstxt gen https://dotenvx.com/sitemap.xml --title 'dotenvx'
+  ```
+
+  </details>
+* <details><summary>`gen --description 'Some description'` - set description</summary><br>
+
+  Set your website's description.
+
+  ```sh
+  $ llmstxt gen https://dotenvx.com/sitemap.xml --description 'This is a description' 
+  ```
+
+  </details>
+
 &nbsp;
@@ -109,3 +127,3 @@
 ```sh
-npx llmstxt gen https://dotenvx.com/sitemap.xml -ep "**/privacy**" -ep "**/terms**" -ep "**/blog/**" -ep "**/stats/**" -ep "**/support/**" -rt 's/\| dotenvx//' > llms.txt
+npx llmstxt gen https://dotenvx.com/sitemap.xml -ep "**/privacy**" -ep "**/terms**" -ep "**/blog/**" -ep "**/stats/**" -ep "**/support/**" -rt 's/\| dotenvx//' -t 'dotenvx' > llms.txt
 ```

src/cli/actions/gen.js

@@ -0,1 +1,2 @@
+const { URL } = require('url')
 const cheerio = require('cheerio')
@@ -5,8 +6,5 @@ const picomatch = require('picomatch')
 const Sitemapper = require('sitemapper')
-const Turndown = require('turndown')
-
 const sitemap = new Sitemapper()
-const turndown = new Turndown()
 
-async function fetchHtml(url) {
+async function fetchHtml (url) {
   try {
@@ -21,3 +19,3 @@ const { body } = await request(url)
 
-async function getTitle(html) {
+async function getTitle (html) {
   try {
@@ -31,3 +29,3 @@ const $ = cheerio.load(html)
 
-async function getDescription(html) {
+async function getDescription (html) {
   try {
@@ -55,3 +53,3 @@ const $ = cheerio.load(html)
 
-function parseSubstitutionCommand(command) {
+function parseSubstitutionCommand (command) {
   const match = command.match(/^s\/(.*?)\/(.*?)\/([gimsuy]*)$/) // Capture optional flags
@@ -65,7 +63,17 @@
   } else {
-    throw new Error("Invalid substitution command format")
+    throw new Error('Invalid substitution command format')
   }
 }
 
-function substituteTitle(title, command) {
+function parseSection(uri) {
+  try {
+    const url = new URL(uri)
+    const segments = url.pathname.split('/').filter(Boolean)
+    return segments[0] || 'ROOT'
+  } catch (_error) {
+    return 'ROOT'
+  }
+}
+
+function substituteTitle (title, command) {
   if (!command || command.length < 1 || !command.startsWith('s/')) {
@@ -80,2 +88,19 @@ return title
 
+function isRootUrl (uri) {
+  try {
+    const url = new URL(uri)
+    return url.pathname === '/'
+  } catch (_error) {
+    return false
+  }
+}
+
+function capitalizeString(str) {
+  if (!str || typeof str !== 'string') {
+    return ''
+  }
+
+  return str.charAt(0).toUpperCase() + str.slice(1).toLowerCase()
+}
+
 async function gen (sitemapUrl) {
@@ -93,3 +118,3 @@ const options = this.opts()
 
-  let lines = []
+  const sections = {}
 
@@ -110,2 +135,3 @@ try {
 
+      // html
       const html = await fetchHtml(url)
@@ -129,13 +155,49 @@ if (!html) {
 
-      lines.push(`- [${title}](${url}): ${description}`)
+      const line = {
+        title,
+        url,
+        description
+      }
+
+      // set up section
+      const section = parseSection(url)
+      sections[section] ||= []
+
+      // add line
+      sections[section].push(line)
+
     }
   } catch (error) {
-    console.error(`Error processing sitemap:`, error.message)
+    console.error('Error processing sitemap:', error.message)
   }
 
-  const md = lines.join('\n')
+  let output = ''
 
-  console.log(md)
+  // handle root
+  const root = sections.ROOT
+  delete sections.ROOT
+
+  output += `# ${options.title || root[0].title}`
+  output += '\n'
+  output += '\n'
+  output += `> ${options.description || root[0].description}`
+  output += '\n'
+  output += '\n'
+
+  // handle sections
+  for (const section in sections) {
+    output += `## ${capitalizeString(section)}`
+    output += '\n'
+    for (const line of sections[section]) {
+      const { title, url, description } = line
+      output += '\n'
+      output += `- [${title}](${url}): ${description}`
+    }
+    output += '\n'
+    output += '\n'
+  }
+
+  console.log(output)
 }
 
 module.exports = gen

src/cli/llmstxt.js

@@ -22,4 +22,6 @@ #!/usr/bin/env node
   .option('-rt, --replace-title <replaceTitle...>', 'replace string(s) from title (default: none)')
+  .option('-t, --title <title>', 'set title (default: root page title)')
+  .option('-d, --description <description>', 'set description (default: root page description)')
   .action(genAction)
 
 program.parse()

No alert changes

Improved metrics

Total package byte prevSize

12046

18.74%

Lines of code

181

43.65%

Number of lines in readme file

128

16.36%

Worsened metrics

Dev dependency count

1

Infinity%

No dependency changes