llmstxt - npm Package Compare versions

Comparing version 0.6.2 to 0.7.0

package.json

 {
-  "version": "0.6.2",
+  "version": "0.7.0",
   "name": "llmstxt",
@@ -4,0 +4,0 @@ "description": "convert `sitemap.xml` to `llms.txt`",

README.md

 # llmstxt
 
-> *generate `llms.txt`*–using your `sitemap.xml`.
+> *generate `llms.txt`*–using your `sitemap.xml`. A `llms.txt` file is a curated list of your website's pages in markdown format, perfect for training or fine-tuning language models with your content.
 
@@ -5,0 +5,0 @@ <p align="center"><br><img src="llmstxt.gif" width="500"><br></p>

src/cli/actions/gen.js

@@ -101,2 +101,37 @@ const { URL } = require('url')
 
+function cleanTitle(title) {
+  if (!title) return '';
+  // Remove leading '|' and whitespace
+  return title.replace(/^\|\s*/, '').trim();
+}
+
+/**
+ * Process URLs in batches with limited concurrency
+ * @param {Array} items - Array of items to process
+ * @param {Function} processor - Async function to process each item
+ * @param {number} concurrency - Maximum number of concurrent operations
+ * @returns {Array} - Results array
+ */
+async function processInBatches(items, processor, concurrency = 10) {
+  const results = [];
+  const totalItems = items.length;
+  let processedItems = 0;
+
+  // Process items in batches
+  for (let i = 0; i < totalItems; i += concurrency) {
+    const batch = items.slice(i, i + concurrency);
+    const batchPromises = batch.map(async (item, index) => {
+      const result = await processor(item, i + index);
+      processedItems++;
+      return result;
+    });
+
+    // Wait for the current batch to complete
+    const batchResults = await Promise.all(batchPromises);
+    results.push(...batchResults);
+  }
+
+  return results.filter(Boolean); // Remove null/undefined results
+}
+
 async function gen (sitemapUrl) {
@@ -117,2 +152,3 @@ const options = this.opts()
   const sections = {}
+  const concurrency = options.concurrency || 5
 
@@ -122,9 +158,10 @@ try {
     const sites = await sitemap.fetch(sitemapUrl)
+    
+    // Define the URL processor function
+    const processUrl = async (url, index) => {
+      spinner.text = `Processing [${index + 1}/${sites.sites.length}]: ${url}`
 
-    for (const url of sites.sites) {
-      spinner.text = url
-
       // path excluded - don't process it
       if (isExcluded(url)) {
-        continue
+        return null;
       }
@@ -134,3 +171,3 @@
       if (includePaths.length > 0 && !isIncluded(url)) {
-        continue
+        return null;
       }
@@ -141,3 +178,3 @@
       if (!html) {
-        continue
+        return null;
       }
@@ -148,3 +185,3 @@
       if (!title) {
-        continue
+        return null;
       }
@@ -154,3 +191,3 @@ for (command of replaceTitle) {
       }
-      title = title.trim()
+      title = cleanTitle(title)
 
@@ -160,14 +197,22 @@ // description
 
-      const line = {
-        title,
-        url,
-        description
-      }
+      // section
+      const section = parseSection(url)
+      
+      return { title, url, description, section };
+    };
 
+    // Process URLs concurrently
+    const results = await processInBatches(sites.sites, processUrl, concurrency);
+    
+    // Organize results into sections
+    for (const result of results) {
+      if (!result) continue;
+      
+      const { title, url, description, section } = result;
+      
       // set up section
-      const section = parseSection(url)
       sections[section] ||= []
 
       // add line
-      sections[section].push(line)
+      sections[section].push({ title, url, description });
     }
@@ -181,13 +226,17 @@ } catch (error) {
   // handle root
-  const root = sections.ROOT
+  const root = sections.ROOT || []
   delete sections.ROOT
 
-  output += `# ${options.title || root[0].title}`
+  // Default values if root doesn't exist
+  const defaultTitle = options.title || 'Documentation'
+  const defaultDescription = options.description || 'Generated documentation'
+
+  output += `# ${options.title || (root.length > 0 ? root[0].title : defaultTitle)}`
   output += '\n'
   output += '\n'
-  output += `> ${options.description || root[0].description}`
+  output += `> ${options.description || (root.length > 0 ? root[0].description : defaultDescription)}`
   output += '\n'
   output += '\n'
 
-  spinner.text = options.title || root[0].title
+  spinner.text = options.title || (root.length > 0 ? root[0].title : defaultTitle)
 
@@ -201,3 +250,3 @@ // handle sections
       output += '\n'
-      output += `- [${title}](${url}): ${description}`
+      output += `- [${title}](${url})${description ? ': ' + description : ''}`
 
@@ -204,0 +253,0 @@ spinner.text = title

src/cli/llmstxt.js

@@ -24,4 +24,5 @@ #!/usr/bin/env node
   .option('-d, --description <description>', 'set description (default: root page description)')
+  .option('-c, --concurrency <concurrency>', 'maximum number of concurrent connections (default: 5)', parseInt)
   .action(genAction)
 
 program.parse()

No alert changes

Improved metrics

Total package byte prevSize

15423

16.53%

Lines of code

228

21.28%

No dependency changes