@prerenderbuddy/cli - npm Package Compare versions

Comparing version 0.1.2 to 0.1.3

ROADMAP.md

+# Public roadmap
+
+This roadmap describes possible directions, not promised dates or release commitments.
+
+## Diagnostic improvements
+
+- additional transparent crawler profiles supported by public documentation;
+- clearer evidence and remediation fields for every finding;
+- more deterministic HTML and discovery-file fixtures;
+- configurable diagnostic thresholds;
+- stable JSON schema work toward `1.0`;
+- URL-list or sitemap-driven batch checks;
+- richer CI annotations and possible SARIF output.
+
+## Explicitly out of scope
+
+- browser rendering or JavaScript execution;
+- private Prerender Buddy API access;
+- hosted monitoring, baselines, incidents, or history;
+- managed crawler routing or DNS onboarding;
+- cache management;
+- proxy, queue, billing, or infrastructure deployment.
+
+The CLI will remain usable without an account, authentication, telemetry, or calls to Prerender Buddy production services.

bin/prerenderbuddy.js

 #!/usr/bin/env node
 
-import { runCli } from '../src/cli.js';
+import { executionErrorResult, runCli } from '../src/cli.js';
 
 runCli(process.argv.slice(2)).catch((error) => {
-  process.stderr.write(`Prerender Buddy check failed: ${error.message}\n`);
+  if (process.argv.includes('--json')) {
+    process.stdout.write(`${JSON.stringify(executionErrorResult(error), null, 2)}\n`);
+  } else {
+    process.stderr.write(`Prerender Buddy check failed: ${error.message}\n`);
+  }
   process.exitCode = 2;
 });

package.json

 {
   "name": "@prerenderbuddy/cli",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Open-source crawler-readability and discovery-file diagnostics for public websites.",
@@ -26,2 +26,3 @@ "homepage": "https://github.com/kopachlager/prerenderbuddy-cli#readme",
     "README.md",
+    "ROADMAP.md",
     "SECURITY.md"
@@ -31,5 +32,6 @@ ],
     "check": "node --check bin/prerenderbuddy.js && node --check src/*.js",
+    "demo:fixture": "node scripts/demo-fixture.js",
     "test": "node --test",
     "test:coverage": "node --test --experimental-test-coverage",
-    "pack:check": "npm pack --dry-run"
+    "pack:check": "node scripts/verify-package.js"
   },
@@ -41,8 +43,11 @@ "engines": {
     "crawler",
+    "technical-seo",
     "javascript-seo",
     "prerendering",
-    "robots.txt",
+    "googlebot",
+    "gptbot",
+    "robots-txt",
     "sitemap",
-    "llms.txt",
-    "cli"
+    "llms-txt",
+    "seo-cli"
   ],
@@ -49,0 +54,0 @@ "license": "Apache-2.0",

README.md

@@ -11,6 +11,44 @@ # Prerender Buddy CLI
 
-The CLI inspects returned HTML, compares browser-style and crawler-style responses, and validates common discovery files. It does not render JavaScript, change a website, require a Prerender Buddy account, or predict search rankings, indexing, AI citations, mentions, or traffic.
+The CLI inspects returned HTML, compares standard and crawler user-agent HTTP responses, and validates common discovery files. It does not render JavaScript, change a website, require a Prerender Buddy account, or predict search rankings, indexing, AI citations, mentions, or traffic.
 
-This is an early public release. Install it from npm or run it from a local checkout.
+Run it without installing:
 
+```bash
+npx @prerenderbuddy/cli check https://example.com --user-agent googlebot
+```
+
+Or install the command globally:
+
+```bash
+npm install --global @prerenderbuddy/cli
+prerenderbuddy check https://example.com
+```
+
+Example output from the included loading-placeholder fixture:
+
+```text
+Prerender Buddy · crawler HTML check · CRITICAL
+URL                https://example.com/app
+Crawler profile    Googlebot
+HTTP               200
+Final URL          https://example.com/app
+Title              Loading application
+Description        Application loading screen.
+H1                 Loading application
+Readable text      41 characters / 5 words
+App-shell signs    root div, bundled assets, module scripts
+
+Issues:
+- CRITICAL [app_shell]: Returned HTML has limited visible content and multiple JavaScript app-shell signals.
+  Why: Crawlers that do not execute JavaScript may receive only the application shell.
+  Evidence: {"readableCharacters":41,"scriptCount":1,"signals":["loading-only visible text","module or bundled application script","root div detected","bundled assets detected","module scripts detected"]}
+  Next: Inspect the raw response and test whether important page content is present before JavaScript executes.
+
+This checks returned HTML only. It does not predict rankings, indexing, citations, mentions, or traffic.
+```
+
+Reproduce that output from a local checkout with `npm run demo:fixture`. The demo injects a static fixture into the normal check and formatting functions; it does not weaken public-URL safety or start a local URL-fetching service.
+
+The finding is a documented heuristic, not proof that a crawler failed. The CLI does not run Chromium, execute page JavaScript, or produce rendered HTML.
+
 ## Requirements
@@ -21,6 +59,5 @@
 
-Run without installing:
+Other commands:
 
 ```bash
-npx @prerenderbuddy/cli check https://example.com
 npx @prerenderbuddy/cli compare https://example.com --user-agent gptbot
@@ -30,10 +67,2 @@ npx @prerenderbuddy/cli files https://example.com
 
-Local checkout usage:
-
-```bash
-node ./bin/prerenderbuddy.js check https://example.com
-node ./bin/prerenderbuddy.js compare https://example.com --user-agent gptbot
-node ./bin/prerenderbuddy.js files https://example.com
-```
-
 ## Commands
@@ -63,6 +92,14 @@
 
-Compares a browser-style response with the selected crawler response. It flags status, metadata, heading, and material text differences. Different output can be legitimate; the result is evidence to review, not an accusation of cloaking.
+Compares a browser-style user-agent HTTP response with the selected crawler user-agent HTTP response. It reports status, metadata, heading, and material text-volume differences separately. Different output can be legitimate; the result is evidence to review, not an accusation of cloaking.
 
-This is not a raw-versus-browser-rendered comparison. The open-source v0.1 package deliberately has no browser engine or connection to Prerender Buddy’s private rendering infrastructure.
+Both sides are ordinary HTTP responses. Neither side executes JavaScript. This is not a raw-versus-browser-rendered comparison, and the package has no browser engine or connection to Prerender Buddy’s private rendering infrastructure.
 
+The default text-ratio tolerance is 30% in either direction. Adjust it for a known-variable site:
+
+```bash
+prerenderbuddy compare https://example.com --text-ratio-threshold 0.20
+```
+
+The comparison normalizes HTML into whitespace-collapsed visible text and reports the exact lengths and metadata values that changed. It does not perform semantic AI comparison or automatically remove cookie notices, timestamps, rotating banners, experiments, personalization, regional content, anti-bot pages, or temporary CDN responses. Review those sources of variation before treating a warning as a regression.
+
 ### Validate discovery files
@@ -108,2 +145,25 @@
 
+### Programmatic use
+
+The same diagnostics are exported as dependency-free ESM functions:
+
+```js
+import {
+  analyzeHtml,
+  checkDiscoveryFiles,
+  checkUrl,
+  compareUrl,
+} from '@prerenderbuddy/cli';
+
+const page = await checkUrl('https://example.com', { userAgent: 'googlebot' });
+const comparison = await compareUrl('https://example.com', {
+  userAgent: 'gptbot',
+  textRatioThreshold: 0.2,
+});
+const files = await checkDiscoveryFiles('https://example.com');
+const localAnalysis = analyzeHtml('<main><h1>Example</h1></main>');
+```
+
+Network functions retain the same public-URL safety, redirect, timeout, and response-size controls as the CLI.
+
 ### GitHub Actions
@@ -118,3 +178,3 @@
 - checks crawler-readable HTML as Googlebot;
-- compares browser-style and GPTBot responses;
+- compares browser-style user-agent and GPTBot HTTP responses;
 - validates `robots.txt`, `sitemap.xml`, and `llms.txt`;
@@ -140,7 +200,24 @@ - fails only on critical findings by default;
 
-## When a managed service is not needed
+## CLI and hosted service
 
+| Capability | Open-source CLI | Hosted Prerender Buddy |
+| --- | --- | --- |
+| One-time public URL diagnostics | Yes | Yes |
+| Local execution and CI | Yes | No |
+| Returned HTML inspection | Yes | Yes |
+| JavaScript execution | No | Yes, for managed crawler-ready rendering |
+| Scheduled monitoring | No | Yes |
+| Baselines, history, and incidents | No | Yes |
+| Managed crawler routing | No | Yes |
+| Cache operations | No | Yes |
+| DNS or proxy onboarding | No | Yes |
+| Account required | No | Yes |
+
+The CLI is independently useful for diagnostics. The hosted service operates rendering, routing, monitoring, and cache workflows when testing shows that a production deployment needs them.
+
+### When a managed service is not needed
+
 If important production routes already return complete, consistent HTML to the crawlers you care about, an additional rendering layer may not be needed. Continue testing after framework, hosting, domain, or deployment changes.
 
-## When Prerender Buddy may help
+### When Prerender Buddy may help
 
@@ -151,2 +228,8 @@ If production tests find missing, partial, crawler-dependent, or unreliable HTML, Prerender Buddy can provide managed crawler-ready rendering. Its hosted service also provides scheduled monitoring, baselines, incidents, history, cache operations, DNS/proxy onboarding, crawler routing, and support.
 
+## Fixtures and heuristic limits
+
+Deterministic fixtures live in [`test/fixtures`](./test/fixtures). They cover healthy HTML, thin application shells, minimal static pages, canvas applications, loading placeholders, hidden script data, cookie banners, crawler-blocked responses, malformed metadata, and discovery-file errors.
+
+Application-shell detection uses observable inputs: readable character count, empty `root` or `app` mount points, loading-only text, module or bundled scripts, and framework markers. It does not identify a framework failure, simulate verified crawler traffic, or prove that a genuine crawler received the same response.
+
 ## Development
@@ -167,4 +250,12 @@
 
+## Next steps
+
+- Run the [browser-based crawler checker](https://prerenderbuddy.com/tools/bot-view-checker).
+- Read the [technical documentation](https://prerenderbuddy.com/docs).
+- Review the [public roadmap](./ROADMAP.md).
+- Report reproducible CLI problems in [GitHub Issues](https://github.com/kopachlager/prerenderbuddy-cli/issues).
+- Use the [hosted Prerender Buddy service](https://prerenderbuddy.com) when diagnostics show that managed rendering or monitoring is needed.
+
 ## License
 
 Apache License 2.0. See [LICENSE](./LICENSE).

src/check.js

@@ -12,2 +12,5 @@ import { fetchPublicText } from './fetch-public.js';
     timeoutMs: options.timeoutMs,
+    fetchFn: options.fetchFn,
+    assertUrlFn: options.assertUrlFn,
+    maxChars: options.maxChars,
   });
@@ -27,2 +30,4 @@ const html = analyzeHtml(response.text);
       contentType: response.contentType,
+      truncated: response.truncated,
+      maxChars: response.maxChars,
     },
@@ -29,0 +34,0 @@ html,

src/cli.js

@@ -22,2 +22,5 @@ import { readFileSync } from 'node:fs';
   --timeout <milliseconds>  request timeout from 1000 to 60000 (default: 15000)
+  --text-ratio-threshold <number>
+                            compare command only; text-volume tolerance from 0.01 to 0.99
+                            (default: 0.30)
   --json                    print machine-readable JSON
@@ -31,4 +34,19 @@ --fail-on <level>         warning or critical
 
+function optionValue(args, index, option) {
+  const value = args[index + 1];
+  if (value === undefined || value.startsWith('--')) {
+    throw new Error(`${option} requires a value.`);
+  }
+  return value;
+}
+
 function parseArgs(args) {
-  const options = { userAgent: 'googlebot', timeoutMs: 15_000, json: false, failOn: null };
+  const options = {
+    userAgent: 'googlebot',
+    timeoutMs: 15_000,
+    textRatioThreshold: 0.3,
+    textRatioThresholdProvided: false,
+    json: false,
+    failOn: null,
+  };
   const positional = [];
@@ -41,5 +59,8 @@
     else if (value === '--version' || value === '-v') options.version = true;
-    else if (value === '--user-agent') options.userAgent = args[++index];
-    else if (value === '--timeout') options.timeoutMs = Number(args[++index]);
-    else if (value === '--fail-on') options.failOn = args[++index];
+    else if (value === '--user-agent') options.userAgent = optionValue(args, index++, value);
+    else if (value === '--timeout') options.timeoutMs = Number(optionValue(args, index++, value));
+    else if (value === '--text-ratio-threshold') {
+      options.textRatioThreshold = Number(optionValue(args, index++, value));
+      options.textRatioThresholdProvided = true;
+    } else if (value === '--fail-on') options.failOn = optionValue(args, index++, value);
     else if (value.startsWith('-')) throw new Error(`Unknown option "${value}".`);
@@ -52,2 +73,7 @@ else positional.push(value);
   }
+  if (!Number.isFinite(options.textRatioThreshold)
+    || options.textRatioThreshold < 0.01
+    || options.textRatioThreshold > 0.99) {
+    throw new Error('--text-ratio-threshold must be a number between 0.01 and 0.99.');
+  }
   if (options.failOn && !['warning', 'critical'].includes(options.failOn)) {
@@ -69,10 +95,22 @@ throw new Error('--fail-on must be warning or critical.');
 
-export async function runCli(args) {
+export async function runCli(args, runtime = {}) {
+  return runCliWithRuntime(args, runtime);
+}
+
+export async function runCliWithRuntime(args, runtime = {}) {
   const { positional, options } = parseArgs(args);
+  const write = runtime.write || ((value) => process.stdout.write(value));
+  const setExitCode = runtime.setExitCode || ((value) => { process.exitCode = value; });
+  const handlers = {
+    check: checkUrl,
+    compare: compareUrl,
+    files: checkDiscoveryFiles,
+    ...runtime.handlers,
+  };
   if (options.help || (!positional.length && !options.version)) {
-    process.stdout.write(`${HELP}\n`);
+    write(`${HELP}\n`);
     return;
   }
   if (options.version) {
-    process.stdout.write(`${VERSION}\n`);
+    write(`${VERSION}\n`);
     return;
@@ -87,14 +125,33 @@ }
   if (positional.length > 2) throw new Error('Only one URL can be checked at a time in v0.1.');
+  if (options.textRatioThresholdProvided && command !== 'compare') {
+    throw new Error('--text-ratio-threshold is supported by the compare command only.');
+  }
 
-  const runOptions = { userAgent: options.userAgent, timeoutMs: options.timeoutMs };
-  const result = command === 'check'
-    ? await checkUrl(url, runOptions)
-    : command === 'compare'
-      ? await compareUrl(url, runOptions)
-      : await checkDiscoveryFiles(url, runOptions);
+  const runOptions = {
+    userAgent: options.userAgent,
+    timeoutMs: options.timeoutMs,
+    textRatioThreshold: options.textRatioThreshold,
+  };
+  const result = await handlers[command](url, runOptions);
 
-  process.stdout.write(`${options.json ? JSON.stringify(result, null, 2) : formatHuman(result)}\n`);
-  if (shouldFail(result.summary, options.failOn)) process.exitCode = 1;
+  write(`${options.json ? JSON.stringify(result, null, 2) : formatHuman(result)}\n`);
+  if (shouldFail(result.summary, options.failOn)) setExitCode(1);
 }
 
+export function executionErrorResult(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  const code = /timed out/i.test(message)
+    ? 'timeout'
+    : /private|blocked network|local and private|credentials|only http and https/i.test(message)
+      ? 'unsafe_target'
+      : /unknown|requires|must be|only one URL|compare command only|public URL is required|invalid url/i.test(message)
+        ? 'invalid_input'
+        : 'request_failed';
+  return {
+    command: null,
+    summary: 'error',
+    error: { code, message },
+  };
+}
+
 export { HELP, parseArgs, shouldFail };

src/compare.js

@@ -6,10 +6,28 @@ import { fetchPublicText } from './fetch-public.js';
 
-function contentDelta(browser, crawler) {
-  const baseline = Math.max(browser.textLength, 1);
+export function contentDelta(standard, crawler, textRatioThreshold = 0.3) {
+  const baseline = Math.max(standard.textLength, 1);
+  const minimum = Number((1 - textRatioThreshold).toFixed(2));
+  const maximum = Number((1 + textRatioThreshold).toFixed(2));
   return {
-    textLength: crawler.textLength - browser.textLength,
+    textLength: crawler.textLength - standard.textLength,
     textRatio: Number((crawler.textLength / baseline).toFixed(2)),
-    titleChanged: crawler.title !== browser.title,
-    descriptionChanged: crawler.description !== browser.description,
-    h1Changed: JSON.stringify(crawler.headings.h1) !== JSON.stringify(browser.headings.h1),
+    textRatioThreshold,
+    acceptedTextRatio: { minimum, maximum },
+    titleChanged: crawler.title !== standard.title,
+    descriptionChanged: crawler.description !== standard.description,
+    h1Changed: JSON.stringify(crawler.headings.h1) !== JSON.stringify(standard.headings.h1),
+    values: {
+      standard: {
+        textLength: standard.textLength,
+        title: standard.title,
+        description: standard.description,
+        h1: standard.headings.h1,
+      },
+      crawler: {
+        textLength: crawler.textLength,
+        title: crawler.title,
+        description: crawler.description,
+        h1: crawler.headings.h1,
+      },
+    },
   };
@@ -22,14 +40,23 @@ }
   const crawlerProfile = getUserAgentProfile(options.userAgent);
+  const textRatioThreshold = options.textRatioThreshold ?? 0.3;
+  const fetchOptions = {
+    timeoutMs: options.timeoutMs,
+    fetchFn: options.fetchFn,
+    assertUrlFn: options.assertUrlFn,
+    maxChars: options.maxChars,
+  };
 
   const [browserResponse, crawlerResponse] = await Promise.all([
-    fetchPublicText(url, { userAgent: browserProfile.value, timeoutMs: options.timeoutMs }),
-    fetchPublicText(url, { userAgent: crawlerProfile.value, timeoutMs: options.timeoutMs }),
+    fetchPublicText(url, { ...fetchOptions, userAgent: browserProfile.value }),
+    fetchPublicText(url, { ...fetchOptions, userAgent: crawlerProfile.value }),
   ]);
   const browser = analyzeHtml(browserResponse.text);
   const crawler = analyzeHtml(crawlerResponse.text);
-  const difference = contentDelta(browser, crawler);
+  const difference = contentDelta(browser, crawler, textRatioThreshold);
+  const textVolumeDiffers = difference.textRatio < difference.acceptedTextRatio.minimum
+    || difference.textRatio > difference.acceptedTextRatio.maximum;
   const materiallyDifferent = browserResponse.statusCode !== crawlerResponse.statusCode
-    || difference.textRatio < 0.7
-    || difference.textRatio > 1.3
+    || textVolumeDiffers
     || difference.titleChanged
+    || difference.descriptionChanged
     || difference.h1Changed;
@@ -42,12 +69,41 @@
       code: 'status_differs',
-      message: `Browser and ${crawlerProfile.label} responses return different status codes.`,
+      message: `Standard and ${crawlerProfile.label} HTTP responses return different status codes.`,
+      why: 'Different status codes can change whether the page is accessible to the selected crawler.',
+      evidence: {
+        standardStatusCode: browserResponse.statusCode,
+        crawlerStatusCode: crawlerResponse.statusCode,
+      },
+      nextStep: 'Confirm whether crawler-specific status handling is intentional and stable.',
     });
   }
-  if (materiallyDifferent) {
+  if (textVolumeDiffers) {
     issues.push({
       severity: 'warning',
-      code: 'crawler_response_differs',
-      message: `The ${crawlerProfile.label} response differs materially from the browser-style response; review whether the difference is intended.`,
+      code: 'text_volume_differs',
+      message: `The ${crawlerProfile.label} response has a materially different readable-text volume.`,
+      why: 'A large text-volume difference can indicate missing content, an interstitial, personalization, or intentional crawler handling.',
+      evidence: {
+        standardCharacters: browser.textLength,
+        crawlerCharacters: crawler.textLength,
+        textRatio: difference.textRatio,
+        acceptedTextRatio: difference.acceptedTextRatio,
+      },
+      nextStep: 'Compare the returned text and rule out banners, regional content, experiments, authentication, or temporary edge responses.',
     });
   }
+  for (const [changed, code, label, standardValue, crawlerValue] of [
+    [difference.titleChanged, 'title_differs', 'title', browser.title, crawler.title],
+    [difference.descriptionChanged, 'description_differs', 'meta description', browser.description, crawler.description],
+    [difference.h1Changed, 'h1_differs', 'H1 headings', browser.headings.h1, crawler.headings.h1],
+  ]) {
+    if (!changed) continue;
+    issues.push({
+      severity: 'warning',
+      code,
+      message: `The ${crawlerProfile.label} ${label} differs from the standard HTTP response.`,
+      why: `Different ${label} values may be intentional, personalized, or caused by crawler-specific response handling.`,
+      evidence: { standard: standardValue, crawler: crawlerValue },
+      nextStep: `Review both ${label} values and confirm that the difference is expected.`,
+    });
+  }
   if (crawler.looksLikeAppShell) {
@@ -57,5 +113,29 @@ issues.push({
       code: 'crawler_app_shell',
-      message: `The ${crawlerProfile.label} response appears to contain a thin JavaScript app shell.`,
+      message: `The ${crawlerProfile.label} response has limited visible content and multiple JavaScript app-shell signals.`,
+      why: 'The selected crawler may receive an application shell without the page’s primary content.',
+      evidence: {
+        readableCharacters: crawler.textLength,
+        scriptCount: crawler.scriptCount,
+        signals: crawler.appShellEvidence,
+      },
+      nextStep: 'Inspect the crawler HTTP response and verify whether primary content is present without JavaScript execution.',
     });
   }
+  if (materiallyDifferent) {
+    issues.push({
+      severity: 'warning',
+      code: 'crawler_response_differs',
+      message: `The ${crawlerProfile.label} response differs materially from the standard HTTP response.`,
+      why: 'This compatibility finding preserves the original pre-1.0 comparison code while specific findings explain each observed difference.',
+      evidence: {
+        statusChanged: browserResponse.statusCode !== crawlerResponse.statusCode,
+        textVolumeChanged: textVolumeDiffers,
+        titleChanged: difference.titleChanged,
+        descriptionChanged: difference.descriptionChanged,
+        h1Changed: difference.h1Changed,
+      },
+      nextStep: 'Review the specific comparison findings and confirm whether each difference is expected.',
+      compatibilityAlias: true,
+    });
+  }
 
@@ -67,2 +147,3 @@ return {
     crawlerProfile: { name: crawlerProfile.name, label: crawlerProfile.label },
+    comparisonMode: 'http-user-agent-responses',
     browser: {
@@ -85,2 +166,3 @@ response: {
     difference,
+    materiallyDifferent,
     issues,
@@ -92,4 +174,4 @@ summary: issues.some((issue) => issue.severity === 'critical')
         : 'pass',
-    note: 'Different output is evidence to review, not proof of cloaking or a ranking problem.',
+    note: 'Both sides are HTTP responses. Neither executes JavaScript. Differences are evidence to review, not proof of cloaking or a ranking problem.',
   };
 }

src/discovery.js

@@ -9,3 +9,3 @@ import { fetchPublicText } from './fetch-public.js';
 
-function parseRobots(text) {
+export function parseRobots(text) {
   const sitemapLines = text
@@ -25,3 +25,3 @@ .split(/\r?\n/)
 
-function parseSitemap(text, expectedHostname) {
+export function parseSitemap(text, expectedHostname) {
   const locations = [...text.matchAll(/<loc\b[^>]*>([\s\S]*?)<\/loc>/gi)]
@@ -78,2 +78,4 @@ .map((match) => match[1].trim());
       maxChars: 1_000_000,
+      fetchFn: options.fetchFn,
+      assertUrlFn: options.assertUrlFn,
     });
@@ -90,2 +92,5 @@ return [name, response];
         message: `${name} returned HTTP ${response.statusCode}.`,
+        why: `${name} could not be read successfully at its conventional public URL.`,
+        evidence: { statusCode: response.statusCode, finalUrl: response.finalUrl },
+        nextStep: `Confirm whether ${name} should exist and that its public URL returns the intended file.`,
       });
@@ -96,2 +101,12 @@ }
       const details = parseRobots(response.text);
+      if (response.ok && response.contentType && !/(?:text\/plain|text\/robots|application\/octet-stream)/i.test(response.contentType)) {
+        issues.push({
+          severity: 'warning',
+          code: 'unexpected_content_type',
+          message: `robots.txt returned ${response.contentType}.`,
+          why: 'An HTML fallback or unexpected media type can hide a missing robots.txt file.',
+          evidence: { contentType: response.contentType },
+          nextStep: 'Return robots.txt as plain text and verify that the route is not serving an HTML fallback.',
+        });
+      }
       if (details.invalidSitemaps.length) {
@@ -102,2 +117,5 @@ issues.push({
           message: 'One or more Sitemap directives are not valid absolute HTTP(S) URLs.',
+          why: 'Crawler sitemap directives should resolve without relying on a document base URL.',
+          evidence: { invalidValues: details.invalidSitemaps },
+          nextStep: 'Replace relative or malformed Sitemap values with absolute HTTP(S) URLs.',
         });
@@ -110,2 +128,12 @@ }
       const details = parseSitemap(response.text, hostname);
+      if (response.ok && response.contentType && !/(?:application|text)\/(?:[a-z0-9.+-]*\+)?xml/i.test(response.contentType)) {
+        issues.push({
+          severity: 'warning',
+          code: 'unexpected_content_type',
+          message: `sitemap.xml returned ${response.contentType}.`,
+          why: 'An HTML fallback or unexpected media type can hide a missing XML sitemap.',
+          evidence: { contentType: response.contentType },
+          nextStep: 'Return sitemap.xml with an XML content type and verify that the route is not serving an HTML fallback.',
+        });
+      }
       if (response.ok && details.locationCount === 0) {
@@ -116,2 +144,5 @@ issues.push({
           message: 'No <loc> URLs were found in sitemap.xml.',
+          why: 'A sitemap without URL locations does not provide discoverable page entries.',
+          evidence: { locationCount: 0 },
+          nextStep: 'Add absolute page URLs or confirm that this is an intentionally empty sitemap index.',
         });
@@ -124,2 +155,5 @@ }
           message: 'One or more sitemap entries are not valid absolute HTTP(S) URLs.',
+          why: 'Relative or malformed sitemap locations may not be interpreted consistently.',
+          evidence: { invalidValues: details.invalidUrls },
+          nextStep: 'Replace invalid <loc> values with absolute HTTP(S) URLs.',
         });
@@ -132,2 +166,5 @@ }
           message: 'One or more sitemap entries use a different hostname.',
+          why: 'Cross-host entries may be intentional, but often indicate a staging or canonical-host mismatch.',
+          evidence: { expectedHostname: hostname, otherHostUrls: details.otherHosts },
+          nextStep: 'Confirm that every hostname is intentional and publicly canonical.',
         });
@@ -143,4 +180,30 @@ }
     };
+    if (response.ok && response.contentType && !/(?:text\/plain|text\/markdown)/i.test(response.contentType)) {
+      issues.push({
+        severity: 'warning',
+        code: 'unexpected_content_type',
+        message: `llms.txt returned ${response.contentType}.`,
+        why: 'An HTML fallback or unexpected media type can hide a missing llms.txt file.',
+        evidence: { contentType: response.contentType },
+        nextStep: 'Return llms.txt as plain text or Markdown and verify that the route is not serving an HTML fallback.',
+      });
+    }
     if (response.ok && !response.text.trim()) {
-      issues.push({ severity: 'warning', code: 'empty_llms', message: 'llms.txt is empty.' });
+      issues.push({
+        severity: 'warning',
+        code: 'empty_llms',
+        message: 'llms.txt is empty.',
+        why: 'An empty file provides no project summary or resource references.',
+        evidence: { characterCount: 0 },
+        nextStep: 'Add useful plain-text or Markdown content, or remove the empty file if it is not used.',
+      });
+    } else if (response.ok && !details.hasHeading) {
+      issues.push({
+        severity: 'warning',
+        code: 'llms_missing_heading',
+        message: 'llms.txt does not contain a Markdown H1 heading.',
+        why: 'A primary heading is a basic structural signal for the proposed llms.txt format.',
+        evidence: { hasHeading: false, characterCount: details.characterCount },
+        nextStep: 'Add one clear Markdown H1 heading near the beginning of the file.',
+      });
     }
@@ -147,0 +210,0 @@ return resultForFile(name, response, details, issues);

src/fetch-public.js

@@ -7,4 +7,40 @@ import { assertPublicUrl, normalizePublicUrl } from './url-safety.js';
 export async function readBoundedText(response, maxChars = DEFAULT_MAX_CHARS) {
-  if (!response.body) return (await response.text()).slice(0, maxChars);
+  return (await readBoundedResult(response, maxChars)).text;
+}
 
+function abortError() {
+  const error = new Error('The operation was aborted.');
+  error.name = 'AbortError';
+  return error;
+}
+
+async function waitForAbort(promise, signal) {
+  if (!signal) return promise;
+  if (signal.aborted) throw abortError();
+
+  let onAbort;
+  const aborted = new Promise((_, reject) => {
+    onAbort = () => reject(abortError());
+    signal.addEventListener('abort', onAbort, { once: true });
+  });
+
+  try {
+    return await Promise.race([promise, aborted]);
+  } finally {
+    signal.removeEventListener('abort', onAbort);
+  }
+}
+
+async function cancelBody(body, signal) {
+  if (!body) return;
+  const cancellation = body.cancel().catch(() => {});
+  await waitForAbort(cancellation, signal);
+}
+
+async function readBoundedResult(response, maxChars = DEFAULT_MAX_CHARS, signal) {
+  if (!response.body) {
+    const text = await waitForAbort(response.text(), signal);
+    return { text: text.slice(0, maxChars), truncated: text.length > maxChars };
+  }
+
   const decoder = new TextDecoder();
@@ -16,4 +52,4 @@ const reader = response.body.getReader();
   try {
-    while (output.length < maxChars) {
-      const { done, value } = await reader.read();
+    while (output.length <= maxChars) {
+      const { done, value } = await waitForAbort(reader.read(), signal);
       if (done) {
@@ -26,5 +62,9 @@ completed = true;
     output += decoder.decode();
-    return output.slice(0, maxChars);
+    return { text: output.slice(0, maxChars), truncated: output.length > maxChars };
   } finally {
-    if (!completed) await reader.cancel().catch(() => {});
+    if (!completed) {
+      const cancellation = reader.cancel().catch(() => {});
+      if (signal?.aborted) void cancellation;
+      else await waitForAbort(cancellation, signal);
+    }
     reader.releaseLock();
@@ -43,14 +83,14 @@ }
     assertUrlFn = assertPublicUrl,
+    setTimeoutFn = setTimeout,
+    clearTimeoutFn = clearTimeout,
   } = options;
 
   let currentUrl = normalizePublicUrl(target);
+  const controller = new AbortController();
+  const timeout = setTimeoutFn(() => controller.abort(), timeoutMs);
 
-  for (let redirects = 0; redirects <= maxRedirects; redirects += 1) {
-    await assertUrlFn(currentUrl);
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), timeoutMs);
-    let response;
-
-    try {
-      response = await fetchFn(currentUrl, {
+  try {
+    for (let redirects = 0; redirects <= maxRedirects; redirects += 1) {
+      await waitForAbort(Promise.resolve().then(() => assertUrlFn(currentUrl)), controller.signal);
+      const response = await waitForAbort(fetchFn(currentUrl, {
         redirect: 'manual',
@@ -62,37 +102,45 @@ signal: controller.signal,
         },
-      });
-    } catch (error) {
-      if (error?.name === 'AbortError') throw new Error(`Request timed out after ${timeoutMs} ms.`);
-      throw error;
-    } finally {
-      clearTimeout(timeout);
-    }
+      }), controller.signal);
 
-    if (!REDIRECT_CODES.has(response.status)) {
-      return {
-        requestedUrl: normalizePublicUrl(target),
-        finalUrl: currentUrl,
-        statusCode: response.status,
-        ok: response.ok,
-        contentType: response.headers.get('content-type') || '',
-        text: await readBoundedText(response, maxChars),
-      };
+      if (!REDIRECT_CODES.has(response.status)) {
+        const body = await readBoundedResult(response, maxChars, controller.signal);
+        return {
+          requestedUrl: normalizePublicUrl(target),
+          finalUrl: currentUrl,
+          statusCode: response.status,
+          ok: response.ok,
+          contentType: response.headers.get('content-type') || '',
+          text: body.text,
+          truncated: body.truncated,
+          maxChars,
+        };
+      }
+
+      await cancelBody(response.body, controller.signal);
+      const location = response.headers.get('location');
+      if (!location) {
+        return {
+          requestedUrl: normalizePublicUrl(target),
+          finalUrl: currentUrl,
+          statusCode: response.status,
+          ok: response.ok,
+          contentType: response.headers.get('content-type') || '',
+          text: '',
+          truncated: false,
+          maxChars,
+        };
+      }
+      if (redirects === maxRedirects) throw new Error('Too many redirects while checking this URL.');
+      currentUrl = normalizePublicUrl(new URL(location, currentUrl).toString());
     }
 
-    const location = response.headers.get('location');
-    if (!location) {
-      return {
-        requestedUrl: normalizePublicUrl(target),
-        finalUrl: currentUrl,
-        statusCode: response.status,
-        ok: response.ok,
-        contentType: response.headers.get('content-type') || '',
-        text: '',
-      };
+    throw new Error('Too many redirects while checking this URL.');
+  } catch (error) {
+    if (controller.signal.aborted || error?.name === 'AbortError') {
+      throw new Error(`Request timed out after ${timeoutMs} ms.`);
     }
-    if (redirects === maxRedirects) throw new Error('Too many redirects while checking this URL.');
-    currentUrl = normalizePublicUrl(new URL(location, currentUrl).toString());
+    throw error;
+  } finally {
+    clearTimeoutFn(timeout);
   }
-
-  throw new Error('Too many redirects while checking this URL.');
 }

src/format.js

@@ -5,5 +5,20 @@ function line(label, value) {
 
+function evidence(value) {
+  if (value === undefined) return '';
+  return typeof value === 'string' ? value : JSON.stringify(value);
+}
+
+function formatIssue(issue) {
+  return [
+    `- ${issue.severity.toUpperCase()} [${issue.code}]: ${issue.message}`,
+    issue.why ? `  Why: ${issue.why}` : null,
+    issue.evidence !== undefined ? `  Evidence: ${evidence(issue.evidence)}` : null,
+    issue.nextStep ? `  Next: ${issue.nextStep}` : null,
+  ].filter(Boolean).join('\n');
+}
+
 function formatIssues(issues) {
   if (!issues.length) return '\nNo material issues detected by this check.';
-  return `\nIssues:\n${issues.map((issue) => `- ${issue.severity.toUpperCase()}: ${issue.message}`).join('\n')}`;
+  const visibleIssues = issues.filter((issue) => !issue.compatibilityAlias);
+  return `\nIssues:\n${visibleIssues.map(formatIssue).join('\n')}`;
 }
@@ -34,8 +49,13 @@
       line('Crawler profile', result.crawlerProfile.label),
-      line('Browser HTTP', result.browser.response.statusCode),
+      line('Standard HTTP', result.browser.response.statusCode),
       line('Crawler HTTP', result.crawler.response.statusCode),
-      line('Browser text', `${result.browser.html.textLength} characters`),
+      line('Standard text', `${result.browser.html.textLength} characters`),
       line('Crawler text', `${result.crawler.html.textLength} characters`),
       line('Text ratio', result.difference.textRatio),
+      line(
+        'Accepted ratio',
+        `${result.difference.acceptedTextRatio.minimum}–${result.difference.acceptedTextRatio.maximum}`,
+      ),
       line('Title changed', result.difference.titleChanged ? 'yes' : 'no'),
+      line('Description changed', result.difference.descriptionChanged ? 'yes' : 'no'),
       line('H1 changed', result.difference.h1Changed ? 'yes' : 'no'),
@@ -51,3 +71,3 @@ formatIssues(result.issues),
     line('HTTP', file.statusCode),
-    ...file.issues.map((issue) => `- ${issue.severity.toUpperCase()}: ${issue.message}`),
+    ...file.issues.map(formatIssue),
   ]);
@@ -54,0 +74,0 @@ return [

src/html.js

@@ -18,2 +18,3 @@ function decodeEntities(value = '') {
     .replace(/<!--[\s\S]*?-->/g, ' ')
+    .replace(/<head\b[\s\S]*?<\/head>/gi, ' ')
     .replace(/<(script|style|noscript|template)\b[\s\S]*?<\/\1>/gi, ' ')
@@ -74,2 +75,18 @@ .replace(/<[^>]+>/g, ' '));
   const textLength = visibleText.length;
+  const hasEmptyMountPoint = /<(?:div|main)\b[^>]*\bid=["'](?:root|app)["'][^>]*>\s*<\/(?:div|main)>/i.test(html);
+  const hasLoadingPlaceholder = textLength < 180
+    && /\b(?:loading|please wait|initializing|starting)\b/i.test(visibleText);
+  const hasModuleOrBundledScript = /<script\b[^>]*(?:type=["']module["']|src=["'][^"']*(?:\/assets\/|bundle|app)[^"']*\.js)/i.test(html);
+  const appShellEvidence = [
+    ...(hasEmptyMountPoint ? ['empty root or app mount point'] : []),
+    ...(hasLoadingPlaceholder ? ['loading-only visible text'] : []),
+    ...(hasModuleOrBundledScript ? ['module or bundled application script'] : []),
+    ...(scriptCount >= 2 ? [`${scriptCount} script elements`] : []),
+    ...signs.map((sign) => `${sign} detected`),
+  ];
+  const looksLikeAppShell = textLength < 300 && (
+    (hasEmptyMountPoint && scriptCount >= 1)
+    || (hasLoadingPlaceholder && hasModuleOrBundledScript)
+    || (textLength < 80 && hasModuleOrBundledScript)
+  );
 
@@ -91,3 +108,4 @@ return {
     frameworkSigns: signs,
-    looksLikeAppShell: textLength < 300 && (scriptCount >= 2 || signs.length > 0),
+    appShellEvidence,
+    looksLikeAppShell,
   };
@@ -99,13 +117,79 @@ }
   if (!response.ok) {
-    issues.push({ severity: 'critical', code: 'http_error', message: `Page returned HTTP ${response.statusCode}.` });
+    issues.push({
+      severity: 'critical',
+      code: 'http_error',
+      message: `Page returned HTTP ${response.statusCode}.`,
+      why: 'An unsuccessful HTTP response can prevent crawlers from accessing the page content.',
+      evidence: { statusCode: response.statusCode },
+      nextStep: 'Confirm that the public URL returns a successful response for the selected crawler profile.',
+    });
   }
+  if (response.contentType && !/(?:text\/html|application\/xhtml\+xml)/i.test(response.contentType)) {
+    issues.push({
+      severity: 'critical',
+      code: 'unexpected_content_type',
+      message: `Page returned ${response.contentType} instead of HTML.`,
+      why: 'HTML diagnostics are not reliable when the response declares a different media type.',
+      evidence: { contentType: response.contentType },
+      nextStep: 'Check the requested route and its Content-Type header.',
+    });
+  }
+  if (response.truncated) {
+    issues.push({
+      severity: 'warning',
+      code: 'response_truncated',
+      message: `Analysis stopped after the configured ${response.maxChars} character response limit.`,
+      why: 'Signals after the response limit were not analysed.',
+      evidence: { maxChars: response.maxChars },
+      nextStep: 'Review the response size and rerun with a focused page when possible.',
+    });
+  }
   if (!summary.title) {
-    issues.push({ severity: 'warning', code: 'missing_title', message: 'Raw HTML is missing a page title.' });
+    issues.push({
+      severity: 'warning',
+      code: 'missing_title',
+      message: 'Returned HTML is missing a page title.',
+      why: 'The title is a primary page-identification signal in the returned HTML.',
+      evidence: { title: '' },
+      nextStep: 'Add a descriptive <title> to the initial HTML response.',
+    });
   }
   if (!summary.description) {
-    issues.push({ severity: 'warning', code: 'missing_description', message: 'Raw HTML is missing a meta description.' });
+    issues.push({
+      severity: 'warning',
+      code: 'missing_description',
+      message: 'Returned HTML is missing a meta description.',
+      why: 'A description helps crawlers and preview systems understand the page summary.',
+      evidence: { description: '' },
+      nextStep: 'Add a page-specific meta description to the initial HTML response.',
+    });
   }
   if (!summary.headings.h1.length) {
-    issues.push({ severity: 'warning', code: 'missing_h1', message: 'Raw HTML is missing an H1 heading.' });
+    issues.push({
+      severity: 'warning',
+      code: 'missing_h1',
+      message: 'Returned HTML is missing an H1 heading.',
+      why: 'A primary heading provides a clear content label in the returned document.',
+      evidence: { h1Count: 0 },
+      nextStep: 'Include the page’s primary heading in the initial HTML response.',
+    });
   }
+  if (summary.canonicalUrl) {
+    let canonicalIsValid = false;
+    try {
+      canonicalIsValid = ['http:', 'https:'].includes(new URL(summary.canonicalUrl).protocol);
+    } catch {
+      canonicalIsValid = false;
+    }
+    if (!canonicalIsValid) {
+      issues.push({
+        severity: 'warning',
+        code: 'invalid_canonical',
+        message: 'Returned HTML contains a canonical URL that is not an absolute HTTP(S) URL.',
+        why: 'A malformed or relative canonical can make the preferred page URL ambiguous.',
+        evidence: { canonicalUrl: summary.canonicalUrl },
+        nextStep: 'Replace the canonical value with the intended absolute public HTTP(S) URL.',
+      });
+    }
+  }
   if (summary.looksLikeAppShell) {
@@ -115,3 +199,10 @@ issues.push({
       code: 'app_shell',
-      message: 'Raw HTML has limited visible content and JavaScript app-shell signs.',
+      message: 'Returned HTML has limited visible content and multiple JavaScript app-shell signals.',
+      why: 'Crawlers that do not execute JavaScript may receive only the application shell.',
+      evidence: {
+        readableCharacters: summary.textLength,
+        scriptCount: summary.scriptCount,
+        signals: summary.appShellEvidence,
+      },
+      nextStep: 'Inspect the raw response and test whether important page content is present before JavaScript executes.',
     });
@@ -122,3 +213,6 @@ } else if (summary.textLength < 300) {
       code: 'thin_html',
-      message: 'Raw HTML contains less than 300 readable characters.',
+      message: 'Returned HTML contains less than 300 readable characters.',
+      why: 'A short response may be legitimate, but it may also omit important page content.',
+      evidence: { readableCharacters: summary.textLength, threshold: 300 },
+      nextStep: 'Review whether the returned text contains the page’s primary information.',
     });
@@ -125,0 +219,0 @@ }

src/index.js

 export { checkUrl } from './check.js';
-export { compareUrl } from './compare.js';
-export { checkDiscoveryFiles } from './discovery.js';
+export { compareUrl, contentDelta } from './compare.js';
+export { checkDiscoveryFiles, parseRobots, parseSitemap } from './discovery.js';
+export { formatHuman } from './format.js';
 export { analyzeHtml, buildHtmlIssues, stripTags } from './html.js';
 export { USER_AGENT_PROFILES, getUserAgentProfile } from './profiles.js';
 export { assertPublicUrl, isBlockedIp, normalizePublicUrl } from './url-safety.js';

src/profiles.js

 export const USER_AGENT_PROFILES = Object.freeze({
   browser: {
-    label: 'Browser',
+    label: 'Browser-style user agent',
     value: 'Mozilla/5.0 (compatible; PrerenderBuddyCLI/0.1; +https://prerenderbuddy.com)',
@@ -5,0 +5,0 @@ },

New alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Fixed alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Improved metrics

Total package byte prevSize

68777

48.48%

Number of package files

17

6.25%

Lines of code

1056

52.6%

Number of lines in readme file

253

56.17%

No dependency changes