@houtini/lm - npm Package Compare versions

Comparing version 3.2.2 to 3.2.3

dist/context-overflow.d.ts

+/**
+ * context-overflow.ts — recover the real context limit from a backend's 400.
+ *
+ * Strict OpenAI-compatible backends (notably vLLM) reject a request when
+ * prompt + max_tokens exceeds the model's context window with an HTTP 400,
+ * rather than silently clamping. houtini-lm sizes its output budget from the
+ * context it detects via /v1/models — but that number can be wrong when a
+ * proxy sits in front (e.g. a LiteLLM router advertising a generic 100k window
+ * while the model actually loaded is 64k). The result is a spurious 400 on a
+ * request that only overshot because the advertised context was too large.
+ *
+ * The server's own error message states the real limit. Parsing it lets the
+ * caller retry once with a corrected budget — robust even when /v1/models
+ * mis-reports the context behind a proxy, and self-correcting across backends.
+ */
+/**
+ * Extract the real maximum context length (tokens) a backend reports in a
+ * context-overflow error message. Returns the limit, or null if the text is not
+ * a recognisable context-overflow error.
+ *
+ * Recognised shapes (case-insensitive):
+ *   vLLM:      "max_completion_tokens=99002 cannot be greater than max_model_len=max_total_tokens=65536"
+ *   OpenAI:    "This model's maximum context length is 65536 tokens"
+ *   generic:   "... context length/size/window ... 65536 ..."
+ *   llama.cpp: "the request exceeds the available context size ... n_ctx = 65536"
+ */
+export declare function parseContextOverflow(text: string): number | null;
+/**
+ * Given the real context limit and a prompt-size estimate, compute a safe
+ * output budget that leaves room for the prompt. Mirrors the caller's original
+ * cap heuristic (≈3 chars/token + per-message overhead + a margin) so a retry
+ * lands comfortably inside the window. Never returns below `floor`.
+ */
+export declare function correctedMaxTokens(realContextLen: number, promptChars: number, messageCount: number, floor?: number): number;

dist/context-overflow.js

+/**
+ * context-overflow.ts — recover the real context limit from a backend's 400.
+ *
+ * Strict OpenAI-compatible backends (notably vLLM) reject a request when
+ * prompt + max_tokens exceeds the model's context window with an HTTP 400,
+ * rather than silently clamping. houtini-lm sizes its output budget from the
+ * context it detects via /v1/models — but that number can be wrong when a
+ * proxy sits in front (e.g. a LiteLLM router advertising a generic 100k window
+ * while the model actually loaded is 64k). The result is a spurious 400 on a
+ * request that only overshot because the advertised context was too large.
+ *
+ * The server's own error message states the real limit. Parsing it lets the
+ * caller retry once with a corrected budget — robust even when /v1/models
+ * mis-reports the context behind a proxy, and self-correcting across backends.
+ */
+/**
+ * Extract the real maximum context length (tokens) a backend reports in a
+ * context-overflow error message. Returns the limit, or null if the text is not
+ * a recognisable context-overflow error.
+ *
+ * Recognised shapes (case-insensitive):
+ *   vLLM:      "max_completion_tokens=99002 cannot be greater than max_model_len=max_total_tokens=65536"
+ *   OpenAI:    "This model's maximum context length is 65536 tokens"
+ *   generic:   "... context length/size/window ... 65536 ..."
+ *   llama.cpp: "the request exceeds the available context size ... n_ctx = 65536"
+ */
+export function parseContextOverflow(text) {
+    if (!text)
+        return null;
+    const patterns = [
+        /max_model_len\D*([\d,]{2,})/i, // vLLM — grabs the final number after max_model_len
+        /maximum context length is\s*([\d,]{2,})/i, // OpenAI
+        /context (?:length|size|window)\D*([\d,]{2,})/i,
+        /n_ctx\s*=\s*([\d,]{2,})/i, // llama.cpp
+    ];
+    for (const p of patterns) {
+        const m = text.match(p);
+        if (m) {
+            // Numbers may carry thousands separators ("65,536"); strip before parsing.
+            const cleaned = m[1].replace(/,/g, '');
+            if (/^\d{2,}$/.test(cleaned)) {
+                const n = Number(cleaned);
+                if (Number.isFinite(n) && n > 0)
+                    return n;
+            }
+        }
+    }
+    return null;
+}
+/**
+ * Given the real context limit and a prompt-size estimate, compute a safe
+ * output budget that leaves room for the prompt. Mirrors the caller's original
+ * cap heuristic (≈3 chars/token + per-message overhead + a margin) so a retry
+ * lands comfortably inside the window. Never returns below `floor`.
+ */
+export function correctedMaxTokens(realContextLen, promptChars, messageCount, floor = 256) {
+    const promptEstimate = Math.ceil(promptChars / 3) + 64 * messageCount + 512;
+    return Math.max(floor, realContextLen - promptEstimate);
+}
+//# sourceMappingURL=context-overflow.js.map

dist/context-overflow.js.map

+{"version":3,"file":"context-overflow.js","sourceRoot":"","sources":["../src/context-overflow.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH;;;;;;;;;;GAUG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,MAAM,QAAQ,GAAa;QACzB,8BAA8B,EAAgB,oDAAoD;QAClG,0CAA0C,EAAI,SAAS;QACvD,+CAA+C;QAC/C,0BAA0B,EAAqB,YAAY;KAC5D,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,EAAE,CAAC;YACN,2EAA2E;YAC3E,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACvC,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC7B,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC1B,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,OAAO,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAChC,cAAsB,EACtB,WAAmB,EACnB,YAAoB,EACpB,KAAK,GAAG,GAAG;IAEX,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,YAAY,GAAG,GAAG,CAAC;IAC5E,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,cAAc,CAAC,CAAC;AAC1D,CAAC"}

package.json

 {
   "name": "@houtini/lm",
-  "version": "3.2.2",
+  "version": "3.2.3",
   "type": "module",
@@ -13,2 +13,3 @@ "description": "MCP server for local LLMs — connects to LM Studio or any OpenAI-compatible endpoint",
     "test:vllm": "node test-vllm-thinking.mjs",
+    "test:overflow": "node test-context-overflow.mjs",
     "build": "tsc && node add-shebang.mjs",
@@ -15,0 +16,0 @@ "dev": "tsc --watch",

server.json

@@ -10,3 +10,3 @@ {
   },
-  "version": "3.2.2",
+  "version": "3.2.3",
   "packages": [
@@ -16,3 +16,3 @@ {
       "identifier": "@houtini/lm",
-      "version": "3.2.2",
+      "version": "3.2.3",
       "transport": [
@@ -19,0 +19,0 @@ {

No alert changes

Improved metrics

Total package byte prevSize

344817

2.61%

Number of package files

19

18.75%

Lines of code

3978

3.08%

No dependency changes