llama-cpp-python - pypi Package Compare versions

Comparing version 0.3.26 to 0.3.27

examples/server/configs/gemma-4-12b-it-qat.json

+{
+  "server": {
+    "host": "0.0.0.0",
+    "port": 8000
+  },
+  "model": {
+    "alias": "gemma-4-12b-it-qat",
+    "from_pretrained": {
+      "repo_id": "unsloth/gemma-4-12B-it-qat-GGUF",
+      "filename": "gemma-4-12B-it-qat-UD-Q4_K_XL.gguf"
+    },
+    "mtmd": {
+      "mmproj_from_pretrained": {
+        "repo_id": "unsloth/gemma-4-12B-it-qat-GGUF",
+        "filename": "mmproj-BF16.gguf"
+      }
+    },
+    "n_ctx": 32768,
+    "max_output_tokens": 4096,
+    "n_seq_max": 8,
+    "n_batch": 512,
+    "n_ubatch": 512,
+    "threads": 8,
+    "threads_batch": 8,
+    "kv_unified": true,
+    "store_logits": false,
+    "use_mmap": true,
+    "use_mlock": false,
+    "n_gpu_layers": -1,
+    "flash_attn": true,
+    "response_schema": {
+      "type": "object",
+      "properties": {
+        "role": {
+          "const": "assistant"
+        },
+        "reasoning_content": {
+          "type": "string",
+          "x-regex": "^(?:<\\|turn>model\\n)?<\\|channel>thought\\n(.*?)<channel\\|>"
+        },
+        "content": {
+          "type": "string",
+          "x-regex": "^(?:<\\|turn>model\\n)?(?:(?:<\\|channel>thought\\n).*?<channel\\|>)?(.*?)(?=<\\|tool_call>|<turn\\|>|$)"
+        },
+        "tool_calls": {
+          "type": "array",
+          "x-regex-iterator": "<\\|tool_call>(call:[^\\{]+\\{.*?\\})<tool_call\\|>",
+          "items": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "function"
+              },
+              "function": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "type": "string",
+                    "x-regex": "^call:(\\w+)\\{"
+                  },
+                  "arguments": {
+                    "type": "object",
+                    "x-regex": "^call:\\w+(\\{.*\\})$",
+                    "x-parser": "gemma4-tool-call",
+                    "additionalProperties": true
+                  }
+                },
+                "required": [
+                  "name",
+                  "arguments"
+                ]
+              }
+            },
+            "required": [
+              "type",
+              "function"
+            ]
+          }
+        }
+      },
+      "required": [
+        "role"
+      ]
+    },
+    "chat_template": [
+      "{%- macro format_parameters(properties, required, filter_keys=false) -%}\n",
+      "    {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}\n",
+      "    {%- set ns = namespace(found_first=false) -%}\n",
+      "    {%- for key, value in properties | dictsort -%}\n",
+      "        {%- set add_comma = false -%}\n",
+      "        {%- if not filter_keys or key not in standard_keys -%}\n",
+      "            {%- if ns.found_first %},{% endif -%}\n",
+      "            {%- set ns.found_first = true -%}\n",
+      "            {{ key }}:{\n",
+      "            {%- if value['description'] -%}\n",
+      "                description:<|\"|>{{ value['description'] }}<|\"|>\n",
+      "                {%- set add_comma = true -%}\n",
+      "            {%- endif -%}\n",
+      "            {%- if value['type'] | upper == 'STRING' -%}\n",
+      "                {%- if value['enum'] -%}\n",
+      "                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n",
+      "                    enum:{{ format_argument(value['enum']) }}\n",
+      "                {%- endif -%}\n",
+      "            {%- elif value['type'] | upper == 'ARRAY' -%}\n",
+      "                {%- if value['items'] is mapping and value['items'] -%}\n",
+      "                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n",
+      "                    items:{\n",
+      "                    {%- set ns_items = namespace(found_first=false) -%}\n",
+      "                    {%- for item_key, item_value in value['items'] | dictsort -%}\n",
+      "                        {%- if item_value is not none -%}\n",
+      "                            {%- if ns_items.found_first %},{% endif -%}\n",
+      "                            {%- set ns_items.found_first = true -%}\n",
+      "                            {%- if item_key == 'properties' -%}\n",
+      "                                properties:{\n",
+      "                                {%- if item_value is mapping -%}\n",
+      "                                    {{- format_parameters(item_value, value['items']['required'] | default([])) -}}\n",
+      "                                {%- endif -%}\n",
+      "                                }\n",
+      "                            {%- elif item_key == 'required' -%}\n",
+      "                                required:[\n",
+      "                                {%- for req_item in item_value -%}\n",
+      "                                    <|\"|>{{- req_item -}}<|\"|>\n",
+      "                                    {%- if not loop.last %},{% endif -%}\n",
+      "                                {%- endfor -%}\n",
+      "                                ]\n",
+      "                            {%- elif item_key == 'type' -%}\n",
+      "                                {%- if item_value is string -%}\n",
+      "                                    type:{{ format_argument(item_value | upper) }}\n",
+      "                                {%- else -%}\n",
+      "                                    type:{{ format_argument(item_value | map('upper') | list) }}\n",
+      "                                {%- endif -%}\n",
+      "                            {%- else -%}\n",
+      "                                {{ item_key }}:{{ format_argument(item_value) }}\n",
+      "                            {%- endif -%}\n",
+      "                        {%- endif -%}\n",
+      "                    {%- endfor -%}\n",
+      "                    }\n",
+      "                {%- endif -%}\n",
+      "            {%- endif -%}\n",
+      "            {%- if value['nullable'] %}\n",
+      "                {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n",
+      "                nullable:true\n",
+      "            {%- endif -%}\n",
+      "            {%- if value['type'] | upper == 'OBJECT' -%}\n",
+      "                {%- if value['properties'] is defined and value['properties'] is mapping -%}\n",
+      "                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n",
+      "                    properties:{\n",
+      "                    {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n",
+      "                    }\n",
+      "                {%- elif value is mapping -%}\n",
+      "                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n",
+      "                    properties:{\n",
+      "                    {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}}\n",
+      "                    }\n",
+      "                {%- endif -%}\n",
+      "                {%- if value['required'] -%}\n",
+      "                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n",
+      "                    required:[\n",
+      "                    {%- for item in value['required'] | default([]) -%}\n",
+      "                        <|\"|>{{- item -}}<|\"|>\n",
+      "                        {%- if not loop.last %},{% endif -%}\n",
+      "                    {%- endfor -%}\n",
+      "                    ]\n",
+      "                {%- endif -%}\n",
+      "            {%- endif -%}\n",
+      "            {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n",
+      "            type:<|\"|>{{ value['type'] | upper }}<|\"|>}\n",
+      "        {%- endif -%}\n",
+      "    {%- endfor -%}\n",
+      "{%- endmacro -%}\n",
+      "{%- macro format_function_declaration(tool_data) -%}\n",
+      "    declaration:{{- tool_data['function']['name'] -}}{description:<|\"|>{{- tool_data['function']['description'] -}}<|\"|>\n",
+      "    {%- set params = tool_data['function']['parameters'] -%}\n",
+      "    {%- if params -%}\n",
+      "        ,parameters:{\n",
+      "        {%- if params['properties'] -%}\n",
+      "            properties:{ {{- format_parameters(params['properties'], params['required']) -}} },\n",
+      "        {%- endif -%}\n",
+      "        {%- if params['required'] -%}\n",
+      "            required:[\n",
+      "            {%- for item in params['required'] -%}\n",
+      "                <|\"|>{{- item -}}<|\"|>\n",
+      "                {{- ',' if not loop.last -}}\n",
+      "            {%- endfor -%}\n",
+      "            ],\n",
+      "        {%- endif -%}\n",
+      "        {%- if params['type'] -%}\n",
+      "            type:<|\"|>{{- params['type'] | upper -}}<|\"|>}\n",
+      "        {%- endif -%}\n",
+      "    {%- endif -%}\n",
+      "    {%- if 'response' in tool_data['function'] -%}\n",
+      "        {%- set response_declaration = tool_data['function']['response'] -%}\n",
+      "        ,response:{\n",
+      "        {%- if response_declaration['description'] -%}\n",
+      "            description:<|\"|>{{- response_declaration['description'] -}}<|\"|>,\n",
+      "        {%- endif -%}\n",
+      "        {%- if response_declaration['type'] | upper == 'OBJECT' -%}\n",
+      "            type:<|\"|>{{- response_declaration['type'] | upper -}}<|\"|>}\n",
+      "        {%- endif -%}\n",
+      "    {%- endif -%}\n",
+      "    }\n",
+      "{%- endmacro -%}\n",
+      "{%- macro format_argument(argument, escape_keys=True) -%}\n",
+      "    {%- if argument is string -%}\n",
+      "        {{- '<|\"|>' + argument + '<|\"|>' -}}\n",
+      "    {%- elif argument is boolean -%}\n",
+      "        {{- 'true' if argument else 'false' -}}\n",
+      "    {%- elif argument is mapping -%}\n",
+      "        {{- '{' -}}\n",
+      "        {%- set ns = namespace(found_first=false) -%}\n",
+      "        {%- for key, value in argument | dictsort -%}\n",
+      "            {%- if ns.found_first %},{% endif -%}\n",
+      "            {%- set ns.found_first = true -%}\n",
+      "            {%- if escape_keys -%}\n",
+      "                {{- '<|\"|>' + key + '<|\"|>' -}}\n",
+      "            {%- else -%}\n",
+      "                {{- key -}}\n",
+      "            {%- endif -%}\n",
+      "            :{{- format_argument(value, escape_keys=escape_keys) -}}\n",
+      "        {%- endfor -%}\n",
+      "        {{- '}' -}}\n",
+      "    {%- elif argument is sequence -%}\n",
+      "        {{- '[' -}}\n",
+      "        {%- for item in argument -%}\n",
+      "            {{- format_argument(item, escape_keys=escape_keys) -}}\n",
+      "            {%- if not loop.last %},{% endif -%}\n",
+      "        {%- endfor -%}\n",
+      "        {{- ']' -}}\n",
+      "    {%- else -%}\n",
+      "        {{- argument -}}\n",
+      "    {%- endif -%}\n",
+      "{%- endmacro -%}\n",
+      "{%- macro strip_thinking(text) -%}\n",
+      "    {%- set ns = namespace(result='') -%}\n",
+      "    {%- for part in text.split('<channel|>') -%}\n",
+      "        {%- if '<|channel>' in part -%}\n",
+      "            {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}\n",
+      "        {%- else -%}\n",
+      "            {%- set ns.result = ns.result + part -%}\n",
+      "        {%- endif -%}\n",
+      "    {%- endfor -%}\n",
+      "    {{- ns.result | trim -}}\n",
+      "{%- endmacro -%}\n",
+      "\n",
+      "{%- macro format_tool_response_block(tool_name, response) -%}\n",
+      "    {{- '<|tool_response>' -}}\n",
+      "    {%- if response is mapping -%}\n",
+      "        {{- 'response:' + tool_name + '{' -}}\n",
+      "        {%- for key, value in response | dictsort -%}\n",
+      "            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n",
+      "            {%- if not loop.last %},{% endif -%}\n",
+      "        {%- endfor -%}\n",
+      "        {{- '}' -}}\n",
+      "    {%- else -%}\n",
+      "        {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}\n",
+      "    {%- endif -%}\n",
+      "    {{- '<tool_response|>' -}}\n",
+      "{%- endmacro -%}\n",
+      "\n",
+      "{%- set ns = namespace(prev_message_type=None) -%}\n",
+      "{%- set loop_messages = messages -%}\n",
+      "{{- bos_token -}}\n",
+      "{#- Handle System/Tool Definitions Block -#}\n",
+      "{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}\n",
+      "    {{- '<|turn>system\\n' -}}\n",
+      "    {#- Inject Thinking token at the very top of the FIRST system turn -#}\n",
+      "    {%- if enable_thinking is defined and enable_thinking -%}\n",
+      "        {{- '<|think|>\\n' -}}\n",
+      "        {%- set ns.prev_message_type = 'think' -%}\n",
+      "    {%- endif -%}\n",
+      "    {%- if messages[0]['role'] in ['system', 'developer'] -%}\n",
+      "        {%- if messages[0]['content'] is string -%}\n",
+      "            {{- messages[0]['content'] | trim -}}\n",
+      "        {%- elif messages[0]['content'] is sequence -%}\n",
+      "            {%- for item in messages[0]['content'] -%}\n",
+      "                {{- item['text'] | trim + ' '-}}\n",
+      "            {%- endfor -%}\n",
+      "        {%- endif -%}\n",
+      "        {%- set loop_messages = messages[1:] -%}\n",
+      "    {%- endif -%}\n",
+      "    {%- if tools -%}\n",
+      "        {%- for tool in tools %}\n",
+      "            {{- '<|tool>' -}}\n",
+      "            {{- format_function_declaration(tool) | trim -}}\n",
+      "            {{- '<tool|>' -}}\n",
+      "        {%- endfor %}\n",
+      "        {%- set ns.prev_message_type = 'tool' -%}\n",
+      "    {%- endif -%}\n",
+      "    {{- '<turn|>\\n' -}}\n",
+      "{%- endif %}\n",
+      "\n",
+      "{#- Pre-scan: find last user message index for reasoning guard -#}\n",
+      "{%- set ns_turn = namespace(last_user_idx=-1) -%}\n",
+      "{%- for i in range(loop_messages | length) -%}\n",
+      "    {%- if loop_messages[i]['role'] == 'user' -%}\n",
+      "        {%- set ns_turn.last_user_idx = i -%}\n",
+      "    {%- endif -%}\n",
+      "{%- endfor -%}\n",
+      "\n",
+      "{#- Loop through messages -#}\n",
+      "{%- for message in loop_messages -%}\n",
+      "    {%- if message['role'] != 'tool' -%}\n",
+      "    {%- set ns.prev_message_type = None -%}\n",
+      "    {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n",
+      "    {#- Detect continuation: suppress duplicate <|turn>model when previous non-tool message was also assistant -#}\n",
+      "    {%- set prev_nt = namespace(role=None, found=false) -%}\n",
+      "    {%- if loop.index0 > 0 -%}\n",
+      "        {%- for j in range(loop.index0 - 1, -1, -1) -%}\n",
+      "            {%- if not prev_nt.found -%}\n",
+      "                {%- if loop_messages[j]['role'] != 'tool' -%}\n",
+      "                    {%- set prev_nt.role = loop_messages[j]['role'] -%}\n",
+      "                    {%- set prev_nt.found = true -%}\n",
+      "                {%- endif -%}\n",
+      "            {%- endif -%}\n",
+      "        {%- endfor -%}\n",
+      "    {%- endif -%}\n",
+      "    {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}\n",
+      "    {%- if not continue_same_model_turn -%}\n",
+      "        {{- '<|turn>' + role + '\\n' }}\n",
+      "    {%- endif -%}\n",
+      "\n",
+      "    {#- Render reasoning/reasoning_content as thinking channel -#}\n",
+      "    {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n",
+      "    {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}\n",
+      "        {{- '<|channel>thought\\n' + thinking_text + '\\n<channel|>' -}}\n",
+      "    {%- endif -%}\n",
+      "\n",
+      "            {%- if message['tool_calls'] -%}\n",
+      "                {%- for tool_call in message['tool_calls'] -%}\n",
+      "                    {%- set function = tool_call['function'] -%}\n",
+      "                    {{- '<|tool_call>call:' + function['name'] + '{' -}}\n",
+      "                    {%- if function['arguments'] is mapping -%}\n",
+      "                        {%- set ns_args = namespace(found_first=false) -%}\n",
+      "                        {%- for key, value in function['arguments'] | dictsort -%}\n",
+      "                            {%- if ns_args.found_first %},{% endif -%}\n",
+      "                            {%- set ns_args.found_first = true -%}\n",
+      "                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n",
+      "                        {%- endfor -%}\n",
+      "                    {%- elif function['arguments'] is string -%}\n",
+      "                        {{- function['arguments'] -}}\n",
+      "                    {%- endif -%}\n",
+      "                    {{- '}<tool_call|>' -}}\n",
+      "                {%- endfor -%}\n",
+      "                {%- set ns.prev_message_type = 'tool_call' -%}\n",
+      "            {%- endif -%}\n",
+      "\n",
+      "            {%- set ns_tr_out = namespace(flag=false) -%}\n",
+      "            {%- if message.get('tool_responses') -%}\n",
+      "                {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n",
+      "                {%- for tool_response in message['tool_responses'] -%}\n",
+      "                    {{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}\n",
+      "                    {%- set ns_tr_out.flag = true -%}\n",
+      "                    {%- set ns.prev_message_type = 'tool_response' -%}\n",
+      "                {%- endfor -%}\n",
+      "            {%- elif message.get('tool_calls') -%}\n",
+      "                {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#}\n",
+      "                {%- set ns_tool_scan = namespace(stopped=false) -%}\n",
+      "                {%- for k in range(loop.index0 + 1, loop_messages | length) -%}\n",
+      "                    {%- if ns_tool_scan.stopped -%}\n",
+      "                    {%- elif loop_messages[k]['role'] != 'tool' -%}\n",
+      "                        {%- set ns_tool_scan.stopped = true -%}\n",
+      "                    {%- else -%}\n",
+      "                        {%- set follow = loop_messages[k] -%}\n",
+      "                        {#- Resolve tool_call_id to function name -#}\n",
+      "                        {%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}\n",
+      "                        {%- for tc in message['tool_calls'] -%}\n",
+      "                            {%- if tc.get('id') == follow.get('tool_call_id') -%}\n",
+      "                                {%- set ns_tname.name = tc['function']['name'] -%}\n",
+      "                            {%- endif -%}\n",
+      "                        {%- endfor -%}\n",
+      "                        {#- Handle content as string or content-parts array -#}\n",
+      "                        {%- set tool_body = follow.get('content') -%}\n",
+      "                        {%- if tool_body is string -%}\n",
+      "                            {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n",
+      "                        {%- elif tool_body is sequence and tool_body is not string -%}\n",
+      "                            {%- set ns_txt = namespace(s='') -%}\n",
+      "                            {%- for part in tool_body -%}\n",
+      "                                {%- if part.get('type') == 'text' -%}\n",
+      "                                    {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}\n",
+      "                                {%- endif -%}\n",
+      "                            {%- endfor -%}\n",
+      "                            {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n",
+      "                            {%- for part in tool_body -%}\n",
+      "                                {%- if part.get('type') == 'image' -%}\n",
+      "                                    {{- '<|image|>' -}}\n",
+      "                                {%- elif part.get('type') == 'audio' -%}\n",
+      "                                    {{- '<|audio|>' -}}\n",
+      "                                {%- elif part.get('type') == 'video' -%}\n",
+      "                                    {{- '<|video|>' -}}\n",
+      "                                {%- endif -%}\n",
+      "                            {%- endfor -%}\n",
+      "                        {%- else -%}\n",
+      "                            {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n",
+      "                        {%- endif -%}\n",
+      "                        {%- set ns_tr_out.flag = true -%}\n",
+      "                        {%- set ns.prev_message_type = 'tool_response' -%}\n",
+      "                    {%- endif -%}\n",
+      "                {%- endfor -%}\n",
+      "            {%- endif -%}\n",
+      "\n",
+      "            {%- set captured_content -%}\n",
+      "            {%- if message['content'] is string -%}\n",
+      "                {%- if role == 'model' -%}\n",
+      "                    {{- strip_thinking(message['content']) -}}\n",
+      "                {%- else -%}\n",
+      "                    {{- message['content'] | trim -}}\n",
+      "                {%- endif -%}\n",
+      "            {%- elif message['content'] is sequence -%}\n",
+      "                {%- for item in message['content'] -%}\n",
+      "                    {%- if item['type'] == 'text' -%}\n",
+      "                        {%- if role == 'model' -%}\n",
+      "                            {{- strip_thinking(item['text']) -}}\n",
+      "                        {%- else -%}\n",
+      "                            {{- item['text'] | trim -}}\n",
+      "                        {%- endif -%}\n",
+      "                    {%- elif item['type'] == 'image' -%}\n",
+      "                        {{- '<|image|>' -}}\n",
+      "                        {%- set ns.prev_message_type = 'image' -%}\n",
+      "                    {%- elif item['type'] == 'audio' -%}\n",
+      "                        {{- '<|audio|>' -}}\n",
+      "                        {%- set ns.prev_message_type = 'audio' -%}\n",
+      "                    {%- elif item['type'] == 'video' -%}\n",
+      "                        {{- '<|video|>' -}}\n",
+      "                        {%- set ns.prev_message_type = 'video' -%}\n",
+      "                    {%- endif -%}\n",
+      "                {%- endfor -%}\n",
+      "            {%- endif -%}\n",
+      "            {%- endset -%}\n",
+      "\n",
+      "            {{- captured_content -}}\n",
+      "            {%- set has_content = captured_content | trim | length > 0 -%}\n",
+      "\n",
+      "        {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n",
+      "            {{- '<|tool_response>' -}}\n",
+      "        {%- elif not (ns_tr_out.flag and not has_content) -%}\n",
+      "            {{- '<turn|>\\n' -}}\n",
+      "        {%- endif -%}\n",
+      "    {%- endif -%}\n",
+      "{%- endfor -%}\n",
+      "\n",
+      "{%- if add_generation_prompt -%}\n",
+      "    {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n",
+      "        {{- '<|turn>model\\n' -}}\n",
+      "        {%- if not enable_thinking | default(false) -%}\n",
+      "            {{- '<|channel>thought\\n<channel|>' -}}\n",
+      "        {%- endif -%}\n",
+      "    {%- endif -%}\n",
+      "{%- endif -%}"
+    ]
+  }
+}

examples/server/configs/gpt-oss-120b.json

+{
+  "server": {
+    "host": "0.0.0.0",
+    "port": 8000
+  },
+  "model": {
+    "alias": "gpt-oss-120b",
+    "from_pretrained": {
+      "repo_id": "ggml-org/gpt-oss-120b-GGUF",
+      "filename": "gpt-oss-120b-mxfp4-00001-of-00003.gguf",
+      "additional_files": [
+        "gpt-oss-120b-mxfp4-00002-of-00003.gguf",
+        "gpt-oss-120b-mxfp4-00003-of-00003.gguf"
+      ]
+    },
+    "n_ctx": 32768,
+    "max_output_tokens": 4096,
+    "n_seq_max": 4,
+    "n_batch": 512,
+    "n_ubatch": 512,
+    "threads": 8,
+    "threads_batch": 8,
+    "kv_unified": true,
+    "store_logits": false,
+    "use_mmap": true,
+    "use_mlock": false,
+    "n_gpu_layers": -1,
+    "flash_attn": true,
+    "response_schema": {
+      "type": "object",
+      "properties": {
+        "role": {
+          "const": "assistant"
+        },
+        "reasoning_content": {
+          "type": "string",
+          "x-regex": "^(?:<\\|start\\|>assistant)?<\\|channel\\|>analysis<\\|message\\|>(.*?)(?=<\\|end\\|>|<\\|start\\|>assistant|$)"
+        },
+        "content": {
+          "type": "string",
+          "x-regex": "^(?:.*?<\\|start\\|>assistant)?<\\|channel\\|>final<\\|message\\|>(.*?)(?=<\\|return\\|>|<\\|end\\|>|$)"
+        },
+        "tool_calls": {
+          "type": "array",
+          "x-regex-iterator": "(<\\|start\\|>assistant to=functions\\.\\w+<\\|channel\\|>commentary json<\\|message\\|>.*?<\\|call\\|>)",
+          "items": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "function"
+              },
+              "function": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "type": "string",
+                    "x-regex": "^<\\|start\\|>assistant to=functions\\.(\\w+)<\\|channel\\|>commentary json<\\|message\\|>"
+                  },
+                  "arguments": {
+                    "type": "object",
+                    "x-regex": "^<\\|start\\|>assistant to=functions\\.\\w+<\\|channel\\|>commentary json<\\|message\\|>(.*?)<\\|call\\|>$",
+                    "x-parser": "json",
+                    "additionalProperties": true
+                  }
+                },
+                "required": [
+                  "name",
+                  "arguments"
+                ]
+              }
+            },
+            "required": [
+              "type",
+              "function"
+            ]
+          }
+        }
+      },
+      "required": [
+        "role"
+      ]
+    },
+    "chat_template": [
+      "{#-\n",
+      "  In addition to the normal inputs of `messages` and `tools`, this template also accepts the\n",
+      "  following kwargs:\n",
+      "  - \"builtin_tools\": A list, can contain \"browser\" and/or \"python\".\n",
+      "  - \"model_identity\": A string that optionally describes the model identity.\n",
+      "  - \"reasoning_effort\": A string that describes the reasoning effort, defaults to \"medium\".\n",
+      " #}\n",
+      "\n",
+      "{#- Tool Definition Rendering ============================================== #}\n",
+      "{%- macro render_typescript_type(param_spec, required_params, is_nullable=false) -%}\n",
+      "    {%- if param_spec.type == \"array\" -%}\n",
+      "        {%- if param_spec['items'] -%}\n",
+      "            {%- if param_spec['items']['type'] == \"string\" -%}\n",
+      "                {{- \"string[]\" }}\n",
+      "            {%- elif param_spec['items']['type'] == \"number\" -%}\n",
+      "                {{- \"number[]\" }}\n",
+      "            {%- elif param_spec['items']['type'] == \"integer\" -%}\n",
+      "                {{- \"number[]\" }}\n",
+      "            {%- elif param_spec['items']['type'] == \"boolean\" -%}\n",
+      "                {{- \"boolean[]\" }}\n",
+      "            {%- else -%}\n",
+      "                {%- set inner_type = render_typescript_type(param_spec['items'], required_params) -%}\n",
+      "                {%- if inner_type == \"object | object\" or inner_type|length > 50 -%}\n",
+      "                    {{- \"any[]\" }}\n",
+      "                {%- else -%}\n",
+      "                    {{- inner_type + \"[]\" }}\n",
+      "                {%- endif -%}\n",
+      "            {%- endif -%}\n",
+      "            {%- if param_spec.nullable -%}\n",
+      "                {{- \" | null\" }}\n",
+      "            {%- endif -%}\n",
+      "        {%- else -%}\n",
+      "            {{- \"any[]\" }}\n",
+      "            {%- if param_spec.nullable -%}\n",
+      "                {{- \" | null\" }}\n",
+      "            {%- endif -%}\n",
+      "        {%- endif -%}\n",
+      "    {%- elif param_spec.type is defined and param_spec.type is iterable and param_spec.type is not string and param_spec.type is not mapping and param_spec.type[0] is defined -%}\n",
+      "        {#- Handle array of types like [\"object\", \"object\"] from Union[dict, list] #}\n",
+      "        {%- if param_spec.type | length > 1 -%}\n",
+      "            {{- param_spec.type | join(\" | \") }}\n",
+      "        {%- else -%}\n",
+      "            {{- param_spec.type[0] }}\n",
+      "        {%- endif -%}\n",
+      "    {%- elif param_spec.oneOf -%}\n",
+      "        {#- Handle oneOf schemas - check for complex unions and fallback to any #}\n",
+      "        {%- set has_object_variants = false -%}\n",
+      "        {%- for variant in param_spec.oneOf -%}\n",
+      "            {%- if variant.type == \"object\" -%}\n",
+      "                {%- set has_object_variants = true -%}\n",
+      "            {%- endif -%}\n",
+      "        {%- endfor -%}\n",
+      "        {%- if has_object_variants and param_spec.oneOf|length > 1 -%}\n",
+      "            {{- \"any\" }}\n",
+      "        {%- else -%}\n",
+      "            {%- for variant in param_spec.oneOf -%}\n",
+      "                {{- render_typescript_type(variant, required_params) -}}\n",
+      "                {%- if variant.description %}\n",
+      "                    {{- \"// \" + variant.description }}\n",
+      "                {%- endif -%}\n",
+      "                {%- if variant.default is defined %}\n",
+      "                    {{ \"// default: \" + variant.default|tojson }}\n",
+      "                {%- endif -%}\n",
+      "                {%- if not loop.last %}\n",
+      "                    {{- \" | \" }}\n",
+      "                {% endif -%}\n",
+      "            {%- endfor -%}\n",
+      "        {%- endif -%}\n",
+      "    {%- elif param_spec.type == \"string\" -%}\n",
+      "        {%- if param_spec.enum -%}\n",
+      "            {{- '\"' + param_spec.enum|join('\" | \"') + '\"' -}}\n",
+      "        {%- else -%}\n",
+      "            {{- \"string\" }}\n",
+      "            {%- if param_spec.nullable %}\n",
+      "                {{- \" | null\" }}\n",
+      "            {%- endif -%}\n",
+      "        {%- endif -%}\n",
+      "    {%- elif param_spec.type == \"number\" -%}\n",
+      "        {{- \"number\" }}\n",
+      "    {%- elif param_spec.type == \"integer\" -%}\n",
+      "        {{- \"number\" }}\n",
+      "    {%- elif param_spec.type == \"boolean\" -%}\n",
+      "        {{- \"boolean\" }}\n",
+      "\n",
+      "    {%- elif param_spec.type == \"object\" -%}\n",
+      "        {%- if param_spec.properties -%}\n",
+      "            {{- \"{\\n\" }}\n",
+      "            {%- for prop_name, prop_spec in param_spec.properties.items() -%}\n",
+      "                {{- prop_name -}}\n",
+      "                {%- if prop_name not in (param_spec.required or []) -%}\n",
+      "                    {{- \"?\" }}\n",
+      "                {%- endif -%}\n",
+      "                {{- \": \" }}\n",
+      "                {{ render_typescript_type(prop_spec, param_spec.required or []) }}\n",
+      "                {%- if not loop.last -%}\n",
+      "                    {{-\", \" }}\n",
+      "                {%- endif -%}\n",
+      "            {%- endfor -%}\n",
+      "            {{- \"}\" }}\n",
+      "        {%- else -%}\n",
+      "            {{- \"object\" }}\n",
+      "        {%- endif -%}\n",
+      "    {%- else -%}\n",
+      "        {{- \"any\" }}\n",
+      "    {%- endif -%}\n",
+      "{%- endmacro -%}\n",
+      "\n",
+      "{%- macro render_tool_namespace(namespace_name, tools) -%}\n",
+      "    {{- \"## \" + namespace_name + \"\\n\\n\" }}\n",
+      "    {{- \"namespace \" + namespace_name + \" {\\n\\n\" }}\n",
+      "    {%- for tool in tools %}\n",
+      "        {%- set tool = tool.function %}\n",
+      "        {{- \"// \" + tool.description + \"\\n\" }}\n",
+      "        {{- \"type \"+ tool.name + \" = \" }}\n",
+      "        {%- if tool.parameters and tool.parameters.properties %}\n",
+      "            {{- \"(_: {\\n\" }}\n",
+      "            {%- for param_name, param_spec in tool.parameters.properties.items() %}\n",
+      "                {%- if param_spec.description %}\n",
+      "                    {{- \"// \" + param_spec.description + \"\\n\" }}\n",
+      "                {%- endif %}\n",
+      "                {{- param_name }}\n",
+      "                {%- if param_name not in (tool.parameters.required or []) -%}\n",
+      "                    {{- \"?\" }}\n",
+      "                {%- endif -%}\n",
+      "                {{- \": \" }}\n",
+      "                {{- render_typescript_type(param_spec, tool.parameters.required or []) }}\n",
+      "                {%- if param_spec.default is defined -%}\n",
+      "                    {%- if param_spec.enum %}\n",
+      "                        {{- \", // default: \" + param_spec.default }}\n",
+      "                    {%- elif param_spec.oneOf %}\n",
+      "                        {{- \"// default: \" + param_spec.default }}\n",
+      "                    {%- else %}\n",
+      "                        {{- \", // default: \" + param_spec.default|tojson }}\n",
+      "                    {%- endif -%}\n",
+      "                {%- endif -%}\n",
+      "                {%- if not loop.last %}\n",
+      "                    {{- \",\\n\" }}\n",
+      "                {%- else %}\n",
+      "                    {{- \",\\n\" }}\n",
+      "                {%- endif -%}\n",
+      "            {%- endfor %}\n",
+      "            {{- \"}) => any;\\n\\n\" }}\n",
+      "        {%- else -%}\n",
+      "            {{- \"() => any;\\n\\n\" }}\n",
+      "        {%- endif -%}\n",
+      "    {%- endfor %}\n",
+      "    {{- \"} // namespace \" + namespace_name }}\n",
+      "{%- endmacro -%}\n",
+      "\n",
+      "{%- macro render_builtin_tools(browser_tool, python_tool) -%}\n",
+      "    {%- if browser_tool %}\n",
+      "        {{- \"## browser\\n\\n\" }}\n",
+      "        {{- \"// Tool for browsing.\\n\" }}\n",
+      "        {{- \"// The `cursor` appears in brackets before each browsing display: `[{cursor}]`.\\n\" }}\n",
+      "        {{- \"// Cite information from the tool using the following format:\\n\" }}\n",
+      "        {{- \"// `\u3010{cursor}\u2020L{line_start}(-L{line_end})?\u3011`, for example: `\u30106\u2020L9-L11\u3011` or `\u30108\u2020L3\u3011`.\\n\" }}\n",
+      "        {{- \"// Do not quote more than 10 words directly from the tool output.\\n\" }}\n",
+      "        {{- \"// sources=web (default: web)\\n\" }}\n",
+      "        {{- \"namespace browser {\\n\\n\" }}\n",
+      "        {{- \"// Searches for information related to `query` and displays `topn` results.\\n\" }}\n",
+      "        {{- \"type search = (_: {\\n\" }}\n",
+      "        {{- \"query: string,\\n\" }}\n",
+      "        {{- \"topn?: number, // default: 10\\n\" }}\n",
+      "        {{- \"source?: string,\\n\" }}\n",
+      "        {{- \"}) => any;\\n\\n\" }}\n",
+      "        {{- \"// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines.\\n\" }}\n",
+      "        {{- \"// Valid link ids are displayed with the formatting: `\u3010{id}\u2020.*\u3011`.\\n\" }}\n",
+      "        {{- \"// If `cursor` is not provided, the most recent page is implied.\\n\" }}\n",
+      "        {{- \"// If `id` is a string, it is treated as a fully qualified URL associated with `source`.\\n\" }}\n",
+      "        {{- \"// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available.\\n\" }}\n",
+      "        {{- \"// Use this function without `id` to scroll to a new location of an opened page.\\n\" }}\n",
+      "        {{- \"type open = (_: {\\n\" }}\n",
+      "        {{- \"id?: number | string, // default: -1\\n\" }}\n",
+      "        {{- \"cursor?: number, // default: -1\\n\" }}\n",
+      "        {{- \"loc?: number, // default: -1\\n\" }}\n",
+      "        {{- \"num_lines?: number, // default: -1\\n\" }}\n",
+      "        {{- \"view_source?: boolean, // default: false\\n\" }}\n",
+      "        {{- \"source?: string,\\n\" }}\n",
+      "        {{- \"}) => any;\\n\\n\" }}\n",
+      "        {{- \"// Finds exact matches of `pattern` in the current page, or the page given by `cursor`.\\n\" }}\n",
+      "        {{- \"type find = (_: {\\n\" }}\n",
+      "        {{- \"pattern: string,\\n\" }}\n",
+      "        {{- \"cursor?: number, // default: -1\\n\" }}\n",
+      "        {{- \"}) => any;\\n\\n\" }}\n",
+      "        {{- \"} // namespace browser\\n\\n\" }}\n",
+      "    {%- endif -%}\n",
+      "\n",
+      "    {%- if python_tool %}\n",
+      "        {{- \"## python\\n\\n\" }}\n",
+      "        {{- \"Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files).\\n\\n\" }}\n",
+      "        {{- \"When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster.\\n\\n\" }}\n",
+      "    {%- endif -%}\n",
+      "{%- endmacro -%}\n",
+      "\n",
+      "{#- System Message Construction ============================================ #}\n",
+      "{%- macro build_system_message() -%}\n",
+      "    {%- if model_identity is not defined %}\n",
+      "        {%- set model_identity = \"You are ChatGPT, a large language model trained by OpenAI.\" %}\n",
+      "    {%- endif %}\n",
+      "    {{- model_identity + \"\\n\" }}\n",
+      "    {{- \"Knowledge cutoff: 2024-06\\n\" }}\n",
+      "    {{- \"Current date: \" + strftime_now(\"%Y-%m-%d\") + \"\\n\\n\" }}\n",
+      "    {%- if reasoning_effort is not string %}\n",
+      "        {%- set reasoning_effort = \"medium\" %}\n",
+      "    {%- endif %}\n",
+      "    {{- \"Reasoning: \" + reasoning_effort + \"\\n\\n\" }}\n",
+      "    {%- if builtin_tools %}\n",
+      "        {{- \"# Tools\\n\\n\" }}\n",
+      "        {%- set available_builtin_tools = namespace(browser=false, python=false) %}\n",
+      "        {%- for tool in builtin_tools %}\n",
+      "            {%- if tool == \"browser\" %}\n",
+      "                {%- set available_builtin_tools.browser = true %}\n",
+      "            {%- elif tool == \"python\" %}\n",
+      "                {%- set available_builtin_tools.python = true %}\n",
+      "            {%- endif %}\n",
+      "        {%- endfor %}\n",
+      "        {{- render_builtin_tools(available_builtin_tools.browser, available_builtin_tools.python) }}\n",
+      "    {%- endif -%}\n",
+      "    {{- \"# Valid channels: analysis, commentary, final. Channel must be included for every message.\" }}\n",
+      "    {%- if tools -%}\n",
+      "        {{- \"\\nCalls to these tools must go to the commentary channel: 'functions'.\" }}\n",
+      "    {%- endif -%}\n",
+      "{%- endmacro -%}\n",
+      "\n",
+      "{#- Main Template Logic ================================================= #}\n",
+      "{#- Set defaults #}\n",
+      "\n",
+      "{#- Render system message #}\n",
+      "{{- \"<|start|>system<|message|>\" }}\n",
+      "{{- build_system_message() }}\n",
+      "{{- \"<|end|>\" }}\n",
+      "\n",
+      "{#- Extract developer message #}\n",
+      "{%- if messages[0].role == \"developer\" or messages[0].role == \"system\" %}\n",
+      "    {%- set developer_message = messages[0].content %}\n",
+      "    {%- set loop_messages = messages[1:] %}\n",
+      "{%- else %}\n",
+      "    {%- set developer_message = \"\" %}\n",
+      "    {%- set loop_messages = messages %}\n",
+      "{%- endif %}\n",
+      "\n",
+      "{#- Render developer message #}\n",
+      "{%- if developer_message or tools %}\n",
+      "    {{- \"<|start|>developer<|message|>\" }}\n",
+      "    {%- if developer_message %}\n",
+      "        {{- \"# Instructions\\n\\n\" }}\n",
+      "        {{- developer_message }}\n",
+      "        {{- \"\\n\\n\" }}\n",
+      "    {%- endif %}\n",
+      "    {%- if tools -%}\n",
+      "        {{- \"# Tools\\n\\n\" }}\n",
+      "        {{- render_tool_namespace(\"functions\", tools) }}\n",
+      "    {%- endif -%}\n",
+      "    {{- \"<|end|>\" }}\n",
+      "{%- endif %}\n",
+      "\n",
+      "{#- Render messages #}\n",
+      "{%- set last_tool_call = namespace(name=none) %}\n",
+      "{%- for message in loop_messages -%}\n",
+      "    {#- At this point only assistant/user/tool messages should remain #}\n",
+      "    {%- if message.role == 'assistant' -%}\n",
+      "        {#- Checks to ensure the messages are being passed in the format we expect #}\n",
+      "        {%- if \"content\" in message %}\n",
+      "            {%- if \"<|channel|>analysis<|message|>\" in message.content or \"<|channel|>final<|message|>\" in message.content %}\n",
+      "                {{- raise_exception(\"You have passed a message containing <|channel|> tags in the content field. Instead of doing this, you should pass analysis messages (the string between '<|message|>' and '<|end|>') in the 'thinking' field, and final messages (the string between '<|message|>' and '<|end|>') in the 'content' field.\") }}\n",
+      "            {%- endif %}\n",
+      "        {%- endif %}\n",
+      "        {%- if \"thinking\" in message %}\n",
+      "            {%- if \"<|channel|>analysis<|message|>\" in message.thinking or \"<|channel|>final<|message|>\" in message.thinking %}\n",
+      "                {{- raise_exception(\"You have passed a message containing <|channel|> tags in the thinking field. Instead of doing this, you should pass analysis messages (the string between '<|message|>' and '<|end|>') in the 'thinking' field, and final messages (the string between '<|message|>' and '<|end|>') in the 'content' field.\") }}\n",
+      "            {%- endif %}\n",
+      "        {%- endif %}\n",
+      "        {%- if \"tool_calls\" in message %}\n",
+      "            {#- We need very careful handling here - we want to drop the tool call analysis message if the model #}\n",
+      "            {#- has output a later <|final|> message, but otherwise we want to retain it. This is the only case #}\n",
+      "            {#- when we render CoT/analysis messages in inference. #}\n",
+      "            {%- set future_final_message = namespace(found=false) %}\n",
+      "            {%- for future_message in loop_messages[loop.index:] %}\n",
+      "                {%- if future_message.role == 'assistant' and \"tool_calls\" not in future_message %}\n",
+      "                    {%- set future_final_message.found = true %}\n",
+      "                {%- endif %}\n",
+      "            {%- endfor %}\n",
+      "            {#- We assume max 1 tool call per message, and so we infer the tool call name #}\n",
+      "            {#- in \"tool\" messages from the most recent assistant tool call name #}\n",
+      "            {%- set tool_call = message.tool_calls[0] %}\n",
+      "            {%- if tool_call.function %}\n",
+      "                {%- set tool_call = tool_call.function %}\n",
+      "            {%- endif %}\n",
+      "            {%- if message.content and message.thinking %}\n",
+      "                {{- raise_exception(\"Cannot pass both content and thinking in an assistant message with tool calls! Put the analysis message in one or the other, but not both.\") }}\n",
+      "            {%- elif message.content and not future_final_message.found %}\n",
+      "                {{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.content + \"<|end|>\" }}\n",
+      "            {%- elif message.thinking and not future_final_message.found %}\n",
+      "                {{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.thinking + \"<|end|>\" }}\n",
+      "            {%- endif %}\n",
+      "            {{- \"<|start|>assistant to=\" }}\n",
+      "            {{- \"functions.\" + tool_call.name + \"<|channel|>commentary \" }}\n",
+      "            {{- (tool_call.content_type if tool_call.content_type is defined else \"json\") + \"<|message|>\" }}\n",
+      "            {{- tool_call.arguments|tojson }}\n",
+      "            {{- \"<|call|>\" }}\n",
+      "            {%- set last_tool_call.name = tool_call.name %}\n",
+      "        {%- elif loop.last and not add_generation_prompt %}\n",
+      "            {#- Only render the CoT if the final turn is an assistant turn and add_generation_prompt is false #}\n",
+      "            {#- This is a situation that should only occur in training, never in inference. #}\n",
+      "            {%- if \"thinking\" in message %}\n",
+      "                {{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.thinking + \"<|end|>\" }}\n",
+      "            {%- endif %}\n",
+      "            {#- <|return|> indicates the end of generation, but <|end|> does not #}\n",
+      "            {#- <|return|> should never be an input to the model, but we include it as the final token #}\n",
+      "            {#- when training, so the model learns to emit it. #}\n",
+      "            {{- \"<|start|>assistant<|channel|>final<|message|>\" + message.content + \"<|return|>\" }}\n",
+      "        {%- else %}\n",
+      "            {#- CoT is dropped during all previous turns, so we never render it for inference #}\n",
+      "            {{- \"<|start|>assistant<|channel|>final<|message|>\" + message.content + \"<|end|>\" }}\n",
+      "            {%- set last_tool_call.name = none %}\n",
+      "        {%- endif %}\n",
+      "    {%- elif message.role == 'tool' -%}\n",
+      "        {%- if last_tool_call.name is none %}\n",
+      "            {{- raise_exception(\"Message has tool role, but there was no previous assistant message with a tool call!\") }}\n",
+      "        {%- endif %}\n",
+      "        {{- \"<|start|>functions.\" + last_tool_call.name }}\n",
+      "        {{- \" to=assistant<|channel|>commentary<|message|>\" + message.content|tojson + \"<|end|>\" }}\n",
+      "    {%- elif message.role == 'user' -%}\n",
+      "        {{- \"<|start|>user<|message|>\" + message.content + \"<|end|>\" }}\n",
+      "    {%- endif -%}\n",
+      "{%- endfor -%}\n",
+      "\n",
+      "{#- Generation prompt #}\n",
+      "{%- if add_generation_prompt -%}\n",
+      "<|start|>assistant\n",
+      "{%- endif -%}"
+    ]
+  }
+}

examples/server/configs/qwen3.5-0.8b.json

+{
+  "server": {
+    "host": "0.0.0.0",
+    "port": 8000
+  },
+  "model": {
+    "alias": "qwen3.5-0.8b-vl",
+    "from_pretrained": {
+      "repo_id": "lmstudio-community/Qwen3.5-0.8B-GGUF",
+      "filename": "Qwen3.5-0.8B-Q8_0.gguf"
+    },
+    "mtmd": {
+      "mmproj_from_pretrained": {
+        "repo_id": "lmstudio-community/Qwen3.5-0.8B-GGUF",
+        "filename": "mmproj-Qwen3.5-0.8B-BF16.gguf"
+      }
+    },
+    "n_ctx": 32768,
+    "max_output_tokens": 4096,
+    "n_seq_max": 64,
+    "n_batch": 128,
+    "n_ubatch": 128,
+    "threads": 4,
+    "threads_batch": 8,
+    "kv_unified": true,
+    "store_logits": false,
+    "use_mmap": true,
+    "use_mlock": true,
+    "response_schema": {
+      "type": "object",
+      "properties": {
+        "role": {
+          "const": "assistant"
+        },
+        "reasoning_content": {
+          "type": "string",
+          "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:<think>\\n)?(.*?)(?=</think>)"
+        },
+        "content": {
+          "type": "string",
+          "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:(?:<think>\\n)?.*?</think>\\s*)?(.*?)(?=\\s*<tool_call>\\n|<\\|im_end\\|>$|$)"
+        },
+        "tool_calls": {
+          "type": "array",
+          "x-regex-iterator": "<tool_call>\\n(.*?)\\n</tool_call>",
+          "items": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "function"
+              },
+              "function": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "type": "string",
+                    "x-regex": "^<function=([^>\\n]+)>\\n"
+                  },
+                  "arguments": {
+                    "type": "object",
+                    "x-regex": "^<function=[^>\\n]+>\\n(.*?)\\n</function>$",
+                    "x-regex-key-value": "<parameter=(?P<key>[^>\\n]+)>\\n(?P<value>.*?)\\n</parameter>",
+                    "additionalProperties": true
+                  }
+                },
+                "required": [
+                  "name",
+                  "arguments"
+                ]
+              }
+            },
+            "required": [
+              "type",
+              "function"
+            ]
+          }
+        }
+      },
+      "required": [
+        "role"
+      ]
+    },
+    "chat_template": [
+      "{%- set image_count = namespace(value=0) %}\n",
+      "{%- set video_count = namespace(value=0) %}\n",
+      "{%- if enable_thinking is not defined and reasoning_effort is string %}\n",
+      "    {%- set qwen_reasoning_effort = reasoning_effort|lower %}\n",
+      "    {%- if qwen_reasoning_effort in ['none', 'minimal', 'low'] %}\n",
+      "        {%- set enable_thinking = false %}\n",
+      "    {%- elif qwen_reasoning_effort in ['medium', 'high'] %}\n",
+      "        {%- set enable_thinking = true %}\n",
+      "    {%- endif %}\n",
+      "{%- endif %}\n",
+      "{%- set forced_tool_name = none %}\n",
+      "{%- if tool_choice is mapping %}\n",
+      "    {%- if tool_choice.function is defined and tool_choice.function.name is defined %}\n",
+      "        {%- set forced_tool_name = tool_choice.function.name %}\n",
+      "    {%- elif tool_choice.name is defined %}\n",
+      "        {%- set forced_tool_name = tool_choice.name %}\n",
+      "    {%- endif %}\n",
+      "{%- elif function_call is mapping and function_call.name is defined %}\n",
+      "    {%- set forced_tool_name = function_call.name %}\n",
+      "{%- endif %}\n",
+      "{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n",
+      "    {%- if content is string %}\n",
+      "        {{- content }}\n",
+      "    {%- elif content is iterable and content is not mapping %}\n",
+      "        {%- for item in content %}\n",
+      "            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n",
+      "                {%- if is_system_content %}\n",
+      "                    {{- raise_exception('System message cannot contain images.') }}\n",
+      "                {%- endif %}\n",
+      "                {%- if do_vision_count %}\n",
+      "                    {%- set image_count.value = image_count.value + 1 %}\n",
+      "                {%- endif %}\n",
+      "                {%- if add_vision_id %}\n",
+      "                    {{- 'Picture ' ~ image_count.value ~ ': ' }}\n",
+      "                {%- endif %}\n",
+      "                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n",
+      "            {%- elif 'video' in item or item.type == 'video' %}\n",
+      "                {%- if is_system_content %}\n",
+      "                    {{- raise_exception('System message cannot contain videos.') }}\n",
+      "                {%- endif %}\n",
+      "                {%- if do_vision_count %}\n",
+      "                    {%- set video_count.value = video_count.value + 1 %}\n",
+      "                {%- endif %}\n",
+      "                {%- if add_vision_id %}\n",
+      "                    {{- 'Video ' ~ video_count.value ~ ': ' }}\n",
+      "                {%- endif %}\n",
+      "                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n",
+      "            {%- elif 'text' in item %}\n",
+      "                {{- item.text }}\n",
+      "            {%- else %}\n",
+      "                {{- raise_exception('Unexpected item type in content.') }}\n",
+      "            {%- endif %}\n",
+      "        {%- endfor %}\n",
+      "    {%- elif content is none or content is undefined %}\n",
+      "        {{- '' }}\n",
+      "    {%- else %}\n",
+      "        {{- raise_exception('Unexpected content type.') }}\n",
+      "    {%- endif %}\n",
+      "{%- endmacro %}\n",
+      "{%- if not messages %}\n",
+      "    {{- raise_exception('No messages provided.') }}\n",
+      "{%- endif %}\n",
+      "{%- if tools and tools is iterable and tools is not mapping %}\n",
+      "    {{- '<|im_start|>system\\n' }}\n",
+      "    {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n",
+      "    {%- for tool in tools %}\n",
+      "        {{- \"\\n\" }}\n",
+      "        {{- tool | tojson }}\n",
+      "    {%- endfor %}\n",
+      "    {{- \"\\n</tools>\" }}\n",
+      "    {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n",
+      "    {%- if messages[0].role == 'system' %}\n",
+      "        {%- set content = render_content(messages[0].content, false, true)|trim %}\n",
+      "        {%- if content %}\n",
+      "            {{- '\\n\\n' + content }}\n",
+      "        {%- endif %}\n",
+      "    {%- endif %}\n",
+      "    {{- '<|im_end|>\\n' }}\n",
+      "{%- else %}\n",
+      "    {%- if messages[0].role == 'system' %}\n",
+      "        {%- set content = render_content(messages[0].content, false, true)|trim %}\n",
+      "        {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n",
+      "    {%- endif %}\n",
+      "{%- endif %}\n",
+      "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n",
+      "{%- for message in messages[::-1] %}\n",
+      "    {%- set index = (messages|length - 1) - loop.index0 %}\n",
+      "    {%- if ns.multi_step_tool and message.role == \"user\" %}\n",
+      "        {%- set content = render_content(message.content, false)|trim %}\n",
+      "        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n",
+      "            {%- set ns.multi_step_tool = false %}\n",
+      "            {%- set ns.last_query_index = index %}\n",
+      "        {%- endif %}\n",
+      "    {%- endif %}\n",
+      "{%- endfor %}\n",
+      "{%- if ns.multi_step_tool %}\n",
+      "    {{- raise_exception('No user query found in messages.') }}\n",
+      "{%- endif %}\n",
+      "{%- for message in messages %}\n",
+      "    {%- set content = render_content(message.content, true)|trim %}\n",
+      "    {%- if message.role == \"system\" %}\n",
+      "        {%- if not loop.first %}\n",
+      "            {{- raise_exception('System message must be at the beginning.') }}\n",
+      "        {%- endif %}\n",
+      "    {%- elif message.role == \"user\" %}\n",
+      "        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n",
+      "    {%- elif message.role == \"assistant\" %}\n",
+      "        {%- set reasoning_content = '' %}\n",
+      "        {%- if message.reasoning_content is string %}\n",
+      "            {%- set reasoning_content = message.reasoning_content %}\n",
+      "        {%- else %}\n",
+      "            {%- if '</think>' in content %}\n",
+      "                {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n",
+      "                {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n",
+      "            {%- endif %}\n",
+      "        {%- endif %}\n",
+      "        {%- set reasoning_content = reasoning_content|trim %}\n",
+      "        {%- if loop.index0 > ns.last_query_index %}\n",
+      "            {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n",
+      "        {%- else %}\n",
+      "            {{- '<|im_start|>' + message.role + '\\n' + content }}\n",
+      "        {%- endif %}\n",
+      "        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n",
+      "            {%- for tool_call in message.tool_calls %}\n",
+      "                {%- if tool_call.function is defined %}\n",
+      "                    {%- set tool_call = tool_call.function %}\n",
+      "                {%- endif %}\n",
+      "                {%- if loop.first %}\n",
+      "                    {%- if content|trim %}\n",
+      "                        {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n",
+      "                    {%- else %}\n",
+      "                        {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n",
+      "                    {%- endif %}\n",
+      "                {%- else %}\n",
+      "                    {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n",
+      "                {%- endif %}\n",
+      "                {%- if tool_call.arguments is defined %}\n",
+      "                    {%- set arguments = tool_call.arguments | from_json if tool_call.arguments is string else tool_call.arguments %}\n",
+      "                    {%- for args_name, args_value in arguments|items %}\n",
+      "                        {{- '<parameter=' + args_name + '>\\n' }}\n",
+      "                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n",
+      "                        {{- args_value }}\n",
+      "                        {{- '\\n</parameter>\\n' }}\n",
+      "                    {%- endfor %}\n",
+      "                {%- endif %}\n",
+      "                {{- '</function>\\n</tool_call>' }}\n",
+      "            {%- endfor %}\n",
+      "        {%- endif %}\n",
+      "        {{- '<|im_end|>\\n' }}\n",
+      "    {%- elif message.role == \"tool\" %}\n",
+      "        {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n",
+      "            {{- '<|im_start|>user' }}\n",
+      "        {%- endif %}\n",
+      "        {{- '\\n<tool_response>\\n' }}\n",
+      "        {{- content }}\n",
+      "        {{- '\\n</tool_response>' }}\n",
+      "        {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n",
+      "            {{- '<|im_end|>\\n' }}\n",
+      "        {%- elif loop.last %}\n",
+      "            {{- '<|im_end|>\\n' }}\n",
+      "        {%- endif %}\n",
+      "    {%- else %}\n",
+      "        {{- raise_exception('Unexpected message role.') }}\n",
+      "    {%- endif %}\n",
+      "{%- endfor %}\n",
+      "{%- if add_generation_prompt %}\n",
+      "    {{- '<|im_start|>assistant\\n' }}\n",
+      "    {%- if forced_tool_name %}\n",
+      "        {{- '<tool_call>\\n<function=' + forced_tool_name + '>\\n' }}\n",
+      "    {%- elif enable_thinking is defined and enable_thinking is false %}\n",
+      "        {{- '<think>\\n\\n</think>\\n\\n' }}\n",
+      "    {%- else %}\n",
+      "        {{- '<think>\\n' }}\n",
+      "    {%- endif %}\n",
+      "{%- endif %}"
+    ]
+  }
+}

examples/server/configs/qwen3.6-27b.json

+{
+  "server": {
+    "host": "0.0.0.0",
+    "port": 8000
+  },
+  "model": {
+    "alias": "qwen3.6-27b",
+    "from_pretrained": {
+      "repo_id": "unsloth/Qwen3.6-27B-GGUF",
+      "filename": "Qwen3.6-27B-Q8_0.gguf"
+    },
+    "mtmd": {
+      "mmproj_from_pretrained": {
+        "repo_id": "unsloth/Qwen3.6-27B-GGUF",
+        "filename": "mmproj-BF16.gguf"
+      }
+    },
+    "n_ctx": 32768,
+    "max_output_tokens": 4096,
+    "n_seq_max": 8,
+    "n_batch": 512,
+    "n_ubatch": 512,
+    "threads": 8,
+    "threads_batch": 8,
+    "kv_unified": true,
+    "store_logits": false,
+    "use_mmap": true,
+    "use_mlock": false,
+    "n_gpu_layers": -1,
+    "flash_attn": true,
+    "response_schema": {
+      "type": "object",
+      "properties": {
+        "role": {
+          "const": "assistant"
+        },
+        "reasoning_content": {
+          "type": "string",
+          "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:<think>\\n)?(.*?)(?=</think>)"
+        },
+        "content": {
+          "type": "string",
+          "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:(?:<think>\\n)?.*?</think>\\s*)?(.*?)(?=\\s*<tool_call>\\n|<\\|im_end\\|>$|$)"
+        },
+        "tool_calls": {
+          "type": "array",
+          "x-regex-iterator": "<tool_call>\\n(.*?)\\n</tool_call>",
+          "items": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "function"
+              },
+              "function": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "type": "string",
+                    "x-regex": "^<function=([^>\\n]+)>\\n"
+                  },
+                  "arguments": {
+                    "type": "object",
+                    "x-regex": "^<function=[^>\\n]+>\\n(.*?)\\n</function>$",
+                    "x-regex-key-value": "<parameter=(?P<key>[^>\\n]+)>\\n(?P<value>.*?)\\n</parameter>",
+                    "additionalProperties": true
+                  }
+                },
+                "required": [
+                  "name",
+                  "arguments"
+                ]
+              }
+            },
+            "required": [
+              "type",
+              "function"
+            ]
+          }
+        }
+      },
+      "required": [
+        "role"
+      ]
+    },
+    "chat_template": [
+      "{%- set image_count = namespace(value=0) %}\n",
+      "{%- set video_count = namespace(value=0) %}\n",
+      "{%- if enable_thinking is not defined and reasoning_effort is string %}\n",
+      "    {%- set qwen_reasoning_effort = reasoning_effort|lower %}\n",
+      "    {%- if qwen_reasoning_effort in ['none', 'minimal', 'low'] %}\n",
+      "        {%- set enable_thinking = false %}\n",
+      "    {%- elif qwen_reasoning_effort in ['medium', 'high'] %}\n",
+      "        {%- set enable_thinking = true %}\n",
+      "    {%- endif %}\n",
+      "{%- endif %}\n",
+      "{%- set forced_tool_name = none %}\n",
+      "{%- if tool_choice is mapping %}\n",
+      "    {%- if tool_choice.function is defined and tool_choice.function.name is defined %}\n",
+      "        {%- set forced_tool_name = tool_choice.function.name %}\n",
+      "    {%- elif tool_choice.name is defined %}\n",
+      "        {%- set forced_tool_name = tool_choice.name %}\n",
+      "    {%- endif %}\n",
+      "{%- elif function_call is mapping and function_call.name is defined %}\n",
+      "    {%- set forced_tool_name = function_call.name %}\n",
+      "{%- endif %}\n",
+      "{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n",
+      "    {%- if content is string %}\n",
+      "        {{- content }}\n",
+      "    {%- elif content is iterable and content is not mapping %}\n",
+      "        {%- for item in content %}\n",
+      "            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n",
+      "                {%- if is_system_content %}\n",
+      "                    {{- raise_exception('System message cannot contain images.') }}\n",
+      "                {%- endif %}\n",
+      "                {%- if do_vision_count %}\n",
+      "                    {%- set image_count.value = image_count.value + 1 %}\n",
+      "                {%- endif %}\n",
+      "                {%- if add_vision_id %}\n",
+      "                    {{- 'Picture ' ~ image_count.value ~ ': ' }}\n",
+      "                {%- endif %}\n",
+      "                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n",
+      "            {%- elif 'video' in item or item.type == 'video' %}\n",
+      "                {%- if is_system_content %}\n",
+      "                    {{- raise_exception('System message cannot contain videos.') }}\n",
+      "                {%- endif %}\n",
+      "                {%- if do_vision_count %}\n",
+      "                    {%- set video_count.value = video_count.value + 1 %}\n",
+      "                {%- endif %}\n",
+      "                {%- if add_vision_id %}\n",
+      "                    {{- 'Video ' ~ video_count.value ~ ': ' }}\n",
+      "                {%- endif %}\n",
+      "                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n",
+      "            {%- elif 'text' in item %}\n",
+      "                {{- item.text }}\n",
+      "            {%- else %}\n",
+      "                {{- raise_exception('Unexpected item type in content.') }}\n",
+      "            {%- endif %}\n",
+      "        {%- endfor %}\n",
+      "    {%- elif content is none or content is undefined %}\n",
+      "        {{- '' }}\n",
+      "    {%- else %}\n",
+      "        {{- raise_exception('Unexpected content type.') }}\n",
+      "    {%- endif %}\n",
+      "{%- endmacro %}\n",
+      "{%- if not messages %}\n",
+      "    {{- raise_exception('No messages provided.') }}\n",
+      "{%- endif %}\n",
+      "{%- if tools and tools is iterable and tools is not mapping %}\n",
+      "    {{- '<|im_start|>system\\n' }}\n",
+      "    {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n",
+      "    {%- for tool in tools %}\n",
+      "        {{- \"\\n\" }}\n",
+      "        {{- tool | tojson }}\n",
+      "    {%- endfor %}\n",
+      "    {{- \"\\n</tools>\" }}\n",
+      "    {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n",
+      "    {%- if messages[0].role == 'system' %}\n",
+      "        {%- set content = render_content(messages[0].content, false, true)|trim %}\n",
+      "        {%- if content %}\n",
+      "            {{- '\\n\\n' + content }}\n",
+      "        {%- endif %}\n",
+      "    {%- endif %}\n",
+      "    {{- '<|im_end|>\\n' }}\n",
+      "{%- else %}\n",
+      "    {%- if messages[0].role == 'system' %}\n",
+      "        {%- set content = render_content(messages[0].content, false, true)|trim %}\n",
+      "        {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n",
+      "    {%- endif %}\n",
+      "{%- endif %}\n",
+      "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n",
+      "{%- for message in messages[::-1] %}\n",
+      "    {%- set index = (messages|length - 1) - loop.index0 %}\n",
+      "    {%- if ns.multi_step_tool and message.role == \"user\" %}\n",
+      "        {%- set content = render_content(message.content, false)|trim %}\n",
+      "        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n",
+      "            {%- set ns.multi_step_tool = false %}\n",
+      "            {%- set ns.last_query_index = index %}\n",
+      "        {%- endif %}\n",
+      "    {%- endif %}\n",
+      "{%- endfor %}\n",
+      "{%- if ns.multi_step_tool %}\n",
+      "    {{- raise_exception('No user query found in messages.') }}\n",
+      "{%- endif %}\n",
+      "{%- for message in messages %}\n",
+      "    {%- set content = render_content(message.content, true)|trim %}\n",
+      "    {%- if message.role == \"system\" %}\n",
+      "        {%- if not loop.first %}\n",
+      "            {{- raise_exception('System message must be at the beginning.') }}\n",
+      "        {%- endif %}\n",
+      "    {%- elif message.role == \"user\" %}\n",
+      "        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n",
+      "    {%- elif message.role == \"assistant\" %}\n",
+      "        {%- set reasoning_content = '' %}\n",
+      "        {%- if message.reasoning_content is string %}\n",
+      "            {%- set reasoning_content = message.reasoning_content %}\n",
+      "        {%- else %}\n",
+      "            {%- if '</think>' in content %}\n",
+      "                {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n",
+      "                {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n",
+      "            {%- endif %}\n",
+      "        {%- endif %}\n",
+      "        {%- set reasoning_content = reasoning_content|trim %}\n",
+      "        {%- if (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) %}\n",
+      "            {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n",
+      "        {%- else %}\n",
+      "            {{- '<|im_start|>' + message.role + '\\n' + content }}\n",
+      "        {%- endif %}\n",
+      "        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n",
+      "            {%- for tool_call in message.tool_calls %}\n",
+      "                {%- if tool_call.function is defined %}\n",
+      "                    {%- set tool_call = tool_call.function %}\n",
+      "                {%- endif %}\n",
+      "                {%- if loop.first %}\n",
+      "                    {%- if content|trim %}\n",
+      "                        {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n",
+      "                    {%- else %}\n",
+      "                        {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n",
+      "                    {%- endif %}\n",
+      "                {%- else %}\n",
+      "                    {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n",
+      "                {%- endif %}\n",
+      "                {%- if tool_call.arguments is defined %}\n",
+      "                    {%- set arguments = tool_call.arguments | from_json if tool_call.arguments is string else tool_call.arguments %}\n",
+      "                    {%- for args_name, args_value in arguments|items %}\n",
+      "                        {{- '<parameter=' + args_name + '>\\n' }}\n",
+      "                        {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}\n",
+      "                        {{- args_value }}\n",
+      "                        {{- '\\n</parameter>\\n' }}\n",
+      "                    {%- endfor %}\n",
+      "                {%- endif %}\n",
+      "                {{- '</function>\\n</tool_call>' }}\n",
+      "            {%- endfor %}\n",
+      "        {%- endif %}\n",
+      "        {{- '<|im_end|>\\n' }}\n",
+      "    {%- elif message.role == \"tool\" %}\n",
+      "        {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n",
+      "            {{- '<|im_start|>user' }}\n",
+      "        {%- endif %}\n",
+      "        {{- '\\n<tool_response>\\n' }}\n",
+      "        {{- content }}\n",
+      "        {{- '\\n</tool_response>' }}\n",
+      "        {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n",
+      "            {{- '<|im_end|>\\n' }}\n",
+      "        {%- elif loop.last %}\n",
+      "            {{- '<|im_end|>\\n' }}\n",
+      "        {%- endif %}\n",
+      "    {%- else %}\n",
+      "        {{- raise_exception('Unexpected message role.') }}\n",
+      "    {%- endif %}\n",
+      "{%- endfor %}\n",
+      "{%- if add_generation_prompt %}\n",
+      "    {{- '<|im_start|>assistant\\n' }}\n",
+      "    {%- if forced_tool_name %}\n",
+      "        {{- '<tool_call>\\n<function=' + forced_tool_name + '>\\n' }}\n",
+      "    {%- elif enable_thinking is defined and enable_thinking is false %}\n",
+      "        {{- '<think>\\n\\n</think>\\n\\n' }}\n",
+      "    {%- else %}\n",
+      "        {{- '<think>\\n' }}\n",
+      "    {%- endif %}\n",
+      "{%- endif %}"
+    ]
+  }
+}

examples/server/configs/qwen3.6-35b-a3b.json

+{
+  "server": {
+    "host": "0.0.0.0",
+    "port": 8000
+  },
+  "model": {
+    "alias": "qwen3.6-35b-a3b",
+    "from_pretrained": {
+      "repo_id": "unsloth/Qwen3.6-35B-A3B-GGUF",
+      "filename": "Qwen3.6-35B-A3B-Q8_0.gguf"
+    },
+    "mtmd": {
+      "mmproj_from_pretrained": {
+        "repo_id": "unsloth/Qwen3.6-35B-A3B-GGUF",
+        "filename": "mmproj-BF16.gguf"
+      }
+    },
+    "n_ctx": 32768,
+    "max_output_tokens": 4096,
+    "n_seq_max": 8,
+    "n_batch": 512,
+    "n_ubatch": 512,
+    "threads": 8,
+    "threads_batch": 8,
+    "kv_unified": true,
+    "store_logits": false,
+    "use_mmap": true,
+    "use_mlock": false,
+    "n_gpu_layers": -1,
+    "flash_attn": true,
+    "response_schema": {
+      "type": "object",
+      "properties": {
+        "role": {
+          "const": "assistant"
+        },
+        "reasoning_content": {
+          "type": "string",
+          "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:<think>\\n)?(.*?)(?=</think>)"
+        },
+        "content": {
+          "type": "string",
+          "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:(?:<think>\\n)?.*?</think>\\s*)?(.*?)(?=\\s*<tool_call>\\n|<\\|im_end\\|>$|$)"
+        },
+        "tool_calls": {
+          "type": "array",
+          "x-regex-iterator": "<tool_call>\\n(.*?)\\n</tool_call>",
+          "items": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "function"
+              },
+              "function": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "type": "string",
+                    "x-regex": "^<function=([^>\\n]+)>\\n"
+                  },
+                  "arguments": {
+                    "type": "object",
+                    "x-regex": "^<function=[^>\\n]+>\\n(.*?)\\n</function>$",
+                    "x-regex-key-value": "<parameter=(?P<key>[^>\\n]+)>\\n(?P<value>.*?)\\n</parameter>",
+                    "additionalProperties": true
+                  }
+                },
+                "required": [
+                  "name",
+                  "arguments"
+                ]
+              }
+            },
+            "required": [
+              "type",
+              "function"
+            ]
+          }
+        }
+      },
+      "required": [
+        "role"
+      ]
+    },
+    "chat_template": [
+      "{%- set image_count = namespace(value=0) %}\n",
+      "{%- set video_count = namespace(value=0) %}\n",
+      "{%- if enable_thinking is not defined and reasoning_effort is string %}\n",
+      "    {%- set qwen_reasoning_effort = reasoning_effort|lower %}\n",
+      "    {%- if qwen_reasoning_effort in ['none', 'minimal', 'low'] %}\n",
+      "        {%- set enable_thinking = false %}\n",
+      "    {%- elif qwen_reasoning_effort in ['medium', 'high'] %}\n",
+      "        {%- set enable_thinking = true %}\n",
+      "    {%- endif %}\n",
+      "{%- endif %}\n",
+      "{%- set forced_tool_name = none %}\n",
+      "{%- if tool_choice is mapping %}\n",
+      "    {%- if tool_choice.function is defined and tool_choice.function.name is defined %}\n",
+      "        {%- set forced_tool_name = tool_choice.function.name %}\n",
+      "    {%- elif tool_choice.name is defined %}\n",
+      "        {%- set forced_tool_name = tool_choice.name %}\n",
+      "    {%- endif %}\n",
+      "{%- elif function_call is mapping and function_call.name is defined %}\n",
+      "    {%- set forced_tool_name = function_call.name %}\n",
+      "{%- endif %}\n",
+      "{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n",
+      "    {%- if content is string %}\n",
+      "        {{- content }}\n",
+      "    {%- elif content is iterable and content is not mapping %}\n",
+      "        {%- for item in content %}\n",
+      "            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n",
+      "                {%- if is_system_content %}\n",
+      "                    {{- raise_exception('System message cannot contain images.') }}\n",
+      "                {%- endif %}\n",
+      "                {%- if do_vision_count %}\n",
+      "                    {%- set image_count.value = image_count.value + 1 %}\n",
+      "                {%- endif %}\n",
+      "                {%- if add_vision_id %}\n",
+      "                    {{- 'Picture ' ~ image_count.value ~ ': ' }}\n",
+      "                {%- endif %}\n",
+      "                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n",
+      "            {%- elif 'video' in item or item.type == 'video' %}\n",
+      "                {%- if is_system_content %}\n",
+      "                    {{- raise_exception('System message cannot contain videos.') }}\n",
+      "                {%- endif %}\n",
+      "                {%- if do_vision_count %}\n",
+      "                    {%- set video_count.value = video_count.value + 1 %}\n",
+      "                {%- endif %}\n",
+      "                {%- if add_vision_id %}\n",
+      "                    {{- 'Video ' ~ video_count.value ~ ': ' }}\n",
+      "                {%- endif %}\n",
+      "                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n",
+      "            {%- elif 'text' in item %}\n",
+      "                {{- item.text }}\n",
+      "            {%- else %}\n",
+      "                {{- raise_exception('Unexpected item type in content.') }}\n",
+      "            {%- endif %}\n",
+      "        {%- endfor %}\n",
+      "    {%- elif content is none or content is undefined %}\n",
+      "        {{- '' }}\n",
+      "    {%- else %}\n",
+      "        {{- raise_exception('Unexpected content type.') }}\n",
+      "    {%- endif %}\n",
+      "{%- endmacro %}\n",
+      "{%- if not messages %}\n",
+      "    {{- raise_exception('No messages provided.') }}\n",
+      "{%- endif %}\n",
+      "{%- if tools and tools is iterable and tools is not mapping %}\n",
+      "    {{- '<|im_start|>system\\n' }}\n",
+      "    {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n",
+      "    {%- for tool in tools %}\n",
+      "        {{- \"\\n\" }}\n",
+      "        {{- tool | tojson }}\n",
+      "    {%- endfor %}\n",
+      "    {{- \"\\n</tools>\" }}\n",
+      "    {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n",
+      "    {%- if messages[0].role == 'system' %}\n",
+      "        {%- set content = render_content(messages[0].content, false, true)|trim %}\n",
+      "        {%- if content %}\n",
+      "            {{- '\\n\\n' + content }}\n",
+      "        {%- endif %}\n",
+      "    {%- endif %}\n",
+      "    {{- '<|im_end|>\\n' }}\n",
+      "{%- else %}\n",
+      "    {%- if messages[0].role == 'system' %}\n",
+      "        {%- set content = render_content(messages[0].content, false, true)|trim %}\n",
+      "        {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n",
+      "    {%- endif %}\n",
+      "{%- endif %}\n",
+      "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n",
+      "{%- for message in messages[::-1] %}\n",
+      "    {%- set index = (messages|length - 1) - loop.index0 %}\n",
+      "    {%- if ns.multi_step_tool and message.role == \"user\" %}\n",
+      "        {%- set content = render_content(message.content, false)|trim %}\n",
+      "        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n",
+      "            {%- set ns.multi_step_tool = false %}\n",
+      "            {%- set ns.last_query_index = index %}\n",
+      "        {%- endif %}\n",
+      "    {%- endif %}\n",
+      "{%- endfor %}\n",
+      "{%- if ns.multi_step_tool %}\n",
+      "    {{- raise_exception('No user query found in messages.') }}\n",
+      "{%- endif %}\n",
+      "{%- for message in messages %}\n",
+      "    {%- set content = render_content(message.content, true)|trim %}\n",
+      "    {%- if message.role == \"system\" %}\n",
+      "        {%- if not loop.first %}\n",
+      "            {{- raise_exception('System message must be at the beginning.') }}\n",
+      "        {%- endif %}\n",
+      "    {%- elif message.role == \"user\" %}\n",
+      "        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n",
+      "    {%- elif message.role == \"assistant\" %}\n",
+      "        {%- set reasoning_content = '' %}\n",
+      "        {%- if message.reasoning_content is string %}\n",
+      "            {%- set reasoning_content = message.reasoning_content %}\n",
+      "        {%- else %}\n",
+      "            {%- if '</think>' in content %}\n",
+      "                {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n",
+      "                {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n",
+      "            {%- endif %}\n",
+      "        {%- endif %}\n",
+      "        {%- set reasoning_content = reasoning_content|trim %}\n",
+      "        {%- if (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) %}\n",
+      "            {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n",
+      "        {%- else %}\n",
+      "            {{- '<|im_start|>' + message.role + '\\n' + content }}\n",
+      "        {%- endif %}\n",
+      "        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n",
+      "            {%- for tool_call in message.tool_calls %}\n",
+      "                {%- if tool_call.function is defined %}\n",
+      "                    {%- set tool_call = tool_call.function %}\n",
+      "                {%- endif %}\n",
+      "                {%- if loop.first %}\n",
+      "                    {%- if content|trim %}\n",
+      "                        {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n",
+      "                    {%- else %}\n",
+      "                        {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n",
+      "                    {%- endif %}\n",
+      "                {%- else %}\n",
+      "                    {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n",
+      "                {%- endif %}\n",
+      "                {%- if tool_call.arguments is defined %}\n",
+      "                    {%- set arguments = tool_call.arguments | from_json if tool_call.arguments is string else tool_call.arguments %}\n",
+      "                    {%- for args_name, args_value in arguments|items %}\n",
+      "                        {{- '<parameter=' + args_name + '>\\n' }}\n",
+      "                        {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}\n",
+      "                        {{- args_value }}\n",
+      "                        {{- '\\n</parameter>\\n' }}\n",
+      "                    {%- endfor %}\n",
+      "                {%- endif %}\n",
+      "                {{- '</function>\\n</tool_call>' }}\n",
+      "            {%- endfor %}\n",
+      "        {%- endif %}\n",
+      "        {{- '<|im_end|>\\n' }}\n",
+      "    {%- elif message.role == \"tool\" %}\n",
+      "        {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n",
+      "            {{- '<|im_start|>user' }}\n",
+      "        {%- endif %}\n",
+      "        {{- '\\n<tool_response>\\n' }}\n",
+      "        {{- content }}\n",
+      "        {{- '\\n</tool_response>' }}\n",
+      "        {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n",
+      "            {{- '<|im_end|>\\n' }}\n",
+      "        {%- elif loop.last %}\n",
+      "            {{- '<|im_end|>\\n' }}\n",
+      "        {%- endif %}\n",
+      "    {%- else %}\n",
+      "        {{- raise_exception('Unexpected message role.') }}\n",
+      "    {%- endif %}\n",
+      "{%- endfor %}\n",
+      "{%- if add_generation_prompt %}\n",
+      "    {{- '<|im_start|>assistant\\n' }}\n",
+      "    {%- if forced_tool_name %}\n",
+      "        {{- '<tool_call>\\n<function=' + forced_tool_name + '>\\n' }}\n",
+      "    {%- elif enable_thinking is defined and enable_thinking is false %}\n",
+      "        {{- '<think>\\n\\n</think>\\n\\n' }}\n",
+      "    {%- else %}\n",
+      "        {{- '<think>\\n' }}\n",
+      "    {%- endif %}\n",
+      "{%- endif %}"
+    ]
+  }
+}

examples/server/README.md

+# Server Example
+
+This example is an updated OpenAI-compatible web server that depends only on the low-level C bindings.
+It supports batched inference, prompt caching, response parsing, `/v1/responses`, disk sequence caching, MTP, LoRA, and multimodal image/audio inputs.
+
+## Setup
+
+The server is a [`uv` inline script](https://docs.astral.sh/uv/guides/scripts/), so `uv` can create the script environment and install the Python dependencies automatically.
+
+```bash
+cd examples/server
+uv run --script server.py -C configs/qwen3.5-0.8b.json
+```
+
+Use `uv run --extra-index-url` to pull a pre-built `llama-cpp-python` binary wheel instead of building from source.
+
+```bash
+cd examples/server
+uv run \
+  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu \
+  --script server.py -C configs/qwen3.5-0.8b.json
+```
+
+Pick the wheel index that matches the backend you want.
+
+| Backend | Wheel index |
+| --- | --- |
+| CPU | `https://abetlen.github.io/llama-cpp-python/whl/cpu` |
+| CUDA 11.8 | `https://abetlen.github.io/llama-cpp-python/whl/cu118` |
+| CUDA 12.1 | `https://abetlen.github.io/llama-cpp-python/whl/cu121` |
+| CUDA 12.2 | `https://abetlen.github.io/llama-cpp-python/whl/cu122` |
+| CUDA 12.3 | `https://abetlen.github.io/llama-cpp-python/whl/cu123` |
+| CUDA 12.4 | `https://abetlen.github.io/llama-cpp-python/whl/cu124` |
+| CUDA 12.5 | `https://abetlen.github.io/llama-cpp-python/whl/cu125` |
+| CUDA 13.0 | `https://abetlen.github.io/llama-cpp-python/whl/cu130` |
+| CUDA 13.2 | `https://abetlen.github.io/llama-cpp-python/whl/cu132` |
+| Metal | `https://abetlen.github.io/llama-cpp-python/whl/metal` |
+| ROCm | `https://abetlen.github.io/llama-cpp-python/whl/rocm72` |
+| Vulkan | `https://abetlen.github.io/llama-cpp-python/whl/vulkan` |
+
+See the repository installation section for the full [pre-built wheel requirements](../../README.md#supported-backends).
+
+## Model Configs
+
+The smallest checked-in example uses Qwen3.5 0.8B so the server can be started on a normal development machine.
+
+| Config | Model | Notes |
+| --- | --- | --- |
+| [`configs/qwen3.5-0.8b.json`](configs/qwen3.5-0.8b.json) | [`lmstudio-community/Qwen3.5-0.8B-GGUF`](https://huggingface.co/lmstudio-community/Qwen3.5-0.8B-GGUF) | Default small multimodal example. |
+| [`configs/gemma-4-12b-it-qat.json`](configs/gemma-4-12b-it-qat.json) | [`unsloth/gemma-4-12B-it-qat-GGUF`](https://huggingface.co/unsloth/gemma-4-12B-it-qat-GGUF) | Larger Gemma 4 QAT multimodal config with projector. |
+| [`configs/qwen3.6-27b.json`](configs/qwen3.6-27b.json) | [`unsloth/Qwen3.6-27B-GGUF`](https://huggingface.co/unsloth/Qwen3.6-27B-GGUF) | Larger Qwen3.6 multimodal config. |
+| [`configs/qwen3.6-35b-a3b.json`](configs/qwen3.6-35b-a3b.json) | [`unsloth/Qwen3.6-35B-A3B-GGUF`](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF) | Larger Qwen3.6 MoE multimodal config. |
+| [`configs/gpt-oss-120b.json`](configs/gpt-oss-120b.json) | [`ggml-org/gpt-oss-120b-GGUF`](https://huggingface.co/ggml-org/gpt-oss-120b-GGUF) | Large text-only split-GGUF config. |
+
+The larger model configs default to `n_gpu_layers: -1` and `flash_attn: true`.
+
+## Client Examples
+
+Point an OpenAI-compatible client at the local `/v1` base URL.
+
+### Chat Completions
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://127.0.0.1:8000/v1", api_key="not-used")
+
+response = client.chat.completions.create(
+    model="qwen3.5-0.8b-vl",
+    messages=[{"role": "user", "content": "What is the capital of France?"}],
+)
+print(response.choices[0].message.content)
+```
+
+### Responses API
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://127.0.0.1:8000/v1", api_key="not-used")
+
+response = client.responses.create(
+    model="qwen3.5-0.8b-vl",
+    input="Write one sentence about why prefix caching helps batched inference.",
+)
+print(response.output_text)
+```
+
+## API Surface
+
+| Endpoint | Purpose | Reference |
+| --- | --- | --- |
+| `POST /v1/completions` | Legacy text completions with streaming, stop sequences, logprobs, penalties, seeds, and grammar-backed JSON output. | [OpenAI Completions API](https://platform.openai.com/docs/api-reference/completions) |
+| `POST /v1/chat/completions` | Chat completions with streaming, tools, forced tool choice, reasoning parsing, multimodal content parts, and structured response parsing. | [OpenAI Chat API](https://platform.openai.com/docs/api-reference/chat) |
+| `POST /v1/responses` | Stateless Responses API compatibility for clients that use response items and response events. | [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) |
+| `WS /v1/responses` | Stateful websocket Responses transport with per-connection `previous_response_id` replay. | [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) |
+| `GET /v1/models` | Returns the configured model alias. | [OpenAI Models API](https://platform.openai.com/docs/api-reference/models) |
+| `GET /healthz` | Returns a simple typed health response. | |
+| `GET /metrics` | Exposes scheduler, cache, draft, and model metrics in Prometheus text format. | [Prometheus exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/) |
+
+## Config Overview
+
+Config files have three top-level sections.
+
+```json
+{
+  "server": {},
+  "model": {},
+  "disk_cache": {}
+}
+```
+
+| Section | Required | Purpose |
+| --- | --- | --- |
+| `server` | No | Uvicorn host and port settings. |
+| `model` | Yes | Model source, llama.cpp runtime settings, chat formatting, LoRA, MTMD, draft decoding, and output parsing. |
+| `disk_cache` | No | Optional serialized sequence cache for repeated prompt prefixes. |
+
+## `server`
+
+Use `server.host` and `server.port` to choose the bind address.
+
+```json
+{
+  "server": {
+    "host": "0.0.0.0",
+    "port": 8000
+  }
+}
+```
+
+| Field | Default | Notes |
+| --- | --- | --- |
+| `host` | `127.0.0.1` | Use `0.0.0.0` to expose the server on the network. |
+| `port` | `8000` | Passed directly to `uvicorn.run()`. |
+
+## `model` Source
+
+Load a local GGUF with `path` or download a GGUF from Hugging Face with `from_pretrained`.
+
+```json
+{
+  "model": {
+    "alias": "qwen3.5-0.8b-vl",
+    "from_pretrained": {
+      "repo_id": "lmstudio-community/Qwen3.5-0.8B-GGUF",
+      "filename": "Qwen3.5-0.8B-Q8_0.gguf"
+    }
+  }
+}
+```
+
+| Field | Notes |
+| --- | --- |
+| `path` | Local GGUF path. |
+| `from_pretrained.repo_id` | Hugging Face model repository. |
+| `from_pretrained.filename` | File name or glob pattern for the GGUF. |
+| `from_pretrained.additional_files` | Extra files to download from the same repository. |
+| `from_pretrained.cache_dir` | Optional Hugging Face cache directory. |
+| `alias` | Model id returned by `/v1/models` and used by OpenAI-compatible clients. |
+
+See the [Hugging Face Hub download guide](https://huggingface.co/docs/huggingface_hub/guides/download) for cache behavior and repository file resolution.
+
+## llama.cpp Runtime Settings
+
+Most model runtime fields map to `llama_model_params` or `llama_context_params` in [`llama.h`](https://github.com/ggml-org/llama.cpp/blob/master/include/llama.h).
+
+```json
+{
+  "model": {
+    "n_ctx": 32768,
+    "n_seq_max": 64,
+    "n_batch": 128,
+    "n_ubatch": 128,
+    "threads": 4,
+    "threads_batch": 8,
+    "kv_unified": true,
+    "use_mmap": true,
+    "use_mlock": true
+  }
+}
+```
+
+| Field | Purpose |
+| --- | --- |
+| `n_ctx` | Total context size. |
+| `n_seq_max` | Maximum number of concurrent llama.cpp sequence ids. |
+| `n_batch` | Logical batch capacity. |
+| `n_ubatch` | Physical microbatch capacity. |
+| `threads` | Decode thread count. |
+| `threads_batch` | Prefill and batch thread count. |
+| `kv_unified` | Selects unified or per-sequence memory layout. |
+| `store_logits` | Keeps logits after decode when needed by sampling or diagnostics. |
+| `use_mmap` | Memory maps model weights. |
+| `use_mlock` | Attempts to lock model pages into RAM. |
+
+GPU and backend-related fields are passed through to llama.cpp when set.
+
+```json
+{
+  "model": {
+    "n_gpu_layers": -1,
+    "split_mode": 1,
+    "main_gpu": 0,
+    "tensor_split": [1.0],
+    "flash_attn": true,
+    "offload_kqv": true,
+    "op_offload": true
+  }
+}
+```
+
+## Chat Template
+
+`model.chat_template` is a Jinja chat template compatible with the style used by [Hugging Face chat templates](https://huggingface.co/docs/transformers/chat_templating).
+
+```json
+{
+  "model": {
+    "chat_template": "{{ bos_token }}{{ messages[0].content }}{{ eos_token }}"
+  }
+}
+```
+
+Use an array of strings when the template is too large to read or edit as one JSON string.
+
+```json
+{
+  "model": {
+    "chat_template": [
+      "{{ bos_token }}",
+      "{{ messages[0].content }}",
+      "{{ eos_token }}"
+    ]
+  }
+}
+```
+
+The checked-in [`configs/qwen3.5-0.8b.json`](configs/qwen3.5-0.8b.json) includes a Qwen3.5 template with reasoning text, tool calls, forced tool choice, image markers, and video markers.
+
+## Response Parsing
+
+`model.response_schema` parses generated text into OpenAI-compatible fields with JSON Schema plus the Hugging Face `x-regex` extensions.
+
+```json
+{
+  "model": {
+    "response_schema": {
+      "type": "object",
+      "properties": {
+        "role": {"const": "assistant"},
+        "content": {
+          "type": "string",
+          "x-regex": "^(.*)$"
+        }
+      },
+      "required": ["role"]
+    }
+  }
+}
+```
+
+Use `x-regex-iterator` and `x-regex-key-value` to parse repeated tool-call blocks.
+
+See [Hugging Face response parsing](https://huggingface.co/docs/transformers/chat_response_parsing) and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/reference) for the underlying schema concepts.
+
+## Multimodal `model.mtmd`
+
+`model.mtmd` loads a llama.cpp multimodal projector and enables OpenAI-style image and audio content parts.
+
+```json
+{
+  "model": {
+    "mtmd": {
+      "mmproj_from_pretrained": {
+        "repo_id": "lmstudio-community/Qwen3.5-0.8B-GGUF",
+        "filename": "mmproj-Qwen3.5-0.8B-BF16.gguf"
+      },
+      "embedding_cache": {
+        "path": ".cache/mtmd-embeddings",
+        "max_bytes": 1073741824
+      },
+      "image_max_bytes": 20971520,
+      "audio_max_bytes": 104857600,
+      "image_timeout_seconds": 10.0
+    }
+  }
+}
+```
+
+| Field | Purpose |
+| --- | --- |
+| `mmproj_path` | Local multimodal projector path. |
+| `mmproj_from_pretrained` | Hugging Face projector source. |
+| `embedding_cache.path` | Directory for cached image and audio embeddings. |
+| `embedding_cache.max_bytes` | Maximum embedding cache size. |
+| `image_max_bytes` | Maximum image payload size. |
+| `audio_max_bytes` | Maximum audio payload size. |
+| `image_timeout_seconds` | Timeout for remote image and audio URL fetches. |
+
+Send image inputs with OpenAI chat content parts.
+
+```json
+{
+  "messages": [
+    {
+      "role": "user",
+      "content": [
+        {"type": "text", "text": "Describe this image."},
+        {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}
+      ]
+    }
+  ]
+}
+```
+
+Send audio inputs as a URL or as base64 `input_audio` content.
+
+```json
+{
+  "messages": [
+    {
+      "role": "user",
+      "content": [
+        {"type": "text", "text": "Transcribe this audio."},
+        {"type": "input_audio", "input_audio": {"data": "...base64...", "format": "wav"}}
+      ]
+    }
+  ]
+}
+```
+
+## Media Loading Policy
+
+Remote `http:` and `https:` media URLs are unrestricted unless `allowed_media_domains` is set.
+
+```json
+{
+  "model": {
+    "mtmd": {
+      "allowed_media_domains": ["example.com", "static.example.com"]
+    }
+  }
+}
+```
+
+Local `file:` media URLs are disabled unless `allowed_local_media_path` is set.
+
+```json
+{
+  "model": {
+    "mtmd": {
+      "allowed_local_media_path": "/srv/llama-cpp-python/media"
+    }
+  }
+}
+```
+
+`allowed_media_domains` matches exact hostnames and does not allow wildcard patterns.
+
+## LoRA `model.loras`
+
+Load LoRA adapters once at startup from local files or Hugging Face.
+
+```json
+{
+  "model": {
+    "loras": [
+      {
+        "from_pretrained": {
+          "repo_id": "example/qwen-lora-gguf",
+          "filename": "adapter.gguf"
+        },
+        "scale": 1.0
+      }
+    ]
+  }
+}
+```
+
+The current implementation does not hot-swap LoRAs per request.
+
+## Draft Decoding
+
+Set `model.draft_model` to enable speculative draft providers.
+
+```json
+{
+  "model": {
+    "draft_model": "prompt-lookup-decoding",
+    "draft_model_num_pred_tokens": 8,
+    "draft_model_max_ngram_size": 4
+  }
+}
+```
+
+### Multi-Token Prediction (MTP)
+
+Use MTP when the loaded model and llama.cpp build expose the required draft state.
+
+```json
+{
+  "model": {
+    "draft_model": "draft-mtp",
+    "draft_model_num_pred_tokens": 2,
+    "draft_model_threads": 4,
+    "draft_model_threads_batch": 8
+  }
+}
+```
+
+MTP currently applies to text-only requests.
+
+## Disk Sequence Cache
+
+`disk_cache` stores serialized llama.cpp sequence state for repeated prompt prefixes.
+
+```json
+{
+  "disk_cache": {
+    "path": ".cache/sequences",
+    "max_bytes": 1073741824,
+    "min_tokens": 128
+  }
+}
+```
+
+| Field | Purpose |
+| --- | --- |
+| `path` | Directory for cached sequence files. |
+| `max_bytes` | Maximum cache size before background cleanup removes entries. |
+| `min_tokens` | Minimum prefix length that is worth saving. |
+
+The cache is versioned by model and context compatibility data and should be treated as ephemeral.

llama_cpp/llama_cpp_ext.py

+"""Experimental bindings for non-public llama.cpp APIs from `llama-ext.h`.
+
+This module is not part of the stable llama-cpp-python public API.
+Downstream code should not import or depend on it directly.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import functools
+
+from typing import Any, Iterable, Union
+
+from . import llama_cpp
+
+_lib = llama_cpp._lib
+
+
+def _ctypes_function_from_names(
+    names: Iterable[str],
+    argtypes: list[Any],
+    restype: Any,
+):
+    """Decorator for extension functions whose exported symbol name can vary by ABI."""
+
+    def decorator(f):
+        missing: list[str] = []
+        for name in names:
+            try:
+                func = getattr(_lib, name)
+            except AttributeError:
+                missing.append(name)
+                continue
+            func.argtypes = argtypes
+            func.restype = restype
+            functools.wraps(f)(func)
+            return func
+        raise AttributeError(
+            f"None of the shared library symbols were found: {', '.join(missing)}"
+        )
+
+    return decorator
+
+
+# LLAMA_API void llama_set_embeddings_pre_norm(struct llama_context * ctx, bool value, bool masked);
+@_ctypes_function_from_names(
+    (
+        "llama_set_embeddings_pre_norm",
+        "_Z29llama_set_embeddings_pre_normP13llama_contextbb",
+        "?llama_set_embeddings_pre_norm@@YAXPEAUllama_context@@_N1@Z",
+    ),
+    [llama_cpp.llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool],
+    None,
+)
+def llama_set_embeddings_pre_norm(
+    ctx: llama_cpp.llama_context_p,
+    value: bool,
+    masked: bool,
+    /,
+):
+    """Set whether the context outputs pre-norm embeddings or not."""
+    ...
+
+
+# LLAMA_API float * llama_get_embeddings_pre_norm(struct llama_context * ctx);
+@_ctypes_function_from_names(
+    (
+        "llama_get_embeddings_pre_norm",
+        "_Z29llama_get_embeddings_pre_normP13llama_context",
+        "?llama_get_embeddings_pre_norm@@YAPEAMPEAUllama_context@@@Z",
+    ),
+    [llama_cpp.llama_context_p_ctypes],
+    ctypes.POINTER(ctypes.c_float),
+)
+def llama_get_embeddings_pre_norm(
+    ctx: llama_cpp.llama_context_p,
+    /,
+):
+    """Get the pre-norm embeddings from the last evaluation."""
+    ...
+
+
+# LLAMA_API float * llama_get_embeddings_pre_norm_ith(struct llama_context * ctx, int32_t i);
+@_ctypes_function_from_names(
+    (
+        "llama_get_embeddings_pre_norm_ith",
+        "_Z33llama_get_embeddings_pre_norm_ithP13llama_contexti",
+        "?llama_get_embeddings_pre_norm_ith@@YAPEAMPEAUllama_context@@H@Z",
+    ),
+    [llama_cpp.llama_context_p_ctypes, ctypes.c_int32],
+    ctypes.POINTER(ctypes.c_float),
+)
+def llama_get_embeddings_pre_norm_ith(
+    ctx: llama_cpp.llama_context_p,
+    i: Union[ctypes.c_int32, int],
+    /,
+):
+    """Get the pre-norm embeddings for the ith output row from the last evaluation."""
+    ...

vendor/llama.cpp/models/templates/LFM2.5-8B-A1B.jinja

+{{- bos_token -}}
+{%- set preserve_thinking = preserve_thinking | default(false) -%}
+
+{%- macro format_arg_value(arg_value) -%}
+    {%- if arg_value is string -%}
+        {{- "'" + arg_value + "'" -}}
+    {%- elif arg_value is mapping -%}
+        {{- arg_value | tojson -}}
+    {%- else -%}
+        {{- arg_value | string -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro parse_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- set _ns = namespace(result="") -%}
+        {%- for item in content -%}
+            {%- if item["type"] == "image" -%}
+                {%- set _ns.result = _ns.result + "<image>" -%}
+            {%- elif item["type"] == "text" -%}
+                {%- set _ns.result = _ns.result + item["text"] -%}
+            {%- else -%}
+                {%- set _ns.result = _ns.result + item | tojson -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- _ns.result -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_calls(tool_calls) -%}
+    {%- set tool_calls_ns = namespace(tool_calls=[]) -%}
+    {%- for tool_call in tool_calls -%}
+        {%- set func_name = tool_call["function"]["name"] -%}
+        {%- set func_args = tool_call["function"]["arguments"] -%}
+        {%- set args_ns = namespace(arg_strings=[]) -%}
+        {%- for arg_name, arg_value in func_args.items() -%}
+            {%- set args_ns.arg_strings = args_ns.arg_strings + [arg_name + "=" + format_arg_value(arg_value)] -%}
+        {%- endfor -%}
+        {%- set tool_calls_ns.tool_calls = tool_calls_ns.tool_calls + [func_name + "(" + (args_ns.arg_strings | join(", ")) + ")"] -%}
+    {%- endfor -%}
+    {{- "<|tool_call_start|>[" + (tool_calls_ns.tool_calls | join(", ")) + "]<|tool_call_end|>" -}}
+{%- endmacro -%}
+
+{%- set ns = namespace(system_prompt="", last_user_index=-1) -%}
+{%- if messages[0]["role"] == "system" -%}
+    {%- if messages[0].get("content") -%}
+        {%- set ns.system_prompt = parse_content(messages[0]["content"]) -%}
+    {%- endif -%}
+    {%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+    {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: [" -%}
+    {%- for tool in tools -%}
+        {%- if tool is not string -%}
+            {%- set tool = tool | tojson -%}
+        {%- endif -%}
+        {%- set ns.system_prompt = ns.system_prompt + tool -%}
+        {%- if not loop.last -%}
+            {%- set ns.system_prompt = ns.system_prompt + ", " -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- set ns.system_prompt = ns.system_prompt + "]" -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+    {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+    {%- if message["role"] == "user" -%}
+        {%- set ns.last_user_index = loop.index0 -%}
+    {%- endif -%}
+{%- endfor -%}
+{%- for message in messages -%}
+    {{- "<|im_start|>" + message.role + "\n" -}}
+    {%- if message.role == "assistant" -%}
+        {%- generation -%}
+        {%- if message.thinking is defined and (preserve_thinking or loop.index0 > ns.last_user_index) -%}
+            {{- "<think>" + message.thinking + "</think>" -}}
+        {%- endif -%}
+        {%- set _cfm_tag = "CONTINUE_FINAL_MESSAGE_TAG " -%}
+        {%- set _has_cfm = false -%}
+        {%- if message.content is defined -%}
+            {%- set content = parse_content(message.content) -%}
+            {%- if not (preserve_thinking or loop.index0 > ns.last_user_index) -%}
+                {%- if "</think>" in content -%}
+                    {%- set content = content.split("</think>")[-1] | trim -%}
+                {%- endif -%}
+            {%- endif -%}
+            {%- if message.tool_calls is defined and content.endswith(_cfm_tag) -%}
+                {%- set _has_cfm = true -%}
+                {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}
+                {{- content[:_trunc_len] -}}
+            {%- else -%}
+                {{- content -}}
+            {%- endif -%}
+        {%- endif -%}
+        {%- if message.tool_calls is defined -%}
+            {{- render_tool_calls(message.tool_calls) -}}
+        {%- endif -%}
+        {%- if _has_cfm -%}
+            {{- _cfm_tag -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+        {%- endgeneration -%}
+    {%- else %}
+        {%- if message.get("content") -%}
+            {{- parse_content(message["content"]) -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+    {%- endif %}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- "<|im_start|>assistant\n" -}}
+{%- endif -%}

vendor/llama.cpp/tools/mtmd/models/granite4-vision.cpp

+#include "models.h"
+#include "../clip-impl.h"
+#include "../clip-model.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <string>
+#include <vector>
+
+/*
+ * Granite Vision 4.1 clip graph
+ *
+ *   Stage 1a: SigLIP vision tower (N layers, post-norm)
+ *   Stage 1b: WindowQFormer blocks (deepstack + spatial)
+ *   Stage 1c: Concatenate and pack outputs
+ *   Stage 1d: Append newline tokens if add_newline is set
+ */
+
+// ---------------------------------------------------------------------------
+// Member method implementations
+// ---------------------------------------------------------------------------
+
+ggml_tensor * clip_graph_granite4_vision::gather(
+        ggml_tensor * src,
+        const std::string & name,
+        int idx_len) {
+    ggml_tensor * idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, idx_len);
+    ggml_set_name(idx, name.c_str());
+    ggml_set_input(idx);
+    return ggml_get_rows(ctx0, src, idx);
+}
+
+ggml_tensor * clip_graph_granite4_vision::interp_down(
+        ggml_tensor * src,
+        int side,
+        int new_side) {
+    const int n_embd = src->ne[0];
+    ggml_tensor * t = ggml_reshape_4d(ctx0, src, n_embd, side, side, 1);
+    t = ggml_cont(ctx0, ggml_permute(ctx0, t, 2, 0, 1, 3));
+    const int kernel = side / new_side;
+    t = ggml_pool_2d(ctx0, t, GGML_OP_POOL_AVG, kernel, kernel, kernel, kernel, 0, 0);
+    t = ggml_cont(ctx0, ggml_permute(ctx0, t, 1, 2, 0, 3));
+    return ggml_reshape_2d(ctx0, t, n_embd, new_side * new_side);
+}
+
+// ---------------------------------------------------------------------------
+// build_block - WindowQFormer block implementation
+// ---------------------------------------------------------------------------
+
+ggml_tensor * clip_graph_granite4_vision::build_block(
+        const qf_block & blk,
+        ggml_tensor * h,
+        int bid,
+        int spatial_offset,
+        int image_side,
+        int window_side,
+        int query_side,
+        float qformer_eps) {
+
+    const int n_embd = h->ne[0];
+    GGML_ASSERT(h->ne[1] == image_side * image_side);
+    const int n = image_side / window_side;
+    const int new_side = n * query_side;
+    const int n_windows = n * n;
+    const int enc_len = window_side * window_side;
+    const int query_len = query_side * query_side;
+
+    auto cbx = [&](ggml_tensor * & t, const char * step) {
+        const std::string name = "g4v_blk" + std::to_string(bid) + "_" + step;
+        ggml_set_name(t, name.c_str());
+    };
+
+    // 1. Top-level LN
+    cbx(h, "inp");
+    ggml_tensor * x = build_norm(h, blk.qf_proj_norm_w, blk.qf_proj_norm_b, NORM_TYPE_NORMAL, eps, bid);
+    cbx(x, "norm");
+
+    // 2. enc = _win(x, image_side, window_side)
+    ggml_tensor * enc;
+    {
+        ggml_tensor * enc_flat = gather(x,
+            "g4v_blk" + std::to_string(bid) + "_win_idx",
+            image_side * image_side);
+        enc = ggml_reshape_3d(ctx0, enc_flat, n_embd, enc_len, n_windows);
+    }
+    cbx(enc, "enc");
+
+    // 3. downsampled = downsampler(x)
+    ggml_tensor * d;
+    (void) spatial_offset;
+    if (spatial_offset >= 0) {
+        d = gather(x,
+            "g4v_blk" + std::to_string(bid) + "_spatial_idx",
+            new_side * new_side);
+    } else {
+        d = interp_down(x, image_side, new_side);
+    }
+    cbx(d, "downsampled");
+
+    // 4. query_embeds = query + _win(d, new_side, query_side)
+    ggml_tensor * q_in;
+    {
+        ggml_tensor * dw_flat = gather(d,
+            "g4v_blk" + std::to_string(bid) + "_qwin_idx",
+            new_side * new_side);
+        ggml_tensor * dw = ggml_reshape_3d(ctx0, dw_flat, n_embd, query_len, n_windows);
+        q_in = ggml_add(ctx0, dw, blk.qf_proj_query);
+    }
+    cbx(q_in, "query_embeds");
+
+    // 5. encoder_embeds = enc + image_positions → (C, enc_len, n_windows)
+    ggml_tensor * e_in = ggml_add(ctx0, enc, blk.qf_proj_img_pos);
+    cbx(e_in, "encoder_embeds");
+
+    // 6. Qformer forward.
+    ggml_tensor * q = build_norm(q_in, blk.qf_proj_post_norm_w, blk.qf_proj_post_norm_b, NORM_TYPE_NORMAL, qformer_eps, bid);
+
+    // Helper for linear projections with window batching
+    auto linear = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) -> ggml_tensor * {
+        ggml_tensor * t = ggml_reshape_2d(ctx0, x, x->ne[0], x->ne[1] * x->ne[2]);
+        t = build_mm(w, t);
+        if (b) t = ggml_add(ctx0, t, b);
+        return t;
+    };
+
+    // Get the single QFormer layer
+    GGML_ASSERT(blk.qf_proj_layers.size() == 1);
+    const auto & pl = blk.qf_proj_layers[0];
+
+    // 6a. Self-attention
+    ggml_tensor * sa_out;
+    {
+        const int d_h = 64;
+        const int n_head = n_embd / d_h;
+        const int nq = q->ne[1];
+        const float scale = 1.0f / std::sqrt((float) d_h);
+
+        ggml_tensor * Q = linear(q, pl.q_w, pl.q_b);
+        ggml_tensor * K = linear(q, pl.k_w, pl.k_b);
+        ggml_tensor * V = linear(q, pl.v_w, pl.v_b);
+
+        Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_windows);
+        K = ggml_reshape_4d(ctx0, K, d_h, n_head, nq, n_windows);
+        V = ggml_reshape_4d(ctx0, V, d_h, n_head, nq, n_windows);
+
+        sa_out = build_attn(pl.o_w, pl.o_b, Q, K, V, nullptr, scale, bid);
+        sa_out = ggml_reshape_3d(ctx0, sa_out, n_embd, nq, n_windows);
+
+        sa_out = ggml_add(ctx0, sa_out, q);
+        sa_out = build_norm(sa_out, pl.ln_1_w, pl.ln_1_b,
+                            NORM_TYPE_NORMAL, qformer_eps, bid);
+    }
+    cbx(sa_out, "sa_out");
+
+    // 6b. Cross-attention
+    ggml_tensor * ca_out;
+    {
+        const int d_h = 64;
+        const int n_head = n_embd / d_h;
+        const int nq = sa_out->ne[1];
+        const int nkv = e_in->ne[1];
+        const float scale = 1.0f / std::sqrt((float) d_h);
+
+        ggml_tensor * Q = linear(sa_out, pl.cross_attn_q_w, pl.cross_attn_q_b);
+        ggml_tensor * K = linear(e_in, pl.cross_attn_k_w, pl.cross_attn_k_b);
+        ggml_tensor * V = linear(e_in, pl.cross_attn_v_w, pl.cross_attn_v_b);
+
+        Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_windows);
+        K = ggml_reshape_4d(ctx0, K, d_h, n_head, nkv, n_windows);
+        V = ggml_reshape_4d(ctx0, V, d_h, n_head, nkv, n_windows);
+
+        ca_out = build_attn(pl.cross_attn_o_w, pl.cross_attn_o_b,
+                            Q, K, V, nullptr, scale, bid);
+        ca_out = ggml_reshape_3d(ctx0, ca_out, n_embd, nq, n_windows);
+
+        ca_out = ggml_add(ctx0, ca_out, sa_out);
+        ca_out = build_norm(ca_out, pl.cross_attn_norm_w, pl.cross_attn_norm_b,
+                            NORM_TYPE_NORMAL, qformer_eps, bid);
+    }
+    cbx(ca_out, "ca_out");
+
+    // 6c. FFN
+    ggml_tensor * ffn;
+    {
+        ggml_tensor * t = ggml_reshape_2d(ctx0, ca_out, n_embd, query_len * n_windows);
+        t = build_mm(pl.ff_up_w, t);
+        if (pl.ff_up_b) t = ggml_add(ctx0, t, pl.ff_up_b);
+        t = ggml_gelu_erf(ctx0, t);
+        t = build_mm(pl.ff_down_w, t);
+        if (pl.ff_down_b) t = ggml_add(ctx0, t, pl.ff_down_b);
+        t = ggml_reshape_3d(ctx0, t, n_embd, query_len, n_windows);
+        ffn = ggml_add(ctx0, t, ca_out);
+        ffn = build_norm(ffn, pl.ln_2_w, pl.ln_2_b, NORM_TYPE_NORMAL, qformer_eps, bid);
+    }
+    cbx(ffn, "qformer_out");
+
+    // 7. _unwin back to raster
+    ggml_tensor * unwinned;
+    {
+        ggml_tensor * flat = ggml_reshape_2d(ctx0, ffn, n_embd, query_len * n_windows);
+        unwinned = gather(flat,
+            "g4v_blk" + std::to_string(bid) + "_unwin_idx",
+            new_side * new_side);
+    }
+    cbx(unwinned, "unwin");
+
+    // 8. out_linear
+    ggml_tensor * out = build_mm(blk.qf_proj_linear_w, unwinned);
+    if (blk.qf_proj_linear_b) out = ggml_add(ctx0, out, blk.qf_proj_linear_b);
+    cbx(out, "out");
+
+    return out;
+}
+
+// ---------------------------------------------------------------------------
+// build() - top-level graph
+// ---------------------------------------------------------------------------
+
+// Build the K-tiled, base-scaled newline row tensor.
+// Shape: (n_mmproj_embd, 1)
+ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0) {
+    const int K = (int) model.qf_proj_blocks.size();
+    GGML_ASSERT(K > 0);
+    GGML_ASSERT(n_mmproj_embd % K == 0);
+    const int projection_dim = n_mmproj_embd / K;
+    GGML_ASSERT(model.image_newline != nullptr);
+    GGML_ASSERT(ggml_nelements(model.image_newline) == projection_dim);
+
+    // Build newline_row[k*projection_dim + d] = nl[d] * (k == 0 ? base : 1.0)
+    ggml_tensor * nl = model.image_newline; // (projection_dim,)
+    ggml_tensor * nl_first_2d = ggml_reshape_2d(ctx0, nl, projection_dim, 1);
+    ggml_tensor * nl_row_2d;
+    if (K == 1) {
+        nl_row_2d = nl_first_2d;
+    } else {
+        ggml_tensor * nl_2d = ggml_reshape_2d(ctx0, nl, projection_dim, 1);
+        ggml_tensor * rest_template = ggml_new_tensor_2d(
+            ctx0, GGML_TYPE_F32, projection_dim, K - 1);
+        ggml_tensor * nl_rest = ggml_repeat(ctx0, nl_2d, rest_template);
+        nl_row_2d = ggml_concat(ctx0, nl_first_2d, nl_rest, 1); // (projection_dim, K)
+    }
+    nl_row_2d = ggml_cont(ctx0, nl_row_2d);
+    return ggml_reshape_2d(ctx0, nl_row_2d, n_mmproj_embd, 1);
+}
+
+// Append a single newline row at the end of the tile output.
+ggml_tensor * clip_graph_granite4_vision::append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output) {
+    // For the single-tile case, append one newline row at the end.
+    // For the multi-tile rowwise case, this will be called per-tile
+    // (though currently only the single-tile path uses it).
+    ggml_tensor * nl_row = build_newline_row(ctx0);
+    return ggml_concat(ctx0, tile_output, nl_row, 1);
+}
+
+ggml_cgraph * clip_graph_granite4_vision::build() {
+    GGML_ASSERT(model.patch_embeddings_0 != nullptr);
+    GGML_ASSERT(model.position_embeddings != nullptr);
+    GGML_ASSERT(model.class_embedding == nullptr);
+    GGML_ASSERT(!model.qf_proj_blocks.empty());
+
+    // --- Stage 1a: SigLIP encoder producing intermediate hidden states ---
+    ggml_tensor * inp = build_inp();
+    inp = ggml_add(ctx0, inp, model.position_embeddings);
+    cb(inp, "pos_embed", -1);
+
+    ggml_tensor * inpL = inp;
+    std::vector<ggml_tensor *> layer_outs(n_layer, nullptr);
+
+    for (int il = 0; il < n_layer; ++il) {
+        const auto & layer = model.layers[il];
+        ggml_tensor * cur = inpL;
+
+        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
+
+        // Self-attention
+        ggml_tensor * Qcur = build_mm(layer.q_w, cur);
+        if (layer.q_b) Qcur = ggml_add(ctx0, Qcur, layer.q_b);
+        ggml_tensor * Kcur = build_mm(layer.k_w, cur);
+        if (layer.k_b) Kcur = ggml_add(ctx0, Kcur, layer.k_b);
+        ggml_tensor * Vcur = build_mm(layer.v_w, cur);
+        if (layer.v_b) Vcur = ggml_add(ctx0, Vcur, layer.v_b);
+
+        Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_patches);
+        Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_patches);
+        Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_patches);
+
+        cur = build_attn(layer.o_w, layer.o_b,
+                         Qcur, Kcur, Vcur, nullptr, kq_scale, il);
+
+        cur = ggml_add(ctx0, cur, inpL);
+        inpL = cur;
+
+        cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
+        cur = build_ffn(cur,
+                        layer.ff_up_w, layer.ff_up_b,
+                        layer.ff_gate_w, layer.ff_gate_b,
+                        layer.ff_down_w, layer.ff_down_b,
+                        hparams.ffn_op, il);
+        cur = ggml_add(ctx0, inpL, cur);
+        cb(cur, "layer_out", il);
+        layer_outs[il] = cur;
+        inpL = cur;
+    }
+
+    // --- Stage 1b/1c: WindowQFormer blocks ---
+    const int projector_count = hparams.vision_feature_layer.size();
+    const float qformer_eps = 1e-12f;
+
+    ggml_tensor * mmproj = nullptr;
+    for (int bid = 0; bid < projector_count; ++bid) {
+        const auto & blk = model.qf_proj_blocks[bid];
+
+        int vlayer = hparams.vision_feature_layer[bid];
+        GGML_ASSERT(vlayer >= 0 && vlayer < n_layer);
+        ggml_tensor * h = layer_outs[vlayer];
+
+        ggml_tensor * stream = build_block(
+            blk, h, bid,
+            hparams.proj_spatial_offsets[bid],
+            n_patches_x,
+            hparams.downsample_window_side,
+            hparams.downsample_query_side,
+            qformer_eps);
+        cb(stream, (std::string("proj_") + std::to_string(bid) + std::string("_v_out")).c_str(), vlayer);
+        mmproj = mmproj ? ggml_concat(ctx0, mmproj, stream, 0) : stream;
+    }
+
+    // --- Stage 1d: Append newline tokens if add_newline is set ---
+    if (add_newline) {
+        mmproj = append_rowwise_newlines(ctx0, mmproj);
+        ggml_set_name(mmproj, "g4v_mmproj_out_nl");
+    } else {
+        ggml_set_name(mmproj, "g4v_mmproj_out");
+    }
+    ggml_build_forward_expand(gf, mmproj);
+
+    return gf;
+}

.github/workflows/build-and-release.yaml

@@ -52,2 +52,5 @@ name: Build Release
           CIBW_REPAIR_WHEEL_COMMAND_LINUX: "LD_LIBRARY_PATH=/project/llama_cpp/lib auditwheel repair -w {dest_dir} {wheel}"
+          # cibuildwheel v3 defaults to manylinux_2_28 images whose current
+          # GCC toolchain emits symbols newer than the policy allows.
+          CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014"
           # The release wheel is tagged py3-none, so one build per platform
@@ -89,2 +92,4 @@ # covers all supported Python versions and avoids duplicate names.
           CIBW_ARCHS: "aarch64"
+          # Keep this consistent with the x86_64 Linux release wheels.
+          CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux2014"
           # Keep native arm64 builds on a portable CPU baseline instead of
@@ -106,23 +111,4 @@ # tuning wheels to the hosted runner.
   build_wheels_riscv64:
-    name: Build riscv64 wheels (${{ matrix.shard.name }})
+    name: Build riscv64 wheel
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        shard:
-          - name: cp310
-            build: "cp310-*"
-            artifact: wheels_riscv64_cp310
-          - name: cp311
-            build: "cp311-*"
-            artifact: wheels_riscv64_cp311
-          - name: cp312
-            build: "cp312-*"
-            artifact: wheels_riscv64_cp312
-          - name: cp313
-            build: "cp313-*"
-            artifact: wheels_riscv64_cp313
-          - name: cp314
-            build: "cp314-*"
-            artifact: wheels_riscv64_cp314
     steps:
@@ -147,5 +133,5 @@ - uses: actions/checkout@v6
           CIBW_ENVIRONMENT: CMAKE_ARGS="-DGGML_NATIVE=off -DGGML_RVV=off -DGGML_RV_ZFH=off -DGGML_RV_ZVFH=off -DGGML_RV_ZICBOP=off -DGGML_RV_ZIHINTPAUSE=off"
-          # Split the emulated riscv64 build into one Python version per job
-          # to minimize wall-clock time without changing the release artifacts.
-          CIBW_BUILD: ${{ matrix.shard.build }}
+          # The release wheel is tagged py3-none, so one riscv64 build is
+          # enough and avoids duplicate same-name release artifacts.
+          CIBW_BUILD: "cp310-*"
         with:
@@ -157,3 +143,3 @@ output-dir: wheelhouse
         with:
-          name: ${{ matrix.shard.artifact }}
+          name: wheels_riscv64
           path: ./wheelhouse/*.whl
@@ -160,0 +146,0 @@

.github/workflows/build-wheels-rocm.yaml

@@ -36,3 +36,2 @@ name: Build Wheels (ROCm)
           python-version: ${{ matrix.pyver }}
-          cache: "pip"
 
@@ -39,0 +38,0 @@ - name: Install build dependencies

.github/workflows/generate-index-from-release.yaml

@@ -43,2 +43,3 @@ name: Wheels Index
           ./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$'
+          ./scripts/releases-to-pep-503.sh index/whl/cu118 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu118$'
           ./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu121$'
@@ -48,4 +49,5 @@ ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$'
           ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$'
-          # ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$'
-          # ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$'
+          ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu125$'
+          ./scripts/releases-to-pep-503.sh index/whl/cu130 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu130$'
+          ./scripts/releases-to-pep-503.sh index/whl/cu132 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu132$'
           ./scripts/releases-to-pep-503.sh index/whl/rocm72 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-rocm72$'
@@ -52,0 +54,0 @@ ./scripts/releases-to-pep-503.sh index/whl/hip-radeon '^[v]?[0-9]+\.[0-9]+\.[0-9]+-hip-radeon$'

CHANGELOG.md

@@ -10,2 +10,7 @@ # Changelog
 
+## [0.3.27]
+
+- feat: update llama.cpp to ggml-org/llama.cpp@465b1f0e7
+- feat(example): Updated server example (batch processing, multi-token prediction, `/v1/responses` api, response parsing) by @abetlen in #2174
+
 ## [0.3.26]
@@ -12,0 +17,0 @@

llama_cpp/__init__.py

 from .llama_cpp import *
 from .llama import *
 
-__version__ = "0.3.26"
+__version__ = "0.3.27"

llama_cpp/mtmd_cpp.py

@@ -554,10 +554,10 @@ from __future__ import annotations
 
-# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname);
+# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder);
 @ctypes_function(
     "mtmd_helper_bitmap_init_from_file",
-    [mtmd_context_p_ctypes, c_char_p],
+    [mtmd_context_p_ctypes, c_char_p, c_bool],
     mtmd_bitmap_p_ctypes,
 )
 def mtmd_helper_bitmap_init_from_file(
-    ctx: mtmd_context_p, fname: bytes, /
+    ctx: mtmd_context_p, fname: bytes, placeholder: Union[c_bool, bool], /
 ) -> Optional[mtmd_bitmap_p]:
@@ -568,6 +568,6 @@ """Initialize an MTMD bitmap from a file."""
 
-# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len);
+# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder);
 @ctypes_function(
     "mtmd_helper_bitmap_init_from_buf",
-    [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t],
+    [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t, c_bool],
     mtmd_bitmap_p_ctypes,
@@ -579,2 +579,3 @@ )
     length: Union[c_size_t, int],
+    placeholder: Union[c_bool, bool],
     /,
@@ -581,0 +582,0 @@ ) -> Optional[mtmd_bitmap_p]: ...

PKG-INFO

 Metadata-Version: 2.1
 Name: llama_cpp_python
-Version: 0.3.26
+Version: 0.3.27
 Summary: Python bindings for the llama.cpp library
@@ -592,2 +592,6 @@ Author-Email: Andrei Betlen <[e-mail]>
 
+Try Gemma 4 12B in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/notebook.ipynb)
+
+Try Gemma 4 12B QAT in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/Gemma4-12B-QAT.ipynb)
+
 Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images.
@@ -594,0 +598,0 @@

README.md

@@ -541,2 +541,6 @@ <p align="center">
 
+Try Gemma 4 12B in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/notebook.ipynb)
+
+Try Gemma 4 12B QAT in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/Gemma4-12B-QAT.ipynb)
+
 Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images.
@@ -543,0 +547,0 @@

scripts/releases-to-pep-503.sh

@@ -57,4 +57,8 @@ #!/bin/bash
 
-# Filter releases by pattern
-releases=$(grep -E "$pattern" "$current_dir/all_releases.txt")
+# Filter releases by pattern. Some backend indexes are valid even when there
+# are no matching releases yet.
+releases=$(grep -E "$pattern" "$current_dir/all_releases.txt" || true)
+if [ -z "$releases" ]; then
+    log_info "No releases found matching pattern: $pattern"
+fi
 
@@ -85,6 +89,2 @@ # Prepare curl headers
 
-    # Get release version from release ie v0.1.0-cu121 -> v0.1.0
-    release_version=$(echo "$release" | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+")
-    echo "    <h2>$release_version</h2>" >> "$output_dir/llama-cpp-python/index.html"
-    
     wheel_urls=$(echo "$response" | jq -r '.assets[] | select(.name | endswith(".whl")) | .browser_download_url')
@@ -96,2 +96,6 @@ if [ -z "$wheel_urls" ]; then
 
+    # Get release version from release ie v0.1.0-cu121 -> v0.1.0
+    release_version=$(echo "$release" | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+")
+    echo "    <h2>$release_version</h2>" >> "$output_dir/llama-cpp-python/index.html"
+
     echo "$wheel_urls" | while read -r asset; do
@@ -98,0 +102,0 @@ echo "    <a href=\"$asset\">$asset</a>" >> "$output_dir/llama-cpp-python/index.html"

vendor/llama.cpp/.github/workflows/build-msys.yml

@@ -30,4 +30,4 @@ name: CI (msys)
         include:
-          - { sys: UCRT64,  env: ucrt-x86_64,  build: Release }
-          - { sys: CLANG64, env: clang-x86_64, build: Release }
+          - { sys: UCRT64,  env: ucrt-x86_64,  compiler: gcc,   build: Release }
+          - { sys: CLANG64, env: clang-x86_64, compiler: clang, build: Release }
 
@@ -52,5 +52,3 @@ steps:
           install: >-
-            base-devel
-            git
-            mingw-w64-${{matrix.env}}-toolchain
+            mingw-w64-${{matrix.env}}-${{matrix.compiler}}
             mingw-w64-${{matrix.env}}-cmake
@@ -57,0 +55,0 @@ mingw-w64-${{matrix.env}}-openblas

vendor/llama.cpp/.github/workflows/docker.yml

@@ -85,4 +85,4 @@ # This workflow uses actions that are not certified by GitHub.
             { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
-            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.1.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
-            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.1.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
+            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
+            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
             { "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
@@ -89,0 +89,0 @@ { "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },

vendor/llama.cpp/.pi/gg/SYSTEM.md

@@ -19,3 +19,3 @@ You are a coding agent. Here are some very important rules that you must follow:
 - When creating a pull request, look for the repository's PR template and follow it
-- For the AI usage disclosure section, write "YES. llama.cpp + pi + [MODEL]"
+- For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]"
 - Ask the user to tell you what model was used and write it in place of [MODEL]
@@ -25,3 +25,3 @@ - Always create the pull requests in draft mode
 Commits:
-- On every commit that you make, include a "Assisted-by: llama.cpp:local pi" tag
+- On every commit that you make, include a "Assisted-by: pi:llama.cpp/[MODEL]" tag
 - Do not explicitly set the git author in commits - rely on the default git config
@@ -28,0 +28,0 @@ - Always use `--no-gpg-sign` when committing

vendor/llama.cpp/common/chat-peg-parser.cpp

@@ -90,2 +90,4 @@ #include "chat-peg-parser.h"
 
+    auto is_word_char = [](char ch) { return std::isalnum(static_cast<unsigned char>(ch)) || ch == '_'; };
+
     for (size_t i = 0; i < input.size(); ++i) {
@@ -155,2 +157,25 @@ char c = input[i];
             }
+        } else if (!in_single_quoted && !in_double_quoted && (c == 'T' || c == 'F' || c == 'N') &&
+                   (i == 0 || !is_word_char(input[i - 1]))) {
+            // Python literals -> JSON; prefix match keeps streamed partials monotonic.
+            static constexpr std::pair<std::string_view, std::string_view> literals[] = {
+                { "True", "true" }, { "False", "false" }, { "None", "null" },
+            };
+            size_t n = 0;
+            while (i + n < input.size() && is_word_char(input[i + n])) {
+                ++n;
+            }
+            std::string_view token(input.data() + i, n);
+            bool matched = false;
+            for (const auto & [py, js] : literals) {
+                if (py.substr(0, n) == token) {
+                    result += js.substr(0, n);
+                    i += n - 1;
+                    matched = true;
+                    break;
+                }
+            }
+            if (!matched) {
+                result += c;
+            }
         } else {
@@ -358,8 +383,4 @@ result += c;
         } else if (!value_content.empty()) {
-            // For potential containers, normalize Python-style single quotes to JSON double quotes
-            bool is_potential_container = value_content[0] == '[' || value_content[0] == '{';
-            if (is_potential_container) {
-                value_content = normalize_container_value(value_content);
-            }
-            value_to_add += value_content;
+            // Pythonic scalars/containers -> JSON.
+            value_to_add += normalize_container_value(value_content);
         }
@@ -472,2 +493,24 @@
 
+// Like python_value(), but the leaf also accepts JSON-cased true/false/null, used by LFM2/LFM2.5
+common_peg_parser common_chat_peg_builder::python_or_json_value() {
+    return rule("python-or-json-value", [this]() {
+        auto ws    = space();
+        auto value = python_or_json_value();
+
+        auto member  = sequence({ python_string(), ws, literal(":"), ws, value });
+        auto members = sequence({ member, zero_or_more(sequence({ ws, literal(","), ws, member })) });
+        auto dict    = rule("python-or-json-dict", [&]() {
+            return sequence({ literal("{"), ws, choice({ literal("}"), sequence({ members, ws, literal("}") }) }), ws });
+        });
+
+        auto elements = sequence({ value, zero_or_more(sequence({ literal(","), ws, value })) });
+        auto array    = rule("python-or-json-array", [&]() {
+            return sequence({ literal("["), ws, choice({ literal("]"), sequence({ elements, ws, literal("]") }) }), ws });
+        });
+
+        return choice({ dict, array, python_string(), python_number(),
+                        python_bool(), python_null(), json_bool(), json_null() });
+    });
+}
+
 // Python-style tool calls: name(arg1="value1", arg2=123)
@@ -477,3 +520,4 @@ // Used only by LFM2 for now, so we don't merge it into autoparser
     const ordered_json & tools,
-    bool                 parallel_tool_calls) {
+    bool                 parallel_tool_calls,
+    bool                 allow_json_literals) {
     if (!tools.is_array() || tools.empty()) {
@@ -512,3 +556,3 @@ return eps();
                 } else {
-                    arg_value_parser = tool_arg_value(python_value());
+                    arg_value_parser = tool_arg_value(allow_json_literals ? python_or_json_value() : python_value());
                 }
@@ -515,0 +559,0 @@

vendor/llama.cpp/common/chat-peg-parser.h

@@ -135,5 +135,9 @@ #pragma once
     common_peg_parser python_style_tool_calls(const nlohmann::ordered_json & tools,
-                                              bool                           parallel_tool_calls);
+                                              bool                           parallel_tool_calls,
+                                              bool                           allow_json_literals);
 
   private:
+    // Python values plus JSON true/false/null.
+    common_peg_parser python_or_json_value();
+
     // Implementation helpers for standard_json_tools — one per JSON tool call layout mode
@@ -199,2 +203,1 @@ common_peg_parser build_json_tools_function_is_key(const nlohmann::ordered_json & tools,
     const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn);
-

vendor/llama.cpp/conversion/__init__.py

@@ -256,2 +256,3 @@ from __future__ import annotations
     "GlmasrModel": "ultravox",
+    "Granite4VisionForConditionalGeneration": "granite",
     "GraniteSpeechForConditionalGeneration": "granite",
@@ -258,0 +259,0 @@ "HunYuanVLForConditionalGeneration": "hunyuan",

vendor/llama.cpp/conversion/gemma.py

@@ -815,6 +815,7 @@ from __future__ import annotations
         # audio params
-        assert self.hparams_audio is not None
-        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.GEMMA4A)
-        self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["feat_in"])
-        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio.get("layer_norm_eps", 1e-6))
+        if self.has_audio_encoder:
+            assert self.hparams_audio is not None
+            self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.GEMMA4A)
+            self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["feat_in"])
+            self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio.get("layer_norm_eps", 1e-6))
 
@@ -821,0 +822,0 @@ def is_audio_tensor(self, name: str) -> bool:

vendor/llama.cpp/conversion/granite.py

 from __future__ import annotations
 
+import re
 from typing import Any, Callable, Iterable, TYPE_CHECKING
@@ -16,3 +17,3 @@
 
[e-mail]("GraniteForCausalLM", "GraniteSpeechForConditionalGeneration")
[e-mail]("GraniteForCausalLM")
 class GraniteModel(LlamaModel):
@@ -50,7 +51,25 @@ """Conversion for IBM's GraniteForCausalLM"""
 
+        # If being used as the base for Granite4 Vision, add deepstack_layer_arr
+        if self.hparams.get("spatial_target_layers") or self.hparams.get("deepstack_layer_map"):
+            normalized_projector_map = Granite4VisionMmprojModel.get_normalized_projector_map(self.hparams)
+            deepstack_mapping_arr = [-1 for _ in range(self.block_count)] # Populate with -1 sentinels
+            for proj_idx, (_, llm_layer, _, _) in enumerate(normalized_projector_map):
+                # Skip the first projector which is handled as the base embedding
+                # stream like normal
+                if proj_idx == 0:
+                    continue
+                deepstack_mapping_arr[llm_layer] = proj_idx
+            self.gguf_writer.add_deepstack_mapping(deepstack_mapping_arr)
+
     @classmethod
     def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
         name, gen = item
-        if name.startswith("encoder."):
-            return None
+        # Skip multimodal tensors
+        if (
+            name.startswith(("encoder."))
+            or "image_" in name
+            or "layerwise_projectors" in name
+            or "spatial_projectors" in name
+        ):
+            return
         return super().filter_tensors(item)
@@ -246,3 +265,4 @@
     def set_vocab(self):
-        self.hparams["pad_vocab_size_multiple"] = 8
+        # For models with no ssm layers, don't pad for mamba2
+        self.hparams["pad_vocab_size_multiple"] = 8 if self._ssm_layers else 1
         Mamba2Model.set_vocab(self)
@@ -332,1 +352,131 @@
         yield from super().modify_tensors(data_torch, name, bid)
+
+
[e-mail]("Granite4VisionForConditionalGeneration")
+class Granite4VisionMmprojModel(MmprojModel):
+    has_vision_encoder = True
+    has_audio_encoder = False
+
+    @staticmethod
+    def get_normalized_projector_map(global_config: dict) -> list[tuple[int, int, str, int]]:
+        """Normalize both deepstack and spatial projector maps to the form:
+        (vision_layer, llm_layer, <type>, type_index)
+
+        This is then used to populate the following mappings:
+        - vision_feature_layers (mmproj hparam): ordered list of all
+          vision_layer values where order corresponds with the order of the
+          stacked projector tensors
+          NOTE: Values may appear multiple times for spatial projectors
+        - tensor_prefix_map (mmproj tensors): mapping from tensor prefixes to
+          the index of the corresponding projector in the stacked tensors
+        - deepstack_layer_arr (llm hparam): per-text-layer array indicating
+          which input vision feature should be injected at that layer
+          (-1 if none)
+
+        Output: (vision_layer, llm_layer, <type>, type_index)
+        """
+        deepstack_map = global_config.get("deepstack_layer_map", [])  # [[vis_layer, llm_layer], ...]
+        spatial_layers = global_config.get("spatial_target_layers", [])  # [llm_layer, ...]
+        n_text_layers = global_config["text_config"]["num_hidden_layers"]
+        n_vision_layers = global_config["vision_config"]["num_hidden_layers"]
+        normalized_projector_map = []
+        if deepstack_map:
+            for deepstack_idx, (vision_layer, llm_layer) in enumerate(sorted(deepstack_map)):
+                if vision_layer < 0:
+                    vision_layer = n_vision_layers + vision_layer
+                if llm_layer < 0:
+                    llm_layer = n_text_layers + llm_layer
+                normalized_projector_map.append((vision_layer, llm_layer, "layerwise", deepstack_idx))
+        if spatial_layers:
+            spatial_vision_layer = global_config.get("spatial_vision_layer", -1)
+            if spatial_vision_layer < 0:
+                spatial_vision_layer = n_vision_layers + spatial_vision_layer
+            for spatial_idx, llm_layer in enumerate(spatial_layers):
+                normalized_projector_map.append((spatial_vision_layer, llm_layer, "spatial", spatial_idx))
+        return list(sorted(normalized_projector_map, key=(lambda entry: entry[1])))
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        normalized_projector_map = self.get_normalized_projector_map(self.global_config)
+        self._n_proj = len(normalized_projector_map)
+
+        self._tensor_prefix_map = {
+            f"model.{proj_type}_projectors.{type_idx}": proj_idx
+            for proj_idx, (_, _, proj_type, type_idx) in enumerate(normalized_projector_map)
+        }
+        self._vision_feature_layers = [vision_layer for vision_layer, _, _, _ in normalized_projector_map]
+        self._spatial_offsets = [
+            type_idx if proj_type == "spatial" else -1
+            for _, _, proj_type, type_idx in normalized_projector_map
+        ]
+
+    def set_gguf_parameters(self):
+        assert self.hparams_vision is not None
+        super().set_gguf_parameters()
+
+        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GRANITE4_VISION)
+
+        # SigLIP encoder hparams
+        self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-6))
+        self.gguf_writer.add_vision_use_gelu(True)
+
+        # Preprocessor
+        self.gguf_writer.add_vision_preproc_image_size(self.hparams.get("image_size", 384))
+
+        # QFormer projector config
+        ds_rate = self.global_config["downsample_rate"]
+        ds_parts = ds_rate.split("/")
+        assert len(ds_parts) == 2, f"Invalid 'downsample_rate' value: {ds_rate}"
+        query_side, window_side = [int(p) for p in ds_parts]
+        self.gguf_writer.add_vision_projector_query_side(query_side)
+        self.gguf_writer.add_vision_projector_window_side(window_side)
+
+        # Set vision feature layers
+        self.gguf_writer.add_vision_feature_layers(self._vision_feature_layers)
+
+        # Set the spatial offests per projector
+        self.gguf_writer.add_vision_spatial_offsets(self._spatial_offsets)
+
+        # Add flattened image grind pinpoints (resolution candidates internally)
+        if pinpoints := self.global_config.get("image_grid_pinpoints"):
+            # Flatten with h, w -> w, h inversion
+            pinpoints = [val for h, w in pinpoints for val in (w, h)]
+            self.gguf_writer.add_vision_image_grid_pinpoints(pinpoints)
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, _ = item
+        if ("vision_model.head" in name or name.startswith("lm_head")):
+            return None
+        return super().filter_tensors(item)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+
+        # Detect projector tensors and bin them
+        projector_idx = None
+        for prefix, proj_idx in self._tensor_prefix_map.items():
+            if name.startswith(prefix):
+                projector_idx = proj_idx
+                break
+        if projector_idx is not None:
+            # If this projector tensor has a block id within the projector,
+            # alias the bid to projector_idx
+            #
+            # TODO: currently, none of the Granite 4 Vision models have
+            # projectors with multiple QFormer layers, so the `layer.{}` index
+            # is always 0. This allows us to simply map to a single `bid` that
+            # matches the projector index. If this changes, we'll need a
+            # convention that merges the two IDs.
+            id_matches = list(re.finditer(r"\.([0-9]+)\.", name))
+            all_ids = [int(m.group(1)) for m in id_matches]
+            assert len(all_ids) >= 1 and len(all_ids) <= 2, "Must have at least 1 and at most 2 ids in tensor names"
+            # If not layer id, just use the projector index
+            new_bid = projector_idx
+            if len(all_ids) == 1:
+                new_name = name[:id_matches[0].span(1)[0]] + str(new_bid) + name[id_matches[0].span(1)[1]:]
+            else: # len(all_ids) == 2
+                new_bid = projector_idx # + all_ids[1]
+                new_name = name[:id_matches[0].span(0)[0]] + name[id_matches[0].span(1)[1]:id_matches[1].span(1)[0]] + str(new_bid) + name[id_matches[1].span(1)[1]:]
+            yield from super().modify_tensors(data_torch, new_name, new_bid)
+            return
+        yield from super().modify_tensors(data_torch, name, bid)

vendor/llama.cpp/convert_lora_to_gguf.py

@@ -315,2 +315,6 @@ #!/usr/bin/env python3
     parser.add_argument(
+        "--trust-remote-code", default=False, action="store_true",
+        help="trust remote code in the model",
+    )
+    parser.add_argument(
         "lora_path", type=Path,
@@ -323,7 +327,7 @@ help="directory containing Hugging Face PEFT LoRA config (adapter_model.json) and weights (adapter_model.safetensors or adapter_model.bin)",
 
-def load_hparams_from_hf(hf_model_id: str) -> tuple[dict[str, Any], Path | None]:
+def load_hparams_from_hf(hf_model_id: str, trust_remote_code: bool) -> tuple[dict[str, Any], Path | None]:
     from huggingface_hub import try_to_load_from_cache
 
     # normally, adapter does not come with base model config, we need to load it from AutoConfig
-    config = AutoConfig.from_pretrained(hf_model_id)
+    config = AutoConfig.from_pretrained(hf_model_id, trust_remote_code=trust_remote_code)
     cache_dir = try_to_load_from_cache(hf_model_id, "config.json")
@@ -377,3 +381,3 @@ cache_dir = Path(cache_dir).parent if isinstance(cache_dir, str) else None
         logger.info(f"Loading base model from Hugging Face: {base_model_id}")
-        hparams, dir_base_model = load_hparams_from_hf(base_model_id)
+        hparams, dir_base_model = load_hparams_from_hf(base_model_id, args.trust_remote_code)
     elif dir_base_model is None:
@@ -384,3 +388,3 @@ if "base_model_name_or_path" in lparams:
             try:
-                hparams, dir_base_model = load_hparams_from_hf(model_id)
+                hparams, dir_base_model = load_hparams_from_hf(model_id, args.trust_remote_code)
             except OSError as e:
@@ -400,3 +404,5 @@ logger.error(f"Failed to load base model config: {e}")
         try:
-            model_class = get_model_class(hparams["architectures"][0])
+            model_arch = hparams.get("text_config", {}).get("architectures", hparams["architectures"])[0]
+            logger.info("Using model architecture: %s", model_arch)
+            model_class = get_model_class(model_arch)
         except NotImplementedError:
@@ -403,0 +409,0 @@ logger.error(f"Model {hparams['architectures'][0]} is not supported")

vendor/llama.cpp/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp

@@ -41,2 +41,3 @@ // SPDX-FileCopyrightText: Copyright 2025-2026 Arm Limited and/or its affiliates <[e-mail]>
 #include "ggml-cpu.h"
+#include "ggml-cpu-impl.h"
 #include "ggml-impl.h"
@@ -65,3 +66,4 @@ #include "ggml-backend-impl.h"
     int thread_hint;    // <= 0 means “no hint”
-} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, 0, -1 };
+    int chunk_multiplier;
+} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, 0, -1, 4 };
 
@@ -191,4 +193,5 @@ static const char* cpu_feature_to_string(cpu_feature f) {
 
-        const char *env_sme     = getenv("GGML_KLEIDIAI_SME");
-        const char *env_threads = getenv("GGML_TOTAL_THREADS");
+        const char *env_sme         = getenv("GGML_KLEIDIAI_SME");
+        const char *env_threads     = getenv("GGML_TOTAL_THREADS");
+        const char *env_chunk_mult  = getenv("GGML_KLEIDIAI_CHUNK_MULTIPLIER");
 
@@ -210,2 +213,10 @@ const bool cpu_has_sme = ggml_cpu_has_sme();
 
+        if (env_chunk_mult) {
+            bool ok = false;
+            int multiplier = parse_uint_env(env_chunk_mult, "GGML_KLEIDIAI_CHUNK_MULTIPLIER", &ok);
+            if (ok && multiplier > 0) {
+                ctx.chunk_multiplier = multiplier;
+            }
+        }
+
         // SME policy:
@@ -303,2 +314,46 @@ // - If CPU doesn't support SME: SME always off.
 
+static inline size_t gcd_size(size_t a, size_t b) {
+    while (b != 0) {
+        const size_t t = a % b;
+        a = b;
+        b = t;
+    }
+    return a;
+}
+
+static inline bool lcm_size(size_t a, size_t b, size_t & result) {
+    if (a == 0 || b == 0) {
+        result = 0;
+        return false;
+    }
+    const size_t g = gcd_size(a, b);
+    const size_t q = a / g;
+    if (q > SIZE_MAX / b) {
+        return false;
+    }
+    result = q * b;
+    return true;
+}
+
+static inline size_t ceil_div_size(size_t a, size_t b) {
+    return b == 0 ? 0 : (a + b - 1) / b;
+}
+
+struct kleidiai_block_args {
+    size_t lhs_bl;
+    size_t rhs_bl;
+    size_t pack_bl;
+};
+
+static inline kleidiai_block_args kleidiai_get_block_args(ggml_type rhs_type) {
+    switch (rhs_type) {
+        case GGML_TYPE_Q4_0:
+            return { QK4_0, QK4_0, QK4_0 };
+        case GGML_TYPE_Q8_0:
+            return { 0, 0, QK8_0 };
+        default:
+            return { 0, 0, 0 };
+    }
+}
+
 static inline bool kleidiai_pack_fallback_allowed() {
@@ -754,4 +809,6 @@ if (ctx.sme_thread_cap <= 0) {
             size_t lhs_offset;
-            size_t n_offset;
-            size_t n_cols;
+            size_t lhs_bl;
+            size_t rhs_bl;
+            size_t pack_bl;
+            size_t lhs_packed_offset0;
             int assigned_threads;
@@ -781,2 +838,4 @@ int thread_begin;
 
+            const kleidiai_block_args block_args = kleidiai_get_block_args(kernels->rhs_type);
+
             runtime[runtime_count] = {
@@ -794,2 +853,5 @@ slot,
                 0,
+                block_args.lhs_bl,
+                block_args.rhs_bl,
+                block_args.pack_bl,
                 0,
@@ -799,3 +861,2 @@ 0,
                 0,
-                0,
                 rhs_ptr
@@ -807,41 +868,4 @@ };
         if (runtime_count == 0) {
-            ggml_kleidiai_kernels * fallback = ggml_kleidiai_select_kernels(ctx.features, dst);
-            if (!fallback) {
-                return false;
-            }
-            kernel_info * kinfo      = is_gemv ? &fallback->gemv : &fallback->gemm;
-            lhs_packing_info * linfo = is_gemv ? &fallback->gemv_lhs_info : &fallback->gemm_lhs_info;
-            rhs_packing_info * rinfo = &fallback->rhs_info;
-            if (!kinfo || !linfo || !linfo->packed_size_ex || !linfo->pack_func_ex ||
-                !kinfo->get_rhs_packed_offset_ex || !kinfo->run_kernel_ex || !kinfo->get_dst_offset ||
-                !rinfo || !rinfo->pack_func_ex || !rinfo->packed_size_ex) {
-                return false;
-            }
-            kernel_chain[0] = fallback;
-            runtime[0] = {
-                0,
-                fallback,
-                kinfo,
-                linfo,
-                kinfo->get_mr(),
-                kinfo->get_nr(),
-                kinfo->get_kr(),
-                kinfo->get_sr(),
-                kinfo->get_n_step(),
-                0,
-                0,
-                0,
-                0,
-                0,
-                0,
-                0,
-                nullptr
-            };
-            size_t rhs_size_fallback = 0;
-            const uint8_t * rhs_base = weight_for_slot(0, rhs_size_fallback);
-            if (!rhs_base) {
-                rhs_base = static_cast<const uint8_t *>(src0->data);
-            }
-            runtime[0].rhs_base = rhs_base;
-            runtime_count = 1;
+            GGML_LOG_WARN("kleidiai: no runtime kernel slot available for supported op %s\n", dst->name);
+            return false;
         }
@@ -859,2 +883,9 @@
         }
+        int non_sme_slot = -1;
+        for (int i = 0; i < runtime_count; ++i) {
+            if ((runtime[i].kernels->required_cpu & CPU_FEATURE_SME) != CPU_FEATURE_SME) {
+                non_sme_slot = i;
+                break;
+            }
+        }
 
@@ -878,3 +909,3 @@ const int sme_cap_limit = ctx.sme_thread_cap;
             if (too_small_for_hybrid && sme_slot != -1) {
-                chosen_slot = sme_slot;
+                chosen_slot = nth_total > sme_cap_limit && non_sme_slot != -1 ? non_sme_slot : sme_slot;
             } else if (runtime_count > 1 && ctx.sme_thread_cap > 0 && nth_total > ctx.sme_thread_cap) {
@@ -885,2 +916,5 @@ chosen_slot = 1;
                 runtime[0] = runtime[chosen_slot];
+                runtime[0].assigned_threads = 0;
+                runtime[0].thread_begin = 0;
+                runtime[0].thread_end = 0;
             }
@@ -912,2 +946,4 @@ runtime_count = runtime_count > 0 ? 1 : 0;
         int fallback_count = 0;
+        // The current hybrid chain is bounded to SME + one non-SME fallback slot.
+        GGML_ASSERT(GGML_KLEIDIAI_MAX_KERNEL_SLOTS == 2);
         for (int i = 0; i < runtime_count; ++i) {
@@ -969,8 +1005,6 @@ if (i == sme_slot) {
         for (int i = 0; i < runtime_count; ++i) {
-            const ggml_type slot_rhs_type = runtime[i].kernels->rhs_type;
-            const size_t slot_pack_size_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :
-                                              slot_rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : 0;
-            runtime[i].lhs_packed_size = runtime[i].lhs_info->packed_size_ex(m, k, slot_pack_size_arg, runtime[i].mr, runtime[i].kr, runtime[i].sr);
+            runtime[i].lhs_packed_size = runtime[i].lhs_info->packed_size_ex(m, k, runtime[i].pack_bl, runtime[i].mr, runtime[i].kr, runtime[i].sr);
             cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);
             runtime[i].lhs_offset = cursor;
+            runtime[i].lhs_packed_offset0 = runtime[i].lhs_info->get_packed_offset_ex(0, k, runtime[i].lhs_bl, runtime[i].mr, runtime[i].kr, runtime[i].sr);
             cursor += runtime[i].lhs_packed_size;
@@ -982,48 +1016,47 @@ }
 
-        size_t assigned_cols = 0;
-        uint64_t weighted_total = 0;
-        if (runtime_count > 1 && sme_slot != -1) {
-            for (int i = 0; i < runtime_count; ++i) {
-                const uint64_t weight = (i == sme_slot) ? (sme_cap << 1) : 1;
-                weighted_total += (uint64_t)runtime[i].assigned_threads * weight;
-            }
-        }
+        size_t common_step = 1;
         for (int i = 0; i < runtime_count; ++i) {
-            runtime[i].n_offset = assigned_cols;
             if (runtime[i].assigned_threads == 0) {
-                runtime[i].n_cols = 0;
                 continue;
             }
-            const size_t remaining_cols = n - assigned_cols;
-            if (remaining_cols == 0) {
-                runtime[i].n_cols = 0;
-                continue;
+            size_t next_step = 0;
+            if (!lcm_size(common_step, runtime[i].n_step ? runtime[i].n_step : 1, next_step)) {
+                return false;
             }
-            const size_t step = runtime[i].n_step ? runtime[i].n_step : 1;
-            size_t target      = 0;
-            if (weighted_total > 0) {
-                const uint64_t weight = (i == sme_slot) ? (sme_cap << 1) : 1;
-                target = (size_t)(((uint64_t)n * runtime[i].assigned_threads * weight) / weighted_total);
-            } else {
-                target = (size_t)(((uint64_t)n * runtime[i].assigned_threads) / nth_total);
-            }
-            target             = std::min(target, remaining_cols);
-            size_t aligned     = round_down(target, step);
-            if (aligned == 0 && remaining_cols >= step) {
-                aligned = step;
-            }
-            runtime[i].n_cols = aligned;
-            assigned_cols += aligned;
+            common_step = next_step;
         }
+        GGML_ASSERT(common_step > 0);
 
-        if (assigned_cols < n) {
-            for (int i = runtime_count - 1; i >= 0; --i) {
-                if (runtime[i].assigned_threads > 0) {
-                    runtime[i].n_cols += n - assigned_cols;
-                    break;
-                }
-            }
+        const bool disable_chunking = ggml_is_numa();
+        const size_t chunk_multiplier = std::max(1, ctx.chunk_multiplier);
+        const size_t chunk_divisor = (nth_total == 1 || disable_chunking) ? (size_t)nth_total : (size_t)nth_total * chunk_multiplier;
+        size_t chunk_cols = align_up(std::max<size_t>(1, ceil_div_size(n, chunk_divisor)), common_step);
+        if (chunk_cols == 0) {
+            chunk_cols = common_step;
         }
+        // If common_step is larger than n, the loop below runs one valid tail chunk
+        // with cols == n.
+        const size_t nchunk_size = std::max<size_t>(1, ceil_div_size(n, chunk_cols));
+        GGML_ASSERT(nchunk_size <= (size_t)INT_MAX);
+        const int nchunk = (int)nchunk_size;
         const size_t dst_stride = dst->nb[1];
 
+        auto run_chunk = [&](runtime_slot & slot, size_t global_start, size_t cols, uint8_t * dst_batch_base) {
+            const size_t rhs_packed_offset = slot.kernel->get_rhs_packed_offset_ex(global_start, k, slot.rhs_bl);
+            const size_t dst_offset        = slot.kernel->get_dst_offset(0, global_start, dst_stride);
+
+            const uint8_t * lhs_ptr = scratch + slot.lhs_offset + slot.lhs_packed_offset0;
+            const uint8_t * rhs_ptr = slot.rhs_base + rhs_packed_offset;
+            float * dst_ptr         = reinterpret_cast<float *>(dst_batch_base + dst_offset);
+
+            slot.kernel->run_kernel_ex(m, cols, k, slot.rhs_bl,
+                                       lhs_ptr,
+                                       rhs_ptr,
+                                       dst_ptr,
+                                       dst_stride,
+                                       sizeof(float),
+                                       -FLT_MAX,
+                                       FLT_MAX);
+        };
+
         for (int64_t batch_idx = 0; batch_idx < ne12; ++batch_idx) {
@@ -1035,5 +1068,2 @@ const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2];
                 runtime_slot & slot = runtime[local_slot];
-                const ggml_type slot_rhs_type = slot.kernels->rhs_type;
-                const size_t slot_lhs_exec_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :
-                                                 slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0;
                 const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)slot.mr);
@@ -1051,4 +1081,4 @@ int64_t max_threads = slot.mr ? (m_roundup_mr / (int64_t)slot.mr) : slot.assigned_threads;
 
-                    const size_t base_packed_off  = slot.lhs_info->get_packed_offset_ex(m_start, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr);
-                    const size_t next_block_off   = slot.lhs_info->get_packed_offset_ex(m_start + slot.mr, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr);
+                    const size_t base_packed_off  = slot.lhs_info->get_packed_offset_ex(m_start, k, slot.lhs_bl, slot.mr, slot.kr, slot.sr);
+                    const size_t next_block_off   = slot.lhs_info->get_packed_offset_ex(m_start + slot.mr, k, slot.lhs_bl, slot.mr, slot.kr, slot.sr);
                     const size_t row_stride_bytes = slot.mr ? (next_block_off - base_packed_off) / slot.mr : 0;
@@ -1070,3 +1100,3 @@
 
-                        slot.lhs_info->pack_func_ex(take, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr, 0, src_ptr, src1->nb[1], dst_ptr);
+                        slot.lhs_info->pack_func_ex(take, k, slot.lhs_bl, slot.mr, slot.kr, slot.sr, 0, src_ptr, src1->nb[1], dst_ptr);
 
@@ -1079,45 +1109,25 @@ cur       += take;
 
+            if (ith_total == 0) {
+                ggml_threadpool_chunk_set(params->threadpool, nth_total);
+            }
+
+            // Publishes both LHS packing and the initialized dynamic chunk queue.
             ggml_barrier(params->threadpool);
 
             runtime_slot & slot = runtime[local_slot];
-            if (slot.n_cols > 0 && slot.assigned_threads > 0) {
-                int64_t active_threads = slot.assigned_threads;
-                const int64_t max_threads = slot.n_step ? (slot.n_cols / slot.n_step) : slot.assigned_threads;
-                if (max_threads > 0) {
-                    active_threads = std::min<int64_t>(active_threads, std::max<int64_t>(1, max_threads));
+            int current_chunk = ith_total;
+            while (current_chunk < nchunk) {
+                const size_t global_start = (size_t)current_chunk * chunk_cols;
+                if (global_start >= n) {
+                    break;
                 }
-                active_threads = std::max<int64_t>(1, active_threads);
 
-                if (local_ith < active_threads) {
-                    const size_t step = slot.n_step ? slot.n_step : 1;
-                    const size_t chunk0 = round_down((size_t)(slot.n_cols / active_threads), step);
-                    const size_t chunkN = slot.n_cols - (active_threads - 1) * chunk0;
-                    const size_t local_start = (size_t)local_ith * chunk0;
-                    const size_t cols = (local_ith == active_threads - 1) ? chunkN : chunk0;
+                const size_t cols = std::min(chunk_cols, n - global_start);
+                if (cols > 0) {
+                    // KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths;
+                    // only non-tail chunks are guaranteed to be n_step-aligned.
+                    run_chunk(slot, global_start, cols, dst_batch_base);
+                }
 
-                    if (cols > 0) {
-                        const ggml_type slot_rhs_type = slot.kernels->rhs_type;
-                        const size_t slot_lhs_exec_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :
-                                                         slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0;
-                        const size_t slot_rhs_block_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :
-                                                          slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0;
-                        const size_t global_start = slot.n_offset + local_start;
-                        const size_t lhs_packed_offset = slot.lhs_info->get_packed_offset_ex(0, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr);
-                        const size_t rhs_packed_offset = slot.kernel->get_rhs_packed_offset_ex(global_start, k, slot_rhs_block_arg);
-                        const size_t dst_offset        = slot.kernel->get_dst_offset(0, global_start, dst_stride);
-
-                        const uint8_t * lhs_ptr = scratch + slot.lhs_offset + lhs_packed_offset;
-                        const uint8_t * rhs_ptr = slot.rhs_base + rhs_packed_offset;
-                        float * dst_ptr         = reinterpret_cast<float *>(dst_batch_base + dst_offset);
-
-                        slot.kernel->run_kernel_ex(m, cols, k, slot_rhs_block_arg,
-                                                   lhs_ptr,
-                                                   rhs_ptr,
-                                                   dst_ptr,
-                                                   dst_stride,
-                                                   sizeof(float),
-                                                   -FLT_MAX,
-                                                   FLT_MAX);
-                    }
-                }
+                current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);
             }
@@ -1124,0 +1134,0 @@

vendor/llama.cpp/gguf-py/gguf/gguf_writer.py

@@ -962,4 +962,9 @@ from __future__ import annotations
     def add_num_deepstack_layers(self, count: int) -> None:
+        """Add scalar deepstack layer count (qwen3vl format)"""
         self.add_uint32(Keys.LLM.NUM_DEEPSTACK_LAYERS.format(arch=self.arch), count)
 
+    def add_deepstack_mapping(self, layers: Sequence[int]) -> None:
+        """Add per-layer deepstack projector indices (Granite4 Vision format)"""
+        self.add_array(Keys.LLM.DEEPSTACK_MAPPING.format(arch=self.arch), list(layers))
+
     def add_rope_dimension_count(self, count: int) -> None:
@@ -1188,2 +1193,11 @@ self.add_uint32(Keys.Rope.DIMENSION_COUNT.format(arch=self.arch), count)
 
+    def add_vision_projector_query_side(self, value: int) -> None:
+        self.add_uint32(Keys.ClipVision.Projector.QUERY_SIDE, value)
+
+    def add_vision_projector_window_side(self, value: int) -> None:
+        self.add_uint32(Keys.ClipVision.Projector.WINDOW_SIDE, value)
+
+    def add_vision_spatial_offsets(self, layers: Sequence[int]) -> None:
+        self.add_array(Keys.ClipVision.Projector.SPATIAL_OFFSETS, layers)
+
     def add_vision_image_mean(self, values: Sequence[float]) -> None:
@@ -1245,2 +1259,8 @@ self.add_array(Keys.ClipVision.IMAGE_MEAN, values)
 
+    def add_vision_feature_layers(self, layers: Sequence[int]) -> None:
+        self.add_array(Keys.ClipVision.FEATURE_LAYERS, layers)
+
+    def add_vision_image_grid_pinpoints(self, layers: Sequence[Sequence[int]]) -> None:
+        self.add_array(Keys.ClipVision.IMAGE_GRID_PINPOINTS, layers)
+
     def add_vision_sam_layers_count(self, value: int) -> None:
@@ -1247,0 +1267,0 @@ self.add_uint32(Keys.ClipVision.SAM.BLOCK_COUNT, value)

vendor/llama.cpp/scripts/ui-assets.cmake

@@ -129,4 +129,18 @@ # Provision UI assets and generate ui.cpp/ui.h.
 
-    if(NOT EXISTS "${UI_SOURCE_DIR}/node_modules")
-        message(STATUS "UI: running npm install (first time)")
+    # npm writes node_modules/.package-lock.json on every successful install,
+    # so a package-lock.json newer than this marker means node_modules is stale
+    set(NPM_MARKER "${UI_SOURCE_DIR}/node_modules/.package-lock.json")
+    set(need_install FALSE)
+    if(NOT EXISTS "${NPM_MARKER}")
+        set(need_install TRUE)
+    else()
+        file(TIMESTAMP "${UI_SOURCE_DIR}/package-lock.json" lock_ts)
+        file(TIMESTAMP "${NPM_MARKER}" marker_ts)
+        if(lock_ts STRGREATER marker_ts)
+            set(need_install TRUE)
+        endif()
+    endif()
+
+    if(need_install)
+        message(STATUS "UI: running npm install")
         execute_process(
@@ -133,0 +147,0 @@ COMMAND ${NPM_EXECUTABLE} install

vendor/llama.cpp/src/llama-adapter.cpp

@@ -44,3 +44,3 @@ #include "llama-adapter.h"
             ggml_init_params params = {
-                /*.mem_size   =*/ hparams.n_layer*ggml_tensor_overhead(),
+                /*.mem_size   =*/ hparams.n_layer()*ggml_tensor_overhead(),
                 /*.mem_buffer =*/ NULL,
@@ -65,5 +65,5 @@ /*.no_alloc   =*/ true,
     // make tensors
-    tensors.reserve(hparams.n_layer);
+    tensors.reserve(hparams.n_layer());
     tensors.push_back(nullptr); // there's never a tensor for layer 0
-    for (size_t il = 1; il < hparams.n_layer; il++) {
+    for (size_t il = 1; il < hparams.n_layer(); il++) {
         ggml_backend_buffer_type_t buft = model.select_buft(il);
@@ -126,3 +126,3 @@ ggml_context * ctx = ctx_for_buft(buft);
 
-    for (size_t il = 1; il < hparams.n_layer; il++) {
+    for (size_t il = 1; il < hparams.n_layer(); il++) {
         assert(tensors[il] != nullptr);
@@ -129,0 +129,0 @@

vendor/llama.cpp/src/llama-arch.h

@@ -203,2 +203,3 @@ #pragma once
     LLM_KV_NUM_DEEPSTACK_LAYERS,
+    LLM_KV_DEEPSTACK_MAPPING,
     LLM_KV_HIDDEN_ACT,
@@ -205,0 +206,0 @@ LLM_KV_POOLING_TYPE,

vendor/llama.cpp/src/llama-hparams.cpp

@@ -10,27 +10,34 @@ #include "llama-hparams.h"
     if (dense_first) {
-        for (uint32_t il = 0; il < n_layer; ++il) {
+        for (uint32_t il = 0; il < n_layer(); ++il) {
             is_swa_impl[il] = n_pattern == 0 || (il % n_pattern != 0);
         }
     } else {
-        for (uint32_t il = 0; il < n_layer; ++il) {
+        for (uint32_t il = 0; il < n_layer(); ++il) {
             is_swa_impl[il] = n_pattern == 0 || (il % n_pattern < (n_pattern - 1));
         }
     }
+
+    for (uint32_t il = n_layer(); il < n_layer_all; ++il) {
+        is_swa_impl[il] = false;
+    }
 }
 
-// TODO: implement
-//void llama_hparams::set_recr_pattern(uint32_t n_pattern, bool dense_first) {
-//    if (dense_first) {
-//        for (uint32_t il = 0; il < n_layer; ++il) {
-//            is_recr_impl[il] = n_pattern == 0 || (il % n_pattern != 0);
-//        }
-//    } else {
-//        for (uint32_t il = 0; il < n_layer; ++il) {
-//            is_recr_impl[il] = n_pattern == 0 || (il % n_pattern < (n_pattern - 1));
-//        }
-//    }
-//}
+void llama_hparams::set_recr_pattern(uint32_t n_pattern, bool dense_first) {
+    if (dense_first) {
+        for (uint32_t il = 0; il < n_layer(); ++il) {
+            is_recr_impl[il] = n_pattern == 0 || (il % n_pattern != 0);
+        }
+    } else {
+        for (uint32_t il = 0; il < n_layer(); ++il) {
+            is_recr_impl[il] = n_pattern == 0 || (il % n_pattern < (n_pattern - 1));
+        }
+    }
 
+    for (uint32_t il = n_layer(); il < n_layer_all; ++il) {
+        is_recr_impl[il] = false;
+    }
+}
+
 bool llama_hparams::is_swa_any() const {
-    for (uint32_t il = 0; il < n_layer; ++il) {
+    for (uint32_t il = 0; il < n_layer_all; ++il) {
         if (is_swa_impl[il]) {
@@ -45,3 +52,3 @@ return true;
 uint32_t llama_hparams::n_head(uint32_t il) const {
-    if (il < n_layer) {
+    if (il < n_layer_all) {
         return n_head_arr[il];
@@ -54,3 +61,3 @@ }
 uint32_t llama_hparams::n_head_kv(uint32_t il) const {
-    if (il < n_layer) {
+    if (il < n_layer_all) {
         return n_head_kv_arr[il];
@@ -63,3 +70,3 @@ }
 uint32_t llama_hparams::n_ff(uint32_t il) const {
-    if (il < n_layer) {
+    if (il < n_layer_all) {
         return n_ff_arr[il];
@@ -83,3 +90,3 @@ }
 uint32_t llama_hparams::n_rot(uint32_t il) const {
-    if (il < n_layer) {
+    if (il < n_layer_all) {
         return is_swa(il) ? n_rot_swa : n_rot_full;
@@ -106,3 +113,3 @@ }
 uint32_t llama_hparams::n_embd_head_k(uint32_t il) const {
-    if (il < n_layer) {
+    if (il < n_layer_all) {
         return is_swa(il) ? n_embd_head_k_swa : n_embd_head_k_full;
@@ -115,3 +122,3 @@ }
 uint32_t llama_hparams::n_embd_head_v(uint32_t il) const {
-    if (il < n_layer) {
+    if (il < n_layer_all) {
         return is_swa(il) ? n_embd_head_v_swa : n_embd_head_v_full;
@@ -137,3 +144,3 @@ }
     const uint32_t val = n_embd_k_gqa();
-    for (uint32_t il = 0; il < n_layer; ++il) {
+    for (uint32_t il = 0; il < n_layer_all; ++il) {
         if (val != n_embd_k_gqa(il)) {
@@ -149,3 +156,3 @@ return true;
     const uint32_t val = n_embd_v_gqa();
-    for (uint32_t il = 0; il < n_layer; ++il) {
+    for (uint32_t il = 0; il < n_layer_all; ++il) {
         if (val != n_embd_v_gqa(il)) {
@@ -161,3 +168,3 @@ return true;
     uint32_t val = n_embd_k_gqa();
-    for (uint32_t il = 0; il < n_layer; ++il) {
+    for (uint32_t il = 0; il < n_layer_all; ++il) {
         val = std::max(val, n_embd_k_gqa(il));
@@ -171,3 +178,3 @@ }
     uint32_t val = n_embd_v_gqa();
-    for (uint32_t il = 0; il < n_layer; ++il) {
+    for (uint32_t il = 0; il < n_layer_all; ++il) {
         val = std::max(val, n_embd_v_gqa(il));
@@ -221,7 +228,7 @@ }
 bool llama_hparams::is_recr(uint32_t il) const {
-    if (il < n_layer) {
+    if (il < n_layer_all) {
         return is_recr_impl[il];
     }
 
-    GGML_ABORT("%s: il (%u) out of bounds (n_layer: %u)\n", __func__, il, n_layer);
+    GGML_ABORT("%s: il (%u) out of bounds (n_layer_all: %u)\n", __func__, il, n_layer_all);
 }
@@ -234,7 +241,7 @@
 bool llama_hparams::is_swa(uint32_t il) const {
-    if (il < n_layer) {
+    if (il < n_layer_all) {
         return is_swa_impl[il];
     }
 
-    GGML_ABORT("fatal error");
+    GGML_ABORT("%s: il (%u) out of bounds (n_layer_all: %u)\n", __func__, il, n_layer_all);
 }
@@ -258,8 +265,2 @@
 bool llama_hparams::has_kv(uint32_t il) const {
-    if (kv_only_nextn) {
-        // MTP head: only the trailing nextn_predict_layers blocks own a KV cache;
-        // the leading trunk blocks are not executed in this graph.
-        return nextn_predict_layers > 0 && il >= (n_layer - nextn_predict_layers);
-    }
-
     if (n_layer_kv_from_start >= 0) {
@@ -277,12 +278,4 @@ if (il < (uint32_t) n_layer_kv_from_start) {
 
-uint32_t llama_hparams::n_layer_kv() const {
-    uint32_t res = 0;
-
-    for (uint32_t il = 0; il < n_layer; ++il) {
-        if (has_kv(il)) {
-            res++;
-        }
-    }
-
-    return res;
+uint32_t llama_hparams::n_layer() const {
+    return n_layer_all - n_layer_nextn;
 }
@@ -289,0 +282,0 @@

vendor/llama.cpp/src/llama-hparams.h

@@ -51,4 +51,4 @@ #pragma once
     uint32_t n_embd;
-    uint32_t n_layer;
-    int32_t  n_layer_kv_from_start = -1; // if non-negative, the first n_layer_kv_from_start layers have KV cache
+    uint32_t n_layer_all;
+    uint32_t n_layer_nextn = 0;
     uint32_t n_expert = 0;
@@ -58,2 +58,5 @@ uint32_t n_expert_used = 0;
 
+    // TODO: this needs to be reworked
+    int32_t  n_layer_kv_from_start = -1; // if non-negative, the first n_layer_kv_from_start layers have KV cache
+
     // different head size for full_attention and SWA layers
@@ -101,6 +104,3 @@ uint32_t n_embd_head_k_full; // dimension of keys (d_k). d_q is assumed to be the same, but there are n_head q heads, and only n_head_kv k-v heads
     uint32_t moe_latent_size      = 0;
-    uint32_t nextn_predict_layers = 0;
 
-    bool kv_only_nextn = false; // if true, only the last nextn_predict_layers blocks have a KV cache (MTP head arches)
-
     float f_norm_eps;
@@ -225,4 +225,14 @@ float f_norm_rms_eps;
     // qwen3vl deepstack
+    // When parsed from GGUF, this implies the first N layers consume the first
+    // N deepstack embeddings. Use deepstack_mapping_arr if you need a more
+    // complex mapping. If using deepstack_mapping_arr, also make sure to set
+    // n_deepstack_layers to the number of unique deepstack layers so that
+    // n_embd_imp is accurate (see granite.cpp).
     uint32_t n_deepstack_layers = 0;
 
+    // deepstack layer array (Granite4 Vision)
+    // -1  => no deepstack
+    // >=0 => input embedding index for deepstack injection
+    std::array<int32_t, LLAMA_MAX_LAYERS> deepstack_mapping_arr;
+
     // gemma4 per-layer embedding
@@ -279,4 +289,3 @@ uint32_t n_embd_per_layer = 0;
 
-    // TODO: implement
-    //void set_recr_pattern(uint32_t n_pattern, bool dense_first = false);
+    void set_recr_pattern(uint32_t n_pattern, bool dense_first = false);
 
@@ -337,4 +346,4 @@ // whether or not the given layer is recurrent (for hybrid models)
 
-    // number of layers for which has_kv() returns true
-    uint32_t n_layer_kv() const;
+    // number of effective layers (excludes nextn layers)
+    uint32_t n_layer() const;
 
@@ -341,0 +350,0 @@ // note that this function uses different SWA parameters from those in the hparams

vendor/llama.cpp/src/llama-memory-recurrent.cpp

@@ -29,3 +29,3 @@ #include "llama-memory-recurrent.h"
     const layer_filter_cb & filter) : hparams(model.hparams), n_seq_max(n_seq_max) {
-    const int32_t n_layer = hparams.n_layer;
+    const int32_t n_layer = hparams.n_layer();
 
@@ -867,3 +867,3 @@ head = 0;
     const uint32_t s_trans = 0;
-    const uint32_t n_layer = hparams.n_layer;
+    const uint32_t n_layer = hparams.n_layer();
 
@@ -1052,4 +1052,4 @@ io.write(&s_trans, sizeof(s_trans));
 
-    if (n_layer != hparams.n_layer) {
-        LLAMA_LOG_ERROR("%s: mismatched layer count (%u instead of %u)\n", __func__, n_layer, hparams.n_layer);
+    if (n_layer != hparams.n_layer()) {
+        LLAMA_LOG_ERROR("%s: mismatched layer count (%u instead of %u)\n", __func__, n_layer, hparams.n_layer());
         return false;
@@ -1056,0 +1056,0 @@ }

vendor/llama.cpp/src/llama-model-saver.cpp

@@ -80,3 +80,3 @@ #include "llama-model-saver.h"
     GGML_ASSERT(model != nullptr || !per_layer);
-    const size_t n_values = per_layer ? size_t(model->hparams.n_layer) : value.size();
+    const size_t n_values = per_layer ? size_t(model->hparams.n_layer()) : value.size();
     GGML_ASSERT(n_values <= value.size());
@@ -210,3 +210,3 @@
     }
-    add_kv(LLM_KV_BLOCK_COUNT,                       hparams.n_layer);
+    add_kv(LLM_KV_BLOCK_COUNT,                       hparams.n_layer_all);
     add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT,         hparams.n_layer_dense_lead);
@@ -232,4 +232,5 @@ add_kv(LLM_KV_FEED_FORWARD_LENGTH,               hparams.n_ff_arr, true);
     add_kv(LLM_KV_MOE_EVERY_N_LAYERS,                hparams.moe_every_n_layers);
-    add_kv(LLM_KV_NEXTN_PREDICT_LAYERS,              hparams.nextn_predict_layers);
+    add_kv(LLM_KV_NEXTN_PREDICT_LAYERS,              hparams.n_layer_nextn);
     add_kv(LLM_KV_NUM_DEEPSTACK_LAYERS,              hparams.n_deepstack_layers);
+    add_kv(LLM_KV_DEEPSTACK_MAPPING,                 hparams.deepstack_mapping_arr);
     add_kv(LLM_KV_POOLING_TYPE,                      uint32_t(hparams.pooling_type));
@@ -236,0 +237,0 @@ add_kv(LLM_KV_LOGIT_SCALE,                       hparams.f_logit_scale);

vendor/llama.cpp/src/llama-model.h

@@ -703,3 +703,4 @@ #pragma once
 #define LLAMA_LOAD_LOCALS \
-    const int     n_layer        = hparams.n_layer;          GGML_UNUSED(n_layer); \
+    const int     n_layer        = hparams.n_layer();        GGML_UNUSED(n_layer); \
+    const int     n_layer_all    = hparams.n_layer_all;      GGML_UNUSED(n_layer_all); \
     const int64_t n_head         = hparams.n_head();         GGML_UNUSED(n_head); \
@@ -706,0 +707,0 @@ const int64_t n_head_kv      = hparams.n_head_kv();      GGML_UNUSED(n_head_kv); \

vendor/llama.cpp/src/llama-quant.cpp

@@ -850,3 +850,3 @@ #include "llama-impl.h"
     }
-    qs.n_ffn_down = qs.n_ffn_gate = qs.n_ffn_up = (int)qs.model.hparams.n_layer;
+    qs.n_ffn_down = qs.n_ffn_gate = qs.n_ffn_up = (int)qs.model.hparams.n_layer();
 }
@@ -1352,3 +1352,3 @@
     model->hparams.n_embd_head_v_full = desc->n_embd_head_v;
-    model->hparams.n_layer            = desc->n_layer;
+    model->hparams.n_layer_all        = desc->n_layer;
     model->hparams.n_expert           = desc->n_expert;
@@ -1355,0 +1355,0 @@

vendor/llama.cpp/src/models/afmoe.cpp

@@ -33,3 +33,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 56: type = LLM_TYPE_6B; break;
@@ -36,0 +36,0 @@ case 32: type = LLM_TYPE_26B; break;

vendor/llama.cpp/src/models/apertus.cpp

@@ -5,8 +5,9 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    ml.get_key_or_arr(LLM_KV_XIELU_ALPHA_N,        hparams.xielu_alpha_n, hparams.n_layer);
-    ml.get_key_or_arr(LLM_KV_XIELU_ALPHA_P,        hparams.xielu_alpha_p, hparams.n_layer);
-    ml.get_key_or_arr(LLM_KV_XIELU_BETA,           hparams.xielu_beta,    hparams.n_layer);
-    ml.get_key_or_arr(LLM_KV_XIELU_EPS,            hparams.xielu_eps,     hparams.n_layer);
 
-    switch (hparams.n_layer) {
+    ml.get_key_or_arr(LLM_KV_XIELU_ALPHA_N, hparams.xielu_alpha_n, hparams.n_layer());
+    ml.get_key_or_arr(LLM_KV_XIELU_ALPHA_P, hparams.xielu_alpha_p, hparams.n_layer());
+    ml.get_key_or_arr(LLM_KV_XIELU_BETA,    hparams.xielu_beta,    hparams.n_layer());
+    ml.get_key_or_arr(LLM_KV_XIELU_EPS,     hparams.xielu_eps,     hparams.n_layer());
+
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_8B; break;
@@ -13,0 +14,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/arcee.cpp

@@ -7,3 +7,3 @@ #include "models.h"
     // Arcee uses the same structure as Llama
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 36: type = LLM_TYPE_4B; break;
@@ -10,0 +10,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/arctic.cpp

@@ -7,3 +7,3 @@ #include "models.h"
     if (hparams.n_expert == 128) {
-        switch (hparams.n_layer) {
+        switch (hparams.n_layer()) {
             case 35: type = LLM_TYPE_10B_128x3_66B; break;
@@ -10,0 +10,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/arwkv7.cpp

@@ -13,3 +13,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 12:
@@ -16,0 +16,0 @@ switch (hparams.n_embd) {

vendor/llama.cpp/src/models/baichuan.cpp

@@ -5,3 +5,3 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_7B; break;
@@ -8,0 +8,0 @@ case 40: type = LLM_TYPE_13B; break;

vendor/llama.cpp/src/models/bailingmoe.cpp

@@ -11,3 +11,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 28: type = LLM_TYPE_16B; break;
@@ -14,0 +14,0 @@ case 88: type = LLM_TYPE_290B; break;

vendor/llama.cpp/src/models/bailingmoe2.cpp

@@ -12,13 +12,9 @@ #include "models.h"
     ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                hparams.expert_gating_func);
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,              hparams.nextn_predict_layers, false);
-    GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,              hparams.n_layer_nextn, false);
 
-    // TODO: when MTP is implemented, this should probably be updated if needed
-    hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 20: type = LLM_TYPE_16B_A1B; break;
-        case 21: type = LLM_TYPE_16B_A1B; break;
         case 32: type = LLM_TYPE_100B_A6B; break;
-        case 33: type = LLM_TYPE_100B_A6B; break;
         default: type = LLM_TYPE_UNKNOWN;
@@ -43,5 +39,5 @@ }
 
-    for (int i = 0; i < n_layer; ++i) {
+    for (int i = 0; i < n_layer_all; ++i) {
         int flags = 0;
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        if (i >= n_layer) {
             // skip all tensors in the NextN layers
@@ -83,3 +79,3 @@ flags |= TENSOR_SKIP;
         // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        if (i >= n_layer) {
             layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags);
@@ -118,4 +114,3 @@ layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags);
 
-    const int n_transformer_layers = n_layer - hparams.nextn_predict_layers;
-    for (int il = 0; il < n_transformer_layers; ++il) {
+    for (int il = 0; il < n_layer; ++il) {
         ggml_tensor * inpSA = inpL;
@@ -153,3 +148,3 @@
 
-        if (il == n_transformer_layers - 1 && inp_out_ids) {
+        if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
@@ -156,0 +151,0 @@ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);

vendor/llama.cpp/src/models/bert.cpp

 #include "models.h"
 
 void llama_model_bert::load_arch_hparams(llama_model_loader & ml) {
-    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 3:
@@ -8,0 +8,0 @@ type = LLM_TYPE_17M; break; // bge-micro

vendor/llama.cpp/src/models/bitnet.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 26: type = LLM_TYPE_3B; break;
@@ -9,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/bloom.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_1B; break;
@@ -9,0 +9,0 @@ case 30:

vendor/llama.cpp/src/models/chameleon.cpp

@@ -9,3 +9,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_7B; break;
@@ -12,0 +12,0 @@ case 48: type = LLM_TYPE_34B; break;

vendor/llama.cpp/src/models/chatglm.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 28: {
@@ -8,0 +9,0 @@ if (hparams.n_head(0) == 16) {

vendor/llama.cpp/src/models/codeshell.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 42: type = LLM_TYPE_7B; break;
@@ -8,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/cogvlm.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_13B; break;
@@ -8,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/cohere2.cpp

@@ -8,2 +8,3 @@ #include "models.h"
     hparams.set_swa_pattern(swa_period);
+
     hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;
@@ -16,3 +17,4 @@ hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,  hparams.f_norm_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_8B; break;
@@ -19,0 +21,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/command-r.cpp

@@ -6,3 +6,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 40: type = LLM_TYPE_35B; break;
@@ -9,0 +10,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/dbrx.cpp

 #include "models.h"
 
 void llama_model_dbrx::load_arch_hparams(llama_model_loader & ml) {
-ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
-ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV,     hparams.f_clamp_kqv);
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+    ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV,     hparams.f_clamp_kqv);
 
-switch (hparams.n_layer) {
-    case 40: type = LLM_TYPE_16x12B; break;
-    default: type = LLM_TYPE_UNKNOWN;
+    switch (hparams.n_layer()) {
+        case 40: type = LLM_TYPE_16x12B; break;
+        default: type = LLM_TYPE_UNKNOWN;
+    }
 }
-        }
 
@@ -13,0 +13,0 @@ void llama_model_dbrx::load_arch_tensors(llama_model_loader &) {

vendor/llama.cpp/src/models/deci.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_7B; break;
@@ -8,0 +9,0 @@ case 80: type = LLM_TYPE_70B; break;

vendor/llama.cpp/src/models/deepseek2.cpp

@@ -8,3 +8,3 @@ #include "models.h"
     // lite variants include DeepSeek-V2-Lite, GigaChat3-10B-A1.8B, Kanana-2-30B-A3B
-    const bool is_lite = (hparams.n_layer == 27 || hparams.n_layer == 26 || (hparams.n_layer == 48 && n_vocab == 128256));
+    const bool is_lite = (hparams.n_layer() == 27 || hparams.n_layer() == 26 || (hparams.n_layer() == 48 && n_vocab == 128256));
 
@@ -27,3 +27,3 @@ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
         // that have no expert_gating_func model parameter set
-        if ((hparams.n_layer == 47 || hparams.n_layer == 48) && n_vocab == 154880) {
+        if ((hparams.n_layer() == 47 || hparams.n_layer() == 48) && n_vocab == 154880) {
             // GLM 4.7 Lite
@@ -48,3 +48,3 @@ hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 27: type = LLM_TYPE_16B; break;
@@ -197,4 +197,3 @@ case 47: type = LLM_TYPE_30B_A3B; break;
 
-    int effective_n_layers = hparams.n_layer - hparams.nextn_predict_layers;
-    for (int il = 0; il < effective_n_layers; ++il) {
+    for (int il = 0; il < n_layer; ++il) {
         ggml_tensor * inpSA = inpL;
@@ -373,3 +372,3 @@
         }
-        if (il == effective_n_layers - 1 && inp_out_ids) {
+        if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
@@ -376,0 +375,0 @@ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);

vendor/llama.cpp/src/models/deepseek2ocr.cpp

@@ -17,3 +17,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 12: type = LLM_TYPE_3B; break;
@@ -20,0 +20,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/deepseek32.cpp

@@ -34,3 +34,3 @@ #include "models.h"
     // Expert gating function
-    ml.get_key(LLM_KV_EXPERT_GATING_FUNC,          hparams.expert_gating_func);
+    ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
 
@@ -44,9 +44,6 @@ if (ml.get_key(LLM_KV_ROPE_SCALING_YARN_LOG_MUL, hparams.rope_yarn_log_mul, 0.0f)) {
     // NextN/MTP parameters
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,        hparams.nextn_predict_layers, false);
-    GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");
 
-    // TODO: when MTP is implemented, this should probably be updated if needed
-    hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
-
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 62: type = LLM_TYPE_685B_A37B; break;
@@ -87,5 +84,5 @@ default: type = LLM_TYPE_UNKNOWN;
 
-    for (int i = 0; i < n_layer; ++i) {
+    for (int i = 0; i < n_layer_all; ++i) {
         int flags = 0;
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        if (i >= n_layer) {
             // skip all tensors in the NextN layers
@@ -148,3 +145,3 @@ // TODO @ngxson : TENSOR_NOT_REQUIRED was a hack, need to remove it later
         // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        if (i >= n_layer) {
             layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags);
@@ -212,4 +209,3 @@ layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags);
 
-    int effective_n_layers = hparams.n_layer - hparams.nextn_predict_layers;
-    for (int il = 0; il < effective_n_layers; ++il) {
+    for (int il = 0; il < n_layer; ++il) {
         ggml_tensor * inpSA = inpL;
@@ -435,3 +431,3 @@
         }
-        if (il == effective_n_layers - 1 && inp_out_ids) {
+        if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
@@ -438,0 +434,0 @@ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);

vendor/llama.cpp/src/models/dots1.cpp

@@ -11,3 +11,4 @@ #include "models.h"
     ml.get_key(LLM_KV_EXPERT_GATING_FUNC,          hparams.expert_gating_func, false);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 62: type = LLM_TYPE_142B; break;
@@ -14,0 +15,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/dream.cpp

@@ -5,4 +5,5 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
     // Dream models are primarily 7B with 28 layers
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 28:
@@ -9,0 +10,0 @@ type = LLM_TYPE_7B;

vendor/llama.cpp/src/models/ernie4-5.cpp

@@ -15,3 +15,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 18: type = LLM_TYPE_0_3B; break;
@@ -18,0 +18,0 @@ case 28: type = LLM_TYPE_21B_A3B; break;

vendor/llama.cpp/src/models/eurobert.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    if (hparams.n_layer == 12) {
+    if (hparams.n_layer() == 12) {
         type = LLM_TYPE_SMALL;  // 0.2B
@@ -9,0 +9,0 @@ }

vendor/llama.cpp/src/models/exaone-moe.cpp

@@ -23,9 +23,8 @@ #include "models.h"
 
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,              hparams.nextn_predict_layers, false);
-    GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_30B_A3B; break;
-        case 48:
-        case 49: type = LLM_TYPE_235B_A22B; break;
+        case 48: type = LLM_TYPE_235B_A22B; break;
         default: type = LLM_TYPE_UNKNOWN;
@@ -54,5 +53,5 @@ }
 
-    for (int i = 0; i < n_layer; ++i) {
+    for (int i = 0; i < n_layer_all; ++i) {
         int flags = 0;
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        if (i >= n_layer) {
             // skip all tensors in the NextN layers
@@ -75,3 +74,3 @@ flags |= TENSOR_SKIP;
         // dense layers for first n_layer_dense_lead layers or nextn_predict_layers layers at the end
-        if (i < (int) hparams.n_layer_dense_lead || (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers)) {
+        if (i < (int) hparams.n_layer_dense_lead || (i >= n_layer)) {
             layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, flags);
@@ -101,3 +100,3 @@ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, flags);
         // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        if (i >= n_layer) {
             layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2 * n_embd, n_embd}, flags);
@@ -137,4 +136,3 @@ layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM,   "weight", i), {n_embd}, flags);
 
-    const int n_transformer_layers = n_layer - hparams.nextn_predict_layers;
-    for (int il = 0; il < n_transformer_layers; ++il) {
+    for (int il = 0; il < n_layer; ++il) {
         ggml_tensor * inpSA = inpL;
@@ -178,3 +176,3 @@
         }
-        if (il == n_transformer_layers - 1 && inp_out_ids) {
+        if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
@@ -181,0 +179,0 @@ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);

vendor/llama.cpp/src/models/exaone.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_8B; break;
@@ -9,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/exaone4.cpp

 #include "models.h"
 
 void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) {
-    if (hparams.n_layer == 64) {    // 32B
+    if (hparams.n_layer() == 64) {    // 32B
         hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
@@ -18,7 +18,7 @@ hparams.n_swa = 4096;
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,        hparams.nextn_predict_layers, false);
-    GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
-    hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,        hparams.n_layer_nextn, false);
 
-    switch (hparams.n_layer) {
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");
+
+    switch (hparams.n_layer()) {
         case 30: type = LLM_TYPE_1_2B; break;
@@ -44,4 +44,4 @@ case 64: type = LLM_TYPE_32B; break;
 
-    for (int i = 0; i < n_layer; ++i) {
-        const bool is_nextn = hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers;
+    for (int i = 0; i < n_layer_all; ++i) {
+        const bool is_nextn = i >= n_layer;
         int flags = 0;
@@ -114,7 +114,3 @@ if (is_nextn) {
 
-    // MTP / NextN tail blocks are loaded for compatibility but not executed (same as exaone-moe).
-    const int n_layer_main = int(n_layer) - int(hparams.nextn_predict_layers);
-    GGML_ASSERT(n_layer_main > 0);
-
-    for (int il = 0; il < n_layer_main; ++il) {
+    for (int il = 0; il < n_layer; ++il) {
         ggml_tensor * inpSA = inpL;
@@ -155,3 +151,3 @@
         }
-        if (il == n_layer_main - 1 && inp_out_ids) {
+        if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
@@ -158,0 +154,0 @@ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);

vendor/llama.cpp/src/models/falcon-h1.cpp

@@ -16,3 +16,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 36:
@@ -19,0 +19,0 @@ type = LLM_TYPE_0_5B; break;

vendor/llama.cpp/src/models/falcon.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_7B; break;
@@ -9,0 +9,0 @@ case 60: type = LLM_TYPE_40B; break;

vendor/llama.cpp/src/models/gemma-embedding.cpp

@@ -24,3 +24,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_0_3B; break;
@@ -27,0 +27,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/gemma.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 18: type = LLM_TYPE_2B; break;
@@ -9,0 +9,0 @@ case 28: type = LLM_TYPE_7B; break;

vendor/llama.cpp/src/models/gemma2.cpp

@@ -19,3 +19,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 26: type = LLM_TYPE_2B; break;
@@ -22,0 +22,0 @@ case 42: type = LLM_TYPE_9B; break;

vendor/llama.cpp/src/models/gemma3.cpp

@@ -20,3 +20,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 18: type = LLM_TYPE_270M; break;
@@ -23,0 +23,0 @@ case 26: type = LLM_TYPE_1B; break;

vendor/llama.cpp/src/models/gemma3n.cpp

@@ -9,4 +9,4 @@ #include "models.h"
 
-    hparams.n_layer_kv_from_start     = 20;
-    hparams.f_attention_scale         = 1.0f;
+    hparams.n_layer_kv_from_start = 20;
+    hparams.f_attention_scale     = 1.0f;
 
@@ -17,3 +17,3 @@ ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,          hparams.rope_freq_base_train_swa, false);
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 30: type = LLM_TYPE_E2B; break;
@@ -20,0 +20,0 @@ case 35: type = LLM_TYPE_E4B; break;

vendor/llama.cpp/src/models/gemma4.cpp

@@ -5,3 +5,3 @@ #include "models.h"
     hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
-    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer);
+    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
 
@@ -11,3 +11,3 @@ uint32_t n_kv_shared_layers = 0;
 
-    hparams.n_layer_kv_from_start = hparams.n_layer - (int32_t)n_kv_shared_layers;
+    hparams.n_layer_kv_from_start = hparams.n_layer_all - (int32_t)n_kv_shared_layers;
     hparams.f_attention_scale     = 1.0f; // Gemma4 uses self.scaling = 1.0 (no pre-attn scaling)
@@ -24,3 +24,3 @@
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 30: type = LLM_TYPE_26B_A4B; break;
@@ -27,0 +27,0 @@ case 35: type = LLM_TYPE_E2B; break;

vendor/llama.cpp/src/models/glm-dsa.cpp

@@ -36,9 +36,6 @@ #include "models.h"
     // NextN/MTP parameters
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,        hparams.nextn_predict_layers, false);
-    GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
 
-    // TODO: when MTP is implemented, this should probably be updated if needed
-    hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
-
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 79: type = LLM_TYPE_744B_A40B; break;
@@ -80,5 +77,5 @@ default: type = LLM_TYPE_UNKNOWN;
 
-    for (int i = 0; i < n_layer; ++i) {
+    for (int i = 0; i < n_layer_all; ++i) {
         int flags = 0;
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        if (i >= n_layer) {
             // skip all tensors in the NextN layers
@@ -140,4 +137,4 @@ // TODO @ngxson : TENSOR_NOT_REQUIRED was a hack, need to remove it later
 
-        // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        // NextN/MTP tensors (preserved but unused) - conditionally load for last n_layer_nextn
+        if (i >= n_layer) {
             layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags);
@@ -144,0 +141,0 @@ layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags);

vendor/llama.cpp/src/models/glm4-moe.cpp

@@ -23,12 +23,9 @@ #include "models.h"
     // NextN/MTP parameters
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,        hparams.nextn_predict_layers, false);
-    GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
 
-    // TODO: when MTP is implemented, this should probably be updated if needed
-    hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
-
-    switch (hparams.n_layer) {
-        case 47: type = LLM_TYPE_106B_A12B; break; // GLM-4.5-Air (46 layers + 1 NextN layer)
+    switch (hparams.n_layer()) {
+        case 46: type = LLM_TYPE_106B_A12B; break; // GLM-4.5-Air
         case 48: type = LLM_TYPE_102B_A12B; break; // Solar Open
-        case 93: type = LLM_TYPE_355B_A32B; break; // GLM-4.5 (92 layers + 1 NextN layer)
+        case 92: type = LLM_TYPE_355B_A32B; break; // GLM-4.5
         default: type = LLM_TYPE_UNKNOWN;
@@ -58,5 +55,5 @@ }
     // but only PROCESS up to last layer (skipping final NextN layer) in forward pass
-    for (int i = 0; i < n_layer; ++i) {
+    for (int i = 0; i < n_layer_all; ++i) {
         int flags = 0;
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        if (i >= n_layer) {
             // skip all tensors in the NextN layers
@@ -121,3 +118,3 @@ flags |= TENSOR_SKIP;
         // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        if (i >= n_layer) {
             layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags);
@@ -167,4 +164,3 @@ layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags);
     // Final layer tensors are loaded but not processed in forward pass
-    const int n_transformer_layers = n_layer - hparams.nextn_predict_layers;
-    for (int il = 0; il < n_transformer_layers; ++il) {
+    for (int il = 0; il < n_layer; ++il) {
         ggml_tensor * inpSA = inpL;
@@ -218,3 +214,3 @@
         }
-        if (il == n_transformer_layers - 1 && inp_out_ids) {
+        if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
@@ -221,0 +217,0 @@ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);

vendor/llama.cpp/src/models/glm4.cpp

@@ -8,9 +8,6 @@ #include "models.h"
     // NextN/MTP parameters (GLM-OCR)
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);
-    GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
 
-    // TODO: when MTP is implemented, this should probably be updated if needed
-    hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
-
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 17: type = LLM_TYPE_1B; break; // GLM-OCR
@@ -36,5 +33,5 @@ case 40: type = LLM_TYPE_9B; break;
 
-    for (int i = 0; i < n_layer; ++i) {
+    for (int i = 0; i < n_layer_all; ++i) {
         int flags = 0;
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        if (i >= n_layer) {
             // skip all tensors in the NextN layers
@@ -60,3 +57,3 @@ flags |= TENSOR_SKIP;
         // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers
-        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
+        if (i >= n_layer) {
             layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags);
@@ -106,4 +103,3 @@ layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags);
     // Final layer tensors are loaded but not processed in forward pass
-    const int n_transformer_layers = n_layer - hparams.nextn_predict_layers;
-    for (int il = 0; il < n_transformer_layers; ++il) {
+    for (int il = 0; il < n_layer; ++il) {
         ggml_tensor * inpSA = inpL;
@@ -147,3 +143,3 @@
         }
-        if (il == n_transformer_layers - 1 && inp_out_ids) {
+        if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
@@ -150,0 +146,0 @@ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);

vendor/llama.cpp/src/models/gpt2.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 12: type = LLM_TYPE_SMALL; break;
@@ -8,0 +9,0 @@ case 24: type = LLM_TYPE_MEDIUM; break;

vendor/llama.cpp/src/models/gptneox.cpp

@@ -6,3 +6,4 @@ #include "models.h"
     ml.get_key(LLM_KV_USE_PARALLEL_RESIDUAL,   hparams.use_par_res);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 6:
@@ -9,0 +10,0 @@ switch (hparams.n_ff()) {

vendor/llama.cpp/src/models/granite-hybrid.cpp

@@ -22,3 +22,3 @@ #include "models.h"
     // A layer is recurrent IFF the n_head_kv value is set to 0
-    for (uint32_t i = 0; i < hparams.n_layer; ++i) {
+    for (uint32_t i = 0; i < hparams.n_layer(); ++i) {
         hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0;
@@ -25,0 +25,0 @@ }

vendor/llama.cpp/src/models/granite-moe.cpp

@@ -15,3 +15,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_3B; break;
@@ -18,0 +18,0 @@ case 40: type = LLM_TYPE_3B; break;

vendor/llama.cpp/src/models/granite.cpp

 #include "models.h"
 
+#include <sstream>
+
 void llama_model_granite::load_arch_hparams(llama_model_loader & ml) {
@@ -10,2 +12,23 @@ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
 
+    // Granite4 Vision uses array deepstack_mapping
+    ml.get_arr(LLM_KV_DEEPSTACK_MAPPING, hparams.deepstack_mapping_arr, false);
+
+    // Count the unique deepstack input indices
+    std::unordered_set<uint32_t> unique_deepstack_idxs;
+    for (const auto val : hparams.deepstack_mapping_arr) {
+        if (val >= 0) {
+            unique_deepstack_idxs.insert(val);
+        }
+    }
+    hparams.n_deepstack_layers = unique_deepstack_idxs.size();
+
+    // Ensure all values are valid (avoid overflow attacks)
+    for (const auto val : unique_deepstack_idxs) {
+        if (val > hparams.n_deepstack_layers) {
+            std::stringstream ss;
+            ss << "Invalid deepstack index: " << val << " > " << hparams.n_deepstack_layers;
+            throw std::runtime_error(ss.str());
+        }
+    }
+
     // Granite uses rope_finetuned as a switch for rope, so default to true
@@ -16,3 +39,3 @@ bool rope_finetuned = true;
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_3B; break;
@@ -117,2 +140,16 @@ case 40: type = LLM_TYPE_3B; break;
     for (int il = 0; il < n_layer; ++il) {
+
+        // Granite Vision 4.1 deepstack: inject the projector stream that
+        // targets decoder layer `il` before the decoder runs.
+        // NOTE: skip the first deepstack layer since that's inpL
+        const auto & deepstack_emb_idx = hparams.deepstack_mapping_arr[il];
+        if (il > 0 && deepstack_emb_idx >= 0) {
+            ggml_tensor * ds = ggml_view_2d(ctx0,
+                res->t_inp_embd, n_embd, n_tokens,
+                res->t_inp_embd->nb[1],
+                deepstack_emb_idx * n_embd * sizeof(float));
+            inpL = ggml_add(ctx0, inpL, ds);
+            cb(inpL, "deepstack_in", il);
+        }
+
         ggml_tensor * inpSA = inpL;
@@ -119,0 +156,0 @@

vendor/llama.cpp/src/models/grok.cpp

@@ -29,3 +29,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 64: type = LLM_TYPE_314B; break;
@@ -32,0 +32,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/grovemoe.cpp

@@ -10,3 +10,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 48: type = LLM_TYPE_30B_A3B; break;
@@ -13,0 +13,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/hunyuan-moe.cpp

@@ -8,3 +8,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_A13B; break;
@@ -11,0 +11,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/internlm2.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_7B; break;
@@ -8,0 +9,0 @@ case 48: type = LLM_TYPE_20B; break;

vendor/llama.cpp/src/models/jais.cpp

@@ -7,3 +7,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_1_3B; break;
@@ -10,0 +10,0 @@ case 40: type = LLM_TYPE_13B; break;

vendor/llama.cpp/src/models/jais2.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_8B; break;
@@ -9,0 +9,0 @@ case 68: type = LLM_TYPE_70B; break;

vendor/llama.cpp/src/models/jamba.cpp

@@ -11,7 +11,7 @@ #include "models.h"
 
-    for (uint32_t i = 0; i < hparams.n_layer; ++i) {
+    for (uint32_t i = 0; i < hparams.n_layer(); ++i) {
         hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0;
     }
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         // TODO: Jamba layers are a bit heterogeneous, so naming this is hard.
@@ -18,0 +18,0 @@ case 12: // 900M  8x???M

vendor/llama.cpp/src/models/jina-bert-v2.cpp

@@ -7,3 +7,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 4:  type = LLM_TYPE_33M;  break; // jina-embeddings-small
@@ -10,0 +10,0 @@ case 12: type = LLM_TYPE_137M; break; // jina-embeddings-base

vendor/llama.cpp/src/models/jina-bert-v3.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24:
@@ -9,0 +9,0 @@ type = LLM_TYPE_558M; break;

vendor/llama.cpp/src/models/kimi-linear.cpp

@@ -17,3 +17,3 @@ #include "models.h"
     // Set n_head_kv = 0 for KDA layers (recurrent), n_head_kv = n_head for MLA layers (attention)
-    for (uint32_t i = 0; i < hparams.n_layer; ++i) {
+    for (uint32_t i = 0; i < hparams.n_layer(); ++i) {
         hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0;  // KDA layers are recurrent
@@ -29,3 +29,3 @@ }
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 27: type = LLM_TYPE_48B_A3B; break; // Kimi-Linear-48B-A3B
@@ -32,0 +32,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/lfm2.cpp

@@ -8,6 +8,9 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    for (uint32_t il = 0; il < hparams.n_layer; ++il) {
+
+    for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
         hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0;
     }
-    hparams.n_layer_dense_lead = hparams.n_layer;
+
+    hparams.n_layer_dense_lead = hparams.n_layer();
+
     switch (hparams.n_ff()) {
@@ -20,5 +23,6 @@ case  4608: type = LLM_TYPE_350M; break;
     }
+
     if (const auto is_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); is_swa && hparams.n_swa > 0) {
         hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
-        for (uint32_t il = 0; il < hparams.n_layer; ++il) {
+        for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
             hparams.is_swa_impl[il] = !hparams.is_recr_impl[il];
@@ -25,0 +29,0 @@ }

vendor/llama.cpp/src/models/lfm2moe.cpp

@@ -12,7 +12,7 @@ #include "models.h"
 
-    for (uint32_t il = 0; il < hparams.n_layer; ++il) {
+    for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
         hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0;
     }
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_8B_A1B;  break;
@@ -19,0 +19,0 @@ case 40: type = LLM_TYPE_24B_A2B; break;

vendor/llama.cpp/src/models/llada-moe.cpp

@@ -5,7 +5,8 @@ #include "models.h"
     ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
 
-    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
     // diffusion language model uses non-causal attention
     hparams.causal_attn = false;
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 16: type = LLM_TYPE_A1_7B; break;
@@ -12,0 +13,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/llada.cpp

@@ -5,4 +5,5 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
     // LLaDA-8B has 32 layers, similar to LLaMA but for diffusion
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32:
@@ -14,2 +15,3 @@ type = LLM_TYPE_8B;
     }
+
     // Set non-causal attention for diffusion models
@@ -16,0 +18,0 @@ hparams.causal_attn = false;

vendor/llama.cpp/src/models/llama.cpp

@@ -10,3 +10,3 @@ #include "models.h"
     if (hparams.n_expert == 8) {
-        switch (hparams.n_layer) {
+        switch (hparams.n_layer()) {
             case 32: type = LLM_TYPE_8x7B; break;
@@ -17,3 +17,3 @@ case 56: type = LLM_TYPE_8x22B; break;
     } else {
-        switch (hparams.n_layer) {
+        switch (hparams.n_layer()) {
             case 16: type = LLM_TYPE_1B; break; // Llama 3.2 1B
@@ -20,0 +20,0 @@ case 22: type = LLM_TYPE_1B; break;

vendor/llama.cpp/src/models/llama4.cpp

@@ -11,3 +11,3 @@ #include "models.h"
         hparams.swa_type             = LLAMA_SWA_TYPE_NONE;
-        hparams.n_no_rope_layer_step = hparams.n_layer; // always use rope
+        hparams.n_no_rope_layer_step = hparams.n_layer(); // always use rope
     } else {
@@ -14,0 +14,0 @@ hparams.swa_type                = LLAMA_SWA_TYPE_CHUNKED;

vendor/llama.cpp/src/models/maincoder.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_1B; break;
@@ -8,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/mamba.cpp

@@ -12,3 +12,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24:
@@ -15,0 +15,0 @@ switch (hparams.n_embd) {

vendor/llama.cpp/src/models/mamba2.cpp

@@ -12,3 +12,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24:
@@ -15,0 +15,0 @@ switch (hparams.n_embd) {

vendor/llama.cpp/src/models/mellum.cpp

@@ -16,3 +16,3 @@ #include "models.h"
         } else {
-            ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer);
+            ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
         }
@@ -28,3 +28,3 @@
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 28: type = LLM_TYPE_12B_A2_5B; break;
@@ -31,0 +31,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/mimo2.cpp

@@ -12,3 +12,3 @@ #include "models.h"
 
-    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer);
+    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
 
@@ -20,7 +20,6 @@ float value_scale = 0.0f;
 
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);
-    GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
-    hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
 
-    switch (hparams.n_layer - hparams.nextn_predict_layers) {
+    switch (hparams.n_layer()) {
         case 48: type = LLM_TYPE_310B_A15B; break;
@@ -40,5 +39,3 @@ default: type = LLM_TYPE_UNKNOWN;
 
-    const uint32_t n_nextn = hparams.nextn_predict_layers;
-
-    for (int i = 0; i < n_layer; ++i) {
+    for (int i = 0; i < n_layer_all; ++i) {
         auto & layer = layers[i];
@@ -50,3 +47,3 @@ uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(i);
         // NextN/MTP layers (the last n_nextn blocks) are preserved but disabled pending support
-        const bool is_nextn = (n_nextn > 0) && (static_cast<uint32_t>(i) >= n_layer - n_nextn);
+        const bool is_nextn = i >= n_layer;
         const int  skip     = is_nextn ? TENSOR_SKIP : 0;
@@ -100,6 +97,3 @@
 
-    // The last hparams.nextn_predict_layers blocks are MTP heads, currently inactive
-    const int n_transformer_layers = n_layer - hparams.nextn_predict_layers;
-
-    for (int il = 0; il < n_transformer_layers; ++il) {
+    for (int il = 0; il < n_layer; ++il) {
         ggml_tensor * inpSA = inpL;
@@ -182,3 +176,3 @@
 
-        if (il == n_transformer_layers - 1 && inp_out_ids) {
+        if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);
@@ -185,0 +179,0 @@ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);

vendor/llama.cpp/src/models/minicpm.cpp

@@ -6,3 +6,3 @@ #include "models.h"
     hparams.f_embedding_scale = 12.0f;
-    hparams.f_residual_scale  = 1.4f / sqrtf(float(hparams.n_layer));
+    hparams.f_residual_scale  = 1.4f / sqrtf(float(hparams.n_layer()));
     hparams.f_logit_scale     = hparams.n_embd ? (256.0f / float(hparams.n_embd)) : 1.0f;
@@ -20,3 +20,3 @@
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 52: type = LLM_TYPE_1B; break;
@@ -23,0 +23,0 @@ case 40: type = LLM_TYPE_2B; break;

vendor/llama.cpp/src/models/minicpm3.cpp

@@ -8,3 +8,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 62: type = LLM_TYPE_4B; break;
@@ -11,0 +11,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/minimax-m2.cpp

@@ -8,3 +8,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 62: type = LLM_TYPE_230B_A10B; break;
@@ -11,0 +11,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/mistral3.cpp

@@ -21,3 +21,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 26: type = LLM_TYPE_3B; break;
@@ -24,0 +24,0 @@ case 34: type = LLM_TYPE_8B; break;

vendor/llama.cpp/src/models/modern-bert.cpp

@@ -25,3 +25,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 12:
@@ -28,0 +28,0 @@ type = LLM_TYPE_47M; break; // granite-embedding-small

vendor/llama.cpp/src/models/mpt.cpp

@@ -8,3 +8,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_7B; break;
@@ -11,0 +11,0 @@ case 48: type = LLM_TYPE_30B; break;

vendor/llama.cpp/src/models/nemotron-h.cpp

@@ -12,3 +12,3 @@ #include "models.h"
     // the n_ff value is set to 0
-    for (uint32_t i = 0; i < hparams.n_layer; ++i) {
+    for (uint32_t i = 0; i < hparams.n_layer(); ++i) {
         hparams.is_recr_impl[i] = (hparams.n_head_kv(i) == 0 && hparams.n_ff(i) == 0);
@@ -26,3 +26,3 @@ }
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 52: type = LLM_TYPE_31B_A3_5B; break; // Nemotron-H_MOE 31B
@@ -29,0 +29,0 @@ case 56: type = LLM_TYPE_9B; break;

vendor/llama.cpp/src/models/nemotron.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_4B; break;
@@ -8,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/neo-bert.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    if (hparams.n_layer == 28) {
+    if (hparams.n_layer() == 28) {
         type = LLM_TYPE_250M;
@@ -9,0 +9,0 @@ }

vendor/llama.cpp/src/models/nomic-bert-moe.cpp

@@ -7,3 +7,3 @@ #include "models.h"
 
-    if (hparams.n_layer == 12 && hparams.n_embd == 768) {
+    if (hparams.n_layer() == 12 && hparams.n_embd == 768) {
         if (arch == LLM_ARCH_NOMIC_BERT) {
@@ -10,0 +10,0 @@ type = LLM_TYPE_137M;

vendor/llama.cpp/src/models/nomic-bert.cpp

@@ -7,3 +7,3 @@ #include "models.h"
 
-    if (hparams.n_layer == 12 && hparams.n_embd == 768) {
+    if (hparams.n_layer() == 12 && hparams.n_embd == 768) {
         if (arch == LLM_ARCH_NOMIC_BERT) {
@@ -10,0 +10,0 @@ type = LLM_TYPE_137M;

vendor/llama.cpp/src/models/olmo.cpp

@@ -7,3 +7,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 22: type = LLM_TYPE_1B; break;
@@ -10,0 +10,0 @@ case 32: type = LLM_TYPE_7B; break;

vendor/llama.cpp/src/models/olmo2.cpp

@@ -20,3 +20,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 16: type = LLM_TYPE_1B; break;
@@ -23,0 +23,0 @@ case 32: type = LLM_TYPE_7B; break;

vendor/llama.cpp/src/models/olmoe.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 16: type = LLM_TYPE_A1_7B; break;
@@ -8,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/openai-moe.cpp

@@ -17,3 +17,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_20B; break;
@@ -20,0 +20,0 @@ case 36: type = LLM_TYPE_120B; break;

vendor/llama.cpp/src/models/openelm.cpp

@@ -6,8 +6,8 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
-    case 16: type = LLM_TYPE_270M; break;
-    case 20: type = LLM_TYPE_450M; break;
-    case 28: type = LLM_TYPE_1B; break;
-    case 36: type = LLM_TYPE_3B; break;
-    default: type = LLM_TYPE_UNKNOWN;
+    switch (hparams.n_layer()) {
+        case 16: type = LLM_TYPE_270M; break;
+        case 20: type = LLM_TYPE_450M; break;
+        case 28: type = LLM_TYPE_1B; break;
+        case 36: type = LLM_TYPE_3B; break;
+        default: type = LLM_TYPE_UNKNOWN;
     }
@@ -14,0 +14,0 @@ }

vendor/llama.cpp/src/models/orion.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 40: type = LLM_TYPE_14B; break;
@@ -9,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/pangu-embed.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 26: type = LLM_TYPE_1B; break; // openPangu-Embedded-1B-V1.1
@@ -8,0 +9,0 @@ case 34: type = LLM_TYPE_7B; break; // openPangu-Embedded-7B-V1.1

vendor/llama.cpp/src/models/phi2.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_1B; break;
@@ -9,0 +9,0 @@ case 32: type = LLM_TYPE_3B; break;

vendor/llama.cpp/src/models/phi3.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_1B; break;
@@ -9,0 +9,0 @@ case 32: type = LLM_TYPE_3B; break;

vendor/llama.cpp/src/models/phimoe.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_16x3_8B; break;
@@ -9,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/plamo.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 40: type = LLM_TYPE_13B; break;
@@ -9,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/plamo2.cpp

@@ -14,7 +14,7 @@ #include "models.h"
 
-    for (uint32_t i = 0; i < hparams.n_layer; ++i) {
+    for (uint32_t i = 0; i < hparams.n_layer(); ++i) {
         hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0;
     }
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 16: type = LLM_TYPE_1B; break;
@@ -21,0 +21,0 @@ case 32:

vendor/llama.cpp/src/models/plamo3.cpp

@@ -16,3 +16,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_2B; break;
@@ -19,0 +19,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/plm.cpp

@@ -6,3 +6,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_1_8B; break;
@@ -9,0 +10,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/qwen.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_7B; break;
@@ -9,0 +9,0 @@ case 40: type = LLM_TYPE_13B; break;

vendor/llama.cpp/src/models/qwen2.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 24: type = hparams.n_embd == 1024 ? LLM_TYPE_0_5B : LLM_TYPE_1B; break;
@@ -8,0 +9,0 @@ case 28: type = hparams.n_embd == 1536 ? LLM_TYPE_1_5B : LLM_TYPE_7B; break;

vendor/llama.cpp/src/models/qwen2moe.cpp

@@ -8,3 +8,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_A2_7B; break;
@@ -11,0 +12,0 @@ case 28: type = LLM_TYPE_57B_A14B; break;

vendor/llama.cpp/src/models/qwen3.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 28: type = hparams.n_embd == 1024 ? LLM_TYPE_0_6B : LLM_TYPE_1_7B; break;
@@ -8,0 +9,0 @@ case 36: type = hparams.n_embd == 2560 ? LLM_TYPE_4B : LLM_TYPE_8B; break;

vendor/llama.cpp/src/models/qwen35.cpp

@@ -16,18 +16,16 @@ #include "models.h"
     // NextN/MTP (Qwen3.5/3.6): extra decoder block appended beyond the main stack
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);
-    GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
 
     // Mark recurrent layers (linear attention layers). MTP layers are dense
     // attention-only and must be flagged non-recurrent.
-    if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer, false)) {
-        const uint32_t n_main = hparams.n_layer - hparams.nextn_predict_layers;
-
+    if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
         uint32_t full_attn_interval = 4;
         ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);
-        for (uint32_t i = 0; i < hparams.n_layer; ++i) {
-            hparams.is_recr_impl[i] = (i < n_main) && ((i + 1) % full_attn_interval != 0);
+        for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
+            hparams.is_recr_impl[i] = (i < hparams.n_layer()) && ((i + 1) % full_attn_interval != 0);
         }
     }
 
-    switch (hparams.n_layer - hparams.nextn_predict_layers) {
+    switch (hparams.n_layer()) {
         case 24: type = hparams.n_embd == 1024 ? LLM_TYPE_0_8B : LLM_TYPE_2B; break;
@@ -43,5 +41,3 @@ case 32: type = hparams.n_embd == 2560 ? LLM_TYPE_4B : LLM_TYPE_9B; break;
 
-    const uint32_t n_main = n_layer - hparams.nextn_predict_layers;
-    const bool mtp_only   = (hparams.nextn_predict_layers > 0) &&
-                            (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
+    const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
     const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
@@ -127,6 +123,6 @@
 
-    for (int i = 0; i < (int) n_main; ++i) {
+    for (int i = 0; i < n_layer; ++i) {
         load_block_trunk(i, trunk_flags);
     }
-    for (int i = (int) n_main; i < n_layer; ++i) {
+    for (int i = n_layer; i < n_layer_all; ++i) {
         load_block_mtp(i);
@@ -165,4 +161,3 @@ }
     // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass.
-    const int n_transformer_layers = n_layer - (int) hparams.nextn_predict_layers;
-    for (int il = 0; il < n_transformer_layers; ++il) {
+    for (int il = 0; il < n_layer; ++il) {
         ggml_tensor * inpSA = inpL;
@@ -184,3 +179,3 @@
 
-        if (il == n_transformer_layers - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {
+        if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
@@ -498,4 +493,4 @@ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
     : llm_graph_context(params) {
-    GGML_ASSERT(hparams.nextn_predict_layers > 0 && "QWEN35 MTP requires nextn_predict_layers > 0");
-    GGML_ASSERT(hparams.nextn_predict_layers == 1 && "QWEN35 MTP currently only supports a single MTP block");
+    GGML_ASSERT(hparams.n_layer_nextn > 0 && "QWEN35 MTP requires n_layer_nextn > 0");
+    GGML_ASSERT(hparams.n_layer_nextn == 1 && "QWEN35 MTP currently only supports a single MTP block");
 
@@ -507,3 +502,3 @@ const int64_t n_embd_head = hparams.n_embd_head_v();
     // layer is stored immediately after the main layers in model.layers[].
-    const int il = (int) hparams.n_layer - (int) hparams.nextn_predict_layers;
+    const int il = hparams.n_layer();
     const auto & layer = model.layers[il];
@@ -510,0 +505,0 @@

vendor/llama.cpp/src/models/qwen35moe.cpp

@@ -19,18 +19,16 @@ #include "models.h"
     // NextN/MTP (Qwen3.5/3.6): extra decoder block appended beyond the main stack
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);
-    GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
 
     // Mark recurrent layers (linear attention layers). MTP layers are dense
     // attention-only and must be flagged non-recurrent.
-    if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer, false)) {
-        const uint32_t n_main = hparams.n_layer - hparams.nextn_predict_layers;
-
+    if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
         uint32_t full_attn_interval = 4;
         ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);
-        for (uint32_t i = 0; i < hparams.n_layer; ++i) {
-            hparams.is_recr_impl[i] = (i < n_main) && ((i + 1) % full_attn_interval != 0);
+        for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
+            hparams.is_recr_impl[i] = (i < hparams.n_layer()) && ((i + 1) % full_attn_interval != 0);
         }
     }
 
-    switch (hparams.n_layer - hparams.nextn_predict_layers) {
+    switch (hparams.n_layer()) {
         case 40: type = LLM_TYPE_35B_A3B; break;
@@ -46,5 +44,3 @@ case 48: type = LLM_TYPE_122B_A10B; break;
 
-    const uint32_t n_main = n_layer - hparams.nextn_predict_layers;
-    const bool mtp_only   = (hparams.nextn_predict_layers > 0) &&
-                            (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
+    const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
     const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
@@ -150,6 +146,6 @@
 
-    for (int i = 0; i < (int) n_main; ++i) {
+    for (int i = 0; i < n_layer; ++i) {
         load_block_trunk(i, trunk_flags);
     }
-    for (int i = (int) n_main; i < n_layer; ++i) {
+    for (int i = n_layer; i < n_layer_all; ++i) {
         load_block_mtp(i);
@@ -188,4 +184,3 @@ }
     // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass.
-    const int n_transformer_layers = n_layer - (int) hparams.nextn_predict_layers;
-    for (int il = 0; il < n_transformer_layers; ++il) {
+    for (int il = 0; il < n_layer; ++il) {
         ggml_tensor * inpSA = inpL;
@@ -207,3 +202,3 @@
 
-        if (il == n_transformer_layers - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {
+        if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
@@ -563,4 +558,4 @@ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
     : llm_graph_context(params) {
-    GGML_ASSERT(hparams.nextn_predict_layers > 0 && "QWEN35MOE MTP requires nextn_predict_layers > 0");
-    GGML_ASSERT(hparams.nextn_predict_layers == 1 && "QWEN35MOE MTP currently only supports a single MTP block");
+    GGML_ASSERT(hparams.n_layer_nextn > 0 && "QWEN35MOE MTP requires n_layer_nextn > 0");
+    GGML_ASSERT(hparams.n_layer_nextn == 1 && "QWEN35MOE MTP currently only supports a single MTP block");
 
@@ -570,3 +565,3 @@ const int64_t n_embd_head = hparams.n_embd_head_v();
 
-    const int il = (int) hparams.n_layer - (int) hparams.nextn_predict_layers;
+    const int il = hparams.n_layer();
     const auto & layer = model.layers[il];
@@ -573,0 +568,0 @@

vendor/llama.cpp/src/models/qwen3moe.cpp

 #include "models.h"
 
 void llama_model_qwen3moe::load_arch_hparams(llama_model_loader & ml) {
-    ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp, false);
+    ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,  hparams.n_ff_exp, false);
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
 
-    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 48: type = LLM_TYPE_30B_A3B; break;
@@ -9,0 +9,0 @@ case 94: type = LLM_TYPE_235B_A22B; break;

vendor/llama.cpp/src/models/qwen3next.cpp

@@ -17,11 +17,11 @@ #include "models.h"
     // Mark recurrent layers (linear attention layers)
-    if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer, false)) {
+    if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
         uint32_t full_attn_interval = 4;
         ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);
-        for (uint32_t i = 0; i < hparams.n_layer; ++i) {
-            hparams.is_recr_impl[i] = ((i + 1) % full_attn_interval != 0);
+        for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
+            hparams.is_recr_impl[i] = (i < hparams.n_layer()) && ((i + 1) % full_attn_interval != 0);
         }
     }
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 48: type = LLM_TYPE_80B_A3B; break;
@@ -28,0 +28,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/qwen3vl.cpp

@@ -7,3 +7,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 28: type = LLM_TYPE_1_7B; break;
@@ -10,0 +11,0 @@ case 36: type = hparams.n_embd == 2560 ? LLM_TYPE_4B : LLM_TYPE_8B; break;

vendor/llama.cpp/src/models/qwen3vlmoe.cpp

@@ -8,3 +8,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 48: type = LLM_TYPE_30B_A3B; break;
@@ -11,0 +12,0 @@ case 94: type = LLM_TYPE_235B_A22B; break;

vendor/llama.cpp/src/models/refact.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_1B; break;
@@ -8,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/rnd1.cpp

@@ -5,8 +5,9 @@ #include "models.h"
     ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
 
-    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 48: type = LLM_TYPE_30B_A3B; break;
         default: type = LLM_TYPE_UNKNOWN;
     }
+
     // Set non-causal attention for diffusion models
@@ -13,0 +14,0 @@ hparams.causal_attn = false;

vendor/llama.cpp/src/models/rwkv6.cpp

@@ -12,3 +12,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_1_6B; break;
@@ -15,0 +15,0 @@ case 32:

vendor/llama.cpp/src/models/rwkv6qwen2.cpp

@@ -12,3 +12,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_1_6B; break;
@@ -15,0 +15,0 @@ case 32:

vendor/llama.cpp/src/models/rwkv7.cpp

@@ -13,3 +13,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 12:
@@ -16,0 +16,0 @@ switch (hparams.n_embd) {

vendor/llama.cpp/src/models/seed-oss.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 64: type = LLM_TYPE_36B; break;
@@ -8,0 +9,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/smallthinker.cpp

@@ -18,3 +18,3 @@ #include "models.h"
         hparams.swa_type             = LLAMA_SWA_TYPE_NONE;
-        hparams.n_no_rope_layer_step = hparams.n_layer;
+        hparams.n_no_rope_layer_step = hparams.n_layer();
     }
@@ -26,3 +26,3 @@
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_4B;  break;
@@ -29,0 +29,0 @@ case 52: type = LLM_TYPE_20B; break;

vendor/llama.cpp/src/models/smollm3.cpp

@@ -7,3 +7,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 36: type = LLM_TYPE_3B; break;
@@ -10,0 +10,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/stablelm.cpp

@@ -6,3 +6,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_1B; break;
@@ -9,0 +9,0 @@ case 32: type = LLM_TYPE_3B; break;

vendor/llama.cpp/src/models/starcoder.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 24: type = LLM_TYPE_1B; break;
@@ -8,0 +9,0 @@ case 36: type = LLM_TYPE_3B; break;

vendor/llama.cpp/src/models/starcoder2.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 30: type = LLM_TYPE_3B; break;
@@ -8,0 +9,0 @@ case 32: type = LLM_TYPE_7B; break;

vendor/llama.cpp/src/models/step35.cpp

@@ -26,12 +26,12 @@ #include "models.h"
 
-    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer);
+    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
 
-    ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP,   hparams.swiglu_clamp_exp,   hparams.n_layer, false);
-    ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer, false);
+    ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP,   hparams.swiglu_clamp_exp,   hparams.n_layer(), false);
+    ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer(), false);
 
     // NextN/MTP (Step3p5): extra decoder block appended beyond the main stack.
-    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);
-    GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
+    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
+    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
 
-    switch (hparams.n_layer - hparams.nextn_predict_layers) {
+    switch (hparams.n_layer()) {
         case 45: type = LLM_TYPE_196B_A11B; break;
@@ -45,11 +45,8 @@ default: type = LLM_TYPE_UNKNOWN;
 
-    const uint32_t n_main = n_layer - hparams.nextn_predict_layers;
-    const bool mtp_only   = (hparams.nextn_predict_layers > 0) &&
-                            (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
+    const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
     // Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP
     // tensors live in a separate file (e.g. user split target/draft). Mark
     // MTP tensors NOT_REQUIRED so the trunk loads cleanly.
-    const std::string mtp_probe = "blk." + std::to_string(n_main) + ".nextn.eh_proj.weight";
-    const bool trunk_only = (hparams.nextn_predict_layers > 0) &&
-                            (ml.get_weight(mtp_probe.c_str()) == nullptr);
+    const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
+    const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
     const int trunk_flags = mtp_only  ? TENSOR_NOT_REQUIRED : 0;
@@ -181,3 +178,3 @@ const int mtp_flags   = trunk_only ? TENSOR_NOT_REQUIRED : 0;
 
-    for (int i = 0; i < (int) n_main; ++i) {
+    for (int i = 0; i < n_layer; ++i) {
         load_block_trunk(i, trunk_flags);
@@ -190,4 +187,4 @@ }
     // path. See scripts/prune_step35_extra_mtp.py for the pruner.
-    for (int i = (int) n_main; i < n_layer; ++i) {
-        load_block_mtp(i, /*is_first_mtp=*/ i == (int) n_main);
+    for (int i = n_layer; i < n_layer_all; ++i) {
+        load_block_mtp(i, /*is_first_mtp=*/ i == n_layer);
     }
@@ -213,4 +210,3 @@ }
     // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass.
-    const int n_transformer_layers = n_layer - (int) hparams.nextn_predict_layers;
-    for (int il = 0; il < n_transformer_layers; ++il) {
+    for (int il = 0; il < n_layer; ++il) {
         ggml_tensor * inpSA = inpL;
@@ -302,3 +298,3 @@
 
-        if (il == n_transformer_layers - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {
+        if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) {
             cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
@@ -383,3 +379,3 @@ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
     : llm_graph_context(params) {
-    GGML_ASSERT(hparams.nextn_predict_layers > 0 && "STEP35 MTP requires nextn_predict_layers > 0");
+    GGML_ASSERT(hparams.n_layer_nextn > 0 && "STEP35 MTP requires n_layer_nextn > 0");
 
@@ -392,3 +388,3 @@ // Single-block MTP only: always run the first trained MTP block (Qwen
     // scripts/prune_step35_extra_mtp.py.
-    const int il       = (int) hparams.n_layer - (int) hparams.nextn_predict_layers;
+    const int il = hparams.n_layer();
     const auto & layer = model.layers[il];
@@ -395,0 +391,0 @@

vendor/llama.cpp/src/models/t5.cpp

@@ -12,6 +12,6 @@ #include "models.h"
 
-    hparams.dec_n_layer = hparams.n_layer;
+    hparams.dec_n_layer = hparams.n_layer();
     ml.get_key(LLM_KV_DECODER_BLOCK_COUNT, hparams.dec_n_layer, false);
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 6:  type = LLM_TYPE_60M;  break; // t5-small
@@ -18,0 +18,0 @@ case 8:  type = LLM_TYPE_80M;  break; // flan-t5-small

vendor/llama.cpp/src/models/talkie.cpp

@@ -7,3 +7,3 @@ #include "models.h"
 
-    switch (hparams.n_layer) {
+    switch (hparams.n_layer()) {
         case 40: type = LLM_TYPE_13B; break;
@@ -10,0 +10,0 @@ default: type = LLM_TYPE_UNKNOWN;

vendor/llama.cpp/src/models/xverse.cpp

@@ -5,3 +5,4 @@ #include "models.h"
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-    switch (hparams.n_layer) {
+
+    switch (hparams.n_layer()) {
         case 32: type = LLM_TYPE_7B; break;
@@ -8,0 +9,0 @@ case 40: type = LLM_TYPE_13B; break;

vendor/llama.cpp/tools/cli/cli.cpp

@@ -400,2 +400,4 @@ #include "chat.h"
 
+    ctx_cli.defaults.sampling = params.sampling;
+
     console::spinner::stop();
@@ -402,0 +404,0 @@ console::log("\n");

vendor/llama.cpp/tools/completion/completion.cpp

@@ -36,8 +36,4 @@ #include "arg.h"
 static llama_context           ** g_ctx;
-static llama_model             ** g_model;
 static common_sampler          ** g_smpl;
 static common_params            * g_params;
-static std::vector<llama_token> * g_input_tokens;
-static std::ostringstream       * g_output_ss;
-static std::vector<llama_token> * g_output_tokens;
 static bool is_interacting  = false;
@@ -140,3 +136,2 @@ static bool need_insert_eot = false;
 
-    g_model = &model;
     g_ctx = &ctx;
@@ -554,5 +549,5 @@ g_smpl = &smpl;
 
-    std::vector<int>   input_tokens;  g_input_tokens  = &input_tokens;
-    std::vector<int>   output_tokens; g_output_tokens = &output_tokens;
-    std::ostringstream output_ss;     g_output_ss     = &output_ss;
+    std::vector<int>   input_tokens;
+    std::vector<int>   output_tokens;
+    std::ostringstream output_ss;
     std::ostringstream assistant_ss; // for storing current assistant message, used in conversation mode
@@ -995,3 +990,3 @@
         llama_state_save_file(ctx, path_session.c_str(), session_tokens.data(), session_tokens.size());
-        LOG_INF("saved final session to %s, n_tokens = %ld\n", path_session.data(), session_tokens.size());
+        LOG_INF("saved final session to %s, n_tokens = %zu\n", path_session.data(), session_tokens.size());
 
@@ -998,0 +993,0 @@ }

vendor/llama.cpp/tools/mtmd/clip-graph.h

@@ -40,2 +40,5 @@ #pragma once
 
+    // TODO [QWEN_VIDEO]: improve this in the future
+    int n_batch = 1;
+
     ggml_context_ptr ctx0_ptr;
@@ -42,0 +45,0 @@ ggml_context * ctx0;

vendor/llama.cpp/tools/mtmd/clip-impl.h

@@ -7,2 +7,3 @@ #pragma once
 
+#include <array>
 #include <climits>
@@ -39,16 +40,18 @@ #include <cstdarg>
 // vision-specific
-#define KEY_VISION_PROJ_TYPE    "clip.vision.projector_type" // for models with mixed modalities
-#define KEY_IMAGE_SIZE          "clip.vision.image_size"
-#define KEY_IMAGE_MIN_PIXELS    "clip.vision.image_min_pixels"
-#define KEY_IMAGE_MAX_PIXELS    "clip.vision.image_max_pixels"
-#define KEY_PREPROC_MIN_TILES   "clip.vision.preproc_min_tiles"
-#define KEY_PREPROC_MAX_TILES   "clip.vision.preproc_max_tiles"
-#define KEY_PREPROC_IMAGE_SIZE  "clip.vision.preproc_image_size"
-#define KEY_PATCH_SIZE          "clip.vision.patch_size"
-#define KEY_IMAGE_MEAN          "clip.vision.image_mean"
-#define KEY_IMAGE_STD           "clip.vision.image_std"
-#define KEY_FEATURE_LAYER       "clip.vision.feature_layer"
-#define KEY_PROJ_SCALE_FACTOR   "clip.vision.projector.scale_factor"
-#define KEY_SPATIAL_MERGE_SIZE  "clip.vision.spatial_merge_size"
-#define KEY_IS_DEEPSTACK_LAYERS "clip.vision.is_deepstack_layers"
+#define KEY_VISION_PROJ_TYPE        "clip.vision.projector_type" // for models with mixed modalities
+#define KEY_IMAGE_SIZE              "clip.vision.image_size"
+#define KEY_IMAGE_MIN_PIXELS        "clip.vision.image_min_pixels"
+#define KEY_IMAGE_MAX_PIXELS        "clip.vision.image_max_pixels"
+#define KEY_PREPROC_MIN_TILES       "clip.vision.preproc_min_tiles"
+#define KEY_PREPROC_MAX_TILES       "clip.vision.preproc_max_tiles"
+#define KEY_PREPROC_IMAGE_SIZE      "clip.vision.preproc_image_size"
+#define KEY_PATCH_SIZE              "clip.vision.patch_size"
+#define KEY_IMAGE_MEAN              "clip.vision.image_mean"
+#define KEY_IMAGE_STD               "clip.vision.image_std"
+#define KEY_FEATURE_LAYER           "clip.vision.feature_layer"
+#define KEY_PROJ_SCALE_FACTOR       "clip.vision.projector.scale_factor"
+#define KEY_PROJ_SAMPLE_QUERY_SIDE  "clip.vision.projector.query_side"
+#define KEY_PROJ_SAMPLE_WINDOW_SIDE "clip.vision.projector.window_side"
+#define KEY_PROJ_SPATIAL_OFFSETS    "clip.vision.projector.spatial_offsets"
+#define KEY_SPATIAL_MERGE_SIZE      "clip.vision.spatial_merge_size"
 
@@ -77,3 +80,2 @@ #define KEY_MM_PATCH_MERGE_TYPE    "clip.vision.mm_patch_merge_type"
 
-
 //
@@ -216,18 +218,24 @@ // tensor name constants
 // qformer projector
-#define TN_QF_PROJ_QUERY   "a.proj_query"
-#define TN_QF_PROJ_NORM    "a.proj_norm.%s"
-#define TN_QF_PROJ_LINEAR  "a.proj_linear.%s"
-#define TN_QF_SELF_ATTN_Q  "a.proj_blk.%d.self_attn_q.%s"
-#define TN_QF_SELF_ATTN_K  "a.proj_blk.%d.self_attn_k.%s"
-#define TN_QF_SELF_ATTN_V  "a.proj_blk.%d.self_attn_v.%s"
-#define TN_QF_SELF_ATTN_O  "a.proj_blk.%d.self_attn_out.%s"
-#define TN_QF_SELF_ATTN_N  "a.proj_blk.%d.self_attn_norm.%s"
-#define TN_QF_CROSS_ATTN_Q "a.proj_blk.%d.cross_attn_q.%s"
-#define TN_QF_CROSS_ATTN_K "a.proj_blk.%d.cross_attn_k.%s"
-#define TN_QF_CROSS_ATTN_V "a.proj_blk.%d.cross_attn_v.%s"
-#define TN_QF_CROSS_ATTN_O "a.proj_blk.%d.cross_attn_out.%s"
-#define TN_QF_CROSS_ATTN_N "a.proj_blk.%d.cross_attn_norm.%s"
-#define TN_QF_FFN_UP       "a.proj_blk.%d.ffn_up.%s"
-#define TN_QF_FFN_DOWN     "a.proj_blk.%d.ffn_down.%s"
-#define TN_QF_FFN_NORM     "a.proj_blk.%d.ffn_norm.%s"
+#define TN_QF_PROJ_QUERY   "%s.proj_query"
+#define TN_QF_PROJ_NORM    "%s.proj_norm.%s"
+#define TN_QF_PROJ_LINEAR  "%s.proj_linear.%s"
+#define TN_QF_SELF_ATTN_Q  "%s.proj_blk.%d.self_attn_q.%s"
+#define TN_QF_SELF_ATTN_K  "%s.proj_blk.%d.self_attn_k.%s"
+#define TN_QF_SELF_ATTN_V  "%s.proj_blk.%d.self_attn_v.%s"
+#define TN_QF_SELF_ATTN_O  "%s.proj_blk.%d.self_attn_out.%s"
+#define TN_QF_SELF_ATTN_N  "%s.proj_blk.%d.self_attn_norm.%s"
+#define TN_QF_CROSS_ATTN_Q "%s.proj_blk.%d.cross_attn_q.%s"
+#define TN_QF_CROSS_ATTN_K "%s.proj_blk.%d.cross_attn_k.%s"
+#define TN_QF_CROSS_ATTN_V "%s.proj_blk.%d.cross_attn_v.%s"
+#define TN_QF_CROSS_ATTN_O "%s.proj_blk.%d.cross_attn_out.%s"
+#define TN_QF_CROSS_ATTN_N "%s.proj_blk.%d.cross_attn_norm.%s"
+#define TN_QF_FFN_UP       "%s.proj_blk.%d.ffn_up.%s"
+#define TN_QF_FFN_DOWN     "%s.proj_blk.%d.ffn_down.%s"
+#define TN_QF_FFN_NORM     "%s.proj_blk.%d.ffn_norm.%s"
+// multi-projector qformer (bid => projector ID)
+#define TN_MULTI_PROJ_IMG_POS   "v.proj_blk.%d.img_pos"
+#define TN_MULTI_PROJ_QUERY     "%s.proj_blk.%d.query"
+#define TN_MULTI_PROJ_LINEAR    "%s.proj_blk.%d.linear.%s"
+#define TN_MULTI_PROJ_NORM      "%s.proj_blk.%d.norm.%s"
+#define TN_MULTI_PROJ_POST_NORM "%s.proj_blk.%d.post_norm.%s"
 
@@ -361,2 +369,3 @@ // gemma4 audio conformer
     PROJECTOR_TYPE_MIMOVL,
+    PROJECTOR_TYPE_GRANITE4_VISION,
     PROJECTOR_TYPE_UNKNOWN,
@@ -415,2 +424,3 @@ };
     { PROJECTOR_TYPE_MIMOVL,     "mimovl"},
+    { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
 };
@@ -429,6 +439,64 @@
 struct clip_image_u8 {
-    int nx;
-    int ny;
+    clip_image_size get_size() const {
+        return { nx, ny };
+    }
 
+    void set_size(clip_image_size size, bool is_placeholder) {
+        nx = size.width;
+        ny = size.height;
+        if (is_placeholder) {
+            buf.clear();
+        } else {
+            buf.resize((size_t) nx * (size_t) ny * 3);
+        }
+    }
+
+    void cpy_buf(const std::vector<uint8_t> & new_buf) {
+        buf = new_buf;
+    }
+
+    const std::vector<uint8_t> & get_ro_buf() const {
+        if (is_placeholder()) {
+            throw std::runtime_error("this clip_image_u8 is a placeholder");
+        }
+        return buf;
+    }
+
+    // note to contributors: NEVER add a get_rw_buf(), it is a DANGEROUS pattern. always use get_pixel / set_pixel for buffer manipulation
+
+    bool is_placeholder() const {
+        return buf.empty();
+    }
+
+    std::array<uint8_t, 3> get_pixel(int x, int y) const {
+        if (is_placeholder()) {
+            // return a dummy value, so that legacy code can still process image without errors
+            return { 0, 0, 0 };
+        }
+        int idx = (y * nx + x) * 3;
+        return { buf[idx], buf[idx + 1], buf[idx + 2] };
+    }
+
+    void set_pixel(int x, int y, const std::array<uint8_t, 3> & rgb) {
+        if (is_placeholder()) {
+            return; // no-op
+        }
+        int idx = (y * nx + x) * 3;
+        buf[idx] = rgb[0];
+        buf[idx + 1] = rgb[1];
+        buf[idx + 2] = rgb[2];
+    }
+
+    size_t n_elements() const {
+        return n_pixels() * 3;
+    }
+
+  private:
     std::vector<uint8_t> buf;
+    int nx = 0;
+    int ny = 0;
+
+    size_t n_pixels() const {
+        return (size_t) nx * (size_t) ny;
+    }
 };
@@ -438,12 +506,88 @@
 //     Memory layout: RGBRGBRGB...
+// For seq, buf.size() == nx*ny*3*nt
+//     Memory layout: RGBRGB...RGBRGB... (nt times)
 // For audio, only one channel is used, buf.size() == nx*ny
 //     nx will be n_frames and ny will be n_mel
 struct clip_image_f32 {
-    int nx;
-    int ny;
+    // marks the global view in e.g., DeepSeek-OCR Models
+    bool add_viewsep = false;
+    // whether a learned newline (or EOI) token should be appended after the image (eg Granite4 Vision)
+    bool add_newline = false;
 
+    clip_image_size get_size() const {
+        return { nx_, ny_ };
+    }
+
+    int nx() const { return nx_; }
+    int ny() const { return ny_; }
+
+    void set_size(clip_image_size size, bool is_placeholder, bool is_audio) {
+        nx_ = size.width;
+        ny_ = size.height;
+        if (is_placeholder) {
+            buf.clear();
+        } else {
+            if (is_audio) {
+                buf.resize((size_t) nx_ * (size_t) ny_);
+            } else {
+                buf.resize((size_t) nx_ * (size_t) ny_ * 3);
+            }
+        }
+    }
+
+    void cpy_buf(const std::vector<float> & new_buf) {
+        buf = new_buf;
+    }
+
+    void from_u8(const clip_image_u8 & img) {
+        auto size = img.get_size();
+        nx_ = size.width;
+        ny_ = size.height;
+        if (img.is_placeholder()) {
+            buf.clear();
+            return; // no-op
+        }
+        buf.resize(img.n_elements());
+        const auto & u8_buf = img.get_ro_buf();
+        for (size_t i = 0; i < img.n_elements(); ++i) {
+            buf[i] = (float) u8_buf[i] / 255.0f;
+        }
+    }
+
+    size_t n_elements() const {
+        return n_pixels() * 3;
+    }
+
+    void normalize(const float mean[3], const float std[3]) {
+        if (is_placeholder()) {
+            return; // no-op
+        }
+        for (size_t i = 0; i < n_pixels(); ++i) {
+            buf[i * 3 + 0] = (buf[i * 3 + 0] - mean[0]) / std[0];
+            buf[i * 3 + 1] = (buf[i * 3 + 1] - mean[1]) / std[1];
+            buf[i * 3 + 2] = (buf[i * 3 + 2] - mean[2]) / std[2];
+        }
+    }
+
+    const std::vector<float> & get_ro_buf() const {
+        if (is_placeholder()) {
+            throw std::runtime_error("this clip_image_f32 is a placeholder");
+        }
+        return buf;
+    }
+
+    // note to contributors: NEVER add a get_rw_buf(), it is a DANGEROUS pattern
+
+    bool is_placeholder() const {
+        return buf.empty();
+    }
+
+  private:
     std::vector<float> buf;
+    int nx_ = 0;
+    int ny_ = 0;
 
-    // marks the global view in e.g., DeepSeek-OCR Models
-    bool add_viewsep = false;
+    size_t n_pixels() const {
+        return (size_t) nx_ * (size_t) ny_;
+    }
 };
@@ -496,2 +640,3 @@
 
+#define LOG_TRC(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
 #define LOG_DBG(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
@@ -498,0 +643,0 @@ #define LOG_INF(...) clip_log_internal(GGML_LOG_LEVEL_INFO,  __VA_ARGS__)

vendor/llama.cpp/tools/mtmd/clip-model.h

@@ -7,2 +7,3 @@ #pragma once
 
+#include <algorithm>
 #include <array>
@@ -94,3 +95,3 @@ #include <vector>
     float rope_theta = 0.0;
-    std::unordered_set<int32_t> vision_feature_layer;
+    std::vector<int32_t> vision_feature_layer;
     int32_t attn_window_size = 0;
@@ -106,2 +107,7 @@ int32_t n_wa_pattern = 0;
 
+    // Granite4 Vision
+    std::vector<int32_t> proj_spatial_offsets;
+    int32_t downsample_query_side;
+    int32_t downsample_window_side;
+
     // audio
@@ -164,2 +170,6 @@ int32_t n_mel_bins = 0; // whisper preprocessor
     }
+
+    bool is_vision_feature_layer(int32_t layer) const {
+        return std::find(vision_feature_layer.begin(), vision_feature_layer.end(), layer) != vision_feature_layer.end();
+    }
 };
@@ -332,2 +342,16 @@
 
+// QFormer projector block for models with 1 (or more) QFormer projectors
+// Granite Speech, Granite4 Vision
+struct qf_block {
+    ggml_tensor * qf_proj_query       = nullptr;
+    ggml_tensor * qf_proj_norm_w      = nullptr;
+    ggml_tensor * qf_proj_norm_b      = nullptr;
+    ggml_tensor * qf_proj_linear_w    = nullptr;
+    ggml_tensor * qf_proj_linear_b    = nullptr;
+    ggml_tensor * qf_proj_post_norm_w = nullptr;
+    ggml_tensor * qf_proj_post_norm_b = nullptr;
+    ggml_tensor * qf_proj_img_pos     = nullptr; // Vision only
+    std::vector<clip_layer> qf_proj_layers;
+};
+
 struct clip_model {
@@ -597,9 +621,4 @@ clip_modality modality = CLIP_MODALITY_VISION;
     ggml_tensor * ctc_out_mid_b = nullptr;
-    // qformer projector
-    ggml_tensor * qf_proj_query    = nullptr;
-    ggml_tensor * qf_proj_norm_w   = nullptr;
-    ggml_tensor * qf_proj_norm_b   = nullptr;
-    ggml_tensor * qf_proj_linear_w = nullptr;
-    ggml_tensor * qf_proj_linear_b = nullptr;
-    std::vector<clip_layer> qf_proj_layers;
+    // qformer projector(s)
+    std::vector<qf_block> qf_proj_blocks;
 
@@ -606,0 +625,0 @@ bool audio_has_avgpool() const {

vendor/llama.cpp/tools/mtmd/clip.h

@@ -20,2 +20,11 @@ #pragma once
     int height;
+    bool operator==(const clip_image_size & other) const {
+        return width == other.width && height == other.height;
+    }
+    bool operator!=(const clip_image_size & other) const {
+        return !(*this == other);
+    }
+    int area() const {
+        return width * height;
+    }
 };
@@ -58,5 +67,2 @@
 
-size_t clip_embd_nbytes(const struct clip_ctx * ctx);
-size_t clip_embd_nbytes_by_img(const struct clip_ctx * ctx, int img_w, int img_h);
-
 int32_t clip_get_image_size (const struct clip_ctx * ctx);
@@ -84,5 +90,2 @@ int32_t clip_get_patch_size (const struct clip_ctx * ctx);
 
-// nx, ny are the output image dimensions
-unsigned char * clip_image_u8_get_data(struct clip_image_u8 * img, uint32_t * nx, uint32_t * ny);
-
 void clip_image_size_free (struct clip_image_size * img_size);
@@ -100,10 +103,2 @@ void clip_image_u8_free (struct clip_image_u8  * img);
 
-/**
- * Build image from pixels decoded by other libraries instead of stb_image.h for better performance.
- * The memory layout is RGBRGBRGB..., input buffer length must be 3*nx*ny bytes
- */
-void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny, struct clip_image_u8 * img);
-
-struct ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx);
-
 bool clip_image_encode      (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec);
@@ -116,10 +111,7 @@ bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec);
 
-bool clip_encode_float_image (struct clip_ctx * ctx, int n_threads, float * img, int h, int w, float * vec);
-
-// use by audio input
-void clip_image_f32_batch_add_mel(struct clip_image_f32_batch * batch, int n_mel, int n_frames, float * mel);
-
 bool clip_has_vision_encoder(const struct clip_ctx * ctx);
 bool clip_has_audio_encoder(const struct clip_ctx * ctx);
 
+int clip_model_n_batch_max(const struct clip_ctx * ctx);
+
 std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx);
@@ -126,0 +118,0 @@

vendor/llama.cpp/tools/mtmd/CMakeLists.txt

@@ -28,2 +28,3 @@ # mtmd
             models/granite-speech.cpp
+            models/granite4-vision.cpp
             models/hunyuanvl.cpp
@@ -30,0 +31,0 @@ models/internvl.cpp

vendor/llama.cpp/tools/mtmd/models/conformer.cpp

 #include "models.h"
 
 ggml_cgraph * clip_graph_conformer::build() {
-    const int n_frames   = img.nx;
+    const int n_frames   = img.nx();
     const int n_pos      = n_frames / 2;
@@ -6,0 +6,0 @@ const int n_pos_embd = (((((n_frames + 1) / 2) + 1) / 2 + 1) / 2) * 2 - 1;

vendor/llama.cpp/tools/mtmd/models/exaone4_5.cpp

@@ -25,4 +25,4 @@ // similar to qwen2vl, except for GQA attention
 
-    GGML_ASSERT(img.nx % (patch_size * 2) == 0);
-    GGML_ASSERT(img.ny % (patch_size * 2) == 0);
+    GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
+    GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
 
@@ -29,0 +29,0 @@ {

vendor/llama.cpp/tools/mtmd/models/glm4v.cpp

@@ -19,4 +19,4 @@ #include "models.h"
 
-    GGML_ASSERT(img.nx % (patch_size * 2) == 0);
-    GGML_ASSERT(img.ny % (patch_size * 2) == 0);
+    GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
+    GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
 
@@ -23,0 +23,0 @@ // second conv dimension

vendor/llama.cpp/tools/mtmd/models/granite-speech.cpp

 #include "models.h"
 
 ggml_cgraph * clip_graph_granite_speech::build() {
-    const int n_frames     = img.nx;
+    const int n_frames     = img.nx();
     const int context_size = hparams.audio_chunk_size;
@@ -202,4 +202,4 @@ const int ctc_layer    = n_layer / 2;
 
-        ggml_tensor * queries = build_norm(model.qf_proj_query,
-            model.qf_proj_norm_w, model.qf_proj_norm_b,
+        ggml_tensor * queries = build_norm(model.qf_proj_blocks[0].qf_proj_query,
+            model.qf_proj_blocks[0].qf_proj_norm_w, model.qf_proj_blocks[0].qf_proj_norm_b,
             NORM_TYPE_NORMAL, proj_eps, -1);
@@ -213,4 +213,4 @@ {
 
-        for (int il = 0; il < (int)model.qf_proj_layers.size(); il++) {
-            const auto & pl = model.qf_proj_layers[il];
+        for (int il = 0; il < (int)model.qf_proj_blocks[0].qf_proj_layers.size(); il++) {
+            const auto & pl = model.qf_proj_blocks[0].qf_proj_layers[il];
 
@@ -270,3 +270,3 @@ // self-attention
         cur = ggml_reshape_2d(ctx0, queries, n_embd, num_queries * nblocks_proj);
-        cur = ggml_add(ctx0, build_mm(model.qf_proj_linear_w, cur), model.qf_proj_linear_b);
+        cur = ggml_add(ctx0, build_mm(model.qf_proj_blocks[0].qf_proj_linear_w, cur), model.qf_proj_blocks[0].qf_proj_linear_b);
         cb(cur, "projector_out", -1);
@@ -273,0 +273,0 @@ }

vendor/llama.cpp/tools/mtmd/models/kimik25.cpp

@@ -10,4 +10,4 @@ #include "models.h"
     ggml_tensor * pos_embd = model.position_embeddings;
-    const int height       = img.ny / patch_size;
-    const int width        = img.nx / patch_size;
+    const int height       = img.ny() / patch_size;
+    const int width        = img.nx() / patch_size;
     const uint32_t mode    = interpolation_mode;
@@ -14,0 +14,0 @@

vendor/llama.cpp/tools/mtmd/models/llava.cpp

@@ -54,3 +54,2 @@ #include "models.h"
     std::vector<ggml_tensor *> embedding_stack;
-    const auto & vision_feature_layer = hparams.vision_feature_layer;
 
@@ -64,3 +63,3 @@ // loop over layers
         // NOTE: 0 index here refers to the input to the encoder.
-        if (vision_feature_layer.find(il) != vision_feature_layer.end()) {
+        if (hparams.is_vision_feature_layer(il)) {
             embedding_stack.push_back(cur);
@@ -140,3 +139,3 @@ }
         // final layer is a vision feature layer
-        if (vision_feature_layer.find(max_feature_layer) != vision_feature_layer.end()) {
+        if (hparams.is_vision_feature_layer(max_feature_layer)) {
             embedding_stack.push_back(inpL);
@@ -143,0 +142,0 @@ }

vendor/llama.cpp/tools/mtmd/models/mimovl.cpp

@@ -59,4 +59,4 @@ #include "models.h"
 
-        GGML_ASSERT(img.nx % (patch_size * 2) == 0);
-        GGML_ASSERT(img.ny % (patch_size * 2) == 0);
+        GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
+        GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
 
@@ -63,0 +63,0 @@ inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);  // [w,h,c,b] -> [c,w,h,b]

vendor/llama.cpp/tools/mtmd/models/models.h

@@ -34,6 +34,7 @@ #pragma once
     ggml_cgraph * build() override;
+    ggml_tensor * build_inp_with_temporal_merge();
 };
 
-struct clip_graph_qwen3vl : clip_graph {
-    clip_graph_qwen3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+struct clip_graph_qwen3vl : clip_graph_qwen2vl {
+    clip_graph_qwen3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen2vl(ctx, img) {}
     ggml_cgraph * build() override;
@@ -215,1 +216,24 @@ };
 };
+
+struct clip_graph_granite4_vision : clip_graph {
+    clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img)
+        : clip_graph(ctx, img),
+          add_newline(img.add_newline) {}
+
+    ggml_cgraph * build() override;
+
+private:
+    // The graph is per-tile since only batch-size 1 is supported in clip. As
+    // such, this value is set at construct time based on the tile that will be
+    // encoded, then used during build to determine how to handle newlines.
+    const bool add_newline;
+
+    ggml_tensor * gather(ggml_tensor * src, const std::string & name, int idx_len);
+    ggml_tensor * interp_down(ggml_tensor * src, int side, int new_side);
+    ggml_tensor * build_block(const qf_block & blk, ggml_tensor * h, int bid,
+                              int spatial_offset, int image_side, int window_side,
+                              int query_side, float qformer_eps);
+
+    ggml_tensor * build_newline_row(ggml_context * ctx0);
+    ggml_tensor * append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output);
+};

vendor/llama.cpp/tools/mtmd/models/qwen2vl.cpp

 #include "models.h"
 
+ggml_tensor * clip_graph_qwen2vl::build_inp_with_temporal_merge() {
+    ggml_tensor * inp_raw = build_inp_raw();
+
+    GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
+    GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
+
+    const size_t nb1 = ggml_row_size(inp_raw->type, img.nx());
+    const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny());
+
+    if (n_batch == 1) {
+        // still image input
+        return ggml_add(ctx0,
+            ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
+            ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
+    } else if (n_batch == 2) {
+        // 2 frames input (video input)
+        ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw,
+                                    img.nx(), img.ny(), 3, nb1, nb2, 0);
+        ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw,
+                                    img.nx(), img.ny(), 3, nb1, nb2,
+                                    nb2 * 3); // move to the second frame
+        return ggml_add(ctx0,
+            ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1),
+            ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1));
+    } else {
+        GGML_ASSERT(false && "n_batch > 2 is not supported");
+    }
+}
+
 ggml_cgraph * clip_graph_qwen2vl::build() {
@@ -19,13 +48,6 @@ GGML_ASSERT(model.patch_bias == nullptr);
 
-    ggml_tensor * inp_raw = build_inp_raw();
-    ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
+    ggml_tensor * inp = build_inp_with_temporal_merge();
 
-    GGML_ASSERT(img.nx % (patch_size * 2) == 0);
-    GGML_ASSERT(img.ny % (patch_size * 2) == 0);
-
     // second conv dimension
     {
-        auto inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
-        inp = ggml_add(ctx0, inp, inp_1);
-
         inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);  // [w, h, c, b] -> [c, w, h, b]
@@ -32,0 +54,0 @@ inp = ggml_cont_4d(

vendor/llama.cpp/tools/mtmd/models/qwen3vl.cpp

@@ -16,13 +16,6 @@ #include "models.h"
 
-    ggml_tensor * inp_raw = build_inp_raw();
-    ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
+    ggml_tensor * inp = build_inp_with_temporal_merge();
 
-    GGML_ASSERT(img.nx % (patch_size * 2) == 0);
-    GGML_ASSERT(img.ny % (patch_size * 2) == 0);
-
-    // second conv dimension
+    // spatial merge
     {
-        auto inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
-        inp = ggml_add(ctx0, inp, inp_1);
-
         inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);  // [w, h, c, b] -> [c, w, h, b]
@@ -29,0 +22,0 @@ inp = ggml_cont_4d(

vendor/llama.cpp/tools/mtmd/models/whisper-enc.cpp

 #include "models.h"
 
 ggml_cgraph * clip_graph_whisper_enc::build() {
-    const int n_frames = img.nx;
+    const int n_frames = img.nx();
     const int n_pos    = n_frames / 2;
@@ -6,0 +6,0 @@ GGML_ASSERT(model.position_embeddings->ne[1] >= n_pos);

vendor/llama.cpp/tools/mtmd/mtmd-cli.cpp

@@ -169,3 +169,3 @@ #include "arg.h"
     bool load_media(const std::string & fname) {
-        mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str()));
+        mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str(), false));
         if (!bmp.ptr) {
@@ -172,0 +172,0 @@ return false;

vendor/llama.cpp/tools/mtmd/mtmd-helper.cpp

@@ -481,3 +481,3 @@ // fix problem with std::min and std::max
 
-mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len) {
+mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) {
     if (audio_helpers::is_audio_file((const char *)buf, len)) {
@@ -494,3 +494,3 @@ std::vector<float> pcmf32;
         }
-        return mtmd_bitmap_init_from_audio(pcmf32.size(), pcmf32.data());
+        return mtmd_bitmap_init_from_audio(pcmf32.size(), placeholder ? nullptr : pcmf32.data());
     }
@@ -507,3 +507,3 @@
         }
-        result = mtmd_bitmap_init(nx, ny, data);
+        result = mtmd_bitmap_init(nx, ny, placeholder ? nullptr : data);
         stbi_image_free(data);
@@ -514,3 +514,3 @@ }
 
-mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname) {
+mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder) {
     std::vector<unsigned char> buf;
@@ -540,3 +540,4 @@ FILE * f = fopen(fname, "rb");
 
-    return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size());
+    return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder);
 }
+

vendor/llama.cpp/tools/mtmd/mtmd-helper.h

@@ -32,3 +32,3 @@ #ifndef MTMD_HELPER_H
 // this function is thread-safe
-MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname);
+MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder);
 
@@ -42,3 +42,3 @@ // helper function to construct a mtmd_bitmap from a buffer containing a file
 // this function is thread-safe
-MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len);
+MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder);
 
@@ -45,0 +45,0 @@ // helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache

vendor/llama.cpp/tools/mtmd/mtmd.h

@@ -136,5 +136,12 @@ #ifndef MTMD_H
 //     the data is in RGBRGBRGB... format
+//     note: some video-capable models (i.e. qwen-vl) can merge consecutive bitmaps
+//           into one chunk, mtmd_tokenize() will automatically handle this
 // if bitmap is audio:
 //     length of data must be n_samples * sizeof(float)
 //     the data is in float format (PCM F32)
+//
+// if data == nullptr:
+//     the bitmap is considered "empty", and will be treated as a placeholder for counting tokens
+//     you can pass the bitmap via mtmd_tokenize(), then call mtmd_*_get_n_tokens() to count the tokens
+//     note: passing a placeholder bitmap to mtmd_encode() will return an error
 MTMD_API mtmd_bitmap *         mtmd_bitmap_init           (uint32_t nx, uint32_t ny, const unsigned char * data);
@@ -141,0 +148,0 @@ MTMD_API mtmd_bitmap *         mtmd_bitmap_init_from_audio(size_t n_samples,         const float         * data);

vendor/llama.cpp/tools/quantize/README.md

@@ -8,51 +8,57 @@ # quantize
 
-You can also use the [GGUF-my-repo](https://huggingface.co/spaces/ggml-org/gguf-my-repo) space on Hugging Face to build your own quants without any setup.
+You can also use the [GGUF-my-repo](https://huggingface.co/spaces/ggml-org/gguf-my-repo) space on Hugging Face to build your own quants without any setup. It syncs from llama.cpp `main` every 6 hours.
 
-Note: It is synced from llama.cpp `main` every 6 hours.
+## Overview
 
-Example usage:
+Quantization is done in two phases:
+- Convert the original model to GGUF format.
+- Quantize the converted GGUF file.
 
-```./llama-quantize [options] input-model-f32.gguf [output-model-quant.gguf] type [threads]```
+If the model supports multimodal inputs (images or audio), you also need to convert and quantize the multimodal encoders and projectors.
 
+To perform these tasks, you need to install the Python requirements:
+
 ```bash
-# from Hugginface, obtain the official meta-llama/Llama-3.1-8B model weights and place them in ./models
-ls ./models
-config.json             model-00001-of-00004.safetensors  model-00004-of-00004.safetensors  README.md                tokenizer.json
-generation_config.json  model-00002-of-00004.safetensors  model.safetensors.index.json      special_tokens_map.json  USE_POLICY.md
-LICENSE                 model-00003-of-00004.safetensors  original                          tokenizer_config.json
+python3 -m pip install -r requirements.txt
+```
 
-# [Optional] for PyTorch .bin models like Mistral-7B
-ls ./models
-<folder containing weights and tokenizer json>
+Or if you use `uv`:
 
-# install Python dependencies
-python3 -m pip install -r requirements.txt
+```bash
+uv pip install -r requirements.txt --index-strategy unsafe-best-match
+```
 
-# convert the model to ggml FP16 format
-python3 convert_hf_to_gguf.py ./models/mymodel/
+## Prepare the input GGUF file
 
-# quantize the model to 4-bits (using Q4_K_M method)
-./llama-quantize ./models/mymodel/ggml-model-f16.gguf ./models/mymodel/ggml-model-Q4_K_M.gguf Q4_K_M
+To convert a model from a Hugging Face repo, you can use a command like the following:
 
-# update the gguf filetype to current version if older version is now unsupported
-./llama-quantize ./models/mymodel/ggml-model-Q4_K_M.gguf ./models/mymodel/ggml-model-Q4_K_M-v2.gguf COPY
 ```
+python convert_hf_to_gguf.py --outfile gemma-4-E2B-it-bf16.gguf --outtype bf16 --remote google/gemma-4-E2B-it
+```
 
-Run the quantized model:
+Notes:
+- In the usual case where the model is distributed in 16-bit format, `--outtype auto` (or omitting `--outtype` entirely) also works well.
+- If you have previously downloaded the model locally, specify the directory and remove the `--remote` flag.
+- For compatibility reasons, the Python requirements install transformers 4, but more and more models (like Gemma 4) require transformers 5. You can safely `pip install -U transformers` to get the latest version.
 
+## Quantize the GGUF
+
+After you have created a high-quality GGUF version of the model, you use `llama-quantize` to apply quantization. For example, quantize to `Q4_K_M` using a command like the following:
+
 ```bash
-# start inference on a gguf model
-./llama-cli -m ./models/mymodel/ggml-model-Q4_K_M.gguf -cnv -p "You are a helpful assistant"
+./build/bin/llama-quantize gemma-4-E2B-it-bf16.gguf gemma-4-E2B-it-Q4_K_M.gguf Q4_K_M
 ```
 
+Various quantization methods are described [later in this document](#quantize).
+
 Options:
-* `--allow-requantize` allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit
-* `--leave-output-tensor` will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing
-* `--pure` disables k-quant mixtures and quantizes all tensors to the same type
-* `--imatrix` uses data in file generated by `llama-imatrix` as importance matrix for quant optimizations (highly recommended)
-* `--include-weights` use an importance matrix for tensor(s) in the list. Cannot be used with `--exclude-weights`
-* `--exclude-weights` use an importance matrix for tensor(s) in the list. Cannot be used with `--include-weights`
+* `--allow-requantize` allow requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit
+* `--leave-output-tensor` leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing
+* `--pure` disable k-quant mixtures and quantizes all tensors to the same type
+* `--imatrix file_name` use data in file_name as importance matrix for quant optimizations
+* `--include-weights tensor_name` use importance matrix for this tensor (can be specified multiple times)
+* `--exclude-weights tensor_name` use importance matrix for the tensors **not** specified (include/exclude cannot be mixed)
 * `--output-tensor-type` use a specific quant type for the output.weight tensor
 * `--token-embedding-type` use a specific quant type for the token embeddings tensor
-* `--keep-split` will generate the quantized model in the same shards as the input file otherwise it will produce a single quantized file
+* `--keep-split` generate the quantized model in the same shards as the input file instead of a single quantized file
 
@@ -62,7 +68,26 @@ Advanced options:
 * `--prune-layers` prune (remove) the layers in the list
-* `--override-kv` option to override model metadata by key in the quantized model. May be specified multiple times
+* `--override-kv` option to override model metadata by key in the quantized model. May be specified multiple times.
 
-Examples:
+## (Optional) Convert the multimodal components
 
+llama.cpp will convert the LLM portion of the source model, which is enough for conversational applications. If the model accepts multimodal inputs and you wish to take advantage of them, you need to create a separate GGUF file. This file is generically known as `mmproj`, for "multimedia projector"; however, it may contain various components such as vision or audio encoders in addition to projections.
+
+Multimodal components are usually much smaller than the LLMs they come with. In addition, their quality has a direct impact on the quality of LLM generations, because these components are in charge of preparing the inputs for the LLM: the closer inputs are to data seen during training, the better LLM results will be.
+
+For these reasons, multimodal components are usually kept in a high-quality format such as bf16 or q8. The impact on speed and memory from using a smaller quant is negligible, but overall quality could be impacted.
+
 ```bash
+python convert_hf_to_gguf.py --mmproj --outfile mmproj-gemma-4-E2B-it-Q8_0.gguf --outtype q8_0 --remote google/gemma-4-E2B-it
+```
+
+## Run the quantized model
+
+
+```bash
+./build/bin/llama cli -m ./gemma-4-E2B-it-Q4_K_M.gguf --mmproj ./mmproj-gemma-4-E2B-it-Q8_0.gguf --image <input_image> --prompt "Describe this image"
+```
+
+## Quantization Examples
+
+```bash
 # naive Q4_K_M quantization using default settings and 8 CPU threads. Output will be "ggml-model-Q4_K_M.gguf"
@@ -69,0 +94,0 @@ ./llama-quantize input-model-f32.gguf q4_k_m 8

vendor/llama.cpp/tools/server/server-common.cpp

@@ -716,6 +716,6 @@ #include "common.h"
 
-server_tokens process_mtmd_prompt(mtmd_context * mctx, std::string prompt, std::vector<raw_buffer> files) {
+server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder) {
     mtmd::bitmaps bitmaps;
     for (auto & file : files) {
-        mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size()));
+        mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder));
         if (!bmp.ptr) {
@@ -722,0 +722,0 @@ throw std::runtime_error("Failed to load image or audio file");

vendor/llama.cpp/tools/server/server-common.h

@@ -261,3 +261,4 @@ #pragma once
 // process mtmd prompt, return the server_tokens containing both text tokens and media chunks
-server_tokens process_mtmd_prompt(mtmd_context * mctx, std::string prompt, std::vector<raw_buffer> files);
+// if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to server_queue)
+server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder = false);
 
@@ -264,0 +265,0 @@ /**

vendor/llama.cpp/tools/server/server-context.h

@@ -113,4 +113,6 @@ #pragma once
     server_http_context::handler_t post_chat_completions;
+    server_http_context::handler_t post_chat_completions_tok;
     server_http_context::handler_t post_control;
     server_http_context::handler_t post_responses_oai;
+    server_http_context::handler_t post_responses_tok_oai;
     server_http_context::handler_t post_transcriptions_oai;
@@ -143,2 +145,3 @@ server_http_context::handler_t post_anthropic_messages;
     std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
+    std::unique_ptr<server_res_generator> handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const server_http_req & req, task_response_type res_type);
 
@@ -145,0 +148,0 @@ // using unique_ptr to allow late initialization of const

vendor/llama.cpp/tools/server/server.cpp

@@ -164,2 +164,4 @@ #include "server-context.h"
         routes.post_apply_template         = models_routes->proxy_post;
+        routes.post_chat_completions_tok   = models_routes->proxy_post;
+        routes.post_responses_tok_oai      = models_routes->proxy_post;
         routes.get_lora_adapters           = models_routes->proxy_get;
@@ -196,3 +198,2 @@ routes.post_lora_adapters          = models_routes->proxy_post;
     ctx_http.post("/v1/messages",              ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API
-    ctx_http.post("/v1/messages/count_tokens", ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting
     ctx_http.post("/infill",                   ex_wrapper(routes.post_infill));
@@ -209,2 +210,8 @@ ctx_http.post("/embedding",                ex_wrapper(routes.post_embeddings)); // legacy
     ctx_http.post("/apply-template",           ex_wrapper(routes.post_apply_template));
+    // token counting
+    ctx_http.post("/chat/completions/input_tokens",    ex_wrapper(routes.post_chat_completions_tok));
+    ctx_http.post("/v1/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok));
+    ctx_http.post("/responses/input_tokens",           ex_wrapper(routes.post_responses_tok_oai));
+    ctx_http.post("/v1/responses/input_tokens",        ex_wrapper(routes.post_responses_tok_oai));
+    ctx_http.post("/v1/messages/count_tokens",         ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting
     // LoRA adapters hotswap
@@ -211,0 +218,0 @@ ctx_http.get ("/lora-adapters",            ex_wrapper(routes.get_lora_adapters));

vendor/llama.cpp/t...rver/tests/unit/test_chat_completion.py

@@ -576,1 +576,17 @@ import pytest
             assert choice["finish_reason"] == "length"
+
+
+def test_chat_completions_token_count():
+    global server
+    server.start()
+    # make sure cache can be reused across multiple choices and multiple requests
+    # ref: https://github.com/ggml-org/llama.cpp/pull/18663
+    for _ in range(2):
+        res = server.make_request("POST", "/chat/completions/input_tokens", data={
+            "messages": [
+                {"role": "system", "content": "Book"},
+                {"role": "user", "content": "What is the best book"},
+            ],
+        })
+        assert res.status_code == 200
+        assert res.body["input_tokens"] > 5

vendor/llama.cpp/tools/server/tests/unit/test_vision_api.py

@@ -101,2 +101,21 @@ import pytest
 
+def test_vision_chat_completion_token_count():
+    global server
+    server.start()
+    res = server.make_request("POST", "/chat/completions/input_tokens", data={
+        "temperature": 0.0,
+        "top_k": 1,
+        "messages": [
+            {"role": "user", "content": [
+                {"type": "text", "text": "What is this:"},
+                {"type": "image_url", "image_url": {
+                    "url": get_img_url("IMG_URL_0"),
+                }},
+            ]},
+        ],
+    })
+    assert res.status_code == 200
+    assert res.body["input_tokens"] > 10
+
+
 @pytest.mark.parametrize(
@@ -103,0 +122,0 @@ "prompt, image_data, success, re_content",

vendor/llama.cpp/tools/ui/.npmrc

 engine-strict=true
+ignore-scripts=true

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

189164852

0.53%

Number of package files

3122

0.39%

Lines of code

134831

13.97%