New:Socket for Asana Is Now Available.Learn more →
llama-cpp-python
Artifacts are distributable versions of a package. It typically includes the source code and metadata
Advanced tools
llama-cpp-python - pypi Package Compare versions
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
| { | ||
| "server": { | ||
| "host": "0.0.0.0", | ||
| "port": 8000 | ||
| }, | ||
| "model": { | ||
| "alias": "bge-small-en-v1.5", | ||
| "from_pretrained": { | ||
| "repo_id": "CompendiumLabs/bge-small-en-v1.5-gguf", | ||
| "filename": "bge-small-en-v1.5-q4_k_m.gguf" | ||
| }, | ||
| "n_ctx": 512, | ||
| "n_seq_max": 16, | ||
| "n_batch": 512, | ||
| "n_ubatch": 512, | ||
| "threads": 4, | ||
| "threads_batch": 8, | ||
| "kv_unified": true, | ||
| "store_logits": false, | ||
| "use_mmap": true | ||
| } | ||
| } |
| #include "models.h" | ||
| void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) { | ||
| hparams.n_embd_inp_impl = hparams.n_embd_out(); | ||
| hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; | ||
| ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); | ||
| uint32_t n_kv_shared_layers = 0; | ||
| ml.get_key(LLM_KV_ATTENTION_SHARED_KV_LAYERS, n_kv_shared_layers, false); | ||
| hparams.f_attention_scale = 1.0f; | ||
| ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false); | ||
| GGML_ASSERT(hparams.n_layer_nextn == hparams.n_layer_all && "n_layer_nextn must be == n_layer_impl"); | ||
| ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); | ||
| ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); | ||
| ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); | ||
| ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa); | ||
| ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa); | ||
| } | ||
| void llama_model_gemma4_assistant::load_arch_tensors(llama_model_loader &) { | ||
| LLAMA_LOAD_LOCALS; | ||
| if (n_embd_head_k != n_embd_head_v) { | ||
| throw std::runtime_error("Gemma 4 assistant requires n_embd_head_k == n_embd_head_v"); | ||
| } | ||
| if (hparams.n_embd_head_k_swa != hparams.n_embd_head_v_swa) { | ||
| throw std::runtime_error("Gemma 4 assistant requires n_embd_head_k_swa == n_embd_head_v_swa"); | ||
| } | ||
| if (hparams.n_embd_out() == n_embd) { | ||
| throw std::runtime_error("Gemma 4 assistant requires embedding_length_out to carry the target hidden size"); | ||
| } | ||
| tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); | ||
| output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED); | ||
| output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); | ||
| const int64_t n_embd_backbone = hparams.n_embd_inp(); | ||
| nextn_proj_post = create_tensor(tn(LLM_TENSOR_NEXTN_PROJ_POST, "weight"), { n_embd, n_embd_backbone }, 0); | ||
| int rope_freqs_flag = 0; | ||
| for (int i = 0; i < n_layer_nextn; ++i) { | ||
| auto & layer = layers[i]; | ||
| const int64_t n_head = hparams.n_head(i); | ||
| const int64_t n_embd_head = hparams.n_embd_head_k(i); | ||
| const int64_t n_ff = hparams.n_ff(i); | ||
| if (i == 0) { | ||
| nextn_proj_pre = create_tensor(tn(LLM_TENSOR_NEXTN_PROJ_PRE, "weight", i), { 2*n_embd_backbone, n_embd }, 0); | ||
| } | ||
| layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0); | ||
| layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head*n_head }, 0); | ||
| layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head*n_head, n_embd }, 0); | ||
| layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), { n_embd_head }, 0); | ||
| layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), { n_embd }, 0); | ||
| layer.out_scale = create_tensor(tn(LLM_TENSOR_LAYER_OUT_SCALE, "weight", i), { 1u }, 0); | ||
| if (!hparams.is_swa(i)) { | ||
| layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), { n_embd_head/2 }, rope_freqs_flag); | ||
| rope_freqs_flag = TENSOR_DUPLICATED; | ||
| } | ||
| layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0); | ||
| layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff }, 0); | ||
| layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff }, 0); | ||
| layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, 0); | ||
| layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), { n_embd }, 0); | ||
| } | ||
| } | ||
| std::unique_ptr<llm_graph_context> llama_model_gemma4_assistant::build_arch_graph(const llm_graph_params & params) const { | ||
| return std::make_unique<graph>(*this, params); | ||
| } | ||
| llama_model_gemma4_assistant::graph::graph(const llama_model & model, const llm_graph_params & params) : | ||
| llm_graph_context(params) { | ||
| const int64_t n_embd_backbone = hparams.n_embd_inp(); | ||
| ggml_tensor * inp_tokens; | ||
| ggml_tensor * inp_h; | ||
| { | ||
| auto inp = std::make_unique<llm_graph_input_embd>(n_embd_backbone); | ||
| inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens); | ||
| cb(inp->tokens, "inp_tokens", -1); | ||
| ggml_set_input(inp->tokens); | ||
| inp_tokens = inp->tokens; | ||
| res->t_inp_tokens = inp->tokens; | ||
| inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_backbone, ubatch.n_tokens); | ||
| cb(inp->embd, "inp_h", -1); | ||
| ggml_set_input(inp->embd); | ||
| inp_h = inp->embd; | ||
| res->t_inp_embd = inp->embd; | ||
| res->add_input(std::move(inp)); | ||
| } | ||
| GGML_ASSERT(cparams.ctx_other != nullptr); | ||
| const auto * model_other = llama_get_model(cparams.ctx_other); | ||
| ggml_tensor * x = ggml_get_rows(ctx0, model_other->tok_embd, inp_tokens); | ||
| x = ggml_scale(ctx0, x, sqrtf((float) n_embd_backbone)); | ||
| cb(x, "inp_embd_target", -1); | ||
| ggml_tensor * xh = ggml_concat(ctx0, x, inp_h, 0); | ||
| cb(xh, "inp_xh", -1); | ||
| ggml_tensor * cur = ggml_mul_mat(ctx0, model.nextn_proj_pre, xh); | ||
| cb(cur, "pre_proj", -1); | ||
| auto * inp_attn = build_attn_inp_kv_iswa(); | ||
| ggml_tensor * inp_pos = build_inp_pos(); | ||
| ggml_tensor * inp_out_ids = build_inp_out_ids(); | ||
| ggml_tensor * inpL = cur; | ||
| for (int il = 0; il < n_layer_nextn; ++il) { | ||
| const bool is_swa = hparams.is_swa(il); | ||
| const int64_t n_embd_head = hparams.n_embd_head_k(il); | ||
| const int64_t n_head = hparams.n_head(il); | ||
| const float freq_base_l = model.get_rope_freq_base(cparams, il); | ||
| const float freq_scale_l = model.get_rope_freq_scale(cparams, il); | ||
| const int n_rot_l = hparams.n_rot(il); | ||
| ggml_tensor * cur_norm = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); | ||
| cb(cur_norm, "attn_norm", il); | ||
| ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur_norm); | ||
| Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); | ||
| Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); | ||
| cb(Qcur, "Qcur_normed", il); | ||
| ggml_tensor * freq_factors = is_swa ? nullptr : model.layers[il].rope_freqs; | ||
| Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, | ||
| freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); | ||
| cb(Qcur, "Qcur_pos", il); | ||
| cur = build_attn(inp_attn, model.layers[il].wo, nullptr, nullptr, | ||
| Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il); | ||
| if (il == n_layer_nextn - 1 && inp_out_ids) { | ||
| cur = ggml_get_rows(ctx0, cur, inp_out_ids); | ||
| inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); | ||
| } | ||
| cur = build_norm(cur, model.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il); | ||
| cb(cur, "attn_post_norm", il); | ||
| ggml_tensor * attn_out = ggml_add(ctx0, cur, inpL); | ||
| cb(attn_out, "attn_out", il); | ||
| cur = build_norm(attn_out, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); | ||
| cb(cur, "ffn_norm", il); | ||
| cur = build_ffn(cur, | ||
| model.layers[il].ffn_up, nullptr, nullptr, | ||
| model.layers[il].ffn_gate, nullptr, nullptr, | ||
| model.layers[il].ffn_down, nullptr, nullptr, | ||
| nullptr, | ||
| LLM_FFN_GELU, LLM_FFN_PAR, il); | ||
| cb(cur, "ffn_out", il); | ||
| cur = build_norm(cur, model.layers[il].ffn_post_norm, nullptr, LLM_NORM_RMS, -1); | ||
| cb(cur, "ffn_post_norm", il); | ||
| cur = ggml_add(ctx0, cur, attn_out); | ||
| cur = ggml_mul(ctx0, cur, model.layers[il].out_scale); | ||
| cb(cur, "out_scaled", il); | ||
| inpL = cur; | ||
| } | ||
| cur = inpL; | ||
| cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); | ||
| cb(cur, "result_norm", -1); | ||
| ggml_tensor * logits = build_lora_mm(model.output, cur); | ||
| cb(logits, "result_output", -1); | ||
| res->t_logits = logits; | ||
| ggml_tensor * h_next = ggml_mul_mat(ctx0, model.nextn_proj_post, cur); | ||
| cb(h_next, "h_nextn", -1); | ||
| res->t_h_nextn = h_next; | ||
| ggml_build_forward_expand(gf, logits); | ||
| ggml_build_forward_expand(gf, h_next); | ||
| } |
+6
-0
@@ -10,2 +10,8 @@ # Changelog | ||
| ## [0.3.28] | ||
| - feat(example): align server MTP support with llama.cpp by @abetlen in #2283 | ||
| - feat: update llama.cpp to ggml-org/llama.cpp@9e3b928fd | ||
| - feat(example): add OpenAI-compatible embeddings endpoint by @abetlen in #2281 | ||
| ## [0.3.27] | ||
@@ -12,0 +18,0 @@ |
| # Server Example | ||
| This example is an updated OpenAI-compatible web server that depends only on the low-level C bindings. | ||
| It supports batched inference, prompt caching, response parsing, `/v1/responses`, disk sequence caching, MTP, LoRA, and multimodal image/audio inputs. | ||
| It supports batched inference, prompt caching, response parsing, `/v1/responses`, `/v1/embeddings`, disk sequence caching, MTP, LoRA, and multimodal image/audio inputs. | ||
@@ -49,2 +49,3 @@ ## Setup | ||
| | --- | --- | --- | | ||
| | [`configs/bge-small-en-v1.5.json`](configs/bge-small-en-v1.5.json) | [`CompendiumLabs/bge-small-en-v1.5-gguf`](https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf) | Small embedding model config for `/v1/embeddings`. | | ||
| | [`configs/qwen3.5-0.8b.json`](configs/qwen3.5-0.8b.json) | [`lmstudio-community/Qwen3.5-0.8B-GGUF`](https://huggingface.co/lmstudio-community/Qwen3.5-0.8B-GGUF) | Default small multimodal example. | | ||
@@ -90,2 +91,23 @@ | [`configs/gemma-4-12b-it-qat.json`](configs/gemma-4-12b-it-qat.json) | [`unsloth/gemma-4-12B-it-qat-GGUF`](https://huggingface.co/unsloth/gemma-4-12B-it-qat-GGUF) | Larger Gemma 4 QAT multimodal config with projector. | | ||
| ### Embeddings | ||
| Start the server with an embedding config before calling `/v1/embeddings`. | ||
| ```bash | ||
| cd examples/server | ||
| uv run --script server.py -C configs/bge-small-en-v1.5.json | ||
| ``` | ||
| ```python | ||
| from openai import OpenAI | ||
| client = OpenAI(base_url="http://127.0.0.1:8000/v1", api_key="not-used") | ||
| response = client.embeddings.create( | ||
| model="bge-small-en-v1.5", | ||
| input=["The food was delicious.", "The meal was excellent."], | ||
| ) | ||
| print(len(response.data[0].embedding)) | ||
| ``` | ||
| ## API Surface | ||
@@ -96,2 +118,3 @@ | ||
| | `POST /v1/completions` | Legacy text completions with streaming, stop sequences, logprobs, penalties, seeds, and grammar-backed JSON output. | [OpenAI Completions API](https://platform.openai.com/docs/api-reference/completions) | | ||
| | `POST /v1/embeddings` | OpenAI-compatible embeddings for embedding-mode GGUF models, including string inputs, token inputs, base64 output, and dimensions truncation. | [OpenAI Embeddings API](https://platform.openai.com/docs/api-reference/embeddings) | | ||
| | `POST /v1/chat/completions` | Chat completions with streaming, tools, forced tool choice, reasoning parsing, multimodal content parts, and structured response parsing. | [OpenAI Chat API](https://platform.openai.com/docs/api-reference/chat) | | ||
@@ -196,2 +219,4 @@ | `POST /v1/responses` | Stateless Responses API compatibility for clients that use response items and response events. | [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) | | ||
| | `kv_unified` | Selects unified or per-sequence memory layout. | | ||
| | `embedding` | Overrides embedding mode; omit to auto-detect pooled embedding GGUFs from model metadata. | | ||
| | `pooling_type` | Overrides pooled embedding behavior for embedding models, such as `1` for mean pooling. | | ||
| | `store_logits` | Keeps logits after decode when needed by sampling or diagnostics. | | ||
@@ -416,2 +441,18 @@ | `use_mmap` | Memory maps model weights. | | ||
| By default `draft-mtp` creates the MTP context from the target model. | ||
| Set `draft_model_path` or `draft_model_from_pretrained` when the model uses a separate assistant GGUF. | ||
| ```json | ||
| { | ||
| "model": { | ||
| "draft_model": "draft-mtp", | ||
| "draft_model_num_pred_tokens": 2, | ||
| "draft_model_from_pretrained": { | ||
| "repo_id": "example/gemma-assistant-GGUF", | ||
| "filename": "assistant.gguf" | ||
| } | ||
| } | ||
| } | ||
| ``` | ||
| MTP currently applies to text-only requests. | ||
@@ -418,0 +459,0 @@ |
| from .llama_cpp import * | ||
| from .llama import * | ||
| __version__ = "0.3.27" | ||
| __version__ = "0.3.28" |
@@ -45,8 +45,8 @@ """Experimental bindings for non-public llama.cpp APIs from `llama-ext.h`. | ||
| # LLAMA_API void llama_set_embeddings_pre_norm(struct llama_context * ctx, bool value, bool masked); | ||
| # LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked); | ||
| @_ctypes_function_from_names( | ||
| ( | ||
| "llama_set_embeddings_pre_norm", | ||
| "_Z29llama_set_embeddings_pre_normP13llama_contextbb", | ||
| "?llama_set_embeddings_pre_norm@@YAXPEAUllama_context@@_N1@Z", | ||
| "llama_set_embeddings_nextn", | ||
| "_Z26llama_set_embeddings_nextnP13llama_contextbb", | ||
| "?llama_set_embeddings_nextn@@YAXPEAUllama_context@@_N1@Z", | ||
| ), | ||
@@ -56,3 +56,3 @@ [llama_cpp.llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool], | ||
| ) | ||
| def llama_set_embeddings_pre_norm( | ||
| def llama_set_embeddings_nextn( | ||
| ctx: llama_cpp.llama_context_p, | ||
@@ -63,12 +63,12 @@ value: bool, | ||
| ): | ||
| """Set whether the context outputs pre-norm embeddings or not.""" | ||
| """Set whether the context outputs nextn embeddings or not.""" | ||
| ... | ||
| # LLAMA_API float * llama_get_embeddings_pre_norm(struct llama_context * ctx); | ||
| # LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx); | ||
| @_ctypes_function_from_names( | ||
| ( | ||
| "llama_get_embeddings_pre_norm", | ||
| "_Z29llama_get_embeddings_pre_normP13llama_context", | ||
| "?llama_get_embeddings_pre_norm@@YAPEAMPEAUllama_context@@@Z", | ||
| "llama_get_embeddings_nextn", | ||
| "_Z26llama_get_embeddings_nextnP13llama_context", | ||
| "?llama_get_embeddings_nextn@@YAPEAMPEAUllama_context@@@Z", | ||
| ), | ||
@@ -78,16 +78,16 @@ [llama_cpp.llama_context_p_ctypes], | ||
| ) | ||
| def llama_get_embeddings_pre_norm( | ||
| def llama_get_embeddings_nextn( | ||
| ctx: llama_cpp.llama_context_p, | ||
| /, | ||
| ): | ||
| """Get the pre-norm embeddings from the last evaluation.""" | ||
| """Get the nextn embeddings from the last evaluation.""" | ||
| ... | ||
| # LLAMA_API float * llama_get_embeddings_pre_norm_ith(struct llama_context * ctx, int32_t i); | ||
| # LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i); | ||
| @_ctypes_function_from_names( | ||
| ( | ||
| "llama_get_embeddings_pre_norm_ith", | ||
| "_Z33llama_get_embeddings_pre_norm_ithP13llama_contexti", | ||
| "?llama_get_embeddings_pre_norm_ith@@YAPEAMPEAUllama_context@@H@Z", | ||
| "llama_get_embeddings_nextn_ith", | ||
| "_Z30llama_get_embeddings_nextn_ithP13llama_contexti", | ||
| "?llama_get_embeddings_nextn_ith@@YAPEAMPEAUllama_context@@H@Z", | ||
| ), | ||
@@ -97,3 +97,3 @@ [llama_cpp.llama_context_p_ctypes, ctypes.c_int32], | ||
| ) | ||
| def llama_get_embeddings_pre_norm_ith( | ||
| def llama_get_embeddings_nextn_ith( | ||
| ctx: llama_cpp.llama_context_p, | ||
@@ -103,3 +103,21 @@ i: Union[ctypes.c_int32, int], | ||
| ): | ||
| """Get the pre-norm embeddings for the ith output row from the last evaluation.""" | ||
| """Get the nextn embeddings for the ith output row from the last evaluation.""" | ||
| ... | ||
| # LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); | ||
| @_ctypes_function_from_names( | ||
| ( | ||
| "llama_get_ctx_other", | ||
| "_Z19llama_get_ctx_otherP13llama_context", | ||
| "?llama_get_ctx_other@@YAPEAUllama_context@@PEAU1@@Z", | ||
| ), | ||
| [llama_cpp.llama_context_p_ctypes], | ||
| llama_cpp.llama_context_p_ctypes, | ||
| ) | ||
| def llama_get_ctx_other( | ||
| ctx: llama_cpp.llama_context_p, | ||
| /, | ||
| ): | ||
| """Get the context linked through llama_context_params.ctx_other.""" | ||
| ... |
+1
-1
| Metadata-Version: 2.1 | ||
| Name: llama_cpp_python | ||
| Version: 0.3.27 | ||
| Version: 0.3.28 | ||
| Summary: Python bindings for the llama.cpp library | ||
@@ -5,0 +5,0 @@ Author-Email: Andrei Betlen <abetlen@gmail.com> |
@@ -772,33 +772,47 @@ #include "sampling.h" | ||
| std::vector<common_sampler_type> common_sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names) { | ||
| std::unordered_map<std::string, common_sampler_type> sampler_canonical_name_map { | ||
| { "dry", COMMON_SAMPLER_TYPE_DRY }, | ||
| { "top_k", COMMON_SAMPLER_TYPE_TOP_K }, | ||
| { "top_p", COMMON_SAMPLER_TYPE_TOP_P }, | ||
| { "top_n_sigma", COMMON_SAMPLER_TYPE_TOP_N_SIGMA }, | ||
| { "typ_p", COMMON_SAMPLER_TYPE_TYPICAL_P }, | ||
| { "min_p", COMMON_SAMPLER_TYPE_MIN_P }, | ||
| { "temperature", COMMON_SAMPLER_TYPE_TEMPERATURE }, | ||
| { "xtc", COMMON_SAMPLER_TYPE_XTC }, | ||
| { "infill", COMMON_SAMPLER_TYPE_INFILL }, | ||
| { "penalties", COMMON_SAMPLER_TYPE_PENALTIES }, | ||
| { "adaptive_p", COMMON_SAMPLER_TYPE_ADAPTIVE_P }, | ||
| }; | ||
| std::vector<common_sampler_type> common_sampler_types_from_names(const std::vector<std::string> & names) { | ||
| // sampler names can be written multiple ways; generate aliases from canonical names | ||
| static const auto sampler_name_map = []{ | ||
| // canonical sampler name mapping | ||
| std::unordered_map<std::string, common_sampler_type> canonical_name_map { | ||
| { "dry", COMMON_SAMPLER_TYPE_DRY }, | ||
| { "top_k", COMMON_SAMPLER_TYPE_TOP_K }, | ||
| { "top_p", COMMON_SAMPLER_TYPE_TOP_P }, | ||
| { "top_n_sigma", COMMON_SAMPLER_TYPE_TOP_N_SIGMA }, | ||
| { "typ_p", COMMON_SAMPLER_TYPE_TYPICAL_P }, | ||
| { "min_p", COMMON_SAMPLER_TYPE_MIN_P }, | ||
| { "temperature", COMMON_SAMPLER_TYPE_TEMPERATURE }, | ||
| { "xtc", COMMON_SAMPLER_TYPE_XTC }, | ||
| { "infill", COMMON_SAMPLER_TYPE_INFILL }, | ||
| { "penalties", COMMON_SAMPLER_TYPE_PENALTIES }, | ||
| { "adaptive_p", COMMON_SAMPLER_TYPE_ADAPTIVE_P } | ||
| }; | ||
| std::unordered_map<std::string, common_sampler_type> alias_name_map; | ||
| for (const auto & entry : canonical_name_map) { | ||
| const std::string & canonical = entry.first; | ||
| if (canonical.find('_') == std::string::npos) { | ||
| continue; | ||
| } | ||
| // kebab-case: "top-k", "min-p", etc. | ||
| { | ||
| std::string kebab_case = canonical; | ||
| std::replace(kebab_case.begin(), kebab_case.end(), '_', '-'); | ||
| alias_name_map.insert({kebab_case, entry.second}); | ||
| } | ||
| // no dash: "topk", "minp", etc. | ||
| { | ||
| std::string no_dash = canonical; | ||
| no_dash.erase(std::remove(no_dash.begin(), no_dash.end(), '_'), no_dash.end()); | ||
| alias_name_map.insert({no_dash, entry.second}); | ||
| } | ||
| } | ||
| // misc. aliases | ||
| alias_name_map.insert({"nucleus", COMMON_SAMPLER_TYPE_TOP_P}); | ||
| alias_name_map.insert({"temp", COMMON_SAMPLER_TYPE_TEMPERATURE}); | ||
| alias_name_map.insert({"typ", COMMON_SAMPLER_TYPE_TYPICAL_P}); | ||
| // include aliases + canonical names in the complete mapping | ||
| alias_name_map.merge(canonical_name_map); | ||
| return alias_name_map; | ||
| }(); | ||
| // since samplers names are written multiple ways | ||
| // make it ready for both system names and input names | ||
| std::unordered_map<std::string, common_sampler_type> sampler_alt_name_map { | ||
| { "top-k", COMMON_SAMPLER_TYPE_TOP_K }, | ||
| { "top-p", COMMON_SAMPLER_TYPE_TOP_P }, | ||
| { "top-n-sigma", COMMON_SAMPLER_TYPE_TOP_N_SIGMA }, | ||
| { "nucleus", COMMON_SAMPLER_TYPE_TOP_P }, | ||
| { "typical-p", COMMON_SAMPLER_TYPE_TYPICAL_P }, | ||
| { "typical", COMMON_SAMPLER_TYPE_TYPICAL_P }, | ||
| { "typ-p", COMMON_SAMPLER_TYPE_TYPICAL_P }, | ||
| { "typ", COMMON_SAMPLER_TYPE_TYPICAL_P }, | ||
| { "min-p", COMMON_SAMPLER_TYPE_MIN_P }, | ||
| { "temp", COMMON_SAMPLER_TYPE_TEMPERATURE }, | ||
| { "adaptive-p", COMMON_SAMPLER_TYPE_ADAPTIVE_P }, | ||
| }; | ||
| std::vector<common_sampler_type> samplers; | ||
@@ -808,15 +822,10 @@ samplers.reserve(names.size()); | ||
| for (const auto & name : names) { | ||
| auto sampler = sampler_canonical_name_map.find(name); | ||
| if (sampler != sampler_canonical_name_map.end()) { | ||
| std::string name_lower = name; | ||
| std::transform(name_lower.begin(), name_lower.end(), name_lower.begin(), ::tolower); | ||
| auto sampler = sampler_name_map.find(name_lower); | ||
| if (sampler != sampler_name_map.end()) { | ||
| samplers.push_back(sampler->second); | ||
| continue; | ||
| } | ||
| if (allow_alt_names) { | ||
| sampler = sampler_alt_name_map.find(name); | ||
| if (sampler != sampler_alt_name_map.end()) { | ||
| samplers.push_back(sampler->second); | ||
| continue; | ||
| } | ||
| } | ||
| LOG_WRN("%s: unable to match sampler by name '%s'\n", __func__, name.c_str()); | ||
| LOG_WRN("%s: unable to match sampler by name '%s'\n", __func__, name_lower.c_str()); | ||
| } | ||
@@ -823,0 +832,0 @@ |
@@ -112,3 +112,3 @@ #pragma once | ||
| std::vector<enum common_sampler_type> common_sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names); | ||
| std::vector<enum common_sampler_type> common_sampler_types_from_names(const std::vector<std::string> & names); | ||
| std::vector<enum common_sampler_type> common_sampler_types_from_chars(const std::string & chars); | ||
@@ -115,0 +115,0 @@ |
@@ -6,3 +6,2 @@ #include "speculative.h" | ||
| #include "llama.h" | ||
| #include "../src/llama-ext.h" // staging API: llama_set_embeddings_nextn / llama_get_embeddings_nextn_ith (used by MTP) | ||
| #include "log.h" | ||
@@ -14,2 +13,4 @@ #include "ngram-cache.h" | ||
| #include "../src/llama-ext.h" // staging API: llama_set_embeddings_nextn / llama_get_embeddings_nextn_ith (used by MTP) | ||
| #include <algorithm> | ||
@@ -63,6 +64,6 @@ #include <cassert> | ||
| const bool vocab_type_tgt = llama_vocab_type(vocab_tgt); | ||
| const auto vocab_type_tgt = llama_vocab_type(vocab_tgt); | ||
| LOG_DBG("%s: vocab_type tgt: %d\n", __func__, vocab_type_tgt); | ||
| const bool vocab_type_dft = llama_vocab_type(vocab_dft); | ||
| const auto vocab_type_dft = llama_vocab_type(vocab_dft); | ||
| LOG_DBG("%s: vocab_type dft: %d\n", __func__, vocab_type_dft); | ||
@@ -424,2 +425,4 @@ | ||
| bool is_mem_shared = false; | ||
| // Per-sequence cross-batch carryover: pair (h_p, x_{p+1}) at MTP pos p+1. | ||
@@ -451,3 +454,5 @@ // The last h-row of one process() call needs the first token of the NEXT | ||
| n_embd = llama_model_n_embd(llama_get_model(ctx_dft)); | ||
| n_embd = llama_model_n_embd_out(llama_get_model(ctx_dft)); | ||
| GGML_ASSERT(n_embd == llama_model_n_embd(llama_get_model(ctx_tgt)) && | ||
| "MTP input row width must match the target h_nextn width"); | ||
@@ -498,2 +503,4 @@ LOG_INF("%s: adding speculative implementation 'draft-mtp'\n", __func__); | ||
| is_mem_shared = llama_get_ctx_other(ctx_dft) == ctx_tgt; | ||
| pending_h.assign(n_seq, std::vector<float>(n_embd, 0.0f)); | ||
@@ -535,5 +542,7 @@ | ||
| } | ||
| auto * ctx_dft = this->params.ctx_dft; | ||
| const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id); | ||
| if (pos_max < N - 1) { | ||
| if (pos_max < N - 1 && !is_mem_shared) { | ||
| LOG_WRN("%s: ctx_dft pos_max=%d < N-1=%d - " | ||
@@ -581,46 +590,40 @@ "process() hook may not have run on every prefill ubatch " | ||
| common_batch_clear(batch); | ||
| // if kv is shared with target (e.g Gemma4), then we can skip this catch-up decode | ||
| if (!is_mem_shared) { | ||
| common_batch_clear(batch); | ||
| for (int k = 0; k < n_tokens; ++k) { | ||
| common_batch_add(batch, batch_in.token[k], batch_in.pos[k], { batch_in.seq_id[k][0] }, 0); | ||
| } | ||
| for (int k = 0; k < n_tokens; ++k) { | ||
| common_batch_add(batch, batch_in.token[k], batch_in.pos[k], { batch_in.seq_id[k][0] }, 0); | ||
| } | ||
| // shift the tgt embeddings to the right by one position | ||
| // assumes that the tokens in the batch are sequential for each sequence | ||
| // i.e. we cannot have seq_id like this: [0, 0, 0, 1, 1, 0, 1, 1] | ||
| // ^--- this is a problem | ||
| // TODO:this is generally true, but would be nice to assert it | ||
| { | ||
| const float * h_tgt = llama_get_embeddings_nextn(ctx_tgt); | ||
| std::memcpy(batch.embd + (size_t) 1 * n_embd, h_tgt, row_bytes * (n_tokens-1)); | ||
| // shift the tgt embeddings to the right by one position | ||
| // assumes that the tokens in the batch are sequential for each sequence | ||
| // i.e. we cannot have seq_id like this: [0, 0, 0, 1, 1, 0, 1, 1] | ||
| // ^--- this is a problem | ||
| // TODO:this is generally true, but would be nice to assert it | ||
| { | ||
| const float * h_tgt = llama_get_embeddings_nextn(ctx_tgt); | ||
| std::memcpy(batch.embd + (size_t) 1 * n_embd, h_tgt, row_bytes * (n_tokens-1)); | ||
| } | ||
| //{ | ||
| // // string with seq_ids in the batch | ||
| // std::stringstream ss; | ||
| // for (int i = 0; i < n_tokens; ++i) { | ||
| // ss << batch_in.seq_id[i][0] << ","; | ||
| // } | ||
| // LOG_WRN("%s: batch_in.seq_id = %s\n", __func__, ss.str().c_str()); | ||
| //} | ||
| } | ||
| // fill the pending embeddings from a previous run | ||
| auto set_h = [&](int idx, const float * h_row) { | ||
| std::memcpy(batch.embd + (size_t) idx * n_embd, h_row, row_bytes); | ||
| }; | ||
| // fill the pending embeddings from a previous run | ||
| auto set_h = [&](int idx, const float * h_row) { | ||
| std::memcpy(batch.embd + (size_t) idx * n_embd, h_row, row_bytes); | ||
| }; | ||
| for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { | ||
| if (i_batch_beg[seq_id] < 0) { | ||
| continue; | ||
| } | ||
| for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { | ||
| if (i_batch_beg[seq_id] < 0) { | ||
| continue; | ||
| set_h(i_batch_beg[seq_id], pending_h[seq_id].data()); | ||
| } | ||
| set_h(i_batch_beg[seq_id], pending_h[seq_id].data()); | ||
| const int32_t rc = llama_decode(ctx_dft, batch); | ||
| if (rc != 0) { | ||
| LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (pos=%d)\n", __func__, (int) rc, (int) batch_in.pos[0]); | ||
| return false; | ||
| } | ||
| } | ||
| const int32_t rc = llama_decode(ctx_dft, batch); | ||
| if (rc != 0) { | ||
| LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (pos=%d)\n", __func__, (int) rc, (int) batch_in.pos[0]); | ||
| return false; | ||
| } | ||
| for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { | ||
@@ -732,3 +735,9 @@ if (i_batch_end[seq_id] < 0) { | ||
| common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); | ||
| if (is_mem_shared) { | ||
| // note: with shared memory (e.g. Gemma4 assistants) we use the same position for all draft tokens | ||
| // ref: https://github.com/huggingface/transformers/blob/effde20942e3f82a1b97449f60b3a48c5ff96145/docs/source/en/model_doc/gemma4_assistant.md?plain=1#L36-L37 | ||
| common_batch_add(batch, id, dp.n_past, { seq_id }, true); | ||
| } else { | ||
| common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); | ||
| } | ||
| std::memcpy(batch.embd + n_embd*(batch.n_tokens - 1), h_row, row_bytes); | ||
@@ -735,0 +744,0 @@ } |
@@ -78,5 +78,7 @@ from __future__ import annotations | ||
| "Gemma3nForConditionalGeneration": "gemma", | ||
| "Gemma4AssistantForCausalLM": "gemma", | ||
| "Gemma4ForConditionalGeneration": "gemma", | ||
| "Gemma4ForCausalLM": "gemma", | ||
| "Gemma4UnifiedForConditionalGeneration": "gemma", | ||
| "Gemma4UnifiedAssistantForCausalLM": "gemma", | ||
| "GemmaForCausalLM": "gemma", | ||
@@ -83,0 +85,0 @@ "Glm4ForCausalLM": "glm", |
@@ -788,2 +788,12 @@ from __future__ import annotations | ||
| @ModelBase.register("Gemma4AssistantForCausalLM", "Gemma4UnifiedAssistantForCausalLM") | ||
| class Gemma4AssistantModel(Gemma4Model): | ||
| model_arch = gguf.MODEL_ARCH.GEMMA4_ASSISTANT | ||
| def set_gguf_parameters(self): | ||
| super().set_gguf_parameters() | ||
| self.gguf_writer.add_embedding_length_out(self.hparams["backbone_hidden_size"]) | ||
| self.gguf_writer.add_nextn_predict_layers(self.block_count) | ||
| @ModelBase.register("Gemma4ForConditionalGeneration") | ||
@@ -790,0 +800,0 @@ class Gemma4VisionAudioModel(MmprojModel): |
@@ -108,4 +108,5 @@ from __future__ import annotations | ||
| if "llama_4_scaling" in hparams: | ||
| gguf_writer.add_attn_temperature_scale(hparams["llama_4_scaling"]["beta"]) | ||
| llama_4_scaling = hparams.get("llama_4_scaling") | ||
| if llama_4_scaling is not None: | ||
| gguf_writer.add_attn_temperature_scale(llama_4_scaling["beta"]) | ||
@@ -112,0 +113,0 @@ |
@@ -241,3 +241,3 @@ #!/usr/bin/env python3 | ||
| model_class = PixtralModel | ||
| elif "moe" in hparams: | ||
| elif hparams.get("moe") is not None: | ||
| from conversion.mistral import MistralMoeModel | ||
@@ -244,0 +244,0 @@ model_class = MistralMoeModel |
@@ -64,2 +64,3 @@ #pragma once | ||
| LLM_ARCH_GEMMA4, | ||
| LLM_ARCH_GEMMA4_ASSISTANT, | ||
| LLM_ARCH_GEMMA_EMBEDDING, | ||
@@ -561,2 +562,4 @@ LLM_ARCH_STARCODER2, | ||
| LLM_TENSOR_INDEXER_ATTN_Q_B, | ||
| LLM_TENSOR_NEXTN_PROJ_PRE, | ||
| LLM_TENSOR_NEXTN_PROJ_POST, | ||
| LLM_TENSOR_NEXTN_EH_PROJ, | ||
@@ -563,0 +566,0 @@ LLM_TENSOR_NEXTN_EMBED_TOKENS, |
@@ -9,2 +9,3 @@ #pragma once | ||
| #include "llama-impl.h" | ||
| #include "llama-memory.h" | ||
@@ -277,3 +278,3 @@ #include "ggml-cpp.h" | ||
| std::unique_ptr<llama_memory_i> memory; | ||
| llama_memory_ptr memory; | ||
@@ -280,0 +281,0 @@ // decode output (2-dimensional array: [n_outputs][n_vocab]) |
@@ -52,2 +52,4 @@ #pragma once | ||
| void * cb_eval_user_data; | ||
| llama_context * ctx_other; | ||
| }; |
@@ -103,1 +103,3 @@ #pragma once | ||
| LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i); | ||
| LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); |
@@ -787,2 +787,3 @@ #pragma once | ||
| const int64_t n_layer; | ||
| const int64_t n_layer_nextn; | ||
| const int64_t n_rot; | ||
@@ -789,0 +790,0 @@ const int64_t n_ctx; // user-specified context size (can be different from n_ctx_train) |
@@ -94,2 +94,6 @@ #include "llama-hparams.h" | ||
| uint32_t llama_hparams::n_embd_inp() const { | ||
| if (n_embd_inp_impl > 0) { | ||
| return n_embd_inp_impl; | ||
| } | ||
| uint32_t n_embd_inp = n_embd; | ||
@@ -96,0 +100,0 @@ |
@@ -188,2 +188,5 @@ #pragma once | ||
| // input embedding dimension (0 = use n_embd) | ||
| uint32_t n_embd_inp_impl = 0; | ||
| // output embedding dimension (0 = use n_embd) | ||
@@ -228,2 +231,3 @@ uint32_t n_embd_out_impl = 0; | ||
| // n_embd_imp is accurate (see granite.cpp). | ||
| // TODO: can be expressed via the `new n_embd_inp_impl` and remove this param | ||
| uint32_t n_deepstack_layers = 0; | ||
@@ -230,0 +234,0 @@ |
@@ -35,3 +35,3 @@ #include "llama-kv-cache-dsa.h" | ||
| v_trans, offload, unified, kv_size, n_seq_max, n_pad, | ||
| n_swa, swa_type, filter, reuse); | ||
| n_swa, swa_type, nullptr, filter, reuse, nullptr); | ||
@@ -53,3 +53,3 @@ // we use llama_kv_cache for caching indexer keys | ||
| v_trans, offload, unified, kv_size, n_seq_max, n_pad, | ||
| n_swa, swa_type, filter, reuse); | ||
| n_swa, swa_type, nullptr, filter, reuse, nullptr); | ||
| } | ||
@@ -56,0 +56,0 @@ |
@@ -26,4 +26,6 @@ #include "llama-kv-cache-iswa.h" | ||
| uint32_t n_pad, | ||
| llama_memory_t mem_other, | ||
| const layer_filter_cb & filter, | ||
| const layer_reuse_cb & reuse) : hparams(model.hparams), unified(unified) { | ||
| const layer_reuse_cb & reuse, | ||
| const layer_share_cb & share) : hparams(model.hparams), unified(unified) { | ||
@@ -63,6 +65,16 @@ // chain filters | ||
| llama_memory_t mem_other_base = nullptr; | ||
| if (mem_other) { | ||
| mem_other_base = static_cast<llama_kv_cache_iswa *>(mem_other)->get_base(); | ||
| } | ||
| llama_memory_t mem_other_swa = nullptr; | ||
| if (mem_other) { | ||
| mem_other_swa = static_cast<llama_kv_cache_iswa *>(mem_other)->get_swa(); | ||
| } | ||
| kv_base = std::make_unique<llama_kv_cache>( | ||
| model, hparams, type_k, type_v, | ||
| v_trans, offload, unified, size_base, n_seq_max, n_pad, | ||
| 0, LLAMA_SWA_TYPE_NONE, filter_base, reuse); | ||
| 0, LLAMA_SWA_TYPE_NONE, mem_other_base, filter_base, reuse, share); | ||
@@ -74,3 +86,3 @@ LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa); | ||
| v_trans, offload, unified, size_swa, n_seq_max, n_pad, | ||
| hparams.n_swa, hparams.swa_type, filter_swa, reuse); | ||
| hparams.n_swa, hparams.swa_type, mem_other_swa, filter_swa, reuse, share); | ||
| } | ||
@@ -77,0 +89,0 @@ |
@@ -28,4 +28,6 @@ #pragma once | ||
| uint32_t n_pad, | ||
| llama_memory_t mem_other, | ||
| const layer_filter_cb & filter, | ||
| const layer_reuse_cb & reuse); | ||
| const layer_reuse_cb & reuse, | ||
| const layer_share_cb & share); | ||
@@ -32,0 +34,0 @@ ~llama_kv_cache_iswa() = default; |
@@ -101,3 +101,3 @@ #pragma once | ||
| const llama_model & model, | ||
| const llama_hparams & hparams, | ||
| const llama_hparams & hparams, | ||
| ggml_type type_k, | ||
@@ -113,4 +113,6 @@ ggml_type type_v, | ||
| llama_swa_type swa_type, | ||
| llama_memory_t mem_other, | ||
| const layer_filter_cb & filter, | ||
| const layer_reuse_cb & reuse); | ||
| const layer_reuse_cb & reuse, | ||
| const layer_share_cb & share); | ||
@@ -269,4 +271,9 @@ ~llama_kv_cache() = default; | ||
| std::vector<llama_kv_cells> v_cells; | ||
| // TODO: temporary until we refactor to be able to share the same cells between 2 kv caches [TAG_KV_CACHE_SHARE_CELLS] | ||
| llama_kv_cache * other; | ||
| std::shared_ptr<llama_kv_cells_vec> v_cells_impl; | ||
| llama_kv_cells_vec & v_cells; | ||
| // maps from a sequence id to a stream id | ||
@@ -273,0 +280,0 @@ std::vector<uint32_t> seq_to_stream; |
@@ -534,1 +534,3 @@ #pragma once | ||
| }; | ||
| using llama_kv_cells_vec = std::vector<llama_kv_cells>; |
@@ -46,5 +46,7 @@ #include "llama-memory-hybrid-iswa.h" | ||
| n_pad, | ||
| nullptr, | ||
| filter_attn == nullptr ? | ||
| [&](int32_t il) { return !hparams.is_recr(il); } | ||
| : filter_attn, | ||
| nullptr, | ||
| nullptr | ||
@@ -51,0 +53,0 @@ )), |
@@ -47,5 +47,7 @@ #include "llama-memory-hybrid.h" | ||
| swa_type, | ||
| nullptr, | ||
| filter_attn == nullptr ? | ||
| [&](int32_t il) { return !hparams.is_recr(il); } | ||
| : filter_attn, | ||
| nullptr, | ||
| nullptr | ||
@@ -52,0 +54,0 @@ )), |
@@ -26,2 +26,4 @@ #pragma once | ||
| llama_context_type ctx_type; | ||
| llama_memory_t mem_other; | ||
| }; | ||
@@ -80,2 +82,4 @@ | ||
| using layer_share_cb = std::function<int32_t(int32_t il)>; | ||
| virtual ~llama_memory_i() = default; | ||
@@ -82,0 +86,0 @@ |
@@ -551,2 +551,6 @@ #pragma once | ||
| // NextN/MTP model-level projections | ||
| struct ggml_tensor * nextn_proj_pre = nullptr; | ||
| struct ggml_tensor * nextn_proj_post = nullptr; | ||
| // classifier | ||
@@ -706,2 +710,3 @@ struct ggml_tensor * cls = nullptr; | ||
| const int n_layer_all = hparams.n_layer_all; GGML_UNUSED(n_layer_all); \ | ||
| const int n_layer_nextn = hparams.n_layer_nextn; GGML_UNUSED(n_layer_nextn); \ | ||
| const int64_t n_head = hparams.n_head(); GGML_UNUSED(n_head); \ | ||
@@ -708,0 +713,0 @@ const int64_t n_head_kv = hparams.n_head_kv(); GGML_UNUSED(n_head_kv); \ |
@@ -158,3 +158,3 @@ #include "models.h" | ||
| void set_input(const llama_ubatch *) override { | ||
| void set_input(const llama_ubatch * /*ubatch*/) override { | ||
| const int64_t n_vocab = arr.size(); | ||
@@ -164,3 +164,5 @@ ggml_backend_tensor_set(logits_bias, arr.data(), 0, n_vocab*ggml_element_size(logits_bias)); | ||
| // bool can_reuse(const llm_graph_params & params) override; | ||
| bool can_reuse(const llm_graph_params & /*params*/) override { | ||
| return true; | ||
| } | ||
@@ -275,3 +277,4 @@ ggml_tensor * logits_bias = nullptr; // F32 [n_vocab] | ||
| // TODO @ngxson : strip unused token right after the last KV layer to speed up prompt processing | ||
| if (il == n_layer - 1 && inp_out_ids) { | ||
| // keep all rows when extracting unmasked nextn embeddings (MTP target needs the hidden state for every token) | ||
| if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) { | ||
| cur = ggml_get_rows(ctx0, cur, inp_out_ids); | ||
@@ -376,3 +379,3 @@ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); | ||
| // TODO @ngxson : improve this | ||
| if (il == n_layer - 1 && inp_out_ids) { | ||
| if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) { | ||
| inp_this_layer = ggml_get_rows(ctx0, inp_this_layer, inp_out_ids); | ||
@@ -408,2 +411,13 @@ } | ||
| // Expose the post-output-norm hidden state (the LM-head input feature) so that | ||
| // MTP draft contexts can read it via llama_get_embeddings_nextn_ith() as the | ||
| // recurrent h input. This matches the reference (transformers/vLLM/SGLang), | ||
| // which feeds the drafter the target's post-final-norm hidden state. | ||
| cb(cur, "h_nextn", -1); | ||
| res->t_h_nextn = cur; | ||
| if (!cparams.embeddings_nextn_masked && inp_out_ids) { | ||
| cur = ggml_get_rows(ctx0, cur, inp_out_ids); | ||
| } | ||
| cb(cur, "result_norm", -1); | ||
@@ -410,0 +424,0 @@ res->t_embd = cur; |
@@ -395,3 +395,3 @@ #include "common.h" | ||
| } | ||
| if (arch == LLM_ARCH_GEMMA4) { | ||
| if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) { | ||
| return false; // FIXME @ngxson | ||
@@ -451,3 +451,3 @@ } | ||
| } | ||
| if (arch == LLM_ARCH_GEMMA4) { | ||
| if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) { | ||
| continue; // FIXME: ISWA KV cache initialization needs more fixture params | ||
@@ -555,3 +555,3 @@ } | ||
| } | ||
| if (arch == LLM_ARCH_GEMMA4) { | ||
| if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) { | ||
| continue; // FIXME: ISWA KV cache initialization needs more fixture params | ||
@@ -558,0 +558,0 @@ } |
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Alert delta unavailable
Currently unable to show alert delta for PyPI packages.
Improved metrics
- Total package byte prevSize
189209999
0.02%- Number of package files
3124
0.06%- Lines of code
135234
0.3%