Sign In

llama-cpp-python

Package Overview
Dependencies
Maintainers
1
Versions
208
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

llama-cpp-python - pypi Package Compare versions

Comparing version
0.3.30
 to
0.3.31
.git/modules/vendo...2e2186a8c6065c9a82a82f87fa481bccfd1.idx

Sorry, the diff of this file is not supported yet

.git/modules/vendo...e2186a8c6065c9a82a82f87fa481bccfd1.pack

Sorry, the diff of this file is not supported yet

.git/modules/vendo...2e2186a8c6065c9a82a82f87fa481bccfd1.rev

Sorry, the diff of this file is not supported yet

.git/modules/vendor/llama.cpp/refs/tags/b9733

Sorry, the diff of this file is not supported yet

.git/objects/pack/...0c1b89155882c0f6c309eb28716473a95c6.idx

Sorry, the diff of this file is not supported yet

.git/objects/pack/...c1b89155882c0f6c309eb28716473a95c6.pack

Sorry, the diff of this file is not supported yet

.git/objects/pack/...0c1b89155882c0f6c309eb28716473a95c6.rev

Sorry, the diff of this file is not supported yet

.git/refs/tags/v0.3.31

Sorry, the diff of this file is not supported yet

+24
vendor/llama.cpp/....tions/windows-setup-openvino/action.yml
name: "Windows - Setup OpenVINO Toolkit"
description: "Setup OpenVINO Toolkit for Windows"
inputs:
path:
description: "Installation path"
required: true
version_major:
description: "OpenVINO major version (e.g., 2026.2)"
required: true
version_full:
description: "OpenVINO full version"
required: true
runs:
using: "composite"
steps:
- name: Download and extract OpenVINO Runtime
shell: powershell
run: |
$url = "https://storage.openvinotoolkit.org/repositories/openvino/packages/${{ inputs.version_major }}/windows/openvino_toolkit_windows_${{ inputs.version_full }}_x86_64.zip"
$out = "openvino.zip"
Invoke-WebRequest -Uri $url -OutFile $out
Expand-Archive -Path $out -DestinationPath ${{ inputs.path }} -Force
Remove-Item $out
+9
vendor/llama.cpp/examples/sycl/update-ops-doc.sh
#!/bin/bash
# MIT license
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: MIT
./build/bin/test-backend-ops support --output csv > docs/ops/SYCL.csv
./scripts/create_ops_docs.py
+8
vendor/llama.cpp/examples/sycl/win-update-ops-doc.bat
@echo off
rem MIT license
rem Copyright (C) 2026 Intel Corporation
rem SPDX-License-Identifier: MIT
build\bin\test-backend-ops support --output csv > docs\ops\SYCL.csv
python scripts\create_ops_docs.py
vendor/llama.cpp/ggml/src/ggml-cuda/col2im-1d.cu

Sorry, the diff of this file is not supported yet

vendor/llama.cpp/ggml/src/ggml-cuda/col2im-1d.cuh

Sorry, the diff of this file is not supported yet

+64
vendor/llama.cpp/ggml/src/ggml-hexagon/htp/hex-profile.h
#ifndef HEX_PROFILE_H
#define HEX_PROFILE_H
#include <stdbool.h>
#include <stdint.h>
#include <qurt.h>
#include "hex-utils.h"
#include "htp-ops.h"
#define HTP_TRACE_EVT_START 0
#define HTP_TRACE_EVT_STOP 1
#ifndef HEX_NUM_PMU_COUNTERS
#define HEX_NUM_PMU_COUNTERS 8
#endif
static inline void hex_get_pmu(uint32_t counters[]) {
#if __HVX_ARCH__ >= 79
asm volatile("%0 = upmucnt0" : "=r"(counters[0]));
asm volatile("%0 = upmucnt1" : "=r"(counters[1]));
asm volatile("%0 = upmucnt2" : "=r"(counters[2]));
asm volatile("%0 = upmucnt3" : "=r"(counters[3]));
asm volatile("%0 = upmucnt4" : "=r"(counters[4]));
asm volatile("%0 = upmucnt5" : "=r"(counters[5]));
asm volatile("%0 = upmucnt6" : "=r"(counters[6]));
asm volatile("%0 = upmucnt7" : "=r"(counters[7]));
#else
counters[0] = qurt_pmu_get(QURT_PMUCNT0);
counters[1] = qurt_pmu_get(QURT_PMUCNT1);
counters[2] = qurt_pmu_get(QURT_PMUCNT2);
counters[3] = qurt_pmu_get(QURT_PMUCNT3);
counters[4] = qurt_pmu_get(QURT_PMUCNT4);
counters[5] = qurt_pmu_get(QURT_PMUCNT5);
counters[6] = qurt_pmu_get(QURT_PMUCNT6);
counters[7] = qurt_pmu_get(QURT_PMUCNT7);
#endif
}
struct htp_thread_trace {
uint32_t count;
uint32_t max_events;
struct htp_trace_desc * events;
};
static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) {
if (tr && tr->events && tr->count < tr->max_events) {
uint32_t idx = tr->count;
tr->events[idx].id = id;
tr->events[idx].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0);
tr->events[idx].cycles = (uint32_t) hex_get_cycles();
tr->count++;
}
}
static inline void htp_trace_event_start(struct htp_thread_trace * tr, uint16_t id, uint16_t info) {
htp_trace_event(tr, id, info, HTP_TRACE_EVT_START);
}
static inline void htp_trace_event_stop(struct htp_thread_trace * tr, uint16_t id, uint16_t info) {
htp_trace_event(tr, id, info, HTP_TRACE_EVT_STOP);
}
#endif /* HEX_PROFILE_H */
+62
vendor/llama.cpp/g...rc/ggml-openvino/openvino/op/add_id.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include <memory>
#include <openvino/core/node.hpp>
#include <openvino/core/node_output.hpp>
#include <openvino/op/add.hpp>
#include <openvino/op/constant.hpp>
#include <openvino/op/convert.hpp>
#include <openvino/op/gather.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/shape_of.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_add_id(const NodeContext & context) {
num_inputs_check(context, 3, 3);
auto input = process_view_input_new(context, 0);
auto bias = process_view_input_new(context, 1);
auto ids = process_view_input_new(context, 2);
// OpenVINO uses reversed GGML dimensions:
// input: [1, n_token, n_used, n_embd]
// bias: [1, 1, n_expert, n_embd]
// ids: [1, 1, n_token, n_used]
auto bias_shape_4d = std::make_shared<ov::op::v3::ShapeOf>(bias, ov::element::i64);
auto ids_shape_4d = std::make_shared<ov::op::v3::ShapeOf>(ids, ov::element::i64);
bias = std::make_shared<ov::op::v1::Reshape>(bias, get_dimensions(bias_shape_4d, {2, 3}), false);
ids = std::make_shared<ov::op::v1::Reshape>(ids, get_dimensions(ids_shape_4d, {2, 3}), false);
if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) {
ids = std::make_shared<ov::op::v0::Convert>(ids, ov::element::i32);
}
auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});
ov::Output<ov::Node> selected_bias = std::make_shared<ov::op::v8::Gather>(bias, ids, gather_axis);
selected_bias = std::make_shared<ov::op::v1::Reshape>(
selected_bias, std::make_shared<ov::op::v3::ShapeOf>(input, ov::element::i64), false);
if (selected_bias.get_element_type() != input.get_element_type()) {
selected_bias = std::make_shared<ov::op::v0::Convert>(selected_bias, input.get_element_type());
}
ov::Output<ov::Node> res = std::make_shared<ov::op::v1::Add>(input, selected_bias);
const auto output_type = context.get_output_type();
if (res.get_element_type() != output_type) {
res = std::make_shared<ov::op::v0::Convert>(res, output_type);
}
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+47
vendor/llama.cpp/g...c/ggml-openvino/openvino/op/argsort.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include "ggml.h"
#include <openvino/frontend/exception.hpp>
#include <openvino/op/constant.hpp>
#include <openvino/op/squeeze.hpp>
#include <openvino/op/topk.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_argsort(const NodeContext & context) {
num_inputs_check(context, 1, 1);
auto input = process_view_input_new(context, 0);
const int32_t order = context.get_output_op_params()[0];
ov::op::v11::TopK::Mode mode;
switch (order) {
case GGML_SORT_ORDER_ASC:
mode = ov::op::v11::TopK::Mode::MIN;
break;
case GGML_SORT_ORDER_DESC:
mode = ov::op::v11::TopK::Mode::MAX;
break;
default:
FRONT_END_OP_CONVERSION_CHECK(false, "Unsupported GGML_OP_ARGSORT order: ", order);
}
auto k = std::make_shared<ov::op::v0::Squeeze>(get_dimensions(input.get_node_shared_ptr(), {3}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
auto topk = std::make_shared<ov::op::v11::TopK>(input, k, 3, mode, ov::op::v11::TopK::SortType::SORT_VALUES,
context.get_output_type(), false);
return rename_outputs_with_suffix({topk->output(1)}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+33
vendor/llama.cpp/g...src/ggml-openvino/openvino/op/clamp.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include <cstring>
#include <openvino/op/clamp.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_clamp(const NodeContext & context) {
num_inputs_check(context, 1, 1);
auto input = process_view_input_new(context, 0);
const int32_t * op_params = context.get_output_op_params();
FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "CLAMP requires output op params");
float min;
float max;
std::memcpy(&min, reinterpret_cast<const float *>(op_params) + 0, sizeof(float));
std::memcpy(&max, reinterpret_cast<const float *>(op_params) + 1, sizeof(float));
auto res = std::make_shared<ov::op::v0::Clamp>(input, min, max);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+48
vendor/llama.cpp/g...rc/ggml-openvino/openvino/op/concat.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include <memory>
#include <openvino/frontend/exception.hpp>
#include <openvino/op/concat.hpp>
#include <openvino/op/convert.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_concat(const NodeContext & context) {
num_inputs_check(context, 2, 2);
const int32_t * op_params = context.get_output_op_params();
FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "CONCAT requires output op params");
const auto output_shape = context.get_output_shape();
FRONT_END_CHECK_IMPLEMENTED(output_shape.rank().is_static(), "CONCAT requires static output rank");
const auto rank = output_shape.rank().get_length();
const int32_t ggml_dim = op_params[0];
FRONT_END_CHECK_IMPLEMENTED(ggml_dim >= 0 && ggml_dim < rank, "CONCAT axis is out of range");
auto input_0 = process_view_input_new(context, 0);
auto input_1 = process_view_input_new(context, 1);
const auto output_type = context.get_output_type();
if (input_0.get_element_type() != output_type) {
input_0 = std::make_shared<ov::op::v0::Convert>(input_0, output_type);
}
if (input_1.get_element_type() != output_type) {
input_1 = std::make_shared<ov::op::v0::Convert>(input_1, output_type);
}
const auto axis = static_cast<int64_t>(rank - 1 - ggml_dim);
auto res = std::make_shared<ov::op::v0::Concat>(OutputVector{input_0, input_1}, axis);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+146
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/op/div.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include "ggml.h"
#include <memory>
#include <openvino/op/constant.hpp>
#include <openvino/op/convert.hpp>
#include <openvino/op/divide.hpp>
#include <openvino/op/multiply.hpp>
#include <openvino/op/shape_of.hpp>
#include <openvino/op/sigmoid.hpp>
#include <openvino/op/tile.hpp>
#include <openvino/op/util/precision_sensitive_attribute.hpp>
#include <vector>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
namespace {
bool is_silu_div_pattern(const ov::Output<ov::Node> & numerator,
const ov::Output<ov::Node> & denominator,
const NodeContext & context) {
if (context.get_input_size() != 2) {
return false;
}
const auto * unary_op = reinterpret_cast<const ggml_unary_op *>(context.get_input_op_params(0));
if (unary_op == nullptr || *unary_op != GGML_UNARY_OP_SILU) {
return false;
}
auto mul = std::dynamic_pointer_cast<ov::op::v1::Multiply>(numerator.get_node_shared_ptr());
if (!mul) {
return false;
}
const auto denom_node = denominator.get_node_shared_ptr();
const auto mul_input_0 = mul->input_value(0).get_node_shared_ptr();
const auto mul_input_1 = mul->input_value(1).get_node_shared_ptr();
auto sigmoid = std::dynamic_pointer_cast<ov::op::v0::Sigmoid>(mul_input_1);
if (mul_input_0 == denom_node && sigmoid && sigmoid->input_value(0).get_node_shared_ptr() == denom_node) {
return true;
}
sigmoid = std::dynamic_pointer_cast<ov::op::v0::Sigmoid>(mul_input_0);
return mul_input_1 == denom_node && sigmoid && sigmoid->input_value(0).get_node_shared_ptr() == denom_node;
}
ov::Output<ov::Node> repeat_input_to_match(const NodeContext & context,
const ov::Output<ov::Node> & input,
const ov::Output<ov::Node> & target,
size_t input_index) {
const auto input_shape = context.get_input_shape(input_index);
const auto target_shape = context.get_input_shape(0);
if (input_shape == target_shape) {
return input;
}
if (input_shape.rank().is_static() && target_shape.rank().is_static()) {
const auto rank = static_cast<size_t>(input_shape.rank().get_length());
std::vector<int64_t> repeats(rank, 1);
bool needs_repeat = false;
for (size_t axis = 0; axis < rank; ++axis) {
FRONT_END_OP_CONVERSION_CHECK(input_shape[axis].is_static() && target_shape[axis].is_static(),
"DIV repeat requires static dimensions on both inputs");
const int64_t input_dim = input_shape[axis].get_length();
const int64_t target_dim = target_shape[axis].get_length();
FRONT_END_OP_CONVERSION_CHECK(input_dim > 0 && target_dim > 0 && target_dim % input_dim == 0,
"DIV input shape ", input_shape, " cannot repeat to match ", target_shape);
repeats[axis] = target_dim / input_dim;
needs_repeat = needs_repeat || repeats[axis] != 1;
}
if (!needs_repeat) {
return input;
}
auto repeats_node = ov::op::v0::Constant::create(ov::element::i64, {repeats.size()}, repeats);
return std::make_shared<ov::op::v0::Tile>(input, repeats_node);
}
auto input_shape_node = std::make_shared<ov::op::v3::ShapeOf>(input, ov::element::i64);
auto target_shape_node = std::make_shared<ov::op::v3::ShapeOf>(target, ov::element::i64);
auto repeats_node = std::make_shared<ov::op::v1::Divide>(target_shape_node, input_shape_node);
return std::make_shared<ov::op::v0::Tile>(input, repeats_node);
}
} // namespace
OutputVector translate_div(const NodeContext & context) {
num_inputs_check(context, 2, 2);
auto input_0 = process_view_input_new(context, 0);
auto input_1 = process_view_input_new(context, 1);
if (is_silu_div_pattern(input_0, input_1, context)) {
ov::Output<ov::Node> res = std::make_shared<ov::op::v0::Sigmoid>(input_1);
if (res.get_element_type() != context.get_output_type()) {
res = std::make_shared<ov::op::v0::Convert>(res, context.get_output_type());
}
return rename_outputs_with_suffix({res}, context.get_name());
}
input_1 = repeat_input_to_match(context, input_1, input_0, 1);
const auto output_type = context.get_output_type();
const bool use_f32_compute = input_0.get_element_type() != ov::element::f32 ||
input_1.get_element_type() != ov::element::f32 || output_type != ov::element::f32;
if (use_f32_compute) {
input_0 = std::make_shared<ov::op::v0::Convert>(input_0, ov::element::f32);
input_1 = std::make_shared<ov::op::v0::Convert>(input_1, ov::element::f32);
}
ov::Output<ov::Node> res = std::make_shared<ov::op::v1::Divide>(input_0, input_1);
if (use_f32_compute) {
// Keep the reciprocal/divide path in FP32. Without this hint, the GPU
// plugin can still compress the subgraph back to FP16 and overflow on
// small shexp gate values (e.g. silu(x) / x in qwen2moe).
ov::mark_as_precision_sensitive(res.get_node_shared_ptr()->input(0));
ov::mark_as_precision_sensitive(res.get_node_shared_ptr()->input(1));
}
if (res.get_element_type() != output_type) {
auto output_convert = std::make_shared<ov::op::v0::Convert>(res, output_type);
if (use_f32_compute) {
ov::mark_as_precision_sensitive(output_convert->input(0));
}
res = output_convert;
}
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+282
vendor/llama.cpp/g...penvino/openvino/op/gated_delta_net.cpp
#include "gated_delta_net.hpp"
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include <cmath>
#include <cstdint>
#include <memory>
#include <openvino/op/add.hpp>
#include <openvino/op/broadcast.hpp>
#include <openvino/op/concat.hpp>
#include <openvino/op/constant.hpp>
#include <openvino/op/exp.hpp>
#include <openvino/op/gather.hpp>
#include <openvino/op/loop.hpp>
#include <openvino/op/matmul.hpp>
#include <openvino/op/multiply.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/squeeze.hpp>
#include <openvino/op/subtract.hpp>
#include <openvino/op/transpose.hpp>
#include <openvino/op/unsqueeze.hpp>
#include <vector>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
static OutputVector translate_gated_delta_net_ref(const NodeContext & context);
OutputVector translate_gated_delta_net(const NodeContext & context) {
// auto v_shape = context.get_input_shape(2).to_shape(); // [B, T, H_v, S_v]
// auto q_shape = context.get_input_shape(0).to_shape(); // [B, T, H_k, S_k]
// // Fused GatedDeltaNet op only supports scalar gate (kda=0).
// // Fall back to reference implementation for per-key-dimension gating.
// // if (kda) {
// // return translate_gated_delta_net_ref(context);
// // }
// auto q = context.get_input(0);
// auto k = context.get_input(1);
// auto v = context.get_input(2);
// auto g = context.get_input(3);
// auto beta = context.get_input(4);
// auto state = context.get_input(5);
// const int64_t B = v_shape[0];
// const int64_t T = v_shape[1];
// const int64_t H_v = v_shape[2];
// const int64_t S_v = v_shape[3];
// const int64_t S_k = q_shape[3];
// // ggml state layout (OV notation): [B, H_v, value_dim, key_dim]
// // GatedDeltaNet op expects: [B, H_v, key_dim, value_dim]
// auto state_reshape_shape =
// ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{B, H_v, S_v, S_k});
// state = std::make_shared<ov::op::v1::Reshape>(state, state_reshape_shape, false);
// auto state_perm = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, 1, 3, 2});
// state = std::make_shared<ov::op::v1::Transpose>(state, state_perm);
// g = std::make_shared<ov::op::v0::Squeeze>(g, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
// beta = std::make_shared<ov::op::v0::Squeeze>(beta, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
// auto gdn = std::make_shared<ov::op::internal::GatedDeltaNet>(q, k, v, state, g, beta);
// auto attn_4d = gdn->output(0);
// auto state_4d = gdn->output(1); // [B, H_v, key_dim, value_dim]
// // Transpose output state back to ggml layout [B, H_v, value_dim, key_dim]
// auto state_transposed = std::make_shared<ov::op::v1::Transpose>(state_4d, state_perm);
// auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
// auto attn = std::make_shared<ov::op::v1::Reshape>(attn_4d, flat_shape_1d, false);
// auto new_state = std::make_shared<ov::op::v1::Reshape>(state_transposed, flat_shape_1d, false);
// auto packed = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{attn, new_state}, 0);
// auto out_shape =
// ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, T * B + S_v * B, S_v * H_v});
// auto res = std::make_shared<ov::op::v1::Reshape>(packed, out_shape, false);
// return rename_outputs_with_suffix({res}, context.get_name());
// The OV version in CI does not have the GatedDeltaNet op, so use reference implementation for now.
return translate_gated_delta_net_ref(context);
}
static OutputVector translate_gated_delta_net_ref(const NodeContext & context) {
num_inputs_check(context, 6, 6);
// Inputs (OV shapes are reversed from ggml):
// ggml: q[S_k, H_k, T, B], k[S_k, H_k, T, B], v[S_v, H_v, T, B]
// OV: q[B, T, H_k, S_k], k[B, T, H_k, S_k], v[B, T, H_v, S_v]
// ggml: g[1 or S_v, H_v, T, B], beta[1, H_v, T, B]
// OV: g[B, T, H_v, 1 or S_v], beta[B, T, H_v, 1]
// ggml: state[S_v, S_v, H_v, B]
// OV: state[B, H_v, S_v, S_v]
auto q = process_view_input_new(context, 0);
auto k = process_view_input_new(context, 1);
auto v = process_view_input_new(context, 2);
auto g = process_view_input_new(context, 3);
auto beta = process_view_input_new(context, 4);
auto state = process_view_input_new(context, 5);
auto v_shape = context.get_input_shape(2).to_shape(); // [B, T, H_v, S_v]
auto q_shape = context.get_input_shape(0).to_shape(); // [B, T, H_k, S_k]
auto g_shape = context.get_input_shape(3).to_shape(); // [B, T, H_v, 1 or S_v]
const int64_t B = v_shape[0];
const int64_t T = v_shape[1];
const int64_t H_v = v_shape[2];
const int64_t S_v = v_shape[3];
const int64_t H_k = q_shape[2];
const bool kda = (g_shape[3] == (size_t) S_v);
const int64_t rq1 = H_v / H_k; // head repeat factor
const float scale = 1.0f / std::sqrt((float) S_v);
auto axis_1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto axis_2 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
// Transpose inputs from [B, T, H, S] to [B, H, T, S] for easier per-head processing
auto perm_0213 = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, 2, 1, 3});
auto q_t = std::make_shared<ov::op::v1::Transpose>(q, perm_0213); // [B, H_k, T, S_k]
auto k_t = std::make_shared<ov::op::v1::Transpose>(k, perm_0213); // [B, H_k, T, S_k]
auto v_t = std::make_shared<ov::op::v1::Transpose>(v, perm_0213); // [B, H_v, T, S_v]
auto g_t = std::make_shared<ov::op::v1::Transpose>(g, perm_0213); // [B, H_v, T, 1 or S_v]
auto beta_t = std::make_shared<ov::op::v1::Transpose>(beta, perm_0213); // [B, H_v, T, 1]
// Broadcast Q, K heads to match V heads if GQA is used (H_v > H_k)
ov::Output<ov::Node> q_bh = q_t;
ov::Output<ov::Node> k_bh = k_t;
if (rq1 > 1) {
auto q_unsq = std::make_shared<ov::op::v0::Unsqueeze>(q_t, axis_2); // [B, H_k, 1, T, S]
auto k_unsq = std::make_shared<ov::op::v0::Unsqueeze>(k_t, axis_2); // [B, H_k, 1, T, S]
auto bcast_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector<int64_t>{1, 1, rq1, 1, 1});
auto q_bcast =
std::make_shared<ov::op::v3::Broadcast>(q_unsq, bcast_shape, ov::op::BroadcastType::BIDIRECTIONAL);
auto k_bcast =
std::make_shared<ov::op::v3::Broadcast>(k_unsq, bcast_shape, ov::op::BroadcastType::BIDIRECTIONAL);
// Transpose [B, H_k, rq1, T, S] -> [B, rq1, H_k, T, S] so that reshape merges
// as [rq1, H_k] giving repeat-blocks pattern matching CPU: iq1 = iv1 % H_k
auto perm_5d = ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector<int64_t>{0, 2, 1, 3, 4});
auto q_transposed = std::make_shared<ov::op::v1::Transpose>(q_bcast, perm_5d);
auto k_transposed = std::make_shared<ov::op::v1::Transpose>(k_bcast, perm_5d);
auto new_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{B, H_v, T, S_v});
q_bh = std::make_shared<ov::op::v1::Reshape>(q_transposed, new_shape, false);
k_bh = std::make_shared<ov::op::v1::Reshape>(k_transposed, new_shape, false);
}
// Merge batch and head dims: [B*H_v, T, S_v]
auto merge_bh = [&](ov::Output<ov::Node> x, int64_t last_dim) {
auto shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{B * H_v, T, last_dim});
return std::make_shared<ov::op::v1::Reshape>(x, shape, false);
};
auto q_m = merge_bh(q_bh, S_v); // [B*H_v, T, S_v]
auto k_m = merge_bh(k_bh, S_v); // [B*H_v, T, S_v]
auto v_m = merge_bh(v_t, S_v); // [B*H_v, T, S_v]
auto g_m = merge_bh(g_t, kda ? S_v : 1); // [B*H_v, T, 1 or S_v]
auto beta_m = merge_bh(beta_t, 1); // [B*H_v, T, 1]
// State: [B, H_v, S_v, S_v] -> [B*H_v, S_v, S_v]
auto state_shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{B * H_v, S_v, S_v});
auto state_m = std::make_shared<ov::op::v1::Reshape>(state, state_shape, false);
auto scale_const = ov::op::v0::Constant::create(ov::element::f32, {}, std::vector<float>{scale});
// --- Build Loop body ---
// Body parameters (no iteration counter needed, use -1 in special ports)
auto body_state = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape::dynamic());
auto body_q = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape::dynamic());
auto body_k = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape::dynamic());
auto body_v = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape::dynamic());
auto body_g = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape::dynamic());
auto body_beta = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape::dynamic());
auto body_iter = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::Shape{1});
// Condition output (always true - we rely on trip_count for termination)
auto body_cond_out = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, std::vector<bool>{true});
// Gather current token from invariant inputs using iteration counter
auto q_t_cur = std::make_shared<ov::op::v8::Gather>(body_q, body_iter, axis_1); // [B*H_v, 1, S_v]
auto k_t_cur = std::make_shared<ov::op::v8::Gather>(body_k, body_iter, axis_1); // [B*H_v, 1, S_v]
auto v_t_cur = std::make_shared<ov::op::v8::Gather>(body_v, body_iter, axis_1); // [B*H_v, 1, S_v]
auto g_t_cur = std::make_shared<ov::op::v8::Gather>(body_g, body_iter, axis_1); // [B*H_v, 1, 1 or S_v]
auto b_t_cur = std::make_shared<ov::op::v8::Gather>(body_beta, body_iter, axis_1); // [B*H_v, 1, 1]
// Squeeze token dim
auto q_cur = std::make_shared<ov::op::v0::Squeeze>(q_t_cur, axis_1); // [B*H_v, S_v]
auto k_cur = std::make_shared<ov::op::v0::Squeeze>(k_t_cur, axis_1); // [B*H_v, S_v]
auto v_cur = std::make_shared<ov::op::v0::Squeeze>(v_t_cur, axis_1); // [B*H_v, S_v]
auto g_cur = std::make_shared<ov::op::v0::Squeeze>(g_t_cur, axis_1); // [B*H_v, 1 or S_v]
auto b_cur = std::make_shared<ov::op::v0::Squeeze>(b_t_cur, axis_1); // [B*H_v, 1]
// Step 1: Apply decay gate to state
auto exp_g = std::make_shared<ov::op::v0::Exp>(g_cur); // [B*H_v, 1 or S_v]
auto exp_g_unsq = std::make_shared<ov::op::v0::Unsqueeze>(exp_g, axis_1); // [B*H_v, 1, 1 or S_v]
auto state_decayed = std::make_shared<ov::op::v1::Multiply>(body_state, exp_g_unsq); // [B*H_v, S_v, S_v]
// Step 2: delta = (v - S @ k) * beta
auto k_col = std::make_shared<ov::op::v0::Unsqueeze>(k_cur, axis_2); // [B*H_v, S_v, 1]
auto sk = std::make_shared<ov::op::v0::MatMul>(state_decayed, k_col, false, false); // [B*H_v, S_v, 1]
auto sk_sq = std::make_shared<ov::op::v0::Squeeze>(sk, axis_2); // [B*H_v, S_v]
auto v_minus_sk = std::make_shared<ov::op::v1::Subtract>(v_cur, sk_sq); // [B*H_v, S_v]
auto delta = std::make_shared<ov::op::v1::Multiply>(v_minus_sk, b_cur); // [B*H_v, S_v]
// Step 3: state += outer(delta, k)
auto delta_col = std::make_shared<ov::op::v0::Unsqueeze>(delta, axis_2); // [B*H_v, S_v, 1]
auto k_row = std::make_shared<ov::op::v0::Unsqueeze>(k_cur, axis_1); // [B*H_v, 1, S_v]
auto outer_prod = std::make_shared<ov::op::v0::MatMul>(delta_col, k_row, false, false); // [B*H_v, S_v, S_v]
auto state_updated = std::make_shared<ov::op::v1::Add>(state_decayed, outer_prod); // [B*H_v, S_v, S_v]
// Step 4: attn_out = S @ q * scale
auto q_col = std::make_shared<ov::op::v0::Unsqueeze>(q_cur, axis_2); // [B*H_v, S_v, 1]
auto sq = std::make_shared<ov::op::v0::MatMul>(state_updated, q_col, false, false); // [B*H_v, S_v, 1]
auto sq_squeezed = std::make_shared<ov::op::v0::Squeeze>(sq, axis_2); // [B*H_v, S_v]
auto attn_out = std::make_shared<ov::op::v1::Multiply>(sq_squeezed, scale_const); // [B*H_v, S_v]
// Unsqueeze attn_out to [B*H_v, 1, S_v] for scan output concatenation
auto attn_out_unsq = std::make_shared<ov::op::v0::Unsqueeze>(attn_out, axis_1); // [B*H_v, 1, S_v]
// --- Assemble Loop ---
// Body: results = [condition, state_updated, attn_out_unsq]
auto body = std::make_shared<ov::Model>(
ov::OutputVector{body_cond_out, state_updated, attn_out_unsq},
ov::ParameterVector{body_iter, body_state, body_q, body_k, body_v, body_g, body_beta});
auto trip_count = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, std::vector<int64_t>{T});
auto exec_cond = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, std::vector<bool>{true});
auto loop = std::make_shared<ov::op::v5::Loop>(trip_count, exec_cond);
loop->set_function(body);
loop->set_special_body_ports(ov::op::v5::Loop::SpecialBodyPorts{0, 0});
// Carried state: feeds back from body output 1 to body_state param
loop->set_merged_input(body_state, state_m, state_updated);
// Invariant inputs: passed through unchanged each iteration
loop->set_invariant_input(body_q, q_m);
loop->set_invariant_input(body_k, k_m);
loop->set_invariant_input(body_v, v_m);
loop->set_invariant_input(body_g, g_m);
loop->set_invariant_input(body_beta, beta_m);
// Loop outputs:
// 1) Final state (last iteration value of state_updated)
auto final_state_out = loop->get_iter_value(state_updated, -1); // [B*H_v, S_v, S_v]
// 2) Concatenated attention outputs across all iterations along axis 1
auto attn_concat_out = loop->get_concatenated_slices(attn_out_unsq, 0, 1, 1, -1, 1); // [B*H_v, T, S_v]
// --- Pack outputs to match ggml layout ---
// ggml output ne = {S_v*H, T*B + S_v*B, 1, 1} -> OV [1, 1, T*B+S_v*B, S_v*H_v]
// attn: [B, T, H_v, S_v] row-major, state: [B, H_v, S_v, S_v] row-major
// attn: [B*H_v, T, S_v] -> [B, H_v, T, S_v] -> transpose to [B, T, H_v, S_v] -> flatten
auto attn_4d_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{B, H_v, T, S_v});
auto attn_4d = std::make_shared<ov::op::v1::Reshape>(attn_concat_out, attn_4d_shape, false);
auto attn_perm = std::make_shared<ov::op::v1::Transpose>(attn_4d, perm_0213); // [B, T, H_v, S_v]
auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector<int64_t>{-1});
auto attn_1d = std::make_shared<ov::op::v1::Reshape>(attn_perm, flat_shape_1d, false);
// state: [B*H_v, S_v, S_v] -> [B, H_v, S_v, S_v] -> flatten
auto state_4d_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{B, H_v, S_v, S_v});
auto state_4d = std::make_shared<ov::op::v1::Reshape>(final_state_out, state_4d_shape, false);
auto state_1d = std::make_shared<ov::op::v1::Reshape>(state_4d, flat_shape_1d, false);
// Concat [attn | state] and reshape to final output
auto packed = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{attn_1d, state_1d}, 0);
auto out_shape =
ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, T * B + S_v * B, S_v * H_v});
auto res = std::make_shared<ov::op::v1::Reshape>(packed, out_shape, false);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+65
vendor/llama.cpp/g...penvino/openvino/op/gated_delta_net.hpp
#pragma once
#include "openvino/op/op.hpp"
namespace ov::op::internal {
/// \note GatedDeltaNet op class is under development and subject to change
///
/// \brief Operator performing Gated Delta Net computation
/// \ingroup ov_ops_cpp_api
class OPENVINO_API GatedDeltaNet : public ov::op::Op {
public:
OPENVINO_OP("GatedDeltaNet")
GatedDeltaNet() = default;
/// \brief Constructs a GatedDeltaNet operation.
///
/// \param query Query tensor input.
/// \param key Key tensor input.
/// \param value Value tensor input.
/// \param recurrent_state Initial recurrent state tensor.
/// \param gate Gate tensor controlling state decay/update.
/// \param beta Beta tensor scaling the delta update.
/// \param fuse_qk_l2norm Enables fusing q/k L2-normalization into this op.
/// \param q_l2_norm_eps Epsilon used for query L2-normalization when fusion is enabled.
/// \param k_l2_norm_eps Epsilon used for key L2-normalization when fusion is enabled.
GatedDeltaNet(const Output<Node>& query,
const Output<Node>& key,
const Output<Node>& value,
const Output<Node>& recurrent_state,
const Output<Node>& gate,
const Output<Node>& beta,
const bool fuse_qk_l2norm = false,
const float q_l2_norm_eps = 1e-6F,
const float k_l2_norm_eps = 1e-6F);
/// \brief Constructs a GatedDeltaNet operation from input vector.
///
/// \param args Input tensor vector in order: query, key, value, recurrent_state, gate, beta.
/// \param fuse_qk_l2norm Enables fusing q/k L2-normalization into this op.
/// \param q_l2_norm_eps Epsilon used for query L2-normalization when fusion is enabled.
/// \param k_l2_norm_eps Epsilon used for key L2-normalization when fusion is enabled.
GatedDeltaNet(const ov::OutputVector& args,
const bool fuse_qk_l2norm = false,
const float q_l2_norm_eps = 1e-6F,
const float k_l2_norm_eps = 1e-6F);
void validate_and_infer_types() override;
bool visit_attributes(AttributeVisitor& visitor) override;
std::shared_ptr<ov::Node> clone_with_new_inputs(const ov::OutputVector& new_args) const override;
bool get_fuse_qk_l2norm() const {
return m_fuse_qk_l2norm;
}
float get_q_l2_norm_eps() const {
return m_q_l2_norm_eps;
}
float get_k_l2_norm_eps() const {
return m_k_l2_norm_eps;
}
private:
bool m_fuse_qk_l2norm = false;
float m_q_l2_norm_eps = 1e-6F;
float m_k_l2_norm_eps = 1e-6F;
};
} // namespace ov::op::internal
+120
vendor/llama.cpp/g...rc/ggml-openvino/openvino/op/im2col.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include "ggml-impl.h"
#include <cstddef>
#include <memory>
#include <openvino/core/shape.hpp>
#include <openvino/core/strides.hpp>
#include <openvino/op/constant.hpp>
#include <openvino/op/convert.hpp>
#include <openvino/op/extractimagepatches.hpp>
#include <openvino/op/pad.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/transpose.hpp>
#include <openvino/op/util/attr_types.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_im2col(const NodeContext & context) {
num_inputs_check(context, 2, 2);
const int32_t * params = context.get_output_op_params();
int32_t s0 = params[0];
int32_t s1 = params[1];
int32_t p0 = params[2];
int32_t p1 = params[3];
int32_t d0 = params[4];
int32_t d1 = params[5];
bool is_2D = params[6] == 1;
ov::Output<Node> res;
ov::Output<Node> image = context.get_input(1);
const ov::Shape kernel_shape = context.get_input(0).get_shape();
const size_t IC = is_2D ? kernel_shape[1] : kernel_shape[2];
const size_t KH = is_2D ? kernel_shape[2] : 1;
const size_t KW = kernel_shape[3];
int32_t stride_w = s0;
int32_t stride_h = is_2D ? s1 : 1;
int32_t pad_w = p0;
int32_t pad_h = is_2D ? p1 : 0;
int32_t dil_w = d0;
int32_t dil_h = is_2D ? d1 : 1;
if (!is_2D) {
// GGML input shape: [IW, IC, N, 1]
// OpenVINO input shape: [1, N, IC, IW]
// Reshape image to: [N, IC, 1, IW]
const ov::Shape image_shape = image.get_shape();
const size_t N = image_shape[1];
const size_t IW = image_shape[3];
auto image_reshape_shape = ov::op::v0::Constant::create(
ov::element::i64, ov::Shape{4},
std::vector<int64_t>{static_cast<int64_t>(N), static_cast<int64_t>(IC), 1, static_cast<int64_t>(IW)});
image = std::make_shared<ov::op::v1::Reshape>(image, image_reshape_shape, false);
}
const ov::Shape patch_sizes = {KH, KW};
const ov::Strides strides = {static_cast<size_t>(stride_h), static_cast<size_t>(stride_w)};
const ov::Shape rates = {static_cast<size_t>(dil_h), static_cast<size_t>(dil_w)};
auto pads_begin =
ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector<int64_t>{0, 0, pad_h, pad_w});
auto pads_end =
ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector<int64_t>{0, 0, pad_h, pad_w});
auto pad = std::make_shared<ov::op::v1::Pad>(image, pads_begin, pads_end, ov::op::PadMode::CONSTANT);
auto patches =
std::make_shared<ov::op::v3::ExtractImagePatches>(pad, patch_sizes, strides, rates, ov::op::PadType::VALID);
// [N, KH*KW*IC, OH, OW] → [N, OH, OW, KH*KW*IC]
auto perm1 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector<int64_t>{0, 2, 3, 1});
auto t1 = std::make_shared<ov::op::v1::Transpose>(patches, perm1);
// [N, OH, OW, KH*KW*IC] → [N, OH, OW, KH*KW, IC]
const ov::Shape out_shape = t1->get_output_shape(0);
const size_t N = out_shape[0];
const size_t OH = out_shape[1];
const size_t OW = out_shape[2];
auto reshape1_shape = ov::op::v0::Constant::create(
ov::element::i64, ov::Shape{5},
std::vector<int64_t>{static_cast<int64_t>(N), static_cast<int64_t>(OH), static_cast<int64_t>(OW),
static_cast<int64_t>(KH * KW), static_cast<int64_t>(IC)});
auto r1 = std::make_shared<ov::op::v1::Reshape>(t1, reshape1_shape, false);
// [N, OH, OW, KH*KW, IC] → [N, OH, OW, IC, KH*KW]
auto perm2 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, std::vector<int64_t>{0, 1, 2, 4, 3});
auto t2 = std::make_shared<ov::op::v1::Transpose>(r1, perm2);
// flatten back to [N, OH, OW, IC*KH*KW]
auto r2_shape = ov::op::v0::Constant::create(
ov::element::i64, ov::Shape{4},
std::vector<int64_t>{static_cast<int64_t>(N), static_cast<int64_t>(OH), static_cast<int64_t>(OW),
static_cast<int64_t>(IC * KH * KW)});
res = std::make_shared<ov::op::v1::Reshape>(t2, r2_shape, false);
if (!is_2D) {
// [N, 1, OW, IC * KW] -> [1, N, OW, IC * KW]
auto final_reshape_shape = ov::op::v0::Constant::create(
ov::element::i64, ov::Shape{4},
std::vector<int64_t>{1, static_cast<int64_t>(N), static_cast<int64_t>(OW), static_cast<int64_t>(IC * KW)});
res = std::make_shared<ov::op::v1::Reshape>(res, final_reshape_shape, false);
}
auto output_type = context.get_output_type();
if (res.get_element_type() != output_type) {
res = std::make_shared<ov::op::v0::Convert>(res, output_type);
}
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+44
vendor/llama.cpp/g...c/ggml-openvino/openvino/op/l2_norm.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include <memory>
#include <openvino/op/constant.hpp>
#include <openvino/op/divide.hpp>
#include <openvino/op/maximum.hpp>
#include <openvino/op/multiply.hpp>
#include <openvino/op/reduce_sum.hpp>
#include <openvino/op/sqrt.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_l2_norm(const NodeContext & context) {
num_inputs_check(context, 1, 1);
auto input_node = process_view_input_new(context, 0);
auto squared = std::make_shared<ov::op::v1::Multiply>(input_node, input_node);
auto sum_squared = std::make_shared<ov::op::v1::ReduceSum>(
squared, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true);
auto l2_norm = std::make_shared<ov::op::v0::Sqrt>(sum_squared);
float eps;
memcpy(&eps, context.get_output_op_params(), sizeof(float));
auto eps_const = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps});
auto clamped_norm = std::make_shared<ov::op::v1::Maximum>(l2_norm, eps_const);
auto res = std::make_shared<ov::op::v1::Divide>(input_node, clamped_norm);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+108
vendor/llama.cpp/g...gml-openvino/openvino/op/mul_mat_id.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include <memory>
#include <openvino/op/broadcast.hpp>
#include <openvino/op/concat.hpp>
#include <openvino/op/constant.hpp>
#include <openvino/op/convert.hpp>
#include <openvino/op/gather.hpp>
#include <openvino/op/matmul.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/shape_of.hpp>
#include <openvino/op/squeeze.hpp>
#include <openvino/op/unsqueeze.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_mul_mat_id(const NodeContext & context) {
num_inputs_check(context, 3, 3);
auto expert_weights = process_view_input_new(context, 0);
auto activations = process_view_input_new(context, 1);
auto ids = process_view_input_new(context, 2);
// OpenVINO sees GGML tensors in reversed dimension order:
// weights: [1, n_expert, m, k]
// activations: [1, n_tokens, n_used_or_1, k]
// ids: [1, 1, n_tokens, n_used]
// Rebuild the logical ranks explicitly from the 4D inputs instead of relying
// on fixed squeeze axes: real graphs can arrive through VIEW/RESHAPE chains
// where singleton axes are still represented differently at this point.
auto expert_weights_shape_4d = std::make_shared<ov::op::v3::ShapeOf>(expert_weights, ov::element::i64);
auto activations_shape_4d = std::make_shared<ov::op::v3::ShapeOf>(activations, ov::element::i64);
auto ids_shape_4d = std::make_shared<ov::op::v3::ShapeOf>(ids, ov::element::i64);
auto expert_weights_shape_3d = get_dimensions(expert_weights_shape_4d, {1, 2, 3});
auto activations_shape_3d = get_dimensions(activations_shape_4d, {1, 2, 3});
auto ids_shape_2d = get_dimensions(ids_shape_4d, {2, 3});
expert_weights = std::make_shared<ov::op::v1::Reshape>(expert_weights, expert_weights_shape_3d, false);
activations = std::make_shared<ov::op::v1::Reshape>(activations, activations_shape_3d, false);
ids = std::make_shared<ov::op::v1::Reshape>(ids, ids_shape_2d, false);
if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) {
ids = std::make_shared<ov::op::v0::Convert>(ids, ov::element::i32);
}
auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});
ov::Output<ov::Node> selected_weights = std::make_shared<ov::op::v8::Gather>(expert_weights, ids, gather_axis);
const auto output_type = context.get_output_type();
if (selected_weights.get_element_type() != ov::element::f32) {
selected_weights = std::make_shared<ov::op::v0::Convert>(selected_weights, ov::element::f32);
}
if (activations.get_element_type() != ov::element::f32) {
activations = std::make_shared<ov::op::v0::Convert>(activations, ov::element::f32);
}
auto activations_shape = std::make_shared<ov::op::v3::ShapeOf>(activations, ov::element::i64);
auto ids_shape = std::make_shared<ov::op::v3::ShapeOf>(ids, ov::element::i64);
ov::Output<ov::Node> acts_target_dims = std::make_shared<ov::op::v0::Concat>(
ov::OutputVector{
get_dimensions(activations_shape, {0}),
get_dimensions(ids_shape, {1}),
get_dimensions(activations_shape, {2}),
},
0);
ov::Output<ov::Node> acts_broadcasted =
std::make_shared<ov::op::v3::Broadcast>(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL);
auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
auto activations_expanded = std::make_shared<ov::op::v0::Unsqueeze>(acts_broadcasted, unsqueeze_axes);
auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto output_shape = context.get_output_shape();
FRONT_END_OP_CONVERSION_CHECK(output_shape.rank().is_static() && output_shape.rank().get_length() == 4,
"Unexpected MUL_MAT_ID output rank");
FRONT_END_OP_CONVERSION_CHECK(output_shape[3].is_static(), "Expected static row dimension for MUL_MAT_ID output");
const auto row_dim_value = output_shape[3].get_length();
auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {row_dim_value});
ov::Output<ov::Node> result =
std::make_shared<ov::op::v0::MatMul>(activations_expanded, selected_weights, false, true);
auto result_target_dims = std::make_shared<ov::op::v0::Concat>(
ov::OutputVector{
batch_dim,
get_dimensions(ids_shape, {0, 1}),
row_dim,
},
0);
result = std::make_shared<ov::op::v1::Reshape>(result, result_target_dims, false);
if (result.get_element_type() != output_type) {
result = std::make_shared<ov::op::v0::Convert>(result, output_type);
}
return rename_outputs_with_suffix({result}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+58
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/op/norm.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include <memory>
#include <openvino/op/add.hpp>
#include <openvino/op/constant.hpp>
#include <openvino/op/divide.hpp>
#include <openvino/op/multiply.hpp>
#include <openvino/op/power.hpp>
#include <openvino/op/reduce_mean.hpp>
#include <openvino/op/sqrt.hpp>
#include <openvino/op/subtract.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_norm(const NodeContext & context) {
num_inputs_check(context, 1, 1);
auto input_node = process_view_input_new(context, 0);
// Step 1: Calculate mean along the last dimension
// mean = reduce_mean(input, axis=-1, keepdims=true)
auto mean = std::make_shared<ov::op::v1::ReduceMean>(
input_node, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true);
// Step 2: Calculate (input - mean)
auto centered = std::make_shared<ov::op::v1::Subtract>(input_node, mean);
// Step 3: Calculate squared differences (input - mean)^2
auto squared = std::make_shared<ov::op::v1::Power>(
centered, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {2.0f}));
// Step 4: Calculate variance = mean((input - mean)^2)
auto variance = std::make_shared<ov::op::v1::ReduceMean>(
squared, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true);
// Step 5: Get epsilon from op_params
float eps;
memcpy(&eps, context.get_output_op_params(), sizeof(float));
// Step 6: Calculate std = sqrt(variance + eps)
auto std_dev = std::make_shared<ov::op::v0::Sqrt>(std::make_shared<ov::op::v1::Add>(
variance, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps})));
// Step 7: Normalize: output = (input - mean) / std
auto res = std::make_shared<ov::op::v1::Divide>(centered, std_dev);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+95
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/op/pad.cpp
#include "../op_table.h"
#include "../utils.h"
#include <array>
#include <cstdint>
#include <openvino/op/constant.hpp>
#include <openvino/op/gather.hpp>
#include <openvino/op/pad.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/shape_of.hpp>
#include <vector>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
namespace {
ov::Output<ov::Node> translate_circular_pad(ov::Output<ov::Node> input,
const std::array<int32_t, 8> & pads,
const ov::Shape & input_shape) {
ov::Output<ov::Node> result = input;
const std::array<int32_t, 4> pads_begin = {pads[6], pads[4], pads[2], pads[0]};
const std::array<int32_t, 4> pads_end = {pads[7], pads[5], pads[3], pads[1]};
for (size_t axis = 0; axis < input_shape.size(); ++axis) {
const int64_t input_dim = static_cast<int64_t>(input_shape[axis]);
const int64_t pad_begin = pads_begin[axis];
const int64_t pad_end = pads_end[axis];
if (pad_begin == 0 && pad_end == 0) {
continue;
}
FRONT_END_CHECK_IMPLEMENTED(input_dim > 0, "Circular PAD requires static non-zero input dimensions");
std::vector<int64_t> indices(static_cast<size_t>(input_dim + pad_begin + pad_end));
for (int64_t index = 0; index < static_cast<int64_t>(indices.size()); ++index) {
int64_t wrapped = (index - pad_begin) % input_dim;
if (wrapped < 0) {
wrapped += input_dim;
}
indices[static_cast<size_t>(index)] = wrapped;
}
auto gather_indices = ov::op::v0::Constant::create(ov::element::i64, {indices.size()}, indices);
auto gather_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {axis});
result = std::make_shared<ov::op::v8::Gather>(result, gather_indices, gather_axis);
}
return result;
}
} // namespace
OutputVector translate_pad(const NodeContext & context) {
num_inputs_check(context, 1, 1);
auto input = process_view_input_new(context, 0);
if (context.get_input_shape(0) == context.get_output_shape()) {
auto input_shape = std::make_shared<ov::op::v3::ShapeOf>(input);
auto res = std::make_shared<ov::op::v1::Reshape>(input, input_shape, false);
return rename_outputs_with_suffix({res}, context.get_name());
}
const int32_t * op_params = context.get_output_op_params();
FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "PAD requires output op params");
const std::array<int32_t, 8> pads = {op_params[0], op_params[1], op_params[2], op_params[3],
op_params[4], op_params[5], op_params[6], op_params[7]};
const bool circular = op_params[8] != 0;
if (circular) {
auto res = translate_circular_pad(input, pads, context.get_input_shape(0).to_shape());
return rename_outputs_with_suffix({res}, context.get_name());
}
const std::vector<int64_t> pads_begin = {pads[6], pads[4], pads[2], pads[0]};
const std::vector<int64_t> pads_end = {pads[7], pads[5], pads[3], pads[1]};
auto pads_begin_node = ov::op::v0::Constant::create(ov::element::i64, {pads_begin.size()}, pads_begin);
auto pads_end_node = ov::op::v0::Constant::create(ov::element::i64, {pads_end.size()}, pads_end);
auto pad_value = ov::op::v0::Constant::create(context.get_input_type(0), ov::Shape{}, {0});
auto res =
std::make_shared<ov::op::v1::Pad>(input, pads_begin_node, pads_end_node, pad_value, ov::op::PadMode::CONSTANT);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+74
vendor/llama.cpp/g...rc/ggml-openvino/openvino/op/repeat.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include "ggml.h"
#include <memory>
#include <openvino/op/broadcast.hpp>
#include <openvino/op/constant.hpp>
#include <openvino/op/divide.hpp>
#include <openvino/op/shape_of.hpp>
#include <openvino/op/tile.hpp>
#include <vector>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
// GGML_OP_REPEAT tiles src[0] to fill the destination shape. Every destination
// dimension is an integer multiple of the corresponding source dimension.
OutputVector translate_repeat(const NodeContext & context) {
num_inputs_check(context, 1, 2);
auto input = process_view_input_new(context, 0);
const auto input_shape = context.get_input_shape(0);
const auto output_shape = context.get_output_shape();
if (input_shape.rank().is_static() && output_shape.rank().is_static() &&
input_shape.rank() == output_shape.rank()) {
const auto rank = static_cast<size_t>(input_shape.rank().get_length());
std::vector<int64_t> repeats(rank, 1);
bool all_static = true;
for (size_t axis = 0; axis < rank; ++axis) {
if (!input_shape[axis].is_static() || !output_shape[axis].is_static()) {
all_static = false;
break;
}
const int64_t input_dim = input_shape[axis].get_length();
const int64_t output_dim = output_shape[axis].get_length();
FRONT_END_OP_CONVERSION_CHECK(input_dim > 0 && output_dim > 0 && output_dim % input_dim == 0,
"REPEAT input shape ", input_shape, " cannot tile to match ", output_shape);
repeats[axis] = output_dim / input_dim;
}
if (all_static) {
auto repeats_node = ov::op::v0::Constant::create(ov::element::i64, {repeats.size()}, repeats);
ov::Output<ov::Node> res = std::make_shared<ov::op::v0::Tile>(input, repeats_node);
return rename_outputs_with_suffix({res}, context.get_name());
}
}
// Dynamic fallback: tile by the ratio of output to input shape.
auto input_shape_node = std::make_shared<ov::op::v3::ShapeOf>(input, ov::element::i64);
std::shared_ptr<ov::Node> target_shape_node;
if (output_shape.rank().is_static() && output_shape.is_static()) {
target_shape_node =
ov::op::v0::Constant::create(ov::element::i64, {output_shape.to_shape().size()}, output_shape.to_shape());
} else {
target_shape_node = std::make_shared<ov::op::v3::ShapeOf>(context.get_input(1), ov::element::i64);
}
auto repeats_node = std::make_shared<ov::op::v1::Divide>(target_shape_node, input_shape_node);
ov::Output<ov::Node> res = std::make_shared<ov::op::v0::Tile>(input, repeats_node);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+59
vendor/llama.cpp/g.../ggml-openvino/openvino/op/ssm_conv.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include <openvino/op/constant.hpp>
#include <openvino/op/group_conv.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/transpose.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_ssm_conv(const NodeContext & context) {
num_inputs_check(context, 2, 2);
auto sx = context.get_input(0); // conv state + input: OV shape [1, n_s, d_inner, ncs]
auto c = context.get_input(1); // conv1d weight: OV shape [1, 1, d_inner, d_conv]
auto sx_shape = context.get_input_shape(0).to_shape(); // [1, n_s, d_inner, ncs]
auto c_shape = context.get_input_shape(1).to_shape(); // [1, 1, d_inner, d_conv]
int64_t n_s = sx_shape[1];
int64_t d_inner = sx_shape[2];
int64_t ncs = sx_shape[3]; // d_conv - 1 + n_t
int64_t d_conv = c_shape[3];
int64_t n_t = ncs - d_conv + 1;
// Reshape sx from [1, n_s, d_inner, ncs] to [n_s, d_inner, ncs] for 1D GroupConvolution
auto sx_new_shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{n_s, d_inner, ncs});
auto sx_reshaped = std::make_shared<ov::op::v1::Reshape>(sx, sx_new_shape, false);
// Reshape c from [1, 1, d_inner, d_conv] to [d_inner, 1, 1, d_conv]
// GroupConvolution filter: [groups, out_channels/groups, in_channels/groups, kernel_size]
auto c_new_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{d_inner, 1, 1, d_conv});
auto c_reshaped = std::make_shared<ov::op::v1::Reshape>(c, c_new_shape, false);
// Depthwise 1D convolution: groups=d_inner, stride=1, no padding, no dilation
// Input: [n_s, d_inner, ncs], Filter: [d_inner, 1, 1, d_conv]
// Output: [n_s, d_inner, n_t]
auto conv = std::make_shared<ov::op::v1::GroupConvolution>(
sx_reshaped, c_reshaped, ov::Strides{1}, ov::CoordinateDiff{0}, ov::CoordinateDiff{0}, ov::Strides{1});
// Transpose from [n_s, d_inner, n_t] to [n_s, n_t, d_inner]
auto perm = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{0, 2, 1});
auto transposed = std::make_shared<ov::op::v1::Transpose>(conv, perm);
// Reshape to output shape [1, n_s, n_t, d_inner]
auto out_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, n_s, n_t, d_inner});
auto res = std::make_shared<ov::op::v1::Reshape>(transposed, out_shape, false);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+27
vendor/llama.cpp/g.../ggml-openvino/openvino/op/sum_rows.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include <memory>
#include <openvino/op/constant.hpp>
#include <openvino/op/reduce_sum.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_sum_rows(const NodeContext & context) {
num_inputs_check(context, 1, 1);
auto input = process_view_input_new(context, 0);
auto res = std::make_shared<ov::op::v1::ReduceSum>(
input, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+38
vendor/llama.cpp/g...openvino/openvino/op/unary_softplus.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include <openvino/op/abs.hpp>
#include <openvino/op/add.hpp>
#include <openvino/op/constant.hpp>
#include <openvino/op/exp.hpp>
#include <openvino/op/log.hpp>
#include <openvino/op/negative.hpp>
#include <openvino/op/relu.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_unary_softplus(const NodeContext & context) {
num_inputs_check(context, 1, 1);
auto input = process_view_input_new(context, 0);
const auto element_type = input.get_element_type();
auto one = ov::op::v0::Constant::create(element_type, ov::Shape{}, {1.0f});
auto positive = std::make_shared<ov::op::v0::Relu>(input);
auto abs = std::make_shared<ov::op::v0::Abs>(input);
auto neg_abs = std::make_shared<ov::op::v0::Negative>(abs);
auto exp_neg_abs = std::make_shared<ov::op::v0::Exp>(neg_abs);
auto log_term = std::make_shared<ov::op::v0::Log>(std::make_shared<ov::op::v1::Add>(one, exp_neg_abs));
auto res = std::make_shared<ov::op::v1::Add>(positive, log_term);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
+158
vendor/llama.cpp/ggml/src/ggml-sycl/conv2d-dw.cpp
#include "conv2d-dw.hpp"
struct conv2d_dw_params {
int in_w, in_h;
int out_w, out_h;
int kernel_w, kernel_h;
int stride_x, stride_y;
int padding_x, padding_y;
int dilation_x, dilation_y;
int channels, batches;
};
struct conv2d_dw_kernel_bounds {
int y_min, y_max;
int x_min, x_max;
};
static inline conv2d_dw_kernel_bounds dw_calculate_kernel_bounds(int out_x, int out_y,
const conv2d_dw_params & p) {
conv2d_dw_kernel_bounds bounds;
bounds.y_min = sycl::max(0, (p.padding_y - out_y * p.stride_y + p.dilation_y - 1) / p.dilation_y);
bounds.y_max = sycl::min(p.kernel_h,
(p.in_h + p.padding_y - out_y * p.stride_y + p.dilation_y - 1) / p.dilation_y);
bounds.x_min = sycl::max(0, (p.padding_x - out_x * p.stride_x + p.dilation_x - 1) / p.dilation_x);
bounds.x_max = sycl::min(p.kernel_w,
(p.in_w + p.padding_x - out_x * p.stride_x + p.dilation_x - 1) / p.dilation_x);
return bounds;
}
static inline int dw_calculate_input_coord(int out_coord, int kern_coord, int stride, int dilation, int padding) {
return out_coord * stride + kern_coord * dilation - padding;
}
// whcn layout: input/output stored as [N, C, H, W]
struct dw_whcn_layout {
static int input_index(int n, int c, int y, int x, const conv2d_dw_params & p) {
return n * (p.channels * p.in_w * p.in_h) + c * p.in_w * p.in_h + y * p.in_w + x;
}
static int kernel_index(int c, int ky, int kx, const conv2d_dw_params & p) {
return c * p.kernel_h * p.kernel_w + ky * p.kernel_w + kx;
}
static int output_index(int n, int c, int y, int x, const conv2d_dw_params & p) {
return n * (p.channels * p.out_w * p.out_h) + c * p.out_w * p.out_h + y * p.out_w + x;
}
static void unpack_indices(int global_idx, const conv2d_dw_params & p,
int & n, int & c, int & out_y, int & out_x) {
out_x = global_idx % p.out_w;
out_y = (global_idx / p.out_w) % p.out_h;
c = (global_idx / (p.out_w * p.out_h)) % p.channels;
n = global_idx / (p.out_w * p.out_h * p.channels);
}
};
// cwhn layout: input/output stored as [N, H, W, C]
struct dw_cwhn_layout {
static int input_index(int n, int c, int y, int x, const conv2d_dw_params & p) {
return n * (p.channels * p.in_w * p.in_h) + (y * p.in_w + x) * p.channels + c;
}
static int kernel_index(int c, int ky, int kx, const conv2d_dw_params & p) {
return (ky * p.kernel_w + kx) * p.channels + c;
}
static int output_index(int n, int c, int y, int x, const conv2d_dw_params & p) {
return n * (p.channels * p.out_w * p.out_h) + y * (p.out_w * p.channels) + x * p.channels + c;
}
static void unpack_indices(int global_idx, const conv2d_dw_params & p,
int & n, int & c, int & out_y, int & out_x) {
c = global_idx % p.channels;
out_x = (global_idx / p.channels) % p.out_w;
out_y = (global_idx / (p.channels * p.out_w)) % p.out_h;
n = global_idx / (p.channels * p.out_w * p.out_h);
}
};
template <typename Layout>
static void conv2d_dw_kernel(const float * input, const float * kernel, float * output,
const conv2d_dw_params p, const sycl::nd_item<3> & item_ct1) {
const int global_idx = item_ct1.get_local_id(2) +
item_ct1.get_group(2) * item_ct1.get_local_range(2);
const int total_elements = p.batches * p.channels * p.out_h * p.out_w;
if (global_idx >= total_elements) {
return;
}
int n, c, out_y, out_x;
Layout::unpack_indices(global_idx, p, n, c, out_y, out_x);
float acc = 0.0f;
const conv2d_dw_kernel_bounds bounds = dw_calculate_kernel_bounds(out_x, out_y, p);
for (int ky = bounds.y_min; ky < bounds.y_max; ++ky) {
const int in_y = dw_calculate_input_coord(out_y, ky, p.stride_y, p.dilation_y, p.padding_y);
for (int kx = bounds.x_min; kx < bounds.x_max; ++kx) {
const int in_x = dw_calculate_input_coord(out_x, kx, p.stride_x, p.dilation_x, p.padding_x);
acc += input[Layout::input_index(n, c, in_y, in_x, p)] *
kernel[Layout::kernel_index(c, ky, kx, p)];
}
}
output[Layout::output_index(n, c, out_y, out_x, p)] = acc;
}
template <typename Layout>
static void conv2d_dw_sycl(const float * x_d, const float * w_d, float * y_d,
const conv2d_dw_params p, const queue_ptr & stream) {
const int total = p.batches * p.channels * p.out_h * p.out_w;
const int num_blocks = (total + SYCL_CONV2D_DW_BLOCK_SIZE - 1) / SYCL_CONV2D_DW_BLOCK_SIZE;
const sycl::range<3> block_dims(1, 1, SYCL_CONV2D_DW_BLOCK_SIZE);
const sycl::range<3> block_nums(1, 1, num_blocks);
stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
[=](sycl::nd_item<3> item_ct1) {
conv2d_dw_kernel<Layout>(x_d, w_d, y_d, p, item_ct1);
});
}
void ggml_sycl_op_conv2d_dw(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
const ggml_tensor * kernel = dst->src[0];
const ggml_tensor * input = dst->src[1];
GGML_ASSERT(kernel->type == GGML_TYPE_F32 && input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32);
const float * w_d = (const float *) kernel->data;
const float * x_d = (const float *) input->data;
float * y_d = (float *) dst->data;
const int32_t * p = (const int32_t *) dst->op_params;
const int stride_x = p[0];
const int stride_y = p[1];
const int padding_x = p[2];
const int padding_y = p[3];
const int dilation_x = p[4];
const int dilation_y = p[5];
const int in_w = input->ne[0];
const int in_h = input->ne[1];
const int kernel_w = kernel->ne[0];
const int kernel_h = kernel->ne[1];
const int out_w = dst->ne[0];
const int out_h = dst->ne[1];
const int channels = dst->ne[2];
const int batches = dst->ne[3];
const conv2d_dw_params params = { in_w, in_h, out_w, out_h, kernel_w, kernel_h,
stride_x, stride_y, padding_x, padding_y,
dilation_x, dilation_y, channels, batches };
const queue_ptr stream = ctx.stream();
if (ggml_is_contiguous(input)) {
conv2d_dw_sycl<dw_whcn_layout>(x_d, w_d, y_d, params, stream);
} else if (ggml_is_contiguous_channels(input)) {
conv2d_dw_sycl<dw_cwhn_layout>(x_d, w_d, y_d, params, stream);
} else {
GGML_ABORT("Unsupported memory layout for conv2d_dw");
}
}
+10
vendor/llama.cpp/ggml/src/ggml-sycl/conv2d-dw.hpp
#ifndef GGML_SYCL_CONV2D_DW_HPP
#define GGML_SYCL_CONV2D_DW_HPP
#include "common.hpp"
#define SYCL_CONV2D_DW_BLOCK_SIZE 256
void ggml_sycl_op_conv2d_dw(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
#endif // GGML_SYCL_CONV2D_DW_HPP
+125
vendor/llama.cpp/ggml/src/ggml-sycl/conv2d-transpose.cpp
#include "conv2d-transpose.hpp"
#include "convert.hpp"
template <typename kernel_t>
static void conv2d_transpose_kernel(const float * input, const kernel_t * kernel, float * output,
const int in_w, const int in_h,
const int out_w, const int out_h,
const int kernel_w, const int kernel_h,
const int stride,
const int c_in, const int c_out, const int batches,
const sycl::nd_item<3> & item_ct1) {
const int global_idx = item_ct1.get_local_id(2) +
item_ct1.get_group(2) * item_ct1.get_local_range(2);
const int total_elements = out_w * out_h * c_out * batches;
if (global_idx >= total_elements) {
return;
}
const int out_x = global_idx % out_w;
const int out_y = (global_idx / out_w) % out_h;
const int c_idx = (global_idx / (out_w * out_h)) % c_out;
const int n_idx = global_idx / (out_w * out_h * c_out);
float acc = 0.0f;
for (int c_in_idx = 0; c_in_idx < c_in; ++c_in_idx) {
for (int kh = 0; kh < kernel_h; ++kh) {
int in_y = out_y - kh;
if (in_y < 0 || in_y % stride) {
continue;
}
in_y /= stride;
if (in_y >= in_h) {
continue;
}
for (int kw = 0; kw < kernel_w; ++kw) {
int in_x = out_x - kw;
if (in_x < 0 || in_x % stride) {
continue;
}
in_x /= stride;
if (in_x >= in_w) {
continue;
}
const int input_idx = (in_w * in_h * c_in) * n_idx + (in_w * in_h) * c_in_idx + in_w * in_y + in_x;
const int kernel_idx = (kernel_h * kernel_w * c_out) * c_in_idx + (kernel_h * kernel_w) * c_idx +
kernel_w * kh + kw;
acc += input[input_idx] * ggml_sycl_cast<float>(kernel[kernel_idx]);
}
}
}
output[(out_w * out_h * c_out) * n_idx + (out_w * out_h) * c_idx + out_w * out_y + out_x] = acc;
}
template <typename kernel_t>
static void conv2d_transpose_sycl(const float * input_d, const kernel_t * kernel_d, float * output_d,
const int in_w, const int in_h,
const int out_w, const int out_h,
const int kernel_w, const int kernel_h,
const int stride,
const int c_in, const int c_out, const int batches,
const queue_ptr & stream) {
const int total = out_w * out_h * c_out * batches;
const int num_blocks = (total + SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE - 1) / SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE;
const sycl::range<3> block_dims(1, 1, SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE);
const sycl::range<3> block_nums(1, 1, num_blocks);
stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
[=](sycl::nd_item<3> item_ct1) {
conv2d_transpose_kernel<kernel_t>(input_d, kernel_d, output_d,
in_w, in_h, out_w, out_h, kernel_w, kernel_h,
stride, c_in, c_out, batches, item_ct1);
});
}
// input: (W, H, C_in, N)
// kernel: (W, H, C_out, C_in)
// output: (W, H, C_out, N)
void ggml_sycl_op_conv2d_transpose(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
const ggml_tensor * kernel = dst->src[0];
const ggml_tensor * input = dst->src[1];
GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32);
GGML_ASSERT(input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32);
GGML_ASSERT(ggml_is_contiguous(input));
GGML_ASSERT(ggml_is_contiguous(kernel));
GGML_ASSERT(ggml_is_contiguous(dst));
const float * input_d = (const float *) input->data;
float * output_d = (float *) dst->data;
const void * kernel_d = kernel->data;
const int input_w = input->ne[0];
const int input_h = input->ne[1];
const int channels_in = input->ne[2];
const int batches = input->ne[3];
const int output_w = dst->ne[0];
const int output_h = dst->ne[1];
const int channels_out = kernel->ne[2];
const int kernel_w = kernel->ne[0];
const int kernel_h = kernel->ne[1];
const int stride = dst->op_params[0];
GGML_ASSERT(channels_in == kernel->ne[3]);
GGML_ASSERT(stride > 0);
const queue_ptr stream = ctx.stream();
if (kernel->type == GGML_TYPE_F16) {
conv2d_transpose_sycl<sycl::half>(input_d, (const sycl::half *) kernel_d, output_d,
input_w, input_h, output_w, output_h, kernel_w, kernel_h,
stride, channels_in, channels_out, batches, stream);
} else {
conv2d_transpose_sycl<float>(input_d, (const float *) kernel_d, output_d,
input_w, input_h, output_w, output_h, kernel_w, kernel_h,
stride, channels_in, channels_out, batches, stream);
}
}
+10
vendor/llama.cpp/ggml/src/ggml-sycl/conv2d-transpose.hpp
#ifndef GGML_SYCL_CONV2D_TRANSPOSE_HPP
#define GGML_SYCL_CONV2D_TRANSPOSE_HPP
#include "common.hpp"
#define SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE 256
void ggml_sycl_op_conv2d_transpose(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
#endif // GGML_SYCL_CONV2D_TRANSPOSE_HPP
+150
vendor/llama.cpp/ggml/src/ggml-sycl/conv2d.cpp
#include "conv2d.hpp"
#include "convert.hpp"
struct conv2d_params {
const int64_t IW, IH;
const int64_t OW, OH;
const int64_t KW, KH;
const int64_t ST_X, ST_Y;
const int64_t PD_X, PD_Y;
const int64_t DL_X, DL_Y;
const int64_t IC, OC;
const int64_t B;
const int64_t TOTAL;
};
struct conv2d_kernel_bounds {
int64_t y_min, y_max;
int64_t x_min, x_max;
};
static inline int64_t conv2d_max64(int64_t a, int64_t b) {
return (a > b) ? a : b;
}
static inline int64_t conv2d_min64(int64_t a, int64_t b) {
return (a < b) ? a : b;
}
static inline conv2d_kernel_bounds calculate_kernel_bounds(int64_t out_x, int64_t out_y, const conv2d_params & P) {
conv2d_kernel_bounds bounds;
bounds.y_min = conv2d_max64(0, (P.PD_Y - out_y * P.ST_Y + P.DL_Y - 1) / P.DL_Y);
bounds.y_max = conv2d_min64(P.KH, (P.IH + P.PD_Y - out_y * P.ST_Y + P.DL_Y - 1) / P.DL_Y);
bounds.x_min = conv2d_max64(0, (P.PD_X - out_x * P.ST_X + P.DL_X - 1) / P.DL_X);
bounds.x_max = conv2d_min64(P.KW, (P.IW + P.PD_X - out_x * P.ST_X + P.DL_X - 1) / P.DL_X);
return bounds;
}
static inline int calculate_input_coord(int64_t out_coord, int64_t kern_coord, int64_t stride,
int64_t dilation, int64_t padding) {
return out_coord * stride + kern_coord * dilation - padding;
}
// whcn layout helpers (matching ggml tensor memory order)
static inline int64_t whcn_input_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv2d_params & P) {
return n * (P.IC * P.IW * P.IH) + c * P.IW * P.IH + y * P.IW + x;
}
static inline int64_t whcn_kernel_index(int64_t c_out, int64_t c_in, int64_t ky, int64_t kx, const conv2d_params & P) {
return c_out * (P.IC * P.KH * P.KW) + c_in * (P.KH * P.KW) + ky * P.KW + kx;
}
static inline int64_t whcn_output_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv2d_params & P) {
return n * (P.OC * P.OW * P.OH) + c * P.OW * P.OH + y * P.OW + x;
}
template <typename T>
static void conv2d_kernel(const float * input, const T * kernel, float * output,
const conv2d_params P, const sycl::nd_item<3> & item_ct1) {
const int64_t global_idx = item_ct1.get_local_id(2) +
item_ct1.get_group(2) * item_ct1.get_local_range(2);
if (global_idx >= P.TOTAL) {
return;
}
const int64_t out_x = global_idx % P.OW;
const int64_t out_y = (global_idx / P.OW) % P.OH;
const int64_t c_out = (global_idx / (P.OW * P.OH)) % P.OC;
const int64_t n = global_idx / (P.OW * P.OH * P.OC);
float acc = 0.0f;
const conv2d_kernel_bounds bounds = calculate_kernel_bounds(out_x, out_y, P);
for (int64_t c_in = 0; c_in < P.IC; ++c_in) {
for (int64_t ky = bounds.y_min; ky < bounds.y_max; ++ky) {
const int64_t in_y = calculate_input_coord(out_y, ky, P.ST_Y, P.DL_Y, P.PD_Y);
for (int64_t kx = bounds.x_min; kx < bounds.x_max; ++kx) {
const int64_t in_x = calculate_input_coord(out_x, kx, P.ST_X, P.DL_X, P.PD_X);
const float input_val = input[whcn_input_index(n, c_in, in_y, in_x, P)];
const T kernel_val = kernel[whcn_kernel_index(c_out, c_in, ky, kx, P)];
acc += input_val * ggml_sycl_cast<float>(kernel_val);
}
}
}
output[whcn_output_index(n, c_out, out_y, out_x, P)] = acc;
}
template <typename T>
static void conv2d_sycl(const float * X_D, const T * K_D, float * Y_D,
const conv2d_params P, const queue_ptr & stream) {
const int num_blocks = (P.TOTAL + SYCL_CONV2D_BLOCK_SIZE - 1) / SYCL_CONV2D_BLOCK_SIZE;
const sycl::range<3> block_dims(1, 1, SYCL_CONV2D_BLOCK_SIZE);
const sycl::range<3> block_nums(1, 1, num_blocks);
stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
[=](sycl::nd_item<3> item_ct1) {
conv2d_kernel<T>(X_D, K_D, Y_D, P, item_ct1);
});
}
void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
const ggml_tensor * kernel = dst->src[0];
const ggml_tensor * input = dst->src[1];
const float * K_D = (const float *) kernel->data;
const float * X_D = (const float *) input->data;
float * Y_D = (float *) dst->data;
GGML_ASSERT(ggml_is_contiguous(kernel));
GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32);
GGML_ASSERT(input->type == GGML_TYPE_F32);
GGML_ASSERT(dst->type == GGML_TYPE_F32);
// same number of input channels
GGML_ASSERT(input->ne[2] == kernel->ne[2]);
const queue_ptr stream = ctx.stream();
const int32_t * p = (const int32_t *) dst->op_params;
const int ST_X = p[0];
const int ST_Y = p[1];
const int PD_X = p[2];
const int PD_Y = p[3];
const int DL_X = p[4];
const int DL_Y = p[5];
// no cwhn layout support
GGML_ASSERT(p[6] == 0);
const int IW = input->ne[0];
const int IH = input->ne[1];
const int OW = dst->ne[0];
const int OH = dst->ne[1];
const int KW = kernel->ne[0];
const int KH = kernel->ne[1];
const int IC = input->ne[2];
const int OC = kernel->ne[3];
const int B = input->ne[3];
const int64_t total = (int64_t) B * OC * OH * OW;
const conv2d_params params = { IW, IH, OW, OH, KW, KH, ST_X, ST_Y, PD_X, PD_Y, DL_X, DL_Y, IC, OC, B, total };
if (kernel->type == GGML_TYPE_F16) {
conv2d_sycl<sycl::half>(X_D, (const sycl::half *) K_D, Y_D, params, stream);
} else {
conv2d_sycl<float>(X_D, K_D, Y_D, params, stream);
}
}
+10
vendor/llama.cpp/ggml/src/ggml-sycl/conv2d.hpp
#ifndef GGML_SYCL_CONV2D_HPP
#define GGML_SYCL_CONV2D_HPP
#include "common.hpp"
#define SYCL_CONV2D_BLOCK_SIZE 256
void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
#endif // GGML_SYCL_CONV2D_HPP
+218
vendor/llama.cpp/ggml/src/ggml-sycl/conv3d.cpp
#include "conv3d.hpp"
static inline int64_t ggml_sycl_conv3d_calc_patch_total(const ggml_tensor * dst, int32_t n) {
return (int64_t) n * dst->ne[0] * dst->ne[1] * dst->ne[2];
}
static inline int64_t ggml_sycl_conv3d_calc_knl_n_total(const ggml_tensor * src0, int32_t c) {
return (int64_t) src0->ne[0] * src0->ne[1] * src0->ne[2] * c;
}
static inline void ggml_sycl_conv3d_write_output(
const ggml_tensor * dst,
const float * src, float * dst_data,
int64_t patch_total, int64_t oc,
int64_t dst_w, int64_t dst_h, int64_t dst_d,
dpct::queue_ptr stream) {
const int64_t dst_nb0 = dst->nb[0];
const int64_t dst_nb1 = dst->nb[1];
const int64_t dst_nb2 = dst->nb[2];
const int64_t dst_nb3 = dst->nb[3];
const int64_t total = patch_total * oc;
const int64_t block_size = 256;
const int64_t num_work_items = ((total + block_size - 1) / block_size) * block_size;
stream->parallel_for(sycl::range<1>(num_work_items), [=](sycl::id<1> id) {
const int64_t i = id[0];
if (i >= total) {
return;
}
const int64_t patch_idx = i / oc;
const int64_t out_ch = i % oc;
const int64_t p_in_batch = patch_idx % (dst_w * dst_h * dst_d);
const int64_t batch_idx = patch_idx / (dst_w * dst_h * dst_d);
const int64_t dst_z = p_in_batch / (dst_w * dst_h);
const int64_t dst_y = (p_in_batch % (dst_w * dst_h)) / dst_w;
const int64_t dst_x = p_in_batch % dst_w;
const int64_t ocn_idx = batch_idx * oc + out_ch;
const int64_t dst_offset = dst_x * dst_nb0 + dst_y * dst_nb1 + dst_z * dst_nb2 + ocn_idx * dst_nb3;
// `src` is a column-major (m x n) GEMM output where m == patch_total, n == oc.
// GEMM stores element (row, col) at index `row + col*m`, so compute index accordingly.
const int64_t src_index = patch_idx + out_ch * patch_total;
const float value = src[src_index];
*(float *)((char *)dst_data + dst_offset) = value;
});
}
void ggml_sycl_op_conv_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32);
GGML_ASSERT(src1->type == GGML_TYPE_F32);
GGML_ASSERT(dst->type == GGML_TYPE_F32);
GGML_ASSERT(ggml_is_contiguous(src0));
GGML_ASSERT(ggml_is_contiguous(src1));
const int32_t * opts = (const int32_t *) dst->op_params;
const int32_t s0 = opts[0];
const int32_t s1 = opts[1];
const int32_t s2 = opts[2];
const int32_t p0 = opts[3];
const int32_t p1 = opts[4];
const int32_t p2 = opts[5];
const int32_t d0 = opts[6];
const int32_t d1 = opts[7];
const int32_t d2 = opts[8];
const int32_t c = opts[9];
const int32_t n = opts[10];
const int32_t oc = opts[11];
const int64_t knl_w = src0->ne[0];
const int64_t knl_h = src0->ne[1];
const int64_t knl_d = src0->ne[2];
const int64_t patch_total = ggml_sycl_conv3d_calc_patch_total(dst, n);
const int64_t knl_n_total = ggml_sycl_conv3d_calc_knl_n_total(src0, c);
const size_t kernel_type_size = ggml_element_size(src0);
ggml_sycl_pool_alloc<float> gemm_output(ctx.pool());
gemm_output.alloc((size_t) patch_total * oc);
ggml_tensor dst_mat = {};
dst_mat.type = GGML_TYPE_F32;
dst_mat.ne[0] = patch_total;
dst_mat.ne[1] = oc;
dst_mat.ne[2] = 1;
dst_mat.ne[3] = 1;
dst_mat.nb[0] = sizeof(float);
dst_mat.nb[1] = dst_mat.nb[0] * dst_mat.ne[0];
dst_mat.nb[2] = dst_mat.nb[1];
dst_mat.nb[3] = dst_mat.nb[2];
dst_mat.data = gemm_output.get();
dst_mat.buffer = dst->buffer;
dst_mat.extra = dst->extra;
dpct::queue_ptr stream = ctx.stream();
// allocate packed arrays: A_packed (k x m), B_packed (k x n)
ggml_sycl_pool_alloc<float> A_packed_alloc(ctx.pool());
ggml_sycl_pool_alloc<float> B_packed_alloc(ctx.pool());
A_packed_alloc.alloc((size_t) knl_n_total * patch_total * sizeof(float));
B_packed_alloc.alloc((size_t) knl_n_total * oc * sizeof(float));
float * A_packed = A_packed_alloc.get();
float * B_packed = B_packed_alloc.get();
const int m = (int) patch_total;
const int n_gemm = (int) oc;
const int k = (int) knl_n_total;
// Combined kernel: im2col -> pack A, and pack B simultaneously
const char * src1_base = (const char *) src1->data;
const int64_t src1_nb0 = src1->nb[0];
const int64_t src1_nb1 = src1->nb[1];
const int64_t src1_nb2 = src1->nb[2];
const int64_t src1_nb3 = src1->nb[3];
// Compute correct strides for src0 as (knl_n_total, oc) matrix
const int64_t src0_packed_nb0 = kernel_type_size;
const int64_t src0_packed_nb1 = kernel_type_size * knl_n_total;
const int64_t KW = knl_w;
const int64_t KH = knl_h;
const int64_t KD = knl_d;
const int64_t PW = dst->ne[0];
const int64_t PH = dst->ne[1];
const int64_t PD = dst->ne[2];
// Pack A (with inline im2col): for each (row, col) in k x m matrix
const int64_t A_total = (int64_t)k * m;
const int64_t A_block_size = 256;
const int64_t A_num_work = ((A_total + A_block_size - 1) / A_block_size) * A_block_size;
stream->parallel_for(sycl::range<1>(A_num_work), [=](sycl::id<1> id) {
const int64_t t = id[0];
if (t >= A_total) return;
const int64_t row = t % k;
const int64_t col = t / k;
// Inline im2col for this element
const int64_t k_index = row;
const int64_t patch_idx = col;
const int64_t ic = k_index / (KD * KH * KW);
const int64_t rem = k_index - ic * (KD * KH * KW);
const int64_t kz = rem / (KH * KW);
const int64_t rem2 = rem - kz * (KH * KW);
const int64_t ky = rem2 / KW;
const int64_t kx = rem2 % KW;
const int64_t p_in_batch = patch_idx % (PW * PH * PD);
const int64_t batch_idx = patch_idx / (PW * PH * PD);
const int64_t dst_z = p_in_batch / (PW * PH);
const int64_t dst_y = (p_in_batch % (PW * PH)) / PW;
const int64_t dst_x = p_in_batch % PW;
const int64_t sx = dst_x * s0 + kx * d0 - p0;
const int64_t sy = dst_y * s1 + ky * d1 - p1;
const int64_t sz = dst_z * s2 + kz * d2 - p2;
float val = 0.0f;
if (sx >= 0 && sx < src1->ne[0] && sy >= 0 && sy < src1->ne[1] && sz >= 0 && sz < src1->ne[2]) {
const int64_t channel_idx = batch_idx * c + ic;
const char * ptr = src1_base + sx * src1_nb0 + sy * src1_nb1 + sz * src1_nb2 + channel_idx * src1_nb3;
val = *(const float *) ptr;
}
A_packed[row + col * (int64_t)k] = val;
});
// Pack B: for each (row, col) in k x n_gemm matrix
const int64_t B_total = (int64_t)k * n_gemm;
const int64_t B_block_size = 256;
const int64_t B_num_work = ((B_total + B_block_size - 1) / B_block_size) * B_block_size;
stream->parallel_for(sycl::range<1>(B_num_work), [=](sycl::id<1> id) {
const int64_t t = id[0];
if (t >= B_total) return;
const int64_t row = t % k;
const int64_t col = t / k;
const char * src_ptr = (const char *) src0->data + row * src0_packed_nb0 + col * src0_packed_nb1;
float v;
if (src0->type == GGML_TYPE_F32) {
v = *(const float *) src_ptr;
} else {
v = sycl::vec<sycl::half, 1>(*(const sycl::half *) src_ptr).convert<float, sycl::rounding_mode::automatic>()[0];
}
B_packed[row + col * (int64_t)k] = v;
});
// GEMM: C = A^T * B where A is (k x m), B is (k x n), C is (m x n)
const float alpha = 1.0f;
const float beta = 0.0f;
const int lda = k;
const int ldb = k;
const int ldc = m;
SYCL_CHECK(CHECK_TRY_ERROR(oneapi::mkl::blas::column_major::gemm(
*stream, oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans,
m, n_gemm, k,
dpct::get_value(&alpha, *stream),
(const float *) A_packed, lda,
(const float *) B_packed, ldb,
dpct::get_value(&beta, *stream),
(float *) dst_mat.data, ldc)));
const float * gemm_data = (const float *) dst_mat.data;
float * dst_data = (float *) dst->data;
ggml_sycl_conv3d_write_output(dst, gemm_data, dst_data, patch_total, oc,
dst->ne[0], dst->ne[1], dst->ne[2], stream);
}
+8
vendor/llama.cpp/ggml/src/ggml-sycl/conv3d.hpp
#ifndef GGML_SYCL_CONV3D_HPP
#define GGML_SYCL_CONV3D_HPP
#include "common.hpp"
void ggml_sycl_op_conv_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
#endif // GGML_SYCL_CONV3D_HPP
+463
vendor/llama.cpp/scripts/snapdragon/ggml-hexagon-trace.py
#!/usr/bin/env python3
import sys
import os
import re
import argparse
import statistics
import logging
from typing import Any, Dict, List, Optional
from collections import defaultdict
logger = logging.getLogger("ggml-hexagon-trace")
op_pattern = re.compile(
r"profile-op\s+(?P<op_name>[A-Z_0-9+]+):\s+.*?\s+:\s+(?P<dims>[\d:x\s\->!]+)\s+:\s+(?P<types>[a-z\d_\s\->x]+)\s+:\s+(?P<strides>[\d:x\s\->!]+)\s+:\s+(?:op-)?usec\s+(?P<usec>\d+)\s+(?:op-)?cycles\s+(?P<cycles>\d+)(?:\s+start\s+(?P<start>\d+))?(?:\s+mhz\s+(?P<mhz>[\d.]+))?(?:\s+pmu\s+\[(?P<pmu>[\d,\s]+)\])?(?:\s+evt\s+\[(?P<evt>[\d,\s]+)\])?"
)
trace_pattern = re.compile(
r"trace-op\s+(?P<op_name>[A-Z_0-9+]+):\s+thread\s+(?P<thread>\d+)\s+event\s+(?P<event>[A-Z_0-9\-]+)\s+info\s+(?P<info>\d+)\s+(?P<state>start|stop)\s+(?P<cycles>\d+)"
)
def normalize_event_name(evt_type):
if evt_type == "HVX_COMP":
return "V-COMP"
if evt_type == "HMX_COMP":
return "M-COMP"
name = evt_type
if name.startswith("HVX_") or name.startswith("HMX_"):
name = name[4:]
return name.replace("_", "-")
class CycleUnwrapper:
def __init__(self):
self.last_raw = None
self.high_part = 0
def unwrap(self, raw):
if self.last_raw is None:
self.last_raw = raw
return raw
diff = raw - self.last_raw
if diff < -0x80000000:
self.high_part += 0x100000000
elif diff > 0x80000000:
self.high_part -= 0x100000000
self.last_raw = raw
return raw + self.high_part
def parse_log(file_path):
try:
if file_path != "-":
f = open(file_path, 'r', encoding='utf-8', errors='ignore')
else:
f = os.fdopen(0, 'r', encoding='utf-8', errors='ignore')
except FileNotFoundError:
logger.error(f"file '{file_path}' not found.")
sys.exit(1)
all_ops: List[Dict[str, Any]] = []
current_op: Optional[Dict[str, Any]] = None
unwrapper = CycleUnwrapper()
line_idx = 0
for line in f:
line_idx += 1
op_match = op_pattern.search(line)
if op_match:
cycles_start_raw = op_match.group('start')
unwrapped_cycles_start = None
if cycles_start_raw:
unwrapped_cycles_start = unwrapper.unwrap(int(cycles_start_raw))
idx = line.find("profile-op ")
op_text = line[idx + 11:].strip() if idx != -1 else line.strip()
current_op = {
'name': op_match.group('op_name'),
'dims': op_match.group('dims').strip() if op_match.group('dims') else '',
'types': op_match.group('types').strip() if op_match.group('types') else '',
'strides': op_match.group('strides').strip() if op_match.group('strides') else '',
'op_text': op_text,
'usec': int(op_match.group('usec')),
'cycles': int(op_match.group('cycles')),
'cycles_start': int(cycles_start_raw) if cycles_start_raw else None,
'unwrapped_cycles_start': unwrapped_cycles_start,
'trace_events': [],
'line_num': line_idx
}
all_ops.append(current_op)
continue
trace_match = trace_pattern.search(line)
if trace_match and current_op:
if trace_match.group('op_name') == current_op['name']:
raw_cyc = int(trace_match.group('cycles'))
current_op['trace_events'].append({
'thread': int(trace_match.group('thread')),
'event': trace_match.group('event'),
'info': int(trace_match.group('info')),
'cycles': raw_cyc,
'unwrapped_cycles': unwrapper.unwrap(raw_cyc),
'state': trace_match.group('state')
})
f.close()
return all_ops
# --- Simple protobuf encoder ---
def write_varint(val):
if val < 0:
val = (1 << 64) + val
res = bytearray()
while True:
towrite = val & 0x7f
val >>= 7
if val > 0:
res.append(towrite | 0x80)
else:
res.append(towrite)
break
return bytes(res)
def pb_field(num, wire, data):
return write_varint((num << 3) | wire) + data
def pb_varint(num, val):
return pb_field(num, 0, write_varint(val))
def pb_length_delimited(num, data):
return pb_field(num, 2, write_varint(len(data)) + data)
def pb_string(num, text):
return pb_length_delimited(num, text.encode('utf-8'))
# Message Encoders
def make_process_descriptor(pid, name):
return pb_varint(1, pid) + pb_string(6, name)
def make_thread_descriptor(pid, tid, name, sort_index=None):
payload = pb_varint(1, pid) + pb_varint(2, tid) + pb_string(5, name)
if sort_index is not None:
payload += pb_varint(3, sort_index)
return payload
def make_track_descriptor(uuid, name=None, parent_uuid=None, thread=None, process=None, sibling_merge_behavior=None, child_ordering=None, sibling_order_rank=None):
payload = pb_varint(1, uuid)
if name is not None:
payload += pb_string(2, name)
if parent_uuid is not None:
payload += pb_varint(5, parent_uuid)
if process is not None:
payload += pb_length_delimited(3, process)
if thread is not None:
payload += pb_length_delimited(4, thread)
if sibling_merge_behavior is not None:
payload += pb_varint(15, sibling_merge_behavior)
if child_ordering is not None:
payload += pb_varint(11, child_ordering)
if sibling_order_rank is not None:
payload += pb_varint(12, sibling_order_rank)
return payload
def make_debug_annotation(name, string_val=None, int_val=None):
payload = pb_string(10, name)
if string_val is not None:
payload += pb_string(6, string_val)
elif int_val is not None:
payload += pb_varint(4, int_val)
return payload
def make_track_event(event_type, track_uuid, name=None, category=None, debug_annotations=None):
payload = pb_varint(9, event_type)
payload += pb_varint(11, track_uuid)
if name is not None:
payload += pb_string(23, name)
if category is not None:
payload += pb_string(22, category)
if debug_annotations is not None:
for da in debug_annotations:
payload += pb_length_delimited(4, da)
return payload
def make_trace_packet(timestamp, track_event=None, track_descriptor=None, seq_id=1):
payload = pb_varint(8, timestamp)
payload += pb_varint(10, seq_id)
if track_event is not None:
payload += pb_length_delimited(11, track_event)
if track_descriptor is not None:
payload += pb_length_delimited(60, track_descriptor)
return payload
def write_trace_packet_to_file(f, packet_bytes):
# Write as field 1 of top-level Trace message
f.write(pb_length_delimited(1, packet_bytes))
# --- End Protobuf Encoder ---
def generate_perfetto_trace(filtered_ops, output_path):
if not filtered_ops:
logger.warning("No operators found after filtering.")
return
# Compute average frequency
frequencies = []
for op in filtered_ops:
if op['usec'] > 0 and op['cycles'] > 0:
frequencies.append(op['cycles'] / op['usec'])
avg_freq_mhz = statistics.mean(frequencies) if frequencies else 1000.0
if avg_freq_mhz <= 0:
avg_freq_mhz = 1000.0
# Assign start and end cycles to each operator
for op in filtered_ops:
op['start_cycles'] = op['unwrapped_cycles_start']
op['end_cycles'] = op['start_cycles'] + op['cycles']
global_min_cyc = min(op['start_cycles'] for op in filtered_ops if op['start_cycles'] is not None)
# Process events
completed_events = []
for op in filtered_ops:
events = op['trace_events']
if not events:
continue
events = sorted(events, key=lambda e: e['unwrapped_cycles'])
active_starts = {}
for e in events:
t = e['thread']
evt = e['event']
info = e['info']
state = e['state']
cyc = e['unwrapped_cycles']
key = (t, evt, info)
if state == 'start':
active_starts[key] = cyc
elif state == 'stop':
if key in active_starts:
start_cyc = active_starts[key]
del active_starts[key]
completed_events.append({
'thread': t,
'event': evt,
'info': info,
'start_cyc': start_cyc,
'end_cyc': cyc,
'op_name': op['name']
})
completed_events.sort(key=lambda e: e['start_cyc'])
# Convert event times to microseconds and apply clamp rounded to 1ns resolution (3 decimals)
for e in completed_events:
start_us = (e['start_cyc'] - global_min_cyc) / avg_freq_mhz
dur_us = (e['end_cyc'] - e['start_cyc']) / avg_freq_mhz
e['ts_ns'] = int(round(start_us * 1000))
e['dur_ns'] = int(round(max(dur_us, 0.1) * 1000))
# Allocate slots (sub-tracks) to prevent overlaps on same virtual track
active_slots = defaultdict(list)
for e in completed_events:
t = e['thread']
evt = e['event']
ts = e['ts_ns']
dur = e['dur_ns']
norm_evt = normalize_event_name(evt)
if norm_evt == "DMA":
track_key = (t, "DMA")
elif t == 10:
track_key = (t, "HMX")
else:
track_key = (t, "HVX")
slots = active_slots[track_key]
allocated_slot = -1
for idx, slot_end_ns in enumerate(slots):
if ts >= slot_end_ns:
slots[idx] = ts + dur
allocated_slot = idx
break
if allocated_slot == -1:
slots.append(ts + dur)
allocated_slot = len(slots) - 1
e['slot'] = allocated_slot
# Generate Track IDs and track definitions
used_tracks = {}
for e in completed_events:
t = e['thread']
evt = e['event']
slot = e['slot']
norm_evt = normalize_event_name(evt)
if norm_evt == "DMA":
track_evt = "DMA"
evt_id = 1
elif t == 10:
track_evt = "HMX"
evt_id = 3
else:
track_evt = "HVX"
evt_id = 2
t_sort = 1 if t == 10 else t + 2
# Unique UUID for each sub-track
if t == 10:
uuid = 20 # HMX thread track UUID
else:
uuid = int(t_sort * 1000000 + evt_id * 1000 + slot)
e['uuid'] = uuid
used_tracks[uuid] = (t, track_evt, slot)
with open(output_path, "wb") as f:
# Define Process with EXPLICIT child sorting
proc_desc = make_process_descriptor(1, "HTP NPU")
proc_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(1, process=proc_desc, child_ordering=3))
write_trace_packet_to_file(f, proc_packet)
# Define Operators Track (UUID = 2) as a thread track at rank 1, tid 8
op_thread_desc = make_thread_descriptor(1, 8, "Ops", sort_index=1)
op_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(2, parent_uuid=1, thread=op_thread_desc))
write_trace_packet_to_file(f, op_packet)
# Define HMX Thread Track (UUID = 20) at rank 2, tid 9
hmx_thread_desc = make_thread_descriptor(1, 9, "HMX", sort_index=2)
hmx_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(20, parent_uuid=1, thread=hmx_thread_desc))
write_trace_packet_to_file(f, hmx_packet)
# Define Thread Tracks (T0, T1, ..., T9)
unique_threads = sorted(list(set(t for (t, _, _) in used_tracks.values() if t != 10)))
for t in unique_threads:
thread_uuid = 10 + t
thread_name = f"T{t}"
# Sort order starts from index 3 (T0 -> 3, T1 -> 4, etc.)
sort_index = 3 + t
tid = 10 + t
thread_desc = make_thread_descriptor(1, tid, thread_name, sort_index=sort_index)
thread_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(
thread_uuid,
parent_uuid=1,
thread=thread_desc,
sibling_order_rank=sort_index,
child_ordering=3 # Explicit child sorting for sub-tracks
))
write_trace_packet_to_file(f, thread_packet)
# Define Track descriptors for sub-tracks parented to thread tracks
for uuid in sorted(used_tracks.keys()):
if uuid == 20:
continue
t, evt, slot = used_tracks[uuid]
name = f"T{t} {evt}"
rank = 0 if evt == "HVX" else 1
parent_thread_uuid = 10 + t
# Sibling merge behavior: 1 (SIBLING_MERGE_BEHAVIOR_BY_TRACK_NAME)
track_desc = make_track_descriptor(
uuid=uuid,
name=name,
parent_uuid=parent_thread_uuid,
sibling_merge_behavior=1,
sibling_order_rank=rank
)
track_packet = make_trace_packet(0, track_descriptor=track_desc)
write_trace_packet_to_file(f, track_packet)
# Emit Operators
last_op_end_ns = 0
for op in filtered_ops:
op_start_ns = int(round(((op['start_cycles'] - global_min_cyc) / avg_freq_mhz) * 1000))
op_dur_ns = int(round((op['cycles'] / avg_freq_mhz) * 1000))
if op_start_ns < last_op_end_ns:
op_start_ns = last_op_end_ns
clamped_dur = max(op_dur_ns, 100) # Clamp to 100ns (0.1us)
# Debug annotations for Ops
debug_annots = []
if 'line_num' in op:
debug_annots.append(make_debug_annotation("line", int_val=op['line_num']))
if 'strides' in op and op['strides']:
debug_annots.append(make_debug_annotation("strides", string_val=op['strides']))
# Slice Begin
evt_begin = make_track_event(1, 2, name=f"{op['name']} ({op['dims']})", category="operator", debug_annotations=debug_annots)
packet_begin = make_trace_packet(op_start_ns, track_event=evt_begin)
write_trace_packet_to_file(f, packet_begin)
# Slice End
evt_end = make_track_event(2, 2)
packet_end = make_trace_packet(op_start_ns + clamped_dur, track_event=evt_end)
write_trace_packet_to_file(f, packet_end)
last_op_end_ns = op_start_ns + clamped_dur
# Emit Thread Trace Events
for e in completed_events:
norm_name = normalize_event_name(e['event'])
name = f"DMA {e['info']}" if norm_name == "DMA" else norm_name
# Slice Begin
evt_begin = make_track_event(1, e['uuid'], name=name, category="trace")
packet_begin = make_trace_packet(e['ts_ns'], track_event=evt_begin)
write_trace_packet_to_file(f, packet_begin)
# Slice End
evt_end = make_track_event(2, e['uuid'])
packet_end = make_trace_packet(e['ts_ns'] + e['dur_ns'], track_event=evt_end)
write_trace_packet_to_file(f, packet_end)
logger.info(f"Successfully generated Perfetto trace at {output_path}")
def main():
parser = argparse.ArgumentParser(description="Convert Hexagon Op profile logs to native Perfetto Protobuf traces.")
parser.add_argument("logfile", help="Path to hex-log profile file")
parser.add_argument("-o", "--output", default="optrace.perfetto-trace", help="Output trace file path (default: optrace.perfetto-trace)")
parser.add_argument("--filter", type=str, help="Regex filter matching against the original profile-op line")
group = parser.add_mutually_exclusive_group()
group.add_argument("--head", type=int, help="Limit to first N ops")
group.add_argument("--tail", type=int, help="Limit to last N ops")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format='%(message)s')
ops = parse_log(args.logfile)
if args.filter:
try:
filter_re = re.compile(args.filter)
except re.error as e:
logger.error(f"Invalid regex filter: {e}")
sys.exit(1)
ops = [op for op in ops if filter_re.search(op['op_text'])]
if args.head is not None:
ops = ops[:args.head]
elif args.tail is not None:
ops = ops[-args.tail:]
generate_perfetto_trace(ops, args.output)
if __name__ == "__main__":
main()
+35
vendor/llama.cpp/tools/mtmd/README-dev.md
# libmtmd dev guide
## History
Please refer to [multimodal.md](../../docs/multimodal.md) for a broader context.
In short:
- `libmtmd` started as a wrapper around `libllava` / `clip.cpp`
- Various components that used to be in `clip.cpp` are moved progressively to mtmd. For example, preprocessor is now part of mtmd
## Terminologies
- mtmd: **M**ul**T**i**M**o**D**al
- bitmap: representing a raw input data, for example: RGB image, PCM audio
- tiles / slices: for llava-uhd-style models, the preprocessor breaks a large input into smaller square images called tiles or slices
- chunk: a mtmd_input_chunk represents a preprocessed input that can then be passed through `mtmd_encode()`
## Pipeline
A typical pipeline of the core libmtmd is as follows:
- A bitmap (RGB image or PCM audio) is created
- Bitmap and the text prompt is provided to `mtmd_tokenize()` that breaks the input into chunks
- The tokenizer function first expands a "lazy" bitmap if it finds one. Typically, this is used by video, so that one media token corresponds to one input bitmap
- For models that support "fused" temporal frames like Qwen-VL, the tokenizer tries to merge pair of consecutive frames into one batch
- The preprocessor will then be called, which produces a list of chunks
- Depending on the model itself, special tokens will be injected to separate image chunks (i.e. llava-uhd-style models)
- Multiple bitmaps may be batched together to form a larger `mtmd_batch()`
- Single image or batch is encoded, via `mtmd_encode()` or `mtmd_batch_encode()`
- Get the output embeddings
## Helper
We provide a set of helper functions via `mtmd_helper` to make using libmtmd easier. The helper provides:
- Image, audio and video file decoding (for example, decode raw JPEG into RGB bitmap)
- Manage `llama_batch` and calls to `llama_decode`
+635
vendor/llama.cpp/tools/server/server-schema.cpp
#include "server-schema.h"
#include "json-schema-to-grammar.h"
namespace server_schema {
//
// llama.cpp-specific completion schema
//
std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params & params_base, task_params & params) {
std::vector<std::unique_ptr<field>> fields;
auto add = [&](field * f) {
fields.emplace_back(f);
};
add((new field_bool("timings_per_token", params.timings_per_token))
->set_desc("Include prompt processing and text generation speed information in each response"));
add((new field_bool("stream", params.stream))
->set_desc("Allows receiving each predicted token in real-time instead of waiting for the completion to finish"));
add((new field_nested("stream_options"))
->add_subfield((new field_bool("include_usage", params.include_usage))
->set_desc("Whether to include usage information in the stream"))
->set_desc("Additional options for streaming responses"));
add((new field_bool("cache_prompt", params.cache_prompt))
->set_desc("Re-use KV cache from a previous request if possible. This way the common prefix does not have to be re-processed, only the suffix that differs between the requests"));
add((new field_bool("return_tokens", params.return_tokens))
->set_desc("Return the raw generated token ids in the `tokens` field"));
add((new field_bool("return_progress", params.return_progress))
->set_desc("Include prompt processing progress events in stream mode"));
add((new field_num("n_predict", params.n_predict))
->set_hard_limits(-1, INT32_MAX)
->add_alias("max_completion_tokens")
->add_alias("max_tokens")
->set_desc("Set the maximum number of tokens to predict. When 0, no tokens will be generated but the prompt is evaluated into the cache"));
add((new field_num("n_indent", params.n_indent))
->set_hard_limits(0, INT32_MAX)
->set_desc("Specify the minimum line indentation for the generated text in number of whitespace characters. Useful for code completion tasks"));
add((new field_num("n_keep", params.n_keep))
->set_hard_limits(-1, INT32_MAX)
->set_desc("Specify the number of tokens from the initial prompt to retain when context size is exceeded. Use -1 to retain all tokens from the prompt"));
add((new field_num("n_discard", params.n_discard))
->set_hard_limits(0, INT32_MAX)
->set_desc("Number of tokens after n_keep that may be discarded when shifting context (0 = half context)"));
add((new field_num("n_cmpl", params.n_cmpl))
->set_hard_limits(1, params_base.n_parallel)
->add_alias("n") // alias "n" as fallback (OpenAI completions API)
->set_desc("Number of completions to generate. If the input has multiple prompts, total outputs will be N prompts times n_cmpl"));
add((new field_num("n_cache_reuse", params.n_cache_reuse))
->set_hard_limits(0, INT32_MAX)
->set_desc("Min chunk size to attempt reusing from the cache via KV shifting. See --cache-reuse arg"));
// TODO: implement t_max_prompt_ms
// add((new field_num("t_max_prompt_ms", params.t_max_prompt_ms))
add((new field_num("t_max_predict_ms", params.t_max_predict_ms))
->set_hard_limits(-1, std::numeric_limits<int64_t>::max())
->set_desc("Set a time limit in milliseconds for the prediction phase. The timeout triggers if generation exceeds this time (measured since the first token) and a newline has been generated. Useful for FIM applications"));
add((new field_json("response_fields"))
->set_desc("A list of response fields to return. Missing fields are omitted without error. Fields with a slash are unnested (e.g. generation_settings/n_predict moves n_predict to the root)")
->set_handler([&](field_eval_context & ctx, const json & data) {
ctx.params.response_fields = json_value(data, "response_fields", std::vector<std::string>());
}));
//
// Sampling params
//
add((new field_num("top_k", params.sampling.top_k))
->set_limits(0, INT32_MAX)
->set_desc("Limit the next token selection to the K most probable tokens (0 = disabled)"));
add((new field_num("top_p", params.sampling.top_p))
->set_limits(0.0f, 1.0f)
->set_desc("Limit the next token selection to a subset of tokens with cumulative probability above threshold P (1.0 = disabled)"));
add((new field_num("min_p", params.sampling.min_p))
->set_limits(0.0f, 1.0f)
->set_desc("The minimum probability for a token to be considered, relative to the probability of the most likely token (0 = disabled)"));
add((new field_num("top_n_sigma", params.sampling.top_n_sigma))
->set_desc("Keep tokens within n standard deviations of the top token logit (< 0 = disabled)"));
add((new field_num("xtc_probability", params.sampling.xtc_probability))
->set_limits(0.0f, 1.0f)
->set_desc("Set the chance for token removal via XTC sampler (0 = disabled)"));
add((new field_num("xtc_threshold", params.sampling.xtc_threshold))
->set_limits(0.0f, 1.0f)
->set_desc("Set a minimum probability threshold for tokens to be removed via XTC sampler (> 0.5 disables XTC)"));
add((new field_num("typical_p", params.sampling.typ_p))
// ->set_limits(0.0f, 1.0f) // what's the valid range?
->set_desc("Enable locally typical sampling with parameter p (1.0 = disabled)"));
add((new field_num("temperature", params.sampling.temp))
->set_limits(0.0f, std::numeric_limits<float>::infinity())
->set_desc("Adjust the randomness of the generated text (0 = greedy)"));
add((new field_num("dynatemp_range", params.sampling.dynatemp_range))
->set_desc("Dynamic temperature range. The final temperature will be in [temperature - range, temperature + range] (0 = disabled)"));
add((new field_num("dynatemp_exponent", params.sampling.dynatemp_exponent))
->set_desc("Dynamic temperature exponent, controls how entropy maps to temperature"));
add((new field_num("repeat_last_n", params.sampling.penalty_last_n))
->set_hard_limits(-1, INT32_MAX)
->set_desc("Last n tokens to consider for penalizing repetition (0 = disabled, -1 = ctx-size)"));
add((new field_num("repeat_penalty", params.sampling.penalty_repeat))
->set_desc("Control the repetition of token sequences in the generated text (1.0 = disabled)"));
add((new field_num("frequency_penalty", params.sampling.penalty_freq))
->set_desc("Repeat alpha frequency penalty (0 = disabled)"));
add((new field_num("presence_penalty", params.sampling.penalty_present))
->set_desc("Repeat alpha presence penalty (0 = disabled)"));
add((new field_num("dry_multiplier", params.sampling.dry_multiplier))
->set_desc("Set the DRY (Don't Repeat Yourself) repetition penalty multiplier (0 = disabled)"));
add((new field_num("dry_base", params.sampling.dry_base))
->set_desc("Set the DRY repetition penalty base value (must be >= 1.0, any values < 1.0 will be replaced with the default value)")
->set_handler([&](field_eval_context & ctx, const json & data) {
float v = data.at("dry_base").get<float>();
ctx.params.sampling.dry_base = (v < 1.0f) ? params_base.sampling.dry_base : v;
}));
add((new field_num("dry_allowed_length", params.sampling.dry_allowed_length))
->set_hard_limits(0, INT32_MAX)
->set_desc("Tokens that extend repetition beyond this length receive exponentially increasing penalty: multiplier * base ^ (sequence_length - allowed_length)"));
add((new field_num("dry_penalty_last_n", params.sampling.dry_penalty_last_n))
->set_hard_limits(-1, INT32_MAX)
->set_desc("How many tokens to scan for repetitions (0 = disabled, -1 = context size)"));
add((new field_num("mirostat", params.sampling.mirostat))
->set_limits(0, 2)
->set_desc("Enable Mirostat sampling, controlling perplexity during text generation (0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)"));
add((new field_num("mirostat_tau", params.sampling.mirostat_tau))
->set_desc("Set the Mirostat target entropy, parameter tau"));
add((new field_num("mirostat_eta", params.sampling.mirostat_eta))
->set_desc("Set the Mirostat learning rate, parameter eta"));
add((new field_num("adaptive_target", params.sampling.adaptive_target))
->set_limits(-std::numeric_limits<float>::max(), 1.0f)
->set_desc("Adaptive sampling target entropy (valid range 0.0 to 1.0; negative = disabled)"));
add((new field_num("adaptive_decay", params.sampling.adaptive_decay))
->set_hard_limits(0.0f, 0.99f)
->set_desc("EMA decay for adaptive sampling; history approximates 1/(1-decay) tokens"));
// seed is uint32_t; field_num uses int32_t so use a handler
add((new field_num("seed", params.sampling.seed))
->set_desc("Set the random number generator (RNG) seed (-1 = random)"));
add((new field_num("n_probs", params.sampling.n_probs))
->add_alias("logprobs") // use "logprobs" if "n_probs" wasn't provided
->set_desc("If greater than 0, output the probabilities of top N tokens for each generated token"));
add((new field_num("min_keep", params.sampling.min_keep))
->set_hard_limits(0, INT32_MAX)
->set_desc("If greater than 0, force samplers to return at least N possible tokens"));
add((new field_bool("backend_sampling", params.sampling.backend_sampling))
->set_desc("Use backend sampling instead of llama.cpp sampling"));
add((new field_bool("post_sampling_probs", params.post_sampling_probs))
->set_desc("Return probabilities of top n_probs tokens after applying the sampling chain"));
//
// Speculative decoding params
//
// TODO: to keep things simple, we disable speculative parameter adjustments for now
#if 0
// TODO: for now, be able to adjust only the draft-model based speculative parameters
add((new field_num("speculative.n_max", params.speculative.draft.n_max))
->set_hard_limits(0, INT32_MAX)
->set_desc("Maximum number of tokens to draft during speculative decoding"));
add((new field_num("speculative.n_min", params.speculative.draft.n_min))
->set_hard_limits(0, INT32_MAX)
->set_desc("Minimum number of draft tokens to use for speculative decoding");
add((new field_num("speculative.p_min", params.speculative.draft.p_min))
->set_hard_limits(0.0f, 1.0f)
->set_desc("Minimum speculative decoding probability for draft tokens (0 = greedy)"));
add((new field_str("speculative.type"))
->set_desc("Speculative decoding method (for debugging and research purposes)")
->set_handler([&](field_eval_context & ctx, const json & data) {
ctx.params.speculative.types = { common_speculative_type_from_name(data.at("speculative.type").get<std::string>()) };
}));
add((new field_num("speculative.ngram_size_n", params.speculative.ngram_simple.size_n))
->set_desc("Ngram size for lookup in ngram-based speculative decoding"));
add((new field_num("speculative.ngram_size_m", params.speculative.ngram_simple.size_m))
->set_desc("Mgram size for speculative tokens in ngram-based speculative decoding"));
add((new field_num("speculative.ngram_min_hits", params.speculative.ngram_simple.min_hits))
->set_desc("Minimum hits at ngram lookup for mgram to be proposed"));
#endif
add((new field_json("lora"))
->set_desc("A list of LoRA adapters to apply to this request. Each entry must have `id` and `scale` fields. Adapters not listed default to scale 0.0")
->set_handler([&](field_eval_context & ctx, const json & data) {
const auto & lora = data.at("lora");
if (!lora.is_array()) {
throw std::runtime_error("Error: 'lora' must be an array of objects with 'id' and 'scale' fields");
}
ctx.params.lora = parse_lora_request(lora);
}));
// sequence breakers for DRY
// Currently, this is not compatible with TextGen WebUI, Koboldcpp and SillyTavern format
// Ref: https://github.com/oobabooga/text-generation-webui/blob/d1af7a41ade7bd3c3a463bfa640725edb818ebaf/extensions/openai/typing.py#L39
add((new field_json("dry_sequence_breakers"))
->set_desc("Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted")
->set_handler([&](field_eval_context & ctx, const json & data) {
ctx.params.sampling.dry_sequence_breakers = json_value(data, "dry_sequence_breakers", std::vector<std::string>());
if (ctx.params.sampling.dry_sequence_breakers.empty()) {
throw std::runtime_error("Error: dry_sequence_breakers must be a non-empty array of strings");
}
}));
// handle both "json_schema" and "grammar"
add((new field_json("json_schema"))
->add_alias("grammar")
->set_desc("Set a JSON schema (json_schema) or GBNF grammar string (grammar) for constrained generation. json_schema takes precedence if both are provided")
->set_handler([&](field_eval_context & ctx, const json & data) {
auto & params = ctx.params;
if (data.contains("json_schema") && !data.contains("grammar")) {
try {
auto schema = json_value(data, "json_schema", json::object());
SRV_DBG("JSON schema: %s\n", schema.dump(2).c_str());
std::string grammar_str = json_schema_to_grammar(schema);
SRV_DBG("Converted grammar: %s\n", grammar_str.c_str());
params.sampling.grammar = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, std::move(grammar_str)};
} catch (const std::exception & e) {
throw std::runtime_error(std::string("\"json_schema\": ") + e.what());
}
} else {
std::string grammar_str = json_value(data, "grammar", std::string());
if (!grammar_str.empty()) {
// grammar_type key is set by the server when converting chat template grammars
std::string grammar_type = json_value(data, "grammar_type", std::string());
if (grammar_type == "tool_calls") {
params.sampling.grammar = {COMMON_GRAMMAR_TYPE_TOOL_CALLS, std::move(grammar_str)};
} else {
// explicit grammar from the user (API field "grammar")
params.sampling.grammar = {COMMON_GRAMMAR_TYPE_USER, std::move(grammar_str)};
}
SRV_DBG("Grammar (%s): %s\n", grammar_type.c_str(), common_grammar_value(params.sampling.grammar).c_str());
}
}
}));
add((new field_bool("grammar_lazy", params.sampling.grammar_lazy))
->set_desc("Whether to apply grammar constraints lazily, only when triggered (instead of at every step)"));
//
// Chat parser params
//
// TODO: change this to string field instead
add((new field_json("chat_format"))
->set_desc("Chat format used internally by the server")
->set_handler([&](field_eval_context & ctx, const json & data) {
ctx.params.chat_parser_params.format = static_cast<common_chat_format>(data.at("chat_format").get<int>());
SRV_INF("Chat format: %s\n", common_chat_format_name(ctx.params.chat_parser_params.format));
}));
add((new field_str("reasoning_format"))
->set_desc("Reasoning format for chain-of-thought models")
->set_handler([&](field_eval_context & ctx, const json & data) {
auto reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get<std::string>());
ctx.params.chat_parser_params.reasoning_format = reasoning_format;
ctx.params.chat_parser_params.reasoning_in_content = ctx.params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);
}));
add((new field_str("generation_prompt"))
->set_desc("Generation prompt appended to the chat template output")
->set_handler([&](field_eval_context & ctx, const json & data) {
std::string s = data.at("generation_prompt").get<std::string>();
ctx.params.chat_parser_params.generation_prompt = s;
ctx.params.sampling.generation_prompt = s;
}));
add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls))
->set_desc("Whether to parse tool calls from the generated output"));
add((new field_str("chat_parser"))
->set_desc("Chat parser configuration string")
->set_handler([&](field_eval_context & ctx, const json & data) {
ctx.params.chat_parser_params.parser.load(data.at("chat_parser").get<std::string>());
}));
add((new field_json("continue_final_message"))
->set_desc("Whether to continue the final message of the chat template")
->set_handler([&](field_eval_context & ctx, const json & data) {
auto continuation = common_chat_continuation_parse(data.at("continue_final_message"));
ctx.params.chat_parser_params.is_continuation = continuation != COMMON_CHAT_CONTINUATION_NONE;
}));
add((new field_bool("echo", params.chat_parser_params.echo))
->set_desc("Whether to echo the input tokens in the output"));
//
// Token-level fields (require vocab)
//
add((new field_json("preserved_tokens"))
->set_desc("List of token strings that must not be split during tokenization")
->set_handler([&](field_eval_context & ctx, const json & data) {
GGML_ASSERT(ctx.vocab != nullptr);
for (const auto & t : data.at("preserved_tokens")) {
auto ids = common_tokenize(ctx.vocab, t.get<std::string>(), false, true);
if (ids.size() == 1) {
ctx.params.sampling.preserved_tokens.insert(ids[0]);
}
}
}));
add((new field_json("grammar_triggers"))
->set_desc("List of strings or patterns that trigger grammar-constrained generation")
->set_handler([&](field_eval_context & ctx, const json & data) {
GGML_ASSERT(ctx.vocab != nullptr);
for (const auto & t : data.at("grammar_triggers")) {
server_grammar_trigger ct(t);
if (ct.value.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {
const auto & word = ct.value.value;
auto ids = common_tokenize(ctx.vocab, word, false, true);
if (ids.size() == 1) {
auto token = ids[0];
if (std::find(ctx.params.sampling.preserved_tokens.begin(), ctx.params.sampling.preserved_tokens.end(), (llama_token) token) == ctx.params.sampling.preserved_tokens.end()) {
throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word);
}
common_grammar_trigger trigger;
trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;
trigger.value = word;
trigger.token = token;
ctx.params.sampling.grammar_triggers.push_back(std::move(trigger));
} else {
ctx.params.sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});
}
} else {
ctx.params.sampling.grammar_triggers.emplace_back(std::move(ct.value));
}
}
if (ctx.params.sampling.grammar_lazy && ctx.params.sampling.grammar_triggers.empty()) {
throw std::runtime_error("Error: no triggers set for lazy grammar!");
}
}));
add((new field_bool("reasoning_control", params.sampling.reasoning_control))
->set_desc("Create the budget sampler on demand so reasoning can be ended at runtime"));
add((new field_num("reasoning_budget_tokens", params.sampling.reasoning_budget_tokens))
->set_hard_limits(-1, INT32_MAX)
->set_desc("Number of tokens in the reasoning budget (-1 = disabled)"));
add((new field_str("reasoning_budget_start_tag"))
->set_desc("Token string marking the start of the reasoning budget section")
->set_handler([&](field_eval_context & ctx, const json & data) {
GGML_ASSERT(ctx.vocab != nullptr);
ctx.params.sampling.reasoning_budget_start = common_tokenize(ctx.vocab, data.at("reasoning_budget_start_tag").get<std::string>(), false, true);
}));
add((new field_str("reasoning_budget_end_tag"))
->set_desc("Token string marking the end of the reasoning budget section")
->set_handler([&](field_eval_context & ctx, const json & data) {
GGML_ASSERT(ctx.vocab != nullptr);
std::string end_tag = data.at("reasoning_budget_end_tag").get<std::string>();
ctx.params.sampling.reasoning_budget_end = common_tokenize(ctx.vocab, end_tag, false, true);
}));
add((new field_str("reasoning_budget_message"))
->set_desc("Message to prepend to the reasoning budget end tag when forcing it")
->set_handler([&](field_eval_context & ctx, const json & data) {
GGML_ASSERT(ctx.vocab != nullptr);
std::string end_tag = json_value(data, "reasoning_budget_end_tag", std::string());
std::string message = data.at("reasoning_budget_message").get<std::string>();
ctx.params.sampling.reasoning_budget_forced = common_tokenize(ctx.vocab, message + end_tag, false, true);
}));
add((new field_json("logit_bias"))
->set_desc("Modify the likelihood of specific tokens. Accepts an array of [token, bias] pairs or an object mapping token to bias. Use false as bias to ban a token")
->set_handler([&](field_eval_context & ctx, const json & data) {
GGML_ASSERT(ctx.vocab != nullptr);
ctx.params.sampling.logit_bias.clear();
const auto & logit_bias = data.at("logit_bias");
const int n_vocab = llama_vocab_n_tokens(ctx.vocab);
auto parse_bias = [](const json & v, float & bias) -> bool {
if (v.is_number()) { bias = v.get<float>(); return true; }
if (v.is_boolean() && !v.get<bool>()) { bias = -INFINITY; return true; }
return false;
};
if (logit_bias.is_array()) {
for (const auto & el : logit_bias) {
if (!el.is_array() || el.size() != 2) continue;
float bias;
if (!parse_bias(el[1], bias)) continue;
if (el[0].is_number_integer()) {
llama_token tok = el[0].get<llama_token>();
if (tok >= 0 && tok < n_vocab) ctx.params.sampling.logit_bias.push_back({tok, bias});
} else if (el[0].is_string()) {
for (auto tok : common_tokenize(ctx.vocab, el[0].get<std::string>(), false))
ctx.params.sampling.logit_bias.push_back({tok, bias});
}
}
} else if (logit_bias.is_object()) {
for (const auto & el : logit_bias.items()) {
float bias;
if (!parse_bias(el.value(), bias)) continue;
char * end;
llama_token tok = strtol(el.key().c_str(), &end, 10);
if (*end == 0) {
if (tok >= 0 && tok < n_vocab) ctx.params.sampling.logit_bias.push_back({tok, bias});
} else {
for (auto t : common_tokenize(ctx.vocab, el.key(), false))
ctx.params.sampling.logit_bias.push_back({t, bias});
}
}
}
}));
add((new field_bool("ignore_eos", params.sampling.ignore_eos))
->set_desc("Ignore the end-of-sequence token and continue generating")
->set_handler([&](field_eval_context & ctx, const json & data) {
GGML_ASSERT(ctx.logit_bias_eog != nullptr);
ctx.params.sampling.ignore_eos = data.at("ignore_eos").get<bool>();
if (ctx.params.sampling.ignore_eos && ctx.logit_bias_eog) {
ctx.params.sampling.logit_bias.insert(
ctx.params.sampling.logit_bias.end(),
ctx.logit_bias_eog->begin(), ctx.logit_bias_eog->end());
}
}));
add((new field_json("stop"))
->set_desc("Specify stopping strings. Generation stops when one is produced, and the string is not included in the output")
->set_handler([&](field_eval_context & ctx, const json & data) {
ctx.params.antiprompt.clear();
const auto & stop = data.at("stop");
if (stop.is_array()) {
for (const auto & word : stop) {
if (!word.empty()) ctx.params.antiprompt.push_back(word);
}
} else if (stop.is_string()) {
ctx.params.antiprompt.push_back(stop.get<std::string>());
}
// fall back to CLI defaults if the request provided no effective stop strings
if (ctx.params.antiprompt.empty()) {
ctx.params.antiprompt = params_base.antiprompt;
}
}));
add((new field_json("samplers"))
->set_desc("The order in which samplers are applied. An array of sampler type names, or a single string of sampler chars")
->set_handler([&](field_eval_context & ctx, const json & data) {
const auto & samplers = data.at("samplers");
if (samplers.is_array()) {
ctx.params.sampling.samplers = common_sampler_types_from_names(samplers);
} else if (samplers.is_string()) {
ctx.params.sampling.samplers = common_sampler_types_from_chars(samplers.get<std::string>());
}
}));
return fields;
}
task_params eval_llama_cmpl_schema(
const llama_vocab * vocab,
const common_params & params_base,
const int n_ctx_slot,
const std::vector<llama_logit_bias> & logit_bias_eog,
const json & data) {
task_params params;
// Sampling parameter defaults are loaded from the global server context (but individual requests can still them)
params.sampling = params_base.sampling;
params.speculative = params_base.speculative;
params.n_keep = params_base.n_keep;
params.n_predict = params_base.n_predict;
params.n_cache_reuse = params_base.n_cache_reuse;
params.cache_prompt = params_base.cache_prompt;
params.antiprompt = params_base.antiprompt;
// enabling this will output extra debug information in the HTTP responses from the server
params.verbose = params_base.verbosity > 9;
params.chat_parser_params.reasoning_format = params_base.reasoning_format;
// create context and schema
field_eval_context ctx(params);
ctx.vocab = vocab;
ctx.logit_bias_eog = &logit_bias_eog;
auto schema = make_llama_cmpl_schema(params_base, params);
// eval all fields in the schema
for (const auto & f : schema) {
f->eval(ctx, data);
}
// post-processing
{
if (params.sampling.penalty_last_n == -1) {
// note: should be the slot's context and not the full context, but it's ok
params.sampling.penalty_last_n = n_ctx_slot;
}
if (params.sampling.dry_penalty_last_n == -1) {
params.sampling.dry_penalty_last_n = n_ctx_slot;
}
// if "reasoning_format" is not provided, its handler will not be called, we will need to handle it here
auto reasoning_format = params.chat_parser_params.reasoning_format;
params.chat_parser_params.reasoning_in_content = params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);
}
// debugging
{
auto budget = params.sampling.reasoning_budget_tokens;
SRV_DBG("reasoning budget: tokens=%d, generation_prompt='%s', start=%zu toks, end=%zu toks, forced=%zu toks\n",
budget, params.sampling.generation_prompt.c_str(),
params.sampling.reasoning_budget_start.size(),
params.sampling.reasoning_budget_end.size(),
params.sampling.reasoning_budget_forced.size());
}
return params;
}
//
// eval() implementations
//
static void handle_with_catch(const char * name, std::function<void()> func) {
try {
func();
} catch (const std::exception & e) {
throw std::invalid_argument(string_format("Field '%s': %s", name, e.what()));
}
}
template <typename T>
void field_num<T>::eval(field_eval_context & ctx, const json & data) {
for (const auto & n : name) {
if (data.contains(n)) {
handle_with_catch(n, [&]() {
if (custom_handler) {
custom_handler(ctx, data);
} else if (!is_hard_limit) {
val = std::max(min, std::min(max, data.at(n).template get<T>()));
} else {
T tmp = data.at(n).template get<T>();
if (tmp < min || tmp > max) {
throw std::invalid_argument(std::string("Value must be between ") + std::to_string(min) + " <= value <= " + std::to_string(max) + ", but got " + std::to_string(tmp));
}
val = tmp;
}
});
return;
}
}
}
void field_str::eval(field_eval_context & ctx, const json & data) {
GGML_ASSERT(custom_handler);
for (const auto & n : name) {
if (data.contains(n)) {
handle_with_catch(n, [&]() {
custom_handler(ctx, data);
});
return;
}
}
}
void field_bool::eval(field_eval_context & ctx, const json & data) {
for (const auto & n : name) {
if (data.contains(n)) {
handle_with_catch(n, [&]() {
if (custom_handler) {
custom_handler(ctx, data);
} else {
val = data.at(n).get<bool>();
}
});
return;
}
}
}
void field_json::eval(field_eval_context & ctx, const json & data) {
GGML_ASSERT(custom_handler);
for (const auto & n : name) {
if (data.contains(n)) {
handle_with_catch(n, [&]() {
custom_handler(ctx, data);
});
return;
}
}
}
void field_nested::eval(field_eval_context & ctx, const json & data) {
for (const auto & n : name) {
if (data.contains(n) && data.at(n).is_object()) {
for (auto & f : subfields) {
f->eval(ctx, data.at(n));
}
return;
}
}
}
} // namespace server_schema
+105
vendor/llama.cpp/tools/server/server-schema.h
#pragma once
#include "server-common.h"
#include "server-task.h"
#include "sampling.h"
#include "speculative.h"
#include <climits>
#include <functional>
#include <limits>
#include <memory>
#include <string>
#include <vector>
namespace server_schema {
struct field_eval_context {
task_params & params;
const llama_vocab * vocab = nullptr;
const std::vector<llama_logit_bias> * logit_bias_eog = nullptr;
field_eval_context(task_params & params) : params(params) {}
};
using field_handler = std::function<void(field_eval_context &, const json &)>;
struct field {
std::vector<const char *> name;
const char * desc = "";
field_handler custom_handler;
field() = default;
field(const char * n) : name({n}) {}
virtual ~field() = default;
field * set_desc(const char * s) {
desc = s;
return this;
}
// if 'name' is present, use it, otherwise look for aliases following the order they were added
field * add_alias(const char * n) {
name.push_back(n);
return this;
}
field * set_handler(field_handler h) { this->custom_handler = h; return this; }
virtual void eval(field_eval_context & ctx, const json & data) = 0;
};
template <typename T = int32_t>
struct field_num : public field {
T & val;
T min = std::numeric_limits<T>::lowest();
T max = std::numeric_limits<T>::max();
bool is_hard_limit = false; // if true, throw error if the value is invalid
field_num(const char * n, T & val) : field(n), val(val) {}
// limits are inclusive, min <= value <= max
field_num * set_limits(T min, T max) {
this->min = min;
this->max = max;
return this;
}
field_num * set_hard_limits(T min, T max) {
set_limits(min, max);
is_hard_limit = true;
return this;
}
virtual void eval(field_eval_context & ctx, const json & data) override;
};
struct field_str : public field {
field_str(const char * n) : field(n) {}
virtual void eval(field_eval_context & ctx, const json & data) override;
};
struct field_bool : public field {
bool & val;
field_bool(const char * n, bool & val) : field(n), val(val) {}
virtual void eval(field_eval_context & ctx, const json & data) override;
};
struct field_json : public field {
field_json(const char * n) : field(n) {}
virtual void eval(field_eval_context & ctx, const json & data) override;
};
struct field_nested : public field {
std::vector<std::unique_ptr<field>> subfields;
field_nested(const char * n) : field(n) {}
field_nested * add_subfield(field * f) {
subfields.emplace_back(std::unique_ptr<field>(f));
return this;
}
virtual void eval(field_eval_context & ctx, const json & data) override;
};
std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(
const common_params & params_base,
task_params & params);
task_params eval_llama_cmpl_schema(
const llama_vocab * vocab,
const common_params & params_base,
const int n_ctx_slot,
const std::vector<llama_logit_bias> & logit_bias_eog,
const json & data);
} // namespace server_schema
+9
vendor/llama.cpp/t.../ui/src/lib/constants/diagram-blocks.ts
// Shared constants for diagram blocks (mermaid and svg) that toggle between a
// rendered view and a source view. The wrapper carries the active mode, css
// drives the visibility, the click handler only flips the attribute.
export const DIAGRAM_VIEW_MODE_ATTR = 'data-view-mode';
export const DIAGRAM_VIEW_RENDERED = 'rendered';
export const DIAGRAM_VIEW_SOURCE = 'source';
export const DIAGRAM_SOURCE_CLASS = 'diagram-source';
export const TOGGLE_SOURCE_BTN_CLASS = 'toggle-source-btn';
+269
vendor/llama.cpp/t...s/stories/ModelsSelector.stories.svelte
<script module lang="ts">
import { defineMeta } from '@storybook/addon-svelte-csf';
import ModelsSelectorList from '$lib/components/app/models/ModelsSelectorList.svelte';
import ModelsSelectorOption from '$lib/components/app/models/ModelsSelectorOption.svelte';
import type { GroupedModelOptions, ModelItem } from '$lib/components/app/models/utils';
import { modelsStore } from '$lib/stores/models.svelte';
import { ServerModelStatus } from '$lib/enums';
const { Story } = defineMeta({
title: 'Components/ModelsSelector',
parameters: {
layout: 'centered'
}
});
const mockModel = (id: string, name: string, orgName?: string, tags?: string[]): ModelOption => ({
id,
name,
model: orgName ? `${orgName}/${name}` : name,
capabilities: [],
parsedId: {
raw: orgName ? `${orgName}/${name}` : name,
orgName: orgName ?? null,
modelName: name,
params: null,
activatedParams: null,
quantization: null,
tags: tags ?? []
},
tags
});
const mockRouterEntry = (modelName: string, status: ServerModelStatus): ApiModelDataEntry => ({
id: modelName,
object: 'model',
owned_by: 'llamacpp',
created: Date.now(),
in_cache: true,
path: `/models/${modelName}`,
status: { value: status }
});
</script>
<script lang="ts">
let selectedModel = $state<string | null>(null);
let activeId = $state<string | null>(null);
function mockModelsStore() {
modelsStore.favoriteModelIds = new Set(['qwen2.5-7b', 'llama3.2-3b']);
// Mock router models with various statuses for ModelLoadedStates story
modelsStore.routerModels = [
mockRouterEntry('meta/Model (loading)', ServerModelStatus.LOADING),
mockRouterEntry('meta/Model (loaded)', ServerModelStatus.LOADED),
mockRouterEntry('meta/Model (sleeping)', ServerModelStatus.SLEEPING),
mockRouterEntry('meta/Model (failed)', ServerModelStatus.FAILED)
];
}
mockModelsStore();
const loadedModels: ModelItem[] = [
{ option: mockModel('llama3.1-8b', 'Llama-3.1-8B-Instruct', 'meta'), flatIndex: 0 },
{ option: mockModel('mistral-7b', 'Mistral-7B-v0.3', 'mistralai'), flatIndex: 1 }
];
const favoriteModels: ModelItem[] = [
{ option: mockModel('qwen2.5-7b', 'Qwen2.5-7B-Instruct', 'Qwen'), flatIndex: 2 },
{ option: mockModel('llama3.2-3b', 'Llama-3.2-3B-Instruct', 'meta'), flatIndex: 3 }
];
const availableModels: ModelItem[] = [
{
option: mockModel('deepseek-coder-6.7b', 'DeepSeek-Coder-6.7B', 'deepseek', ['coding']),
flatIndex: 4
},
{ option: mockModel('gemma-2-9b', 'Gemma-2-9B-IT', 'google'), flatIndex: 5 },
{ option: mockModel('phi-3-mini', 'Phi-3-mini-4k', 'microsoft'), flatIndex: 6 },
{ option: mockModel('codellama-7b', 'CodeLlama-7B', 'codellama', ['coding']), flatIndex: 7 },
{ option: mockModel('neural-chat-7b', 'Neural-Chat-7B-v3-3', 'intel'), flatIndex: 8 }
];
const groupedOptions: GroupedModelOptions = {
loaded: loadedModels,
favorites: favoriteModels,
available: [
{
orgName: 'deepseek',
items: [availableModels[0]]
},
{
orgName: 'google',
items: [availableModels[1]]
},
{
orgName: 'microsoft',
items: [availableModels[2]]
},
{
orgName: 'codellama',
items: [availableModels[3]]
},
{
orgName: 'intel',
items: [availableModels[4]]
}
]
};
function handleSelect(modelId: string) {
const opt = [...loadedModels, ...favoriteModels, ...availableModels].find(
(m) => m.option.id === modelId
);
if (opt) {
selectedModel = opt.option.model;
activeId = modelId;
}
}
</script>
<Story name="List">
<div class="w-80 rounded-lg border border-border bg-popover p-2 shadow-md">
<ModelsSelectorList
groups={groupedOptions}
currentModel={selectedModel}
{activeId}
onSelect={handleSelect}
onInfoClick={(modelName) => console.log('Info clicked:', modelName)}
/>
</div>
</Story>
<Story name="SingleLoaded">
<div class="w-80 rounded-lg border border-border bg-popover p-2 shadow-md">
<ModelsSelectorList
groups={{
loaded: [loadedModels[0]],
favorites: [],
available: []
}}
currentModel={null}
activeId={null}
onSelect={handleSelect}
onInfoClick={(modelName) => console.log('Info clicked:', modelName)}
/>
</div>
</Story>
<Story name="WithFavoritesOnly">
<div class="w-80 rounded-lg border border-border bg-popover p-2 shadow-md">
<ModelsSelectorList
groups={{
loaded: [],
favorites: favoriteModels,
available: []
}}
currentModel={null}
activeId={null}
onSelect={handleSelect}
onInfoClick={(modelName) => console.log('Info clicked:', modelName)}
/>
</div>
</Story>
<Story name="ModelLoadedStates">
<div class="w-80 rounded-lg border border-border bg-popover p-2 shadow-md">
<div class="px-2 py-2 text-[13px] font-semibold text-muted-foreground/70 select-none">
Server model states
</div>
<ModelsSelectorOption
option={mockModel('model-idle', 'Model (idle)', 'meta')}
isSelected={false}
isHighlighted={false}
isFav={false}
hideOrgName={true}
onSelect={() => {}}
onMouseEnter={() => {}}
onKeyDown={() => {}}
/>
<ModelsSelectorOption
option={mockModel('model-loading', 'Model (loading)', 'meta')}
isSelected={false}
isHighlighted={false}
isFav={false}
hideOrgName={true}
onSelect={() => {}}
onMouseEnter={() => {}}
onKeyDown={() => {}}
/>
<ModelsSelectorOption
option={mockModel('model-loaded', 'Model (loaded)', 'meta')}
isSelected={false}
isHighlighted={false}
isFav={false}
hideOrgName={true}
onSelect={() => {}}
onMouseEnter={() => {}}
onKeyDown={() => {}}
/>
<ModelsSelectorOption
option={mockModel('model-sleeping', 'Model (sleeping)', 'meta')}
isSelected={false}
isHighlighted={false}
isFav={false}
hideOrgName={true}
onSelect={() => {}}
onMouseEnter={() => {}}
onKeyDown={() => {}}
/>
<ModelsSelectorOption
option={mockModel('model-failed', 'Model (failed)', 'meta')}
isSelected={false}
isHighlighted={false}
isFav={false}
hideOrgName={true}
onSelect={() => {}}
onMouseEnter={() => {}}
onKeyDown={() => {}}
/>
</div>
</Story>
<Story name="ModelSelectedStates">
<div class="w-80 rounded-lg border border-border bg-popover p-2 shadow-md">
<div class="px-2 py-2 text-[13px] font-semibold text-muted-foreground/70 select-none">
Selection states
</div>
<ModelsSelectorOption
option={mockModel('normal-model', 'Normal Model', 'meta')}
isSelected={false}
isHighlighted={false}
isFav={false}
hideOrgName={true}
onSelect={() => {}}
onMouseEnter={() => {}}
onKeyDown={() => {}}
/>
<ModelsSelectorOption
option={mockModel('selected-model', 'Selected Model', 'meta')}
isSelected={true}
isHighlighted={false}
isFav={false}
hideOrgName={true}
onSelect={() => {}}
onMouseEnter={() => {}}
onKeyDown={() => {}}
/>
<ModelsSelectorOption
option={mockModel('highlighted-model', 'Highlighted Model', 'meta')}
isSelected={false}
isHighlighted={true}
isFav={false}
hideOrgName={true}
onSelect={() => {}}
onMouseEnter={() => {}}
onKeyDown={() => {}}
/>
<ModelsSelectorOption
option={mockModel('fav-model', 'Favorite Model', 'Qwen')}
isSelected={false}
isHighlighted={false}
isFav={true}
hideOrgName={true}
onSelect={() => {}}
onMouseEnter={() => {}}
onKeyDown={() => {}}
/>
</div>
</Story>
+4
-0
CHANGELOG.md

@@ -10,2 +10,6 @@ # Changelog

## [0.3.31]
- feat: update llama.cpp to ggml-org/llama.cpp@f449e0553
## [0.3.30]

@@ -12,0 +16,0 @@ 

+1
-1
llama_cpp/__init__.py
from .llama_cpp import *
from .llama import *
__version__ = "0.3.30"
__version__ = "0.3.31"
+1
-1
PKG-INFO
Metadata-Version: 2.1
Name: llama_cpp_python
Version: 0.3.30
Version: 0.3.31
Summary: Python bindings for the llama.cpp library

@@ -5,0 +5,0 @@ Author-Email: Andrei Betlen <abetlen@gmail.com>

+16
-0
vendor/llama.cpp/.devops/cann.Dockerfile

@@ -16,2 +16,16 @@ # ==============================================================================

# ==============================================================================
ARG NODE_VERSION=24
FROM docker.io/node:$NODE_VERSION AS web
ARG APP_VERSION
WORKDIR /app/tools/ui
COPY tools/ui/package.json tools/ui/package-lock.json ./
RUN npm ci
COPY tools/ui/ ./
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
FROM ${CANN_BASE_IMAGE} AS build

@@ -30,2 +44,4 @@

COPY --from=web /app/tools/ui/dist tools/ui/dist
# -- Set CANN environment variables (required for compilation) --

@@ -32,0 +48,0 @@ # Using ENV instead of `source` allows environment variables to persist across the entire image layer

+16
-0
vendor/llama.cpp/.devops/cpu.Dockerfile

@@ -6,2 +6,16 @@ ARG UBUNTU_VERSION=24.04

ARG NODE_VERSION=24
FROM docker.io/node:$NODE_VERSION AS web
ARG APP_VERSION
WORKDIR /app/tools/ui
COPY tools/ui/package.json tools/ui/package-lock.json ./
RUN npm ci
COPY tools/ui/ ./
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build

@@ -20,2 +34,4 @@

COPY --from=web /app/tools/ui/dist tools/ui/dist
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \

@@ -22,0 +38,0 @@ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON; \

+16
-0
vendor/llama.cpp/.devops/cuda.Dockerfile

@@ -14,2 +14,16 @@ ARG UBUNTU_VERSION=24.04

ARG NODE_VERSION=24
FROM docker.io/node:$NODE_VERSION AS web
ARG APP_VERSION
WORKDIR /app/tools/ui
COPY tools/ui/package.json tools/ui/package-lock.json ./
RUN npm ci
COPY tools/ui/ ./
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
FROM ${BASE_CUDA_DEV_CONTAINER} AS build

@@ -30,2 +44,4 @@

COPY --from=web /app/tools/ui/dist tools/ui/dist
RUN if [ "${CUDA_DOCKER_ARCH}" != "default" ]; then \

@@ -32,0 +48,0 @@ export CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH}"; \

+16
-0
vendor/llama.cpp/.devops/intel.Dockerfile

@@ -8,2 +8,16 @@ ARG ONEAPI_VERSION=2025.3.3-0-devel-ubuntu24.04

ARG NODE_VERSION=24
FROM docker.io/node:$NODE_VERSION AS web
ARG APP_VERSION
WORKDIR /app/tools/ui
COPY tools/ui/package.json tools/ui/package-lock.json ./
RUN npm ci
COPY tools/ui/ ./
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build

@@ -26,2 +40,4 @@

COPY --from=web /app/tools/ui/dist tools/ui/dist
RUN if [ "${GGML_SYCL_F16}" = "ON" ]; then \

@@ -28,0 +44,0 @@ echo "GGML_SYCL_F16 is set" \

+16
-0
vendor/llama.cpp/.devops/musa.Dockerfile

@@ -13,2 +13,16 @@ ARG UBUNTU_VERSION=22.04

ARG NODE_VERSION=24
FROM docker.io/node:$NODE_VERSION AS web
ARG APP_VERSION
WORKDIR /app/tools/ui
COPY tools/ui/package.json tools/ui/package-lock.json ./
RUN npm ci
COPY tools/ui/ ./
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
FROM ${BASE_MUSA_DEV_CONTAINER} AS build

@@ -33,2 +47,4 @@

COPY --from=web /app/tools/ui/dist tools/ui/dist
RUN if [ "${MUSA_DOCKER_ARCH}" != "default" ]; then \

@@ -35,0 +51,0 @@ export CMAKE_ARGS="-DMUSA_ARCHITECTURES=${MUSA_DOCKER_ARCH}"; \

+79
-46
vendor/llama.cpp/.devops/openvino.Dockerfile

@@ -1,15 +0,15 @@

ARG OPENVINO_VERSION_MAJOR=2026.0
ARG OPENVINO_VERSION_FULL=2026.0.0.20965.c6d6a13a886
ARG OPENVINO_VERSION_MAJOR=2026.2
ARG OPENVINO_VERSION_FULL=2026.2.0.21903.52ddc073857
ARG UBUNTU_VERSION=24.04
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
ARG IGC_VERSION=v2.30.1
ARG IGC_VERSION_FULL=2_2.30.1+20950
ARG COMPUTE_RUNTIME_VERSION=26.09.37435.1
ARG COMPUTE_RUNTIME_VERSION_FULL=26.09.37435.1-0
ARG IGDGMM_VERSION=22.9.0
ARG IGC_VERSION=v2.34.4
ARG IGC_VERSION_FULL=2_2.34.4+21428
ARG COMPUTE_RUNTIME_VERSION=26.18.38308.1
ARG COMPUTE_RUNTIME_VERSION_FULL=26.18.38308.1-0
ARG IGDGMM_VERSION=22.10.0
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
ARG NPU_DRIVER_VERSION=v1.32.0
ARG NPU_DRIVER_FULL=v1.32.0.20260402-23905121947
ARG NPU_DRIVER_VERSION=v1.33.0
ARG NPU_DRIVER_FULL=v1.33.0.20260529-26625960453
ARG LIBZE1_VERSION=1.27.0-1~24.04~ppa2

@@ -25,2 +25,16 @@

ARG NODE_VERSION=24
FROM docker.io/node:$NODE_VERSION AS web
ARG APP_VERSION
WORKDIR /app/tools/ui
COPY tools/ui/package.json tools/ui/package-lock.json ./
RUN npm ci
COPY tools/ui/ ./
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
## Build Image

@@ -50,9 +64,14 @@ FROM docker.io/ubuntu:${UBUNTU_VERSION} AS build

# Install OpenVINO for Ubuntu 24.04
# OpenVINO toolkit and GPU/NPU drivers are cached via BuildKit cache mounts to avoid re-downloading on rebuilds.
# Install OpenVINO for Ubuntu 24.04.
ARG OPENVINO_VERSION_MAJOR
ARG OPENVINO_VERSION_FULL
RUN mkdir -p /opt/intel && \
wget https://storage.openvinotoolkit.org/repositories/openvino/packages/${OPENVINO_VERSION_MAJOR}/linux/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz && \
tar -xf openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz && \
mv openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64 /opt/intel/openvino_${OPENVINO_VERSION_MAJOR} && \
RUN --mount=type=cache,target=/var/cache/openvino,sharing=locked \
mkdir -p /opt/intel && \
TGZ=/var/cache/openvino/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz && \
if [ ! -f "$TGZ" ]; then \
wget -O "$TGZ" https://storage.openvinotoolkit.org/repositories/openvino/packages/${OPENVINO_VERSION_MAJOR}/linux/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz; \
fi && \
tar -xf "$TGZ" -C /opt/intel/ && \
mv /opt/intel/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64 /opt/intel/openvino_${OPENVINO_VERSION_MAJOR} && \
cd /opt/intel/openvino_${OPENVINO_VERSION_MAJOR} && \

@@ -69,2 +88,4 @@ echo "Y" | ./install_dependencies/install_openvino_dependencies.sh && \

COPY --from=web /app/tools/ui/dist tools/ui/dist
# Build Stage

@@ -74,10 +95,10 @@ RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \

-DCMAKE_BUILD_TYPE=Release \
-DLLAMA_BUILD_TESTS=OFF \
-DGGML_OPENVINO=ON && \
cmake --build build/ReleaseOV -j$(nproc)"
cmake --build build/ReleaseOV --parallel "
# Copy all necessary libraries
# Copy all necessary libraries (build outputs + OpenVINO runtime libs)
RUN mkdir -p /app/lib && \
find build/ReleaseOV -name '*.so*' -exec cp {} /app/lib \; && \
find ${OpenVINO_DIR}/runtime/lib/intel64 -name '*.so*' -exec cp -P {} /app/lib \; 2>/dev/null || \
find ${OpenVINO_DIR}/lib/intel64 -name '*.so*' -exec cp -P {} /app/lib \;
find build/ReleaseOV -name '*.so*' -exec cp -P {} /app/lib \; && \
find "${OpenVINO_DIR}/runtime/lib/intel64" -name '*.so*' -exec cp -P {} /app/lib \;

@@ -127,14 +148,18 @@ # Create runtime directories and copy binaries

ARG IGDGMM_VERSION
RUN mkdir /tmp/neo/ && cd /tmp/neo/ \
&& wget https://github.com/intel/intel-graphics-compiler/releases/download/${IGC_VERSION}/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \
&& wget https://github.com/intel/intel-graphics-compiler/releases/download/${IGC_VERSION}/intel-igc-opencl-${IGC_VERSION_FULL}_amd64.deb \
&& wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-ocloc-dbgsym_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.ddeb \
&& wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-ocloc_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \
&& wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-opencl-icd-dbgsym_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.ddeb \
&& wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-opencl-icd_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \
&& wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libigdgmm12_${IGDGMM_VERSION}_amd64.deb \
&& wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libze-intel-gpu1-dbgsym_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.ddeb \
&& wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libze-intel-gpu1_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \
&& dpkg --install *.deb \
&& rm -rf /tmp/neo/
RUN --mount=type=cache,target=/var/cache/intel-gpu,sharing=locked \
set -eux; \
cd /var/cache/intel-gpu; \
for url in \
https://github.com/intel/intel-graphics-compiler/releases/download/${IGC_VERSION}/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \
https://github.com/intel/intel-graphics-compiler/releases/download/${IGC_VERSION}/intel-igc-opencl-${IGC_VERSION_FULL}_amd64.deb \
https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-ocloc_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \
https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-opencl-icd_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \
https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libigdgmm12_${IGDGMM_VERSION}_amd64.deb \
https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libze-intel-gpu1_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb ; do \
f=$(basename "$url"); \
[ -f "$f" ] || wget -q -O "$f" "$url"; \
done; \
apt-get update; \
apt-get install -y --no-install-recommends ./*.deb; \
rm -rf /var/lib/apt/lists/*

@@ -145,13 +170,17 @@ # Install NPU drivers

ARG LIBZE1_VERSION
RUN mkdir /tmp/npu/ && cd /tmp/npu/ \
&& wget https://github.com/intel/linux-npu-driver/releases/download/${NPU_DRIVER_VERSION}/linux-npu-driver-${NPU_DRIVER_FULL}-ubuntu2404.tar.gz \
&& tar -xf linux-npu-driver-${NPU_DRIVER_FULL}-ubuntu2404.tar.gz \
&& dpkg --install *.deb \
&& rm -rf /tmp/npu/
RUN --mount=type=cache,target=/var/cache/intel-npu,sharing=locked \
set -eux; \
TGZ=/var/cache/intel-npu/linux-npu-driver-${NPU_DRIVER_FULL}-ubuntu2404.tar.gz; \
if [ ! -f "$TGZ" ]; then \
wget -q -O "$TGZ" https://github.com/intel/linux-npu-driver/releases/download/${NPU_DRIVER_VERSION}/linux-npu-driver-${NPU_DRIVER_FULL}-ubuntu2404.tar.gz; \
fi; \
DEB=/var/cache/intel-npu/libze1_${LIBZE1_VERSION}_amd64.deb; \
if [ ! -f "$DEB" ]; then \
wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260324T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
fi; \
mkdir /tmp/npu/ && cd /tmp/npu/ && tar -xf "$TGZ" && cp "$DEB" .; \
apt-get update; \
apt-get install -y --no-install-recommends ./*.deb; \
rm -rf /tmp/npu/ /var/lib/apt/lists/*
RUN cd /tmp \
&& wget https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260324T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb \
&& dpkg --install libze1_${LIBZE1_VERSION}_amd64.deb \
&& rm libze1_${LIBZE1_VERSION}_amd64.deb
COPY --from=build /app/lib/ /app/

@@ -175,5 +204,5 @@

python3-pip && \
python3 -m venv /ov-venv && \
/ov-venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel && \
/ov-venv/bin/pip install --no-cache-dir -r requirements.txt && \
python3 -m venv /openvino-venv && \
/openvino-venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel && \
/openvino-venv/bin/pip install --no-cache-dir -r requirements.txt && \
apt-get autoremove -y && \

@@ -185,9 +214,13 @@ apt-get clean && \

ENTRYPOINT ["/bin/bash", "-c", "source /ov-venv/bin/activate && exec /app/tools.sh \"$@\"", "--"]
# Activate the venv
ENV VIRTUAL_ENV=/openvino-venv \
PATH=/openvino-venv/bin:$PATH
ENTRYPOINT ["/app/tools.sh"]
### Light, CLI only
FROM base AS light
COPY --from=build /app/full/llama-cli /app/
COPY --from=build /app/full/llama-cli /app/full/llama-completion /app/

@@ -194,0 +227,0 @@ WORKDIR /app

+16
-0
vendor/llama.cpp/.devops/rocm.Dockerfile

@@ -14,2 +14,16 @@ ARG UBUNTU_VERSION=24.04

ARG NODE_VERSION=24
FROM docker.io/node:$NODE_VERSION AS web
ARG APP_VERSION
WORKDIR /app/tools/ui
COPY tools/ui/package.json tools/ui/package-lock.json ./
RUN npm ci
COPY tools/ui/ ./
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
### Build image

@@ -42,2 +56,4 @@ FROM ${BASE_ROCM_DEV_CONTAINER} AS build

COPY --from=web /app/tools/ui/dist tools/ui/dist
RUN HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \

@@ -44,0 +60,0 @@ cmake -S . -B build \

+16
-0
vendor/llama.cpp/.devops/s390x.Dockerfile

@@ -7,2 +7,16 @@ ARG GCC_VERSION=15.2.0

ARG NODE_VERSION=24
FROM docker.io/node:$NODE_VERSION AS web
ARG APP_VERSION
WORKDIR /app/tools/ui
COPY tools/ui/package.json tools/ui/package-lock.json ./
RUN npm ci
COPY tools/ui/ ./
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
### Build Llama.cpp stage

@@ -24,2 +38,4 @@ FROM docker.io/gcc:${GCC_VERSION} AS build

COPY --from=web /app/tools/ui/dist tools/ui/dist
RUN --mount=type=cache,target=/root/.ccache \

@@ -26,0 +42,0 @@ --mount=type=cache,target=/app/build \

+16
-0
vendor/llama.cpp/.devops/vulkan.Dockerfile

@@ -6,2 +6,16 @@ ARG UBUNTU_VERSION=26.04

ARG NODE_VERSION=24
FROM docker.io/node:$NODE_VERSION AS web
ARG APP_VERSION
WORKDIR /app/tools/ui
COPY tools/ui/package.json tools/ui/package-lock.json ./
RUN npm ci
COPY tools/ui/ ./
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build

@@ -21,2 +35,4 @@

COPY --from=web /app/tools/ui/dist tools/ui/dist
RUN cmake -B build -DGGML_NATIVE=OFF -DGGML_VULKAN=ON -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON && \

@@ -23,0 +39,0 @@ cmake --build build --config Release -j$(nproc)

+16
-0
vendor/llama.cpp/.devops/zendnn.Dockerfile

@@ -6,2 +6,16 @@ ARG UBUNTU_VERSION=24.04

ARG NODE_VERSION=24
FROM docker.io/node:$NODE_VERSION AS web
ARG APP_VERSION
WORKDIR /app/tools/ui
COPY tools/ui/package.json tools/ui/package-lock.json ./
RUN npm ci
COPY tools/ui/ ./
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build

@@ -18,2 +32,4 @@

COPY --from=web /app/tools/ui/dist tools/ui/dist
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_ZENDNN=ON && \

@@ -20,0 +36,0 @@ cmake --build build -j $(nproc)

+3
-0
vendor/llama.cpp/.dockerignore

@@ -13,2 +13,5 @@ *.o

tools/ui/node_modules/
tools/ui/dist/
models/*

@@ -15,0 +18,0 @@ 

+30
-2
vendor/llama.cpp/.github/workflows/build-cache.yml

@@ -71,4 +71,4 @@ name: Build Actions Cache

# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.0"
OPENVINO_VERSION_FULL: "2026.0.0.20965.c6d6a13a886"
OPENVINO_VERSION_MAJOR: "2026.2"
OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857"

@@ -95,2 +95,30 @@ steps:

windows-2022-openvino-cache:
runs-on: windows-2022
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.2"
OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857"
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Setup Cache
uses: actions/cache@v5
id: cache-openvino
with:
path: ./openvino_toolkit
key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}
- name: Setup OpenVINO Toolkit
if: steps.cache-openvino.outputs.cache-hit != 'true'
uses: ./.github/actions/windows-setup-openvino
with:
path: ./openvino_toolkit
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
version_full: ${{ env.OPENVINO_VERSION_FULL }}
windows-2022-rocm-cache:

@@ -97,0 +125,0 @@ runs-on: windows-2022

+81
-8
vendor/llama.cpp/.github/workflows/build-openvino.yml

@@ -40,10 +40,6 @@ name: CI (openvino)

concurrency:
group: openvino-gpu-${{ github.head_ref || github.ref }}
cancel-in-progress: false
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.0"
OPENVINO_VERSION_FULL: "2026.0.0.20965.c6d6a13a886"
OPENVINO_VERSION_MAJOR: "2026.2"
OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857"

@@ -82,3 +78,3 @@ steps:

-DGGML_OPENVINO=ON
time cmake --build build/ReleaseOV --config Release -j $(nproc)
time cmake --build build/ReleaseOV --config Release --parallel

@@ -98,2 +94,79 @@ - name: Test (CPU)

export GGML_OPENVINO_DEVICE=GPU
ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs" --verbose --timeout 2000
ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs" --verbose --timeout 3000
openvino-windows-2022:
runs-on: windows-2022
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.2"
OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857"
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
with:
key: openvino-windows-2022
variant: ccache
evict-old-files: 1d
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
- name: Setup Cache
uses: actions/cache@v5
id: cache-openvino
with:
path: ./openvino_toolkit
key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}
- name: Setup OpenVINO Toolkit
if: steps.cache-openvino.outputs.cache-hit != 'true'
uses: ./.github/actions/windows-setup-openvino
with:
path: ./openvino_toolkit
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
version_full: ${{ env.OPENVINO_VERSION_FULL }}
- name: Install OpenCL using vcpkg
shell: powershell
run: |
git clone https://github.com/microsoft/vcpkg C:\vcpkg
C:\vcpkg\bootstrap-vcpkg.bat
C:\vcpkg\vcpkg install opencl
- name: Build
id: cmake_build
shell: cmd
run: |
REM Find extracted OpenVINO folder dynamically
for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
if not exist "%OPENVINO_ROOT%\runtime\cmake\OpenVINOConfig.cmake" (
echo ERROR: OpenVINOConfig.cmake not found
exit /b 1
)
call "%OPENVINO_ROOT%\setupvars.bat"
cmake -B build\ReleaseOV -G "Visual Studio 17 2022" ^
-A x64 ^
-DCMAKE_BUILD_TYPE=Release ^
-DGGML_OPENVINO=ON ^
-DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake
cmake --build build\ReleaseOV --config Release -- /m
- name: Test (CPU)
id: cmake_test_cpu
shell: cmd
# TODO: fix and re-enable the `test-llama-archs` test below
run: |
REM Find extracted OpenVINO folder dynamically
for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
call "%OPENVINO_ROOT%\setupvars.bat"
cd build
ctest --test-dir ReleaseOV -L main -E "test-llama-archs" -C Release --verbose --timeout 3000
+2
-6
vendor/llama.cpp/.github/workflows/build-self-hosted.yml

@@ -267,10 +267,6 @@ name: CI (self-hosted)

concurrency:
group: openvino-gpu-${{ github.head_ref || github.ref }}
cancel-in-progress: false
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.0"
OPENVINO_VERSION_FULL: "2026.0.0.20965.c6d6a13a886"
OPENVINO_VERSION_MAJOR: "2026.2"
OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857"

@@ -277,0 +273,0 @@ steps:

+111
-4
vendor/llama.cpp/.github/workflows/release.yml

@@ -49,2 +49,4 @@ name: Release

- id: check
env:
COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
run: |

@@ -54,3 +56,3 @@ if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then

elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/master" ]]; then
if echo "${{ github.event.head_commit.message }}" | grep -q '\[no release\]'; then
if echo "$COMMIT_MESSAGE" | grep -q '\[no release\]'; then
echo "should_release=false" >> $GITHUB_OUTPUT

@@ -448,5 +450,5 @@ else

env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.0"
OPENVINO_VERSION_FULL: "2026.0.0.20965.c6d6a13a886"
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.2"
OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857"

@@ -534,2 +536,105 @@ steps:

windows-openvino:
runs-on: windows-2022
outputs:
openvino_version: ${{ steps.openvino_version.outputs.value }}
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
OPENVINO_VERSION_MAJOR: "2026.2"
OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857"
steps:
- name: Set OpenVINO version output
id: openvino_version
shell: bash
run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT
- name: Clone
id: checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: "24"
cache: "npm"
cache-dependency-path: "tools/ui/package-lock.json"
- name: ccache
uses: ggml-org/ccache-action@v1.2.21
with:
key: release-windows-2022-openvino
variant: ccache
evict-old-files: 1d
- name: Setup Cache
uses: actions/cache@v5
id: cache-openvino
with:
path: ./openvino_toolkit
key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}
- name: Setup OpenVINO Toolkit
if: steps.cache-openvino.outputs.cache-hit != 'true'
uses: ./.github/actions/windows-setup-openvino
with:
path: ./openvino_toolkit
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
version_full: ${{ env.OPENVINO_VERSION_FULL }}
- name: Install OpenCL using vcpkg
shell: powershell
run: |
git clone https://github.com/microsoft/vcpkg C:\vcpkg
C:\vcpkg\bootstrap-vcpkg.bat
C:\vcpkg\vcpkg install opencl
- name: Build
id: cmake_build
shell: cmd
run: |
REM Find extracted OpenVINO folder dynamically
for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
if not exist "%OPENVINO_ROOT%\runtime\cmake\OpenVINOConfig.cmake" (
echo ERROR: OpenVINOConfig.cmake not found
exit /b 1
)
call "%OPENVINO_ROOT%\setupvars.bat"
cmake -B build\ReleaseOV -G "Visual Studio 17 2022" ^
-A x64 ^
-DCMAKE_BUILD_TYPE=Release ^
-DGGML_OPENVINO=ON ^
-DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake
cmake --build build\ReleaseOV --config Release -- /m
- name: ccache-clear
uses: ./.github/actions/ccache-clear
with:
key: release-windows-2022-openvino
- name: Determine tag name
id: tag
uses: ./.github/actions/get-tag-name
- name: Pack artifacts
id: pack_artifacts
shell: powershell
run: |
Copy-Item LICENSE .\build\ReleaseOV\bin\
7z a -snl llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip .\build\ReleaseOV\bin\*
- name: Upload artifacts
uses: actions/upload-artifact@v6
with:
path: llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
name: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip
windows-cpu:

@@ -1410,2 +1515,3 @@ needs: [check-release]

- windows-hip
- windows-openvino
- ubuntu-22-rocm

@@ -1532,2 +1638,3 @@ - ubuntu-cpu

- [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
- [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip)
- [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip)

@@ -1534,0 +1641,0 @@ - [Windows x64 (HIP)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-hip-radeon-x64.zip)

+0
-10
vendor/llama.cpp/.pi/gg/SYSTEM.md

@@ -28,11 +28,1 @@ You are a coding agent. Here are some very important rules that you must follow:

- Never `git push` without explicit confirmation from the user
Resources (read on demand):
- [CONTRIBUTING.md](CONTRIBUTING.md)
- [Build documentation](docs/build.md)
- [Server usage documentation](tools/server/README.md)
- [Server development documentation](tools/server/README-dev.md)
- [PEG parser](docs/development/parsing.md)
- [Auto parser](docs/autoparser.md)
- [Jinja engine](common/jinja/README.md)
- [PR template](.github/pull_request_template.md)
+26
-13
vendor/llama.cpp/app/llama.cpp

@@ -23,3 +23,3 @@ #include "build-info.h"

// hands the update over to the install script, which downloads and swaps the binary
// Self-update is only supported for binaries built with llama-install.sh
static int llama_update(int argc, char ** argv) {

@@ -29,2 +29,3 @@ (void) argc;

#ifdef LLAMA_INSTALL_BUILD
#if defined(_WIN32)

@@ -35,2 +36,6 @@ return system("powershell -NoProfile -ExecutionPolicy Bypass -Command \"irm https://llama.app/install.ps1 | iex\"");

#endif
#else
printf("Updates are available only when installed from https://llama.app\n");
return 1;
#endif
}

@@ -52,17 +57,25 @@

#ifdef LLAMA_INSTALL_BUILD
#define UPDATE_HIDDEN false
#else
#define UPDATE_HIDDEN true
#endif
static const command cmds[] = {
{"serve", "HTTP API server", {"server"}, false, llama_server },
{"cli", "Command-line interactive interface", {"client"}, false, llama_cli },
{"update", "Update llama to the latest release", {}, false, llama_update },
{"completion", "Text completion", {"complete"}, true, llama_completion },
{"bench", "Benchmark prompt processing and text generation", {}, true, llama_bench },
{"batched-bench", "Benchmark batched decoding performance", {}, true, llama_batched_bench},
{"fit-params", "Compute parameters to fit a model in device memory", {}, true, llama_fit_params },
{"quantize", "Quantize a model", {}, true, llama_quantize },
{"perplexity", "Compute model perplexity and KL divergence", {}, true, llama_perplexity },
{"version", "Show version", {}, false, version },
{"licenses", "Show third-party licenses", {"credits"}, false, licenses },
{"help", "Show available commands", {}, false, help },
{"serve", "HTTP API server", {"server"}, false, llama_server },
{"cli", "Command-line interactive interface", {"client"}, false, llama_cli },
{"update", "Update llama to the latest release", {}, UPDATE_HIDDEN, llama_update },
{"completion", "Text completion", {"complete"}, true, llama_completion },
{"bench", "Benchmark prompt processing and text generation", {}, true, llama_bench },
{"batched-bench", "Benchmark batched decoding performance", {}, true, llama_batched_bench},
{"fit-params", "Compute parameters to fit a model in device memory", {}, true, llama_fit_params },
{"quantize", "Quantize a model", {}, true, llama_quantize },
{"perplexity", "Compute model perplexity and KL divergence", {}, true, llama_perplexity },
{"version", "Show version", {}, false, version },
{"licenses", "Show third-party licenses", {"credits"}, false, licenses },
{"help", "Show available commands", {}, false, help },
};
#undef UPDATE_HIDDEN
static int version(int argc, char ** argv) {

@@ -69,0 +82,0 @@ printf("%s\n", llama_build_info());

+23
-12
vendor/llama.cpp/common/common.h

@@ -298,3 +298,12 @@ // Various helper functions and utilities

std::string docker_repo = ""; // Docker repo // NOLINT
std::string name = ""; // in format <user>/<model>[:<tag>] (tag is optional) // NOLINT
std::string get_name() {
if (!hf_repo.empty()) {
return hf_repo;
}
if (!docker_repo.empty()) {
return docker_repo;
}
return path;
}
};

@@ -367,3 +376,3 @@

bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP;
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3;
});

@@ -629,8 +638,2 @@

bool ui = true;
// Deprecated: use ui, ui_mcp_proxy, ui_config_json instead
bool webui = ui;
bool webui_mcp_proxy = false;
std::string webui_config_json;
bool ui_mcp_proxy = false;

@@ -648,6 +651,7 @@ std::string ui_config_json;

// router server configs
std::string models_dir = ""; // directory containing models for the router server
std::string models_preset = ""; // directory containing model presets for the router server
int models_max = 4; // maximum number of models to load simultaneously
bool models_autoload = true; // automatically load models when requested via the router server
std::string models_dir = ""; // directory containing models for the router server
std::string models_preset = ""; // directory containing model presets for the router server
int models_max = 4; // maximum number of models to load simultaneously
bool models_autoload = true; // automatically load models when requested via the router server
std::string models_preset_hf = ""; // show a warning about remote presets on router loaded (if not empty)

@@ -854,2 +858,5 @@ bool log_json = false;

// fs open, also handle UTF8 on Windows
std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode);
//

@@ -1072,2 +1079,6 @@ // TTY utils

// (optional) speculative-decoding implementation state stashed with the checkpoint
// (e.g. eagle3's deferred-boundary g_embd row)
std::vector<uint8_t> data_spec;
size_t size() const;

@@ -1074,0 +1085,0 @@ 

+120
-17
vendor/llama.cpp/common/download.cpp

@@ -699,2 +699,3 @@ #include "arg.h"

hf_cache::hf_file mtp;
hf_cache::hf_file preset; // if set, only this file is downloaded
};

@@ -721,2 +722,10 @@

// if preset.ini exists in the repo root, download only that file
for (const auto & f : all) {
if (f.path == "preset.ini") {
plan.preset = f;
return plan;
}
}
hf_cache::hf_file primary;

@@ -799,11 +808,16 @@

hf = get_hf_plan(model, opts, download_mmproj, download_mtp);
for (const auto & f : hf.model_files) {
tasks.push_back({f.url, f.local_path});
if (!hf.preset.path.empty()) {
// if preset.ini exists, only download that file alone
tasks.push_back({hf.preset.url, hf.preset.local_path});
} else {
for (const auto & f : hf.model_files) {
tasks.push_back({f.url, f.local_path});
}
if (!hf.mmproj.path.empty()) {
tasks.push_back({hf.mmproj.url, hf.mmproj.local_path});
}
if (!hf.mtp.path.empty()) {
tasks.push_back({hf.mtp.url, hf.mtp.local_path});
}
}
if (!hf.mmproj.path.empty()) {
tasks.push_back({hf.mmproj.url, hf.mmproj.local_path});
}
if (!hf.mtp.path.empty()) {
tasks.push_back({hf.mtp.url, hf.mtp.local_path});
}
} else if (!model.url.empty()) {

@@ -841,13 +855,18 @@ tasks = get_url_tasks(model);

if (is_hf) {
for (const auto & f : hf.model_files) {
hf_cache::finalize_file(f);
}
result.model_path = hf.primary.final_path;
if (!hf.preset.path.empty()) {
// if preset.ini is used, do not set other paths
result.preset_path = hf_cache::finalize_file(hf.preset);
} else {
for (const auto & f : hf.model_files) {
hf_cache::finalize_file(f);
}
result.model_path = hf.primary.final_path;
if (!hf.mmproj.path.empty()) {
result.mmproj_path = hf_cache::finalize_file(hf.mmproj);
}
if (!hf.mmproj.path.empty()) {
result.mmproj_path = hf_cache::finalize_file(hf.mmproj);
}
if (!hf.mtp.path.empty()) {
result.mtp_path = hf_cache::finalize_file(hf.mtp);
if (!hf.mtp.path.empty()) {
result.mtp_path = hf_cache::finalize_file(hf.mtp);
}
}

@@ -1004,1 +1023,85 @@ } else {

}
bool common_download_remove(const std::string & hf_repo_with_tag) {
namespace fs = std::filesystem;
auto [repo_id, tag] = common_download_split_repo_tag(hf_repo_with_tag);
if (tag.empty()) {
return hf_cache::remove_cached_repo(repo_id);
}
std::string tag_upper = tag;
for (char & c : tag_upper) {
c = (char) std::toupper((unsigned char) c);
}
auto files = hf_cache::get_cached_files(repo_id);
if (files.empty()) {
return false;
}
// collect snapshot entries whose tag matches
std::vector<fs::path> to_remove;
for (const auto & f : files) {
auto split = get_gguf_split_info(f.path);
if (split.tag == tag_upper) {
to_remove.emplace_back(f.local_path);
}
}
if (to_remove.empty()) {
return false;
}
// resolve blob paths from symlinks before deleting snapshot entries
std::vector<fs::path> blobs_to_check;
for (const auto & p : to_remove) {
std::error_code ec;
if (fs::is_symlink(p, ec)) {
auto target = fs::read_symlink(p, ec);
if (!ec) {
blobs_to_check.push_back((p.parent_path() / target).lexically_normal());
}
}
}
// remove snapshot entries
for (const auto & p : to_remove) {
std::error_code ec;
fs::remove(p, ec);
if (ec) {
LOG_WRN("%s: failed to remove %s: %s\n", __func__, p.string().c_str(), ec.message().c_str());
}
}
if (blobs_to_check.empty()) {
return true;
}
// collect blobs still referenced by remaining snapshot entries
std::unordered_set<std::string> still_referenced;
for (const auto & f : hf_cache::get_cached_files(repo_id)) {
fs::path p(f.local_path);
std::error_code ec;
if (fs::is_symlink(p, ec)) {
auto target = fs::read_symlink(p, ec);
if (!ec) {
still_referenced.insert((p.parent_path() / target).lexically_normal().string());
}
}
}
// remove orphaned blobs
for (const auto & blob : blobs_to_check) {
if (still_referenced.find(blob.string()) == still_referenced.end()) {
std::error_code ec;
fs::remove(blob, ec);
if (ec) {
LOG_WRN("%s: failed to remove blob %s: %s\n", __func__, blob.string().c_str(), ec.message().c_str());
}
}
}
return true;
}
+8
-0
vendor/llama.cpp/common/download.h

@@ -66,2 +66,3 @@ #pragma once

std::string mtp_path;
std::string preset_path;
};

@@ -119,1 +120,8 @@

std::string common_docker_resolve_model(const std::string & docker);
// Remove a cached model from disk
// input format: "user/model" or "user/model:tag"
// - if tag is omitted, removes the entire repo cache directory
// - if tag is present, removes only files matching that tag (and orphaned blobs)
// returns true if anything was removed
bool common_download_remove(const std::string & hf_repo_with_tag);
+15
-0
vendor/llama.cpp/common/hf-cache.cpp

@@ -498,2 +498,17 @@ #include "hf-cache.h"

bool remove_cached_repo(const std::string & repo_id) {
if (!is_valid_repo_id(repo_id)) {
LOG_WRN("%s: invalid repository: %s\n", __func__, repo_id.c_str());
return false;
}
fs::path repo_path = get_repo_path(repo_id);
std::error_code ec;
auto removed = fs::remove_all(repo_path, ec);
if (ec) {
LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());
return false;
}
return removed > 0;
}
} // namespace hf_cache
+3
-0
vendor/llama.cpp/common/hf-cache.h

@@ -32,2 +32,5 @@ #pragma once

// Remove the entire cached directory for a repo, returns true if removed
bool remove_cached_repo(const std::string & repo_id);
} // namespace hf_cache
+81
-75
vendor/llama.cpp/common/log.cpp

@@ -14,4 +14,9 @@ #include "common.h"

#include <vector>
#include <algorithm>
#if defined(_WIN32)
# define WIN32_LEAN_AND_MEAN
# ifndef NOMINMAX
# define NOMINMAX
# endif
# include <io.h>

@@ -66,13 +71,12 @@ # include <windows.h>

struct common_log_entry {
enum ggml_log_level level;
enum ggml_log_level level {GGML_LOG_LEVEL_INFO};
bool prefix;
std::vector<char> msg;
int64_t timestamp;
int64_t timestamp { 0 };
bool is_end { false }; // signals the worker thread to stop
bool prefix { false };
std::vector<char> msg;
common_log_entry(size_t size = 256) : msg(size) { }
// signals the worker thread to stop
bool is_end;
void print(FILE * file = nullptr) const {

@@ -127,18 +131,11 @@ FILE * fcur = file;

struct common_log {
// default capacity - will be expanded if needed
common_log() : common_log(256) {}
common_log(size_t capacity) {
file = nullptr;
prefix = false;
// default capacity
common_log(size_t capacity = 512) {
file = nullptr;
prefix = false;
timestamps = false;
running = false;
t_start = t_us();
running = false;
t_start = t_us();
// initial message size - will be expanded if longer messages arrive
entries.resize(capacity);
for (auto & entry : entries) {
entry.msg.resize(256);
}
queue.resize(capacity, common_log_entry(256));
head = 0;

@@ -158,5 +155,6 @@ tail = 0;

private:
std::mutex mtx;
std::thread thrd;
std::condition_variable cv;
std::mutex mtx;
std::thread thrd;
std::condition_variable cv_new; // new entry
std::condition_variable cv_full; // wait on full

@@ -171,14 +169,43 @@ FILE * file;

// ring buffer of entries
std::vector<common_log_entry> entries;
// queue of entries
std::vector<common_log_entry> queue;
size_t head;
size_t tail;
// worker thread copies into this
common_log_entry cur;
bool print_entry(const common_log_entry & e) const {
if (e.is_end) return true;
e.print();
if (file) {
e.print(file);
}
return false;
}
bool flush_queue(size_t start_head, size_t end_tail, size_t & out_head) const {
bool stop = false;
size_t h = start_head;
while (h != end_tail && !stop) {
stop = print_entry(queue[h]);
h = (h + 1) % queue.size();
}
out_head = h;
return stop;
}
public:
bool is_full() const {
return ((tail + 1) % queue.size()) == head;
}
bool is_empty() const {
return head == tail;
}
void add(enum ggml_log_level level, const char * fmt, va_list args) {
std::lock_guard<std::mutex> lock(mtx);
std::unique_lock<std::mutex> lock(mtx);
// block if the queue is full
cv_full.wait(lock, [this]() { return !running || !is_full(); });
if (!running) {

@@ -189,3 +216,3 @@ // discard messages while the worker thread is paused

auto & entry = entries[tail];
auto & entry = queue[tail];

@@ -225,4 +252,5 @@ {

entry.level = level;
entry.prefix = prefix;
entry.is_end = false;
entry.level = level;
entry.prefix = prefix;
entry.timestamp = 0;

@@ -232,28 +260,5 @@ if (timestamps) {

}
entry.is_end = false;
tail = (tail + 1) % entries.size();
if (tail == head) {
// expand the buffer
std::vector<common_log_entry> new_entries(2*entries.size());
size_t new_tail = 0;
do {
new_entries[new_tail] = std::move(entries[head]);
head = (head + 1) % entries.size();
new_tail = (new_tail + 1);
} while (head != tail);
head = 0;
tail = new_tail;
for (size_t i = tail; i < new_entries.size(); i++) {
new_entries[i].msg.resize(256);
}
entries = std::move(new_entries);
}
cv.notify_one();
tail = (tail + 1) % queue.size();
cv_new.notify_one();
}

@@ -272,18 +277,19 @@

while (true) {
{
std::unique_lock<std::mutex> lock(mtx);
cv.wait(lock, [this]() { return head != tail; });
cur = entries[head];
std::unique_lock<std::mutex> lock(mtx);
cv_new.wait(lock, [this]() { return !is_empty(); });
head = (head + 1) % entries.size();
}
size_t cached_head = head;
size_t cached_tail = tail;
if (cur.is_end) {
break;
}
lock.unlock(); // drop the lock during flush
cur.print(); // stdout and stderr
size_t next_head;
bool stop = flush_queue(cached_head, cached_tail, next_head);
if (file) {
cur.print(file);
lock.lock();
head = next_head;
cv_full.notify_all();
if (stop) {
break;
}

@@ -305,9 +311,9 @@ }

// push an entry to signal the worker thread to stop
{
auto & entry = entries[tail];
entry.is_end = true;
auto & entry = queue[tail];
entry.is_end = true;
tail = (tail + 1) % queue.size();
tail = (tail + 1) % entries.size();
}
cv.notify_one();
// wakeup everyone
cv_new.notify_one();
cv_full.notify_all();
}

@@ -314,0 +320,0 @@ 

+1
-49
vendor/llama.cpp/common/preset.cpp

@@ -19,44 +19,2 @@ #include "arg.h"

// only allow a subset of args for remote presets for security reasons
// do not add more args unless absolutely necessary
// args that output to files are strictly prohibited
static std::set<std::string> get_remote_preset_whitelist(const std::map<std::string, common_arg> & key_to_opt) {
static const std::set<std::string> allowed_options = {
"model-url",
"hf-repo",
"hf-repo-draft",
"hf-repo-v", // vocoder
"hf-file-v", // vocoder
"mmproj-url",
"pooling",
"jinja",
"batch-size",
"ubatch-size",
"cache-reuse",
"chat-template-kwargs",
"mmap",
// note: sampling params are automatically allowed by default
// negated args will be added automatically if the positive arg is specified above
};
std::set<std::string> allowed_keys;
for (const auto & it : key_to_opt) {
const std::string & key = it.first;
const common_arg & opt = it.second;
if (allowed_options.find(key) != allowed_options.end() || opt.is_sampling) {
allowed_keys.insert(key);
// also add variant keys (args without leading dashes and env vars)
for (const auto & arg : opt.get_args()) {
allowed_keys.insert(rm_leading_dashes(arg));
}
for (const auto & env : opt.get_env()) {
allowed_keys.insert(env);
}
}
}
return allowed_keys;
}
std::vector<std::string> common_preset::to_args(const std::string & bin_path) const {

@@ -304,12 +262,6 @@ std::vector<std::string> args;

common_preset_context::common_preset_context(llama_example ex, bool only_remote_allowed)
common_preset_context::common_preset_context(llama_example ex)
: ctx_params(common_params_parser_init(default_params, ex)) {
common_params_add_preset_options(ctx_params.options);
key_to_opt = get_map_key_opt(ctx_params);
// setup allowed keys if only_remote_allowed is true
if (only_remote_allowed) {
filter_allowed_keys = true;
allowed_keys = get_remote_preset_whitelist(key_to_opt);
}
}

@@ -316,0 +268,0 @@ 

+1
-1
vendor/llama.cpp/common/preset.h

@@ -63,3 +63,3 @@ #pragma once

// if only_remote_allowed is true, only accept whitelisted keys
common_preset_context(llama_example ex, bool only_remote_allowed = false);
common_preset_context(llama_example ex);

@@ -66,0 +66,0 @@ // load presets from INI file

+3
-0
vendor/llama.cpp/common/sampling.cpp

@@ -262,2 +262,5 @@ #include "sampling.h"

}
if (!grmr && !grammar_str.empty()) {
throw std::runtime_error("failed to parse grammar");
}

@@ -264,0 +267,0 @@ // Compute prefill tokens from the generation prompt

+4
-0
vendor/llama.cpp/common/speculative.h

@@ -71,2 +71,6 @@ #pragma once

// (optional) get/set internal state
bool common_speculative_get_state(common_speculative * spec, llama_seq_id seq_id, std::vector<uint8_t> & data);
void common_speculative_set_state(common_speculative * spec, llama_seq_id seq_id, const std::vector<uint8_t> & data);
// print statistics about the speculative decoding

@@ -73,0 +77,0 @@ void common_speculative_print_stats(const common_speculative * spec);

+534
-131
vendor/llama.cpp/docs/backend/OPENVINO.md

@@ -15,2 +15,21 @@ # OpenVINO Backend for llama.cpp

## Contents
- [Supported Devices](#supported-devices)
- [Supported Model Precisions](#supported-model-precisions)
- [Supported Llama.cpp Tools](#supported-llamacpp-tools)
- [Validated Models](#validated-models)
- [Build Instructions](#build-instructions)
- [0. Prerequisites](#0-prerequisites)
- [1. Install OpenVINO Runtime](#1-install-openvino-runtime)
- [2. Build llama.cpp with OpenVINO Backend](#2-build-llamacpp-with-openvino-backend)
- [Automated Ubuntu Build Script](#automated-ubuntu-build-script)
- [Automated Windows Build Script](#automated-windows-build-script)
- [3. Download Sample Model](#3-download-sample-model)
- [4. Run Inference with OpenVINO Backend](#4-run-inference-with-openvino-backend)
- [5. Docker Build](#5-docker-build)
- [GGML OpenVINO Backend Runtime Configurations](#ggml-openvino-backend-runtime-configurations)
- [Known Limitations](#known-limitations)
- [Work in Progress](#work-in-progress)
## Supported Devices

@@ -35,4 +54,4 @@

- `Q4_K_M`
- `Q5_K` (converted to Q8_0_C at runtime)
- `Q6_K` (converted to Q8_0_C at runtime)
- `Q5_K` (converted to `Q8_0_C` at runtime)
- `Q6_K` (converted to `Q8_0_C` at runtime)

@@ -42,45 +61,92 @@ > [!NOTE]

## Quantization Support Details
### CPU and GPU
- **`Q4_0`, `Q4_1`, `Q4_K_M`, `Q6_K` models are supported**
**CPU and GPU Quantization Details:**
- `Q5_K` and `Q6_K` tensors are converted to `Q8_0_C`
### NPU
- **Primary supported quantization scheme is `Q4_0`**
**NPU Quantization Details:**
- Primary supported quantization scheme is `Q4_0`
- `Q6_K` tensors are requantized to `Q4_0_128` in general. For embedding weights, `Q6_K` tensors are requantized to `Q8_0_C` except for the token embedding matrix which is dequantized to fp16
### Additional Notes
**Additional Notes:**
- Both `Q4_0` and `Q4_1` models use `Q6_K` for the token embedding tensor and the final matmul weight tensor (often the same tensor)
- `Q4_0` models may produce some `Q4_1` tensors if an imatrix is provided during quantization using `llama-quantize`
- `Q4_K_M` models may include both `Q6_K` and `Q5_K` tensors (observed in Phi-3)
- `Q5_1` tensors are dequantized natively (weights, scales, and zero-points extracted directly)
## Supported Llama.cpp Tools
The OpenVINO backend integrates with the standard llama.cpp tools listed below.
However, all the tools coverage across all devices is not uniform and exhaustive validation is work in progress.
- llama-bench
- llama-cli
- llama-completion
- llama-embedding
- llama-perplexity
- llama-run
- llama-server
- llama-simple
## Validated Models
The following models were validated on Intel® Core™ Ultra Series 2. While our testing was limited, the OpenVINO backend is expected to work across a broad range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html).
- Use `GGML_OPENVINO_STATEFUL_EXECUTION=1` when using GPU device.
- `-fa 1` is required when running llama-bench with the OpenVINO backend.
- Additional model support, quantization formats and validations are work in progress.
Although, the validated models below were tested with `llama-cli` using the `Q4_K_M` quantization format on Intel® Core™ Ultra Series 2 (Lunar Lake), the OpenVINO backend is expected to work across a broader range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), [supported model precisions](#supported-model-precisions), [supported llama.cpp tools](#supported-llamacpp-tools) and additional model architectures.
| Model | Validated | Known Issues |
| :------| :---------- | :-------------|
| [Llama-3.2-1B-Instruct](https://huggingface.co/unsloth/Llama-3.2-1B-Instruct-GGUF/) | `FP16`, `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/GPU/NPU | — |
| [Meta-Llama-3.1-8B-Instruct](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | `Q8_0`, `Q4_K_M` on CPU/GPU/NPU | `Q4_0_8_8`, `Q4_0_4_8`, `Q4_0_4_4` fail |
| [Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf) | `FP16`, `Q4` on CPU/NPU | GPU unsupported for `FP16` and `Q4` (`llama-cli`, `llama-bench`) |
| [Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | `FP16`, `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/GPU/NPU | — |
| [Qwen3-8B-Instruct](https://huggingface.co/Qwen/Qwen3-8B-GGUF) | `FP16`, `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/NPU; GPU works via `llama-bench` | GPU `llama-cli` unsupported for all quantizations |
| [MiniCPM-V-2_6-GGUF](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `Q4_0` on CPU/GPU/NPU | — |
| [DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/GPU/NPU | — |
| [Hunyuan-7B-Instruct](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | CPU: `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M`; GPU: `Q8_0`, `Q4_0`, `Q4_1`; NPU (`llama-bench` only): `Q4_0`, `Q4_1`, `Q4_K_M` | GPU `Q4_K_M` unsupported; NPU `llama-cli` unsupported |
| [Mistral-7B-Instruct-v0.3](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF/) | CPU/GPU: `Q8_0`, `Q4_K_M`; NPU: `Q8_0`, `Q4_K_M` (via `llama-bench`) | NPU `llama-cli` unsupported for `Q8_0`, `Q4_K_M` |
> [!NOTE]
> Extensive accuracy validation, performance optimizations, and broader architecture coverage are work in progress.
**Legend & Test Configuration:**
- **Status:** ✓ = Passed | ✗ = Failed or Unsupported
- **Execution Modes:**
- **SL** = Stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`)
- **SF** = Stateful (`GGML_OPENVINO_STATEFUL_EXECUTION=1`)
- Note: The NPU operates in stateless mode only.
- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel OpenCL GPU Driver 26.18.38308.1 | Intel NPU Driver 1.33.0.
- See [Known Limitations](#known-limitations) for context on observed failures.
| Model | CPU (SL / SF) | GPU (SL / SF) | NPU (SL) |
| :--- | :---: | :---: | :---: |
| [bartowski/Llama-3.2-1B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/Llama-3.2-3B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/Meta-Llama-3.1-8B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✗ | ✓ |
| | | | |
| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
| | | | |
| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| | | | |
| [bartowski/Mistral-7B-Instruct-v0.3-Q4_K_M](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [QuantFactory/Ministral-3b-instruct.Q4_K_M](https://huggingface.co/QuantFactory/Ministral-3b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/Ministral-8B-Instruct-2410-Q4_K_M](https://huggingface.co/bartowski/Ministral-8B-Instruct-2410-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| | | | |
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ |
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ / ✓ | ✗ / ✗ | ✗ |
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ / ✓ | ✓ / ✗ | ✓ |
| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
| | | | |
| [gpustack/bge-m3-Q4_K_M.gguf](https://huggingface.co/gpustack/bge-m3-GGUF) | ✓ | ✗ | ✗ |
## Build Instructions
### Prerequisites
### 0. Prerequisites
- Linux or Windows system with Intel hardware (CPU, GPU, or NPU)
- **For Intel GPU or NPU Usage**: Install the appropriate hardware drivers for your Intel GPU or NPU. For detailed instructions, see: [Additional Configurations for Hardware Acceleration](https://docs.openvino.ai/2025/get-started/install-openvino/configurations.html).
- **For Intel GPU or NPU Usage**: Install the appropriate hardware drivers for your Intel GPU or NPU. For detailed instructions, see: [Additional Configurations for Hardware Acceleration](https://docs.openvino.ai/2026/get-started/install-openvino/configurations.html).

@@ -125,50 +191,372 @@ - **Linux:**

- Verify OpenVINO is initialized properly:
```bash
echo $OpenVINO_DIR
```
### 2. Build llama.cpp with OpenVINO Backend
Clone llama.cpp repo and build :
```bash
git clone https://github.com/ggml-org/llama.cpp
cd llama.cpp
```
- **Linux:**
```bash
source /opt/intel/openvino/setupvars.sh
cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON
cmake --build build/ReleaseOV --parallel
```
<details>
<summary>📦 Click to expand OpenVINO installation from an archive file on Ubuntu</summary>
<br>
- **Windows:** Open a **Developer Command Prompt for VS 2022** (so the MSVC toolchain is on `PATH`), then run:
```bash
wget https://raw.githubusercontent.com/ravi9/misc-scripts/main/openvino/ov-archive-install/install-openvino-from-archive.sh
chmod +x install-openvino-from-archive.sh
./install-openvino-from-archive.sh
```
```cmd
C:\Intel\openvino\setupvars.bat
cmake -B build\ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake
cmake --build build\ReleaseOV --parallel
```
Verify OpenVINO is initialized properly:
```bash
echo $OpenVINO_DIR
```
</details>
> [!NOTE]
> The Windows install path is `C:\Intel\openvino` (no spaces) to avoid quoting problems some CMake/Ninja toolchains have with `C:\Program Files (x86)\...`. Adjust to wherever you installed OpenVINO Runtime. From `cmd`, run `C:\Intel\openvino\setupvars.bat`; from PowerShell, run `& "C:\Intel\openvino\setupvars.ps1"` instead. Once the build is finished you can launch the binaries from any `cmd` or `PowerShell` window after sourcing the matching `setupvars` script for that shell.
#### Automated Ubuntu Build Script
### 2. Build llama.cpp with OpenVINO Backend
For Ubuntu24 users, the following shell script automates the prerequisite installs (build tools, OpenCL ICD), the OpenVINO Runtime download/extract/setup, and the Ninja-based llama.cpp build.
Save the following as `ubuntu-llamacpp-ov-install.sh` next to where you want the `llama.cpp` folder to land, then run it:
Clone the OpenVINO-enabled llama.cpp fork and build it:
```bash
chmod +x ubuntu-llamacpp-ov-install.sh
./ubuntu-llamacpp-ov-install.sh
```
<details>
<summary>Click to expand <code>ubuntu-llamacpp-ov-install.sh</code></summary>
```bash
git clone https://github.com/ggml-org/llama.cpp
cd llama.cpp
#!/usr/bin/env bash
# ============================================
# llama.cpp OpenVINO Build Script (Ninja)
# ============================================
set -euo pipefail
OPENVINO_VERSION_MAJOR="2026.2"
OPENVINO_VERSION_FULL="2026.2.0.21903.52ddc073857"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OPENVINO_INSTALL_DIR="/opt/intel/openvino_${OPENVINO_VERSION_MAJOR}"
OPENVINO_LINK_DIR="/opt/intel/openvino"
OPENVINO_TGZ="${SCRIPT_DIR}/openvino.tgz"
OPENVINO_URL="https://storage.openvinotoolkit.org/repositories/openvino/packages/${OPENVINO_VERSION_MAJOR}/linux/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz"
echo "============================================"
echo "Installing prerequisites (apt)..."
echo "============================================"
sudo apt-get update
sudo apt-get install -y \
build-essential libcurl4-openssl-dev libtbb12 \
cmake ninja-build python3-pip \
curl wget tar git
echo "============================================"
echo "Installing OpenCL runtime + headers..."
echo "============================================"
sudo apt-get install -y \
ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd
cd "${SCRIPT_DIR}"
# ============================================
# Clone llama.cpp if missing
# ============================================
if [[ ! -f "llama.cpp/CMakeLists.txt" ]]; then
echo "Cloning llama.cpp..."
git clone https://github.com/ggml-org/llama.cpp
fi
# ============================================
# Setup OpenVINO: download & extract to /opt/intel/openvino_${OPENVINO_VERSION_MAJOR},
# then point /opt/intel/openvino at it via symlink so the active version is swappable.
# ============================================
if [[ -f "${OPENVINO_INSTALL_DIR}/setupvars.sh" ]]; then
echo "OpenVINO ${OPENVINO_VERSION_MAJOR} already installed at ${OPENVINO_INSTALL_DIR}. Skipping download."
else
echo "OpenVINO not found at ${OPENVINO_INSTALL_DIR}. Starting download..."
curl -L -o "${OPENVINO_TGZ}" "${OPENVINO_URL}"
echo "Extracting OpenVINO to ${OPENVINO_INSTALL_DIR}..."
sudo mkdir -p "${OPENVINO_INSTALL_DIR}"
sudo tar -xzf "${OPENVINO_TGZ}" -C "${OPENVINO_INSTALL_DIR}" --strip-components=1
rm -f "${OPENVINO_TGZ}"
fi
# Refresh symlink: /opt/intel/openvino -> /opt/intel/openvino_${OPENVINO_VERSION_MAJOR}
sudo ln -sfn "${OPENVINO_INSTALL_DIR}" "${OPENVINO_LINK_DIR}"
OPENVINO_ROOT="${OPENVINO_LINK_DIR}"
echo "OpenVINO Ready: ${OPENVINO_ROOT} -> ${OPENVINO_INSTALL_DIR}"
# Install OpenVINO's own runtime dependencies (one-time per system).
if [[ -x "${OPENVINO_ROOT}/install_dependencies/install_openvino_dependencies.sh" ]]; then
echo "============================================"
echo "Installing OpenVINO runtime dependencies..."
echo "============================================"
echo "Y" | sudo -E "${OPENVINO_ROOT}/install_dependencies/install_openvino_dependencies.sh"
fi
# ============================================
# Clean old build cache
# ============================================
cd "${SCRIPT_DIR}/llama.cpp"
if [[ -d "build/ReleaseOV" ]]; then
echo "Removing old build directory..."
rm -rf "build/ReleaseOV"
fi
echo "============================================"
echo "Configuring with CMake..."
echo "============================================"
# shellcheck disable=SC1091
source "${OPENVINO_ROOT}/setupvars.sh"
cmake -B build/ReleaseOV -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DGGML_OPENVINO=ON
cmake --build build/ReleaseOV --parallel
echo "============================================"
echo "Build completed successfully!"
echo "============================================"
echo "Binaries: $(pwd)/build/ReleaseOV/bin"
echo
echo "NOTE: To run, source setupvars.sh and pick a device:"
echo " source /opt/intel/openvino/setupvars.sh"
echo " export GGML_OPENVINO_DEVICE=CPU # or GPU / NPU"
echo " ./build/ReleaseOV/bin/llama-cli -m model.gguf"
```
- **Linux:**
```bash
source /opt/intel/openvino/setupvars.sh
cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON
cmake --build build/ReleaseOV --parallel
```
> [!NOTE]
> The script pins OpenVINO `2026.2` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release.
- **Windows:**
```cmd
# x64 Native Tools Command Prompt for VS 2022
"C:\Program Files (x86)\Intel\openvino_2026.0\setupvars.bat"
cmake -B build\ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON -DLLAMA_CURL=OFF -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake
cmake --build build\ReleaseOV --parallel
```
</details>
#### Automated Windows Build Script
For Windows users, the following `.bat` script automates the prerequisite installs (Git, Ninja, CMake, Visual Studio 2022 Build Tools, vcpkg + OpenCL), the OpenVINO Runtime download/extract, and the Ninja-based llama.cpp build.
Save the following as `windows-llamacpp-ov-install.bat` next to where you want the `llama.cpp` to land, then run it from either **Command Prompt** or **PowerShell**:
```cmd
:: Command Prompt
windows-llamacpp-ov-install.bat
```
```powershell
# PowerShell
.\windows-llamacpp-ov-install.bat
```
<details>
<summary>Click to expand <code>windows-llamacpp-ov-install.bat</code></summary>
```bat
@echo off
setlocal enabledelayedexpansion
REM ============================================
REM llama.cpp OpenVINO Build Script (Ninja)
REM ============================================
set "OPENVINO_VERSION_MAJOR=2026.2"
set "OPENVINO_VERSION_FULL=2026.2.0.21903.52ddc073857"
set "SCRIPT_DIR=%~dp0"
set "VCPKG_DIR=C:\vcpkg"
set "OPENVINO_INSTALL_DIR=C:\Intel\openvino_%OPENVINO_VERSION_MAJOR%"
set "OPENVINO_LINK_DIR=C:\Intel\openvino"
set "OPENVINO_ZIP=%SCRIPT_DIR%openvino.zip"
set "OPENVINO_EXTRACT_TMP=%SCRIPT_DIR%openvino_extract_tmp"
set "OPENVINO_URL=https://storage.openvinotoolkit.org/repositories/openvino/packages/%OPENVINO_VERSION_MAJOR%/windows/openvino_toolkit_windows_%OPENVINO_VERSION_FULL%_x86_64.zip"
echo ============================================
echo Installing prerequisites...
echo ============================================
winget install --id Git.Git -e --accept-source-agreements --accept-package-agreements 2>nul
winget install --id Ninja-build.Ninja -e --accept-source-agreements --accept-package-agreements 2>nul
winget install --id Kitware.CMake -e --accept-source-agreements --accept-package-agreements 2>nul
REM Ensure Visual Studio Build Tools are installed.
echo Checking for Visual Studio Build Tools...
set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
set "VS_INSTALLED="
if exist "%VSWHERE%" (
for /f "usebackq tokens=*" %%i in (`"%VSWHERE%" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath 2^>nul`) do (
set "VS_INSTALLED=%%i"
)
)
if defined VS_INSTALLED (
echo Visual Studio with VC++ x86/x64 tools already present at "!VS_INSTALLED!". Skipping winget install.
) else (
winget install --id Microsoft.VisualStudio.2022.BuildTools -e --override "--wait --passive --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended" --accept-source-agreements --accept-package-agreements
if errorlevel 1 (
echo WARNING: winget could not install Visual Studio Build Tools automatically.
echo Install manually from https://aka.ms/vs/17/release/vs_BuildTools.exe ^(select the "Desktop development with C++" workload^)
echo and re-run this script from a "Developer Command Prompt for VS 2022".
)
)
echo ============================================
echo Installing OpenCL via vcpkg...
echo ============================================
if not exist "%VCPKG_DIR%" (
git clone https://github.com/microsoft/vcpkg "%VCPKG_DIR%"
cd /d "%VCPKG_DIR%"
call bootstrap-vcpkg.bat
call vcpkg integrate install
)
cd /d "%VCPKG_DIR%"
call vcpkg install opencl
cd /d "%SCRIPT_DIR%"
REM ============================================
REM Clone llama.cpp if missing
REM ============================================
if not exist "llama.cpp\CMakeLists.txt" (
echo Cloning llama.cpp...
git clone https://github.com/ggml-org/llama.cpp
)
cd /d "llama.cpp"
set "SCRIPT_DIR=%CD%"
REM ============================================
REM Setup OpenVINO: download & extract to C:\Intel\openvino_%OPENVINO_VERSION_MAJOR%,
REM then point C:\Intel\openvino at it via a directory junction (mklink /J).
REM ============================================
if exist "%OPENVINO_INSTALL_DIR%\setupvars.bat" (
echo OpenVINO %OPENVINO_VERSION_MAJOR% already installed at "%OPENVINO_INSTALL_DIR%". Skipping download.
) else (
echo OpenVINO not found at "%OPENVINO_INSTALL_DIR%". Starting download...
curl -L -o "%OPENVINO_ZIP%" "%OPENVINO_URL%"
if errorlevel 1 (
echo ERROR: Download failed.
exit /b 1
)
echo Extracting OpenVINO...
if exist "%OPENVINO_EXTRACT_TMP%" rmdir /s /q "%OPENVINO_EXTRACT_TMP%"
mkdir "%OPENVINO_EXTRACT_TMP%"
tar -xf "%OPENVINO_ZIP%" -C "%OPENVINO_EXTRACT_TMP%"
if errorlevel 1 (
echo ERROR: Extraction failed.
exit /b 1
)
REM Move the single top-level folder contents into the versioned install dir.
REM NOTE: delayed expansion (!VAR!) is required because the surrounding else( ... )
REM block is parsed once up-front, so %OPENVINO_EXTRACTED% would expand to "" here
REM and xcopy would then treat "\*" as C:\* and fail with "Cannot perform a cyclic copy".
set "OPENVINO_EXTRACTED="
for /d %%i in ("%OPENVINO_EXTRACT_TMP%\*") do set "OPENVINO_EXTRACTED=%%i"
if not defined OPENVINO_EXTRACTED (
echo ERROR: Could not locate extracted OpenVINO folder under "%OPENVINO_EXTRACT_TMP%".
exit /b 1
)
if not exist "%OPENVINO_INSTALL_DIR%" mkdir "%OPENVINO_INSTALL_DIR%"
xcopy /e /i /y /q "!OPENVINO_EXTRACTED!\*" "%OPENVINO_INSTALL_DIR%\" >nul
if errorlevel 1 (
echo ERROR: Failed to copy OpenVINO from "!OPENVINO_EXTRACTED!" to "%OPENVINO_INSTALL_DIR%".
echo Re-run this script from an elevated Command Prompt ^(Run as administrator^) if access is denied.
exit /b 1
)
rmdir /s /q "%OPENVINO_EXTRACT_TMP%"
del "%OPENVINO_ZIP%"
)
REM Refresh junction: C:\Intel\openvino -> C:\Intel\openvino_<version>.
REM `mklink /J` creates a directory junction (no admin / Developer Mode required).
if exist "%OPENVINO_LINK_DIR%" rmdir "%OPENVINO_LINK_DIR%"
mklink /J "%OPENVINO_LINK_DIR%" "%OPENVINO_INSTALL_DIR%" >nul
if errorlevel 1 (
echo ERROR: Failed to create junction "%OPENVINO_LINK_DIR%" -^> "%OPENVINO_INSTALL_DIR%".
echo If "%OPENVINO_LINK_DIR%" already exists as a regular non-empty folder, remove it manually and re-run.
exit /b 1
)
set "OPENVINO_ROOT=%OPENVINO_LINK_DIR%"
echo OpenVINO Ready: %OPENVINO_ROOT% -^> %OPENVINO_INSTALL_DIR%
echo ============================================
echo Setting up compiler environment...
echo ============================================
REM Locate Visual Studio Build Tools vcvars64.bat
set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
if exist "%VSWHERE%" (
for /f "usebackq tokens=*" %%i in (`"%VSWHERE%" -latest -products Microsoft.VisualStudio.Product.BuildTools -property installationPath`) do (
set "VS_PATH=%%i"
)
)
if defined VS_PATH (
call "%VS_PATH%\VC\Auxiliary\Build\vcvars64.bat" >nul
) else (
echo WARNING: Visual Studio Build Tools not found. Compiler may be missing.
)
REM ============================================
REM Clean old build cache
REM ============================================
if exist "build\ReleaseOV" (
echo Removing old build directory ...
rmdir /s /q "build\ReleaseOV"
)
echo ============================================
echo Configuring with CMake...
echo ============================================
call "%OPENVINO_ROOT%\setupvars.bat" >nul 2>nul
cmake -B build\ReleaseOV -G Ninja ^
-DCMAKE_BUILD_TYPE=Release ^
-DGGML_OPENVINO=ON ^
-DCMAKE_TOOLCHAIN_FILE="%VCPKG_DIR%\scripts\buildsystems\vcpkg.cmake"
if errorlevel 1 (
echo If you continue to face CMAKE errors, make sure to install:
echo winget install Microsoft.VisualStudio.2022.BuildTools
echo Then run the "Developer Command Prompt for VS 2022" and launch this script from there.
exit /b 1
)
cmake --build build\ReleaseOV --config Release
if errorlevel 1 exit /b 1
echo ============================================
echo Build completed successfully!
echo ============================================
echo Binaries: %CD%\build\ReleaseOV\bin
echo.
echo NOTE: To run, source setupvars.bat and pick a device:
echo call "C:\Intel\openvino\setupvars.bat"
echo set GGML_OPENVINO_DEVICE=CPU ^&^& REM or GPU / NPU
echo build\ReleaseOV\bin\llama-cli.exe -m model.gguf
echo.
endlocal
```
> [!NOTE]
> Use `x64 Native Tools Command Prompt` for Windows build. After building, you could use either `cmd` or `PowerShell` to run the OpenVINO backend.
> The script pins OpenVINO `2026.2` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**.
</details>
### 3. Download Sample Model
Download models for testing:
Download sample model for testing.

@@ -178,12 +566,12 @@ ```bash

mkdir -p ~/models/
wget https://huggingface.co/unsloth/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf \
-O ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf
wget https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf \
-O ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
# Windows PowerShell
mkdir C:\models
Invoke-WebRequest -Uri https://huggingface.co/unsloth/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf -OutFile C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf
Invoke-WebRequest -Uri https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf -OutFile C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf
# Windows Command Line
mkdir C:\models
curl -L https://huggingface.co/unsloth/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf -o C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf
curl -L https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf -o C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf
```

@@ -204,18 +592,18 @@

export GGML_OPENVINO_DEVICE=GPU
# Enable stateful execution with GPU device to avoid known stateless execution failures.
# Optional: enable stateful execution for improved GPU performance (recommended).
export GGML_OPENVINO_STATEFUL_EXECUTION=1
# To run llama-simple:
./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf -n 50 "The story of AI is "
./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -n 50 "The story of AI is "
# To run in chat mode:
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf -c 1024
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024
# To run llama-bench, -fa 1 is needed
GGML_OPENVINO_STATEFUL_EXECUTION=1 GGML_OPENVINO_DEVICE=GPU ./build/ReleaseOV/bin/llama-bench -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf -fa 1
GGML_OPENVINO_STATEFUL_EXECUTION=1 GGML_OPENVINO_DEVICE=GPU ./build/ReleaseOV/bin/llama-bench -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -fa 1
# NPU: keep context small to avoid failures from very large model context windows.
export GGML_OPENVINO_DEVICE=NPU
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf -c 512
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 512
# Windows Command Line
set GGML_OPENVINO_DEVICE=GPU
# Enable stateful execution with GPU device to avoid known stateless execution failures.
# Optional: enable stateful execution for improved GPU performance (recommended).
set GGML_OPENVINO_STATEFUL_EXECUTION=1

@@ -227,7 +615,7 @@ # Windows PowerShell

# To run llama-simple
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf" -n 50 "The story of AI is "
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
# To run in chat mode:
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf" -c 1024
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 1024
# To run llama-bench, -fa 1 is needed
build\ReleaseOV\bin\llama-bench.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf" -fa 1
build\ReleaseOV\bin\llama-bench.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -fa 1

@@ -239,3 +627,3 @@ # NPU: keep context small to avoid failures from very large model context windows.

$env:GGML_OPENVINO_DEVICE = "NPU"
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf" -c 512
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 512
```

@@ -245,24 +633,4 @@ > [!NOTE]

### Known Issues and Current Workarounds
### 5. Docker Build
- GPU stateless execution is currently affected by a known issue.
- Workaround: set `GGML_OPENVINO_STATEFUL_EXECUTION=1` when using GPU device.
- NPU failures can happen when context size is too large. Recent llama.cpp behavior may resolve context size to the model training context (for example, 131072 for Llama 3.2 1B), which is too large for current NPU usage and can also stress laptop CPU/GPU on larger models. To inspect the selected context size, run `llama-cli` or `llama-server` with `-lv 3`.
- Workaround: explicitly set context size, for ex. `-c 1024` for NPU runs. Performance will be better with lower context size.
- Additional NPU limitations:
- Model caching is not yet supported.
- `llama-server -np > 1` (multiple parallel sequences) is not supported.
- `llama-perplexity` is only supported with `-b 512` or smaller.
- `--context-shift` with `llama-cli` is currently not supported with OpenVINO backend across CPU, GPU, and NPU devices.
- Encoder models (embedding, reranking) are not supported with the current OpenVINO backend implementation.
- `-fa 1` is required when running llama-bench with the OpenVINO backend.
- `GGML_OPENVINO_STATEFUL_EXECUTION=1 GGML_OPENVINO_DEVICE=GPU ./llama-bench -fa 1`
- `llama-server` with OpenVINO backend supports only one chat session/thread, when `GGML_OPENVINO_STATEFUL_EXECUTION=1` is enabled.
> [!NOTE]
> The OpenVINO backend is actively under development. Fixes are underway, and this document will continue to be updated as issues are resolved.
### Docker Build
You can build and run llama.cpp with OpenVINO backend using Docker.

@@ -284,3 +652,3 @@

# If you are behind a proxy:
docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --target=light -t llama-openvino:light -f .devops/openvino.Dockerfile .
docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --target=server -t llama-openvino:server -f .devops/openvino.Dockerfile .
```

@@ -294,3 +662,3 @@

# Run Docker container
docker run --rm -it -v ~/models:/models llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_0.gguf
docker run --rm -it -v ~/models:/models llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf

@@ -301,3 +669,3 @@ # With Intel GPU access (iGPU or dGPU)

--env=GGML_OPENVINO_DEVICE=GPU --env=GGML_OPENVINO_STATEFUL_EXECUTION=1 \
llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_0.gguf
llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf

@@ -308,3 +676,3 @@ # With Intel NPU access

--env=GGML_OPENVINO_DEVICE=NPU \
llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_0.gguf
llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
```

@@ -317,13 +685,26 @@

```bash
# Run the Server Docker container
docker run --rm -it -p 8080:8080 -v ~/models:/models llama-openvino:server --no-warmup -m /models/Llama-3.2-1B-Instruct-Q4_0.gguf -c 1024
# Run the llama-openvino:server Docker container (CPU)
docker run --rm -it -p 8080:8080 -v ~/models:/models llama-openvino:server --no-warmup -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024 --host 0.0.0.0
# Run the llama-openvino:server Docker container with Intel GPU access (iGPU or dGPU)
docker run --rm -it -v ~/models:/models \
--device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
-p 8080:8080 --env=GGML_OPENVINO_DEVICE=GPU \
llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --host 0.0.0.0
# Run the llama-openvino:server Docker container with Intel NPU access
docker run --rm -it -v ~/models:/models \
--device=/dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
-p 8080:8080 --env=GGML_OPENVINO_DEVICE=NPU \
llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --host 0.0.0.0
# Or Using llama-server executable
./build/ReleaseOV/bin/llama-server -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf --port 8080 -c 1024
./build/ReleaseOV/bin/llama-server -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --port 8080 -c 1024
# Option 1: Open your browser to http://localhost:8080 to access the web UI for the llama.cpp server.
# Option 2: In a NEW terminal, test the server with curl
# If you are behind a proxy, make sure to set NO_PROXY to avoid proxy for localhost
export NO_PROXY=localhost,127.0.0.1
# Option 1: Open your browser to http://localhost:8080 to access the web UI for the llama.cpp server.
# Option 2: In a NEW terminal, test the server with curl
# Test health endpoint

@@ -337,21 +718,23 @@ curl -f http://localhost:8080/health

## Runtime Configuration
## GGML OpenVINO Backend Runtime Configurations
The OpenVINO backend can be configured using the following environment variables at runtime to control device selection, caching, debugging, and profiling behavior.
Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `1`) to enable; unset, empty, `0`, negative, or non-numeric values are treated as disabled.
### Configuration Options
| Variable | Type | Default | Description |
|-----------------------------------|-----------|------------|-------------------------------------------------------------------------------------------------------------|
| `GGML_OPENVINO_DEVICE` | String | `CPU` | Specify the target device (CPU, GPU, NPU). On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html). When set to **NPU**, static compilation mode is enabled for optimal performance. |
| `GGML_OPENVINO_CACHE_DIR` | String | `not set` | Directory for OpenVINO model caching (recommended: `/tmp/ov_cache`). Enables model caching when set. **Not supported on NPU devices.** |
| `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| Integer | `256` | Token chunk size for **NPU** prefill (NPU-only; ignored on CPU/GPU). Must be a positive integer; otherwise the default is used. |
| `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Enable stateful KV cache for better performance. Recommended on CPU, GPU. |
| `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. |
| `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. |
| `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. |
| `GGML_OPENVINO_PROFILING` | Boolean | `0` | Enable execution-time profiling. |
| `GGML_OPENVINO_DUMP_CGRAPH` | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. |
| `GGML_OPENVINO_DUMP_IR` | Boolean | `0` | Serialize OpenVINO IR files with timestamps. |
| `GGML_OPENVINO_DEBUG_INPUT` | Boolean | `0` | Enable input debugging and print input tensor info. |
| `GGML_OPENVINO_DEBUG_OUTPUT` | Boolean | `0` | Enable output debugging and print output tensor info. |
| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | Boolean | `0` | Print tensor address map once. |
| Variable | Default | Description |
|-----------------------------------|------------|-------------------------------------------------------------------------------------------------------------|
| `GGML_OPENVINO_DEVICE` | `CPU` | Specify the target device (CPU, GPU, NPU). On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html). When set to **NPU**, static compilation mode is enabled for optimal performance. |
| `GGML_OPENVINO_CACHE_DIR` | `not set` | Directory for OpenVINO model caching (recommended: `/tmp/ov_cache`). Enables model caching when set. **Not supported on NPU devices.** |
| `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| `256` | Token chunk size for **NPU** prefill. |
| `GGML_OPENVINO_STATEFUL_EXECUTION`| `0` | Enable stateful KV cache on for better performance. Recommended on CPU, GPU. |
| `GGML_OPENVINO_PROFILING` | `0` | Enable execution-time profiling. |
| `GGML_OPENVINO_DUMP_CGRAPH` | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. |
| `GGML_OPENVINO_DUMP_IR` | `0` | Serialize OpenVINO IR files with timestamps. |
| `GGML_OPENVINO_DEBUG_INPUT` | `0` | Enable input debugging and print input tensor info. |
| `GGML_OPENVINO_DEBUG_OUTPUT` | `0` | Enable output debugging and print output tensor info. |
| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | `0` | Print tensor address map once. |
> [!NOTE]

@@ -373,3 +756,3 @@ >`GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature to allow stateful execution for managing the KV cache internally inside the OpenVINO model, improving performance on CPUs and GPUs. Stateful execution is not effective on NPUs, and not all models currently support this feature. This feature is experimental and has been validated only with the llama-simple, llama-cli, llama-bench, and llama-run applications and is recommended to enable for the best performance. Other applications, such as llama-server and llama-perplexity, are not yet supported.

./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf -n 50 "The story of AI is "
./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -n 50 "The story of AI is "

@@ -388,16 +771,36 @@ # Windows Command Line

build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf" -n 50 "The story of AI is "
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
```
## Llama.cpp Tools
## Known Limitations
The following tools work with the OpenVINO backend on CPU, GPU, NPU:
- llama-bench
- llama-cli
- llama-completion
- llama-perplexity
- llama-server
- llama-simple
**General (all devices)**
- Llama.cpp OpenVINO backend currently supports a subset of GGML ops and text-only models. Unsupported ops or unsupported op shapes/cases fail during OpenVINO translation.
- Multimodal features (audio/image/video) are a work in progress.
- Limited Embedding and Reranking model support.
- Llama.cpp tool coverage across CPU/GPU/NPU is not uniform.
**Tool-specific**
- `llama-bench`: requires `-fa 1` (flash-attention).
- `llama-cli --context-shift`: stateless only (`GGML_OPENVINO_STATEFUL_EXECUTION=0`). In stateful mode the KV cache is owned by the OpenVINO model and cannot be shifted externally.
- `llama-server`: only one chat session/thread when `GGML_OPENVINO_STATEFUL_EXECUTION=1`.
**GPU-specific**
- `llama-server -np > 1`: concurrent requests are batched together, which may slightly reduce per-request throughput.
**NPU-specific**
- Default context resolves to the model's training context (e.g. 131072 for Llama 3.2 1B), which can OOM or fail or degrade performance on NPU. Inspect the resolved value with `-lv 3`.
- **Workaround:** Pass an explicit `-c <N>`, e.g. `-c 1024`.
- NPU device uses a static graph with a fixed prefill chunk size (defaults to 256), configurable with `GGML_OPENVINO_PREFILL_CHUNK_SIZE`. Large prefill/batch settings may need tuning.
- `llama-server -np > 1` (multiple parallel sequences) is not supported.
- `llama-perplexity`: requires `-b 512` or smaller.
> [!NOTE]
> The OpenVINO backend is actively under development. Fixes and improvements are underway, and this document will continue to be updated.
## Work in Progress

@@ -404,0 +807,0 @@ 

+63
-2
vendor/llama.cpp/docs/backend/SYCL.md

@@ -164,2 +164,60 @@ # llama.cpp for SYCL

## Quick Development WOW
This chapter is for quick development & try with SYCL backend on Intel GPU.
You need to install following sofeware before development:
- Intel GPU driver
- oneAPI package
- other development tools.
Please refer to [Linux](#linux) or [Windows](#windows-1) for above installation and resolve the trouble in usage. There are the detailed guide.
- Linux
```
## build from source code
./examples/sycl/build.sh
## run CONV_2D_DW unit test cases
./build/bin/test-backend-ops -b SYCL0 -o CONV_2D_DW
## run all unit test cases
./build/bin/test-backend-ops -b SYCL0
## run with LLM on the first GPU
./examples/sycl/test.sh -mg 0 -m xxxx.gguf
## run service with LLM on the first GPU
export ONEAPI_DEVICE_SELECTOR="level_zero:0"
./examples/sycl/start-svr.sh -m xxxx.gguf
## update the docs/ops.md for new/update OPs
./examples/sycl/update-ops-doc.sh
```
- Windows
```
## build from source code
examples\sycl\win-build-sycl.bat
## run CONV_2D_DW unit test cases
build\bin\test-backend-ops.exe -b SYCL0 -o CONV_2D_DW
## run all unit test cases
build\bin\test-backend-ops.exe -b SYCL0
## run LLM on the first GPU
examples\sycl\win-test.bat -mg 0 -m xxxx.gguf
## run service with LLM on the first GPU
set ONEAPI_DEVICE_SELECTOR="level_zero:0"
examples\sycl\win-start-svr.bat -m xxxx.gguf
## update the docs/ops.md for new/update OPs
examples\sycl\win-update-ops-doc.bat
```
## Linux

@@ -705,3 +763,3 @@

| GGML_SYCL_HOST_MEM_FALLBACK | ON *(default)* \|OFF *(Optional)* | Allow host memory fallback when device memory is full during quantized weight reorder. Enables inference to continue at reduced speed (reading over PCIe) instead of failing. Requires Linux kernel 6.8+. |
| GGML_SYCL_SUPPORT_LEVEL_ZERO | ON *(default)* \|OFF *(Optional)* | Enable Level Zero API for device memory allocation. Requires Level Zero headers/library at build time and Intel GPU driver (Level Zero runtime) at run time. Reduces system RAM usage during multi-GPU inference. |
| GGML_SYCL_SUPPORT_LEVEL_ZERO_API | ON *(default)* \|OFF *(Optional)* | Support to use Level Zero API for device memory allocation. Requires Level Zero headers/library at build time and Intel GPU driver (Level Zero runtime) at run time. Reduces system RAM usage during multi-GPU inference. SYCL backend always runs on Level Zero running time even if it's set as OFF (The SYCL api will be usage for memory allocation).|
| CMAKE_C_COMPILER | `icx` *(Linux)*, `icx/cl` *(Windows)* | Set `icx` compiler for SYCL code path. |

@@ -717,6 +775,7 @@ | CMAKE_CXX_COMPILER | `icpx` *(Linux)*, `icx` *(Windows)* | Set `icpx/icx` compiler for SYCL code path. |

| GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function by macro: GGML_SYCL_DEBUG |
| GGML_SYCL_DEV2DEV_MEMCPY | 0 (default) or 1 | Choose the SYCL or L0 API in dev2dev memory copy.<br>Value: <br>* 0: SYCL API (default)<br>* 1: L0 API -- L0 API is found to lead to abnormal crash in some case. This debug flag is used to check the issue.|
| GGML_SYCL_ENABLE_FLASH_ATTN | 1 (default) or 0| Enable Flash-Attention. It can reduce memory usage. The performance impact depends on the LLM.|
| GGML_SYCL_DISABLE_OPT | 0 (default) or 1 | Disable optimize features for Intel GPUs. (Recommended to 1 for Intel devices older than Gen 10) |
| GGML_SYCL_DISABLE_GRAPH | 0 or 1 (default) | Disable running computations through SYCL Graphs feature. Disabled by default because SYCL Graph is still on development, no better performance. |
| GGML_SYCL_ENABLE_LEVEL_ZERO | 1 (default) or 0 | Use Level Zero API for device memory allocation instead of SYCL. Reduces system RAM usage on Intel dGPUs by avoiding DMA-buf/TTM host memory staging. Requires GGML_SYCL_SUPPORT_LEVEL_ZERO=ON at build time. |
| GGML_SYCL_USE_LEVEL_ZERO_API | 1 (default) or 0 | Use Level Zero API for device memory allocation instead of SYCL. Reduces system RAM usage on Intel dGPUs by avoiding DMA-buf/TTM host memory staging. Requires GGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON at build time. SYCL backend always runs on Level Zero running time even if it's set as OFF (The SYCL api will be usage for memory allocation).|
| GGML_SYCL_DISABLE_DNN | 0 (default) or 1 | Disable running computations through oneDNN and always use oneMKL. |

@@ -726,2 +785,3 @@ | GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. |

| UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Allow SYCL/Unified Runtime Level Zero device allocations larger than 4 GiB. llama.cpp's direct Level Zero allocation path requests the relaxed maximum-size limit itself when GGML_SYCL_ENABLE_LEVEL_ZERO=1. |
| GGML_SYCL_USM_SYSTEM | 0 (default) or 1 | Enable experimental support for [USM system allocations](https://github.khronos.org/SYCL_Reference/iface/usm_basic_concept.html#system-allocations) for large GPU buffers. This requires enough host memory for model weights and caches, an Intel Xe2+ GPU such as BMG or newer and supported on Linux only, with CONFIG_DRM_XE_GPUSVM enabled. |

@@ -737,2 +797,3 @@ ## Compile-time Flags

## Design Rule

@@ -739,0 +800,0 @@ 

+3
-2
vendor/llama.cpp/docs/multimodal.md
# Multimodal
llama.cpp supports multimodal input via `libmtmd`. Currently, there are 2 tools support this feature:
- [llama-mtmd-cli](../tools/mtmd/README.md)
- [llama-cli](../tools/cli/README.md)
- [llama-server](../tools/server/README.md) via OpenAI-compatible `/chat/completions` API
- [llama-mtmd-cli](../tools/mtmd/README.md), for testing and development
Currently, we support **image** and **audio** input. Audio is highly experimental and may have reduced quality.
Currently, we support **image**, **audio** and **video** input.

@@ -9,0 +10,0 @@ To enable it, you can use one of the 2 methods below:

+10
-10
vendor/llama.cpp/docs/ops.md

@@ -26,12 +26,12 @@ # GGML Operations

| CEIL | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
| CLAMP | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ❌ |
| CLAMP | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ❌ | ❌ |
| COL2IM_1D | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
| CONCAT | ❌ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
| CONT | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
| CONV_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ |
| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
| CONV_3D | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
| CONV_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
| CONV_3D | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
| CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
| COS | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
| COS | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
| COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |

@@ -69,3 +69,3 @@ | CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |

| LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | ✅ | 🟡 | ❌ | ❌ | ❌ |
| LOG | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
| LOG | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
| MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |

@@ -104,3 +104,3 @@ | MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |

| SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
| SIN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
| SIN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
| SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |

@@ -110,4 +110,4 @@ | SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |

| SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
| SQR | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
| SQRT | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
| SQR | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
| SQRT | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
| SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |

@@ -114,0 +114,0 @@ | SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |

+34
-36
vendor/llama.cpp/docs/preset.md

@@ -11,53 +11,51 @@ # llama.cpp INI Presets

### Using a Remote Preset
### Using a Hugging Face Preset
> [!NOTE]
> [!IMPORTANT]
>
> This feature is currently only supported via the `-hf` option.
> Please only use presets that you can trust! Unknown presets may be unsafe
For GGUF models hosted on Hugging Face, you can include a `preset.ini` file in the root directory of the repository to define specific configurations for that model.
You can push your preset to Hugging Face Hub and share with other users by:
1. Creating an empty model repository on Hugging Face
2. Creating a `preset.ini` file in the root directory of the repository
Example:
Example of a `preset.ini`:
```ini
hf-repo-draft = username/my-draft-model-GGUF
temp = 0.5
top-k = 20
top-p = 0.95
```
[*]
ctx-size = 0
mmap = 1
kv-unified = 1
parallel = 4
spec-default = 1
For security reasons, only certain options are allowed. Please refer to [preset.cpp](../common/preset.cpp) for the complete list of permitted options.
[Qwen3.5-4B]
hf = unsloth/Qwen3.5-4B-GGUF:Q4_K_M
ctx-size = 262144
batch-size = 2048
ubatch-size = 2048
top-p = 1.0
top-k = 0
min-p = 0.01
temp = 1.0
Example usage:
Assuming your repository `username/my-model-with-preset` contains a `preset.ini` with the configuration above:
```sh
llama-cli -hf username/my-model-with-preset
# This is equivalent to:
llama-cli -hf username/my-model-with-preset \
--hf-repo-draft username/my-draft-model-GGUF \
--temp 0.5 \
--top-k 20 \
--top-p 0.95
[gpt-oss-120b-hf]
hf = ggml-org/gpt-oss-120b-GGUF
ctx-size = 262144
batch-size = 2048
ubatch-size = 2048
top-p = 1.0
top-k = 0
min-p = 0.01
temp = 1.0
chat-template-kwargs = {"reasoning_effort": "high"}
```
You can also override preset arguments by specifying them on the command line:
The preset will be loaded similarly to the `--models-preset` option. Therefore, you can also override certain params via CLI arguments:
```sh
# Force temp = 0.1, overriding the preset value
llama-cli -hf username/my-model-with-preset --temp 0.1
llama-cli -hf username/my-preset --temp 0.1
```
If you want to define multiple preset configurations for one or more GGUF models, you can create a blank HF repo for each preset. Each HF repo should contain a `preset.ini` file that references the actual model(s):
```ini
hf-repo = user/my-model-main
hf-repo-draft = user/my-model-draft
temp = 0.8
ctx-size = 1024
; (and other configurations)
```
### Named presets

@@ -64,0 +62,0 @@ 

+2
-2
vendor/llama.cpp/ggml/CMakeLists.txt

@@ -8,3 +8,3 @@ cmake_minimum_required(VERSION 3.14...3.28) # for add_link_options and implicit target directories.

set(GGML_VERSION_MINOR 15)
set(GGML_VERSION_PATCH 1)
set(GGML_VERSION_PATCH 2)
set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")

@@ -253,3 +253,3 @@

option(GGML_SYCL_HOST_MEM_FALLBACK "ggml: allow host memory fallback in SYCL reorder (requires kernel 6.8+)" ON)
option(GGML_SYCL_SUPPORT_LEVEL_ZERO "ggml: use Level Zero API in SYCL backend" ON)
option(GGML_SYCL_SUPPORT_LEVEL_ZERO_API "ggml: use Level Zero API in SYCL backend" ON)
option(GGML_SYCL_DNN "ggml: enable oneDNN in the SYCL backend" ON)

@@ -256,0 +256,0 @@ set (GGML_SYCL_TARGET "INTEL" CACHE STRING

+8
-1
vendor/llama.cpp/ggml/src/CMakeLists.txt

@@ -441,3 +441,10 @@ include(CheckCXXCompilerFlag)

ggml_add_cpu_backend_variant(power10 POWER10 VSX)
ggml_add_cpu_backend_variant(power11 POWER11 VSX)
# POWER11 backend: only if compiler supports -mcpu=power11
check_cxx_compiler_flag("-mcpu=power11" GGML_CXX_SUPPORTS_POWER11)
if (GGML_CXX_SUPPORTS_POWER11)
message(STATUS "Compiler supports -mcpu=power11, enabling POWER11 backend")
ggml_add_cpu_backend_variant(power11 POWER11 VSX)
else()
message(STATUS "Skipping POWER11 backend: compiler does not support -mcpu=power11")
endif()
else()

@@ -444,0 +451,0 @@ message(FATAL_ERROR "Unsupported PowerPC target OS: ${CMAKE_SYSTEM_NAME}")

+1
-1
vendor/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt

@@ -392,3 +392,3 @@ function(ggml_add_cpu_backend_features cpu_name arch)

if (EXTRACTED_NUMBER GREATER_EQUAL 10)
if (EXTRACTED_NUMBER EQUAL 10 OR EXTRACTED_NUMBER EQUAL 11)
list(APPEND ARCH_FLAGS -mcpu=power10)

@@ -395,0 +395,0 @@ elseif (EXTRACTED_NUMBER EQUAL 9)

+1
-1
vendor/llama.cpp/ggml/src/ggml-hexagon/htp/CMakeLists.txt

@@ -40,4 +40,4 @@ cmake_minimum_required(VERSION 3.22.2)

target_sources(${HTP_LIB} PRIVATE
hmx-flash-attn-ops.c
hmx-matmul-ops.c
hmx-flash-attn-ops.c
hmx-queue.c

@@ -44,0 +44,0 @@ )

+4
-0
vendor/llama.cpp/ggml/src/ggml-hexagon/htp/flash-attn-ops.c

@@ -342,2 +342,5 @@ #pragma clang diagnostic ignored "-Wunused-variable"

struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0);
dma_queue * dma = octx->ctx->dma[ith];

@@ -619,2 +622,3 @@

}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0);
}

@@ -621,0 +625,0 @@ 

+10
-3
vendor/llama.cpp/ggml/src/ggml-hexagon/htp/hex-dma.h

@@ -9,2 +9,4 @@ #ifndef HTP_DMA_H

#include "hex-profile.h"
#ifdef __cplusplus

@@ -92,2 +94,3 @@ extern "C" {

uint32_t idx_mask;
struct htp_thread_trace * trace;
} dma_queue;

@@ -157,2 +160,3 @@

if (size) {
htp_trace_event_start(q->trace, HTP_TRACE_EVT_DMA, q->push_idx);
dmlink(q->tail, desc);

@@ -208,2 +212,3 @@ q->tail = (dma_descriptor_2d *) desc;

if (nrows) {
htp_trace_event_start(q->trace, HTP_TRACE_EVT_DMA, q->push_idx);
dmlink(q->tail, desc);

@@ -230,6 +235,8 @@ q->tail = desc;

// Wait for desc to complete
while (!desc->done) {
// FARF(ERROR, "dma-pop: waiting for DMA : %u\n", q->pop_idx);
dmpoll();
if (!desc->done) {
while (!desc->done) {
dmpoll();
}
}
htp_trace_event_stop(q->trace, HTP_TRACE_EVT_DMA, q->pop_idx);

@@ -236,0 +243,0 @@ dptr = q->dptr[q->pop_idx];

+0
-27
vendor/llama.cpp/ggml/src/ggml-hexagon/htp/hex-utils.h

@@ -110,29 +110,2 @@ #ifndef HEX_UTILS_H

#ifndef HEX_NUM_PMU_COUNTERS
#define HEX_NUM_PMU_COUNTERS 8
#endif
static inline void hex_get_pmu(uint32_t counters[]) {
#if __HVX_ARCH__ >= 79
asm volatile("%0 = upmucnt0" : "=r"(counters[0]));
asm volatile("%0 = upmucnt1" : "=r"(counters[1]));
asm volatile("%0 = upmucnt2" : "=r"(counters[2]));
asm volatile("%0 = upmucnt3" : "=r"(counters[3]));
asm volatile("%0 = upmucnt4" : "=r"(counters[4]));
asm volatile("%0 = upmucnt5" : "=r"(counters[5]));
asm volatile("%0 = upmucnt6" : "=r"(counters[6]));
asm volatile("%0 = upmucnt7" : "=r"(counters[7]));
#else
counters[0] = qurt_pmu_get(QURT_PMUCNT0);
counters[1] = qurt_pmu_get(QURT_PMUCNT1);
counters[2] = qurt_pmu_get(QURT_PMUCNT2);
counters[3] = qurt_pmu_get(QURT_PMUCNT3);
counters[4] = qurt_pmu_get(QURT_PMUCNT4);
counters[5] = qurt_pmu_get(QURT_PMUCNT5);
counters[6] = qurt_pmu_get(QURT_PMUCNT6);
counters[7] = qurt_pmu_get(QURT_PMUCNT7);
// qurt_pmu_get_pmucnt(counters);
#endif
}
#endif /* HEX_UTILS_H */
+2
-0
vendor/llama.cpp/ggml/src/ggml-hexagon/htp/hmx-queue.c

@@ -47,3 +47,5 @@ #pragma clang diagnostic ignored "-Wunused-function"

hmx_lock(q);
htp_trace_event_start(q->trace, HTP_TRACE_EVT_HMX_COMP, ir);
d->func(d->data);
htp_trace_event_stop(q->trace, HTP_TRACE_EVT_HMX_COMP, ir);
break;

@@ -50,0 +52,0 @@ }

+2
-0
vendor/llama.cpp/ggml/src/ggml-hexagon/htp/hmx-queue.h

@@ -14,2 +14,3 @@ #ifndef HMX_QUEUE_H

#include "hex-utils.h"
#include "hex-profile.h"

@@ -51,2 +52,3 @@ #ifdef __cplusplus

bool hmx_locked;
struct htp_thread_trace * trace;
};

@@ -53,0 +55,0 @@ 

+2
-0
vendor/llama.cpp/ggml/src/ggml-hexagon/htp/htp-ctx.h

@@ -7,2 +7,3 @@ #ifndef HTP_CTX_H

#include "htp-ops.h"
#include "hex-profile.h"
#include "worker-pool.h"

@@ -74,2 +75,3 @@

uint32_t profiler;
struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1];

@@ -76,0 +78,0 @@ uint8_t * vtcm_base;

+31
-4
vendor/llama.cpp/ggml/src/ggml-hexagon/htp/htp-ops.h

@@ -149,2 +149,8 @@ #ifndef HTP_OPS_H

#ifndef HTP_MAX_NTHREADS
#define HTP_MAX_NTHREADS 10
#endif
#define HTP_TRACE_MAX_EVENTS 256
enum htp_profiler_mode {

@@ -154,4 +160,24 @@ HTP_PROF_DISABLED = 0,

HTP_PROF_PMU = 2,
HTP_PROF_TRACE = 3,
};
enum htp_trace_event_id {
HTP_TRACE_EVT_DMA = 0,
HTP_TRACE_EVT_HVX_COMP = 20,
HTP_TRACE_EVT_HVX_A_QUANT = 21,
HTP_TRACE_EVT_HVX_A_PREP = 22,
HTP_TRACE_EVT_HVX_W_DEQUANT = 23,
HTP_TRACE_EVT_HVX_W_PREP = 24,
HTP_TRACE_EVT_HVX_O_PROC = 25,
HTP_TRACE_EVT_HMX_COMP = 40,
};
struct htp_trace_desc {
uint32_t cycles; // lower 32-bits of cycle counter
uint16_t id; // Event ID
uint16_t info; // bit 15: is_stop. bits 14-0: tile/chunk index or other metadata.
};
#define HTP_PROF_PMU_NCNT 8

@@ -163,4 +189,4 @@

uint32_t usecs; // Number of usec
uint32_t cycles; // Number of cycles
uint32_t pad; // Unused
uint32_t cycles_start; // Start cycle counter
uint32_t cycles_stop; // Stop cycle counter
uint32_t pmu[HTP_PROF_PMU_NCNT]; // PMU counters

@@ -174,3 +200,3 @@ };

uint32_t n_ops; // Number of ops
uint32_t flags; // unused
uint32_t n_traces; // Number of trace descriptors per thread
uint32_t pad; // unused

@@ -188,3 +214,4 @@ // struct htp_buf_desc bufs[]; -- dspqueue buf 0

uint32_t n_ops; // Number of op profile descriptors
uint32_t pad; // unused
uint32_t n_traces[HTP_MAX_NTHREADS + 1];
uint8_t pad[8]; // align to 8 bytes
// struct htp_prof_desc profs[]; -- dspqueue buf 0

@@ -191,0 +218,0 @@ };

+41
-9
vendor/llama.cpp/ggml/src/ggml-hexagon/htp/main.c

@@ -403,3 +403,5 @@ #pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"

ctx->hmx_queue = hmx_queue_create(16, ctx->vtcm_rctx);
if (!ctx->hmx_queue) {
if (ctx->hmx_queue) {
ctx->hmx_queue->trace = &ctx->trace[HTP_MAX_NTHREADS];
} else {
FARF(ERROR, "hmx-queue-create failed");

@@ -429,2 +431,5 @@ ctx->hmx_enabled = false;

ctx->dma[i] = dma_queue_create(256); // queue depth
if (ctx->dma[i]) {
ctx->dma[i]->trace = &ctx->trace[i];
}
}

@@ -507,3 +512,4 @@

uint64_t usecs;
uint64_t cycles;
uint64_t cycles_start;
uint64_t cycles_stop;
uint32_t pmu_counters[HEX_NUM_PMU_COUNTERS];

@@ -518,4 +524,5 @@ };

case HTP_PROF_BASIC:
case HTP_PROF_TRACE:
d->usecs = HAP_perf_get_qtimer_count();
d->cycles = hex_get_cycles();
d->cycles_start = hex_get_cycles();
break;

@@ -537,4 +544,5 @@ default:

case HTP_PROF_BASIC:
case HTP_PROF_TRACE:
d->usecs = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - d->usecs);
d->cycles = hex_get_cycles() - d->cycles;
d->cycles_stop = hex_get_cycles();
break;

@@ -853,10 +861,11 @@ default:

const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops;
const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(struct htp_trace_desc);
if (dbuf.size < b_size + t_size + o_size + p_size) {
FARF(ERROR, "invalid opbatch memory block size %u", dbuf.size);
if (dbuf.size < b_size + t_size + o_size + p_size + tr_size) {
FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf.size, b_size + t_size + o_size + p_size + tr_size);
break;
}
FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u : m-size %u b-size %u t-size %u o-size %u", req.id,
n_bufs, n_tens, n_ops, dbuf.size, b_size, t_size, o_size);
FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req.id,
n_bufs, n_tens, n_ops, req.n_traces, dbuf.size, b_size, t_size, o_size);

@@ -878,2 +887,16 @@ // Setup descriptor pointers

if (ctx->profiler == HTP_PROF_TRACE) {
memset(ctx->trace, 0, sizeof(ctx->trace));
struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size);
for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
ctx->trace[t].events = &trace_events[t * req.n_traces];
ctx->trace[t].max_events = req.n_traces;
}
} else {
for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
ctx->trace[t].events = NULL;
ctx->trace[t].max_events = 0;
}
}
for (uint32_t i=0; i < n_ops; i++) {

@@ -896,3 +919,4 @@ struct profile_data prof;

pds[i].usecs = prof.usecs;
pds[i].cycles = prof.cycles;
pds[i].cycles_start = prof.cycles_start;
pds[i].cycles_stop = prof.cycles_stop;
for (int j = 0; j < HEX_NUM_PMU_COUNTERS; j++) {

@@ -910,2 +934,10 @@ pds[i].pmu[j] = prof.pmu_counters[j];

rsp.n_ops = n_ops;
memset(rsp.pad, 0, sizeof(rsp.pad));
if (ctx->profiler == HTP_PROF_TRACE) {
for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
rsp.n_traces[t] = ctx->trace[t].count;
}
} else {
memset(rsp.n_traces, 0, sizeof(rsp.n_traces));
}

@@ -912,0 +944,0 @@ dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;

+1
-0
vendor/llama.cpp/ggml/src/ggml-metal/ggml-metal-device.h

@@ -118,2 +118,3 @@ #pragma once

struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat (ggml_metal_library_t lib, enum ggml_type tsrc);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_concat (ggml_metal_library_t lib, enum ggml_type tsrc);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary (ggml_metal_library_t lib, const struct ggml_tensor * op);

@@ -120,0 +121,0 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu (ggml_metal_library_t lib, const struct ggml_tensor * op);

+2
-4
vendor/llama.cpp/ggml/src/ggml-openvino/CMakeLists.txt

@@ -1,6 +0,4 @@

find_package(OpenVINO REQUIRED)
find_package(OpenVINO REQUIRED COMPONENTS Runtime Threading)
find_package(OpenCL REQUIRED)
include("${OpenVINO_DIR}/../3rdparty/tbb/lib/cmake/TBB/TBBConfig.cmake")
file(GLOB_RECURSE GGML_HEADERS_OPENVINO "*.h" "*.hpp")

@@ -14,3 +12,3 @@ file(GLOB_RECURSE GGML_SOURCES_OPENVINO "*.cpp")

target_link_libraries(ggml-openvino PRIVATE openvino::runtime TBB::tbb OpenCL::OpenCL)
target_link_libraries(ggml-openvino PRIVATE openvino::runtime openvino::threading OpenCL::OpenCL)

@@ -17,0 +15,0 @@ if (GGML_OPENVINO)

+76
-23
vendor/llama.cpp/ggml/src/ggml-openvino/ggml-decoder.h
#pragma once
#include "ggml-quants.h"
#include "ggml-backend-impl.h"
#include "ggml-backend.h"
#include "ggml.h"

@@ -17,10 +18,9 @@ #include "openvino/decoder.h"

int ctx = -1;
int ctx_swa = -1;
int ctx_per_seq = -1;
int ctx_per_seq_swa = -1;
int n_seq = 1;
int n_heads = -1;
int n_heads_kv = -1;
int head_size = -1;
int32_t rope_params[15];
bool mixed_rope_params = false;
std::vector<int> swa_layers;

@@ -32,3 +32,4 @@

bool same_rope_params(const ModelParams & other) const {
return memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0;
return mixed_rope_params == other.mixed_rope_params &&
memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0;
}

@@ -61,2 +62,3 @@

std::map<std::string, ggml_tensor *> node_inputs;
std::map<std::string, std::vector<std::pair<std::string, ggml_tensor *>>> node_inputs_views;
std::vector<std::string> node_inputs_names;

@@ -68,2 +70,3 @@ ggml_tensor * node_output;

};
// Graph decoder

@@ -76,2 +79,3 @@ GgmlOvDecoder(ggml_cgraph * cgraph,

bool is_stateful = false,
bool model_is_splitted = false,
bool is_prefill = false,

@@ -92,2 +96,38 @@ int prefill_chunk_size = 256);

virtual size_t get_view_input_size(int node_idx, const std::string & name) const override;
virtual size_t get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const override;
virtual size_t get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const override;
virtual std::vector<size_t> get_view_input_stride(int node_idx,
const std::string & name,
size_t view_index) const override;
virtual std::vector<size_t> get_view_input_src_stride(int node_idx,
const std::string & name,
size_t view_index) const override;
virtual ov::Shape get_view_input_ggml_shape(int node_idx,
const std::string & name,
size_t view_index) const override;
virtual ov::Shape get_view_input_src_ggml_shape(int node_idx,
const std::string & name,
size_t view_index) const override;
virtual ov::PartialShape get_view_input_ov_shape(int node_idx,
const std::string & name,
size_t view_index) const override;
virtual ov::PartialShape get_view_input_src_ov_shape(int node_idx,
const std::string & name,
size_t view_index) const override;
virtual std::string get_view_input_name(int node_idx, const std::string & name, size_t view_index) const override;
virtual std::string get_view_input_src_name(int node_idx,
const std::string & name,
size_t view_index) const override;
virtual ov::element::Type get_input_type(int node_idx, const std::string & name) const override;

@@ -115,2 +155,4 @@

virtual std::vector<size_t> get_output_stride(int node_idx) const override;
virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const override;

@@ -120,2 +162,4 @@

virtual size_t get_output_op_offset(int node_idx) const override;
virtual std::vector<std::string> get_output_names(int node_idx) const override;

@@ -131,4 +175,7 @@

virtual void visit_subgraph(std::function<void(std::shared_ptr<GgmlDecoder>, int node_idx)> node_visitor) const override;
virtual int32_t get_op_dynamic_dim(int node_idx) const override;
virtual void visit_subgraph(
std::function<void(std::shared_ptr<GgmlDecoder>, int node_idx)> node_visitor) const override;
ggml_tensor * get_input_ggml_tensor(const std::string & name) const { return m_inputs.at(name); }

@@ -154,5 +201,3 @@

virtual std::vector<std::string> get_model_output_names() const override {
return m_model_output_names;
}
virtual std::vector<std::string> get_model_output_names() const override { return m_model_output_names; }

@@ -163,4 +208,2 @@ const std::map<std::string, ggml_tensor *> & get_model_outputs() const { return m_model_outputs; }

virtual int get_ctx_swa_size() const { return m_model_params.ctx_swa; }
virtual int get_ctx_per_seq() const { return m_model_params.ctx_per_seq; }

@@ -183,2 +226,4 @@

virtual bool has_mixed_rope_params() const override { return m_model_params.mixed_rope_params; }
virtual std::map<std::string, std::string> get_kv_param_res_names() const override;

@@ -190,4 +235,10 @@

ov::PartialShape get_graph_input_shape(const ggml_tensor * op, const ggml_tensor * input) const;
int get_static_n_tokens() const { return m_is_prefill ? m_prefill_chunk_size : 1; }
virtual bool is_splited_model() const override { return m_model_is_splitted; }
ov::PartialShape get_graph_input_shape(const ggml_tensor * op,
const ggml_tensor * input,
int dynamic_dim_index = -1) const;
static void dump_cgraph(const ggml_cgraph * cgraph, std::string & filename);

@@ -221,2 +272,3 @@

int m_prefill_chunk_size = 0;
bool m_model_is_splitted = false; // label the cgraph is splited or not

@@ -244,3 +296,4 @@ static ov::Shape get_shape(const ggml_tensor * tensor);

inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]);
return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) ||
(op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]);
}

@@ -253,3 +306,4 @@

inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_SET_ROWS && op->src[2] == tensor;
return tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY ||
(op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor);
}

@@ -262,9 +316,7 @@

inline static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE;
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE &&
op->src[1]->op == GGML_OP_NONE;
}
static std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) {
if (is_inp_tok(tensor, op)) {
return "inp_tokens";
}
std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) {
if (is_inp_pos(tensor, op)) {

@@ -276,6 +328,3 @@ return "inp_pos";

}
if (is_output_idx(tensor, op)) {
return "inp_out_ids";
}
if (is_inp_mask(tensor, op)) {
if (is_stateful() && is_inp_mask(tensor, op)) {
return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa";

@@ -293,2 +342,5 @@ }

// Infer and propagate dynamic-dimension indices for all tensors in the GGML graph.
void compute_node_dynamic_dims();
void validate_cgraph() const;

@@ -306,2 +358,3 @@

std::vector<NodeInfo> m_node_info_list;
std::map<ggml_tensor *, int> m_node_dynamic_dims;

@@ -314,2 +367,2 @@ ModelParams m_model_params;

int extract_layer_from_name(const std::string & name);
std::optional<int> extract_layer_from_name(const std::string & name);
+57
-3
vendor/llama.cpp/g...c/ggml-openvino/ggml-openvino-extra.cpp

@@ -6,2 +6,3 @@ #include "ggml-openvino-extra.h"

#include <cstdlib>
#include <cstring>

@@ -26,3 +27,34 @@ #include <openvino/runtime/intel_gpu/ocl/ocl.hpp>

}
device_name = getenv("GGML_OPENVINO_DEVICE") ? getenv("GGML_OPENVINO_DEVICE") : "CPU";
// All recognized GGML_OPENVINO_* env vars. Their values are cached here
// once at backend init time and read back via ggml_openvino_getenv_str()
// (raw string) or ggml_openvino_getenv_int() (integer / boolean toggle).
static constexpr const char * env_var_names[] = {
// String values (use ggml_openvino_getenv_str)
"GGML_OPENVINO_DEVICE",
"GGML_OPENVINO_CACHE_DIR",
// Integer values (use ggml_openvino_getenv_int)
"GGML_OPENVINO_PREFILL_CHUNK_SIZE",
// Boolean toggles (treated as int flags via ggml_openvino_getenv_int)
"GGML_OPENVINO_STATEFUL_EXECUTION",
"GGML_OPENVINO_PROFILING",
"GGML_OPENVINO_DUMP_CGRAPH",
"GGML_OPENVINO_DUMP_IR",
"GGML_OPENVINO_DEBUG_INPUT",
"GGML_OPENVINO_DEBUG_OUTPUT",
"GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS",
"GGML_OPENVINO_ENABLE_CACHE",
"GGML_OPENVINO_DISABLE_CACHE",
"GGML_OPENVINO_DISABLE_KV_SLICE",
"GGML_OPENVINO_MANUAL_GQA_ATTN",
};
for (const char * const & env_var : env_var_names) {
auto * env = getenv(env_var);
if (env) {
environment_variables[env_var] = env;
}
}
device_name = ggml_openvino_getenv_str("GGML_OPENVINO_DEVICE", "CPU");
auto available_devices = ov_singleton_core().get_available_devices();

@@ -35,3 +67,3 @@ if (std::find(available_devices.begin(), available_devices.end(), device_name) == available_devices.end()) {

auto * cache_dir = getenv("GGML_OPENVINO_CACHE_DIR");
const char * cache_dir = ggml_openvino_getenv_str("GGML_OPENVINO_CACHE_DIR");
if (device_name == "NPU") {

@@ -125,2 +157,19 @@ compile_config = {

// Get the value of a GGML_OPENVINO_* env var as a string. Returns
// default_value when the var is unset or set to an empty string.
const char * ggml_openvino_getenv_str(const char * var, const char * default_value) {
auto & env_map = ggml_openvino_get_device_config().environment_variables;
auto it = env_map.find(var);
return (it == env_map.end() || it->second.empty()) ? default_value : it->second.c_str();
}
// Get the value of a GGML_OPENVINO_* env var as an int (via std::atoi).
// Returns default_value (0) when the var is unset or empty. Used for both
// integer settings (e.g. GGML_OPENVINO_PREFILL_CHUNK_SIZE) and boolean
// toggles: "0" disables, any non-zero integer enables.
int ggml_openvino_getenv_int(const char * var, int default_value) {
const char * v = ggml_openvino_getenv_str(var, nullptr);
return v ? std::atoi(v) : default_value;
}
// Check if running on NPU

@@ -180,3 +229,4 @@ bool ggml_openvino_is_npu() {

if (strncmp(tensor->name, "token_embd.weight", 17) == 0) {
return ((ggml_openvino_is_npu() && tensor->type == GGML_TYPE_Q6_K) ? ExtraQuantType::F16 : ExtraQuantType::Q8_0_C);
return ((ggml_openvino_is_npu() && tensor->type == GGML_TYPE_Q6_K) ? ExtraQuantType::F16 :
ExtraQuantType::Q8_0_C);
}

@@ -306,2 +356,6 @@ if (strncmp(tensor->name, "output.weight", 13) == 0) {

case GGML_TYPE_Q5_1:
// u8 weights (5-bit values), asymmetric (scale + zero point)
break;
case GGML_TYPE_Q6_K:

@@ -308,0 +362,0 @@ layout.weights_per_block = 16;

+29
-8
vendor/llama.cpp/g...src/ggml-openvino/ggml-openvino-extra.h

@@ -67,2 +67,3 @@ #pragma once

ov::AnyMap compile_config;
std::unordered_map<std::string, std::string> environment_variables;
cl_command_queue cl_queue = nullptr;

@@ -83,2 +84,18 @@

// Environment variable accessors. All GGML_OPENVINO_* env vars are read once
// during backend init and cached on the device config; consumers must go
// through these helpers (never call ::getenv directly) so behavior stays
// consistent and centralized.
//
// Use ggml_openvino_getenv_str() for string / path values
// (e.g. GGML_OPENVINO_DEVICE, GGML_OPENVINO_CACHE_DIR). The optional
// default_value is returned when the var is unset or empty.
//
// Use ggml_openvino_getenv_int() for boolean toggles and integer settings.
// It returns std::atoi(value) when set, otherwise default_value. For
// boolean use, `if (ggml_openvino_getenv_int(name))` is true iff the value
// is a non-zero integer (so "0" disables, "1" enables).
const char * ggml_openvino_getenv_str(const char * var, const char * default_value = nullptr);
int ggml_openvino_getenv_int(const char * var, int default_value = 0);
// Check if running on NPU

@@ -120,5 +137,5 @@ bool ggml_openvino_is_npu();

struct ggml_openvino_quantized_weight_extra : public ggml_openvino_extra_base {
ov::Tensor weights; // U4 or U8 extracted weights
ov::Tensor scales; // F16 scales
ov::Tensor zp; // U4 or U8 zero points (same type as weights)
ov::Tensor weights; // U4 or U8 extracted weights
ov::Tensor scales; // F16 scales
ov::Tensor zp; // U4 or U8 zero points (same type as weights)
std::shared_ptr<ov::Node> weight_node; // Pre-built OpenVINO weight subgraph

@@ -138,4 +155,5 @@

explicit ggml_openvino_tensor_extra(std::shared_ptr<ov::Tensor> t)
: ggml_openvino_extra_base(Type::TENSOR), tensor(std::move(t)) {}
explicit ggml_openvino_tensor_extra(std::shared_ptr<ov::Tensor> t) :
ggml_openvino_extra_base(Type::TENSOR),
tensor(std::move(t)) {}
};

@@ -159,7 +177,7 @@

int64_t weights_per_block; // weights per scale/zp block
bool is_symmetric; // true for symmetric quantization
bool is_symmetric; // true for symmetric quantization
// Requantization info
bool is_requant = false; // true if this tensor needs requantization
std::optional<ExtraQuantType> requant_type; // target requant type if is_requant
bool is_requant = false; // true if this tensor needs requantization
std::optional<ExtraQuantType> requant_type; // target requant type if is_requant
};

@@ -172,2 +190,5 @@

// Check if a tensor's buffer uses remote (device) memory (e.g. GPU USM)
bool ggml_openvino_buffer_is_remote(const ggml_tensor * tensor);
// Register an extra with the tensor's OpenVINO buffer context for proper lifetime management.

@@ -174,0 +195,0 @@ // This sets tensor->extra and tracks the extra in the buffer context for cleanup.

+302
-58
vendor/llama.cpp/ggml/src/ggml-openvino/ggml-openvino.cpp

@@ -7,2 +7,3 @@ #include "ggml-openvino.h"

#include "ggml-openvino-extra.h"
#include "ggml-openvino/openvino/op_table.h"
#include "ggml-openvino/utils.h"

@@ -13,4 +14,4 @@ #include "ggml-quants.h"

#include <atomic>
#include <cstdint>
#include <cstdlib>
#include <cstdint>
#include <cstring>

@@ -151,4 +152,3 @@ #include <memory>

static bool is_stateful_enabled() {
static const auto * stateful = getenv("GGML_OPENVINO_STATEFUL_EXECUTION");
return stateful && *stateful != '\0' && strcmp(stateful, "0") != 0;
return ggml_openvino_getenv_int("GGML_OPENVINO_STATEFUL_EXECUTION") != 0;
}

@@ -373,7 +373,5 @@

if (src_ctx->is_remote) {
cl_int err =
mem_cpy_fn(queue, CL_TRUE, dst->data, src->data, ggml_nbytes(src), 0, nullptr, nullptr);
cl_int err = mem_cpy_fn(queue, CL_TRUE, dst->data, src->data, ggml_nbytes(src), 0, nullptr, nullptr);
if (err != CL_SUCCESS) {
GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL (device-to-device) failed with error %d\n", __func__,
err);
GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL (device-to-device) failed with error %d\n", __func__, err);
return false;

@@ -586,2 +584,13 @@ }

bool ggml_openvino_buffer_is_remote(const ggml_tensor * tensor) {
if (tensor == nullptr || tensor->buffer == nullptr) {
return false;
}
if (!ggml_backend_buffer_is_openvino(tensor->buffer)) {
return false;
}
auto * ctx = static_cast<ggml_backend_openvino_buffer_context *>(tensor->buffer->context);
return ctx->is_remote;
}
void ggml_openvino_buffer_register_extra(ggml_tensor * tensor, ggml_openvino_extra_base * extra) {

@@ -793,2 +802,14 @@ GGML_ASSERT(tensor != nullptr);

static bool has_non_contiguous_view_input(const ggml_tensor * op) {
for (int i = 0; i < GGML_MAX_SRC; i++) {
if (op->src[i] == nullptr) {
break;
}
if (op->src[i]->op == GGML_OP_VIEW && !ggml_is_contiguous(op->src[i])) {
return true;
}
}
return false;
}
static bool is_supported_flash_attn_pattern(const ggml_tensor * op) {

@@ -806,4 +827,74 @@ // pattern of q,k,v should be q->op==PERMUTE, q->src[0]->op==VIEW, q->src[0]->src[0]->view_src==nullptr

static bool is_gemma3n_flash_attn_pattern(const ggml_tensor * op) {
if (!is_supported_flash_attn_pattern(op)) {
return false;
}
const ggml_tensor * q_base =
op->src[0] != nullptr && op->src[0]->src[0] != nullptr ? op->src[0]->src[0]->src[0] : nullptr;
const ggml_tensor * k_base =
op->src[1] != nullptr && op->src[1]->src[0] != nullptr ? op->src[1]->src[0]->src[0] : nullptr;
const ggml_tensor * v_base =
op->src[2] != nullptr && op->src[2]->src[0] != nullptr ? op->src[2]->src[0]->src[0] : nullptr;
if (q_base == nullptr || q_base->op != GGML_OP_ROPE) {
return false;
}
// gemma3n direct attention path (no KV cache): q=ROPE, k=ROPE, v=RMS_NORM
// Only match this specific pattern to avoid falsely catching other models
// (e.g. Gemma4) that also use scale=1.0 with KV-cache backed attention.
const bool is_qkv_direct =
k_base != nullptr && v_base != nullptr && k_base->op == GGML_OP_ROPE && v_base->op == GGML_OP_RMS_NORM;
return is_qkv_direct;
}
static bool checked_mul_size(size_t a, size_t b, size_t & out) {
if (a == 0 || b == 0) {
out = 0;
return true;
}
if (a > SIZE_MAX / b) {
return false;
}
out = a * b;
return true;
}
static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) {
const ggml_tensor * as = op->src[0];
const ggml_tensor * ids = op->src[2];
if (as == nullptr || ids == nullptr) {
return true;
}
// The current OpenVINO translation materializes selected expert weights with
// shape [n_tokens, n_used, rows, k]. Skip cases that would create a very
// large temporary on GPU and let the scheduler fall back instead.
size_t tmp_elems = 1;
if (!checked_mul_size(tmp_elems, static_cast<size_t>(ids->ne[1]), tmp_elems) ||
!checked_mul_size(tmp_elems, static_cast<size_t>(ids->ne[0]), tmp_elems) ||
!checked_mul_size(tmp_elems, static_cast<size_t>(as->ne[1]), tmp_elems) ||
!checked_mul_size(tmp_elems, static_cast<size_t>(as->ne[0]), tmp_elems)) {
return true;
}
size_t tmp_bytes = 0;
if (!checked_mul_size(tmp_elems, sizeof(float), tmp_bytes)) {
return true;
}
static constexpr size_t mul_mat_id_tmp_limit = 1ULL << 30; // 1 GiB
return tmp_bytes > mul_mat_id_tmp_limit;
}
static bool is_op_unsupported_case(const ggml_tensor * op) {
switch (op->op) {
case GGML_OP_CONCAT: {
if (op->type == GGML_TYPE_I64) {
return true;
}
break;
}
case GGML_OP_GET_ROWS:

@@ -814,6 +905,26 @@ case GGML_OP_SET_ROWS: {

}
if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K)) {
// ERR = 0.000000306 > 0.000000100 GET_ROWS(type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=0)
// ERR = 0.000000197 > 0.000000100 GET_ROWS(type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=0)
return true;
}
// Keep the MoE routing weights gather on CPU for GPU runs. Splitting
// only at the later SUM/CLAMP/DIV nodes still leaves this routing path
// numerically unstable for arctic-style MoE graphs.
if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) {
return true;
}
break;
}
case GGML_OP_RESHAPE: {
if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0 ||
strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) {
return true;
}
break;
}
case GGML_OP_ADD:
case GGML_OP_MUL: {
case GGML_OP_MUL:
case GGML_OP_SUB: {
if (op->src[1]->op == GGML_OP_PERMUTE) {

@@ -829,2 +940,38 @@ return true;

}
case GGML_OP_ADD_ID: {
// Keep support aligned with the CPU backend implementation, which only handles f32 inputs/output and i32 ids.
if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32 ||
op->src[2]->type != GGML_TYPE_I32) {
return true;
}
break;
}
case GGML_OP_DIV: {
bool requires_broadcast = false;
for (int i = 0; i < 4; i++) {
if (op->src[0]->ne[i] == op->src[1]->ne[i]) {
continue;
}
if (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1) {
return true;
}
requires_broadcast = true;
}
// The GPU plugin can fuse broadcast DIV into the preceding FFN GEMM path
// and produce infs for per-channel scale vectors. Keep those DIVs on CPU
// until the fused GPU kernel is reliable. (falied case llama-arch-test mpt)
if (requires_broadcast && ggml_openvino_get_device_name() == "GPU") {
return true;
}
// qwen3next MoE weight normalization is numerically sensitive on the GPU
// path. Keep the normalization divide on CPU to match the reference.
if (strncmp(op->name, "ffn_moe_weights_norm", sizeof("ffn_moe_weights_norm") - 1) == 0) {
return true;
}
break;
}
case GGML_OP_SOFT_MAX: {

@@ -835,21 +982,34 @@ if (op->src[2] != nullptr) {

}
float scale = 1.0f;
float max_bias = 0.0f;
const auto * op_params = op->op_params;
memcpy(&scale, (const float *) op_params + 0, sizeof(float));
memcpy(&max_bias, (const float *) op_params + 1, sizeof(float));
if (max_bias > 0) {
// GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with max_bias > 0\n");
if (strncmp(op->name, "ffn_moe_probs", sizeof("ffn_moe_probs") - 1) == 0) {
return true;
}
// GPU execution of the MoE routing weights softmax is numerically unstable
// when fused with the surrounding GET_ROWS/reshape path. Keep this softmax
// on CPU so the scheduler splits at the same boundary that restores parity.
if (op->src[0] != nullptr && op->src[0]->op == GGML_OP_RESHAPE && op->src[0]->src[0] != nullptr &&
strncmp(op->src[0]->src[0]->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) {
return true;
}
break;
}
case GGML_OP_FLASH_ATTN_EXT: {
if (op->src[4] != nullptr) {
// GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with sinks\n");
case GGML_OP_SUM_ROWS: {
if (strncmp(op->name, "ffn_moe_weights_sum", sizeof("ffn_moe_weights_sum") - 1) == 0) {
return true;
}
if (!is_supported_flash_attn_pattern(op)) {
// if the input is PERMUTE skip
if (op->src[0]->op == GGML_OP_PERMUTE) {
return true;
}
break;
}
case GGML_OP_CLAMP: {
if (strncmp(op->name, "ffn_moe_weights_sum_clamped", sizeof("ffn_moe_weights_sum_clamped") - 1) == 0) {
return true;
}
break;
}
case GGML_OP_FLASH_ATTN_EXT: {
float scale = 1.0f;

@@ -862,2 +1022,17 @@ float max_bias = 0.0f;

memcpy(&logit_softcap, (const float *) op_params + 2, sizeof(float));
// Keep gemma3n flash-attn pattern on CPU for GPU runs to avoid
// accuracy drift in the OpenVINO path. Restrict by scale=1.0 to avoid
// affecting non-gemma3n models such as Llama-3.2.
if (fabsf(scale - 1.0f) < 1e-6f && is_gemma3n_flash_attn_pattern(op)) {
return true;
}
if (op->src[4] != nullptr) {
// GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with sinks\n");
return true;
}
if (!is_supported_flash_attn_pattern(op)) {
return true;
}
if (max_bias > 0) {

@@ -882,12 +1057,19 @@ // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with max_bias > 0\n");

case GGML_OP_CPY: {
if (op->src[1] != op) {
// GGML_LOG_WARN("OpenVINO backend only supports CPY that is a cast\n");
if (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16) {
// GGML_LOG_WARN("OpenVINO backend does not support CPY with non-contiguous data or bf16 types\n");
return true;
}
// op test case with non-contiguous src or dst
if ((op->ne[0] == 3 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) ||
(op->ne[0] == 1 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) ||
(op->ne[0] == 2 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2)) {
return true;
}
break;
}
case GGML_OP_MUL_MAT: {
if (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16) {
// Has accuracy issue, try enabling this and see `test-backend-ops -o "MUL_MAT"`
// GGML_LOG_WARN("OpenVINO backend does not support MUL_MAT with two F16 tensors\n");
if (ggml_openvino_get_device_name() == "GPU" && op->src[1]->op == GGML_OP_SOFT_MAX &&
op->src[0]->op == GGML_OP_CONT && op->src[0]->src[0] != nullptr &&
op->src[0]->src[0]->op == GGML_OP_TRANSPOSE && op->src[0]->src[0]->src[0] != nullptr &&
op->src[0]->src[0]->src[0]->op == GGML_OP_PERMUTE) {
return true;

@@ -898,11 +1080,14 @@ }

}
if (op->src[0]->op == GGML_OP_PERMUTE || op->src[1]->op == GGML_OP_PERMUTE) {
if (op->src[0]->op == GGML_OP_VIEW && op->src[1]->op == GGML_OP_VIEW) {
return true;
}
if (ggml_is_quantized(op->src[0]->type) && op->src[0]->ne[1] == 1) {
// MUL_MAT(type_a=q4_0,type_b=f32,m=1,n=2048,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1)
// triggers a bug in ov matmul_shape_inference.hpp
break;
}
case GGML_OP_MUL_MAT_ID: {
if (strncmp(op->name, "ffn_moe_gate_up", sizeof("ffn_moe_gate_up") - 1) == 0 ||
strncmp(op->name, "ffn_moe_down", sizeof("ffn_moe_down") - 1) == 0) {
return true;
}
if (op->src[0]->op == GGML_OP_VIEW && op->src[1]->op == GGML_OP_VIEW) {
if (mul_mat_id_requires_large_tmp(op)) {
return true;

@@ -925,3 +1110,3 @@ }

}
if (op->type != GGML_TYPE_F32) {
if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) {
// GGML_LOG_WARN("OpenVINO backend does not support ROPE with type %s\n", ggml_type_name(op->type));

@@ -947,12 +1132,51 @@ return true;

}
default:
case GGML_OP_TRANSPOSE: {
// if the type is bf16, will return true
if (op->type == GGML_TYPE_BF16) {
// GGML_LOG_WARN("OpenVINO backend does not support CONT with BF16 type\n");
return true;
}
break;
}
if (op->op == GGML_OP_GET_ROWS) {
if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K)) {
// ERR = 0.000000306 > 0.000000100 GET_ROWS(type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=0)
// ERR = 0.000000197 > 0.000000100 GET_ROWS(type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=0)
case GGML_OP_GATED_DELTA_NET: {
// enable after https://github.com/openvinotoolkit/openvino/pull/35917 is included in OV release
return true;
// if (ggml_openvino_get_device_name() == "GPU" && op->src[0]->ne[2] > 1) {
// // CVS-186471
// return true;
// }
if (op->src[2]->op == GGML_OP_PERMUTE) {
return true;
}
// kda (per-key-dimension gating) not supported by fused GatedDeltaNet op
if (op->src[3]->ne[0] != 1) {
return true;
}
// v_repeat > 1 (GQA): ggml uses modulo head mapping (h_q = h_v % H_k)
// but the fused op uses consecutive mapping (h_q = h_v / group_size)
if (op->src[2]->ne[1] != op->src[0]->ne[1]) {
return true;
}
// K > 1 (multiple state snapshots) not supported by fused op
if (op->src[5]->ne[1] > 1) {
return true;
}
break;
}
case GGML_OP_SSM_CONV: {
// qwen3next is numerically unstable with OpenVINO SSM_CONV.
// Keep this op on CPU until the OpenVINO implementation is fixed.
return true;
}
case GGML_OP_VIEW: {
// Skip TOPK_MOE fused tests until it is fully supported
// the argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe
if (strcmp(op->name, "selected_experts") == 0) {
return true;
}
break;
}
default:
break;
}
return false;

@@ -964,20 +1188,43 @@ }

static std::set<ggml_type> supported_types{GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64,
GGML_TYPE_I32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K,
GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K};
static std::unordered_set<ggml_type> supported_types{
GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, GGML_TYPE_I32, GGML_TYPE_Q4_0,
GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K};
static const std::set<ggml_op> supported_ops{GGML_OP_NONE, GGML_OP_ADD, GGML_OP_MUL, GGML_OP_MUL_MAT, GGML_OP_VIEW,
/*GGML_OP_CONT,*/ GGML_OP_RESHAPE, GGML_OP_PERMUTE, GGML_OP_TRANSPOSE,
GGML_OP_GET_ROWS, GGML_OP_ROPE, GGML_OP_RMS_NORM, GGML_OP_SCALE,
// softmax is not updated due to replaced by flash_attn_ext
// GGML_OP_SOFT_MAX,
GGML_OP_SET_ROWS, GGML_OP_FLASH_ATTN_EXT, GGML_OP_CPY};
static const std::set<ggml_unary_op> supported_unary_ops{
GGML_UNARY_OP_GELU,
GGML_UNARY_OP_SILU,
// derive supported op sets from the op_table map, keys in
// the map use the full macro name (e.g. "GGML_OP_ADD"), while
// the ggml_*_op_name() helpers return only the trailing part (e.g. "ADD").
// each set is built once and cached.
static const auto build_supported_sets = [] {
const auto & table = ov::frontend::ggml::get_supported_ops();
std::unordered_set<ggml_op> ops;
std::unordered_set<ggml_unary_op> unary_ops;
std::unordered_set<ggml_glu_op> glu_ops;
// GGML_OP_NONE has no translator but is always safe to add to the supported set.
ops.insert(GGML_OP_NONE);
for (int i = 0; i < GGML_OP_COUNT; ++i) {
const std::string key = std::string("GGML_OP_") + ggml_op_name(static_cast<ggml_op>(i));
if (table.count(key)) {
ops.insert(static_cast<ggml_op>(i));
}
}
for (int i = 0; i < GGML_UNARY_OP_COUNT; ++i) {
const std::string key = std::string("GGML_UNARY_OP_") + ggml_unary_op_name(static_cast<ggml_unary_op>(i));
if (table.count(key)) {
unary_ops.insert(static_cast<ggml_unary_op>(i));
}
}
for (int i = 0; i < GGML_GLU_OP_COUNT; ++i) {
const std::string key = std::string("GGML_GLU_OP_") + ggml_glu_op_name(static_cast<ggml_glu_op>(i));
if (table.count(key)) {
glu_ops.insert(static_cast<ggml_glu_op>(i));
}
}
return std::make_tuple(ops, unary_ops, glu_ops);
};
static const std::set<ggml_glu_op> supported_glu_ops{
GGML_GLU_OP_SWIGLU,
GGML_GLU_OP_GEGLU,
};
static const auto supported_sets = build_supported_sets();
static const auto & supported_ops = std::get<0>(supported_sets);
static const auto & supported_unary_ops = std::get<1>(supported_sets);
static const auto & supported_glu_ops = std::get<2>(supported_sets);

@@ -991,7 +1238,2 @@ switch (op->op) {

}
if (has_view_op_input(op)) {
// GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n",
// ggml_unary_op_name(ggml_get_unary_op(op)));
return false;
}
break;

@@ -1023,4 +1265,3 @@ }

static std::set<ggml_op> ops_not_support_view_input{
GGML_OP_GET_ROWS,
GGML_OP_RMS_NORM,
GGML_OP_L2_NORM,
};

@@ -1031,2 +1272,5 @@ if (ops_not_support_view_input.find(op->op) != ops_not_support_view_input.end() && has_view_op_input(op)) {

}
if (op->op == GGML_OP_RMS_NORM && has_non_contiguous_view_input(op)) {
return false;
}
}

@@ -1033,0 +1277,0 @@ }

+66
-0
vendor/llama.cpp/ggml/src/ggml-openvino/ggml-quants.cpp

@@ -129,2 +129,64 @@ #include "ggml-quants.h"

// Extracts (weight, scales, zp) from Q5_1 tensors.
// Data layout is: |16 bit scale|16 bit min|32 bit qh (5th bits)|32 x 4bit low nibbles|.
// Reconstructed quant q in [0,31]: q = (low nibble) | (qh_bit << 4). Dequant: w*d + m.
// Weights are stored as u8 (5-bit values do not fit u4), matching make_int8_weights.
void extract_q5_1_data(const ggml_tensor * tensor,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr,
bool use_bias) {
const uint64_t bytes_per_block = 24; // 2 scale + 2 min + 4 qh + 16 (32x0.5) weights
const int qk = 32;
auto * data = static_cast<uint8_t *>(tensor->data);
auto * weights = static_cast<uint8_t *>(weights_arr.data()); // u8 weights, one byte per weight
auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
// Read a 16-bit little-endian value without aliasing/const-qual violations.
auto read_u16 = [](const uint8_t * p) {
uint16_t v;
memcpy(&v, p, sizeof(v));
return v;
};
auto unpack_block = [&](const uint8_t * block, uint8_t * dst) {
uint32_t qh;
memcpy(&qh, block + 4, sizeof(uint32_t));
const uint8_t * qs = block + 8;
for (int j = 0; j < qk / 2; ++j) {
const uint8_t lo = qs[j] & 0x0F;
const uint8_t hi = qs[j] >> 4;
const uint8_t bit_lo = (qh >> j) & 1;
const uint8_t bit_hi = (qh >> (j + qk / 2)) & 1;
dst[j] = lo | (bit_lo << 4); // first 16 weights
dst[j + qk / 2] = hi | (bit_hi << 4); // last 16 weights
}
};
if (use_bias) {
// Store bias (min) directly as f16: dequant w*d + m
auto * bias = zp_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
ov::parallel_for(scales_arr.get_size(), [&](size_t i) {
const uint8_t * block = data + i * bytes_per_block;
float scale = static_cast<float>(ov::float16::from_bits(read_u16(block)));
float min = static_cast<float>(ov::float16::from_bits(read_u16(block + 2)));
scales[i] = ov::float16(scale);
bias[i] = ov::float16(min);
unpack_block(block, weights + i * qk);
});
} else {
auto * zp = static_cast<uint8_t *>(zp_arr.data()); // u8 zero points
ov::parallel_for(scales_arr.get_size(), [&](size_t i) {
const uint8_t * block = data + i * bytes_per_block;
float scale = static_cast<float>(ov::float16::from_bits(read_u16(block)));
float min = static_cast<float>(ov::float16::from_bits(read_u16(block + 2)));
scales[i] = ov::float16(scale);
// zp = -min / scale (dequant: (w - zp) * s == w*s + min)
zp[i] = (scale != 0.0f) ? (uint8_t) std::lround(-min / scale) : 0;
unpack_block(block, weights + i * qk);
});
}
}
// Extracts (weight, scales, zp) from Q8_0 tensors.

@@ -581,2 +643,3 @@ // Data layout is: |16 bit scale|32 x 8bit weights|.

case GGML_TYPE_Q8_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q5_K:

@@ -606,2 +669,5 @@ is_u4 = false;

break;
case GGML_TYPE_Q5_1:
extract_q5_1_data(&temp_tensor, weights, scales, zp, use_bias);
break;
case GGML_TYPE_Q8_0:

@@ -608,0 +674,0 @@ extract_q8_0_data(&temp_tensor, weights, scales, zp);

+10
-4
vendor/llama.cpp/ggml/src/ggml-openvino/ggml-quants.h

@@ -9,3 +9,3 @@ #pragma once

void unpack_32_4(const uint8_t* data, uint8_t* dst);
void unpack_32_4(const uint8_t * data, uint8_t * dst);

@@ -23,2 +23,8 @@ void extract_q4_0_data(const ggml_tensor * tensor,

void extract_q5_1_data(const ggml_tensor * tensor,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
ov::Tensor & zp_arr,
bool use_bias = false);
void extract_q8_0_data(const ggml_tensor * tensor,

@@ -29,3 +35,3 @@ ov::Tensor & weights_arr,

void unpack_256_4(const uint8_t* data, uint8_t* dst);
void unpack_256_4(const uint8_t * data, uint8_t * dst);

@@ -151,4 +157,4 @@ void extract_q4_k_data(const ggml_tensor * tensor,

// From <openvino>/src/common/transformations/include/transformations/utils/utils.hpp
bool get_single_value(const std::shared_ptr<ov::op::v0::Constant>& const_node,
float& value,
bool get_single_value(const std::shared_ptr<ov::op::v0::Constant> & const_node,
float & value,
bool check_value_range = true);

@@ -155,0 +161,0 @@ } // namespace util

+56
-16
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/decoder.h

@@ -6,2 +6,4 @@ #pragma once

#include <openvino/core/node.hpp>
#include <openvino/core/partial_shape.hpp>
#include <openvino/core/shape.hpp>
#include <openvino/frontend/decoder.hpp>

@@ -16,10 +18,38 @@ #include <string>

public:
virtual ov::Any get_attribute(const std::string& name) const = 0;
virtual ov::Any get_attribute(const std::string & name) const = 0;
virtual PartialShape get_input_shape(int node_idx, const std::string& name) const = 0;
virtual PartialShape get_input_shape(int node_idx, const std::string & name) const = 0;
virtual std::vector<size_t> get_input_stride(int node_idx, const std::string& name) const = 0;
virtual std::vector<size_t> get_input_stride(int node_idx, const std::string & name) const = 0;
virtual element::Type get_input_type(int node_idx, const std::string& name) const = 0;
virtual size_t get_view_input_size(int node_idx, const std::string & name) const = 0;
virtual size_t get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const = 0;
virtual size_t get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const = 0;
virtual std::vector<size_t> get_view_input_stride(int node_idx,
const std::string & name,
size_t view_index) const = 0;
virtual std::vector<size_t> get_view_input_src_stride(int node_idx,
const std::string & name,
size_t view_index) const = 0;
virtual Shape get_view_input_ggml_shape(int node_idx, const std::string & name, size_t view_index) const = 0;
virtual Shape get_view_input_src_ggml_shape(int node_idx, const std::string & name, size_t view_index) const = 0;
virtual PartialShape get_view_input_ov_shape(int node_idx, const std::string & name, size_t view_index) const = 0;
virtual PartialShape get_view_input_src_ov_shape(int node_idx,
const std::string & name,
size_t view_index) const = 0;
virtual std::string get_view_input_name(int node_idx, const std::string & name, size_t view_index) const = 0;
virtual std::string get_view_input_src_name(int node_idx, const std::string & name, size_t view_index) const = 0;
virtual element::Type get_input_type(int node_idx, const std::string & name) const = 0;
virtual size_t get_input_size() const = 0;

@@ -30,5 +60,5 @@

virtual void get_input_node(size_t input_port_idx,
std::string& producer_name,
std::string& producer_output_port_name,
size_t& producer_output_port_index) const = 0;
std::string & producer_name,
std::string & producer_output_port_name,
size_t & producer_output_port_index) const = 0;

@@ -41,15 +71,19 @@ virtual std::vector<std::string> get_input_names(int node_idx) const = 0;

virtual int32_t* get_input_op_params(int node_idx, const std::string& name) const = 0;
virtual std::vector<size_t> get_output_stride(int node_idx) const = 0;
virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const = 0;
virtual int32_t * get_output_op_params(int node_idx) const = 0;
virtual size_t get_output_op_offset(int node_idx) const = 0;
virtual std::vector<std::string> get_output_names(int node_idx) const = 0;
virtual const std::string& get_op_type() const = 0;
virtual const std::string & get_op_type() const = 0;
virtual const std::string& get_op_type(int node_idx) const = 0;
virtual const std::string & get_op_type(int node_idx) const = 0;
virtual const std::string& get_op_name() const = 0;
virtual const std::string & get_op_name() const = 0;
virtual const std::string& get_op_name(int node_idx) const = 0;
virtual const std::string & get_op_name(int node_idx) const = 0;

@@ -60,9 +94,11 @@ virtual void visit_subgraph(std::function<void(std::shared_ptr<GgmlDecoder>, int node_idx)> node_visitor) const = 0;

virtual const std::map<std::string, std::shared_ptr<ov::Node>>& get_model_inputs() const = 0;
virtual const std::map<std::string, std::shared_ptr<ov::Node>>& get_model_extra_inputs() const = 0;
virtual const std::map<std::string, std::shared_ptr<ov::Node>>& get_model_weights() const = 0;
virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_inputs() const = 0;
virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_extra_inputs() const = 0;
virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_weights() const = 0;
virtual std::vector<std::string> get_model_output_names() const = 0;
virtual int32_t* get_rope_params() const = 0;
virtual int32_t * get_rope_params() const = 0;
virtual bool has_mixed_rope_params() const = 0;
virtual std::map<std::string, std::string> get_kv_param_res_names() const = 0;

@@ -74,3 +110,7 @@

virtual bool is_splited_model() const = 0;
virtual int is_swa_layer(int layer) const = 0;
virtual int32_t get_op_dynamic_dim(int node_idx) const = 0;
};

@@ -77,0 +117,0 @@ 

+1
-1
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/frontend.h

@@ -18,3 +18,3 @@ // Copyright (C) 2018-2024 Intel Corporation

static std::shared_ptr<Model> convert(const InputModel::Ptr& model, bool naive = false);
static std::shared_ptr<Model> convert(const InputModel::Ptr & model, bool naive = false);
};

@@ -21,0 +21,0 @@ 

+4
-4
vendor/llama.cpp/g...rc/ggml-openvino/openvino/input_model.h
#pragma once
#include "decoder.h"
#include <openvino/frontend/input_model.hpp>
#include "decoder.h"
namespace ov {

@@ -19,5 +19,5 @@ namespace frontend {

public:
explicit InputModel(const std::shared_ptr<GgmlDecoder>& gdecoder);
explicit InputModel(const std::shared_ptr<GgmlDecoder> & gdecoder);
const std::shared_ptr<GgmlDecoder>& get_model_decoder() const;
const std::shared_ptr<GgmlDecoder> & get_model_decoder() const;

@@ -24,0 +24,0 @@ private:

+94
-37
vendor/llama.cpp/g...c/ggml-openvino/openvino/node_context.h
#pragma once
#include "decoder.h"
#include <cstdint>

@@ -7,4 +9,2 @@ #include <openvino/frontend/node_context.hpp>

#include "decoder.h"
namespace ov {

@@ -20,11 +20,11 @@ namespace frontend {

public:
NodeContext(const std::shared_ptr<GgmlDecoder>& decoder,
std::shared_ptr<TensorMap>& tensor_map,
NodeContext(const std::shared_ptr<GgmlDecoder> & decoder,
std::shared_ptr<TensorMap> & tensor_map,
int node_idx,
TranslateSession* translate_session = nullptr)
: ov::frontend::NodeContext(decoder->get_op_type(node_idx)),
m_decoder(decoder),
m_tensor_map(tensor_map),
m_node_idx(node_idx),
m_translate_session(translate_session) {
TranslateSession * translate_session = nullptr) :
ov::frontend::NodeContext(decoder->get_op_type(node_idx)),
m_decoder(decoder),
m_tensor_map(tensor_map),
m_node_idx(node_idx),
m_translate_session(translate_session) {
m_input_names = decoder->get_input_names(m_node_idx);

@@ -34,11 +34,7 @@ m_output_names = decoder->get_output_names(m_node_idx);

TranslateSession* get_translate_session() const {
return m_translate_session;
}
TranslateSession * get_translate_session() const { return m_translate_session; }
const std::vector<std::string>& get_input_names() const { return m_input_names; }
const std::vector<std::string> & get_input_names() const { return m_input_names; }
size_t get_input_size() const override {
return m_decoder->get_input_size(m_node_idx);
}
size_t get_input_size() const override { return m_decoder->get_input_size(m_node_idx); }

@@ -61,17 +57,86 @@ ov::element::Type get_input_type(size_t index) const {

int32_t* get_input_op_params(size_t index) const {
int32_t * get_input_op_params(size_t index) const {
return m_decoder->get_input_op_params(m_node_idx, m_input_names[index]);
}
int32_t * get_output_op_params() const { return m_decoder->get_output_op_params(m_node_idx); }
size_t get_view_input_size(size_t index) const {
return m_decoder->get_view_input_size(m_node_idx, m_input_names[index]);
}
ov::element::Type get_output_type() const {
return m_decoder->get_output_type(m_node_idx);
size_t get_view_input_offset(size_t index, size_t view_index) const {
return m_decoder->get_view_input_offset(m_node_idx, m_input_names[index], view_index);
}
size_t get_view_input_src_offset(size_t index, size_t view_index) const {
return m_decoder->get_view_input_src_offset(m_node_idx, m_input_names[index], view_index);
}
std::vector<size_t> get_view_input_stride(size_t index, size_t view_index) const {
return m_decoder->get_view_input_stride(m_node_idx, m_input_names[index], view_index);
}
std::vector<size_t> get_view_input_src_stride(size_t index, size_t view_index) const {
return m_decoder->get_view_input_src_stride(m_node_idx, m_input_names[index], view_index);
}
ov::Shape get_view_input_ggml_shape(size_t index, size_t view_index) const {
return m_decoder->get_view_input_ggml_shape(m_node_idx, m_input_names[index], view_index);
}
ov::Shape get_view_input_src_ggml_shape(size_t index, size_t view_index) const {
return m_decoder->get_view_input_src_ggml_shape(m_node_idx, m_input_names[index], view_index);
}
ov::PartialShape get_view_input_ov_shape(size_t index, size_t view_index) const {
return m_decoder->get_view_input_ov_shape(m_node_idx, m_input_names[index], view_index);
}
ov::PartialShape get_view_input_src_ov_shape(size_t index, size_t view_index) const {
return m_decoder->get_view_input_src_ov_shape(m_node_idx, m_input_names[index], view_index);
}
std::string get_view_input_name(size_t index, size_t view_index) const {
return m_decoder->get_view_input_name(m_node_idx, m_input_names[index], view_index);
}
std::string get_view_input_src_name(size_t index, size_t view_index) const {
return m_decoder->get_view_input_src_name(m_node_idx, m_input_names[index], view_index);
}
int32_t get_op_dynamic_dim() const { return m_decoder->get_op_dynamic_dim(m_node_idx); }
int32_t * get_output_op_params() const { return m_decoder->get_output_op_params(m_node_idx); }
size_t get_output_op_offset() const { return m_decoder->get_output_op_offset(m_node_idx); }
ov::element::Type get_output_type() const { return m_decoder->get_output_type(m_node_idx); }
std::vector<size_t> get_output_stride() const { return m_decoder->get_output_stride(m_node_idx); }
Output<Node> get_input(int idx) const override {
// Check if this input is a VIEW
size_t view_input_size = m_decoder->get_view_input_size(m_node_idx, m_input_names[idx]);
if (view_input_size > 0) {
// This is a VIEW input, get the base tensor name (last element in the chain)
std::string base_name =
m_decoder->get_view_input_src_name(m_node_idx, m_input_names[idx], view_input_size - 1);
// Check if the VIEW has been resolved (translate_view produced a Slice)
auto view_it = m_tensor_map->find(m_input_names[idx]);
if (!base_name.empty() && view_it != m_tensor_map->end()) {
auto base_it = m_tensor_map->find(base_name);
if (base_it != m_tensor_map->end() &&
view_it->second.get_node_shared_ptr() != base_it->second.get_node_shared_ptr()) {
return view_it->second;
}
return base_it->second;
}
if (!base_name.empty()) {
return m_tensor_map->at(base_name);
}
}
// Not a VIEW or failed to get base name, use the original logic
return m_tensor_map->at(m_input_names[idx]);
}
Output<Node> get_input(const std::string& name) const override {
Output<Node> get_input(const std::string & name) const override {
if (m_tensor_map->find(name) == m_tensor_map->end()) {

@@ -83,17 +148,9 @@ throw std::runtime_error("'" + name + "' not found in tensor map.");

bool has_input(const std::string& name) const {
return m_tensor_map->find(name) != m_tensor_map->end();
}
bool has_input(const std::string & name) const { return m_tensor_map->find(name) != m_tensor_map->end(); }
const std::string& get_name() const override {
return m_decoder->get_op_name(m_node_idx);
}
const std::string & get_name() const override { return m_decoder->get_op_name(m_node_idx); }
ov::Any get_attribute_as_any(const std::string& name) const override {
return m_decoder->get_attribute(name);
}
ov::Any get_attribute_as_any(const std::string & name) const override { return m_decoder->get_attribute(name); }
int get_op_case() const {
return m_decoder->get_op_case(m_node_idx);
}
int get_op_case() const { return m_decoder->get_op_case(m_node_idx); }

@@ -106,5 +163,5 @@ bool is_static() const { return m_decoder->is_static(); }

std::shared_ptr<GgmlDecoder> m_decoder;
std::shared_ptr<TensorMap>& m_tensor_map;
std::shared_ptr<TensorMap> & m_tensor_map;
int m_node_idx;
TranslateSession* m_translate_session;
TranslateSession * m_translate_session;
std::vector<std::string> m_input_names;

@@ -114,3 +171,3 @@ std::vector<std::string> m_output_names;

using CreatorFunction = std::function<ov::OutputVector(const ov::frontend::ggml::NodeContext&)>;
using CreatorFunction = std::function<ov::OutputVector(const ov::frontend::ggml::NodeContext &)>;

@@ -117,0 +174,0 @@ } // namespace ggml

+40
-23
vendor/llama.cpp/g...src/ggml-openvino/openvino/op_table.cpp

@@ -8,5 +8,7 @@ #include "op_table.h"

#include <openvino/op/gather.hpp>
#include <openvino/op/gelu.hpp>
#include <openvino/op/matmul.hpp>
#include <openvino/op/multiply.hpp>
#include <openvino/op/subtract.hpp>
#include <openvino/op/tanh.hpp>

@@ -20,25 +22,40 @@ namespace ov {

return {
{"GGML_OP_ADD", op::translate_1to1_match_2_inputs<v1::Add> },
{"GGML_OP_ADD1", op::translate_1to1_match_2_inputs<v1::Add> },
{"GGML_OP_CONT", op::translate_cont },
{"GGML_OP_DIV", op::translate_1to1_match_2_inputs<v1::Divide> },
{"GGML_OP_GET_ROWS", op::translate_get_rows },
{"GGML_OP_MUL", op::translate_1to1_match_2_inputs<v1::Multiply>},
{"GGML_OP_MUL_MAT", op::translate_mulmat },
{"GGML_OP_PERMUTE", op::translate_permute },
{"GGML_OP_RESHAPE", op::translate_reshape },
{"GGML_OP_RMS_NORM", op::translate_rms_norm },
{"GGML_OP_ROPE", op::translate_rope },
{"GGML_OP_SCALE", op::translate_scale },
{"GGML_OP_SOFT_MAX", op::translate_soft_max },
{"GGML_OP_SUB", op::translate_1to1_match_2_inputs<v1::Subtract>},
{"GGML_OP_TRANSPOSE", op::translate_transpose },
{"GGML_UNARY_OP_GELU", op::translate_unary_gelu },
{"GGML_UNARY_OP_SILU", op::translate_unary_silu },
{"GGML_OP_VIEW", op::translate_view },
{"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu },
{"GGML_GLU_OP_GEGLU", op::translate_glu_geglu },
{"GGML_OP_SET_ROWS", op::translate_set_rows },
{"GGML_OP_CPY", op::translate_cpy },
{"GGML_OP_FLASH_ATTN_EXT", op::translate_flash_attn_ext },
{"GGML_OP_ADD", op::translate_1to1_match_2_inputs<v1::Add> },
{"GGML_OP_ADD1", op::translate_1to1_match_2_inputs<v1::Add> },
{"GGML_OP_ADD_ID", op::translate_add_id },
{"GGML_OP_CONCAT", op::translate_concat },
{"GGML_OP_CONT", op::translate_cont },
{"GGML_OP_DIV", op::translate_div },
{"GGML_OP_GET_ROWS", op::translate_get_rows },
{"GGML_OP_IM2COL", op::translate_im2col },
{"GGML_OP_MUL", op::translate_1to1_match_2_inputs<v1::Multiply>},
{"GGML_OP_MUL_MAT", op::translate_mulmat },
{"GGML_OP_MUL_MAT_ID", op::translate_mul_mat_id },
{"GGML_OP_PERMUTE", op::translate_permute },
{"GGML_OP_RESHAPE", op::translate_reshape },
{"GGML_OP_RMS_NORM", op::translate_rms_norm },
{"GGML_OP_NORM", op::translate_norm },
{"GGML_OP_L2_NORM", op::translate_l2_norm },
{"GGML_OP_SUM_ROWS", op::translate_sum_rows },
{"GGML_OP_ROPE", op::translate_rope },
{"GGML_OP_SCALE", op::translate_scale },
{"GGML_OP_SOFT_MAX", op::translate_soft_max },
{"GGML_OP_ARGSORT", op::translate_argsort },
{"GGML_OP_SUB", op::translate_1to1_match_2_inputs<v1::Subtract>},
{"GGML_OP_TRANSPOSE", op::translate_transpose },
{"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input<v7::Gelu> },
{"GGML_UNARY_OP_SILU", op::translate_unary_silu },
{"GGML_UNARY_OP_SOFTPLUS", op::translate_unary_softplus },
{"GGML_UNARY_OP_TANH", op::translate_1to1_match_1_input<v0::Tanh> },
{"GGML_OP_VIEW", op::translate_view },
{"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu },
{"GGML_GLU_OP_GEGLU", op::translate_glu_geglu },
{"GGML_OP_SET_ROWS", op::translate_set_rows },
{"GGML_OP_CPY", op::translate_cpy },
{"GGML_OP_FLASH_ATTN_EXT", op::translate_flash_attn_ext },
{"GGML_OP_CLAMP", op::translate_clamp },
{"GGML_OP_PAD", op::translate_pad },
{"GGML_OP_SSM_CONV", op::translate_ssm_conv },
{"GGML_OP_GATED_DELTA_NET", op::translate_gated_delta_net },
{"GGML_OP_REPEAT", op::translate_repeat },
};

@@ -45,0 +62,0 @@ }

+17
-5
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/op_table.h

@@ -11,16 +11,22 @@ #pragma once

#define GGML_OP_CONVERTER(op) OutputVector op(const NodeContext& context)
#define GGML_OP_CONVERTER(op) OutputVector op(const NodeContext & context)
GGML_OP_CONVERTER(translate_add);
GGML_OP_CONVERTER(translate_cont);
GGML_OP_CONVERTER(translate_concat);
GGML_OP_CONVERTER(translate_add_id);
GGML_OP_CONVERTER(translate_div);
GGML_OP_CONVERTER(translate_get_rows);
GGML_OP_CONVERTER(translate_mul);
GGML_OP_CONVERTER(translate_im2col);
GGML_OP_CONVERTER(translate_mulmat);
GGML_OP_CONVERTER(translate_mul_mat_id);
GGML_OP_CONVERTER(translate_permute);
GGML_OP_CONVERTER(translate_reshape);
GGML_OP_CONVERTER(translate_rms_norm);
GGML_OP_CONVERTER(translate_norm);
GGML_OP_CONVERTER(translate_l2_norm);
GGML_OP_CONVERTER(translate_sum_rows);
GGML_OP_CONVERTER(translate_rope);
GGML_OP_CONVERTER(translate_scale);
GGML_OP_CONVERTER(translate_unary_silu);
GGML_OP_CONVERTER(translate_unary_gelu);
GGML_OP_CONVERTER(translate_unary_softplus);
GGML_OP_CONVERTER(translate_soft_max);

@@ -33,5 +39,11 @@ GGML_OP_CONVERTER(translate_transpose);

GGML_OP_CONVERTER(translate_cpy);
GGML_OP_CONVERTER(translate_argsort);
GGML_OP_CONVERTER(translate_flash_attn_ext);
GGML_OP_CONVERTER(translate_clamp);
GGML_OP_CONVERTER(translate_pad);
GGML_OP_CONVERTER(translate_ssm_conv);
GGML_OP_CONVERTER(translate_gated_delta_net);
GGML_OP_CONVERTER(translate_repeat);
} // namespace op
} // namespace op

@@ -38,0 +50,0 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops();

+8
-16
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/op/cont.cpp

@@ -21,23 +21,15 @@

int op_case = context.get_op_case();
FRONT_END_CHECK_IMPLEMENTED(op_case == 1 || op_case == 2 || op_case == 3, "Unsupported CONT case");
auto src_shape = context.get_input_shape(0).to_shape();
auto dst_shape = context.get_output_shape().to_shape();
ov::Output<Node> res;
if (op_case == 1) {
// The input comes from a PERMUTE
throw std::runtime_error("Code of this case might be outdated");
dst_shape[1] = -1;
res = std::make_shared<ov::op::v1::Reshape>(
context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {dst_shape.size()}, dst_shape), false);
} else if (op_case == 2) {
// The input comes from a TRANSPOSE
return {context.get_input(0)};
} else {
// The input comes from a VIEW
res = process_view_input(context, 0);
if (context.get_op_dynamic_dim() != -1) {
dst_shape[3 - context.get_op_dynamic_dim()] = -1;
}
auto input = process_view_input_new(context, 0);
ov::Output<Node> res;
res = std::make_shared<ov::op::v1::Reshape>(
input, ov::op::v0::Constant::create(ov::element::i64, {dst_shape.size()}, dst_shape), false);
return rename_outputs_with_suffix({res}, context.get_name());

@@ -44,0 +36,0 @@ }

+14
-1
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/op/cpy.cpp

@@ -6,3 +6,5 @@ #include "../node_context.h"

#include <memory>
#include <openvino/op/constant.hpp>
#include <openvino/op/convert.hpp>
#include <openvino/op/reshape.hpp>

@@ -15,3 +17,14 @@ namespace ov {

OutputVector translate_cpy(const NodeContext & context) {
auto res = std::make_shared<ov::op::v0::Convert>(context.get_input(0), context.get_output_type());
auto input = process_view_input_new(context, 0);
auto input_shape = context.get_input_shape(0);
auto output_shape = context.get_output_shape();
// Non-cast CPY may need a reshape (e.g. [3,192,1,1] -> [576,1,1,1])
if (input_shape != output_shape) {
auto new_shape = ov::op::v0::Constant::create(
ov::element::i64, {static_cast<size_t>(output_shape.rank().get_length())}, output_shape.to_shape());
input = std::make_shared<ov::op::v1::Reshape>(input, new_shape, false);
}
auto res = std::make_shared<ov::op::v0::Convert>(input, context.get_output_type());
return rename_outputs_with_suffix({res}, context.get_name());

@@ -18,0 +31,0 @@ }

+108
-21
vendor/llama.cpp/g...openvino/openvino/op/flash_attn_ext.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include "ggml-openvino/ggml-openvino-extra.h"
#include <cstdint>
#include <cstdlib>
#include <memory>
#include <openvino/op/add.hpp>
#include <openvino/op/broadcast.hpp>

@@ -11,4 +14,7 @@ #include <openvino/op/concat.hpp>

#include <openvino/op/convert.hpp>
#include <openvino/op/matmul.hpp>
#include <openvino/op/multiply.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/scaled_dot_product_attention.hpp>
#include <openvino/op/softmax.hpp>
#include <openvino/op/transpose.hpp>

@@ -38,3 +44,5 @@ #include <openvino/op/unsqueeze.hpp>

ov::Output<ov::Node> mask_sliced, res;
ov::Output<ov::Node> res;
// For stateful
std::string mask_name = "KQ_mask_sliced";

@@ -45,18 +53,92 @@ if (context.get_input_names()[3].find("swa") != std::string::npos) {

if (context.has_input(mask_name)) {
mask_sliced = context.get_input(mask_name);
} else {
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
auto token_len = get_dimensions(q, {2});
mask_sliced = std::make_shared<ov::op::v8::Slice>(mask, zero, token_len, one, two);
mask = context.get_input(mask_name);
}
if (mask_sliced.get_element_type() != ov::element::f16) {
mask_sliced = std::make_shared<ov::op::v0::Convert>(mask_sliced, ov::element::f16);
if (mask.get_element_type() != ov::element::f16) {
mask = std::make_shared<ov::op::v0::Convert>(mask, ov::element::f16);
}
auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output<Node> kv) {
int64_t factor = num_heads / num_heads_kv;
if (factor > 1 && num_heads_kv > 1) {
//auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output<Node> kv) {
// int64_t factor = num_heads / num_heads_kv;
// if (factor > 1 && num_heads_kv > 1) {
auto q_shape = context.get_input_shape(0).to_shape();
auto k_shape = context.get_input_shape(1).to_shape();
const int64_t num_heads = q_shape[1];
const int64_t num_heads_kv = k_shape[1];
const int64_t head_size = q_shape[3];
const int64_t factor = num_heads / num_heads_kv;
// Manual GQA attention: enabled by default on GPU in stateless mode.
// Set GGML_OPENVINO_MANUAL_GQA_ATTN to a positive value (e.g. 1) to force-enable,
// or to 0 to force-disable. Unset falls back to the device-based default.
static const bool manual_gqa_enabled = []() {
const char * env = ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN");
if (env != nullptr) {
return ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0;
}
const char * dev = ggml_openvino_getenv_str("GGML_OPENVINO_DEVICE");
return dev != nullptr && std::string(dev) == "GPU";
}();
const bool use_manual_gqa_attention =
manual_gqa_enabled && factor > 1 && num_heads_kv > 1 && !context.is_stateful();
if (use_manual_gqa_attention) {
// Q, K, V arrive as [B, n_heads(_kv), S, head_size], where B is the active
// batch (n_seq_active) and may be > 1 (llama-perplexity, llama-server -np > 1)
// or dynamic. Reshape to
// K_r: [B, num_heads_kv, 1, S, head_size]
// Q_r: [B, num_heads_kv, factor, S_q, head_size]
// and let MatMul broadcast across the factor dim without materialising
// an expanded K/V. The leading 0 + special_zero=true copies B at runtime,
// so this is correct for B == 1, B > 1, and dynamic B alike. Only the head
// dims and head_size are baked in as literals; the sequence dim stays -1.
auto k_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5},
std::vector<int64_t>{0, num_heads_kv, 1, -1, head_size});
auto v_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5},
std::vector<int64_t>{0, num_heads_kv, 1, -1, head_size});
auto q_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5},
std::vector<int64_t>{0, num_heads_kv, factor, -1, head_size});
auto k_r = std::make_shared<ov::op::v1::Reshape>(k, k_5d_shape, true);
auto v_r = std::make_shared<ov::op::v1::Reshape>(v, v_5d_shape, true);
auto q_r = std::make_shared<ov::op::v1::Reshape>(q, q_5d_shape, true);
// QK^T → [B, num_heads_kv, factor, S_q, S_k]
auto qk = std::make_shared<ov::op::v0::MatMul>(q_r, k_r, /*tA=*/false, /*tB=*/true);
auto qk_scaled = std::make_shared<ov::op::v1::Multiply>(qk, scale_node);
// Mask arrives as [B, 1, S_q, S_k]. Unsqueeze a factor axis at position 2 to
// get [B, 1, 1, S_q, S_k], which NUMPY-broadcasts cleanly against the
// [B, num_heads_kv, factor, S_q, S_k] scores: B==B, then 1→num_heads_kv and
// 1→factor on the head dims.
auto mask_unsq1 =
std::make_shared<ov::op::v0::Unsqueeze>(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
// mask_unsq1: [B, 1, 1, S_q, S_k] (rank 5)
ov::Output<ov::Node> qk_masked = std::make_shared<ov::op::v1::Add>(qk_scaled, mask_unsq1);
auto softmax = std::make_shared<ov::op::v8::Softmax>(qk_masked, /*axis=*/-1);
// softmax @ V → [B, num_heads_kv, factor, S_q, head_size]
auto attn = std::make_shared<ov::op::v0::MatMul>(softmax, v_r);
// Reshape back to [B, num_heads, S_q, head_size] (combine num_heads_kv * factor).
// Leading 0 + special_zero=true copies B at runtime.
auto out_4d_shape =
ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, num_heads, -1, head_size});
auto out_4d = std::make_shared<ov::op::v1::Reshape>(attn, out_4d_shape, true);
// The standard SDPA path's downstream is Transpose(0,2,1,3) → Convert(f32).
// Replicate it here so callers see the same output layout/dtype.
res = std::make_shared<ov::op::v1::Transpose>(
out_4d, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
res = std::make_shared<ov::op::v0::Convert>(res, ov::element::f32);
return rename_outputs_with_suffix({res}, context.get_name());
}
// Default path: explicit Broadcast → SDPA. Kept as the fallback because
// (a) it goes through the GPU plugin's micro-SDPA fast path (FlashAttention
// tiles via DPAS), and (b) the manual path above is still being validated.
auto tile_kv = [&](int64_t n_heads, int64_t n_heads_kv, int64_t hs, ov::Output<Node> kv) {
int64_t f = n_heads / n_heads_kv;
if (f > 1 && n_heads_kv > 1) {
ov::Output<ov::Node> kv_broadcast_shape, kv_unsqueezed, new_kv_shape;

@@ -66,6 +148,9 @@ auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2});

kv_broadcast_shape = ov::op::v0::Constant::create(
ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1});
kv_broadcast_shape = ov::op::v0::Constant::create(ov::element::i64, {5},
{(int64_t) 1, (int64_t) 1, f, (int64_t) 1, (int64_t) 1});
new_kv_shape =
ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, num_heads, (int64_t) -1, head_size});
ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, n_heads, (int64_t) -1, hs});
// ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1});
//new_kv_shape =
// ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, num_heads, (int64_t) -1, head_size});

@@ -79,8 +164,10 @@ kv = std::make_shared<ov::op::v3::Broadcast>(kv_unsqueezed, kv_broadcast_shape,

auto q_shape = context.get_input_shape(0).to_shape();
auto k_shape = context.get_input_shape(1).to_shape();
k = tile_kv(q_shape[1], k_shape[1], q_shape[3], k);
v = tile_kv(q_shape[1], k_shape[1], q_shape[3], v);
//auto q_shape = context.get_input_shape(0).to_shape();
//auto k_shape = context.get_input_shape(1).to_shape();
//k = tile_kv(q_shape[1], k_shape[1], q_shape[3], k);
//v = tile_kv(q_shape[1], k_shape[1], q_shape[3], v);
k = tile_kv(num_heads, num_heads_kv, head_size, k);
v = tile_kv(num_heads, num_heads_kv, head_size, v);
auto sdpa = std::make_shared<ov::op::v13::ScaledDotProductAttention>(q, k, v, mask_sliced, scale_node, false);
auto sdpa = std::make_shared<ov::op::v13::ScaledDotProductAttention>(q, k, v, mask, scale_node, false);
res = std::make_shared<ov::op::v1::Transpose>(sdpa,

@@ -87,0 +174,0 @@ ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));

+2
-9
vendor/llama.cpp/g.../ggml-openvino/openvino/op/get_rows.cpp

@@ -21,13 +21,6 @@ #include "../node_context.h"

int op_case = context.get_op_case();
Output<Node> res;
auto data = context.get_input(0);
auto indices = context.get_input(1);
auto data = process_view_input_new(context, 0);
auto indices = process_view_input_new(context, 1);
if (op_case == 2) {
// The input comes from a VIEW
indices = process_view_input(context, 1);
}
// data[1,b,x,y] ind[1,1,b,x'] test-backend-ops case

@@ -34,0 +27,0 @@ // data[x,y] ind[1,1,1,x'] normal case

+21
-7
vendor/llama.cpp/g...ggml-openvino/openvino/op/glu_geglu.cpp

@@ -7,2 +7,3 @@ #include "../node_context.h"

#include <openvino/core/node_output.hpp>
#include <openvino/op/clamp.hpp>
#include <openvino/op/constant.hpp>

@@ -25,4 +26,6 @@ #include <openvino/op/gelu.hpp>

if (context.get_input_size() == 2) {
src0 = context.get_input(0);
src1 = context.get_input(1);
// Inputs may be VIEW slices of a combined gate_up tensor (MoE experts):
// resolve them so each half has its real sliced shape, not the base tensor.
src0 = process_view_input_new(context, 0);
src1 = process_view_input_new(context, 1);
} else {

@@ -32,3 +35,4 @@ // GGML splits along ne[0] (OV last axis) using floor division: nc = ne[0] / 2.

// Use Slice instead of Split to handle odd dimensions correctly.
auto combined = context.get_input(0);
// Resolve a VIEW input (e.g. non-contiguous slice) to its real shape first.
auto combined = process_view_input_new(context, 0);
auto combined_shape = combined.get_partial_shape();

@@ -38,8 +42,8 @@ int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length();

auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc});
auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc});

@@ -56,2 +60,12 @@ src0 = std::make_shared<ov::op::v8::Slice>(combined, start0, stop0, step, axis);

if (context.is_static()) {
// TODO: Temporary solution for NPU accuracy issue due to fp16 overflow
// To be removed once permanent solution is implemented
// Justification:
// For |x| > 5, GELU(x) ≈ max(x, 0) (behaves like ReLU)
// So Clamp(-10, 10) only affects values where GELU would return ≈ x anyway.
// The only loss: values > 10 get mapped to 10 instead of x.
// In practice, FFN intermediates rarely exceed 10 after GEGLU gating.
src0 = std::make_shared<ov::op::v0::Clamp>(src0, -10.0, 10.0);
}
auto gelu = std::make_shared<ov::op::v7::Gelu>(src0);

@@ -58,0 +72,0 @@ auto res = std::make_shared<ov::op::v1::Multiply>(gelu, src1);

+10
-7
vendor/llama.cpp/g...gml-openvino/openvino/op/glu_swiglu.cpp

@@ -24,4 +24,6 @@ #include "../node_context.h"

if (context.get_input_size() == 2) {
src0 = context.get_input(0);
src1 = context.get_input(1);
// Inputs may be VIEW slices of a combined gate_up tensor (MoE experts):
// resolve them so each half has its real sliced shape, not the base tensor.
src0 = process_view_input_new(context, 0);
src1 = process_view_input_new(context, 1);
} else {

@@ -31,3 +33,4 @@ // GGML splits along ne[0] (OV last axis) using floor division: nc = ne[0] / 2.

// Use Slice instead of Split to handle odd dimensions correctly.
auto combined = context.get_input(0);
// Resolve a VIEW input (e.g. non-contiguous slice) to its real shape first.
auto combined = process_view_input_new(context, 0);
auto combined_shape = combined.get_partial_shape();

@@ -37,8 +40,8 @@ int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length();

auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc});
auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc});

@@ -45,0 +48,0 @@ src0 = std::make_shared<ov::op::v8::Slice>(combined, start0, stop0, step, axis);

+19
-9
vendor/llama.cpp/g...rc/ggml-openvino/openvino/op/mulmat.cpp

@@ -33,13 +33,12 @@ #include "../node_context.h"

ov::Output<Node> res;
ov::Output<ov::Node> B = context.get_input(0);
ov::Output<ov::Node> A = context.get_input(1);
bool transpose_b = true;
if (op_case == 2) {
B = B.get_node_shared_ptr()->input_value(0);
transpose_b = false;
} else if (op_case == 3) {
ov::Output<ov::Node> B;
ov::Output<ov::Node> A;
if (op_case == 3) {
B = process_view_input(context, 0);
A = process_view_input(context, 1);
} else {
B = process_view_input_new(context, 0);
A = process_view_input_new(context, 1);
}
if (A.get_element_type() != B.get_element_type()) {

@@ -59,2 +58,3 @@ B = std::make_shared<ov::op::v0::Convert>(context.get_input(0), context.get_input_type(1));

Output<Node> Z = A_batch_larger ? B : A;
auto Z_shape = A_batch_larger ? B_shape : A_shape;
int64_t factor = batch_large / batch_small;

@@ -72,3 +72,7 @@ if (factor > 1 && batch_small > 1) {

auto new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4},
{(int64_t) 0, batch_large, (int64_t) -1, (int64_t) A_shape[3]});
{(int64_t) 0, batch_large, (int64_t) -1, (int64_t) Z_shape[3]});
if (op_case == 2) {
new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4},
{(int64_t) 0, batch_large, (int64_t) Z_shape[2], (int64_t) -1});
}

@@ -85,4 +89,10 @@ auto Z_broadcasted = std::make_shared<ov::op::v3::Broadcast>(Z_unsqueezed, broadcast_shape,

bool transpose_b = true;
res = std::make_shared<ov::op::v0::MatMul>(A, B, false, transpose_b);
const auto output_type = context.get_output_type();
if (res.get_element_type() != output_type) {
res = std::make_shared<ov::op::v0::Convert>(res, output_type);
}
return rename_outputs_with_suffix({res}, context.get_name());

@@ -89,0 +99,0 @@ }

+58
-13
vendor/llama.cpp/g...c/ggml-openvino/openvino/op/permute.cpp

@@ -15,2 +15,3 @@ #include "../node_context.h"

#include <openvino/op/transpose.hpp>
#include <vector>

@@ -26,12 +27,29 @@ namespace ov {

int op_case = context.get_op_case();
FRONT_END_CHECK_IMPLEMENTED(op_case == 1 || op_case == 2 || op_case == 3 || op_case == 4,
"Unsupported PERMUTE case");
FRONT_END_CHECK_IMPLEMENTED(op_case != 0, "Unsupported PERMUTE case");
// op_case 1 is trivial permute
// op_case 2 is to permute Q. It has a preceding VIEW that reshapes Q to restore the sequqence dimension
// op_case 3 4 it to permute KV cache in the default layout
// op_case 5 6 is to permute V cache when `-fa off`, where v_trans=true
ov::Output<Node> res;
auto src = context.get_input(0);
auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3});
ov::Output<Node> src;
if (op_case == 3 || op_case == 4 || op_case == 5 || op_case == 6) {
src = context.get_input(0);
} else {
src = process_view_input_new(context, 0);
}
std::vector<int64_t> perm_values{0, 2, 1, 3};
const int32_t * op_params = context.get_output_op_params();
if (op_params != nullptr) {
for (size_t input_axis = 0; input_axis < perm_values.size(); ++input_axis) {
const size_t output_axis = static_cast<size_t>(op_params[input_axis]);
perm_values[perm_values.size() - 1 - output_axis] =
static_cast<int64_t>(perm_values.size() - 1 - input_axis);
}
}
auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values);
if (op_case == 1 || context.is_stateful()) {
res = std::make_shared<ov::op::v1::Transpose>(src, perm);
} else if (op_case == 4) {
} else if (op_case == 2) {
auto output_shape = context.get_output_shape().to_shape();

@@ -59,2 +77,6 @@ auto n_heads = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[1]});

int64_t n_heads = output_shape[1];
if (op_case == 5 || op_case == 6) {
head_size = output_shape[2];
n_heads = output_shape[1];
}
int64_t ctx_per_seq = cache_shape[2].is_static() ? cache_shape[2].get_length() : -1;

@@ -66,3 +88,3 @@ int64_t n_seq = cache_shape[1].get_length();

attention_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[2]});
} else if (op_case == 2) {
} else if (op_case == 3 || op_case == 5) {
attention_size = context.get_input("attention_size");

@@ -87,14 +109,37 @@ } else {

// 1. reshape to [n_seq, ctx_per_seq, n_heads, head_size]
// 1. reshape to [n_seq, ctx_per_seq, n_heads, head_size] (for `-fa off` [n_seq, n_heads, head_size, ctx_per_seq])
// 2. slice out the active sequences
// 3. slice out the attention part in each sequence
// 4. permute
// 4. permute (skip for `-fa off`)
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto src_reshaped = std::make_shared<ov::op::v1::Reshape>(
src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}), false);
auto slice1 = std::make_shared<ov::op::v8::Slice>(src_reshaped, seq_active_start, seq_active_end, one, zero);
auto slice2 = std::make_shared<ov::op::v8::Slice>(slice1, zero, attention_size, one, one);
res = std::make_shared<ov::op::v1::Transpose>(slice2, perm);
if (op_case == 3 || op_case == 4) {
auto src_reshaped = std::make_shared<ov::op::v1::Reshape>(
src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}),
false);
ov::Output<ov::Node> after_seq_slice;
if (n_seq == 1) {
after_seq_slice = src_reshaped;
} else {
after_seq_slice =
std::make_shared<ov::op::v8::Slice>(src_reshaped, seq_active_start, seq_active_end, one, zero);
}
auto slice2 = std::make_shared<ov::op::v8::Slice>(after_seq_slice, zero, attention_size, one, one);
res = std::make_shared<ov::op::v1::Transpose>(slice2, perm);
} else {
auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
auto src_reshaped = std::make_shared<ov::op::v1::Reshape>(
src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, n_heads, head_size, ctx_per_seq}),
false);
ov::Output<ov::Node> after_seq_slice;
if (n_seq == 1) {
after_seq_slice = src_reshaped;
} else {
after_seq_slice =
std::make_shared<ov::op::v8::Slice>(src_reshaped, seq_active_start, seq_active_end, one, zero);
}
auto slice2 = std::make_shared<ov::op::v8::Slice>(after_seq_slice, zero, attention_size, one, three);
res = slice2;
}
}

@@ -101,0 +146,0 @@ return rename_outputs_with_suffix({res}, context.get_name());

+13
-6
vendor/llama.cpp/g...c/ggml-openvino/openvino/op/reshape.cpp

@@ -13,3 +13,2 @@ #include "../node_context.h"

#include <openvino/op/reshape.hpp>
#include <stdexcept>
#include <vector>

@@ -24,3 +23,4 @@

num_inputs_check(context, 1, 1);
if (context.get_input_shape(0) == context.get_output_shape()) {
if (context.get_input(0).get_partial_shape().is_static() &&
context.get_input_shape(0) == context.get_output_shape()) {
return {context.get_input(0)};

@@ -39,8 +39,8 @@ }

new_shape_node = ov::op::v0::Constant::create(
ov::element::i64, {3},
std::vector<int64_t>{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
ov::element::i64, {3}, std::vector<int64_t>{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
} else {
new_shape_node = ov::op::v0::Constant::create(
ov::element::i64, {4},
std::vector<int64_t>{(int64_t) output_shape[0], -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
std::vector<int64_t>{(int64_t) output_shape[0], -1, (int64_t) output_shape[2],
(int64_t) output_shape[3]});
}

@@ -53,3 +53,10 @@ } else if (op_case == 2) {

} else if (op_case == 3) {
throw std::runtime_error("might be outdated RESHAPE case");
// - 14: [ 1, 1024, 1, 1] RESHAPE Vcur-0 (reshaped) (reshaped)
// [ 512, 2, 1, 1] 0: RESHAPE Vcur-0 (reshaped)
// - 15: [ 1, 524288, 1, 1] RESHAPE cache_v_l0 (reshaped)
// [ 512, 1024, 1, 1] 0: NONE cache_v_l0
// - 16: [ 1, 524288, 1, 1] SET_ROWS cache_v_l0 (reshaped) (view)
// [ 1, 1024, 1, 1] 0: RESHAPE Vcur-0 (reshaped) (reshaped)
// [ 1024, 1, 1, 1] 1: NONE leaf_11
// [ 1, 524288, 1, 1] 2: RESHAPE cache_v_l0 (reshaped)
new_shape_node = ov::op::v0::Constant::create(

@@ -56,0 +63,0 @@ ov::element::i64, {4}, std::vector<int64_t>{(int64_t) output_shape[0], (int64_t) output_shape[1], -1, 1});

+1
-1
vendor/llama.cpp/g.../ggml-openvino/openvino/op/rms_norm.cpp

@@ -22,3 +22,3 @@ #include "../node_context.h"

auto input_node = context.get_input(0);
auto input_node = process_view_input_new(context, 0);
auto square = std::make_shared<ov::op::v1::Power>(

@@ -25,0 +25,0 @@ input_node, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {2.0f}));

+131
-35
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/op/rope.cpp

@@ -10,2 +10,3 @@ #include "../node_context.h"

#include <openvino/op/add.hpp>
#include <openvino/op/broadcast.hpp>
#include <openvino/op/concat.hpp>

@@ -42,4 +43,3 @@ #include <openvino/op/constant.hpp>

int32_t * op_params = context.get_output_op_params();
const int mode = (op_case & 0xFFFF0000) >> 16;
op_case = (op_case & 0x0000FFFF);
const int mode = op_case;

@@ -61,3 +61,3 @@ constexpr int TYPE_NORMAL = 0;

}
auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode == TYPE_IMROPE);
auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode == TYPE_IMROPE, false);
sin_theta_node = sin_cos.first;

@@ -67,6 +67,4 @@ cos_theta_node = sin_cos.second;

if (op_case == 2) {
// The input comes from a VIEW
int slice_len = output_shape[2] * output_shape[3];
data_node = process_view_input(context, 0, slice_len).get_node_shared_ptr();
if (context.get_view_input_size(0) > 0) {
data_node = process_view_input_new(context, 0).get_node_shared_ptr();
if (context.is_stateful()) {

@@ -78,3 +76,4 @@ auto data_shape = ov::op::v0::Constant::create(

auto data_shape = ov::op::v0::Constant::create(
ov::element::i64, {4}, std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
ov::element::i64, {4},
std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
data_node = std::make_shared<ov::op::v1::Reshape>(data_node, data_shape, false);

@@ -84,31 +83,123 @@ }

auto output_type = context.get_output_type();
if (data_node->get_element_type() != ov::element::f32) {
data_node = std::make_shared<ov::op::v0::Convert>(data_node, ov::element::f32);
}
// TODO(openvino-gpu-rope-fusion): TEMPORARY WORKAROUND - do NOT revert until the
// OpenVINO GPU plugin is updated.
//
// For TYPE_NORMAL rope (both stateful and stateless) we emit the Flux-style
// interleaved pattern below so the GPU plugin's RoPEFusionFlux matcher folds it
// into ov::op::internal::RoPE. The matcher requires rank-4 inputs, which is why
// the original even/odd Slice translation (kept in the `else if (mode ==
// TYPE_NORMAL)` branch below for reference) does not get fused.
//
// Once the GPU plugin's RoPE fusion is extended to also recognize the original
// even/odd Slice form, this Flux rewrite should be removed and both modes should
// be restored to the captured even/odd translation. Until then, keep both paths:
// the active Flux rewrite here and the previous translation preserved below.
if (mode == TYPE_NORMAL) {
auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]});
Output<Node> even_slice;
Output<Node> odd_slice;
int32_t unsqueeze_dim = context.is_stateful() ? 3 : 4;
even_slice = std::make_shared<ov::op::v8::Slice>(data_node, zero, end, two, neg_one);
odd_slice = std::make_shared<ov::op::v8::Slice>(data_node, one, end, two, neg_one);
// Emit the Flux-style interleaved-RoPE pattern so the GPU plugin's
// RoPEFusionFlux matcher folds this subgraph into ov::op::internal::RoPE:
// x_paired = Reshape(x, [1, S, n_heads, head_size/2, 2])
// x0, x1 = Split(x_paired, axis=-1, num_splits=2)
// x1_neg = x1 * -1
// x_rotated = Reshape(Concat([x1_neg, x0], axis=-1), [1, S, n_heads, head_size])
// y = x * t_cos + x_rotated * t_sin
// Mathematically equivalent to the even/odd Slice form below.
//
// RoPEFusionFlux requires rank_equals(4) on x, t_cos and t_sin. The cos/sin
// tables are already built rank-4 ([1, S, 1, head_size/2]) for both modes. In
// stateful mode the data arrives rank-3 ([S, n_heads, head_size]), so lift it
// to rank-4 ([1, S, n_heads, head_size]) here. Stateful RoPE already produced
// rank-4 output, so downstream attention is unaffected.
if (context.is_stateful()) {
auto r4_shape = ov::op::v0::Constant::create(
ov::element::i64, {4},
std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
data_node = std::make_shared<ov::op::v1::Reshape>(data_node, r4_shape, false);
}
const int64_t head_size = static_cast<int64_t>(output_shape[3]);
const int64_t n_heads = static_cast<int64_t>(output_shape[2]);
const int64_t half = head_size / 2;
Output<Node> first_half =
std::make_shared<ov::op::v1::Subtract>(std::make_shared<ov::op::v1::Multiply>(even_slice, cos_theta_node),
std::make_shared<ov::op::v1::Multiply>(odd_slice, sin_theta_node));
Output<Node> second_half =
std::make_shared<ov::op::v1::Add>(std::make_shared<ov::op::v1::Multiply>(even_slice, sin_theta_node),
std::make_shared<ov::op::v1::Multiply>(odd_slice, cos_theta_node));
auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f});
first_half = std::make_shared<ov::op::v0::Unsqueeze>(first_half,
ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim}));
second_half = std::make_shared<ov::op::v0::Unsqueeze>(second_half,
ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim}));
auto stack = std::make_shared<ov::op::v0::Concat>(OutputVector{first_half, second_half}, unsqueeze_dim);
auto paired_shape =
ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector<int64_t>{1, -1, n_heads, half, 2});
auto x_paired = std::make_shared<ov::op::v1::Reshape>(data_node, paired_shape, false);
auto data_shape = ov::op::v0::Constant::create(
ov::element::i64, {4}, std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
res = std::make_shared<ov::op::v1::Reshape>(stack, data_shape, false);
} else if (mode == TYPE_NEOX) {
auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1});
auto data_split = std::make_shared<ov::op::v1::Split>(x_paired, split_axis, 2);
Output<Node> x0 = data_split->outputs()[0];
Output<Node> x1 = data_split->outputs()[1];
auto x1_neg = std::make_shared<ov::op::v1::Multiply>(x1, neg_one_f);
auto x_rotated_paired = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{x1_neg, x0}, -1);
auto flat_shape =
ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, -1, n_heads, head_size});
auto x_rotated = std::make_shared<ov::op::v1::Reshape>(x_rotated_paired, flat_shape, false);
// Expand cos/sin from [..., head_size/2] to [..., head_size] by repeating each
// entry twice. Use special_zero on the final Reshape so the seq dim passes
// through dynamically. Final rank is 4 to satisfy the matcher's predicate.
auto expand_cos_sin = [&](Output<Node> cs) {
auto cs_unsq =
std::make_shared<ov::op::v0::Unsqueeze>(cs, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}));
auto bcast_target =
ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector<int64_t>{1, 1, 1, half, 2});
auto bcast =
std::make_shared<ov::op::v3::Broadcast>(cs_unsq, bcast_target, ov::op::BroadcastType::BIDIRECTIONAL);
auto flat = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, 0, 0, head_size});
return std::make_shared<ov::op::v1::Reshape>(bcast, flat, true);
};
Output<Node> cos_full = expand_cos_sin(cos_theta_node);
Output<Node> sin_full = expand_cos_sin(sin_theta_node);
auto y1 = std::make_shared<ov::op::v1::Multiply>(data_node, cos_full);
auto y2 = std::make_shared<ov::op::v1::Multiply>(x_rotated, sin_full);
res = std::make_shared<ov::op::v1::Add>(y1, y2);
}
// PRESERVED PREVIOUS TRANSLATION - Re-enable this branch (and remove the Flux branch above) once
// the GPU plugin's RoPE fusion is updated to recognize the even/odd Slice form;
// see the TODO(openvino-gpu-rope-fusion) note above. Do not delete.
//
// Original even/odd Slice form. In stateless mode it ran on rank-4 data
// ([1, S, n_heads, head_size]); in stateful mode on rank-3 data
// ([S, n_heads, head_size]). Either way it does not match RoPEFusionFlux
// (which needs rank-4 x in the interleaved layout), so the RoPE stays as
// discrete elementwise ops.
//
// } else if (mode == TYPE_NORMAL) {
// auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
// auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
// auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
// auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
// auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]});
// Output<Node> even_slice;
// Output<Node> odd_slice;
// // stateful data is rank 3 (unsqueeze at axis 3), stateless is rank 4 (axis 4)
// int32_t unsqueeze_dim = context.is_stateful() ? 3 : 4;
// even_slice = std::make_shared<ov::op::v8::Slice>(data_node, zero, end, two, neg_one);
// odd_slice = std::make_shared<ov::op::v8::Slice>(data_node, one, end, two, neg_one);
//
// Output<Node> first_half =
// std::make_shared<ov::op::v1::Subtract>(std::make_shared<ov::op::v1::Multiply>(even_slice, cos_theta_node),
// std::make_shared<ov::op::v1::Multiply>(odd_slice, sin_theta_node));
// Output<Node> second_half =
// std::make_shared<ov::op::v1::Add>(std::make_shared<ov::op::v1::Multiply>(even_slice, sin_theta_node),
// std::make_shared<ov::op::v1::Multiply>(odd_slice, cos_theta_node));
//
// first_half = std::make_shared<ov::op::v0::Unsqueeze>(first_half,
// ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim}));
// second_half = std::make_shared<ov::op::v0::Unsqueeze>(second_half,
// ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim}));
// auto stack = std::make_shared<ov::op::v0::Concat>(OutputVector{first_half, second_half}, unsqueeze_dim);
//
// auto data_shape = ov::op::v0::Constant::create(
// ov::element::i64, {4}, std::vector<int64_t>{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]});
// res = std::make_shared<ov::op::v1::Reshape>(stack, data_shape, false);
else if (mode == TYPE_NEOX) {
auto data_split = std::make_shared<ov::op::v1::Split>(

@@ -129,4 +220,5 @@ data_node, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1}), 2);

} else if (mode == TYPE_IMROPE) {
int64_t n_dims = data_node->get_shape()[3];
auto cos_sin_shape = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{4}, std::vector<int64_t>{1,-1,1,(n_dims >> 1)});
int64_t n_dims = data_node->get_output_partial_shape(0)[3].get_length();
auto cos_sin_shape = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{4},
std::vector<int64_t>{1, -1, 1, (n_dims >> 1)});
auto cos_reshaped = std::make_shared<ov::op::v1::Reshape>(cos_theta_node, cos_sin_shape, true);

@@ -150,2 +242,6 @@ auto sin_reshaped = std::make_shared<ov::op::v1::Reshape>(sin_theta_node, cos_sin_shape, true);

if (res.get_element_type() != output_type) {
res = std::make_shared<ov::op::v0::Convert>(res, output_type);
}
return rename_outputs_with_suffix({res}, context.get_name());

@@ -152,0 +248,0 @@ }

+3
-3
vendor/llama.cpp/g.../ggml-openvino/openvino/op/set_rows.cpp

@@ -31,3 +31,3 @@ #include "../node_context.h"

auto data = context.get_input(0);
auto data = process_view_input_new(context, 0);
auto indices = context.get_input(1);

@@ -38,3 +38,3 @@ auto dst = context.get_input(2);

auto dst_shape = context.get_output_shape().to_shape();
auto row_size = context.get_input_shape(2)[3].get_length();

@@ -46,3 +46,3 @@ auto ind_squeezed =

ov::op::v0::Constant::create(ov::element::i64, {4},
{(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) dst_shape[3]}),
{(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}),
false);

@@ -49,0 +49,0 @@ auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2});

+67
-50
vendor/llama.cpp/g...c/ggml-openvino/openvino/op/softmax.cpp

@@ -5,14 +5,12 @@ #include "../node_context.h"

#include <climits>
#include <cmath>
#include <cstdint>
#include <cstring>
#include <memory>
#include <openvino/core/node.hpp>
#include <openvino/core/node_output.hpp>
#include <openvino/frontend/exception.hpp>
#include <openvino/op/add.hpp>
#include <openvino/op/concat.hpp>
#include <openvino/op/constant.hpp>
#include <openvino/op/convert.hpp>
#include <openvino/op/matmul.hpp>
#include <openvino/op/multiply.hpp>
#include <openvino/op/slice.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/softmax.hpp>

@@ -26,60 +24,79 @@ #include <vector>

// Reimplementation of GGML_OP_SOFT_MAX semantics for OpenVINO backend:
// 1) logits = src0 * scale
// 2) logits += mask (if provided)
// 3) softmax over the last dimension
OutputVector translate_soft_max(const NodeContext & context) {
// TODO code is outdated
num_inputs_check(context, 1, 2);
auto input_node = context.get_input(0).get_node_shared_ptr();
ov::Output<Node> res;
float scale = 1.0f;
float max_bias = 0.0f;
auto * op_params = context.get_output_op_params();
memcpy(&scale, (float *) op_params + 0, sizeof(float));
memcpy(&max_bias, (float *) op_params + 1, sizeof(float));
auto src0_shape = context.get_input_shape(0).get_shape();
const uint32_t h = src0_shape[2];
const uint32_t n_head = src0_shape[0];
const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));
memcpy(&scale, (float *) context.get_output_op_params() + 0, sizeof(float));
memcpy(&max_bias, (float *) context.get_output_op_params() + 1, sizeof(float));
const float m0 = powf(2.0f, -(max_bias) / n_head_log2);
const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
const float slope =
(max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2 * (h - n_head_log2) + 1) : 1.0f;
ov::Output<ov::Node> logits = context.get_input(0);
auto scale_node = std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{}, std::vector<float>{scale});
auto scaled_input = std::make_shared<ov::op::v1::Multiply>(input_node, scale_node);
if (context.get_input_size() < 2) {
res = std::make_shared<ov::op::v8::Softmax>(scaled_input, 2);
return rename_outputs_with_suffix({res}, context.get_name());
// Apply scale first: logits = src0 * scale
if (scale != 1.0f) {
auto scale_const =
std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{}, std::vector<float>{scale});
logits = std::make_shared<ov::op::v1::Multiply>(logits, scale_const);
}
ov::Output<ov::Node> mask_node_sliced;
if (context.has_input("KQ_mask_sliced")) {
mask_node_sliced = context.get_input("KQ_mask_sliced");
} else {
auto token_len = get_dimensions(input_node, {1});
auto mask_node = context.get_input(1);
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
mask_node_sliced = std::make_shared<ov::op::v8::Slice>(mask_node, zero, token_len, one, one);
}
FRONT_END_CHECK_IMPLEMENTED(!(max_bias > 0.0f && context.get_input_size() < 2),
"OpenVINO softmax ALiBi path requires mask input");
if (mask_node_sliced.get_element_type() != context.get_output_type()) {
mask_node_sliced = std::make_shared<ov::op::v0::Convert>(mask_node_sliced, context.get_output_type());
}
// Optional mask add: logits += mask
// For max_bias > 0 (ALiBi), apply per-head slope to mask before adding.
if (context.get_input_size() > 1) {
ov::Output<ov::Node> mask = context.get_input(1);
Output<Node> slope_mask;
if (slope != 1.0f) {
auto slope_node =
std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{}, std::vector<float>{slope});
slope_mask = std::make_shared<ov::op::v1::Multiply>(mask_node_sliced, slope_node);
throw std::runtime_error("Slope != 1.0f in softmax has not been tested, verify it before use.");
// For stateful
std::string mask_name = "KQ_mask_sliced";
if (context.get_input_names()[1].find("swa") != std::string::npos) {
mask_name = "KQ_mask_swa_sliced";
}
if (context.has_input(mask_name)) {
mask = context.get_input(mask_name);
}
if (mask.get_element_type() != logits.get_element_type()) {
mask = std::make_shared<ov::op::v0::Convert>(mask, logits.get_element_type());
}
if (max_bias > 0.0f) {
auto out_shape = context.get_output_shape().to_shape();
FRONT_END_CHECK_IMPLEMENTED(out_shape.size() == 4, "OpenVINO softmax ALiBi path expects rank-4 tensor");
const uint32_t n_head = static_cast<uint32_t>(out_shape[1]);
FRONT_END_CHECK_IMPLEMENTED(n_head > 0, "OpenVINO softmax ALiBi path expects n_head > 0");
const uint32_t n_head_log2 = 1u << static_cast<uint32_t>(std::floor(std::log2(static_cast<float>(n_head))));
const float m0 = std::pow(2.0f, -(max_bias) / static_cast<float>(n_head_log2));
const float m1 = std::pow(2.0f, -(max_bias / 2.0f) / static_cast<float>(n_head_log2));
std::vector<float> slopes(n_head);
for (uint32_t h = 0; h < n_head; ++h) {
slopes[h] = h < n_head_log2 ? std::pow(m0, static_cast<float>(h + 1)) :
std::pow(m1, static_cast<float>(2 * (h - n_head_log2) + 1));
}
ov::Output<ov::Node> slope_node =
std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{n_head}, slopes);
if (slope_node.get_element_type() != mask.get_element_type()) {
slope_node = std::make_shared<ov::op::v0::Convert>(slope_node, mask.get_element_type());
}
auto slope_shape = std::make_shared<ov::op::v0::Constant>(
ov::element::i64, ov::Shape{4}, std::vector<int64_t>{1, static_cast<int64_t>(n_head), 1, 1});
auto slope_4d = std::make_shared<ov::op::v1::Reshape>(slope_node, slope_shape, false);
mask = std::make_shared<ov::op::v1::Multiply>(mask, slope_4d);
}
logits = std::make_shared<ov::op::v1::Add>(logits, mask);
}
slope_mask = mask_node_sliced;
auto input_slope_mask_node = std::make_shared<ov::op::v1::Add>(scaled_input, slope_mask);
// Softmax along last dimension (equivalent to ggml softmax over ne[0]).
auto res = std::make_shared<ov::op::v8::Softmax>(logits, -1);
res = std::make_shared<ov::op::v8::Softmax>(input_slope_mask_node, 2);
return rename_outputs_with_suffix({res}, context.get_name());

@@ -86,0 +103,0 @@ }

+32
-1
vendor/llama.cpp/g...ggml-openvino/openvino/op/transpose.cpp

@@ -15,4 +15,35 @@ #include "../node_context.h"

// Compute permute order from input/output shape and stride information
// so it adapts to different input and output layouts.
auto input_shape = context.get_input_shape(0).to_shape();
auto input_stride = context.get_input_stride(0);
auto output_shape = context.get_output_shape().to_shape();
auto output_stride = context.get_output_stride();
// Compute permute order by matching output and input stride rankings.
// Build <stride, dim_index> pairs.
std::vector<std::pair<size_t, int>> output_stride_dims;
std::vector<std::pair<size_t, int>> input_stride_dims;
for (int i = 0; i < 4; ++i) {
output_stride_dims.push_back({output_stride[i], i});
input_stride_dims.push_back({input_stride[i], i});
}
// Sort by stride in descending order.
std::sort(output_stride_dims.rbegin(), output_stride_dims.rend());
std::sort(input_stride_dims.rbegin(), input_stride_dims.rend());
// Build permute order.
std::vector<int64_t> permute_order(4);
for (int i = 0; i < 4; ++i) {
int output_dim = output_stride_dims[i].second;
int input_dim = input_stride_dims[i].second;
permute_order[output_dim] = input_dim;
}
auto input = process_view_input_new(context, 0);
auto res = std::make_shared<ov::op::v1::Transpose>(
context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 1, 3, 2}));
input, ov::op::v0::Constant::create(ov::element::i64, {4}, permute_order));
return rename_outputs_with_suffix({res}, context.get_name());

@@ -19,0 +50,0 @@ }

+1
-1
vendor/llama.cpp/g...gml-openvino/openvino/op/unary_silu.cpp

@@ -17,3 +17,3 @@ #include "../node_context.h"

auto input = context.get_input(0);
auto input = process_view_input_new(context, 0);
auto sigmoid = std::make_shared<ov::op::v0::Sigmoid>(input);

@@ -20,0 +20,0 @@ auto res = std::make_shared<ov::op::v1::Multiply>(input, sigmoid);

+90
-25
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/op/view.cpp
#include "../op_table.h"
#include "../utils.h"
#include <openvino/op/constant.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/slice.hpp>
#include <set>
namespace ov {

@@ -12,38 +17,98 @@ namespace frontend {

if (context.get_op_case() == 2) {
auto dst_shape = context.get_output_shape().to_shape();
return rename_outputs_with_suffix({process_view_input(context, 0, dst_shape[2] * dst_shape[3])},
context.get_name());
if (!context.is_static()) {
return {context.get_input(0)};
}
// op_case 3
if (context.get_op_case() == 3) {
auto input = context.get_input(0);
auto input_ov_shape = input.get_partial_shape();
auto input_llama_shape = context.get_input_shape(0).to_shape();
auto input = context.get_input(0);
auto src_shape = context.get_input_shape(0);
auto dst_shape = context.get_output_shape();
// if the input ov shape size is different from the input llama shape size, it means the input is already reshaped and we need to reshape it back to the original shape before slicing
if (input_ov_shape.size() != input_llama_shape.size()) {
input = std::make_shared<ov::op::v1::Reshape>(input, ov::op::v0::Constant::create(ov::element::i64, {input_llama_shape.size()}, input_llama_shape), false);
if (src_shape.rank().is_dynamic() || dst_shape.rank().is_dynamic()) {
return {input};
}
int64_t src_elems = 1, dst_elems = 1;
for (int64_t i = 0; i < src_shape.rank().get_length(); ++i) {
if (src_shape[i].is_dynamic()) {
return {input};
}
src_elems *= src_shape[i].get_length();
}
for (int64_t i = 0; i < dst_shape.rank().get_length(); ++i) {
if (dst_shape[i].is_dynamic()) {
return {input};
}
dst_elems *= dst_shape[i].get_length();
}
auto dst_shape = context.get_output_shape().to_shape();
if (dst_elems >= src_elems) {
return {input};
}
// find the index of dst_shape that is different from input shape, and use that index to slice the input
int slice_dim = -1;
for (size_t i = 0; i < dst_shape.size(); ++i) {
if (dst_shape[i] != input_llama_shape[i]) {
slice_dim = i;
auto src_stride = context.get_input_stride(0);
auto dst_stride = context.get_output_stride();
size_t view_offset = context.get_output_op_offset();
bool same_stride = (src_stride.size() == dst_stride.size());
if (same_stride) {
for (size_t i = 0; i < src_stride.size(); ++i) {
if (src_stride[i] != dst_stride[i]) {
same_stride = false;
break;
}
}
}
auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {dst_shape[slice_dim]});
auto stride = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim});
auto sliced = std::make_shared<ov::op::v8::Slice>(input, begin, end, stride, axes);
return {sliced};
if (!same_stride) {
return {input};
}
return {context.get_input(0)};
auto src_ov_shape = src_shape.to_shape();
auto dst_ov_shape = dst_shape.to_shape();
size_t ndims = src_ov_shape.size();
if (dst_ov_shape.size() != ndims) {
return {input};
}
std::vector<int> diff_dims;
for (size_t i = 0; i < ndims; ++i) {
if (src_ov_shape[i] != dst_ov_shape[i]) {
diff_dims.push_back(static_cast<int>(i));
}
}
if (diff_dims.size() != 1) {
return {input};
}
int slice_dim = diff_dims[0];
int64_t dim_size = static_cast<int64_t>(src_ov_shape[slice_dim]);
size_t ov_stride_for_dim = 1;
for (size_t i = slice_dim + 1; i < ndims; ++i) {
ov_stride_for_dim *= src_ov_shape[i];
}
size_t elem_size = src_stride.back();
if (elem_size == 0) {
elem_size = 1;
}
int64_t begin_val = 0;
if (ov_stride_for_dim > 0 && elem_size > 0) {
begin_val = static_cast<int64_t>((view_offset / elem_size) / ov_stride_for_dim);
}
int64_t end_val = begin_val + static_cast<int64_t>(dst_ov_shape[slice_dim]);
if (begin_val < 0 || end_val > dim_size) {
return {input};
}
auto sliced =
std::make_shared<ov::op::v8::Slice>(input, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim}));
sliced->set_friendly_name(context.get_output_name());
return {sliced->output(0)};
}

@@ -50,0 +115,0 @@ 

+1
-1
vendor/llama.cpp/g...ecompression_convert_constant_folding.h
#pragma once
#include "mark_decompression_convert_constant_folding.h"
#include "openvino/core/visibility.hpp"
#include "openvino/pass/matcher_pass.hpp"
#include "openvino/core/visibility.hpp"

@@ -7,0 +7,0 @@ #ifdef OPENVINO_STATIC_LIBRARY

+43
-40
vendor/llama.cpp/g...openvino/openvino/translate_session.cpp

@@ -16,2 +16,3 @@ #include "translate_session.h"

#include <openvino/core/preprocess/pre_post_process.hpp>
#include <openvino/core/type/element_type.hpp>
#include <openvino/op/add.hpp>

@@ -81,5 +82,4 @@ #include <openvino/op/broadcast.hpp>

void add_sliced_mask(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) {
auto create_sliced_mask = [&](const std::string & mask_name, const std::string & sliced_name, bool is_static) {
void add_sliced_mask_stateful(TensorMap & tensor_map) {
auto create_sliced_mask = [&](const std::string & mask_name, const std::string & sliced_name) {
if ((tensor_map.find(mask_name) != tensor_map.end()) &&

@@ -89,29 +89,22 @@ (tensor_map.find("token_len_per_seq") != tensor_map.end())) {

auto mask = tensor_map.at(mask_name).get_node_shared_ptr();
std::shared_ptr<ov::Node> mask_sliced;
if (is_static) {
mask_sliced = mask;
} else if (ggml_model_decoder.is_stateful()) {
auto zero_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {0,0});
auto one_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {1,1});
auto zero_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto three_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
auto neg_one_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto axes = ov::op::v0::Constant::create(ov::element::i64, {2}, {-2,-1});
auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr();
auto gather_inp_pos = std::make_shared<ov::op::v8::Gather>(inp_pos, neg_one_1d, three_1d);
auto reshaped_inp_pos = std::make_shared<ov::op::v1::Reshape>(gather_inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), false);
auto inp_pos_incremented = std::make_shared<ov::op::v1::Add>(reshaped_inp_pos, ov::op::v0::Constant::create(ov::element::i32, ov::Shape{1}, {1}));
auto stop = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{token_len_per_seq, std::make_shared<v1::ConvertLike>(inp_pos_incremented, token_len_per_seq)}, 0);
mask_sliced =
std::make_shared<ov::op::v8::Slice>(mask, zero_2d, stop, one_2d, axes);
mask_sliced = std::make_shared<ov::op::v0::Convert>(mask_sliced, ov::element::f16);
mask_sliced->set_friendly_name(sliced_name);
} else {
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
mask_sliced = std::make_shared<ov::op::v8::Slice>(mask, zero, token_len_per_seq, one, two);
mask_sliced = std::make_shared<ov::op::v0::Convert>(mask_sliced, ov::element::f16);
mask_sliced->set_friendly_name(sliced_name);
}
std::shared_ptr<ov::Node> mask_sliced = mask;
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr();
auto last_inp_pos = std::make_shared<ov::op::v8::Gather>(inp_pos, neg_one, three);
auto last_inp_pos_1d = std::make_shared<ov::op::v1::Reshape>(
last_inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), false);
auto last_inp_pos_cvt = std::make_shared<ov::op::v0::Convert>(last_inp_pos_1d, ov::element::i64);
auto last_inp_pos_inc = std::make_shared<ov::op::v1::Add>(last_inp_pos_cvt, one);
mask_sliced = std::make_shared<ov::op::v8::Slice>(mask, zero, last_inp_pos_inc, step, axes);
mask_sliced = std::make_shared<ov::op::v0::Convert>(mask_sliced, ov::element::f16);
mask_sliced->set_friendly_name(sliced_name);
tensor_map.insert({sliced_name, mask_sliced->output(0)});

@@ -121,7 +114,14 @@ }

create_sliced_mask("self_kq_mask", "KQ_mask_sliced", ggml_model_decoder.is_static());
create_sliced_mask("self_kq_mask_swa", "KQ_mask_swa_sliced", ggml_model_decoder.is_static());
create_sliced_mask("self_kq_mask", "KQ_mask_sliced");
create_sliced_mask("self_kq_mask_swa", "KQ_mask_swa_sliced");
}
void add_rope_sin_cos(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) {
// When ROPE ops in the graph have divergent op_params (e.g. gemma4's mixed
// SWA/non-SWA layers with different n_dims or freq_base), a shared sin/cos
// precompute cannot broadcast across every ROPE use. Skip it here and let
// translate_rope() build sin/cos per-op from its own op_params.
if (ggml_model_decoder.has_mixed_rope_params()) {
return;
}
int32_t * rope_params = ggml_model_decoder.get_rope_params();

@@ -149,4 +149,7 @@ if (tensor_map.find("inp_pos") == tensor_map.end() || rope_params == nullptr) {

void preprocess(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) {
add_sliced_mask(tensor_map, ggml_model_decoder);
add_rope_sin_cos(tensor_map, ggml_model_decoder);
if (ggml_model_decoder.is_stateful()) {
add_sliced_mask_stateful(tensor_map);
}
// This optimization is error-prone
// add_rope_sin_cos(tensor_map, ggml_model_decoder);
}

@@ -296,7 +299,7 @@

std::map<std::string, int> model_output_indexes;
for (size_t i=0; i<output_names.size(); i++) {
for (size_t i = 0; i < output_names.size(); i++) {
model_output_indexes.insert(std::make_pair(output_names[i], i));
}
ov::preprocess::PrePostProcessor ppp(model);
for (size_t i=0; i<model->get_output_size(); i++) {
for (size_t i = 0; i < model->get_output_size(); i++) {
auto output_friendly_name = model->output(i).get_node_shared_ptr()->get_friendly_name();

@@ -306,6 +309,6 @@ auto output_id = model_output_indexes[output_friendly_name];

auto decoder_output_shape = ggml_model_decoder->get_output_shape(output_id);
if (model_output_shape.rank().is_static() && decoder_output_shape.rank().is_static()
&& model_output_shape.rank().get_length() + 1 == decoder_output_shape.rank().get_length()
&& decoder_output_shape[0].is_static() && decoder_output_shape[0].get_length() == 1) {
ppp.output(i).postprocess().custom([](const ov::Output<ov::Node>& node) {
if (model_output_shape.rank().is_static() && decoder_output_shape.rank().is_static() &&
model_output_shape.rank().get_length() + 1 == decoder_output_shape.rank().get_length() &&
decoder_output_shape[0].is_static() && decoder_output_shape[0].get_length() == 1) {
ppp.output(i).postprocess().custom([](const ov::Output<ov::Node> & node) {
auto axes = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{1}, {0});

@@ -312,0 +315,0 @@ return std::make_shared<ov::op::v0::Unsqueeze>(node, axes);

+5
-4
vendor/llama.cpp/g...l-openvino/openvino/translate_session.h

@@ -12,7 +12,8 @@ #pragma once

public:
TranslateSession(const frontend::InputModel::Ptr& input_model,
const std::unordered_map<std::string, CreatorFunction>& translator_map, bool naive = false);
TranslateSession(const frontend::InputModel::Ptr & input_model,
const std::unordered_map<std::string, CreatorFunction> & translator_map,
bool naive = false);
std::shared_ptr<Model> get_converted_model();
std::shared_ptr<Model> translate_graph(const frontend::InputModel::Ptr& input_model);
std::shared_ptr<Model> translate_graph(const frontend::InputModel::Ptr & input_model);

@@ -22,3 +23,3 @@ private:

const frontend::InputModel::Ptr m_input_model;
const std::unordered_map<std::string, CreatorFunction>& m_translator_map;
const std::unordered_map<std::string, CreatorFunction> & m_translator_map;
std::shared_ptr<Model> m_ov_model;

@@ -25,0 +26,0 @@ bool m_naive;

+548
-3
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/utils.cpp

@@ -20,2 +20,3 @@ #include "utils.h"

#include <openvino/op/sin.hpp>
#include <openvino/op/split.hpp>
#include <openvino/op/squeeze.hpp>

@@ -127,3 +128,4 @@ #include <openvino/op/subtract.hpp>

if (stateful) {
inp_pos = std::make_shared<ov::op::v0::Squeeze>(inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
inp_pos =
std::make_shared<ov::op::v0::Squeeze>(inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
inp_pos = std::make_shared<ov::op::v0::Convert>(inp_pos, ov::element::f32);

@@ -217,4 +219,5 @@ auto pos_perm =

theta = std::make_shared<ov::op::v1::Add>(std::make_shared<ov::op::v1::Multiply>(theta_interp, one_minus_ramp),
std::make_shared<ov::op::v1::Multiply>(theta_extrap, ramp_mix));
theta =
std::make_shared<ov::op::v1::Add>(std::make_shared<ov::op::v1::Multiply>(theta_interp, one_minus_ramp),
std::make_shared<ov::op::v1::Multiply>(theta_extrap, ramp_mix));
mscale *= (1.0f + 0.1f * std::log(1.0f / freq_scale));

@@ -258,4 +261,546 @@ }

ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int input_index) {
auto input = context.get_input(input_index);
// Check if this input has view inputs
size_t view_input_size = context.get_view_input_size(input_index);
if (view_input_size == 0) {
// No view inputs, return the input as is
return input;
}
// If translate_view already resolved this VIEW (produced a Slice), the input
// will already have the expected shape — skip re-slicing.
auto expected_ov_shape = context.get_view_input_ov_shape(input_index, 0);
auto actual_shape = input.get_partial_shape();
if (expected_ov_shape.rank().is_static() && actual_shape.rank().is_static() &&
expected_ov_shape.rank() == actual_shape.rank()) {
bool shapes_match = true;
for (int64_t i = 0; i < expected_ov_shape.rank().get_length(); ++i) {
if (!expected_ov_shape[i].is_static() || !actual_shape[i].is_static()) {
shapes_match = false;
break;
}
if (expected_ov_shape[i] != actual_shape[i]) {
shapes_match = false;
break;
}
}
if (shapes_match) {
return input;
}
}
// In static mode, use Split instead of Slice for single-dimension reductions.
// This ensures NPUW's FOLD doesn't parametrize per-layer slice indices (which
// would introduce dynamic shapes). A shared Split node sits outside the repeated
// subgraph boundary; each layer receives one of its output ports.
if (context.is_static() && view_input_size == 1) {
auto view_stride_v = context.get_view_input_stride(input_index, 0);
auto view_src_stride_v = context.get_view_input_src_stride(input_index, 0);
auto view_ggml_shape = context.get_view_input_ggml_shape(input_index, 0);
auto view_src_ggml_shape = context.get_view_input_src_ggml_shape(input_index, 0);
auto view_offset = context.get_view_input_offset(input_index, 0);
auto view_src_offset = context.get_view_input_src_offset(input_index, 0);
size_t ndims = view_ggml_shape.size();
std::vector<int> diff_dims;
if (view_src_ggml_shape.size() == ndims) {
for (size_t i = 0; i < ndims; ++i) {
if (view_ggml_shape[i] != view_src_ggml_shape[i]) {
diff_dims.push_back(static_cast<int>(i));
}
}
}
if (diff_dims.size() == 1) {
int split_dim = diff_dims[0];
int64_t num_splits = static_cast<int64_t>(view_src_ggml_shape[split_dim]);
int64_t chunk_size = static_cast<int64_t>(view_ggml_shape[split_dim]);
// Only apply when slicing exactly 1 element from a multi-element dimension
if (chunk_size == 1 && num_splits > 1) {
// Check suffix strides match (dimensions after split_dim)
bool suffix_ok = view_stride_v.size() == view_src_stride_v.size();
if (suffix_ok) {
for (size_t i = static_cast<size_t>(split_dim) + 1; i < ndims; ++i) {
if (view_stride_v[i] != view_src_stride_v[i]) {
suffix_ok = false;
break;
}
}
}
if (suffix_ok && view_src_stride_v[split_dim] > 0) {
size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0;
int64_t split_index = static_cast<int64_t>(relative_offset / view_src_stride_v[split_dim]);
if (split_index >= 0 && split_index < num_splits) {
auto src_node = input.get_node_shared_ptr();
std::string rt_key = "split_dim_" + std::to_string(split_dim);
auto & rt_info = src_node->get_rt_info();
if (rt_info.find(rt_key) == rt_info.end()) {
auto axis_const =
ov::op::v0::Constant::create(ov::element::i64, {}, {static_cast<int64_t>(split_dim)});
auto split_node =
std::make_shared<ov::op::v1::Split>(input, axis_const, static_cast<size_t>(num_splits));
split_node->set_friendly_name(src_node->get_friendly_name() + "_split");
rt_info[rt_key] = split_node;
}
auto split_node = rt_info[rt_key].as<std::shared_ptr<ov::op::v1::Split>>();
return split_node->output(static_cast<size_t>(split_index));
}
}
}
}
}
// Lambda function to process a single view operation
auto process_single_view =
[](ov::Output<ov::Node> current, size_t view_offset, const std::vector<size_t> & view_stride,
const ov::Shape & view_ggml_shape, const ov::PartialShape & view_ov_shape, const std::string & view_name,
size_t view_src_offset, const std::vector<size_t> & view_src_stride, const ov::Shape & view_src_ggml_shape,
const ov::PartialShape & view_src_ov_shape, const std::string & view_src_name) -> ov::Output<ov::Node> {
auto build_reshape_pattern = [](const ov::PartialShape & target_ov_shape,
const ov::Shape & target_ggml_shape) -> std::vector<int64_t> {
const size_t ndims = target_ggml_shape.size();
std::vector<int64_t> reshape_pattern(ndims);
size_t dynamic_dims = 0;
if (target_ov_shape.rank().is_static() &&
target_ov_shape.rank().get_length() == static_cast<int64_t>(ndims)) {
for (size_t i = 0; i < ndims; ++i) {
if (target_ov_shape[i].is_static()) {
reshape_pattern[i] = target_ov_shape[i].get_length();
} else {
reshape_pattern[i] = -1;
++dynamic_dims;
}
}
} else {
dynamic_dims = 2;
}
if (dynamic_dims > 1) {
for (size_t i = 0; i < ndims; ++i) {
reshape_pattern[i] = static_cast<int64_t>(target_ggml_shape[i]);
}
}
return reshape_pattern;
};
auto build_prefix_tail_reshape_pattern = [](const ov::PartialShape & target_ov_shape,
const ov::Shape & target_ggml_shape, size_t prefix_dims,
int64_t tail_dim) -> std::vector<int64_t> {
std::vector<int64_t> reshape_pattern(prefix_dims + 1);
size_t dynamic_dims = 0;
if (target_ov_shape.rank().is_static() &&
target_ov_shape.rank().get_length() == static_cast<int64_t>(target_ggml_shape.size())) {
for (size_t i = 0; i < prefix_dims; ++i) {
if (target_ov_shape[i].is_static()) {
reshape_pattern[i] = target_ov_shape[i].get_length();
} else {
reshape_pattern[i] = -1;
++dynamic_dims;
}
}
} else {
dynamic_dims = 2;
}
if (dynamic_dims > 1) {
for (size_t i = 0; i < prefix_dims; ++i) {
reshape_pattern[i] = static_cast<int64_t>(target_ggml_shape[i]);
}
}
reshape_pattern[prefix_dims] = tail_dim;
return reshape_pattern;
};
bool same_stride = view_stride.size() == view_src_stride.size();
if (same_stride) {
for (size_t i = 0; i < view_stride.size(); ++i) {
if (view_stride[i] != view_src_stride[i]) {
same_stride = false;
break;
}
}
}
bool same_ggml_shape = view_ggml_shape.size() == view_src_ggml_shape.size();
if (same_ggml_shape) {
for (size_t i = 0; i < view_ggml_shape.size(); ++i) {
if (view_ggml_shape[i] != view_src_ggml_shape[i]) {
same_ggml_shape = false;
break;
}
}
}
if (same_stride && same_ggml_shape) {
return current;
}
if (same_stride) {
const size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0;
const size_t ndims = view_stride.size();
std::vector<int> diff_dims;
if (view_ggml_shape.size() == ndims && view_src_ggml_shape.size() == ndims) {
for (size_t i = 0; i < ndims; ++i) {
if (view_ggml_shape[i] != view_src_ggml_shape[i]) {
diff_dims.push_back(static_cast<int>(i));
}
}
}
if (diff_dims.size() == 1) {
const int slice_dim = diff_dims[0];
const int64_t dim_size = static_cast<int64_t>(view_src_ggml_shape[slice_dim]);
if (view_stride[slice_dim] > 0 && relative_offset % view_stride[slice_dim] == 0) {
const int64_t begin_val = static_cast<int64_t>((relative_offset / view_stride[slice_dim]) %
static_cast<size_t>(dim_size));
const int64_t end_val = begin_val + static_cast<int64_t>(view_ggml_shape[slice_dim]);
if (begin_val >= 0 && end_val <= dim_size) {
auto sliced = std::make_shared<ov::op::v8::Slice>(
current, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim}));
if (view_ov_shape.is_static()) {
auto reshaped = std::make_shared<ov::op::v1::Reshape>(
sliced,
ov::op::v0::Constant::create(ov::element::i64, {ndims}, view_ov_shape.to_shape()),
false);
reshaped->set_friendly_name(view_name);
return reshaped;
}
sliced->set_friendly_name(view_name);
return sliced;
}
}
int64_t tail_src_elems = 1;
int64_t tail_dst_elems = 1;
for (size_t i = slice_dim; i < ndims; ++i) {
tail_src_elems *= static_cast<int64_t>(view_src_ggml_shape[i]);
tail_dst_elems *= static_cast<int64_t>(view_ggml_shape[i]);
}
const size_t elem_stride = view_stride[ndims - 1];
int64_t tail_begin = 0;
if (elem_stride > 0) {
tail_begin =
static_cast<int64_t>((relative_offset / elem_stride) % static_cast<size_t>(tail_src_elems));
}
const int64_t tail_end = tail_begin + tail_dst_elems;
if (tail_begin >= 0 && tail_end <= tail_src_elems) {
std::vector<int64_t> flat_shape;
for (int i = 0; i < slice_dim; ++i) {
flat_shape.push_back(static_cast<int64_t>(view_src_ggml_shape[i]));
}
flat_shape.push_back(tail_src_elems);
const size_t flat_ndims = flat_shape.size();
auto flat = std::make_shared<ov::op::v1::Reshape>(
current, ov::op::v0::Constant::create(ov::element::i64, {flat_ndims}, flat_shape), false);
auto sliced = std::make_shared<ov::op::v8::Slice>(
flat, ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_begin}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_end}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim}));
if (view_ov_shape.is_static()) {
auto reshaped = std::make_shared<ov::op::v1::Reshape>(
sliced, ov::op::v0::Constant::create(ov::element::i64, {ndims}, view_ov_shape.to_shape()),
false);
reshaped->set_friendly_name(view_name);
return reshaped;
}
sliced->set_friendly_name(view_name);
return sliced;
}
}
std::vector<int64_t> begin(ndims, 0);
std::vector<int64_t> end(ndims, 0);
std::vector<int64_t> step(ndims, 1);
std::vector<int64_t> axes(ndims, 0);
size_t remaining_offset = relative_offset;
for (size_t i = 0; i < ndims; ++i) {
axes[i] = static_cast<int64_t>(i);
if (view_stride[i] > 0) {
begin[i] = static_cast<int64_t>(remaining_offset / view_stride[i]);
remaining_offset %= view_stride[i];
}
end[i] = begin[i] + static_cast<int64_t>(view_ggml_shape[i]);
}
bool in_bounds = view_src_ggml_shape.size() == ndims && view_ggml_shape.size() == ndims;
if (in_bounds) {
for (size_t i = 0; i < ndims; ++i) {
if (end[i] > static_cast<int64_t>(view_src_ggml_shape[i])) {
in_bounds = false;
break;
}
}
}
if (in_bounds && remaining_offset == 0) {
auto sliced = std::make_shared<ov::op::v8::Slice>(
current, ov::op::v0::Constant::create(ov::element::i64, {ndims}, begin),
ov::op::v0::Constant::create(ov::element::i64, {ndims}, end),
ov::op::v0::Constant::create(ov::element::i64, {ndims}, step),
ov::op::v0::Constant::create(ov::element::i64, {ndims}, axes));
sliced->set_friendly_name(view_name);
return sliced;
}
} else {
bool same_rank = view_stride.size() == view_src_stride.size() &&
view_ggml_shape.size() == view_src_ggml_shape.size() &&
view_stride.size() == view_ggml_shape.size();
const size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0;
if (same_rank) {
const size_t ndims = view_ggml_shape.size();
std::vector<int> diff_dims;
for (size_t i = 0; i < ndims; ++i) {
if (view_ggml_shape[i] != view_src_ggml_shape[i]) {
diff_dims.push_back(static_cast<int>(i));
}
}
if (diff_dims.size() == 1) {
const size_t slice_dim = static_cast<size_t>(diff_dims[0]);
bool suffix_stride_match = true;
for (size_t i = slice_dim + 1; i < ndims; ++i) {
if (view_stride[i] != view_src_stride[i]) {
suffix_stride_match = false;
break;
}
}
if (suffix_stride_match && view_src_stride[slice_dim] > 0 &&
relative_offset % view_src_stride[slice_dim] == 0) {
const int64_t begin_val = static_cast<int64_t>(relative_offset / view_src_stride[slice_dim]);
const int64_t end_val = begin_val + static_cast<int64_t>(view_ggml_shape[slice_dim]);
const int64_t dim_size = static_cast<int64_t>(view_src_ggml_shape[slice_dim]);
if (begin_val >= 0 && end_val <= dim_size) {
auto sliced = std::make_shared<ov::op::v8::Slice>(
current, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {static_cast<int64_t>(slice_dim)}));
sliced->set_friendly_name(view_name);
return sliced;
}
}
}
}
size_t view_elems = 1;
size_t src_elems = 1;
if (same_rank) {
for (size_t i = 0; i < view_ggml_shape.size(); ++i) {
view_elems *= view_ggml_shape[i];
src_elems *= view_src_ggml_shape[i];
}
}
bool same_num_elements = same_rank && view_elems == src_elems;
if (same_rank && relative_offset == 0 && same_num_elements) {
auto reshape_pattern = build_reshape_pattern(view_ov_shape, view_ggml_shape);
auto reshaped = std::make_shared<ov::op::v1::Reshape>(
current, ov::op::v0::Constant::create(ov::element::i64, {reshape_pattern.size()}, reshape_pattern),
false);
reshaped->set_friendly_name(view_name);
return reshaped;
}
if (same_rank) {
const size_t ndims = view_ggml_shape.size();
// Match views that can be expressed as a regular strided slice over the
// already reconstructed source tensor, e.g. offset on one axis plus step > 1
// on another axis.
bool is_regular_slice = view_src_ggml_shape.size() == ndims;
std::vector<int64_t> begin(ndims, 0);
std::vector<int64_t> end(ndims, 0);
std::vector<int64_t> step(ndims, 1);
std::vector<int64_t> axes(ndims, 0);
size_t remaining_offset = relative_offset;
if (is_regular_slice) {
for (size_t i = 0; i < ndims; ++i) {
axes[i] = static_cast<int64_t>(i);
if (view_src_stride[i] == 0 || view_stride[i] == 0 ||
view_stride[i] % view_src_stride[i] != 0) {
is_regular_slice = false;
break;
}
step[i] = static_cast<int64_t>(view_stride[i] / view_src_stride[i]);
if (step[i] <= 0) {
is_regular_slice = false;
break;
}
begin[i] = static_cast<int64_t>(remaining_offset / view_src_stride[i]);
remaining_offset %= view_src_stride[i];
if (view_ggml_shape[i] == 0) {
end[i] = begin[i];
continue;
}
end[i] = begin[i] + step[i] * static_cast<int64_t>(view_ggml_shape[i] - 1) + 1;
if (begin[i] < 0 || end[i] > static_cast<int64_t>(view_src_ggml_shape[i])) {
is_regular_slice = false;
break;
}
}
}
if (is_regular_slice && remaining_offset == 0) {
auto sliced = std::make_shared<ov::op::v8::Slice>(
current, ov::op::v0::Constant::create(ov::element::i64, {ndims}, begin),
ov::op::v0::Constant::create(ov::element::i64, {ndims}, end),
ov::op::v0::Constant::create(ov::element::i64, {ndims}, step),
ov::op::v0::Constant::create(ov::element::i64, {ndims}, axes));
sliced->set_friendly_name(view_name);
return sliced;
}
const size_t elem_stride = view_src_stride.back();
const bool aligned_offset = elem_stride > 0 && relative_offset % elem_stride == 0;
if (aligned_offset) {
size_t suffix_start = 0;
size_t expected_stride = elem_stride;
for (int i = static_cast<int>(ndims) - 1; i >= 0; --i) {
if (view_stride[i] != expected_stride) {
suffix_start = static_cast<size_t>(i + 1);
break;
}
expected_stride *= view_ggml_shape[i];
}
size_t prefix_elems = 1;
size_t suffix_elems = 1;
for (size_t i = 0; i < suffix_start; ++i) {
prefix_elems *= view_ggml_shape[i];
}
for (size_t i = suffix_start; i < ndims; ++i) {
suffix_elems *= view_ggml_shape[i];
}
if (prefix_elems > 0 && src_elems % prefix_elems == 0) {
const size_t src_tail_elems = src_elems / prefix_elems;
const int64_t tail_begin = static_cast<int64_t>(relative_offset / elem_stride);
const int64_t tail_end = tail_begin + static_cast<int64_t>(suffix_elems);
if (tail_begin >= 0 && tail_end <= static_cast<int64_t>(src_tail_elems)) {
auto prefix_tail_pattern = build_prefix_tail_reshape_pattern(
view_ov_shape, view_ggml_shape, suffix_start, static_cast<int64_t>(src_tail_elems));
auto prefix_tail = std::make_shared<ov::op::v1::Reshape>(
current,
ov::op::v0::Constant::create(ov::element::i64, {prefix_tail_pattern.size()},
prefix_tail_pattern),
false);
ov::Output<ov::Node> selected = prefix_tail;
if (tail_begin != 0 || tail_end != static_cast<int64_t>(src_tail_elems)) {
selected = std::make_shared<ov::op::v8::Slice>(
prefix_tail, ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_begin}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_end}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
ov::op::v0::Constant::create(ov::element::i64, {1},
{static_cast<int64_t>(suffix_start)}));
}
auto reshape_pattern = build_reshape_pattern(view_ov_shape, view_ggml_shape);
auto reshaped = std::make_shared<ov::op::v1::Reshape>(
selected,
ov::op::v0::Constant::create(ov::element::i64, {reshape_pattern.size()},
reshape_pattern),
false);
reshaped->set_friendly_name(view_name);
return reshaped;
}
}
}
}
return current;
}
(void) view_name;
(void) view_src_ov_shape;
(void) view_src_name;
return current;
};
// Process views from the base tensor (last) to the current view (first)
// Start with the base tensor
ov::Output<ov::Node> current = input;
// Process each view in reverse order (from base to current)
for (int view_idx = view_input_size - 1; view_idx >= 0; view_idx--) {
auto view_offset = context.get_view_input_offset(input_index, view_idx);
auto view_stride = context.get_view_input_stride(input_index, view_idx);
auto view_ggml_shape = context.get_view_input_ggml_shape(input_index, view_idx);
auto view_ov_shape = context.get_view_input_ov_shape(input_index, view_idx);
auto view_name = context.get_view_input_name(input_index, view_idx);
// print view info
// std::cout << "View " << view_idx << ": name = " << view_name << ", offset = " << view_offset << ", stride = ["
// << view_stride[0] << "," << view_stride[1] << "," << view_stride[2] << "," << view_stride[3]
// << "], ggml shape = [" << view_ggml_shape[0] << "," << view_ggml_shape[1] << ","
// << view_ggml_shape[2] << "," << view_ggml_shape[3] << "], ov shape = " << view_ov_shape << std::endl;
auto view_src_offset = context.get_view_input_src_offset(input_index, view_idx);
auto view_src_stride = context.get_view_input_src_stride(input_index, view_idx);
auto view_src_ggml_shape = context.get_view_input_src_ggml_shape(input_index, view_idx);
auto view_src_ov_shape = context.get_view_input_src_ov_shape(input_index, view_idx);
auto view_src_name = context.get_view_input_src_name(input_index, view_idx);
// print source view info
// std::cout << "View " << view_idx << ": source name = " << view_src_name
// << ", source offset = " << view_src_offset << ", source stride = [" << view_src_stride[0] << ","
// << view_src_stride[1] << "," << view_src_stride[2] << "," << view_src_stride[3]
// << "], source ggml shape = [" << view_src_ggml_shape[0] << "," << view_src_ggml_shape[1] << ","
// << view_src_ggml_shape[2] << "," << view_src_ggml_shape[3]
// << "], source ov shape = " << view_src_ov_shape << std::endl;
current = process_single_view(current, view_offset, view_stride, view_ggml_shape, view_ov_shape, view_name,
view_src_offset, view_src_stride, view_src_ggml_shape, view_src_ov_shape,
view_src_name);
}
return current;
}
} // namespace ggml
} // namespace frontend
} // namespace ov
+28
-26
vendor/llama.cpp/ggml/src/ggml-openvino/openvino/utils.h
#pragma once
#include "node_context.h"
#include <memory>

@@ -9,4 +11,2 @@ #include <openvino/core/node.hpp>

#include "node_context.h"
namespace ov {

@@ -20,26 +20,19 @@ namespace frontend {

void num_inputs_check(const NodeContext& context, size_t min_inputs, size_t max_inputs);
void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs);
int non_cont_dim(std::vector<size_t> ne, std::vector<size_t> nb);
template <typename T>
std::vector<int> argsort_descend(const std::vector<T>& v) {
template <typename T> std::vector<int> argsort_descend(const std::vector<T> & v) {
std::vector<int> idx(v.size());
std::iota(idx.begin(), idx.end(), 0);
std::sort(idx.begin(), idx.end(), [&v](int i1, int i2) {
return v[i1] > v[i2];
});
std::sort(idx.begin(), idx.end(), [&v](int i1, int i2) { return v[i1] > v[i2]; });
return idx;
}
template <typename T>
std::vector<T> sorted_descend(std::vector<T> v) {
std::sort(v.begin(), v.end(), [](T a, T b) {
return a > b;
});
template <typename T> std::vector<T> sorted_descend(std::vector<T> v) {
std::sort(v.begin(), v.end(), [](T a, T b) { return a > b; });
return v;
}
template <typename T>
bool is_permuted(const std::vector<T>& strides) {
template <typename T> bool is_permuted(const std::vector<T> & strides) {
for (size_t i = 0; i < strides.size() - 1; ++i) {

@@ -53,4 +46,3 @@ if (strides[i] < strides[i + 1]) {

template <typename T>
std::vector<T> permute(const std::vector<T>& x, const std::vector<int>& perm) {
template <typename T> std::vector<T> permute(const std::vector<T> & x, const std::vector<int> & perm) {
std::vector<T> result;

@@ -64,9 +56,9 @@ result.reserve(perm.size());

std::shared_ptr<ov::Node> get_dimensions(const std::shared_ptr<ov::op::v3::ShapeOf>& shape,
const std::vector<int>& dims);
std::shared_ptr<ov::Node> get_dimensions(const std::shared_ptr<ov::Node>& node, const std::vector<int>& dims);
std::shared_ptr<ov::Node> get_dimensions(const std::shared_ptr<ov::op::v3::ShapeOf> & shape,
const std::vector<int> & dims);
std::shared_ptr<ov::Node> get_dimensions(const std::shared_ptr<ov::Node> & node, const std::vector<int> & dims);
OutputVector rename_outputs_with_suffix(const OutputVector& outputs, const std::string& suffix);
OutputVector rename_outputs_with_suffix(const OutputVector & outputs, const std::string & suffix);
std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t* rope_params,
std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params,
std::shared_ptr<ov::Node> inp_pos,

@@ -77,11 +69,21 @@ std::shared_ptr<ov::Node> rope_freqs_weight = nullptr,

ov::Output<ov::Node> process_view_input(const NodeContext& context, int input_index, int slice_len = 0);
ov::Output<ov::Node> process_view_input(const NodeContext & context, int input_index, int slice_len = 0);
ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int input_index);
namespace op {
template <typename T>
OutputVector translate_1to1_match_2_inputs(const NodeContext& context) {
template <typename T> OutputVector translate_1to1_match_2_inputs(const NodeContext & context) {
num_inputs_check(context, 2, 2);
auto res = std::make_shared<T>(context.get_input(0), context.get_input(1));
auto input_0 = process_view_input_new(context, 0);
auto input_1 = process_view_input_new(context, 1);
auto res = std::make_shared<T>(input_0, input_1);
return rename_outputs_with_suffix({res}, context.get_name());
}
template <typename T> OutputVector translate_1to1_match_1_input(const NodeContext & context) {
num_inputs_check(context, 1, 1);
auto input = process_view_input_new(context, 0);
auto res = std::make_shared<T>(input);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op

@@ -88,0 +90,0 @@ 

+383
-94
vendor/llama.cpp/ggml/src/ggml-openvino/utils.cpp

@@ -17,2 +17,3 @@ #include "utils.h"

#include <cstring>
#include <fstream>
#include <iomanip>

@@ -29,5 +30,7 @@ #include <iostream>

#include <openvino/runtime/infer_request.hpp>
#include <openvino/runtime/intel_gpu/ocl/ocl.hpp>
#include <openvino/runtime/intel_npu/properties.hpp>
#include <openvino/runtime/properties.hpp>
#include <openvino/runtime/tensor.hpp>
#include <optional>
#include <string>

@@ -44,3 +47,3 @@ #include <unordered_map>

try {
if (getenv("GGML_OPENVINO_DUMP_CGRAPH")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_CGRAPH")) {
std::string filename = "cgraph_ov.txt";

@@ -68,2 +71,71 @@ GgmlOvDecoder::dump_cgraph(cgraph, filename);

// For a KV cache input, return an ov::Tensor sized to n_kv (== attention_size
// for that layer) instead of the fully-allocated ctx_per_seq. Pre-conditions:
// * non-static (CPU/GPU) backend, single sequence, seq_active_start == 0
// * ggml KV layout is a contiguous [1, 1, ctx_per_seq, n_heads_kv*head_size]
// so the first n_kv rows are the live prefix and shrinking the ctx axis
// gives a valid tensor over the same host storage
// * not an SWA layer (ring cache): once the window has wrapped the first
// n_kv rows no longer contain the live prefix
// On any unmet pre-condition returns std::nullopt; the caller falls back to
// the full-size tensor.
static std::optional<ov::Tensor> try_make_kv_sliced_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
const std::string & name,
const ggml_tensor * ggml_tensor) {
static const bool kv_slice_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE");
if (kv_slice_disabled) {
return std::nullopt;
}
if (ggml_decoder->is_static() || ggml_decoder->is_stateful()) {
return std::nullopt;
}
if (ggml_tensor->op != GGML_OP_NONE || ggml_tensor->view_src != nullptr) {
return std::nullopt;
}
const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor);
if (!GgmlOvDecoder::is_kvcache(ggml_tensor, op)) {
return std::nullopt;
}
const auto & compute_params = ggml_decoder->get_compute_params();
if (compute_params.n_seq_active != 1 || compute_params.seq_active_start != 0) {
return std::nullopt;
}
int layer;
if (auto layer_opt = extract_layer_from_name(name); layer_opt.has_value()) {
layer = layer_opt.value();
} else {
return std::nullopt;
}
const bool is_swa = ggml_decoder->is_swa_layer(layer);
if (is_swa) {
return std::nullopt;
}
const int ctx_per_seq = ggml_decoder->get_ctx_per_seq();
const int n_kv = compute_params.attention_size;
if (ctx_per_seq <= 0 || n_kv <= 0 || n_kv >= ctx_per_seq) {
return std::nullopt;
}
ov::Shape full_shape = ggml_decoder->get_shape(ggml_tensor);
if (full_shape.size() != 4 || full_shape[0] != 1 || full_shape[1] != 1 ||
static_cast<int>(full_shape[2]) != ctx_per_seq) {
return std::nullopt;
}
ov::Shape sliced_shape = full_shape;
sliced_shape[2] = static_cast<size_t>(n_kv);
// Disabling for now as gpu has bug with in-place ScatterUpdate with remote tensors, can re-enable once CVS-186519 is fixed
// if (ggml_openvino_buffer_is_remote(ggml_tensor)) {
// auto remote_context = ggml_openvino_get_remote_context();
// auto gpu_context = remote_context->as<ov::intel_gpu::ocl::ClContext>();
// return gpu_context.create_tensor(ggml_decoder->get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data);
// }
return ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data);
}
ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,

@@ -73,2 +145,15 @@ std::shared_ptr<ov::InferRequest> infer_request,

const ggml_tensor * ggml_tensor) {
if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, std::string(ggml_tensor->name), ggml_tensor)) {
return *sliced;
}
// Disabling for now as gpu has bug with in-place ScatterUpdate with remote tensors, can re-enable once CVS-186519 is fixed
// if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) {
// auto * extra_base = static_cast<ggml_openvino_extra_base *>(ggml_tensor->extra);
// if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) {
// auto * tensor_extra = static_cast<ggml_openvino_tensor_extra *>(extra_base);
// return *tensor_extra->tensor;
// }
// }
auto output_type = ggml_decoder->get_ov_type(ggml_tensor);

@@ -94,3 +179,5 @@ ov::Shape output_shape;

if (is_naive(cgraph)) {
return naive_compute(cgraph, core, device, config);
if (!is_model_splitted(cgraph)) {
return naive_compute(cgraph, core, device, config);
}
}

@@ -107,3 +194,4 @@

graph_key key(cgraph);
bool cache_hit;
static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE");
bool cache_hit = false;

@@ -114,2 +202,3 @@ int64_t decoder_end_time;

int64_t infer_end_time;
int64_t ov_raw_infer_start;

@@ -120,3 +209,3 @@ {

{
if (cache_enabled) {
std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);

@@ -132,2 +221,6 @@ auto it = r_ctx->decoder_cache.find(key);

}
} else {
auto mutex = std::make_shared<std::mutex>();
entry = std::make_shared<decoder_runtime_ctx>(mutex);
cache_hit = false;
}

@@ -140,5 +233,10 @@

old_m_params = ggml_decoder->get_model_params();
cache_hit = old_m_params.can_reuse_dynamically(m_params);
if (!ggml_decoder->is_splited_model()) {
cache_hit = old_m_params.can_reuse_dynamically(m_params);
}
}
std::vector<std::string> ov_input_names;
std::vector<std::string> ov_output_names;
if (cache_hit) {

@@ -155,2 +253,4 @@ std::map<std::string, std::shared_ptr<ov::Node>> model_weights;

infer_request = r_ctx->infer_request_cache.at(key);
ov_input_names = r_ctx->ov_input_names_cache.at(key);
ov_output_names = r_ctx->ov_output_names_cache.at(key);
}

@@ -177,10 +277,11 @@

} catch (...) {
GGML_LOG_ERROR("GGML OpenVINO backend stateful inference failed: no input found for the state\n");
GGML_LOG_ERROR(
"GGML OpenVINO backend stateful inference failed: no input found for the state\n");
return GGML_STATUS_FAILED;
}
auto kv_tensor = get_ov_input_tensor(ggml_decoder, state_name);
kv_tensor.set_shape({state_tensor_shape[0], kv_tensor.get_shape()[2],
state_tensor_shape[2], state_tensor_shape[3]});
state_tensor = kv_tensor;
state_tensor_shape = state_tensor.get_shape();
kv_tensor.set_shape({state_tensor_shape[0], kv_tensor.get_shape()[2], state_tensor_shape[2],
state_tensor_shape[3]});
state_tensor = kv_tensor;
state_tensor_shape = state_tensor.get_shape();
}

@@ -193,3 +294,3 @@ ov::Coordinate begin = {0, 0, 0, 0};

}
r_ctx->stateful_kv_size = pos_data[0] + 1;
r_ctx->stateful_kv_size = pos_data[0] + pos_shape[3];
}

@@ -202,6 +303,7 @@ }

} else {
{
if (cache_enabled) {
std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
r_ctx->infer_request_cache.erase(key);
}
bool model_is_splitted = is_model_splitted(cgraph);

@@ -211,3 +313,4 @@ std::shared_ptr<ov::Model> model;

ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static, stateful);
ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static,
stateful, model_is_splitted);
decoder_end_time = ggml_time_us();

@@ -220,3 +323,3 @@

if (getenv("GGML_OPENVINO_DUMP_IR")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) {
char timestamped_filename[64];

@@ -239,4 +342,2 @@ auto timestamp = (long long) ggml_time_us();

std::vector<std::string> ov_input_names;
std::vector<std::string> ov_output_names;
for (const auto & ov_param : model->get_parameters()) {

@@ -249,10 +350,10 @@ ov_input_names.push_back(ov_param->get_friendly_name());

{
if (cache_enabled) {
std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
r_ctx->infer_request_cache[key] = infer_request;
r_ctx->ov_input_names_cache[key] = std::move(ov_input_names);
r_ctx->ov_output_names_cache[key] = std::move(ov_output_names);
r_ctx->ov_input_names_cache[key] = ov_input_names;
r_ctx->ov_output_names_cache[key] = ov_output_names;
}
if (stateful) {
if (stateful && cache_enabled) {
const auto * inp_pos = get_inp_pos_tensor(cgraph);

@@ -262,4 +363,4 @@ auto pos_shape = ggml_decoder->get_shape(inp_pos);

const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names();
for (const auto& pair : kv_param_res_names) {
r_ctx->kv_state_input_name_map[pair.first+pair.second] = pair.first;
for (const auto & pair : kv_param_res_names) {
r_ctx->kv_state_input_name_map[pair.first + pair.second] = pair.first;
}

@@ -269,10 +370,2 @@ }

std::vector<std::string> ov_input_names;
std::vector<std::string> ov_output_names;
{
std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
ov_input_names = r_ctx->ov_input_names_cache[key];
ov_output_names = r_ctx->ov_output_names_cache[key];
}
for (size_t i = 0; i < ov_input_names.size(); i++) {

@@ -283,3 +376,3 @@ auto param_name = ov_input_names[i];

if (getenv("GGML_OPENVINO_DEBUG_INPUT")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) {
print_input_tensor_info(param_name, input_tensor);

@@ -291,2 +384,5 @@ }

auto * ggml_tensor = ggml_decoder->get_model_outputs().at(ov_output_names[i]);
if (ggml_nbytes(ggml_tensor) == 0) {
continue;
}
auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);

@@ -296,6 +392,7 @@ infer_request->set_output_tensor(i, output_tensor);

ov_raw_infer_start = ggml_time_us();
infer_request->infer();
infer_end_time = ggml_time_us();
if (getenv("GGML_OPENVINO_DEBUG_OUTPUT")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT")) {
for (size_t i = 0; i < ov_output_names.size(); i++) {

@@ -307,10 +404,12 @@ const auto output_tensor = infer_request->get_output_tensor(i);

if (getenv("GGML_OPENVINO_PROFILING")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) {
GGML_LOG_INFO("\nGGML OpenVINO Backend: \n");
GGML_LOG_INFO(" - Graph decoder time: %ld ms \n", (decoder_end_time - start_time) / 1000);
GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0);
if (!cache_hit) {
GGML_LOG_INFO(" - Graph conversion time: %ld ms \n", (conversion_end_time - decoder_end_time) / 1000);
GGML_LOG_INFO(" - Graph compile time: %ld ms \n", (compile_end_time - conversion_end_time) / 1000);
GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n",
(conversion_end_time - decoder_end_time) / 1000.0);
GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0);
}
GGML_LOG_INFO(" - Graph inference time: %ld ms \n", (infer_end_time - compile_end_time) / 1000);
GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0);
GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", (infer_end_time - ov_raw_infer_start) / 1000.0);
}

@@ -326,7 +425,7 @@ }

auto get_prefill_chunk_size = [] {
const char * chunk_size_str = getenv("GGML_OPENVINO_PREFILL_CHUNK_SIZE");
if (chunk_size_str && atoi(chunk_size_str) > 0) {
return atoi(chunk_size_str);
}
return 256;
static const int chunk_size = []() {
int env_prefill_chunk_size = ggml_openvino_getenv_int("GGML_OPENVINO_PREFILL_CHUNK_SIZE");
return env_prefill_chunk_size > 0 ? env_prefill_chunk_size : 256;
}();
return chunk_size;
};

@@ -337,3 +436,4 @@

static auto stateful = false;
static auto prefill_chunk_size = get_prefill_chunk_size();
auto prefill_chunk_size = get_prefill_chunk_size();
const auto & config = ggml_openvino_get_compile_config();

@@ -356,3 +456,4 @@

graph_key key(cgraph);
bool cache_hit;
static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE");
bool cache_hit = false;

@@ -363,2 +464,4 @@ int64_t decoder_end_time;

int64_t infer_end_time;
int64_t ov_raw_infer_start;
int64_t ov_raw_infer_total = 0;

@@ -368,3 +471,3 @@ std::shared_ptr<decoder_runtime_ctx> entry;

{
if (cache_enabled) {
std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);

@@ -380,2 +483,6 @@ auto it = r_ctx->decoder_cache.find(key);

}
} else {
auto mutex = std::make_shared<std::mutex>();
entry = std::make_shared<decoder_runtime_ctx>(mutex);
cache_hit = false;
}

@@ -391,2 +498,5 @@

std::vector<std::string> ov_input_names_local;
std::vector<std::string> ov_output_names_local;
if (cache_hit) {

@@ -405,2 +515,4 @@ std::map<std::string, std::shared_ptr<ov::Node>> model_weights;

is_prefill ? r_ctx->infer_request_cache_prefill.at(key) : r_ctx->infer_request_cache.at(key);
ov_input_names_local = r_ctx->ov_input_names_cache.at(key);
ov_output_names_local = r_ctx->ov_output_names_cache.at(key);
}

@@ -412,3 +524,3 @@

} else {
{
if (cache_enabled) {
std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);

@@ -422,6 +534,10 @@ r_ctx->infer_request_cache.erase(key);

auto ggml_decoder_prefill = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights,
is_static, stateful, true, prefill_chunk_size);
if (m_params.n_heads_kv == -1) {
// graph is not a LLM, e.g. context-shift graph
prefill_chunk_size = inp_pos->ne[0];
}
auto ggml_decoder_prefill = std::make_shared<GgmlOvDecoder>(
cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size);
auto ggml_decoder_decode = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static,
stateful, false, prefill_chunk_size);
stateful, false, false, prefill_chunk_size);
decoder_end_time = ggml_time_us();

@@ -438,3 +554,3 @@

if (getenv("GGML_OPENVINO_DUMP_IR")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) {
char timestamped_filename[64];

@@ -468,28 +584,18 @@ auto timestamp = (long long) ggml_time_us();

std::vector<std::string> ov_input_names;
std::vector<std::string> ov_output_names;
for (const auto & ov_param : model->get_parameters()) {
ov_input_names.push_back(ov_param->get_friendly_name());
ov_input_names_local.push_back(ov_param->get_friendly_name());
}
for (const auto & ov_output : model->get_results()) {
ov_output_names.push_back(ov_output->get_friendly_name());
ov_output_names_local.push_back(ov_output->get_friendly_name());
}
{
if (cache_enabled) {
std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
r_ctx->infer_request_cache_prefill[key] = infer_request_prefill;
r_ctx->infer_request_cache[key] = infer_request_decode;
r_ctx->ov_input_names_cache[key] = std::move(ov_input_names);
r_ctx->ov_output_names_cache[key] = std::move(ov_output_names);
r_ctx->ov_input_names_cache[key] = ov_input_names_local;
r_ctx->ov_output_names_cache[key] = ov_output_names_local;
}
}
std::vector<std::string> ov_input_names_local;
std::vector<std::string> ov_output_names_local;
{
std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
ov_input_names_local = r_ctx->ov_input_names_cache[key];
ov_output_names_local = r_ctx->ov_output_names_cache[key];
}
if (is_prefill) {

@@ -503,3 +609,3 @@ auto inp_len = inp_pos->ne[0];

if (getenv("GGML_OPENVINO_DEBUG_INPUT")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) {
const auto input_tensor = infer_request->get_input_tensor(i);

@@ -516,5 +622,7 @@ print_input_tensor_info(param_name, input_tensor);

ov_raw_infer_start = ggml_time_us();
infer_request->infer();
ov_raw_infer_total += ggml_time_us() - ov_raw_infer_start;
if (getenv("GGML_OPENVINO_DEBUG_OUTPUT")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT")) {
for (size_t i = 0; i < ov_output_names_local.size(); i++) {

@@ -533,3 +641,3 @@ const auto output_tensor = infer_request->get_output_tensor(i);

if (getenv("GGML_OPENVINO_DEBUG_INPUT")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) {
const auto input_tensor = infer_request->get_input_tensor(i);

@@ -546,6 +654,8 @@ print_input_tensor_info(param_name, input_tensor);

ov_raw_infer_start = ggml_time_us();
infer_request->infer();
infer_end_time = ggml_time_us();
ov_raw_infer_total = infer_end_time - ov_raw_infer_start;
if (getenv("GGML_OPENVINO_DEBUG_OUTPUT")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT")) {
for (size_t i = 0; i < ov_output_names_local.size(); i++) {

@@ -558,10 +668,11 @@ const auto output_tensor = infer_request->get_output_tensor(i);

if (getenv("GGML_OPENVINO_PROFILING")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) {
GGML_LOG_INFO("\nGGML OpenVINO Backend: \n");
GGML_LOG_INFO(" - Graph decoder time: %ld ms \n", (decoder_end_time - start_time) / 1000);
GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0);
if (!cache_hit) {
GGML_LOG_INFO(" - Graph conversion time: %ld ms \n", (conversion_end_time - decoder_end_time) / 1000);
GGML_LOG_INFO(" - Graph compile time: %ld ms \n", (compile_end_time - conversion_end_time) / 1000);
GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", (conversion_end_time - decoder_end_time) / 1000.0);
GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0);
}
GGML_LOG_INFO(" - Graph inference time: %ld ms \n", (infer_end_time - compile_end_time) / 1000);
GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0);
GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", ov_raw_infer_total / 1000.0);
}

@@ -572,2 +683,57 @@

// Detect whether a cgraph is a split subgraph or not.
// Step 1 compares each node's recorded use_count with actual fan-out references in node->src.
// Step 2 verifies that node inputs come from model nodes/weights/leafs; external sources imply split.
bool is_model_splitted(ggml_cgraph * cgraph) {
// check the nodes of the model are used by the following nodes, through compare the node's use count and the count of nodes that use it as input. If does not match, return true, else return false.
for (int i = 0; i < cgraph->n_nodes; i++) {
ggml_tensor * node = cgraph->nodes[i];
int use_count = cgraph->use_counts[ggml_hash_find(&cgraph->visited_hash_set, node)];
// TODO: this is a workround for the tests case from llama.cpp, fix should from the root cause in the future.
if ((cgraph->n_nodes <= 1 && use_count == 0) ||
(cgraph->n_nodes <= 1 && node->op == GGML_OP_VIEW && use_count == 1 && node->src[0] != nullptr &&
node->src[0]->op == GGML_OP_NONE)) {
return false;
}
if (cgraph->n_nodes == 1 &&
(cgraph->nodes[0]->op == GGML_OP_TRANSPOSE || cgraph->nodes[0]->op == GGML_OP_PERMUTE)) {
return false;
}
int input_use_count = 0;
for (int j = 0; j < cgraph->n_nodes; j++) {
ggml_tensor * other_node = cgraph->nodes[j];
for (int k = 0; k < GGML_MAX_SRC; k++) {
if (other_node->src[k] == node) {
input_use_count++;
}
}
}
if (use_count != input_use_count && node->op != GGML_OP_NONE) {
return true;
}
}
// if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check.
auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, true);
std::set<ggml_tensor *> model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes);
// leaf nodes
std::set<ggml_tensor *> model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs);
for (int i = 0; i < cgraph->n_nodes; i++) {
ggml_tensor * node = cgraph->nodes[i];
for (int j = 0; j < GGML_MAX_SRC; j++) {
ggml_tensor * src = node->src[j];
// the src is also not the model weights, we think the model is splitted.
// the src is also not in model leafs, we think the model is splitted.
if (src != nullptr && model_nodes.find(src) == model_nodes.end() &&
model_weights.find(std::string(src->name)) == model_weights.end() && !model_leafs.empty() == false &&
model_leafs.find(src) == model_leafs.end()) {
if (GgmlOvDecoder::is_inp_tok(src, node)) {
return false;
}
return true;
}
}
}
return false;
}
bool is_naive(ggml_cgraph * cgraph) {

@@ -597,3 +763,3 @@ constexpr int naive_graph_size_threshold = 20;

auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive);
if (getenv("GGML_OPENVINO_DUMP_IR")) {
if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) {
ov::serialize(model, "IR_naive.xml");

@@ -625,10 +791,13 @@ }

// Use get_output_tensor + memcpy instead of set_output_tensor to avoid memory overwritten
// when i/o buffer overlaps, e.g. the cgraph is a single PERMUTE
infer_request->infer();
auto ov_results = model->get_results();
for (size_t i = 0; i < ov_results.size(); i++) {
auto output_tensor = infer_request->get_output_tensor(i);
auto * ggml_tensor = decoder->get_model_outputs().at(ov_results[i]->get_friendly_name());
auto output_tensor = create_ov_output_tensor(decoder, infer_request, i, ggml_tensor);
infer_request->set_output_tensor(i, output_tensor);
std::memcpy(ggml_tensor->data, output_tensor.data(), output_tensor.get_byte_size());
}
infer_request->infer();
return GGML_STATUS_SUCCESS;

@@ -638,13 +807,57 @@ }

namespace {
template <typename T> void set_zero_diagonal(std::vector<T> & matrix, size_t rows, size_t cols, T zero_value = T{}) {
for (size_t i = 0; i < rows; ++i) {
size_t diag_col = std::min(i, cols - 1);
matrix[i * cols + diag_col] = zero_value;
}
}
ov::Tensor make_contiguous_split_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
const struct ggml_tensor * ggml_tensor,
const ov::Shape & input_shape) {
const size_t element_size = ggml_type_size(ggml_tensor->type);
const size_t block_size = ggml_blck_size(ggml_tensor->type);
GGML_ASSERT(block_size == 1 && "non-contiguous split inputs must be plain element types");
const struct ggml_tensor * source_tensor = ggml_tensor->view_src != nullptr ? ggml_tensor->view_src : ggml_tensor;
const size_t source_offset = ggml_tensor->view_src != nullptr ? ggml_tensor->view_offs : 0;
std::vector<uint8_t> source_data(ggml_nbytes(source_tensor));
ggml_backend_tensor_get(source_tensor, source_data.data(), 0, source_data.size());
ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);
auto * dst = static_cast<uint8_t *>(input_tensor.data());
size_t dst_offset = 0;
for (size_t i3 = 0; i3 < static_cast<size_t>(ggml_tensor->ne[3]); ++i3) {
for (size_t i2 = 0; i2 < static_cast<size_t>(ggml_tensor->ne[2]); ++i2) {
for (size_t i1 = 0; i1 < static_cast<size_t>(ggml_tensor->ne[1]); ++i1) {
for (size_t i0 = 0; i0 < static_cast<size_t>(ggml_tensor->ne[0]); ++i0) {
const size_t src_offset = source_offset + i3 * ggml_tensor->nb[3] + i2 * ggml_tensor->nb[2] +
i1 * ggml_tensor->nb[1] + i0 * ggml_tensor->nb[0];
std::memcpy(dst + dst_offset, source_data.data() + src_offset, element_size);
dst_offset += element_size;
}
}
}
}
return input_tensor;
}
ov::Tensor convert_ggml_input_to_ov(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & name) {
const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name);
if (ggml_tensor->extra != nullptr) {
// GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str());
if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, name, ggml_tensor)) {
return *sliced;
}
if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) {
auto * extra_base = static_cast<ggml_openvino_extra_base *>(ggml_tensor->extra);
if (extra_base->type != ggml_openvino_extra_base::Type::TENSOR) {
throw std::runtime_error("ggml tensor extra is not of type TENSOR for input: " + name);
if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) {
// GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str());
auto * tensor_extra = static_cast<ggml_openvino_tensor_extra *>(extra_base);
return *tensor_extra->tensor;
}
auto * tensor_extra = static_cast<ggml_openvino_tensor_extra *>(extra_base);
return *tensor_extra->tensor;
}

@@ -655,3 +868,3 @@

ov::Shape input_shape;
if (ggml_tensor->op == GGML_OP_VIEW) {
if (ggml_tensor->op == GGML_OP_VIEW && !ggml_decoder->is_splited_model()) {
// This case is added to make test-backend-ops work

@@ -662,2 +875,7 @@ input_shape = ggml_decoder->get_shape(ggml_tensor->view_src);

}
if (ggml_decoder->is_splited_model() && !ggml_is_contiguous(ggml_tensor)) {
return make_contiguous_split_input_tensor(ggml_decoder, ggml_tensor, input_shape);
}
auto input_tensor = ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape, input_data);

@@ -711,2 +929,10 @@ return input_tensor;

size_t context_size = ggml_decoder->get_ctx_size();
if (ggml_tensor->type == GGML_TYPE_F16) {
std::vector<ggml_fp16_t> padded_data =
pad_input<ggml_fp16_t>(ggml_tensor, 1, context_size, GGML_FP32_TO_FP16(-INFINITY));
ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, 1, context_size});
std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t));
return input_tensor;
}
std::vector<float> padded_data = pad_input<float>(ggml_tensor, 1, context_size, -INFINITY);

@@ -780,5 +1006,16 @@ ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, 1, context_size});

size_t rows = ggml_tensor->ne[1];
float * ggml_data = (float *) ggml_tensor->data + chunk_index * chunk_size * cols;
size_t chunk_valid_rows = std::min(chunk_size, rows - chunk_index * chunk_size);
size_t context_size = ggml_decoder->get_ctx_size();
if (ggml_tensor->type == GGML_TYPE_F16) {
const auto * ggml_data =
static_cast<const ggml_fp16_t *>(ggml_tensor->data) + chunk_index * chunk_size * cols;
std::vector<ggml_fp16_t> padded_data = pad_input<ggml_fp16_t>(ggml_data, chunk_valid_rows, cols, chunk_size,
context_size, GGML_FP32_TO_FP16(-INFINITY));
set_zero_diagonal(padded_data, chunk_size, context_size, GGML_FP32_TO_FP16(0.0f));
ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, chunk_size, context_size});
std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t));
return input_tensor;
}
const auto * ggml_data = static_cast<const float *>(ggml_tensor->data) + chunk_index * chunk_size * cols;
std::vector<float> padded_data =

@@ -806,2 +1043,61 @@ pad_input<float>(ggml_data, chunk_valid_rows, cols, chunk_size, context_size, -INFINITY);

bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string & file_path) {
if (tensor == nullptr || tensor->data == nullptr) {
return false;
}
std::ofstream out(file_path);
if (!out.is_open()) {
return false;
}
const size_t n = ggml_nelements(tensor);
out << "name: " << tensor->name << ", type: " << ggml_type_name(tensor->type) << ", shape: [" << tensor->ne[0]
<< ", " << tensor->ne[1] << ", " << tensor->ne[2] << ", " << tensor->ne[3] << "]" << ", elements: " << n
<< ", data:" << '\n';
switch (tensor->type) {
case GGML_TYPE_F32: {
const auto * data = static_cast<const float *>(tensor->data);
for (size_t i = 0; i < n; ++i) {
out << data[i] << '\n';
}
break;
}
case GGML_TYPE_F16: {
const auto * data = static_cast<const ggml_fp16_t *>(tensor->data);
for (size_t i = 0; i < n; ++i) {
out << ggml_fp16_to_fp32(data[i]) << '\n';
}
break;
}
case GGML_TYPE_BF16: {
const auto * data = static_cast<const ggml_bf16_t *>(tensor->data);
for (size_t i = 0; i < n; ++i) {
out << ggml_bf16_to_fp32(data[i]) << '\n';
}
break;
}
case GGML_TYPE_I32: {
const auto * data = static_cast<const int32_t *>(tensor->data);
for (size_t i = 0; i < n; ++i) {
out << data[i] << '\n';
}
break;
}
case GGML_TYPE_I64: {
const auto * data = static_cast<const int64_t *>(tensor->data);
for (size_t i = 0; i < n; ++i) {
out << data[i] << '\n';
}
break;
}
default:
out << "unsupported tensor type for text dump" << '\n';
return false;
}
return true;
}
void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor) {

@@ -903,9 +1199,2 @@ std::cout << "Input name: " << name << ", Input shape: " << tensor.get_shape() << ", Address: " << tensor.data()

void set_zero_diagonal(std::vector<float> & matrix, size_t rows, size_t cols) {
for (size_t i = 0; i < rows; ++i) {
size_t diag_col = std::min(i, cols - 1);
matrix[i * cols + diag_col] = 0.0f;
}
}
const ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) {

@@ -912,0 +1201,0 @@ for (int i = 0; i < cgraph->n_nodes; ++i) {

+11
-8
vendor/llama.cpp/ggml/src/ggml-openvino/utils.h

@@ -1,2 +0,1 @@

#include "ggml-backend-impl.h"
#include "ggml-decoder.h"

@@ -48,2 +47,3 @@ #include "ggml-impl.h"

decoder_runtime_ctx(std::shared_ptr<std::mutex> mutex) : mutex(std::move(mutex)) {}
std::shared_ptr<std::mutex> mutex;

@@ -68,7 +68,3 @@ std::shared_ptr<GgmlOvDecoder> ptr;

ov_runtime_context() :
device("CPU"),
stateful(false),
stateful_kv_size(0),
backend_count(0) {}
ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0), backend_count(0) {}

@@ -92,2 +88,4 @@ void clear_caches() {

bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string & file_path);
void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor);

@@ -123,4 +121,2 @@

void set_zero_diagonal(std::vector<float> & matrix, size_t rows, size_t cols);
const ggml_tensor * get_inp_pos_tensor(struct ggml_cgraph * cgraph);

@@ -144,2 +140,9 @@

/**
* @brief Heuristically checks whether the given computation graph is a split-model fragment.
* @param cgraph Pointer to the GGML computation graph to analyze.
* @return true if the graph is identified as split; otherwise false.
*/
bool is_model_splitted(struct ggml_cgraph * cgraph);
enum ggml_status naive_compute(struct ggml_cgraph * cgraph,

@@ -146,0 +149,0 @@ ov::Core & core,

+1
-0
vendor/llama.cpp/ggml/src/ggml-sycl/backend.hpp

@@ -20,2 +20,3 @@ //

#include "conv.hpp"
#include "conv3d.hpp"
#include "convert.hpp"

@@ -22,0 +23,0 @@ #include "count-equal.hpp"

+5
-5
vendor/llama.cpp/ggml/src/ggml-sycl/CMakeLists.txt

@@ -42,4 +42,4 @@ message(STATUS "GGML_SYCL_TARGET=${GGML_SYCL_TARGET}")

endif()
# Level Zero SDK path for Windows (only when GGML_SYCL_SUPPORT_LEVEL_ZERO is enabled)
if(GGML_SYCL_SUPPORT_LEVEL_ZERO)
# Level Zero SDK path for Windows (only when GGML_SYCL_SUPPORT_LEVEL_ZERO_API is enabled)
if(GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
if(DEFINED ENV{LEVEL_ZERO_V1_SDK_PATH})

@@ -109,4 +109,4 @@ set(LEVEL_ZERO_V1_SDK_PATH $ENV{LEVEL_ZERO_V1_SDK_PATH})

message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO ${GGML_SYCL_SUPPORT_LEVEL_ZERO}")
if (GGML_SYCL_SUPPORT_LEVEL_ZERO)
message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO_API ${GGML_SYCL_SUPPORT_LEVEL_ZERO_API}")
if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
# Link against Level Zero loader for direct device memory allocation.

@@ -119,3 +119,3 @@ # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging

target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB})
target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO)
target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}")

@@ -122,0 +122,0 @@ message(STATUS "Level Zero headers found: ${LEVEL_ZERO_INCLUDE_DIR}")

+5
-5
vendor/llama.cpp/ggml/src/ggml-sycl/common.cpp

@@ -15,3 +15,3 @@ //

#include <sycl/backend.hpp>
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
#include <level_zero/ze_api.h>

@@ -88,5 +88,5 @@ #endif

#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
static bool ggml_sycl_use_level_zero_device_alloc(sycl::queue &q) {
return g_ggml_sycl_enable_level_zero &&
return g_ggml_sycl_use_level_zero_api &&
q.get_device().is_gpu() &&

@@ -100,3 +100,3 @@ q.get_backend() == sycl::backend::ext_oneapi_level_zero;

void * ggml_sycl_malloc_device(size_t size, sycl::queue &q) {
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
if (ggml_sycl_use_level_zero_device_alloc(q)) {

@@ -133,3 +133,3 @@ void *ptr = nullptr;

if (!ptr) return;
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
if (ggml_sycl_use_level_zero_device_alloc(q)) {

@@ -136,0 +136,0 @@ auto ze_ctx = sycl::get_native<sycl::backend::ext_oneapi_level_zero>(q.get_context());

+15
-1
vendor/llama.cpp/ggml/src/ggml-sycl/common.hpp

@@ -65,2 +65,3 @@ //

extern int g_ggml_sycl_enable_flash_attention;
extern int g_ggml_sycl_dev2dev_memcpy;

@@ -130,2 +131,7 @@

enum ggml_sycl_dev2dev_memcpy_mode {
DEV2DEV_MEMCPY_SYCL = 0,
DEV2DEV_MEMCPY_L0 = 1,
};
static_assert(sizeof(sycl::half) == sizeof(ggml_fp16_t), "wrong fp16 size");

@@ -235,2 +241,3 @@

optimize_feature opt_feature;
bool usm_system_support; // support for USM system allocations
};

@@ -323,3 +330,3 @@

extern int g_ggml_sycl_enable_level_zero;
extern int g_ggml_sycl_use_level_zero_api;
void * ggml_sycl_malloc_device(size_t size, sycl::queue &q);

@@ -330,2 +337,7 @@ void ggml_sycl_free_device(void *ptr, sycl::queue &q);

struct mmid_row_mapping {
int32_t i1;
int32_t i2;
};
namespace sycl_ex = sycl::ext::oneapi::experimental;

@@ -428,2 +440,4 @@ struct ggml_backend_sycl_context {

std::vector<mmid_row_mapping> mmid_row_mapping_host;
static std::unique_ptr<ggml_sycl_pool> new_pool_for_device(queue_ptr qptr, int device);

@@ -430,0 +444,0 @@ 

+6
-0
vendor/llama.cpp/ggml/src/ggml-sycl/convert.cpp

@@ -645,2 +645,4 @@ #include "convert.hpp"

switch (type) {
case GGML_TYPE_Q1_0:
return dequantize_block_sycl<QK1_0, QR1_0, dequantize_q1_0>;
case GGML_TYPE_Q4_0:

@@ -728,2 +730,4 @@ if (dst->src[0]->extra &&

switch (type) {
case GGML_TYPE_Q1_0:
return dequantize_block_sycl<QK1_0, QR1_0, dequantize_q1_0>;
case GGML_TYPE_Q4_0:

@@ -835,2 +839,4 @@ if (dst->src[0]->extra &&

#endif
case GGML_TYPE_Q1_0:
return dequantize_block_nc_sycl<QK1_0, QR1_0, dequantize_q1_0>;
case GGML_TYPE_Q4_0:

@@ -837,0 +843,0 @@ return dequantize_block_nc_sycl<QK4_0, QR4_0, dequantize_q4_0>;

+15
-0
vendor/llama.cpp/ggml/src/ggml-sycl/dequantize.hpp

@@ -73,2 +73,17 @@ //

static __dpct_inline__ void dequantize_q1_0_reorder(const void *d_ptr, const int64_t ib, const void *qs,
const int iqs, dfloat2 &v) {
// Q1_0 reorder layout: scale values followed by quantized bits
const dfloat d = (const dfloat)*((const sycl::half*)d_ptr+ib);
const int bit_index_0 = iqs + 0;
const int bit_index_1 = iqs + 1;
const int bit_0 = (*((const uint8_t *)qs + bit_index_0 / 8) >> (bit_index_0 % 8)) & 1;
const int bit_1 = (*((const uint8_t *)qs + bit_index_1 / 8) >> (bit_index_1 % 8)) & 1;
v.x() = (2 * bit_0 - 1) * d;
v.y() = (2 * bit_1 - 1) * d;
}
static __dpct_inline__ void dequantize_q4_1(const void *vx, const int64_t ib,

@@ -75,0 +90,0 @@ const int iqs, dfloat2 &v) {

+45
-9
vendor/llama.cpp/ggml/src/ggml-sycl/outprod.cpp
#include "outprod.hpp"
#include "convert.hpp"

@@ -8,3 +9,3 @@ void ggml_sycl_op_out_prod(ggml_backend_sycl_context& ctx, ggml_tensor* dst) {

GGML_ASSERT(src0->type == GGML_TYPE_F32);
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_Q1_0);
GGML_ASSERT(src1->type == GGML_TYPE_F32);

@@ -24,8 +25,28 @@ GGML_ASSERT(dst->type == GGML_TYPE_F32);

GGML_ASSERT(ne1 == ne10); // Output cols match src1 cols
GGML_ASSERT(ne2 == ne12);
GGML_ASSERT(ne3 == ne13);
GGML_ASSERT(ne2 % ne02 == 0);
GGML_ASSERT(ne3 % ne03 == 0);
// Get data pointers
const float* src0_d = (const float*)src0->data;
const float* src1_d = (const float*)src1->data;
float* dst_d = (float*)dst->data;
const float * src0_d = (const float *) src0->data;
const float * src1_d = (const float *) src1->data;
float * dst_d = (float *) dst->data;
ggml_sycl_pool_alloc<float> src0_as_f32(ctx.pool());
int64_t src0_nb02 = nb02;
int64_t src0_nb03 = nb03;
if (src0->type == GGML_TYPE_Q1_0) {
scope_op_debug_print scope_dbg_print(__func__, "/to_fp32_sycl", dst, /*num_src=*/2,
" : converting src0 Q1_0 to fp32");
src0_d = src0_as_f32.alloc(ne00 * ne01 * ne02 * ne03);
const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(src0->type, dst);
GGML_ASSERT(to_fp32_sycl != nullptr);
to_fp32_sycl(src0->data, const_cast<float *>(src0_d), ne00 * ne01 * ne02 * ne03, stream);
// Dequantized src0 buffer is contiguous fp32 [ne00, ne01, ne02, ne03].
src0_nb02 = ne00 * ne01 * (int64_t) sizeof(float);
src0_nb03 = ne00 * ne01 * ne02 * (int64_t) sizeof(float);
}
// GEMM parameters

@@ -40,8 +61,23 @@ const float alpha = 1.0f;

const int64_t r2 = ne2 / ne02;
const int64_t r3 = ne3 / ne03;
try {
// Perform matrix multiplication using oneMKL GEMM
oneapi::mkl::blas::column_major::gemm(*stream, oneapi::mkl::transpose::nontrans, src1_op,
ne0, ne1, ne01, alpha, src0_d, ne00, src1_d, ldb, beta, dst_d, ne0);
}
catch (sycl::exception const& exc) {
// OUT_PROD applies independently to each (i2, i3) destination plane.
for (int64_t i3 = 0; i3 < ne3; ++i3) {
for (int64_t i2 = 0; i2 < ne2; ++i2) {
const int64_t i03 = i3 / r3;
const int64_t i02 = i2 / r2;
const float * src0_plane = (const float *) ((const char *) src0_d + i02 * src0_nb02 + i03 * src0_nb03);
const float * src1_plane = (const float *) ((const char *) src1_d + i2 * nb12 + i3 * nb13);
float * dst_plane = (float *) ((char *) dst_d + i2 * nb2 + i3 * nb3);
// Perform matrix multiplication using oneMKL GEMM
oneapi::mkl::blas::column_major::gemm(*stream, oneapi::mkl::transpose::nontrans, src1_op,
ne0, ne1, ne01, alpha, src0_plane, ne00,
src1_plane, ldb, beta, dst_plane, ne0);
}
}
} catch (sycl::exception const& exc) {
std::cerr << exc.what() << std::endl;

@@ -48,0 +84,0 @@ GGML_ASSERT(false);

+35
-0
vendor/llama.cpp/ggml/src/ggml-sycl/vecdotq.hpp

@@ -312,2 +312,37 @@ //

#define VDR_Q1_0_Q8_1_MMVQ 1
#define VDR_Q1_0_Q8_1_MMQ 4
static __dpct_inline__ float
vec_dot_q1_0_q8_1(const void *__restrict__ vbq,
const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
const block_q1_0 * bq1_0 = (const block_q1_0 *) vbq;
const block_q8_1 * bq8_1_chunk = bq8_1 + iqs;
const float d1 = bq1_0->d;
const int v = get_int_from_uint8_aligned(bq1_0->qs, iqs);
int vi_bytes[8];
#pragma unroll
for (int j = 0; j < 8; ++j) {
const int shift = j * 4;
const int bits4 = (v >> shift) & 0x0F;
const int b0 = (bits4 & 0x01) ? 1 : -1;
const int b1 = (bits4 & 0x02) ? 1 : -1;
const int b2 = (bits4 & 0x04) ? 1 : -1;
const int b3 = (bits4 & 0x08) ? 1 : -1;
vi_bytes[j] = (b0 & 0xFF) | ((b1 & 0xFF) << 8) | ((b2 & 0xFF) << 16) | ((b3 & 0xFF) << 24);
}
int sumi = 0;
#pragma unroll
for (int j = 0; j < 8; ++j) {
const int u = get_int_from_int8_aligned(bq8_1_chunk->qs, j);
sumi = ggml_sycl_dp4a(vi_bytes[j], u, sumi);
}
return d1 * bq8_1_chunk->ds[0] * sumi;
}
// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called

@@ -314,0 +349,0 @@ // MMVQ = mul_mat_vec_q, MMQ = mul_mat_q

+270
-27
vendor/llama.cpp/scripts/snapdragon/ggml-hexagon-profile.py

@@ -9,2 +9,3 @@ #!/usr/bin/env python3

import logging
from typing import Any, Dict, List, Optional

@@ -29,8 +30,43 @@ from collections import defaultdict

op_pattern = re.compile(
r"profile-op\s+(?P<op_name>[A-Z_0-9+]+):\s+.*?\s+:\s+(?P<dims>[\d:x\s\->!]+)\s+:\s+(?P<types>[a-z\d_\s\->x]+)\s+:\s+.*?\s+(?:op-)?usec\s+(?P<usec>\d+)\s+(?:op-)?cycles\s+(?P<cycles>\d+)(?:\s+pmu\s+\[(?P<pmu>[\d,\s]+)\])?"
r"profile-op\s+(?P<op_name>[A-Z_0-9+]+):\s+.*?\s+:\s+(?P<dims>[\d:x\s\->!]+)\s+:\s+(?P<types>[a-z\d_\s\->x]+)\s+:\s+.*?\s+(?:op-)?usec\s+(?P<usec>\d+)\s+(?:op-)?cycles\s+(?P<cycles>\d+)(?:\s+start\s+(?P<start>\d+))?(?:\s+mhz\s+(?P<mhz>[\d.]+))?(?:\s+pmu\s+\[(?P<pmu>[\d,\s]+)\])?(?:\s+evt\s+\[(?P<evt>[\d,\s]+)\])?"
)
trace_pattern = re.compile(
r"trace-op\s+(?P<op_name>[A-Z_0-9+]+):\s+thread\s+(?P<thread>\d+)\s+event\s+(?P<event>[A-Z_0-9\-]+)\s+info\s+(?P<info>\d+)\s+(?P<state>start|stop)\s+(?P<cycles>\d+)"
)
logger = logging.getLogger("ggml-hexagon-profile")
def normalize_event_name(evt_type):
if evt_type == "HVX_COMP":
return "V-COMP"
if evt_type == "HMX_COMP":
return "M-COMP"
# Strip HVX_ or HMX_ prefixes
name = evt_type
if name.startswith("HVX_") or name.startswith("HMX_"):
name = name[4:]
return name.replace("_", "-")
class CycleUnwrapper:
def __init__(self):
self.last_raw = None
self.high_part = 0
def unwrap(self, raw):
if self.last_raw is None:
self.last_raw = raw
return raw
diff = raw - self.last_raw
if diff < -0x80000000:
self.high_part += 0x100000000
elif diff > 0x80000000:
self.high_part -= 0x100000000
self.last_raw = raw
return raw + self.high_part
def parse_log(file_path, pmu_index=None):

@@ -46,31 +82,207 @@ try:

all_ops = []
all_ops: List[Dict[str, Any]] = []
current_op: Optional[Dict[str, Any]] = None
timestamp_pattern = re.compile(r"^(?P<min>\d+)\.(?P<sec>\d+)\.(?P<ms>\d+)\.(?P<us>\d+)\s+[A-Z]\s+")
unwrapper = CycleUnwrapper()
for line in f:
match = op_pattern.search(line)
if not match: continue
ts_match = timestamp_pattern.match(line)
abs_usec = 0
if ts_match:
abs_usec = (
(int(ts_match.group('min')) * 60 + int(ts_match.group('sec'))) * 1000000
+ int(ts_match.group('ms')) * 1000
+ int(ts_match.group('us'))
)
pmu_raw = match.group('pmu')
pmu_val = None
if pmu_raw and pmu_index is not None:
try:
pmu_list = [int(x.strip()) for x in pmu_raw.split(',')]
if len(pmu_list) > pmu_index:
pmu_val = pmu_list[pmu_index]
except (ValueError, IndexError):
pmu_val = None
op_match = op_pattern.search(line)
if op_match:
pmu_raw = op_match.group('pmu')
pmu_val = None
if pmu_raw and pmu_index is not None:
try:
pmu_list = [int(x.strip()) for x in pmu_raw.split(',')]
if len(pmu_list) > pmu_index:
pmu_val = pmu_list[pmu_index]
except (ValueError, IndexError):
pmu_val = None
all_ops.append({
'name': match.group('op_name'),
'dims': match.group('dims').strip(),
'types': match.group('types').strip(),
'usec': int(match.group('usec')),
'cycles': int(match.group('cycles')),
'pmu_val': pmu_val
})
evt_raw = op_match.group('evt')
evt_val = None
if evt_raw:
try:
evt_val = [int(x.strip()) for x in evt_raw.split(',')]
except ValueError:
evt_val = None
cycles_start_raw = op_match.group('start')
unwrapped_cycles_start = None
if cycles_start_raw:
unwrapped_cycles_start = unwrapper.unwrap(int(cycles_start_raw))
idx = line.find("profile-op ")
op_text = line[idx + 11:].strip() if idx != -1 else line.strip()
current_op = {
'name': op_match.group('op_name'),
'dims': op_match.group('dims').strip(),
'types': op_match.group('types').strip(),
'op_text': op_text,
'usec': int(op_match.group('usec')),
'cycles': int(op_match.group('cycles')),
'cycles_start': int(cycles_start_raw) if cycles_start_raw else None,
'unwrapped_cycles_start': unwrapped_cycles_start,
'pmu_val': pmu_val,
'evt_val': evt_val,
'abs_usec': abs_usec,
'trace_events': []
}
all_ops.append(current_op)
continue
trace_match = trace_pattern.search(line)
if trace_match and current_op:
if trace_match.group('op_name') == current_op['name']:
raw_cyc = int(trace_match.group('cycles'))
current_op['trace_events'].append({
'thread': int(trace_match.group('thread')),
'event': trace_match.group('event'),
'info': int(trace_match.group('info')),
'cycles': raw_cyc,
'unwrapped_cycles': unwrapper.unwrap(raw_cyc),
'state': trace_match.group('state')
})
f.close()
return all_ops
def print_ascii_timeline(op_name, dims, types, usec, cycles, events, evt_val=None):
evt_str = ""
if evt_val:
evt_str = " - evt [" + ",".join(str(x) for x in evt_val) + "]"
logger.info("=" * 100)
logger.info(f"{op_name} ({dims} : {types}) - {usec} usec {cycles} cycles{evt_str}")
logger.info("=" * 100)
events = sorted(events, key=lambda e: e['cycles'])
if not events:
logger.info(" No trace events recorded.")
return
min_cycles = events[0]['cycles']
logger.info("Cycles %-30s" % "EventDetails" + " ".join(f"T{i:<2}" for i in range(10)) + " HMX")
logger.info("-" * 100)
thread_stacks = [[] for _ in range(11)]
for e in events:
t = e['thread']
if t < 0 or t > 10:
continue
if e['cycles'] >= min_cycles:
rel_cycles = e['cycles'] - min_cycles
else:
rel_cycles = (e['cycles'] + 0x100000000) - min_cycles
state = e['state']
evt_type = e['event']
# Determine char representing the event
norm_evt = normalize_event_name(evt_type)
char = '?'
if norm_evt == 'V-COMP':
char = 'V'
elif norm_evt == 'M-COMP':
char = 'H'
elif norm_evt == 'A-QUANT':
char = 'Q'
elif norm_evt == 'A-PREP':
char = 'A'
elif norm_evt == 'W-DEQUANT':
char = 'D'
elif norm_evt == 'O-PROC':
char = 'O'
elif norm_evt == 'W-PREP':
char = 'P'
elif norm_evt == 'DMA':
char = 'M'
if state == 'start':
thread_stacks[t].append(char)
elif state == 'stop':
if thread_stacks[t]:
if thread_stacks[t][-1] == char:
thread_stacks[t].pop()
elif char in thread_stacks[t]:
thread_stacks[t].remove(char)
else:
thread_stacks[t].pop()
cols = []
for i in range(11):
if thread_stacks[i]:
cols.append(f"[{thread_stacks[i][-1]}]")
else:
cols.append(" | ")
evt_desc = f"T{t}: {evt_type} {state} ({e['info']})"
logger.info(f"{rel_cycles:10d} %-30s" % evt_desc + " ".join(cols[:10]) + " " + cols[10])
logger.info("-" * 100)
def print_ascii_summary(op_name, dims, types, usec, cycles, events, evt_val=None):
evt_str = ""
if evt_val:
evt_str = " - evt [" + ",".join(str(x) for x in evt_val) + "]"
logger.info("=" * 100)
logger.info(f"{op_name} ({dims} : {types}) - {usec} usec {cycles} cycles{evt_str}")
logger.info("=" * 100)
events = sorted(events, key=lambda e: e['cycles'])
if not events:
logger.info(" No trace events recorded.")
return
active_starts = {}
thread_totals = defaultdict(lambda: defaultdict(int))
for e in events:
t = e['thread']
evt = e['event']
info = e['info']
cyc = e['cycles']
state = e['state']
key = (t, evt, info)
if state == 'start':
active_starts[key] = cyc
elif state == 'stop':
if key in active_starts:
start_cyc = active_starts[key]
del active_starts[key]
if cyc >= start_cyc:
dur = cyc - start_cyc
else:
dur = (cyc + 0x100000000) - start_cyc
norm_evt = normalize_event_name(evt)
thread_totals[t][norm_evt] += dur
for t in sorted(thread_totals.keys()):
thread_name = f"Thread {t} (HVX)" if t != 10 else "Thread 10 (HMX)"
sorted_evts = sorted(thread_totals[t].items(), key=lambda item: item[0])
evt_strs = []
for evt, dur in sorted_evts:
pct = (dur / cycles * 100) if cycles > 0 else 0
evt_strs.append(f"{evt} {dur} ({pct:.1f}%)")
logger.info(f" {thread_name:<16}: " + " | ".join(evt_strs))
def generate_report(ops, top_n, width_overrides, sort_col, pmu_name=None):

@@ -121,3 +333,2 @@ if not ops:

actual_sort_key = COL_MAP[sort_col][2]
# We sort numeric fields descending, strings (op/dims) ascending
is_numeric = actual_sort_key.startswith("_") or actual_sort_key == "count"

@@ -139,3 +350,3 @@ sorted_groups = sorted(group_stats, key=lambda x: x[actual_sort_key], reverse=is_numeric)[:top_n]

natural_width = max([len(row[data_key]) for row in sorted_groups] + [len(header_text)])
natural_width = max([len(str(row[data_key])) for row in sorted_groups] + [len(header_text)])
target_width = width_overrides.get(col_name, natural_width)

@@ -160,3 +371,3 @@

for i, key in enumerate(final_keys):
val = group[key]
val = str(group[key])
if len(val) > final_widths[i]:

@@ -176,3 +387,10 @@ val = val[:final_widths[i] - 3] + "..."

parser.add_argument("--width", action='append', default=['dims:40'], help="Override column width, e.g. --width dims:50")
parser.add_argument("--timeline", type=str, nargs='?', const='summary', choices=["summary", "diagram"],
help="Output ASCII art event summary or timing diagram (default: summary)")
parser.add_argument("--filter", type=str, help="Regex filter matching against the original profile-op line")
group = parser.add_mutually_exclusive_group()
group.add_argument("--head", type=int, help="Limit to first N ops")
group.add_argument("--tail", type=int, help="Limit to last N ops")
args = parser.parse_args()

@@ -182,3 +400,2 @@

# Sort validation: can't sort by PMU if index isn't provided
if "pmu" in args.sort and args.pmu_index is None:

@@ -199,6 +416,32 @@ logger.error(f"Cannot sort by '{args.sort}' without --pmu-index.")

ops = parse_log(args.logfile, pmu_index=args.pmu_index)
generate_report(ops, args.top, overrides, args.sort, pmu_name=final_pmu_name)
if args.filter:
try:
filter_re = re.compile(args.filter)
except re.error as e:
logger.error(f"Invalid regex filter: {e}")
sys.exit(1)
ops = [op for op in ops if filter_re.search(op['op_text'])]
if args.head is not None:
ops = ops[:args.head]
elif args.tail is not None:
ops = ops[-args.tail:]
if args.timeline:
logger.info(f"\n# ASCII Timing {args.timeline.capitalize()}\n")
printed_cnt = 0
for op in ops:
if args.timeline == "summary":
print_ascii_summary(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events'], op.get('evt_val'))
elif args.timeline == "diagram":
print_ascii_timeline(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events'], op.get('evt_val'))
printed_cnt += 1
if printed_cnt >= args.top:
break
else:
generate_report(ops, args.top, overrides, args.sort, pmu_name=final_pmu_name)
if __name__ == "__main__":
main()
+1
-1
vendor/llama.cpp/scripts/sync_vendor.py

@@ -8,3 +8,3 @@ #!/usr/bin/env python3

HTTPLIB_VERSION = "refs/tags/v0.47.0"
HTTPLIB_VERSION = "refs/tags/v0.48.0"

@@ -11,0 +11,0 @@ vendor = {

+23
-4
vendor/llama.cpp/scripts/ui-assets.cmake

@@ -23,2 +23,3 @@ # Provision UI assets and generate ui.cpp/ui.h.

set(SRC_DIST_DIR "${UI_SOURCE_DIR}/dist")
set(WORK_DIR "${UI_BINARY_DIR}/ui-src")
set(STAMP_FILE "${UI_BINARY_DIR}/.ui-stamp")

@@ -68,2 +69,18 @@ set(UI_CPP "${UI_BINARY_DIR}/ui.cpp")

function(stage_sources)
if(EXISTS "${WORK_DIR}")
file(GLOB staged RELATIVE "${WORK_DIR}" "${WORK_DIR}/*")
list(REMOVE_ITEM staged "node_modules")
foreach(entry ${staged})
file(REMOVE_RECURSE "${WORK_DIR}/${entry}")
endforeach()
endif()
file(COPY "${UI_SOURCE_DIR}/"
DESTINATION "${WORK_DIR}"
NO_SOURCE_PERMISSIONS
PATTERN "node_modules" EXCLUDE
)
endfunction()
function(npm_build out_var)

@@ -94,5 +111,7 @@ set(${out_var} FALSE PARENT_SCOPE)

stage_sources()
# npm writes node_modules/.package-lock.json on every successful install,
# so a package-lock.json newer than this marker means node_modules is stale
set(NPM_MARKER "${UI_SOURCE_DIR}/node_modules/.package-lock.json")
set(NPM_MARKER "${WORK_DIR}/node_modules/.package-lock.json")
set(need_install FALSE)

@@ -102,3 +121,3 @@ if(NOT EXISTS "${NPM_MARKER}")

else()
file(TIMESTAMP "${UI_SOURCE_DIR}/package-lock.json" lock_ts)
file(TIMESTAMP "${WORK_DIR}/package-lock.json" lock_ts)
file(TIMESTAMP "${NPM_MARKER}" marker_ts)

@@ -114,3 +133,3 @@ if(lock_ts STRGREATER marker_ts)

COMMAND ${NPM_EXECUTABLE} install
WORKING_DIRECTORY "${UI_SOURCE_DIR}"
WORKING_DIRECTORY "${WORK_DIR}"
RESULT_VARIABLE rc

@@ -132,3 +151,3 @@ ERROR_VARIABLE err

${NPM_EXECUTABLE} run build
WORKING_DIRECTORY "${UI_SOURCE_DIR}"
WORKING_DIRECTORY "${WORK_DIR}"
RESULT_VARIABLE rc

@@ -135,0 +154,0 @@ ERROR_VARIABLE err

+3
-2
vendor/llama.cpp/src/llama-graph.h

@@ -856,7 +856,8 @@ #pragma once

// do mat_mul_id, while optionally apply lora
// do mat_mul_id, while optionally apply lora and per-expert scale
ggml_tensor * build_lora_mm_id(
ggml_tensor * w, // ggml_tensor * as
ggml_tensor * cur, // ggml_tensor * b
ggml_tensor * ids) const;
ggml_tensor * ids,
ggml_tensor * w_s = nullptr) const;

@@ -863,0 +864,0 @@ ggml_tensor * build_norm(

+4
-0
vendor/llama.cpp/src/llama-hparams.cpp

@@ -107,2 +107,6 @@ #include "llama-hparams.h"

uint32_t llama_hparams::n_embd_inp_enc() const {
return n_embd_inp_enc_impl > 0 ? n_embd_inp_enc_impl : n_embd_inp();
}
uint32_t llama_hparams::n_embd_out() const {

@@ -109,0 +113,0 @@ return n_embd_out_impl > 0 ? n_embd_out_impl : n_embd;

+7
-0
vendor/llama.cpp/src/llama-hparams.h

@@ -192,2 +192,6 @@ #pragma once

// encoder input embedding dimension (0 = use n_embd_inp())
// e.g. the eagle3 encoder fuses target_layers * target_hidden features
uint32_t n_embd_inp_enc_impl = 0;
// output embedding dimension (0 = use n_embd)

@@ -309,2 +313,5 @@ uint32_t n_embd_out_impl = 0;

// dimension of the encoder input embeddings
uint32_t n_embd_inp_enc() const;
// dimension of output embeddings

@@ -311,0 +318,0 @@ uint32_t n_embd_out() const;

+1
-1
vendor/llama.cpp/src/llama.cpp

@@ -252,3 +252,3 @@ #include "llama.h"

// if using single GPU mode, remove all except the main GPU
if (params.split_mode == LLAMA_SPLIT_MODE_NONE) {
if (params.split_mode == LLAMA_SPLIT_MODE_NONE && !model->devices.empty()) {
if (params.main_gpu < 0) {

@@ -255,0 +255,0 @@ model->devices.clear();

+4
-4
vendor/llama.cpp/src/models/eagle3.cpp

@@ -22,3 +22,3 @@ #include "models.h"

hparams.n_embd_inp_impl = (uint32_t) target_layer_ids.size() * n_embd_tgt;
hparams.n_embd_inp_enc_impl = (uint32_t) target_layer_ids.size() * n_embd_tgt;

@@ -38,3 +38,3 @@ // eagle3 norm_before_residual (optional, default false)

const int64_t n_embd_inp = hparams.n_embd_inp();
const int64_t n_embd_inp = hparams.n_embd_inp_enc();
const int64_t n_embd_attn_input = 2 * n_embd;

@@ -114,4 +114,4 @@

// Data will be provided via ubatch->embd in encode_eagle3_features()
auto inp_target = std::make_unique<llm_graph_input_embd>(hparams.n_embd_inp());
inp_target->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32,hparams.n_embd_inp(), n_tokens);
auto inp_target = std::make_unique<llm_graph_input_embd>(hparams.n_embd_inp_enc());
inp_target->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp_enc(), n_tokens);
ggml_set_input(inp_target->embd);

@@ -118,0 +118,0 @@ 

+2
-0
vendor/llama.cpp/src/models/qwen35.cpp

@@ -159,2 +159,4 @@ #include "models.h"

for (int il = 0; il < n_layer; ++il) {
res->t_layer_inp[il] = inpL;
ggml_tensor * inpSA = inpL;

@@ -161,0 +163,0 @@ 

+2
-0
vendor/llama.cpp/src/models/qwen35moe.cpp

@@ -182,2 +182,4 @@ #include "models.h"

for (int il = 0; il < n_layer; ++il) {
res->t_layer_inp[il] = inpL;
ggml_tensor * inpSA = inpL;

@@ -184,0 +186,0 @@ 

+1
-1
vendor/llama.cpp/tools/cli/cli.cpp

@@ -205,3 +205,3 @@ #include "chat.h"

std::string load_input_file(const std::string & fname, bool is_media) {
std::ifstream file(fname, std::ios::binary);
std::ifstream file = fs_open_ifstream(fname, std::ios::binary);
if (!file) {

@@ -208,0 +208,0 @@ return "";

+2
-1
vendor/llama.cpp/tools/cli/README.md

@@ -164,3 +164,3 @@ # llama.cpp/tools/cli

| `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_OFFLOAD) |
| `--image, --audio FILE` | path to an image or audio file. use with multimodal models, use comma-separated values for multiple files |
| `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files |
| `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) |

@@ -178,2 +178,3 @@ | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) |

| `--simple-io` | use basic IO for better compatibility in subprocesses and limited consoles |
| `--log-prompts-dir PATH` | Log prompts to directory (only used for debugging, default: disabled) |
| `--spec-draft-hf, -hfd, -hfrd, --hf-repo-draft <user>/<model>[:quant]` | Same as --hf-repo, but for the draft model (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_HF_REPO) |

@@ -180,0 +181,0 @@ | `--spec-draft-threads, -td, --threads-draft N` | number of threads to use during generation (default: same as --threads) |

+6
-8
vendor/llama.cpp/tools/export-lora/README.md

@@ -9,7 +9,6 @@ # export-lora

options:
-m, --model model path from which to load base model (default '')
--lora FNAME path to LoRA adapter (can be repeated to use multiple adapters)
--lora-scaled FNAME S path to LoRA adapter with user defined scaling S (can be repeated to use multiple adapters)
-t, --threads N number of threads to use during computation (default: 4)
-o, --output FNAME output file (default: 'ggml-lora-merged-f16.gguf')
-m, --model FNAME model path from which to load base model
--lora FNAME path to LoRA adapter (use comma-separated values to load multiple adapters)
--lora-scaled FNAME:SCALE,... path to LoRA adapter with user defined scaling (format: FNAME:SCALE,...)
-o, --output, --output-file FNAME output file (default: 'ggml-lora-merged-f16.gguf')
```

@@ -26,3 +25,3 @@

Multiple LORA adapters can be applied by passing multiple `--lora FNAME` or `--lora-scaled FNAME S` command line parameters:
Multiple LORA adapters can be applied by passing comma-separated values to `--lora FNAME` or `--lora-scaled FNAME:SCALE,...`:

@@ -33,4 +32,3 @@ ```bash

-o your_merged_model.gguf \
--lora-scaled lora_task_A.gguf 0.5 \
--lora-scaled lora_task_B.gguf 0.5
--lora-scaled lora_task_A.gguf:0.5,lora_task_B.gguf:0.5
```
+76
-81
vendor/llama.cpp/tools/mtmd/clip-impl.h

@@ -16,3 +16,11 @@ #pragma once

#include <memory>
#include <fstream>
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#endif
// Internal header for clip.cpp

@@ -371,52 +379,52 @@

static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_MLP, "mlp" },
{ PROJECTOR_TYPE_LDP, "ldp" },
{ PROJECTOR_TYPE_LDPV2, "ldpv2"},
{ PROJECTOR_TYPE_MINICPMV, "resampler"},
{ PROJECTOR_TYPE_GLM_EDGE, "adapter"},
{ PROJECTOR_TYPE_QWEN2VL, "qwen2vl_merger"},
{ PROJECTOR_TYPE_QWEN25VL, "qwen2.5vl_merger"},
{ PROJECTOR_TYPE_QWEN3VL, "qwen3vl_merger"},
{ PROJECTOR_TYPE_STEP3VL, "step3vl"},
{ PROJECTOR_TYPE_GEMMA3, "gemma3"},
{ PROJECTOR_TYPE_GEMMA3NV, "gemma3nv"},
{ PROJECTOR_TYPE_GEMMA3NA, "gemma3na"},
{ PROJECTOR_TYPE_GEMMA4V, "gemma4v"},
{ PROJECTOR_TYPE_GEMMA4A, "gemma4a"},
{ PROJECTOR_TYPE_GEMMA4UV, "gemma4uv"},
{ PROJECTOR_TYPE_GEMMA4UA, "gemma4ua"},
{ PROJECTOR_TYPE_PHI4, "phi4"},
{ PROJECTOR_TYPE_IDEFICS3, "idefics3"},
{ PROJECTOR_TYPE_PIXTRAL, "pixtral"},
{ PROJECTOR_TYPE_ULTRAVOX, "ultravox"},
{ PROJECTOR_TYPE_INTERNVL, "internvl"},
{ PROJECTOR_TYPE_LLAMA4, "llama4"},
{ PROJECTOR_TYPE_QWEN2A, "qwen2a"},
{ PROJECTOR_TYPE_QWEN3A, "qwen3a"},
{ PROJECTOR_TYPE_GLMA, "glma"},
{ PROJECTOR_TYPE_QWEN25O, "qwen2.5o"},
{ PROJECTOR_TYPE_VOXTRAL, "voxtral"},
{ PROJECTOR_TYPE_MERALION, "meralion"},
{ PROJECTOR_TYPE_MUSIC_FLAMINGO, "musicflamingo"},
{ PROJECTOR_TYPE_LFM2, "lfm2"},
{ PROJECTOR_TYPE_KIMIVL, "kimivl"},
{ PROJECTOR_TYPE_PADDLEOCR, "paddleocr"},
{ PROJECTOR_TYPE_LIGHTONOCR,"lightonocr"},
{ PROJECTOR_TYPE_COGVLM, "cogvlm"},
{ PROJECTOR_TYPE_JANUS_PRO, "janus_pro"},
{ PROJECTOR_TYPE_DOTS_OCR, "dots_ocr"},
{ PROJECTOR_TYPE_DEEPSEEKOCR,"deepseekocr"},
{ PROJECTOR_TYPE_DEEPSEEKOCR2,"deepseekocr2"},
{ PROJECTOR_TYPE_LFM2A, "lfm2a"},
{ PROJECTOR_TYPE_GLM4V, "glm4v"},
{ PROJECTOR_TYPE_YOUTUVL, "youtuvl"},
{ PROJECTOR_TYPE_YASA2, "yasa2"},
{ PROJECTOR_TYPE_KIMIK25, "kimik25"},
{ PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},
{ PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"},
{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},
{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},
{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},
{ PROJECTOR_TYPE_MIMOVL, "mimovl"},
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
{ PROJECTOR_TYPE_MLP, "mlp" },
{ PROJECTOR_TYPE_LDP, "ldp" },
{ PROJECTOR_TYPE_LDPV2, "ldpv2"},
{ PROJECTOR_TYPE_MINICPMV, "resampler"},
{ PROJECTOR_TYPE_GLM_EDGE, "adapter"},
{ PROJECTOR_TYPE_QWEN2VL, "qwen2vl_merger"},
{ PROJECTOR_TYPE_QWEN25VL, "qwen2.5vl_merger"},
{ PROJECTOR_TYPE_QWEN3VL, "qwen3vl_merger"},
{ PROJECTOR_TYPE_STEP3VL, "step3vl"},
{ PROJECTOR_TYPE_GEMMA3, "gemma3"},
{ PROJECTOR_TYPE_GEMMA3NV, "gemma3nv"},
{ PROJECTOR_TYPE_GEMMA3NA, "gemma3na"},
{ PROJECTOR_TYPE_GEMMA4V, "gemma4v"},
{ PROJECTOR_TYPE_GEMMA4A, "gemma4a"},
{ PROJECTOR_TYPE_GEMMA4UV, "gemma4uv"},
{ PROJECTOR_TYPE_GEMMA4UA, "gemma4ua"},
{ PROJECTOR_TYPE_PHI4, "phi4"},
{ PROJECTOR_TYPE_IDEFICS3, "idefics3"},
{ PROJECTOR_TYPE_PIXTRAL, "pixtral"},
{ PROJECTOR_TYPE_ULTRAVOX, "ultravox"},
{ PROJECTOR_TYPE_INTERNVL, "internvl"},
{ PROJECTOR_TYPE_LLAMA4, "llama4"},
{ PROJECTOR_TYPE_QWEN2A, "qwen2a"},
{ PROJECTOR_TYPE_QWEN3A, "qwen3a"},
{ PROJECTOR_TYPE_GLMA, "glma"},
{ PROJECTOR_TYPE_QWEN25O, "qwen2.5o"},
{ PROJECTOR_TYPE_VOXTRAL, "voxtral"},
{ PROJECTOR_TYPE_MERALION, "meralion"},
{ PROJECTOR_TYPE_MUSIC_FLAMINGO, "musicflamingo"},
{ PROJECTOR_TYPE_LFM2, "lfm2"},
{ PROJECTOR_TYPE_KIMIVL, "kimivl"},
{ PROJECTOR_TYPE_PADDLEOCR, "paddleocr"},
{ PROJECTOR_TYPE_LIGHTONOCR, "lightonocr"},
{ PROJECTOR_TYPE_COGVLM, "cogvlm"},
{ PROJECTOR_TYPE_JANUS_PRO, "janus_pro"},
{ PROJECTOR_TYPE_DOTS_OCR, "dots_ocr"},
{ PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"},
{ PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"},
{ PROJECTOR_TYPE_LFM2A, "lfm2a"},
{ PROJECTOR_TYPE_GLM4V, "glm4v"},
{ PROJECTOR_TYPE_YOUTUVL, "youtuvl"},
{ PROJECTOR_TYPE_YASA2, "yasa2"},
{ PROJECTOR_TYPE_KIMIK25, "kimik25"},
{ PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},
{ PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"},
{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},
{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},
{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},
{ PROJECTOR_TYPE_MIMOVL, "mimovl"},
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
};

@@ -645,33 +653,6 @@

// wrapper for clip_image_size
struct clip_image_size_deleter {
void operator()(clip_image_size * val) { clip_image_size_free(val); }
};
typedef std::unique_ptr<clip_image_size, clip_image_size_deleter> clip_image_size_ptr;
// wrapper for clip_image_u8
struct clip_image_u8_deleter {
void operator()(clip_image_u8 * val) { clip_image_u8_free(val); }
};
typedef std::unique_ptr<clip_image_u8, clip_image_u8_deleter> clip_image_u8_ptr;
// wrapper for clip_image_f32
struct clip_image_f32_deleter {
void operator()(clip_image_f32 * val) { clip_image_f32_free(val); }
};
typedef std::unique_ptr<clip_image_f32, clip_image_f32_deleter> clip_image_f32_ptr;
struct clip_image_u8_batch {
std::vector<clip_image_u8_ptr> entries;
};
struct clip_image_f32_batch {
std::vector<clip_image_f32_ptr> entries;
std::vector<clip_image_f32> entries;
bool is_audio = false;
// for llava-uhd style models, we need to know the grid size
// note: entries.size() == grid_x * grid_y + 1 (one overview image)
int grid_x = 0;
int grid_y = 0;
clip_image_f32_batch clone() const {

@@ -681,8 +662,6 @@ clip_image_f32_batch new_batch{

/* is_audio */ is_audio,
/* grid_x */ grid_x,
/* grid_y */ grid_y,
};
new_batch.entries.reserve(entries.size());
for (const auto & entry : entries) {
new_batch.entries.emplace_back(new clip_image_f32(*entry));
new_batch.entries.emplace_back(entry); // copy
}

@@ -697,2 +676,18 @@ return new_batch;

#ifdef _WIN32
static std::ifstream open_ifstream_binary(const std::string & fname) {
int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0);
if (!wlen) {
throw std::runtime_error("failed to convert filename to UTF-16: " + fname);
}
std::vector<wchar_t> wfname(wlen);
(void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen);
return std::ifstream(wfname.data(), std::ios::binary);
}
#else
static std::ifstream open_ifstream_binary(const std::string & fname) {
return std::ifstream(fname, std::ios::binary);
}
#endif
static std::string string_format(const char * fmt, ...) {

@@ -699,0 +694,0 @@ va_list ap;

+8
-26
vendor/llama.cpp/tools/mtmd/clip.h

@@ -27,2 +27,5 @@ #pragma once

int area() const {
// avoid overflow when computing area
GGML_ASSERT(width >= 0 && width <= 46000);
GGML_ASSERT(height >= 0 && height <= 46000);
return width * height;

@@ -33,3 +36,2 @@ }

struct clip_image_f32;
struct clip_image_u8_batch;
struct clip_image_f32_batch;

@@ -68,15 +70,11 @@

int32_t clip_get_image_size (const struct clip_ctx * ctx);
int32_t clip_get_patch_size (const struct clip_ctx * ctx);
int32_t clip_get_hidden_size(const struct clip_ctx * ctx);
// TODO: should be enum, not string
const char * clip_patch_merge_type(const struct clip_ctx * ctx);
int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * img);
int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img);
// for M-RoPE, this will be the number of token positions in X and Y directions
// for other models, X will be the total number of tokens and Y will be 1
int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 * img);
int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 * img);
int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img);
int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img);

@@ -86,20 +84,4 @@ // this should be equal to the embedding dimension of the text model

struct clip_image_size * clip_image_size_init(void);
struct clip_image_u8 * clip_image_u8_init (void);
struct clip_image_f32 * clip_image_f32_init(void);
struct clip_image_f32_batch * clip_image_f32_batch_init(void); // only used by libllava
void clip_image_size_free (struct clip_image_size * img_size);
void clip_image_u8_free (struct clip_image_u8 * img);
void clip_image_f32_free(struct clip_image_f32 * img);
void clip_image_u8_batch_free (struct clip_image_u8_batch * batch);
void clip_image_f32_batch_free(struct clip_image_f32_batch * batch);
// use for accessing underlay data of clip_image_f32_batch
size_t clip_image_f32_batch_n_images(const struct clip_image_f32_batch * batch); // equivalent to batch->size()
size_t clip_image_f32_batch_nx(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->nx
size_t clip_image_f32_batch_ny(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->ny
struct clip_image_f32 * clip_image_f32_get_img(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->data
bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, std::vector<float> & out_vec);
// TODO: remove clip_image_encode() and always use batched version
bool clip_image_encode (struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector<float> & out_vec);
bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector<float> & out_batch_embd);

@@ -106,0 +88,0 @@ 

+11
-7
vendor/llama.cpp/tools/mtmd/models/internvl.cpp

@@ -11,3 +11,5 @@ #include "models.h"

// add CLS token
inp = ggml_concat(ctx0, inp, model.class_embedding, 1);
ggml_tensor * cls_repeated = ggml_repeat_4d(ctx0, model.class_embedding,
model.class_embedding->ne[0], 1, n_batch, 1);
inp = ggml_concat(ctx0, inp, cls_repeated, 1);

@@ -28,5 +30,6 @@ // The larger models use a different ViT, which uses RMS norm instead of layer norm

// remove CLS token
cur = ggml_view_2d(ctx0, cur,
n_embd, n_patches,
ggml_row_size(cur->type, n_embd), 0);
cur = ggml_view_3d(ctx0, cur,
n_embd, n_patches, n_batch,
cur->nb[1], cur->nb[2], 0);
cur = ggml_cont(ctx0, cur);

@@ -36,3 +39,3 @@ // pixel shuffle

const int scale_factor = model.hparams.n_merge;
const int bsz = 1; // batch size, always 1 for now since we don't support batching
const int bsz = n_batch;
const int height = n_patches_y;

@@ -50,5 +53,6 @@ const int width = n_patches_x;

// flatten to 2D
cur = ggml_cont_2d(ctx0, cur,
cur = ggml_cont_3d(ctx0, cur,
n_embd * scale_factor * scale_factor,
cur->ne[1] * cur->ne[2]);
cur->ne[1] * cur->ne[2],
cur->ne[3]);
}

@@ -55,0 +59,0 @@ 

+1
-0
vendor/llama.cpp/tools/mtmd/models/models.h

@@ -83,2 +83,3 @@ #pragma once

ggml_cgraph * build() override;
bool support_batch() const override { return true; }
};

@@ -85,0 +86,0 @@ 

+76
-63
vendor/llama.cpp/tools/mtmd/mtmd-audio.cpp

@@ -35,4 +35,4 @@ #include "mtmd-audio.h"

void mtmd_audio_cache::fill_mel_filterbank_matrix(int n_mel,
int n_fft,
void mtmd_audio_cache::fill_mel_filterbank_matrix(int64_t n_mel,
int64_t n_fft,
int sample_rate,

@@ -90,7 +90,12 @@ float fmin,

const int n_fft_bins = n_fft / 2 + 1;
const int64_t n_fft_bins = n_fft / 2 + 1;
// Validate allocation size
if ((size_t)n_mel * (size_t)n_fft_bins > SIZE_MAX) {
GGML_ASSERT(false && "mel filterbank allocation too large");
}
// filterbank
std::vector<float> out(n_mel * n_fft_bins, 0);
for (int m = 0; m < n_mel; ++m) {
std::vector<float> out((size_t)n_mel * (size_t)n_fft_bins, 0);
for (int64_t m = 0; m < n_mel; ++m) {
const double f_left = hz_pts[m];

@@ -271,4 +276,4 @@ const double f_center = hz_pts[m + 1];

struct filter_params {
int32_t n_mel;
int32_t n_fft_bins;
int64_t n_mel;
int64_t n_fft_bins;
int32_t hann_window_size;

@@ -299,4 +304,4 @@ int32_t hop_length;

int n_fft_bins = params.n_fft_bins;
int i = ith;
int64_t n_fft_bins = params.n_fft_bins;
int64_t i = ith;

@@ -309,7 +314,8 @@ const auto & filters = cache.filters;

// calculate FFT only when fft_in are not all zero
for (; i < std::min(n_samples / frame_step + 1, out.n_len); i += n_threads) {
const int offset = i * frame_step;
for (; i < std::min((int64_t)(n_samples / frame_step + 1), out.n_len); i += n_threads) {
const int64_t offset = i * frame_step;
// apply Hann window (~10% faster)
for (int j = 0; j < std::min(frame_size, n_samples - offset); j++) {
const int valid_len = std::min(frame_size, std::max(0, n_samples - (int)offset));
for (int j = 0; j < valid_len; j++) {
fft_in[j] = hann[j] * samples[offset + j];

@@ -319,4 +325,4 @@ }

// fill the rest with zeros
if (n_samples - offset < frame_size) {
std::fill(fft_in.begin() + (n_samples - offset), fft_in.end(), 0.0);
if (valid_len < frame_size) {
std::fill(fft_in.begin() + valid_len, fft_in.end(), 0.0);
}

@@ -334,3 +340,3 @@

// mel spectrogram
for (int j = 0; j < out.n_mel; j++) {
for (int64_t j = 0; j < out.n_mel; j++) {
double sum = 0.0;

@@ -349,3 +355,3 @@ // unroll loop (suggested by GH user @lunixbochs)

for (; k < n_fft_bins; k++) {
sum += fft_out[k] * filters.data[j * n_fft_bins + k];
sum += fft_out[k] * filters.data[(size_t)j * n_fft_bins + k];
}

@@ -356,3 +362,3 @@ sum = std::max(sum, (double)params.mel_floor);

: log10(sum);
out.data[j * out.n_len + i] = sum;
out.data[(size_t)j * out.n_len + i] = sum;
}

@@ -364,4 +370,4 @@ }

for (; i < out.n_len; i += n_threads) {
for (int j = 0; j < out.n_mel; j++) {
out.data[j * out.n_len + i] = sum;
for (int64_t j = 0; j < out.n_mel; j++) {
out.data[(size_t)j * out.n_len + i] = sum;
}

@@ -450,7 +456,12 @@ }

out.n_len = (n_samples - frame_size) / frame_step + 1;
// TODO: handle these checks better
if (out.n_mel > 0 && (unsigned long)out.n_len > SIZE_MAX / out.n_mel) {
LOG_ERR("%s: size overflow\n", __func__);
// Validate dimensions before allocation to prevent integer overflow
if (out.n_mel <= 0 || out.n_len <= 0) {
LOG_ERR("%s: invalid mel dimensions n_mel=%lld n_len=%lld\n", __func__, (long long)out.n_mel, (long long)out.n_len);
return false;
}
const size_t total_size = (size_t)out.n_mel * (size_t)out.n_len;
if (total_size > SIZE_MAX / sizeof(float)) {
LOG_ERR("%s: size overflow: n_mel=%lld n_len=%lld\n", __func__, (long long)out.n_mel, (long long)out.n_len);
return false;
}
if (n_samples < frame_size) {

@@ -460,3 +471,3 @@ LOG_ERR("%s: not enough samples after padding\n", __func__);

}
out.data.resize(out.n_mel * out.n_len);
out.data.resize(total_size);

@@ -479,9 +490,9 @@ {

const int effective_n_len = n_samples_in / frame_step;
const int64_t effective_n_len = n_samples_in / frame_step;
if (params.norm_per_feature) {
GGML_ASSERT(effective_n_len > 1);
for (int i = 0; i < out.n_mel; i++) {
for (int64_t i = 0; i < out.n_mel; i++) {
double mean = 0;
for (int j = 0; j < effective_n_len; ++j) {
mean += out.data[i * out.n_len + j];
for (int64_t j = 0; j < effective_n_len; ++j) {
mean += out.data[(size_t)i * out.n_len + j];
}

@@ -491,4 +502,4 @@ mean /= effective_n_len;

double var = 0.0;
for (int j = 0; j < effective_n_len; ++j) {
const double value = out.data[i * out.n_len + j] - mean;
for (int64_t j = 0; j < effective_n_len; ++j) {
const double value = out.data[(size_t)i * out.n_len + j] - mean;
var += value * value;

@@ -499,4 +510,4 @@ }

for (int j = 0; j < effective_n_len; ++j) {
auto &value = out.data[i * out.n_len + j];
for (int64_t j = 0; j < effective_n_len; ++j) {
auto &value = out.data[(size_t)i * out.n_len + j];
value = (value - mean) / mstd;

@@ -506,4 +517,4 @@ }

// pad the rest with zeros
for (int j = effective_n_len; j < out.n_len; ++j) {
out.data[i * out.n_len + j] = 0.0;
for (int64_t j = effective_n_len; j < out.n_len; ++j) {
out.data[(size_t)i * out.n_len + j] = 0.0;
}

@@ -514,3 +525,4 @@ }

double mmax = -1e20;
for (int i = 0; i < out.n_mel*out.n_len; i++) {
const size_t mel_size = (size_t)out.n_mel * (size_t)out.n_len;
for (size_t i = 0; i < mel_size; i++) {
if (out.data[i] > mmax) {

@@ -523,3 +535,3 @@ mmax = out.data[i];

for (int i = 0; i < out.n_mel*out.n_len; i++) {
for (size_t i = 0; i < mel_size; i++) {
if (out.data[i] < mmax) {

@@ -603,3 +615,3 @@ out.data[i] = mmax;

if (DEBUG) {
printf("output: n_mel = %d, n_len = %d\n", out_full.n_mel, out_full.n_len);
printf("output: n_mel = %d, n_len = %d\n", (int) out_full.n_mel, (int) out_full.n_len);
}

@@ -609,4 +621,4 @@ const size_t frames_per_chunk = 3000;

for (size_t off = 0; off < (size_t) out_full.n_len; off += frames_per_chunk) {
int n_len = std::min(frames_per_chunk, (size_t) out_full.n_len - off);
if ((size_t) n_len < frames_per_chunk) {
int64_t n_len = std::min((int64_t)frames_per_chunk, out_full.n_len - (int64_t)off);
if (n_len < (int64_t)frames_per_chunk) {
break; // last incomplete chunk will always be a padded chunk, safe to ignore

@@ -619,6 +631,6 @@ }

out_chunk.n_len_org = out_full.n_mel; // unused
out_chunk.data.reserve(out_chunk.n_mel * out_chunk.n_len);
out_chunk.data.reserve((size_t)out_chunk.n_mel * (size_t)out_chunk.n_len);
for (int i = 0; i < out_full.n_mel; i++) {
auto src = out_full.data.begin() + i * out_full.n_len + off;
for (int64_t i = 0; i < out_full.n_mel; i++) {
auto src = out_full.data.begin() + (size_t)i * out_full.n_len + off;
out_chunk.data.insert(out_chunk.data.end(), src, src + frames_per_chunk);

@@ -705,4 +717,4 @@ }

// We take min(mel_full.n_len, n_samples/hop + 1) to avoid including excess frames.
const int n_eff = std::min(mel_full.n_len,
(int)(n_samples / hparams.audio_hop_len) + 1);
const int64_t n_eff = std::min(mel_full.n_len,
(int64_t)(n_samples / hparams.audio_hop_len) + 1);

@@ -715,6 +727,6 @@ // Split into inference windows matching n_window_infer=800 from model config.

for (int off = 0; off < n_eff; off += window_size) {
const int win_eff = std::min(window_size, n_eff - off);
const int n_chunks = (win_eff + chunk_size - 1) / chunk_size;
const int n_padded = n_chunks * chunk_size;
for (int64_t off = 0; off < n_eff; off += window_size) {
const int64_t win_eff = std::min((int64_t)window_size, n_eff - off);
const int64_t n_chunks = (win_eff + chunk_size - 1) / chunk_size;
const int64_t n_padded = n_chunks * chunk_size;

@@ -725,5 +737,5 @@ mtmd_audio_mel out;

out.n_len_org = win_eff;
out.data.assign(out.n_mel * out.n_len, 0.0f);
for (int m = 0; m < out.n_mel; m++) {
const int copy_len = std::min(win_eff, mel_full.n_len - off);
out.data.assign((size_t)out.n_mel * (size_t)out.n_len, 0.0f);
for (int64_t m = 0; m < out.n_mel; m++) {
const int64_t copy_len = std::min((int64_t)win_eff, mel_full.n_len - off);
if (copy_len > 0) {

@@ -850,3 +862,4 @@ std::copy(mel_full.data.begin() + (size_t)m * mel_full.n_len + off,

double mmax = -1e20;
for (int i = 0; i < mel.n_mel * mel.n_len; i++) {
const size_t mel_size = (size_t)mel.n_mel * (size_t)mel.n_len;
for (size_t i = 0; i < mel_size; i++) {
if (mel.data[i] > mmax) {

@@ -858,3 +871,3 @@ mmax = mel.data[i];

for (int i = 0; i < mel.n_mel * mel.n_len; i++) {
for (size_t i = 0; i < mel_size; i++) {
if (mel.data[i] < mmax) {

@@ -866,8 +879,8 @@ mel.data[i] = mmax;

int n_frames = mel.n_len;
int64_t n_frames = mel.n_len;
if (n_frames % 2 == 1) {
n_frames--;
}
const int n_mel = mel.n_mel;
const int n_stacked = n_frames / 2;
const int64_t n_mel = mel.n_mel;
const int64_t n_stacked = n_frames / 2;

@@ -877,9 +890,9 @@ mtmd_audio_mel stacked;

stacked.n_len = n_stacked;
stacked.n_len_org = (int)n_samples;
stacked.data.resize(2 * n_mel * n_stacked);
stacked.n_len_org = (int64_t)n_samples;
stacked.data.resize((size_t)2 * (size_t)n_mel * (size_t)n_stacked);
for (int t = 0; t < n_stacked; t++) {
for (int m = 0; m < n_mel; m++) {
stacked.data[m * n_stacked + t] = mel.data[m * mel.n_len + 2 * t];
stacked.data[(m + n_mel) * n_stacked + t] = mel.data[m * mel.n_len + 2 * t + 1];
for (int64_t t = 0; t < n_stacked; t++) {
for (int64_t m = 0; m < n_mel; m++) {
stacked.data[(size_t)m * n_stacked + t] = mel.data[(size_t)m * mel.n_len + 2 * t];
stacked.data[(size_t)(m + n_mel) * n_stacked + t] = mel.data[(size_t)m * mel.n_len + 2 * t + 1];
}

@@ -952,4 +965,4 @@ }

// PyTorch: unfold(size=frame_length+1, step=hop) on semicausal-padded waveform
const int pt_frames = (n_with_left - (hparams.audio_window_len + 1)) / hop + 1;
const int n_padded_needed = (pt_frames - 1) * hop + fft_size;
const int64_t pt_frames = (n_with_left - (hparams.audio_window_len + 1)) / hop + 1;
const int64_t n_padded_needed = (pt_frames - 1) * hop + fft_size;
const int total_pad = std::max((int)(n_padded_needed - (int)chunk_len), pad_left);

@@ -956,0 +969,0 @@ std::vector<float> padded_samples(total_pad + chunk_len, 0.0f);

+7
-7
vendor/llama.cpp/tools/mtmd/mtmd-audio.h

@@ -13,5 +13,5 @@ #pragma once

struct mtmd_audio_mel {
int n_len;
int n_len_org;
int n_mel;
int64_t n_len;
int64_t n_len_org;
int64_t n_mel;

@@ -22,4 +22,4 @@ std::vector<float> data;

struct mtmd_audio_mel_filters {
int32_t n_mel;
int32_t n_fft;
int64_t n_mel;
int64_t n_fft;

@@ -44,4 +44,4 @@ std::vector<float> data;

// n_fft_bins must be (N_fft / 2 + 1). Example: if N_fft=512 -> n_fft_bins=257.
void fill_mel_filterbank_matrix(int n_mel,
int n_fft,
void fill_mel_filterbank_matrix(int64_t n_mel,
int64_t n_fft,
int sample_rate, // e.g. 16000

@@ -48,0 +48,0 @@ float fmin = 0.0f, // e.g. 0.0

+105
-15
vendor/llama.cpp/tools/mtmd/mtmd-cli.cpp

@@ -35,5 +35,5 @@ #include "arg.h"

/**
* Please note that this is NOT a production-ready stuff.
* Please note that this is NOT a production-ready binary.
* It is a playground for trying multimodal support in llama.cpp.
* For contributors: please keep this code simple and easy to understand.
* For contributors: please keep this code simple and easy to understand. Do not add unnecessary complexity. The goal is to have a simple CLI for testing multimodal support.
*/

@@ -69,2 +69,10 @@

// this is only used by tests.sh to capture the response ; it's not meant to be used in production
static void inject_test_response_marker() {
const char * env = std::getenv("MTMD_TEST_RESPONSE_MARKER");
if (env) {
LOG("%s\n", env);
}
}
struct mtmd_cli_context {

@@ -84,2 +92,4 @@ mtmd::context_ptr ctx_vision;

mtmd::batch_ptr mbatch;
// chat template

@@ -239,2 +249,4 @@ common_chat_templates_ptr tmpls;

static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {
inject_test_response_marker();
bool add_bos = ctx.chat_history.empty();

@@ -266,17 +278,92 @@ auto formatted_chat = chat_add_and_format(ctx, msg);

llama_pos new_n_past;
if (mtmd_helper_eval_chunks(ctx.ctx_vision.get(),
ctx.lctx, // lctx
chunks.ptr.get(), // chunks
ctx.n_past, // n_past
0, // seq_id
ctx.n_batch, // n_batch
true, // logits_last
&new_n_past)) {
LOG_ERR("Unable to eval prompt\n");
return 1;
// batch encode all media chunks, then decode each
size_t n_chunks = mtmd_input_chunks_size(chunks.ptr.get());
for (size_t i = 0; i < n_chunks; i++) {
auto chunk = mtmd_input_chunks_get(chunks.ptr.get(), i);
auto chunk_type = mtmd_input_chunk_get_type(chunk);
if (chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
// decode text chunk
llama_pos new_n_past = ctx.n_past;
res = mtmd_helper_eval_chunk_single(ctx.ctx_vision.get(),
ctx.lctx,
chunk,
ctx.n_past,
0, // seq_id
ctx.n_batch,
i == n_chunks - 1, // logits_last
&new_n_past);
if (res != 0) {
LOG_ERR("Unable to eval text chunk %zu\n", i);
return 1;
}
ctx.n_past = new_n_past;
} else {
// media chunk: try to get embd from existing batch, or create a new batch
float * embd = nullptr;
if (ctx.mbatch) {
embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk);
if (embd) {
LOG_DBG("found embd for media chunk %zu in existing batch\n", i);
} else {
LOG_DBG("media chunk %zu not found in existing batch, creating new batch\n", i);
}
}
if (!embd) {
// create and encode a new batch with as many media chunks as possible
ctx.mbatch.reset(mtmd_batch_init(ctx.ctx_vision.get()));
res = mtmd_batch_add_chunk(ctx.mbatch.get(), chunk);
GGML_ASSERT(res == 0); // first chunk must always succeed
int n_added = 1;
// add as many subsequent media chunks as possible
for (size_t j = i + 1; j < n_chunks; j++) {
auto next_chunk = mtmd_input_chunks_get(chunks.ptr.get(), j);
auto next_type = mtmd_input_chunk_get_type(next_chunk);
if (next_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
break; // text chunk splits the batch
}
res = mtmd_batch_add_chunk(ctx.mbatch.get(), next_chunk);
if (res != 0) {
break; // batch full or incompatible
}
n_added++;
}
int64_t time_start = ggml_time_ms();
LOG_INF("encoding mtmd batch, n_chunks = %d (done = %zu, total = %zu)\n", n_added, i, n_chunks);
res = mtmd_batch_encode(ctx.mbatch.get());
if (res != 0) {
LOG_ERR("Failed to encode mtmd batch, res = %d\n", res);
return 1;
}
LOG_INF("mtmd batch encoding done in %d ms\n", (int)(ggml_time_ms() - time_start));
embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk);
}
GGML_ASSERT(embd != nullptr);
llama_pos new_n_past = ctx.n_past;
res = mtmd_helper_decode_image_chunk(ctx.ctx_vision.get(),
ctx.lctx,
chunk,
embd,
ctx.n_past,
0, // seq_id
ctx.n_batch,
&new_n_past,
nullptr, // callback
nullptr // user_data
);
if (res != 0) {
LOG_ERR("Unable to decode media chunk %zu\n", i);
return 1;
}
ctx.n_past = new_n_past;
}
}
ctx.n_past = new_n_past;
LOG("\n");

@@ -317,2 +404,5 @@

console::init(params.simple_io, params.use_color);
atexit([]() { console::cleanup(); });
// Ctrl+C handling

@@ -319,0 +409,0 @@ {

+17
-1
vendor/llama.cpp/tools/mtmd/mtmd-helper.cpp

@@ -585,4 +585,18 @@ // fix problem with std::min and std::max

mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder) {
std::vector<unsigned char> buf;
#ifdef _WIN32
int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
if (!wlen) {
LOG_ERR("Unable to convert filename to UTF-16: %s\n", fname);
return {nullptr, nullptr};
}
std::vector<wchar_t> wfname(wlen);
wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, wfname.data(), wlen);
if (!wlen) {
LOG_ERR("Unable to convert filename to UTF-16: %s\n", fname);
return {nullptr, nullptr};
}
FILE * f = _wfopen(wfname.data(), L"rb");
#else
FILE * f = fopen(fname, "rb");
#endif
if (!f) {

@@ -593,2 +607,4 @@ LOG_ERR("Unable to open file %s: %s\n", fname, strerror(errno));

std::vector<unsigned char> buf;
fseek(f, 0, SEEK_END);

@@ -595,0 +611,0 @@ long file_size = ftell(f);

+42
-15
vendor/llama.cpp/tools/mtmd/mtmd-image.h

@@ -11,2 +11,20 @@ #pragma once

struct mtmd_image_preproc_out {
std::vector<clip_image_f32> entries;
// grid size is required for llava-uhd style models
clip_image_f32 overview; // overview image (downscaled image)
int grid_x = 0;
int grid_y = 0;
void append(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true);
void append(const clip_hparams & hparams, const std::vector<clip_image_u8> & imgs, bool normalized = true);
void append(const clip_hparams & hparams, clip_image_f32 & img, bool normalized = true);
void append_overview(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true);
bool has_overview() const {
return overview.nx() > 0 || overview.ny() > 0;
}
};
// base class, models must inherit from this class

@@ -19,6 +37,3 @@ struct mtmd_image_preprocessor {

virtual ~mtmd_image_preprocessor() = default;
virtual bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) = 0;
void img_u8_to_f32(const clip_image_u8 & src, clip_image_f32 & dst, const float mean[3], const float std[3]);
void img_u8_to_f32(const clip_image_u8 & src, clip_image_f32 & dst);
virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) = 0;
};

@@ -44,6 +59,8 @@

* +--> [slice 3] --> [slice 4]
*
* NOTE: for the ordering of overview, set "ov_img_first" on the mtmd_context
*/
struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {
mtmd_image_preprocessor_llava_uhd(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;

@@ -66,3 +83,7 @@ struct slice_coordinates {

std::vector<clip_image_u8_ptr> slice_image(const clip_image_u8 & img, const slice_instructions & inst, bool overview_first = true);
struct slice_output {
clip_image_u8 overview;
std::vector<clip_image_u8> slices;
};
slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst);

@@ -98,3 +119,3 @@ private:

mtmd_image_preprocessor_fixed_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};

@@ -107,3 +128,3 @@

mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};

@@ -114,3 +135,3 @@

mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};

@@ -141,3 +162,3 @@

mtmd_image_preprocessor_idefics3(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};

@@ -147,3 +168,3 @@

mtmd_image_preprocessor_internvl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};

@@ -153,3 +174,3 @@

mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};

@@ -166,3 +187,3 @@

mtmd_image_preprocessor_deepseekocr2(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;

@@ -182,3 +203,3 @@ private:

mtmd_image_preprocessor_step3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
static slice_instructions build_slice_instructions(const clip_hparams & params, const clip_image_size & prepared_size);

@@ -210,3 +231,9 @@

mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};
// similar to llava_uhd, but has add_newline
struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd {
mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};
+43
-9
vendor/llama.cpp/tools/mtmd/tests.sh

@@ -16,2 +16,4 @@ #!/usr/bin/env bash

export MTMD_TEST_RESPONSE_MARKER="<MTMD_TEST_RESPONSE_MARKER>"
# Check if the first argument is "big", then run test with big models

@@ -32,2 +34,11 @@ # This is useful if we're running the script on a larger machine, so we can test the big models

USE_VIDEO=false
if [ "${1:-}" = "video" ]; then
USE_VIDEO=true
echo "Using video as input..."
# behavior of USE_VIDEO:
# do NOT check if the output contains "new york", only verify if the exit code is 0
# when printing the result, print the OK/FAIL line then print the generated text
fi
# Check if the second argument is "flash", then enable flash attention

@@ -55,9 +66,16 @@ # This is useful to test if flash attention off works correctly

fi
if [ "$USE_VIDEO" = true ]; then
arr_file+=("test-3.mp4")
else
arr_file+=("test-1.jpeg")
fi
arr_prefix+=("[vision]")
arr_hf+=("$hf")
arr_extra_args+=("$extra_args")
arr_file+=("test-1.jpeg")
}
add_test_audio() {
if [ "$USE_VIDEO" = true ]; then
return 0
fi
local hf=$1

@@ -172,15 +190,31 @@ shift

output=$(eval "$cmd" 2>&1 | tee /dev/tty)
exit_code=0
output=$(eval "$cmd" 2>&1 | tee /dev/tty) || exit_code=$?
echo "$output" > $SCRIPT_DIR/output/$bin-$(echo "$hf" | tr '/' '-').log
# either contains "new york" or both "men" and "walk"
if echo "$output" | grep -iq "new york" \
|| (echo "$output" | grep -iq "men" && echo "$output" | grep -iq "walk")
then
result="$prefix \033[32mOK\033[0m: $hf"
if [ "$USE_VIDEO" = true ]; then
# for video, only check exit code; do not grep for "new york"
if [ $exit_code -eq 0 ]; then
result="$prefix \033[32mOK\033[0m: $hf"
else
result="$prefix \033[31mFAIL\033[0m: $hf"
fi
# append generated text (after the response marker)
generated_text=$(echo "$output" | sed "1,/${MTMD_TEST_RESPONSE_MARKER}/d" | tail -10)
if [ -n "$generated_text" ]; then
result+="\n$generated_text"
fi
echo -e "$result"
else
result="$prefix \033[31mFAIL\033[0m: $hf"
# either contains "new york" or both "men" and "walk"
if echo "$output" | grep -iq "new york" \
|| (echo "$output" | grep -iq "men" && echo "$output" | grep -iq "walk")
then
result="$prefix \033[32mOK\033[0m: $hf"
else
result="$prefix \033[31mFAIL\033[0m: $hf"
fi
echo -e "$result"
fi
echo -e "$result"
arr_res+=("$result")

@@ -187,0 +221,0 @@ 

+2
-0
vendor/llama.cpp/tools/server/CMakeLists.txt

@@ -20,2 +20,4 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})

server-tools.h
server-schema.cpp
server-schema.h
)

@@ -22,0 +24,0 @@ 

+29
-0
vendor/llama.cpp/tools/server/README-dev.md

@@ -183,2 +183,31 @@ # llama-server Development Documentation

### Router mode: how child <--> router communicates
Upon spawning a new child process using `subprocess`, both child and router listen to the stdout/stderr (combined)
For the direction from child to router:
- Generic messages are logs, it will be forwarded to router's stdout
- Special state update messages are prefixed by `cmd_child_to_router:state:`, followed by a JSON. See `server_models::handle_child_state` for more
For the direction from router to child:
- When server sends `cmd_router_to_child:exit`, the child should exit gracefully --> if after `DEFAULT_STOP_TIMEOUT` and the child is still running, force-kill it
### Model management API (router mode)
Model management API was added via PR [#23976](https://github.com/ggml-org/llama.cpp/pull/23976)
The main goal of this API is to allow downloading models and/or removing models from the web UI. It relies on the model cache infrastructure under the hood to manage the list of models dynamically.
Instead of building everything from the ground up (like what most AI agents will do when you ask them to implement a similar feature), we built on top of existing, already well-engineered components inside the codebase:
- Model cache infrastructure as mentioned above (`common/download.h`)
- Server response queue (`server-queue.h`). We use this feature to broadcast events to SSE clients.
- Server router thread management (`server-models.h`). We re-use the same thread model that is used for managing subprocess life cycle, except that we don't create a new subprocess, but launch the download right inside the thread.
The flow for downloading a new model:
- POST request comes in --> `post_router_models` --> validation
- `server_models::download()` is called
- Sets up a new thread `inst.th` and runs the download inside
- If a stop request comes in, set `stop_download` to `true`
- Otherwise, upon completion, we call `load_models()` to refresh the list of models
### Notable Related PRs

@@ -185,0 +214,0 @@ 

+25
-11
vendor/llama.cpp/tools/server/server-common.cpp

@@ -15,2 +15,3 @@ #include "common.h"

#include <fstream>
#include <limits>

@@ -1242,3 +1243,3 @@ json format_error_response(const std::string & message, const enum error_type type) {

std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int idx) {
std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int idx, size_t n_top) {
std::vector<llama_token_data> cur;

@@ -1262,17 +1263,30 @@

// sort tokens by logits
std::sort(cur.begin(), cur.end(), [](const llama_token_data & a, const llama_token_data & b) {
return a.logit > b.logit;
});
// sort tokens by logits (partial: only the leading `n_top` need ordering)
if (n_top > cur.size()) {
n_top = cur.size();
}
if (n_top > 0) {
std::partial_sort(cur.begin(), cur.begin() + n_top, cur.end(),
[](const llama_token_data & a, const llama_token_data & b) {
return a.logit > b.logit;
});
}
// apply softmax
float max_l = cur[0].logit;
float max_l = -std::numeric_limits<float>::infinity();
if (n_top > 0) {
max_l = cur[0].logit; // partial_sort guarantees the absolute maximum is at index 0
} else {
for (const auto & t : cur) {
max_l = std::max(max_l, t.logit);
}
}
float cum_sum = 0.0f;
for (size_t i = 0; i < cur.size(); ++i) {
float p = expf(cur[i].logit - max_l);
cur[i].p = p;
for (auto & t : cur) {
float p = expf(t.logit - max_l);
t.p = p;
cum_sum += p;
}
for (size_t i = 0; i < cur.size(); ++i) {
cur[i].p /= cum_sum;
for (auto & t : cur) {
t.p /= cum_sum;
}

@@ -1279,0 +1293,0 @@ 

+1
-1
vendor/llama.cpp/tools/server/server-common.h

@@ -329,3 +329,3 @@ #pragma once

std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int idx);
std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int idx, size_t n_top);

@@ -332,0 +332,0 @@ std::string safe_json_to_str(const json & data);

+28
-5
vendor/llama.cpp/tools/server/server-context.h

@@ -25,4 +25,3 @@ #pragma once

bool has_inp_video;
json json_ui_settings; // Primary: new name
json json_webui_settings; // Deprecated: use json_ui_settings instead (kept for backward compat)
json json_ui_settings;
int slot_n_ctx;

@@ -57,2 +56,27 @@ enum llama_pooling_type pooling_type;

enum server_state {
// SERVER_STATE_DOWNLOADING,
SERVER_STATE_LOADING,
SERVER_STATE_READY,
SERVER_STATE_SLEEPING,
};
static std::string server_state_to_str(server_state state) {
switch (state) {
case SERVER_STATE_LOADING: return "loading";
case SERVER_STATE_READY: return "ready";
case SERVER_STATE_SLEEPING: return "sleeping";
default: GGML_ASSERT(false && "invalid server_state");
}
}
static server_state server_state_from_str(const std::string & str) {
if (str == "loading") return SERVER_STATE_LOADING;
if (str == "ready") return SERVER_STATE_READY;
if (str == "sleeping") return SERVER_STATE_SLEEPING;
GGML_ASSERT(false && "invalid server_state string");
}
using server_state_callback_t = std::function<void(server_state, json /* payload */)>;
struct server_context {

@@ -85,5 +109,4 @@ std::unique_ptr<server_context_impl> impl;

// register a callback to be called when sleeping state changes
// must be set before load_model() is called
void on_sleeping_changed(std::function<void(bool)> callback);
// note: must be set before load_model() is called
void set_state_callback(server_state_callback_t callback);
};

@@ -90,0 +113,0 @@ 

+19
-0
vendor/llama.cpp/tools/server/server-http.cpp

@@ -495,2 +495,4 @@ #include "common.h"

res.status = response->status;
// Tell Nginx to not buffer any streamed response
response->headers["X-Accel-Buffering"] = "no";
set_headers(res, response->headers);

@@ -592,2 +594,19 @@ const std::string content_type = response->content_type;

void server_http_context::del(const std::string & path, const server_http_context::handler_t & handler) const {
handlers.emplace(path, handler);
pimpl->srv->Delete(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) {
server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{
get_params(req),
get_headers(req),
req.path,
build_query_string(req),
req.body,
{},
req.is_connection_closed
});
server_http_res_ptr response = handler(*request);
process_handler_response(std::move(request), response, res);
});
}
//

@@ -594,0 +613,0 @@ // Vertex AI Prediction protocol (AIP_PREDICT_ROUTE)

+1
-0
vendor/llama.cpp/tools/server/server-http.h

@@ -89,2 +89,3 @@ #pragma once

void post(const std::string & path, const handler_t & handler) const;
void del(const std::string & path, const handler_t & handler) const;

@@ -91,0 +92,0 @@ // Register the Google Cloud Platform (Vertex AI) compat (AIP_PREDICT_ROUTE env var, or /predict)

+85
-39
vendor/llama.cpp/tools/server/server-models.h
#pragma once
#include "common.h"
#include "download.h"
#include "preset.h"
#include "server-common.h"
#include "server-http.h"
#include "server-queue.h"

@@ -17,2 +19,4 @@ #include <mutex>

*
* DOWNLOADING ──► DOWNLOADED ──► (replaced by new instance)
*
* UNLOADED ──► LOADING ──► LOADED ◄──── SLEEPING

@@ -26,2 +30,4 @@ * ▲ │ │ ▲

// TODO: also add downloading state when the logic is added
SERVER_MODEL_STATUS_DOWNLOADING,
SERVER_MODEL_STATUS_DOWNLOADED,
SERVER_MODEL_STATUS_UNLOADED,

@@ -33,29 +39,31 @@ SERVER_MODEL_STATUS_LOADING,

static server_model_status server_model_status_from_string(const std::string & status_str) {
if (status_str == "unloaded") {
return SERVER_MODEL_STATUS_UNLOADED;
}
if (status_str == "loading") {
return SERVER_MODEL_STATUS_LOADING;
}
if (status_str == "loaded") {
return SERVER_MODEL_STATUS_LOADED;
}
if (status_str == "sleeping") {
return SERVER_MODEL_STATUS_SLEEPING;
}
throw std::runtime_error("invalid server model status");
}
enum server_model_source {
SERVER_MODEL_SOURCE_PRESET,
SERVER_MODEL_SOURCE_MODELS_DIR,
SERVER_MODEL_SOURCE_CACHE,
};
static std::string server_model_status_to_string(server_model_status status) {
switch (status) {
case SERVER_MODEL_STATUS_UNLOADED: return "unloaded";
case SERVER_MODEL_STATUS_LOADING: return "loading";
case SERVER_MODEL_STATUS_LOADED: return "loaded";
case SERVER_MODEL_STATUS_SLEEPING: return "sleeping";
default: return "unknown";
case SERVER_MODEL_STATUS_DOWNLOADING: return "downloading";
case SERVER_MODEL_STATUS_DOWNLOADED: return "downloaded";
case SERVER_MODEL_STATUS_UNLOADED: return "unloaded";
case SERVER_MODEL_STATUS_LOADING: return "loading";
case SERVER_MODEL_STATUS_LOADED: return "loaded";
case SERVER_MODEL_STATUS_SLEEPING: return "sleeping";
default: return "unknown";
}
}
static std::string server_model_source_to_string(server_model_source source) {
switch (source) {
case SERVER_MODEL_SOURCE_PRESET: return "preset";
case SERVER_MODEL_SOURCE_MODELS_DIR: return "models_dir";
case SERVER_MODEL_SOURCE_CACHE: return "cache";
default: return "unknown";
}
}
struct server_model_meta {
server_model_source source = SERVER_MODEL_SOURCE_CACHE;
common_preset preset;

@@ -69,7 +77,7 @@ std::string name;

std::vector<std::string> args; // args passed to the model instance, will be populated by render_args()
json loaded_info; // info to be reflected via /v1/models endpoint
json loaded_info; // info to be reflected via /v1/models endpoint ; if in DOWNLOADING state, it should contain download progress info
int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED)
int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown
mtmd_caps multimodal; // multimodal capabilities
bool need_download = false; // whether the model needs to be downloaded before loading
// bool need_download = false; // whether the model needs to be downloaded before loading // TODO @ngxson: implement this

@@ -92,8 +100,11 @@ bool is_ready() const {

struct subprocess_s;
struct server_models_routes;
struct server_subproc; // defined in server-models.cpp
struct server_models {
friend struct server_models_routes;
private:
struct instance_t {
std::shared_ptr<subprocess_s> subproc; // shared between main thread and monitoring thread
std::shared_ptr<server_subproc> subproc; // shared between main thread and monitoring thread
std::thread th;

@@ -115,2 +126,5 @@ server_model_meta meta;

// if true, the next get_meta() will trigger a reload of model list
bool need_reload = false;
common_preset_context ctx_preset;

@@ -131,5 +145,10 @@

// notify SSE clients
void notify_sse(const std::string & event, const std::string & model_id, const json & data = nullptr);
public:
server_models(const common_params & params, int argc, char ** argv);
server_response sse; // for real-time updates via SSE endpoint
// (re-)load the list of models from various sources and prepare the metadata mapping

@@ -157,9 +176,24 @@ // - if this is called the first time, simply populate the metadata

// download a new model, progress is reported via SSE
// to stop the download, call unload()
void download(common_params_model && model, common_download_opts && opts);
// update the status of a model instance (thread-safe)
void update_status(const std::string & name, server_model_status status, int exit_code);
void update_loaded_info(const std::string & name, std::string & raw_info);
struct update_status_args {
server_model_status status;
int exit_code = 0; // only valid if status == UNLOADED
json loaded_info = nullptr;
};
void update_status(const std::string & name, const update_status_args & args);
void update_download_progress(const std::string & name, const common_download_progress & progress, bool done, bool ok = true);
// remove a cache model from disk and update the list (thread-safe)
// note: only cache models can be removed; returns false if the model doesn't exist or is not a cache model
bool remove(const std::string & name);
// wait until the model instance is fully loaded (thread-safe)
// note: predicate is called while holding the lock
// return when the model no longer in "loading" state
void wait_until_loading_finished(const std::string & name);
void wait(const std::string & name, std::function<bool(const server_model_meta &)> predicate);
void wait(std::unique_lock<std::mutex> & lk, const std::string & name, std::function<bool(const server_model_meta &)> predicate);

@@ -174,11 +208,23 @@ // ensure the model is in ready state (thread-safe)

// handle message sent from server_child::notify_to_router()
// raw input must starts with CMD_CHILD_TO_ROUTER_STATE, followed by a JSON string
// this function is not thread-safe, must be called from instance's monitoring thread
// payload per state:
// state = loading -> payload = {} (TODO: add progress info)
// state = ready -> payload = model_info (json), or {} if wakeup from sleeping
// state = sleeping -> payload = {}
void handle_child_state(const std::string & name, const std::string & raw_input);
};
struct server_child {
// return true if the current process is a child server instance
static bool is_child_server();
bool is_child();
// notify the router server that a model instance is ready
// register the shutdown_handler to be called by the router
// return the monitoring thread (to be joined by the caller)
static std::thread setup_child_server(const std::function<void(int)> & shutdown_handler, const json & model_info);
std::thread setup(const std::function<void(int)> & shutdown_handler);
// notify the router server that the sleeping state has changed
static void notify_router_sleeping_state(bool sleeping);
// notify router server for status changes (e.g. loading, downloading, sleeping, etc.)
// message will be handled by server_models::handle_child_state() on the router side
void notify_to_router(const std::string & state_name, const json & payload);
};

@@ -188,11 +234,8 @@

common_params params;
json ui_settings = json::object(); // Primary: new name
json webui_settings = json::object(); // Deprecated: use ui_settings (kept for compat)
json ui_settings = json::object(); // Primary: new name
std::atomic<bool> stopping = false; // for graceful disconnecting SSE clients during shutdown
server_models models;
server_models_routes(const common_params & params, int argc, char ** argv)
: params(params), models(params, argc, argv) {
// Support both new ui_config_json and deprecated webui_config_json
const std::string & cfg = !this->params.ui_config_json.empty()
? this->params.ui_config_json
: this->params.webui_config_json;
const std::string & cfg = this->params.ui_config_json;
if (!cfg.empty()) {

@@ -202,3 +245,2 @@ try {

ui_settings = json_settings;
webui_settings = json_settings; // Deprecated: keep in sync
} catch (const std::exception & e) {

@@ -220,2 +262,6 @@ LOG_ERR("%s: failed to parse UI config: %s\n", __func__, e.what());

server_http_context::handler_t post_router_models_unload;
// management API
server_http_context::handler_t get_router_models_sse;
server_http_context::handler_t post_router_models;
server_http_context::handler_t del_router_models;
};

@@ -222,0 +268,0 @@ 

+11
-0
vendor/llama.cpp/tools/server/server-queue.cpp

@@ -334,2 +334,13 @@ #include "server-task.h"

void server_response::broadcast(server_task_result_ptr && result) {
std::unique_lock<std::mutex> lock(mutex_results);
for (const auto & id_task : waiting_task_ids) {
RES_DBG("task id = %d pushed to result queue\n", id_task);
server_task_result_ptr res_copy(result->clone());
res_copy->id = id_task; // override id with target task id
queue_results.emplace_back(std::move(res_copy));
}
condition_results.notify_all();
}
void server_response::terminate() {

@@ -336,0 +347,0 @@ running = false;

+5
-1
vendor/llama.cpp/tools/server/server-queue.h

@@ -157,2 +157,6 @@ #pragma once

// broadcast a new result to all waiting tasks
// (used by router mode)
void broadcast(server_task_result_ptr && result);
// terminate the waiting loop

@@ -162,3 +166,3 @@ void terminate();

// utility class to make working with server_queue and server_response easier
// RAII wrapper to make working with server_queue and server_response easier
// it provides a generator-like API for server responses

@@ -165,0 +169,0 @@ // support pooling connection state and aggregating multiple results

+12
-7
vendor/llama.cpp/tools/server/server-task.h

@@ -213,9 +213,2 @@ #pragma once

static task_params params_from_json_cmpl(
const llama_vocab * vocab,
const common_params & params_base,
const int n_ctx_slot,
const std::vector<llama_logit_bias> & logit_bias_eog,
const json & data);
// utility function

@@ -316,2 +309,5 @@ static std::unordered_set<int> get_list_id(const std::vector<server_task> & tasks) {

virtual ~server_task_result() = default;
virtual server_task_result * clone() const {
GGML_ABORT("not implemented for this task type");
}
};

@@ -654,1 +650,10 @@

};
// used exclusively by router mode
struct server_task_result_router : server_task_result {
json data;
virtual json to_json() override { return data; }
virtual server_task_result * clone() const override {
return new server_task_result_router(*this);
}
};
+32
-14
vendor/llama.cpp/tools/server/server.cpp

@@ -93,4 +93,6 @@ #include "server-context.h"

// router server never loads a model and must not touch the GPU
const bool is_router_server = params.model.path.empty()
&& params.model.hf_repo.empty();
// skip device enumeration so the CUDA primary context stays uncreated
const bool is_router_server = params.model.path.empty();
common_params_print_info(params, !is_router_server);

@@ -117,4 +119,5 @@

// for consistency between server router mode and single-model mode, we set the same model name as alias
if (params.model_alias.empty() && !params.model.name.empty()) {
params.model_alias.insert(params.model.name);
auto model_name = params.model.get_name();
if (params.model_alias.empty() && !model_name.empty()) {
params.model_alias.insert(model_name);
}

@@ -179,4 +182,7 @@

ctx_http.post("/models", ex_wrapper(models_routes->post_router_models));
ctx_http.post("/models/load", ex_wrapper(models_routes->post_router_models_load));
ctx_http.post("/models/unload", ex_wrapper(models_routes->post_router_models_unload));
ctx_http.get ("/models/sse", ex_wrapper(models_routes->get_router_models_sse));
ctx_http.del ("/models", ex_wrapper(models_routes->del_router_models));
}

@@ -230,4 +236,3 @@

// CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)
// Supports both new ui_mcp_proxy and deprecated webui_mcp_proxy fields
if (params.ui_mcp_proxy || params.webui_mcp_proxy) {
if (params.ui_mcp_proxy) {
SRV_WRN("%s", "-----------------\n");

@@ -260,2 +265,3 @@ SRV_WRN("%s", "CORS proxy is enabled, do not expose server to untrusted environments\n");

server_child child; // only used in non-router mode
std::function<void()> clean_up;

@@ -269,2 +275,3 @@

if (models_routes.has_value()) {
models_routes->stopping.store(true); // maybe redundant, but just to be safe
models_routes->models.unload_all();

@@ -283,2 +290,6 @@ }

shutdown_handler = [&](int) {
if (models_routes.has_value()) {
// important to disconnect any SSE clients
models_routes->stopping.store(true);
}
ctx_http.stop();

@@ -303,11 +314,12 @@ };

// setup communication child --> router if necessary
if (child.is_child()) {
ctx_server.set_state_callback([&](server_state state, json payload) {
child.notify_to_router(server_state_to_str(state), payload);
});
}
// load the model
SRV_INF("%s", "loading model\n");
if (server_models::is_child_server()) {
ctx_server.on_sleeping_changed([&](bool sleeping) {
server_models::notify_router_sleeping_state(sleeping);
});
}
if (!ctx_server.load_model(params)) {

@@ -352,2 +364,8 @@ clean_up();

SRV_WRN("%s", " it is not recommended to use this mode in untrusted environments\n");
if (!params.models_preset_hf.empty()) {
SRV_WRN( "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str());
SRV_WRN("%s", " please only use presets that you can trust! Unknown presets may be unsafe\n");
}
if (ctx_http.thread.joinable()) {

@@ -364,5 +382,5 @@ ctx_http.thread.join(); // keep the main thread alive

std::thread monitor_thread;
if (server_models::is_child_server()) {
json model_info = routes.get_model_info();
monitor_thread = server_models::setup_child_server(shutdown_handler, model_info);
if (child.is_child()) {
monitor_thread = child.setup(shutdown_handler);
child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());
}

@@ -369,0 +387,0 @@ 

+4
-4
vendor/llama.cpp/tools/server/tests/unit/test_basic.py

@@ -82,5 +82,5 @@ import pytest

def test_no_webui():
def test_no_ui():
global server
# default: webui enabled
# default: UI enabled
server.start()

@@ -93,4 +93,4 @@ url = f"http://{server.server_host}:{server.server_port}"

# with --no-webui
server.no_webui = True
# with --no-ui, the UI should be disabled
server.no_ui = True
server.start()

@@ -97,0 +97,0 @@ res = requests.get(url)

+14
-0
vendor/llama.cpp/t...rver/tests/unit/test_chat_completion.py

@@ -310,2 +310,16 @@ import pytest

def test_completion_with_invalid_grammar():
global server
server.start()
res = server.make_request("POST", "/chat/completions", data={
"max_tokens": 8,
"messages": [
{"role": "user", "content": "Does not matter what I say, does it?"},
],
"grammar": "root ::= this is (not valid GBNF",
})
assert res.status_code == 400, res.body
assert "error" in res.body
@pytest.mark.parametrize("messages", [

@@ -312,0 +326,0 @@ None,

+3
-3
vendor/llama.cpp/tools/server/tests/unit/test_proxy.py

@@ -15,3 +15,3 @@ import pytest

global server
server.webui_mcp_proxy = False
server.ui_mcp_proxy = False
server.start()

@@ -25,3 +25,3 @@

global server
server.webui_mcp_proxy = True
server.ui_mcp_proxy = True
server.start()

@@ -37,3 +37,3 @@

global server
server.webui_mcp_proxy = True
server.ui_mcp_proxy = True
server.start()

@@ -40,0 +40,0 @@ 

+96
-0
vendor/llama.cpp/tools/server/tests/unit/test_router.py

@@ -0,1 +1,2 @@

import threading
import pytest

@@ -256,1 +257,96 @@ from utils import *

os.remove(preset_path)
MODEL_DOWNLOAD_ID = "ggml-org/test-model-router-download:F16"
MODEL_DOWNLOAD_TIMEOUT = 300
def _listen_sse(server: ServerProcess, collected: list, stop: threading.Event):
"""Collect /models/sse events into `collected` until `stop` is set."""
url = f"http://{server.server_host}:{server.server_port}/models/sse"
try:
with requests.get(url, stream=True, timeout=MODEL_DOWNLOAD_TIMEOUT) as resp:
for line_bytes in resp.iter_lines():
if stop.is_set():
break
line = line_bytes.decode("utf-8")
if line.startswith("data: "):
collected.append(json.loads(line[6:]))
except Exception:
pass
def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: int) -> bool:
deadline = time.time() + timeout
while time.time() < deadline:
if any(e.get("event") == event_type and e.get("model") == model for e in collected):
return True
time.sleep(0.5)
return False
def test_router_download_model():
"""Case 1: download a model, verify SSE events and GET /models."""
global server
server.start()
# Ensure the model is not present before we start
server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
sse_events: list = []
stop = threading.Event()
sse_thread = threading.Thread(
target=_listen_sse, args=(server, sse_events, stop), daemon=True
)
sse_thread.start()
# Trigger the download
res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})
assert res.status_code == 200
assert res.body.get("success") is True
# Wait for download_finished SSE event
finished = _wait_for_sse_event(
sse_events, "download_finished", MODEL_DOWNLOAD_ID, MODEL_DOWNLOAD_TIMEOUT
)
stop.set()
assert finished, "Never received download_finished SSE event"
assert any(
e.get("event") == "download_progress" and e.get("model") == MODEL_DOWNLOAD_ID
for e in sse_events
), "No download_progress events received"
# Model should now appear in GET /models
ids = _get_model_ids(is_reload=False)
assert MODEL_DOWNLOAD_ID in ids, f"{MODEL_DOWNLOAD_ID} not found in /models after download"
def test_router_delete_model():
"""Case 2: delete the downloaded model, verify it disappears from GET /models."""
global server
server.start()
# Ensure the model exists (download it if needed)
if MODEL_DOWNLOAD_ID not in _get_model_ids(is_reload=False):
res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})
assert res.status_code == 200
sse_events: list = []
stop = threading.Event()
threading.Thread(
target=_listen_sse, args=(server, sse_events, stop), daemon=True
).start()
finished = _wait_for_sse_event(
sse_events, "download_finished", MODEL_DOWNLOAD_ID, MODEL_DOWNLOAD_TIMEOUT
)
stop.set()
assert finished, "Model did not finish downloading before delete test"
# Delete the model
del_res = server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
assert del_res.status_code == 200
assert del_res.body.get("success") is True
# Model should no longer appear in GET /models
ids = _get_model_ids(is_reload=False)
assert MODEL_DOWNLOAD_ID not in ids, f"{MODEL_DOWNLOAD_ID} still present after deletion"
+9
-6
vendor/llama.cpp/tools/server/tests/utils.py

@@ -97,3 +97,3 @@ #!/usr/bin/env python3

spec_draft_n_max: int | None = None
no_webui: bool | None = None
no_ui: bool | None = None
jinja: bool | None = None

@@ -111,3 +111,3 @@ reasoning_format: Literal['deepseek', 'none', 'nothink'] | None = None

log_path: str | None = None
webui_mcp_proxy: bool = False
ui_mcp_proxy: bool = False
backend_sampling: bool = False

@@ -230,4 +230,4 @@ gcp_compat: bool = False

server_args.extend(["--spec-draft-n-min", self.spec_draft_n_min])
if self.no_webui:
server_args.append("--no-webui")
if self.no_ui:
server_args.append("--no-ui")
if self.no_models_autoload:

@@ -257,4 +257,4 @@ server_args.append("--no-models-autoload")

server_args.append("--no-cache-idle-slots")
if self.webui_mcp_proxy:
server_args.append("--webui-mcp-proxy")
if self.ui_mcp_proxy:
server_args.append("--ui-mcp-proxy")
if self.backend_sampling:

@@ -347,2 +347,5 @@ server_args.append("--backend_sampling")

parse_body = True
elif method == "DELETE":
response = requests.delete(url, headers=headers, timeout=timeout)
parse_body = True
elif method == "OPTIONS":

@@ -349,0 +352,0 @@ response = requests.options(url, headers=headers, timeout=timeout)

+1
-0
vendor/llama.cpp/tools/ui/package.json

@@ -62,2 +62,3 @@ {

"eslint-plugin-svelte": "3.19.0",
"fflate": "0.8.3",
"globals": "16.5.0",

@@ -64,0 +65,0 @@ "highlight.js": "11.11.1",

+1
-1
vendor/llama.cpp/tools/ui/scripts/git-hooks/pre-commit.sh

@@ -30,3 +30,3 @@ #!/usr/bin/env bash

if [ -n "$staged_ui" ]; then
echo "$staged_ui" | xargs npx --no-install prettier --write
echo "$staged_ui" | xargs npm run format
format_ok=$?

@@ -33,0 +33,0 @@ # Re-stage formatted files

+1
-0
vendor/llama.cpp/tools/ui/scripts/git-hooks/pre-push.sh

@@ -60,2 +60,3 @@ #!/usr/bin/env bash

fi
if [ $test_ok -ne 0 ]; then

@@ -62,0 +63,0 @@ echo "❌ Tests failed"

+42
-6
vendor/llama.cpp/t...nt/MarkdownContent/markdown-content.css

@@ -303,3 +303,4 @@ .markdown-block--unstable {

.markdown-content :global(.copy-code-btn),
.markdown-content :global(.preview-code-btn) {
.markdown-content :global(.preview-code-btn),
.markdown-content :global(.toggle-source-btn) {
display: flex;

@@ -316,3 +317,4 @@ align-items: center;

.markdown-content :global(.copy-code-btn:hover),
.markdown-content :global(.preview-code-btn:hover) {
.markdown-content :global(.preview-code-btn:hover),
.markdown-content :global(.toggle-source-btn:hover) {
transform: scale(1.05);

@@ -322,6 +324,12 @@ }

.markdown-content :global(.copy-code-btn:active),
.markdown-content :global(.preview-code-btn:active) {
.markdown-content :global(.preview-code-btn:active),
.markdown-content :global(.toggle-source-btn:active) {
transform: scale(0.95);
}
/* Pressed state marks the source view as active */
.markdown-content :global(.toggle-source-btn[aria-pressed='true']) {
color: var(--primary);
}
.markdown-content :global(.code-block-wrapper pre) {

@@ -635,4 +643,4 @@ background: transparent;

display: flex;
align-items: center;
justify-content: center;
align-items: safe center;
justify-content: safe center;
padding: 3rem 1rem 1rem;

@@ -652,3 +660,5 @@ }

/* Diagram block uses same header styling as code blocks */
/* Diagram block uses same header styling as code blocks. The header floats over
scrollable diagram content and stays transparent, so the overflow shows up to
the box edge. It keeps a z-index so it stays the click target above content. */
.markdown-content :global(.mermaid-block-wrapper .code-block-header),

@@ -665,2 +675,3 @@ .markdown-content :global(.svg-block-wrapper .code-block-header) {

right: 0;
z-index: 2;
}

@@ -692,2 +703,27 @@

/* Source view stays hidden while the block renders, css swaps the two views
from the wrapper mode so the click handler only flips one attribute. The view
reuses the code block scroll container, so it matches the app code blocks. */
.markdown-content :global(.diagram-source) {
display: none;
text-align: left;
}
.markdown-content :global(.diagram-source pre) {
background: transparent;
margin: 0;
border-radius: 0;
border: none;
font-size: 0.875rem;
}
.markdown-content :global([data-view-mode='source'] .mermaid-scroll-container),
.markdown-content :global([data-view-mode='source'] .svg-scroll-container) {
display: none;
}
.markdown-content :global([data-view-mode='source'] .diagram-source) {
display: block;
}
/* Streaming mermaid block - empty preview box */

@@ -694,0 +730,0 @@ .mermaid-streaming-block {

+28
-1
vendor/llama.cpp/t.../MarkdownContent/MarkdownContent.svelte

@@ -44,2 +44,3 @@ <script lang="ts">

SETTINGS_KEYS,
CODE_BLOCK_HEADER_CLASS,
MERMAID_WRAPPER_CLASS,

@@ -57,3 +58,7 @@ MERMAID_BLOCK_CLASS,

SVG_RENDERED_ATTR,
SVG_INLINE_SHADOW_STYLE
SVG_INLINE_SHADOW_STYLE,
TOGGLE_SOURCE_BTN_CLASS,
DIAGRAM_VIEW_MODE_ATTR,
DIAGRAM_VIEW_RENDERED,
DIAGRAM_VIEW_SOURCE
} from '$lib/constants';

@@ -506,2 +511,19 @@ import { ColorMode, UrlProtocol } from '$lib/enums';

// Toggle a diagram block between its rendered view and its source view.
// Shared by mermaid and svg, css drives the visibility from the wrapper mode.
const toggleBtn = target.closest(`.${TOGGLE_SOURCE_BTN_CLASS}`);
if (toggleBtn) {
event.preventDefault();
event.stopPropagation();
const wrapper = toggleBtn.closest(`.${MERMAID_WRAPPER_CLASS}, .${SVG_WRAPPER_CLASS}`);
if (!wrapper) return;
const isSource = wrapper.getAttribute(DIAGRAM_VIEW_MODE_ATTR) === DIAGRAM_VIEW_SOURCE;
const next = isSource ? DIAGRAM_VIEW_RENDERED : DIAGRAM_VIEW_SOURCE;
wrapper.setAttribute(DIAGRAM_VIEW_MODE_ATTR, next);
toggleBtn.setAttribute('aria-pressed', String(!isSource));
return;
}
// Check if clicking on copy or preview button in mermaid block

@@ -579,2 +601,7 @@ const copyBtn = target.closest(`.${MERMAID_WRAPPER_CLASS} .copy-code-btn`);

// A click on the header chrome targets the action buttons, never the
// diagram. Guard so a header click can not fall through to the click to
// zoom branches below, whatever the scroll position or stacking.
if (target.closest(`.${CODE_BLOCK_HEADER_CLASS}`)) return;
// Open preview when clicking the svg block itself. A final block carries its

@@ -581,0 +608,0 @@ // source, a streaming block does not and is mirrored live into the dialog.

+63
-7
vendor/llama.cpp/t...tent/plugins/rehype/code-block-utils.ts

@@ -10,8 +10,12 @@ /**

CODE_BLOCK_ACTIONS_CLASS,
CODE_BLOCK_SCROLL_CONTAINER_CLASS,
CODE_LANGUAGE_CLASS,
COPY_CODE_BTN_CLASS,
PREVIEW_CODE_BTN_CLASS,
TOGGLE_SOURCE_BTN_CLASS,
DIAGRAM_SOURCE_CLASS,
RELATIVE_CLASS,
COPY_ICON_SVG,
PREVIEW_ICON_SVG
PREVIEW_ICON_SVG,
CODE_ICON_SVG
} from '$lib/constants';

@@ -36,3 +40,4 @@

/**
* Creates a button element with icon.
* Creates a button element with icon. Extra properties merge onto the button,
* which lets a stateful button carry attributes like aria-pressed.
*/

@@ -44,3 +49,4 @@ export function createButton(

id: string,
idAttribute: string
idAttribute: string,
extraProperties: Record<string, string> = {}
): Element {

@@ -54,3 +60,4 @@ return {

title,
type: 'button'
type: 'button',
...extraProperties
},

@@ -80,2 +87,48 @@ children: [createIconElement(iconSvg)]

/**
* Creates a button that toggles a diagram block between its rendered view and
* its source view. aria-pressed starts false, the rendered view is the default.
*/
export function createToggleSourceButton(
id: string,
idAttribute: string,
title: string = 'Toggle source'
): Element {
return createButton(TOGGLE_SOURCE_BTN_CLASS, title, CODE_ICON_SVG, id, idAttribute, {
'aria-pressed': 'false'
});
}
/**
* Creates a source view for a diagram block. It reuses the code block scroll
* container so it matches the app code blocks, and wraps the highlighted code
* element captured at transform time. A missing code element falls back to a
* plain code node built from the raw source.
*/
export function createSourceView(
codeElement: Element | undefined,
source: string,
language: string
): Element {
const code: Element = codeElement ?? {
type: 'element',
tagName: 'code',
properties: { className: ['hljs', `language-${language}`] },
children: [{ type: 'text', value: source }]
};
return {
type: 'element',
tagName: 'div',
properties: { className: [DIAGRAM_SOURCE_CLASS, CODE_BLOCK_SCROLL_CONTAINER_CLASS] },
children: [
{
type: 'element',
tagName: 'pre',
properties: {},
children: [code]
}
]
};
}
/**
* Creates a block header with language label and action buttons.

@@ -124,3 +177,5 @@ */

/**
* Creates a wrapper element with header and scroll container.
* Creates a wrapper element with header and scroll container. Extra children
* append after the scroll container, which lets a block carry a source view
* alongside its rendered output.
*/

@@ -132,3 +187,4 @@ export function createWrapper(

scrollContainerClass: string,
additionalAttributes?: Record<string, string>
additionalAttributes?: Record<string, string>,
extraChildren: Element[] = []
): Element {

@@ -142,3 +198,3 @@ return {

} as Element['properties'],
children: [header, createScrollContainer(preElement, scrollContainerClass)]
children: [header, createScrollContainer(preElement, scrollContainerClass), ...extraChildren]
};

@@ -145,0 +201,0 @@ }

+14
-2
vendor/llama.cpp/t...lugins/rehype/enhance-mermaid-blocks.ts

@@ -22,4 +22,7 @@ /**

MERMAID_SYNTAX_ATTR,
MERMAID_ID_ATTR
MERMAID_ID_ATTR,
DIAGRAM_VIEW_MODE_ATTR,
DIAGRAM_VIEW_RENDERED
} from '$lib/constants';
import type { DiagramPreData } from './pre-transform';
import {

@@ -29,2 +32,4 @@ createBlockHeader,

createPreviewButton,
createToggleSourceButton,
createSourceView,
createWrapper,

@@ -80,2 +85,3 @@ generateBlockId

createCopyButton(mermaidId, MERMAID_ID_ATTR, 'Copy mermaid syntax'),
createToggleSourceButton(mermaidId, MERMAID_ID_ATTR, 'Toggle mermaid source'),
createPreviewButton(mermaidId, MERMAID_ID_ATTR, 'Preview diagram')

@@ -85,2 +91,4 @@ ];

const header = createBlockHeader(MERMAID_LANGUAGE, mermaidId, MERMAID_ID_ATTR, actions);
const preservedCode = (node.data as DiagramPreData | undefined)?.sourceCode;
const sourceView = createSourceView(preservedCode, diagramText, MERMAID_LANGUAGE);
const wrapper = createWrapper(

@@ -91,3 +99,7 @@ header,

MERMAID_SCROLL_CONTAINER_CLASS,
{ [MERMAID_ID_ATTR]: mermaidId }
{
[MERMAID_ID_ATTR]: mermaidId,
[DIAGRAM_VIEW_MODE_ATTR]: DIAGRAM_VIEW_RENDERED
},
[sourceView]
);

@@ -94,0 +106,0 @@ 

+20
-4
vendor/llama.cpp/t...nt/plugins/rehype/enhance-svg-blocks.ts

@@ -21,4 +21,7 @@ /**

SVG_SOURCE_ATTR,
SVG_ID_ATTR
SVG_ID_ATTR,
DIAGRAM_VIEW_MODE_ATTR,
DIAGRAM_VIEW_RENDERED
} from '$lib/constants';
import type { DiagramPreData } from './pre-transform';
import {

@@ -28,2 +31,4 @@ createBlockHeader,

createPreviewButton,
createToggleSourceButton,
createSourceView,
createWrapper,

@@ -70,2 +75,3 @@ generateBlockId

createCopyButton(svgId, SVG_ID_ATTR, 'Copy svg source'),
createToggleSourceButton(svgId, SVG_ID_ATTR, 'Toggle svg source'),
createPreviewButton(svgId, SVG_ID_ATTR, 'Preview svg')

@@ -75,5 +81,15 @@ ];

const header = createBlockHeader(SVG_LANGUAGE, svgId, SVG_ID_ATTR, actions);
const wrapper = createWrapper(header, node, SVG_WRAPPER_CLASS, SVG_SCROLL_CONTAINER_CLASS, {
[SVG_ID_ATTR]: svgId
});
const preservedCode = (node.data as DiagramPreData | undefined)?.sourceCode;
const sourceView = createSourceView(preservedCode, svgSource, SVG_LANGUAGE);
const wrapper = createWrapper(
header,
node,
SVG_WRAPPER_CLASS,
SVG_SCROLL_CONTAINER_CLASS,
{
[SVG_ID_ATTR]: svgId,
[DIAGRAM_VIEW_MODE_ATTR]: DIAGRAM_VIEW_RENDERED
},
[sourceView]
);

@@ -80,0 +96,0 @@ // Replace pre with wrapper in parent

+13
-1
vendor/llama.cpp/t...Content/plugins/rehype/pre-transform.ts

@@ -6,2 +6,11 @@ import type { Plugin } from 'unified';

/**
* Metadata a diagram pre carries on its unist data field. The source code holds
* the highlighted code element captured before the pre became a render target,
* which the enhancer reuses to build a matching source view.
*/
export interface DiagramPreData {
sourceCode: Element;
}
/**
* Recursively extracts all text content from a HAST node.

@@ -73,3 +82,6 @@ * Handles nested elements (e.g., span wrappers from syntax highlighting).

},
children: [{ type: 'text', value: text } as Text]
children: [{ type: 'text', value: text } as Text],
// Keep the highlighted code element so the block can offer a source
// view that matches the app code blocks without re highlighting.
data: { sourceCode: codeElement } satisfies DiagramPreData
};

@@ -76,0 +88,0 @@ 

+27
-17
vendor/llama.cpp/t.../app/models/ModelsSelectorOption.svelte

@@ -82,3 +82,3 @@ <script lang="ts">

<div
class="pointer-events-none flex items-center justify-center gap-0.75 pl-2 opacity-0 group-hover:pointer-events-auto group-hover:opacity-100"
class="pointer-events-none flex items-center justify-center gap-0.75 pl-2 opacity-0 group-hover:pointer-events-auto group-hover:opacity-100 [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100"
onclick={(e) => e.stopPropagation()}

@@ -117,8 +117,12 @@ >

{#if isLoading}
<Loader2 class="h-4 w-4 animate-spin text-muted-foreground" />
<div class="flex w-4 [@media(pointer:coarse)]:w-5 items-center justify-center">
<Loader2 class="h-4 w-4 animate-spin text-muted-foreground" />
</div>
{:else if isFailed}
<div class="flex w-4 items-center justify-center">
<CircleAlert class="h-3.5 w-3.5 text-red-500 group-hover:hidden" />
<div class="flex w-4 [@media(pointer:coarse)]:w-auto items-center justify-center">
<CircleAlert
class="h-3.5 w-3.5 text-red-500 group-hover:hidden [@media(pointer:coarse)]:hidden"
/>
<div class="hidden group-hover:flex">
<div class="hidden group-hover:flex [@media(pointer:coarse)]:flex">
<ActionIcon

@@ -135,6 +139,8 @@ iconSize="h-2.5 w-2.5"

{:else if isSleeping}
<div class="flex w-4 items-center justify-center">
<span class="h-2 w-2 rounded-full bg-orange-400 group-hover:hidden"></span>
<div class="flex w-4 [@media(pointer:coarse)]:w-auto items-center justify-center">
<span
class="h-2 w-2 rounded-full bg-orange-400 group-hover:hidden [@media(pointer:coarse)]:hidden"
></span>
<div class="hidden group-hover:flex">
<div class="hidden group-hover:flex [@media(pointer:coarse)]:flex">
<ActionIcon

@@ -144,3 +150,3 @@ iconSize="h-2.5 w-2.5"

tooltip="Unload model"
class="h-3 w-3 text-red-500 hover:text-red-600"
class="h-3 w-3 text-red-500 hover:text-red-600 [@media(pointer:coarse)]:text-amber-500 [@media(pointer:coarse)]:hover:text-amber-600"
onclick={(e) => {

@@ -154,6 +160,8 @@ e?.stopPropagation();

{:else if isLoaded}
<div class="flex w-4 items-center justify-center">
<span class="h-2 w-2 rounded-full bg-green-500 group-hover:hidden"></span>
<div class="flex w-4 [@media(pointer:coarse)]:w-auto items-center justify-center">
<span
class="h-2 w-2 rounded-full bg-green-500 group-hover:hidden [@media(pointer:coarse)]:hidden"
></span>
<div class="hidden group-hover:flex">
<div class="hidden group-hover:flex [@media(pointer:coarse)]:flex">
<ActionIcon

@@ -163,3 +171,3 @@ iconSize="h-2.5 w-2.5"

tooltip="Unload model"
class="h-3 w-3 text-red-500 hover:text-red-600"
class="h-3 w-3 text-red-500 hover:text-red-600 [@media(pointer:coarse)]:text-green-500 [@media(pointer:coarse)]:hover:text-green-600"
onclick={() => modelsStore.unloadModel(option.model)}

@@ -171,6 +179,8 @@ stopPropagationOnClick

{:else}
<div class="flex w-4 items-center justify-center">
<span class="h-2 w-2 rounded-full bg-muted-foreground/50 group-hover:hidden"></span>
<div class="flex w-4 [@media(pointer:coarse)]:w-auto items-center justify-center">
<span
class="h-2 w-2 rounded-full bg-muted-foreground/50 group-hover:hidden [@media(pointer:coarse)]:hidden"
></span>
<div class="hidden group-hover:flex">
<div class="hidden group-hover:flex [@media(pointer:coarse)]:flex">
<ActionIcon

@@ -180,3 +190,3 @@ iconSize="h-2.5 w-2.5"

tooltip="Load model"
class="h-3 w-3"
class="h-3 w-3 [@media(pointer:coarse)]:text-muted-foreground"
onclick={() => modelsStore.loadModel(option.model)}

@@ -183,0 +193,0 @@ stopPropagationOnClick

+1
-1
vendor/llama.cpp/t...s/app/models/ModelsSelectorSheet.svelte

@@ -69,3 +69,3 @@ <script lang="ts">

class={[
`inline-flex cursor-pointer items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
`inline-flex cursor-pointer items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 max-sm:px-3 max-sm:py-2 text-xs max-sm:text-sm shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
!ms.isCurrentModelInCache

@@ -72,0 +72,0 @@ ? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400'

+13
-25
vendor/llama.cpp/t...Chat/SettingsChatImportExportTab.svelte

@@ -135,3 +135,3 @@ <script lang="ts">

try {
const allData: ExportedConversations = await Promise.all(
const allData: ExportedConversation[] = await Promise.all(
selectedConversations.map(async (conv) => {

@@ -143,3 +143,7 @@ const messages = await conversationsStore.getConversationMessages(conv.id);

conversationsStore.downloadConversationFile(allData);
if (allData.length === 1) {
conversationsStore.downloadConversationFile(allData[0]);
} else {
conversationsStore.downloadConversationsArchive(allData);
}

@@ -161,3 +165,3 @@ exportedConversations = selectedConversations;

input.type = HtmlInputType.FILE;
input.accept = FileExtensionText.JSON;
input.accept = `${FileExtensionText.JSON},${FileExtensionText.JSONL},${FileExtensionText.ZIP}`;

@@ -169,26 +173,10 @@ input.onchange = async (e) => {

try {
const text = await file.text();
const parsedData = JSON.parse(text);
let importedData: ExportedConversations;
const importedData = await conversationsStore.parseImportFile(file);
if (Array.isArray(parsedData)) {
importedData = parsedData;
} else if (
parsedData &&
typeof parsedData === 'object' &&
'conv' in parsedData &&
'messages' in parsedData
) {
// Single conversation object
importedData = [parsedData];
} else {
throw new Error(
'Invalid file format: expected array of conversations or single conversation object'
);
if (importedData.length === 0) {
throw new Error('No conversations found in file');
}
fullImportData = importedData;
availableConversations = importedData.map(
(item: { conv: DatabaseConversation; messages: DatabaseMessage[] }) => item.conv
);
availableConversations = importedData.map((item) => item.conv);
messageCountMap = createMessageCountMap(importedData);

@@ -265,3 +253,3 @@ showImportDialog = true;

title="Export"
description="Download your conversations as a JSON file. This includes all messages, attachments, and conversation history."
description="Download your conversations as a ZIP of JSONL files. This includes all messages, attachments, and conversation history."
IconComponent={Download}

@@ -275,3 +263,3 @@ buttonText="Export conversations"

title="Import"
description="Import one or more conversations from a previously exported JSON file. This will merge with your existing conversations."
description="Import one or more conversations from a previously exported ZIP or JSONL file. This will merge with your existing conversations."
IconComponent={Upload}

@@ -278,0 +266,0 @@ buttonText="Import conversations"

+2
-0
vendor/llama.cpp/tools/ui/src/lib/constants/icons.ts

@@ -42,1 +42,3 @@ /**

export const PREVIEW_ICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-eye lucide-eye-icon"><path d="M2.062 12.345a1 1 0 0 1 0-.69C3.5 7.73 7.36 5 12 5s8.5 2.73 9.938 6.655a1 1 0 0 1 0 .69C20.5 16.27 16.64 19 12 19s-8.5-2.73-9.938-6.655"/><circle cx="12" cy="12" r="3"/></svg>`;
export const CODE_ICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-code lucide-code-icon"><path d="m16 18 6-6-6-6"/><path d="m8 6-6 6 6 6"/></svg>`;
+1
-0
vendor/llama.cpp/tools/ui/src/lib/constants/index.ts

@@ -33,2 +33,3 @@ // Central constants export file

export * from './svg-blocks';
export * from './diagram-blocks';
export * from './max-bundle-size';

@@ -35,0 +36,0 @@ export * from './mcp';

+5
-0
vendor/llama.cpp/tools/ui/src/lib/constants/mcp.ts

@@ -87,1 +87,6 @@ import { Zap, Globe, Radio } from '@lucide/svelte';

};
/** Standard SSE endpoint path indicators */
export const MCP_SSE_ENDPOINT = '/sse';
export const MCP_SSE_ENDPOINT_SLASH = '/sse/';
export const MCP_SSE_ENDPOINT_QUERY = '/sse?';
+5
-1
vendor/llama.cpp/tools/ui/src/lib/enums/files.enums.ts

@@ -126,2 +126,4 @@ /**

JSON = '.json',
JSONL = '.jsonl',
ZIP = '.zip',
XML = '.xml',

@@ -183,3 +185,4 @@ YAML = '.yaml',

PDF = 'application/pdf',
OCTET_STREAM = 'application/octet-stream'
OCTET_STREAM = 'application/octet-stream',
ZIP = 'application/zip'
}

@@ -231,2 +234,3 @@

JSON = 'application/json',
JSONL = 'application/jsonl',
XML_TEXT = 'text/xml',

@@ -233,0 +237,0 @@ XML_APP = 'application/xml',

+58
-1
vendor/llama.cpp/tools/ui/src/lib/services/mcp.service.ts

@@ -19,3 +19,4 @@ import { Client } from '@modelcontextprotocol/sdk/client';

DEFAULT_IMAGE_MIME_TYPE,
MCP_PARTIAL_REDACT_HEADERS
MCP_PARTIAL_REDACT_HEADERS,
CORS_PROXY_ENDPOINT
} from '$lib/constants';

@@ -240,2 +241,32 @@ import {

fetch: async (input, init) => {
if (useProxy && typeof window !== 'undefined') {
let requestUrlStr = '';
if (typeof input === 'string') {
requestUrlStr = input;
} else if (input instanceof URL) {
requestUrlStr = input.href;
}
if (requestUrlStr) {
const parsedRequestUrl = new URL(requestUrlStr, window.location.origin);
if (
parsedRequestUrl.origin === window.location.origin &&
!parsedRequestUrl.pathname.includes(CORS_PROXY_ENDPOINT)
) {
const originalConfigUrl = new URL(config.url);
const realTargetUrl = new URL(
parsedRequestUrl.pathname + parsedRequestUrl.search,
originalConfigUrl.origin
);
const proxiedUrl = buildProxiedUrl(realTargetUrl.href);
if (typeof input === 'string') {
input = proxiedUrl.href;
} else if (input instanceof URL) {
input = proxiedUrl;
}
}
}
}
const startedAt = performance.now();

@@ -408,2 +439,28 @@ const requestHeaders = new Headers(baseInit.headers);

if (config.transport === MCPTransportType.SSE) {
const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url);
const { fetch: diagnosticFetch, disable: stopPhaseLogging } = this.createDiagnosticFetch(
serverName,
config,
requestInit,
url,
useProxy,
onLog
);
if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) {
console.log(`[MCPService] Creating SSE transport for ${url.href}`);
}
return {
transport: new SSEClientTransport(url, {
requestInit,
fetch: diagnosticFetch,
eventSourceInit: { fetch: diagnosticFetch }
}),
type: MCPTransportType.SSE,
stopPhaseLogging
};
}
const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url);

@@ -410,0 +467,0 @@ const { fetch: diagnosticFetch, disable: stopPhaseLogging } = this.createDiagnosticFetch(

+162
-18
vendor/llama.cpp/t.../src/lib/stores/conversations.svelte.ts

@@ -29,4 +29,12 @@ /**

import type { McpServerOverride } from '$lib/types/database';
import { MessageRole, HtmlInputType, FileExtensionText, ReasoningEffort } from '$lib/enums';
import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate';
import {
MessageRole,
HtmlInputType,
FileExtensionText,
MimeTypeText,
MimeTypeApplication,
ReasoningEffort
} from '$lib/enums';
import {
ISO_DATE_TIME_SEPARATOR,

@@ -938,16 +946,114 @@ ISO_DATE_TIME_SEPARATOR_REPLACEMENT,

const trimmedConvId = conversation.id?.slice(0, EXPORT_CONV_ID_TRIM_LENGTH) ?? '';
return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}.json`;
return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}${FileExtensionText.JSONL}`;
}
/**
* Serializes a session (a conversation with its messages) as JSONL.
* The first line is the session header (a `type: 'session'` record carrying the
* conversation properties); each subsequent line is a single message.
* @param data - The exported conversation payload
* @returns The JSONL string (one record per line)
*/
serializeSessionToJsonl(data: ExportedConversation): string {
const { conv, messages } = data;
const sessionLine = JSON.stringify({ type: 'session', harness: 'llama.app', ...conv });
const messageLines = messages.map((message: DatabaseMessage) => {
// `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it.
const { toolCalls, ...rest } = message;
const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest;
return JSON.stringify({ type: 'message', message: normalized });
});
return [sessionLine, ...messageLines].join('\n');
}
/**
* Parses the JSONL session format produced by {@link serializeSessionToJsonl}.
* A `type: 'session'` line starts a new session; following `type: 'message'`
* lines are appended to it. Supports multiple sessions in a single file.
* @param text - The JSONL file contents
* @returns The parsed conversations with their messages
*/
parseSessionsJsonl(text: string): ExportedConversation[] {
const sessions: ExportedConversation[] = [];
let current: ExportedConversation | null = null;
for (const line of text.split('\n')) {
const trimmed = line.trim();
if (!trimmed) continue;
const record = JSON.parse(trimmed);
if (record.type === 'session') {
// Drop the discriminator and harness marker; the rest is the conversation.
const conv = { ...record };
delete conv.type;
delete conv.harness;
current = { conv: conv as DatabaseConversation, messages: [] };
sessions.push(current);
} else if (record.type === 'message') {
if (!current) {
throw new Error('Invalid JSONL: message record before any session record');
}
const message = record.message as DatabaseMessage;
// `toolCalls` is parsed to an array on export; the DB stores it as a string.
if (message.toolCalls !== undefined && typeof message.toolCalls !== 'string') {
message.toolCalls = JSON.stringify(message.toolCalls);
}
current.messages.push(message);
}
// Ignore unknown record types for forward compatibility.
}
return sessions;
}
/**
* Parses an import file into conversations, accepting the current `.jsonl` and
* `.zip` formats as well as the legacy `.json` format.
* @param file - The user-selected file
* @returns The parsed conversations with their messages
*/
async parseImportFile(file: File): Promise<ExportedConversation[]> {
const name = file.name.toLowerCase();
if (name.endsWith(FileExtensionText.ZIP)) {
const entries = unzipSync(new Uint8Array(await file.arrayBuffer()));
const sessions: ExportedConversation[] = [];
for (const [entryName, bytes] of Object.entries(entries)) {
if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue;
sessions.push(...this.parseSessionsJsonl(strFromU8(bytes)));
}
return sessions;
}
const text = await file.text();
if (name.endsWith(FileExtensionText.JSONL)) {
return this.parseSessionsJsonl(text);
}
// Legacy JSON format: an array of conversations or a single conversation object.
const parsed = JSON.parse(text);
if (Array.isArray(parsed)) {
return parsed;
}
if (parsed && typeof parsed === 'object' && 'conv' in parsed && 'messages' in parsed) {
return [parsed];
}
throw new Error(
'Invalid file format: expected array of conversations or single conversation object'
);
}
/**
* Triggers a browser download of the provided exported conversation data
* @param data - The exported conversation payload (either a single conversation or array of them)
* @param data - The exported conversation payload (a single conversation with its messages)
* @param filename - Filename; if omitted, a deterministic name is generated
*/
downloadConversationFile(data: ExportedConversations, filename?: string): void {
// Choose the first conversation or message
const conversation =
'conv' in data ? data.conv : Array.isArray(data) ? data[0]?.conv : undefined;
const msgs =
'messages' in data ? data.messages : Array.isArray(data) ? data[0]?.messages : undefined;
downloadConversationFile(data: ExportedConversation, filename?: string): void {
const { conv: conversation, messages: msgs } = data;

@@ -959,17 +1065,55 @@ if (!conversation) {

let downloadFilename: string;
const downloadFilename = filename ?? this.generateConversationFilename(conversation, msgs);
if (filename) {
downloadFilename = filename;
} else if (Array.isArray(data) && data.length > 1) {
downloadFilename = `${new Date().toISOString().split(ISO_DATE_TIME_SEPARATOR)[0]}_conversations.json`;
} else {
downloadFilename = this.generateConversationFilename(conversation, msgs);
const jsonl = this.serializeSessionToJsonl(data);
const blob = new Blob([jsonl], { type: MimeTypeText.JSONL });
this.triggerDownload(blob, downloadFilename);
}
/**
* Triggers a browser download of multiple conversations as a `.zip`, one
* `.jsonl` file per conversation.
* @param data - The conversations to export
*/
downloadConversationsArchive(data: ExportedConversation[]): void {
if (data.length === 0) {
console.error('Invalid data: no conversations to export');
return;
}
const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
const usedNames = new SvelteSet<string>();
const files: Record<string, Uint8Array> = {};
for (const session of data) {
const baseName = this.generateConversationFilename(session.conv, session.messages);
// Disambiguate any duplicate filenames within the archive.
let entryName = baseName;
let suffix = 1;
while (usedNames.has(entryName)) {
entryName = baseName.replace(
new RegExp(`${FileExtensionText.JSONL}$`),
`_${suffix++}${FileExtensionText.JSONL}`
);
}
usedNames.add(entryName);
files[entryName] = strToU8(this.serializeSessionToJsonl(session));
}
const archiveName = `${new Date().toISOString().split(ISO_DATE_TIME_SEPARATOR)[0]}_conversations${FileExtensionText.ZIP}`;
const zipped = zipSync(files);
const blob = new Blob([zipped], { type: MimeTypeApplication.ZIP });
this.triggerDownload(blob, archiveName);
}
/**
* Triggers a browser download of a blob under the given filename.
*/
private triggerDownload(blob: Blob, filename: string): void {
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = downloadFilename;
a.download = filename;
document.body.appendChild(a);

@@ -976,0 +1120,0 @@ a.click();

+19
-4
vendor/llama.cpp/tools/ui/src/lib/utils/mcp.ts

@@ -22,3 +22,6 @@ import type { MCPServerSettingsEntry, MCPResourceContent, MCPResourceInfo } from '$lib/types';

RESOURCE_TEXT_CONTENT_SEPARATOR,
DEFAULT_RESOURCE_FILENAME
DEFAULT_RESOURCE_FILENAME,
MCP_SSE_ENDPOINT,
MCP_SSE_ENDPOINT_SLASH,
MCP_SSE_ENDPOINT_QUERY
} from '$lib/constants';

@@ -45,6 +48,18 @@ import {

return normalized.startsWith(UrlProtocol.WEBSOCKET) ||
if (
normalized.startsWith(UrlProtocol.WEBSOCKET) ||
normalized.startsWith(UrlProtocol.WEBSOCKET_SECURE)
? MCPTransportType.WEBSOCKET
: MCPTransportType.STREAMABLE_HTTP;
) {
return MCPTransportType.WEBSOCKET;
}
if (
normalized.endsWith(MCP_SSE_ENDPOINT) ||
normalized.endsWith(MCP_SSE_ENDPOINT_SLASH) ||
normalized.includes(MCP_SSE_ENDPOINT_QUERY)
) {
return MCPTransportType.SSE;
}
return MCPTransportType.STREAMABLE_HTTP;
}

@@ -51,0 +66,0 @@ 

+1
-1
vendor/llama.cpp/vendor/cpp-httplib/CMakeLists.txt

@@ -44,3 +44,3 @@ set(TARGET cpp-httplib)

set(BORINGSSL_GIT "https://boringssl.googlesource.com/boringssl" CACHE STRING "BoringSSL git repository")
set(BORINGSSL_VERSION "0.20260526.0" CACHE STRING "BoringSSL version")
set(BORINGSSL_VERSION "0.20260616.0" CACHE STRING "BoringSSL version")

@@ -47,0 +47,0 @@ message(STATUS "Fetching BoringSSL version ${BORINGSSL_VERSION}")

.git/modules/vendor/llama.cpp/FETCH_HEAD

Sorry, the diff of this file is not supported yet

.git/modules/vendo.../9f9ce78aae0962f3a45341863824d082b78cf8

Sorry, the diff of this file is not supported yet

.git/modules/vendo.../a5e3015b59c5792195803c771e6e572a891d77

Sorry, the diff of this file is not supported yet

.git/modules/vendo.../a686951b4a73250bd2f745576cca26c7027c8b

Sorry, the diff of this file is not supported yet

.git/modules/vendo.../7ebf0c9caa5a6105988faebcdd588319dc12b3

Sorry, the diff of this file is not supported yet

.git/modules/vendo.../0ac7693f6efe5b923820d48978263d1caff18e

Sorry, the diff of this file is not supported yet

.git/modules/vendo.../6f2b55024aef84c7fd09860893a35411ad58d2

Sorry, the diff of this file is not supported yet

.git/modules/vendo.../ae39a109667e0d7b3de76d099ef2a988699710

Sorry, the diff of this file is not supported yet

.git/modules/vendo.../f29d85e32707c045f6146f544ae08306d483db

Sorry, the diff of this file is not supported yet

.git/modules/vendo.../74315559b75e84b65717ae993192615ab73e1e

Sorry, the diff of this file is not supported yet

.git/modules/vendo.../546102cd88478f8393c8f09567c3ba38533a8f

Sorry, the diff of this file is not supported yet

.git/modules/vendo.../a74b299085cd00013804f7fca2e03441b2da20

Sorry, the diff of this file is not supported yet

.git/modules/vendo...1f8b17e0ab85cc2f4122557f565a62e9529.idx

Sorry, the diff of this file is not supported yet

.git/modules/vendo...f8b17e0ab85cc2f4122557f565a62e9529.pack

Sorry, the diff of this file is not supported yet

.git/modules/vendo...1f8b17e0ab85cc2f4122557f565a62e9529.rev

Sorry, the diff of this file is not supported yet

.git/objects/pack/...5a575c80cf36f6e50c680b76866ea73f243.idx

Sorry, the diff of this file is not supported yet

.git/objects/pack/...a575c80cf36f6e50c680b76866ea73f243.pack

Sorry, the diff of this file is not supported yet

.git/objects/pack/...5a575c80cf36f6e50c680b76866ea73f243.rev

Sorry, the diff of this file is not supported yet

.git/refs/tags/v0.3.30

Sorry, the diff of this file is not supported yet

-34
vendor/llama.cpp/ggml/src/ggml-hexagon/htp/hmx-profile.h
// Conditional fine-grained profiling macros for HMX operations.
//
// Define ENABLE_PROFILE_TIMERS (via compiler flag or before including this
// header) to instrument sub-operation latencies with HAP qtimer. When the
// macro is not defined the TIMER_* helpers expand to nothing so there is zero
// overhead.
//
// Usage:
// TIMER_DEFINE(my_phase); // declare accumulator variable
// TIMER_START(my_phase); // snapshot start time
// ... work ...
// TIMER_STOP(my_phase); // accumulate elapsed ticks
// FARF(ALWAYS, "my_phase: %lld us", TIMER_US(my_phase));
#ifndef HMX_PROFILE_H
#define HMX_PROFILE_H
#include <HAP_perf.h>
// #define ENABLE_PROFILE_TIMERS
#if defined(ENABLE_PROFILE_TIMERS)
# define TIMER_DEFINE(name) int64_t name##_ticks = 0
# define TIMER_START(name) int64_t name##_t0 = HAP_perf_get_qtimer_count()
# define TIMER_STOP(name) name##_ticks += HAP_perf_get_qtimer_count() - name##_t0
# define TIMER_US(name) HAP_perf_qtimer_count_to_us(name##_ticks)
#else
# define TIMER_DEFINE(name)
# define TIMER_START(name)
# define TIMER_STOP(name)
# define TIMER_US(name) 0LL
#endif
#endif // HMX_PROFILE_H
-25
vendor/llama.cpp/g...gml-openvino/openvino/op/unary_gelu.cpp
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
#include <openvino/core/node_output.hpp>
#include <openvino/op/gelu.hpp>
namespace ov {
namespace frontend {
namespace ggml {
namespace op {
OutputVector translate_unary_gelu(const NodeContext & context) {
num_inputs_check(context, 1, 1);
auto input = context.get_input(0);
auto res = std::make_shared<ov::op::v7::Gelu>(input);
return rename_outputs_with_suffix({res}, context.get_name());
}
} // namespace op
} // namespace ggml
} // namespace frontend
} // namespace ov
.git/config

Sorry, the diff of this file is not supported yet

.git/FETCH_HEAD

Sorry, the diff of this file is not supported yet

.git/HEAD

Sorry, the diff of this file is not supported yet

.git/index

Sorry, the diff of this file is not supported yet

.git/logs/HEAD

Sorry, the diff of this file is not supported yet

.git/modules/vendor/llama.cpp/config

Sorry, the diff of this file is not supported yet

.git/modules/vendor/llama.cpp/HEAD

Sorry, the diff of this file is not supported yet

.git/modules/vendor/llama.cpp/index

Sorry, the diff of this file is not supported yet

.git/modules/vendor/llama.cpp/logs/HEAD

Sorry, the diff of this file is not supported yet

.git/modules/vendor/llama.cpp/logs/refs/heads/master

Sorry, the diff of this file is not supported yet

.git/modules/vendor/llama.cpp/logs/refs/remotes/origin/HEAD

Sorry, the diff of this file is not supported yet

.git/modules/vendor/llama.cpp/packed-refs

Sorry, the diff of this file is not supported yet

.git/modules/vendor/llama.cpp/refs/heads/master

Sorry, the diff of this file is not supported yet

.git/modules/vendor/llama.cpp/shallow

Sorry, the diff of this file is not supported yet

.git/shallow

Sorry, the diff of this file is not supported yet

vendor/llama.cpp/common/arg.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/common/common.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/common/speculative.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/docs/ops/SYCL.csv

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-cpu/llamafile/sgemm.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu

Sorry, the diff of this file is not supported yet

vendor/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/g...c/ggml-hexagon/htp/hmx-flash-attn-ops.c

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-hexagon/htp/matmul-ops.c

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-metal/ggml-metal-device.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-metal/ggml-metal-device.m

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-metal/ggml-metal-ops.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-metal/ggml-metal.metal

Sorry, the diff of this file is not supported yet

vendor/llama.cpp/ggml/src/ggml-opencl/ggml-opencl.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/g...gml-opencl/kernels/mul_mv_f16_f32_l4.cl

Sorry, the diff of this file is not supported yet

vendor/llama.cpp/ggml/src/ggml-openvino/.clang-format

Sorry, the diff of this file is not supported yet

vendor/llama.cpp/ggml/src/ggml-openvino/ggml-decoder.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-sycl/dmmv.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-sycl/dpct/helper.hpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-sycl/ggml-sycl.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-sycl/mmvq.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/g...n/vulkan-shaders/vulkan-shaders-gen.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml-webgpu/ggml-webgpu.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/ggml/src/ggml.c

Sorry, the diff of this file is too big to display

vendor/llama.cpp/scripts/sync-ggml.last

Sorry, the diff of this file is not supported yet

vendor/llama.cpp/src/llama-context.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/src/llama-graph.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/tools/mtmd/clip.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/tools/mtmd/mtmd-image.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/tools/mtmd/mtmd.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/tools/server/README.md

Sorry, the diff of this file is too big to display

vendor/llama.cpp/tools/server/server-context.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/tools/server/server-models.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/tools/server/server-task.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/tools/ui/package-lock.json

Sorry, the diff of this file is too big to display

vendor/llama.cpp/vendor/cpp-httplib/httplib.cpp

Sorry, the diff of this file is too big to display

vendor/llama.cpp/vendor/cpp-httplib/httplib.h

Sorry, the diff of this file is too big to display