Skip to content

feat(vulkan): Add native Vulkan support for TurboQuant KV Cache (turbo3/turbo4) #158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Yvi71 wants to merge 4 commits into from
Closed

feat(vulkan): Add native Vulkan support for TurboQuant KV Cache (turbo3/turbo4) #158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0b05974
hip: bypass memory pool for flash attention f16 temp buffers
TheTom Apr 18, 2026
57f6b93
Merge pull request #92 from TheTom/fix/hip-fa-pool-retention
TheTom Apr 20, 2026
bd0d153
Merge origin/master into feature/turboquant-kv-cache
May 27, 2026
19b1864
feat(vulkan): add native Vulkan support for TurboQuant KV Cache (turb…
May 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions ggml/src/ggml-backend.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ size_t ggml_backend_get_max_size(ggml_backend_t backend) {
void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
GGML_ASSERT(backend);
GGML_ASSERT(tensor);
GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
GGML_ASSERT((tensor->data != NULL || (tensor->view_src != NULL && tensor->view_src->data != NULL)) && "tensor not allocated");
GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");

if (backend->iface.set_tensor_async == NULL) {
Expand All @@ -268,7 +268,7 @@ void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor *
void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
GGML_ASSERT(backend);
GGML_ASSERT(tensor);
GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
GGML_ASSERT((tensor->data != NULL || (tensor->view_src != NULL && tensor->view_src->data != NULL)) && "tensor not allocated");
GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");

if (backend->iface.get_tensor_async == NULL) {
Expand All @@ -283,7 +283,7 @@ void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor
size_t n_copies, size_t stride_tensor, size_t stride_data) {
GGML_ASSERT(backend);
GGML_ASSERT(tensor);
GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
GGML_ASSERT((tensor->data != NULL || (tensor->view_src != NULL && tensor->view_src->data != NULL)) && "tensor not allocated");

if (n_copies <= 1 || backend->iface.set_tensor_2d_async == NULL) {
for (size_t i = 0; i < n_copies; i++) {
Expand All @@ -295,7 +295,7 @@ void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor
return;
}

GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
GGML_ASSERT((tensor->data != NULL || (tensor->view_src != NULL && tensor->view_src->data != NULL)) && "tensor not allocated");
GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor write out of bounds");
backend->iface.set_tensor_2d_async(backend, tensor, data, offset, size, n_copies, stride_tensor, stride_data);
}
Expand All @@ -304,7 +304,7 @@ void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_
size_t n_copies, size_t stride_tensor, size_t stride_data) {
GGML_ASSERT(backend);
GGML_ASSERT(tensor);
GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
GGML_ASSERT((tensor->data != NULL || (tensor->view_src != NULL && tensor->view_src->data != NULL)) && "tensor not allocated");

if (n_copies <= 1 || backend->iface.set_tensor_2d_async == NULL) {
for (size_t i = 0; i < n_copies; i++) {
Expand All @@ -316,7 +316,7 @@ void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_
return;
}

GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
GGML_ASSERT((tensor->data != NULL || (tensor->view_src != NULL && tensor->view_src->data != NULL)) && "tensor not allocated");
GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor write out of bounds");
backend->iface.get_tensor_2d_async(backend, tensor, data, offset, size, n_copies, stride_tensor, stride_data);
}
Expand All @@ -330,7 +330,7 @@ void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, siz
return;
}

GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
GGML_ASSERT((tensor->data != NULL || (tensor->view_src != NULL && tensor->view_src->data != NULL)) && "tensor not allocated");
GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");

buf->iface.set_tensor(buf, tensor, data, offset, size);
Expand All @@ -345,7 +345,7 @@ void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, siz
return;
}

GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
GGML_ASSERT((tensor->data != NULL || (tensor->view_src != NULL && tensor->view_src->data != NULL)) && "tensor not allocated");
GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");

buf->iface.get_tensor(buf, tensor, data, offset, size);
Expand All @@ -367,7 +367,7 @@ void ggml_backend_tensor_set_2d(struct ggml_tensor * tensor, const void * data,
return;
}

GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
GGML_ASSERT((tensor->data != NULL || (tensor->view_src != NULL && tensor->view_src->data != NULL)) && "tensor not allocated");
GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor write out of bounds");

buf->iface.set_tensor_2d(buf, tensor, data, offset, size, n_copies, stride_tensor, stride_data);
Expand All @@ -389,7 +389,7 @@ void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data,
return;
}

GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
GGML_ASSERT((tensor->data != NULL || (tensor->view_src != NULL && tensor->view_src->data != NULL)) && "tensor not allocated");
GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor read out of bounds");

buf->iface.get_tensor_2d(buf, tensor, data, offset, size, n_copies, stride_tensor, stride_data);
Expand All @@ -404,7 +404,7 @@ void ggml_backend_tensor_memset(struct ggml_tensor * tensor, uint8_t value, size
}

GGML_ASSERT(buf != NULL && "tensor buffer not set");
GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
GGML_ASSERT((tensor->data != NULL || (tensor->view_src != NULL && tensor->view_src->data != NULL)) && "tensor not allocated");
GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");
GGML_ASSERT(buf->iface.memset_tensor != NULL && "memset not implemented by backend buffer");

Expand Down
13 changes: 9 additions & 4 deletions ggml/src/ggml-cuda/fattn-common.cuh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1299,14 +1299,19 @@ void launch_fattn(

#ifdef GGML_USE_HIP
// HIP/ROCm: bypass the memory pool for f16 temp buffers.
// The legacy pool (ggml_cuda_pool_leg) retains peak-sized allocations permanently.
// For quantized KV dequant, this means the f16 temp buffer stays allocated,
// consuming more VRAM than the quantized KV compression saves — causing OOM.
// Using raw alloc+free ensures the memory is released after the kernel completes.
// The legacy pool (ggml_cuda_pool_leg) retains peak-sized allocations permanently
// because free() stores buffers for reuse rather than releasing them.
// On HIP without VMM support (RDNA 3/4), this means the f16 dequant temp buffers
// for quantized KV stay allocated after use, consuming more VRAM than the KV
// compression saves — causing OOM before f16 at equivalent context lengths.
// Using raw cudaMalloc/cudaFree ensures memory is released after the kernel completes.
// Ref: https://github.com/ggml-org/llama.cpp/issues/22107
struct hip_f16_alloc {
half * ptr = nullptr;
cudaStream_t stream;
hip_f16_alloc(cudaStream_t s) : stream(s) {}
hip_f16_alloc(const hip_f16_alloc &) = delete;
hip_f16_alloc & operator=(const hip_f16_alloc &) = delete;
~hip_f16_alloc() {
if (ptr) {
// Cast to void: hipStreamSynchronize / hipFree are [[nodiscard]] under
Expand Down
8 changes: 8 additions & 0 deletions ggml/src/ggml-vulkan/ggml-vulkan.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4600,6 +4600,8 @@ static void ggml_vk_load_shaders(vk_device& device) {
ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_Q5_0], "set_rows_q5_0" #itype, set_rows_q5_0 ## itype ## _len, set_rows_q5_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \
ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_Q5_1], "set_rows_q5_1" #itype, set_rows_q5_1 ## itype ## _len, set_rows_q5_1 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \
ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_Q8_0], "set_rows_q8_0" #itype, set_rows_q8_0 ## itype ## _len, set_rows_q8_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \
ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_TURBO3_0], "set_rows_turbo3_0" #itype, set_rows_turbo3_0 ## itype ## _len, set_rows_turbo3_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \
ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_TURBO4_0], "set_rows_turbo4_0" #itype, set_rows_turbo4_0 ## itype ## _len, set_rows_turbo4_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \
ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_IQ4_NL], "set_rows_iq4_nl" #itype, set_rows_iq4_nl ## itype ## _len, set_rows_iq4_nl ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true);

SET_ROWS(_i32)
Expand Down Expand Up @@ -15814,6 +15816,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_TURBO3_0:
case GGML_TYPE_TURBO4_0:
case GGML_TYPE_IQ4_NL:
return true;
default:
Expand All @@ -15838,6 +15842,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_TURBO3_0:
case GGML_TYPE_TURBO4_0:
case GGML_TYPE_IQ4_NL:
return true;
default:
Expand All @@ -15853,6 +15859,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_TURBO3_0:
case GGML_TYPE_TURBO4_0:
case GGML_TYPE_IQ4_NL:
return true;
default:
Expand Down
2 changes: 1 addition & 1 deletion ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -749,7 +749,7 @@ void process_shaders() {
string_to_spv("cpy_" + t + "_f32", "copy_from_quant.comp", {{"DATA_A_" + to_uppercase(t), "1"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
}

for (std::string t : {"f32", "f16", "bf16", "q1_0", "q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "iq4_nl"}) {
for (std::string t : {"f32", "f16", "bf16", "q1_0", "q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "iq4_nl", "turbo3_0", "turbo4_0"}) {
string_to_spv("set_rows_" + t + "_i32", "copy_to_quant.comp", {{"SET_ROWS", "1"}, {"DATA_A_" + to_uppercase(t), "1"}, {"B_TYPE", "uint"}, {"B_SIZE", "32"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
string_to_spv("set_rows_" + t + "_i64", "copy_to_quant.comp", {{"SET_ROWS", "1"}, {"DATA_A_" + to_uppercase(t), "1"}, {"B_TYPE", "uvec2"}, {"B_SIZE", "64"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
}
Expand Down