# In your llama.cpp CMakeLists.txt, add:
add_subdirectory(path/to/quant.cpp turboquant)
target_link_libraries(llama PRIVATE turboquant)

#include "integrations/llamacpp/tq_kv_cache.cpp"

int main() {
    tq_ggml_register_types();
    // ... rest of llama.cpp init
}

./llama-cli -m model.gguf --kv-cache-type tq-uniform-4b

All CLI names also accept short forms: turbo3, polar4, uniform4, qjl1, etc.

quant.cpp hooks into llama.cpp's KV cache layer:

Normal:  key_states (FP16) -> KV cache (FP16) -> attention
TurboQ:  key_states (FP16) -> quantize -> KV cache (3-4 bit) -> attention

The quantization is transparent to the rest of llama.cpp's pipeline. During prefill and token generation, key/value vectors are quantized before being stored in the KV cache. During attention computation, the quantized keys are used directly via the vec_dot callback (dequantize + dot product), avoiding full materialization of FP32 keys.

# In llama.cpp's CMakeLists.txt:
add_subdirectory(path/to/quant.cpp turboquant)
target_link_libraries(llama PRIVATE turboquant)

Build quant.cpp first:

cd /path/to/quant.cpp
cmake -B build -DCMAKE_BUILD_TYPE=Release
cmake --build build -j$(nproc)

Then in llama.cpp's CMakeLists.txt:

option(LLAMA_TURBOQUANT "Enable quant.cpp KV cache compression" OFF)

if(LLAMA_TURBOQUANT)
    set(TURBOQUANT_LIB_PATH "" CACHE PATH "Path to libturboquant.a")
    set(TURBOQUANT_INCLUDE_PATH "" CACHE PATH "Path to quant.cpp include/")

    add_library(turboquant STATIC IMPORTED)
    set_target_properties(turboquant PROPERTIES
        IMPORTED_LOCATION "${TURBOQUANT_LIB_PATH}/libturboquant.a"
    )

    target_sources(llama PRIVATE tq_kv_cache.cpp)
    target_include_directories(llama PRIVATE ${TURBOQUANT_INCLUDE_PATH})
    target_link_libraries(llama PRIVATE turboquant)
    target_compile_definitions(llama PRIVATE LLAMA_TURBOQUANT=1)
endif()

Build with:

cmake -B build \
  -DLLAMA_TURBOQUANT=ON \
  -DTURBOQUANT_LIB_PATH=/path/to/quant.cpp/build \
  -DTURBOQUANT_INCLUDE_PATH=/path/to/quant.cpp/include
cmake --build build -j$(nproc)

#include "tq_ggml_type.h"

// Initialize quant.cpp types (call once at startup)
tq_ggml_register_types();

// Parse CLI argument
tq_type kv_type = tq_parse_kv_cache_type("turbo3");

// Create context
tq_context_t* ctx = tq_llamacpp_create_context();

// Quantize keys during prefill
size_t key_buf_size = tq_quantize_keys_size(n_tokens, head_dim, kv_type);
void* key_buf = malloc(key_buf_size);
tq_llamacpp_quantize_keys(ctx, key_data, n_tokens, head_dim,
                           kv_type, key_buf, key_buf_size);

// Compute attention during decode
float* scores = (float*)malloc(seq_len * sizeof(float));
tq_llamacpp_attention(ctx, query, key_buf, seq_len, head_dim,
                       kv_type, scores);

// Print memory savings
tq_llamacpp_print_config(kv_type, 4, n_heads, head_dim, max_seq_len);

// Cleanup
free(scores);
free(key_buf);
tq_free(ctx);

llama.cpp main loop
    |
    v
KV cache manager (modified)
    |
    +-- tq_ggml_register_types()      [startup: register GGML type IDs]
    +-- tq_parse_kv_cache_type()      [CLI: parse --kv-cache-type arg]
    +-- tq_llamacpp_quantize_keys()   [prefill/decode: compress keys]
    +-- tq_llamacpp_quantize_values() [prefill/decode: compress values]
    +-- tq_llamacpp_attention()       [decode: Q*K^T from quantized cache]
    |
    v
libturboquant.a
    |
    +-- CPU generic (reference C implementation)
    +-- CPU AVX2 (x86 optimized, auto-detected)
    +-- CPU NEON (ARM optimized, auto-detected)
    +-- CUDA (GPU, optional via TQ_BUILD_CUDA)
    +-- Metal (Apple GPU, optional via TQ_BUILD_METAL)

Q: Build fails with "turboquant.h not found" A: Ensure TURBOQUANT_INCLUDE_PATH points to the directory containing turboquant/turboquant.h (i.e., the include/ directory).

Q: Linker errors about undefined tq_ symbols* A: Make sure libturboquant.a is built and TURBOQUANT_LIB_PATH is correct. Also ensure -lm is linked (quant.cpp uses libm for math functions).

Q: Quality degradation with QJL 1-bit A: QJL 1-bit provides maximum compression but lower quality. Use turbo3 or turbo4 for near-lossless quality with good compression.

Q: How do I choose between turbo3 and turbo4? A: turbo3 provides the best compression (5.6x) while maintaining good quality. turbo4 uses one more bit for higher quality at slightly less compression. For most use cases, turbo3 is the recommended default.

Apache 2.0 -- same as quant.cpp