#include <math.h>

#include <float.h>

#include <stdio.h>

#include <ctype.h>

#include <limits.h>

#include <stdlib.h>

#include <string.h>

#ifndef _WIN32

#include <unistd.h>

#include <stdint.h>

#include <stdbool.h>

#include <stdarg.h>

#include <stddef.h>

#include <time.h>

#include "sqlite-memory.h"

#include "dbmem-utils.h"

#include "dbmem-embed.h"

#include "dbmem-parser.h"

#include "dbmem-search.h"

#ifndef SQLITE_CORE

#define DBMEM_TYPE_VALUE 500

#define DBMEM_LOCAL_PROVIDER "local"

#define DBMEM_SAVEPOINT_NAME "memory_transaction"

#define DBMEM_SETTINGS_KEY_PROVIDER "provider"

#define DBMEM_SETTINGS_KEY_MODEL "model"

#define DBMEM_SETTINGS_KEY_DIMENSION "dimension"

#define DBMEM_SETTINGS_KEY_MAX_TOKENS "max_tokens"

#define DBMEM_SETTINGS_KEY_OVERLAY_TOKENS "overlay_tokens"

#define DBMEM_SETTINGS_KEY_CHARS_PER_TOKENS "chars_per_tokens"

#define DBMEM_SETTINGS_KEY_SAVE_CONTEXT "save_content"

#define DBMEM_SETTINGS_KEY_SKIP_SEMANTIC "skip_semantic"

#define DBMEM_SETTINGS_KEY_SKIP_HTML "skip_html"

#define DBMEM_SETTINGS_KEY_EXTENSIONS "extensions"

#define DBMEM_SETTINGS_KEY_ENGINE_WARMUP "engine_warmup"

#define DBMEM_SETTINGS_KEY_MAX_RESULTS "max_results"

#define DBMEM_SETTINGS_KEY_FTS_ENABLED "fts_enabled"

#define DBMEM_SETTINGS_KEY_VECTOR_WEIGHT "vector_weight"

#define DBMEM_SETTINGS_KEY_TEXT_WEIGHT "text_weight"

#define DBMEM_SETTINGS_KEY_MIN_SCORE "min_score"

#define DBMEM_SETTINGS_KEY_UPDATE_ACCESS "update_access"

#define DBMEM_SETTINGS_KEY_EMBEDDING_CACHE "embedding_cache"

#define DBMEM_SETTINGS_KEY_CACHE_MAX_ENTRIES "cache_max_entries"

#define DBMEM_SETTINGS_KEY_SEARCH_OVERSAMPLE "search_oversample"

#define DEFAULT_CHARS_PER_TOKEN 4 // Approximate number of characters per token (GPT ≈ 4, Claude ≈ 3.5)

#define DEFAULT_MAX_TOKENS 400 // Maximum tokens per chunk

#define DEFAULT_OVERLAY_TOKENS 80 // Token overlap between consecutive chunks

#define DEFAULT_MAX_SNIPPET_CHARS 700 // Maximum characters for search result snippets

#define DEFAULT_MAX_RESULTS 20 // Maximum number of search results

#define DEFAULT_VECTOR_WEIGHT 0.6 // Semantic similarity weight in hybrid search scoring

#define DEFAULT_TEXT_WEIGHT 0.4 // Full-text match weight in hybrid search scoring

#define DEFAULT_MIN_SCORE 0.7 // Minimum score threshold to filter irrelevant results

struct dbmem_context {

};

static bool fts5_is_available = true;

static int dbmem_settings_write (sqlite3 *db, const char *key, const char *text_value, sqlite3_int64 int_value, const sqlite3_value *sql_value, int bind_type) {

static const char *sql = "REPLACE INTO dbmem_settings (key, value) VALUES (?1, ?2);";

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_text(vm, 1, key, -1, NULL);

if (rc != SQLITE_OK) goto cleanup;

switch (bind_type) {

case SQLITE_TEXT: rc = sqlite3_bind_text(vm, 2, text_value, -1, NULL); break;

case SQLITE_INTEGER: rc = sqlite3_bind_int64(vm, 2, int_value); break;

case DBMEM_TYPE_VALUE: rc = sqlite3_bind_value(vm, 2, sql_value); break;

default: rc = SQLITE_MISUSE; goto cleanup;

}

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

cleanup:

if (rc != SQLITE_OK) DEBUG_DBMEM("Error in dbmem_settings_write: %s", sqlite3_errmsg(db));

if (vm) sqlite3_finalize(vm);

return rc;

}

static int dbmem_settings_write_text (sqlite3 *db, const char *key, const char *value) {

return dbmem_settings_write(db, key, value, 0, NULL, SQLITE_TEXT);

}

static int dbmem_settings_write_int (sqlite3 *db, const char *key, sqlite3_int64 value) {

return dbmem_settings_write(db, key, NULL, value, NULL, SQLITE_INTEGER);

}

static int dbmem_settings_write_value (sqlite3 *db, const char *key, sqlite3_value *value) {

return dbmem_settings_write(db, key, NULL, 0, value, DBMEM_TYPE_VALUE);

}

static int dbmem_settings_sync (dbmem_context *ctx, const char *key, sqlite3_value *value) {

if (!value) return 0;

if (strcasecmp(key, DBMEM_SETTINGS_KEY_MAX_TOKENS) == 0) {

int n = sqlite3_value_int(value);

if (n > 0) ctx->max_tokens = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_OVERLAY_TOKENS) == 0) {

int n = sqlite3_value_int(value);

if (n > 0) ctx->overlay_tokens = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_CHARS_PER_TOKENS) == 0) {

int n = sqlite3_value_int(value);

if (n > 0) ctx->chars_per_tokens = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_DIMENSION) == 0) {

int n = sqlite3_value_int(value);

if (n > 0) {ctx->dimension = n; ctx->dimension_saved = true;}

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_SAVE_CONTEXT) == 0) {

int n = sqlite3_value_int(value);

ctx->save_content = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_SKIP_SEMANTIC) == 0) {

int n = sqlite3_value_int(value);

ctx->skip_semantic = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_SKIP_HTML) == 0) {

int n = sqlite3_value_int(value);

ctx->skip_html = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_ENGINE_WARMUP) == 0) {

int n = sqlite3_value_int(value);

ctx->engine_warmup = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_FTS_ENABLED) == 0) {

int n = sqlite3_value_int(value);

ctx->perform_fts = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_MAX_RESULTS) == 0) {

int n = sqlite3_value_int(value);

if (n > 0) ctx->max_results = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_VECTOR_WEIGHT) == 0) {

double n = sqlite3_value_double(value);

if (n > 0) ctx->vector_weight = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_TEXT_WEIGHT) == 0) {

double n = sqlite3_value_double(value);

if (n > 0) ctx->text_weight = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_MIN_SCORE) == 0) {

double n = sqlite3_value_double(value);

if (n > 0) ctx->min_score = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_UPDATE_ACCESS) == 0) {

int n = sqlite3_value_int(value);

ctx->update_access = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_EMBEDDING_CACHE) == 0) {

int n = sqlite3_value_int(value);

ctx->embedding_cache = (n > 0) ? 1 : 0;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_CACHE_MAX_ENTRIES) == 0) {

int n = sqlite3_value_int(value);

if (n >= 0) ctx->cache_max_entries = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_SEARCH_OVERSAMPLE) == 0) {

int n = sqlite3_value_int(value);

if (n >= 0) ctx->search_oversample = n;

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_PROVIDER) == 0) {

char *provider = dbmem_strdup((const char *)sqlite3_value_text(value));

if (provider) {

if (ctx->provider) dbmemory_free(ctx->provider);

ctx->provider = provider;

}

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_MODEL) == 0) {

char *model = dbmem_strdup((const char *)sqlite3_value_text(value));

if (model) {

if (ctx->model) dbmemory_free(ctx->model);

ctx->model = model;

}

return 0;

}

if (strcasecmp(key, DBMEM_SETTINGS_KEY_EXTENSIONS) == 0) {

char *extensions = dbmem_strdup((const char *)sqlite3_value_text(value));

if (extensions) {

if (ctx->extensions) dbmemory_free(ctx->extensions);

ctx->extensions = extensions;

}

return 0;

}

return 0;

}

void dbmem_settings_load (sqlite3 *db, dbmem_context *ctx) {

const char *sql = "SELECT key, value FROM dbmem_settings;";

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

while (1) {

// no error handling here

rc = sqlite3_step(vm);

if (rc != SQLITE_ROW) break;

const char *key = (const char *)sqlite3_column_text(vm, 0);

if (!key) continue;

dbmem_settings_sync(ctx, key, sqlite3_column_value(vm, 1));

}

cleanup:

if (vm) sqlite3_finalize(vm);

return;

}

static int dbmem_database_init (sqlite3 *db) {

const char *sql = "CREATE TABLE IF NOT EXISTS dbmem_settings (key TEXT PRIMARY KEY, value TEXT);";

int rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

if (rc != SQLITE_OK) return rc;

sql = "CREATE TABLE IF NOT EXISTS dbmem_content (hash INTEGER PRIMARY KEY NOT NULL, path TEXT NOT NULL UNIQUE, value TEXT DEFAULT NULL, length INTEGER NOT NULL, context TEXT DEFAULT NULL, created_at INTEGER DEFAULT 0, last_accessed INTEGER DEFAULT 0);";

rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

if (rc != SQLITE_OK) return rc;

sql = "CREATE TABLE IF NOT EXISTS dbmem_vault (hash INTEGER NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, PRIMARY KEY (hash, seq));";

rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

if (rc != SQLITE_OK) return rc;

sql = "CREATE TABLE IF NOT EXISTS dbmem_cache (text_hash INTEGER NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, PRIMARY KEY (text_hash, provider, model));";

rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

if (rc != SQLITE_OK) return rc;

sql = "CREATE VIRTUAL TABLE IF NOT EXISTS dbmem_vault_fts USING fts5 (content, hash UNINDEXED, seq UNINDEXED, context UNINDEXED);";

rc = sqlite3_exec(db, sql, NULL, NULL, NULL);

if (rc != SQLITE_OK) {

fts5_is_available = false;

rc = SQLITE_OK;

}

// explicitly allows extension loading (only available when linked statically)

// when loaded dynamically, the calling application must enable extension loading

#if defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)

rc = sqlite3_enable_load_extension(db, 1);

if (rc != SQLITE_OK) return rc;

#endif

return rc;

}

static bool dbmem_database_check_if_stored (sqlite3 *db, uint64_t hash, int64_t len) {

static const char *sql = "SELECT length FROM dbmem_content WHERE hash=? LIMIT 1;";

bool result = false;

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 1, (sqlite3_int64)hash);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

else if (rc != SQLITE_ROW) goto cleanup;

// SQLITE_ROW case

sqlite3_int64 saved_len = sqlite3_column_int64(vm, 0);

result = (saved_len == len);

cleanup:

if (vm) sqlite3_finalize(vm);

return result;

}

static void dbmem_database_delete_hash (sqlite3 *db, sqlite3_int64 hash) {

sqlite3_stmt *vm = NULL;

if (fts5_is_available) {

sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault_fts WHERE hash=?1;", -1, &vm, NULL);

sqlite3_bind_int64(vm, 1, hash);

sqlite3_step(vm);

sqlite3_finalize(vm);

}

sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault WHERE hash=?1;", -1, &vm, NULL);

sqlite3_bind_int64(vm, 1, hash);

sqlite3_step(vm);

sqlite3_finalize(vm);

sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE hash=?1;", -1, &vm, NULL);

sqlite3_bind_int64(vm, 1, hash);

sqlite3_step(vm);

sqlite3_finalize(vm);

}

static void dbmem_database_delete_stale_path (sqlite3 *db, const char *path, uint64_t new_hash) {

if (!path) return;

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, "SELECT hash FROM dbmem_content WHERE path=?1;", -1, &vm, NULL);

if (rc != SQLITE_OK) return;

sqlite3_bind_text(vm, 1, path, -1, SQLITE_STATIC);

rc = sqlite3_step(vm);

if (rc == SQLITE_ROW) {

sqlite3_int64 old_hash = sqlite3_column_int64(vm, 0);

sqlite3_finalize(vm);

if ((uint64_t)old_hash != new_hash) {

dbmem_database_delete_hash(db, old_hash);

}

} else {

sqlite3_finalize(vm);

}

}

static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t hash, const char *buffer, int64_t len) {

static const char *sql = "INSERT INTO dbmem_content (hash, path, value, length, context, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6);";

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 1, (sqlite3_int64)hash);

if (rc != SQLITE_OK) goto cleanup;

const char *path = ctx->path;

char uuid[DBMEM_UUID_STR_MAXLEN];

if (path == NULL) path = dbmem_uuid_v7(uuid);

rc = sqlite3_bind_text(vm, 2, path, -1, SQLITE_STATIC);

if (rc != SQLITE_OK) goto cleanup;

if (len > (int64_t)INT_MAX) { rc = SQLITE_TOOBIG; goto cleanup; }

rc = (ctx->save_content) ? sqlite3_bind_text(vm, 3, buffer, (int)len, SQLITE_STATIC) : sqlite3_bind_null(vm, 3);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 4, (sqlite3_int64)len);

if (rc != SQLITE_OK) goto cleanup;

rc = (ctx->context) ? sqlite3_bind_text(vm, 5, ctx->context, -1, SQLITE_STATIC) : sqlite3_bind_null(vm, 5);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 6, (sqlite3_int64)time(NULL));

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

cleanup:

if (rc != SQLITE_OK) DEBUG_DBMEM_ALWAYS("Error in dbmem_database_add_entry: %s", sqlite3_errmsg(ctx->db));

if (vm) sqlite3_finalize(vm);

return rc;

}

static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *result, size_t offset, size_t length, size_t index) {

static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES (?1, ?2, ?3, ?4, ?5);";

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(ctx->db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 1, (sqlite3_int64)ctx->hash);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 2, (sqlite3_int64)index);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_blob(vm, 3, result->embedding, (int)(result->n_embd * sizeof(float)), SQLITE_STATIC);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 4, (sqlite3_int64)offset);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 5, (sqlite3_int64)length);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

cleanup:

if (rc != SQLITE_OK) DEBUG_DBMEM_ALWAYS("Error in dbmem_database_add_chunk: %s", sqlite3_errmsg(ctx->db));

if (vm) sqlite3_finalize(vm);

return rc;

}

static int dbmem_database_add_fts5 (dbmem_context *ctx, const char *text, size_t text_len, size_t index) {

static const char *sql = "INSERT INTO dbmem_vault_fts (content, hash, seq, context) VALUES (?1, ?2, ?3, ?4);";

sqlite3_stmt *vm = NULL;

int rc = sqlite3_prepare_v2(ctx->db, sql, -1, &vm, NULL);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_text(vm, 1, text, (int)text_len, SQLITE_STATIC);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 2, (sqlite3_int64)ctx->hash);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_bind_int64(vm, 3, (sqlite3_int64)index);

if (rc != SQLITE_OK) goto cleanup;

rc = (ctx->context) ? sqlite3_bind_text(vm, 4, ctx->context, -1, SQLITE_STATIC) : sqlite3_bind_null(vm, 4);

if (rc != SQLITE_OK) goto cleanup;

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

cleanup:

if (rc != SQLITE_OK) DEBUG_DBMEM_ALWAYS("Error in dbmem_database_add_fts5: %s", sqlite3_errmsg(ctx->db));

if (vm) sqlite3_finalize(vm);

return rc;

}

static int dbmem_database_begin_transaction (sqlite3 *db) {

return sqlite3_exec(db, "SAVEPOINT " DBMEM_SAVEPOINT_NAME ";", NULL, NULL, NULL);

}

static int dbmem_database_commit_transaction (sqlite3 *db) {

return sqlite3_exec(db, "RELEASE " DBMEM_SAVEPOINT_NAME ";", NULL, NULL, NULL);

}

static int dbmem_database_rollback_transaction (sqlite3 *db) {

return sqlite3_exec(db, "ROLLBACK TO " DBMEM_SAVEPOINT_NAME "; RELEASE " DBMEM_SAVEPOINT_NAME ";", NULL, NULL, NULL);

}

static void *dbmem_context_create (sqlite3 *db) {

dbmem_context *ctx = (dbmem_context *)dbmemory_zeroalloc(sizeof(dbmem_context));

if (!ctx) return NULL;

ctx->db = db;

ctx->chars_per_tokens = DEFAULT_CHARS_PER_TOKEN;

ctx->max_tokens = DEFAULT_MAX_TOKENS;

ctx->overlay_tokens = DEFAULT_OVERLAY_TOKENS;

ctx->snippet_max_chars = DEFAULT_MAX_SNIPPET_CHARS;

ctx->skip_html = true;

ctx->save_content = true;

ctx->engine_warmup = false;

ctx->perform_fts = fts5_is_available;

ctx->max_results = DEFAULT_MAX_RESULTS;

ctx->vector_weight = DEFAULT_VECTOR_WEIGHT;

ctx->text_weight = DEFAULT_TEXT_WEIGHT;

ctx->min_score = DEFAULT_MIN_SCORE;

ctx->update_access = true;

ctx->embedding_cache = true;

return (void *)ctx;

}

static void dbmem_context_free (void *ptr) {

if (!ptr) return;

dbmem_context *ctx = (dbmem_context *)ptr;

if (ctx->provider) dbmemory_free(ctx->provider);

if (ctx->model) dbmemory_free(ctx->model);

if (ctx->api_key) dbmemory_free(ctx->api_key);

if (ctx->extensions) dbmemory_free(ctx->extensions);

if (ctx->cache_buffer) dbmemory_free(ctx->cache_buffer);

// custom provider

if (ctx->custom_engine && ctx->custom_provider.free) ctx->custom_provider.free(ctx->custom_engine, ctx->custom_provider.xdata);

if (ctx->custom_provider_name) dbmemory_free(ctx->custom_provider_name);

#ifndef DBMEM_OMIT_LOCAL_ENGINE

if (ctx->l_engine) dbmem_local_engine_free(ctx->l_engine);

#endif

#ifndef DBMEM_OMIT_REMOTE_ENGINE

if (ctx->r_engine) dbmem_remote_engine_free(ctx->r_engine);

#endif

dbmemory_free(ctx);

}

static void dbmem_context_reset_temp_values (dbmem_context *ctx) {

ctx->counter = 0;

ctx->hash = 0;

ctx->context = NULL;

ctx->path = NULL;

ctx->error_msg[0] = 0;

}

void *dbmem_context_engine (dbmem_context *ctx, bool *is_local) {

if (ctx->is_custom) {

if (is_local) *is_local = false;

return ctx->custom_engine;

}

if (is_local) *is_local = ctx->is_local;

return (ctx->is_local) ? (void *)ctx->l_engine : (void *)ctx->r_engine;

}

bool dbmem_context_is_custom (dbmem_context *ctx) {

return ctx->is_custom;

}

int dbmem_context_custom_compute (dbmem_context *ctx, const char *text, int text_len, embedding_result_t *result) {

dbmem_embedding_result_t cr = {0};

int rc = ctx->custom_provider.compute(ctx->custom_engine, text, text_len, ctx->custom_provider.xdata, &cr);

if (rc != 0) return rc;

result->n_tokens = cr.n_tokens;

result->n_tokens_truncated = cr.n_tokens_truncated;

result->n_embd = cr.n_embd;

result->embedding = cr.embedding;

return 0;

}

bool dbmem_context_load_vector (dbmem_context *ctx) {

if (ctx->vector_extension_available) return true;

// check if sqlite-vector is loaded

// there's no built-in way to verify if sqlite-vector has already been already loaded for this specific database connection

// the workaround is to attempt to execute vector_version and check for an error

// an error indicates that initialization has not been performed

if (sqlite3_exec(ctx->db, "SELECT vector_version();", NULL, NULL, NULL) != SQLITE_OK) {

dbmem_context_set_error(ctx, "SQLite-vector extension not found, make sure to load it before using the memory_search function");

return false;

}

if (ctx->dimension == 0) {

dbmem_context_set_error(ctx, "SQLite-vector extension cannot be loaded because embedding dimension is not specified");

return false;

}

// In the future can check for quantization options and embedding type here

char sql[1024];

snprintf(sql, sizeof(sql), "SELECT vector_init('dbmem_vault', 'embedding', 'type=FLOAT32,distance=COSINE,dimension=%d');", ctx->dimension);

int rc = sqlite3_exec(ctx->db, sql, NULL, NULL, NULL);

if (rc != SQLITE_OK) {

dbmem_context_set_error(ctx, sqlite3_errmsg(ctx->db));

return false;

}

ctx->vector_extension_available = true;

return true;

}

bool dbmem_context_sync_available (dbmem_context *ctx) {

if (ctx->sync_extension_available) return true;

// there's no built-in way to verify if sqlite-sync has already been already loaded for this specific database connection

// the workaround is to attempt to execute cloudsync_version and check for an error (an error indicates that initialization has not been performed)

if (sqlite3_exec(ctx->db, "SELECT cloudsync_version();", NULL, NULL, NULL) != SQLITE_OK) {

snprintf(ctx->error_msg, DBMEM_ERRBUF_SIZE, "%s", "SQLite-sync extension not found, make sure to load it before using the memory_sync function");

return false;

}

ctx->sync_extension_available = true;

return true;

}

bool dbmem_context_perform_fts (dbmem_context *ctx) {

if (!fts5_is_available) return false;

return ctx->perform_fts;

}

int dbmem_context_max_results (dbmem_context *ctx) {

return ctx->max_results;

}

double dbmem_context_vector_weight (dbmem_context *ctx) {

return ctx->vector_weight;

}

double dbmem_context_text_weight (dbmem_context *ctx) {

return ctx->text_weight;

}

double dbmem_context_min_score (dbmem_context *ctx) {

return ctx->min_score;

}

bool dbmem_context_update_access (dbmem_context *ctx) {

return ctx->update_access;

}

int dbmem_context_search_oversample (dbmem_context *ctx) {

return ctx->search_oversample;

}

const char *dbmem_context_errmsg (dbmem_context *ctx) {

return ctx->error_msg;

}

const char *dbmem_context_apikey (dbmem_context *ctx) {

return ctx->api_key;

}

void dbmem_context_set_error (dbmem_context *ctx, const char *str) {

snprintf(ctx->error_msg, DBMEM_ERRBUF_SIZE, "%s", str);

}

void dbmem_context_set_errorf (dbmem_context *ctx, const char *fmt, ...) {

va_list ap;

va_start(ap, fmt);

vsnprintf(ctx->error_msg, DBMEM_ERRBUF_SIZE, fmt, ap);

va_end(ap);

}

static void dbmem_delete (sqlite3_context *context, int argc, sqlite3_value **argv) {

UNUSED_PARAM(argc);

if (sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {

sqlite3_result_error(context, "The function memory_delete expects one argument of type INTEGER (hash)", SQLITE_ERROR);

return;

}

sqlite3_int64 hash = sqlite3_value_int64(argv[0]);

sqlite3 *db = sqlite3_context_db_handle(context);

int rc = dbmem_database_begin_transaction(db);

if (rc != SQLITE_OK) {

sqlite3_result_error(context, sqlite3_errmsg(db), -1);

return;

}

// Delete from FTS first (if available)

if (fts5_is_available) {

sqlite3_stmt *vm = NULL;

rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault_fts WHERE hash = ?1;", -1, &vm, NULL);

if (rc == SQLITE_OK) {

sqlite3_bind_int64(vm, 1, hash);

sqlite3_step(vm);

sqlite3_finalize(vm);

}

}

// Delete from vault

sqlite3_stmt *vm = NULL;

rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault WHERE hash = ?1;", -1, &vm, NULL);

if (rc != SQLITE_OK) goto rollback;

sqlite3_bind_int64(vm, 1, hash);

rc = sqlite3_step(vm);

sqlite3_finalize(vm);

if (rc != SQLITE_DONE) goto rollback;

// Delete from content

rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE hash = ?1;", -1, &vm, NULL);

if (rc != SQLITE_OK) goto rollback;

sqlite3_bind_int64(vm, 1, hash);

rc = sqlite3_step(vm);

sqlite3_finalize(vm);

if (rc != SQLITE_DONE) goto rollback;

int changes = sqlite3_changes(db);

dbmem_database_commit_transaction(db);

sqlite3_result_int(context, changes);

return;

rollback:

dbmem_database_rollback_transaction(db);

sqlite3_result_error(context, sqlite3_errmsg(db), -1);

}

static void dbmem_delete_context (sqlite3_context *context, int argc, sqlite3_value **argv) {

UNUSED_PARAM(argc);

if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) {

sqlite3_result_error(context, "The function memory_delete_context expects one argument of type TEXT (context)", SQLITE_ERROR);

return;

}

const char *ctx_name = (const char *)sqlite3_value_text(argv[0]);

sqlite3 *db = sqlite3_context_db_handle(context);

int rc = dbmem_database_begin_transaction(db);

if (rc != SQLITE_OK) {

sqlite3_result_error(context, sqlite3_errmsg(db), -1);

return;

}

// Delete from FTS first (if available)

if (fts5_is_available) {

sqlite3_stmt *vm = NULL;

rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault_fts WHERE hash IN (SELECT hash FROM dbmem_content WHERE context = ?1);", -1, &vm, NULL);

if (rc == SQLITE_OK) {

sqlite3_bind_text(vm, 1, ctx_name, -1, SQLITE_STATIC);

sqlite3_step(vm);

sqlite3_finalize(vm);

}

}

// Delete from vault

sqlite3_stmt *vm = NULL;

rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_vault WHERE hash IN (SELECT hash FROM dbmem_content WHERE context = ?1);", -1, &vm, NULL);

if (rc != SQLITE_OK) goto rollback;

sqlite3_bind_text(vm, 1, ctx_name, -1, SQLITE_STATIC);

rc = sqlite3_step(vm);

sqlite3_finalize(vm);

if (rc != SQLITE_DONE) goto rollback;

// Delete from content

rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_content WHERE context = ?1;", -1, &vm, NULL);

if (rc != SQLITE_OK) goto rollback;

sqlite3_bind_text(vm, 1, ctx_name, -1, SQLITE_STATIC);

rc = sqlite3_step(vm);

sqlite3_finalize(vm);

if (rc != SQLITE_DONE) goto rollback;

int changes = sqlite3_changes(db);

dbmem_database_commit_transaction(db);

sqlite3_result_int(context, changes);

return;

rollback:

dbmem_database_rollback_transaction(db);

sqlite3_result_error(context, sqlite3_errmsg(db), -1);

}

static void dbmem_clear (sqlite3_context *context, int argc, sqlite3_value **argv) {

UNUSED_PARAM(argc); UNUSED_PARAM(argv);

sqlite3 *db = sqlite3_context_db_handle(context);

int rc = dbmem_database_begin_transaction(db);

if (rc != SQLITE_OK) {

sqlite3_result_error(context, sqlite3_errmsg(db), -1);

return;

}

// Delete from FTS first (if available)

if (fts5_is_available) {

rc = sqlite3_exec(db, "DELETE FROM dbmem_vault_fts;", NULL, NULL, NULL);

if (rc != SQLITE_OK) goto rollback;

}

// Delete from vault

rc = sqlite3_exec(db, "DELETE FROM dbmem_vault;", NULL, NULL, NULL);

if (rc != SQLITE_OK) goto rollback;

// Delete from content

rc = sqlite3_exec(db, "DELETE FROM dbmem_content;", NULL, NULL, NULL);

if (rc != SQLITE_OK) goto rollback;

dbmem_database_commit_transaction(db);

sqlite3_result_int(context, 1);

return;

rollback:

dbmem_database_rollback_transaction(db);

sqlite3_result_error(context, sqlite3_errmsg(db), -1);

}

static void dbmem_cache_clear (sqlite3_context *context, int argc, sqlite3_value **argv) {

sqlite3 *db = sqlite3_context_db_handle(context);

int rc;

if (argc == 0) {

rc = sqlite3_exec(db, "DELETE FROM dbmem_cache;", NULL, NULL, NULL);

} else if (argc == 2) {

if (sqlite3_value_type(argv[0]) != SQLITE_TEXT || sqlite3_value_type(argv[1]) != SQLITE_TEXT) {

sqlite3_result_error(context, "The function memory_cache_clear expects two arguments of type TEXT (provider, model)", SQLITE_ERROR);

return;

}

const char *provider = (const char *)sqlite3_value_text(argv[0]);

const char *model = (const char *)sqlite3_value_text(argv[1]);

sqlite3_stmt *vm = NULL;

rc = sqlite3_prepare_v2(db, "DELETE FROM dbmem_cache WHERE provider=?1 AND model=?2;", -1, &vm, NULL);

if (rc == SQLITE_OK) {

sqlite3_bind_text(vm, 1, provider, -1, SQLITE_STATIC);

sqlite3_bind_text(vm, 2, model, -1, SQLITE_STATIC);

rc = sqlite3_step(vm);

if (rc == SQLITE_DONE) rc = SQLITE_OK;

}

if (vm) sqlite3_finalize(vm);

} else {

sqlite3_result_error(context, "The function memory_cache_clear expects 0 or 2 arguments", SQLITE_ERROR);

return;

}

if (rc != SQLITE_OK) {

sqlite3_result_error(context, sqlite3_errmsg(db), -1);

return;

}

sqlite3_result_int(context, sqlite3_changes(db));

}

static void dbmem_version (sqlite3_context *context, int argc, sqlite3_value **argv) {

UNUSED_PARAM(argc); UNUSED_PARAM(argv);

sqlite3_result_text(context, SQLITE_DBMEMORY_VERSION, -1, NULL);

}

static int dbmem_reindex(dbmem_context *ctx);

static void dbmem_set_model (sqlite3_context *context, int argc, sqlite3_value **argv) {

// 2 TEXT arguments: provider and model

// if provider is local then model is the full path to the model to use

// options are saved into settings

// sanity check type

if ((sqlite3_value_type(argv[0]) != SQLITE_TEXT) || (sqlite3_value_type(argv[1]) != SQLITE_TEXT)) {

sqlite3_result_error(context, "The function memory_set_model expects two arguments of type TEXT", SQLITE_ERROR);

return;

}

// retrieve arguments

const char *provider = (const char *)sqlite3_value_text(argv[0]);

const char *model = (const char *)sqlite3_value_text(argv[1]);

// retrieve context

dbmem_context *ctx = (dbmem_context *)sqlite3_user_data(context);

// detect model change (only if a model was previously configured)

bool model_changed = false;

if (ctx->provider && ctx->model) {

model_changed = (strcasecmp(ctx->provider, provider) != 0 || strcasecmp(ctx->model, model) != 0);

}

bool is_local_provider = (strcasecmp(provider, DBMEM_LOCAL_PROVIDER) == 0);

// check if a custom provider matches

bool is_custom_provider = (ctx->custom_provider_name && ctx->custom_provider.compute &&

strcasecmp(provider, ctx->custom_provider_name) == 0);

if (!is_custom_provider) {

#ifdef DBMEM_OMIT_LOCAL_ENGINE

if (is_local_provider) {

sqlite3_result_error(context, "Local provider cannot be set because SQLite-memory was compiled without local provider support", SQLITE_ERROR);

return;

}

#endif

#ifdef DBMEM_OMIT_REMOTE_ENGINE

if (!is_local_provider) {

sqlite3_result_error(context, "Remote provider cannot be set because SQLite-memory was compiled without remote provider support", SQLITE_ERROR);

return;

}

#endif

}

// custom provider path

if (is_custom_provider) {

// free previous custom engine if any

if (ctx->custom_engine && ctx->custom_provider.free) ctx->custom_provider.free(ctx->custom_engine, ctx->custom_provider.xdata);

ctx->custom_engine = NULL;

ctx->custom_engine = ctx->custom_provider.init(model, ctx->api_key, ctx->custom_provider.xdata, ctx->error_msg);

if (ctx->custom_engine == NULL) {

sqlite3_result_error(context, ctx->error_msg, -1);

return;

}

ctx->is_custom = true;

ctx->is_local = false;

}

// if provider is local then make sure model file exists

#ifndef DBMEM_OMIT_LOCAL_ENGINE

if (!is_custom_provider && is_local_provider) {

if (dbmem_file_exists(model) == false) {

sqlite3_result_error(context, "Local model not found in the specified path", SQLITE_ERROR);

return;

}

if (ctx->l_engine) dbmem_local_engine_free(ctx->l_engine);

ctx->l_engine = NULL;

ctx->l_engine = dbmem_local_engine_init(ctx, model, ctx->error_msg);

if (ctx->l_engine == NULL) {

sqlite3_result_error(context, ctx->error_msg, -1);

return;

}

if (ctx->engine_warmup) {

dbmem_local_engine_warmup(ctx->l_engine);

}

ctx->is_local = true;

ctx->is_custom = false;

}

#endif

#ifndef DBMEM_OMIT_REMOTE_ENGINE

if (!is_custom_provider && !is_local_provider) {

if (ctx->r_engine) dbmem_remote_engine_free(ctx->r_engine);

ctx->r_engine = NULL;