{

auto dot_pos = full_name.find('.');

if (dot_pos == std::string::npos) {

return {"public", full_name}; // Default to public schema if not specified

}

return {full_name.substr(0, dot_pos), full_name.substr(dot_pos + 1)};

const std::vector<Datum>& values,

const std::vector<uint8_t>& nulls,

int32_t t_len,

icm::string_map<nd::array>& row)

{

TupleDesc tupdesc = table_data.get_tuple_descriptor();

for (auto i = 0; i < table_data.num_columns(); ++i) {

Form_pg_attribute attr = TupleDescAttr(tupdesc, i);

if (attr == nullptr) {

ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Invalid attribute at position %d", i)));

}

// Skip SERIAL columns as they are auto-generated

if (attr->attidentity == 'a' || attr->attisdropped || attr->attgenerated == ATTRIBUTE_GENERATED_STORED) {

continue;

}

const auto column_name = table_data.get_atttypename(i);

// Skip if column is not in the input tuple

if (i >= t_len || nulls[i] == 1) {

row[column_name] = nd::none(nd::dtype::unknown, 0);

continue;

}

row[column_name] =

pg::utils::datum_to_nd(values[i], table_data.get_base_atttypid(i), table_data.get_atttypmod(i));

}

{

icm::string_map<> creds_map;

if (creds_guc_string != nullptr && std::strlen(creds_guc_string) > 0) {

try {

auto creds_json = icm::json::parse(creds_guc_string);

for (auto it = creds_json.begin(); it != creds_json.end(); ++it) {

std::string key_str(it.key());

if (it.value().is_string()) {

creds_map[key_str] = it.value().get<std::string>();

} else {

elog(WARNING, "Credential value for key '%s' is not a string, skipping", key_str.c_str());

}

}

} catch (const std::exception& e) {

elog(WARNING, "Failed to parse deeplake.creds: %s. Using environment variables.", e.what());

}

}

return creds_map;

{

if (root_path_guc_string != nullptr && std::strlen(root_path_guc_string) > 0) {

return std::string(root_path_guc_string);

}

return "";

{

DefineCustomStringVariable(

"deeplake.creds",

"JSON string containing storage credentials for DeepLake datasets",

"Credentials are used for all dataset operations in the current session. "

"Example: SET deeplake.creds = '{\"aws_access_key_id\": \"...\", \"aws_secret_access_key\": \"...\"}';",

&creds_guc_string, // linked C variable

"", // default value (empty)

PGC_USERSET, // context - can be set by any user

0, // flags

nullptr, // check_hook

nullptr, // assign_hook

nullptr // show_hook

);

DefineCustomStringVariable(

"deeplake.root_path",

"Root path for DeepLake datasets",

"Defines the root directory where datasets will be created when no explicit path is provided. "

"Supports local paths and cloud storage (s3://, gcs://, azure://). "

"Example: SET deeplake.root_path = 's3://my-bucket/datasets';",

&root_path_guc_string, // linked C variable

"", // default value (empty)

PGC_USERSET, // context - can be set by any user

0, // flags

nullptr, // check_hook

nullptr, // assign_hook

nullptr // show_hook

);

{

const std::string& table_name = table_data.get_table_name();

const std::string ds_path = table_data.get_dataset_path().url();

pg::utils::memory_context_switcher context_switcher;

pg::utils::pg_try([&]() {

StringInfoData buf;

initStringInfo(&buf);

appendStringInfo(&buf,

"INSERT INTO public.pg_deeplake_tables (table_oid, table_name, ds_path) "

"VALUES (%u, %s, %s) "

"ON CONFLICT DO NOTHING",

table_data.get_table_oid(),

quote_literal_cstr(table_name.c_str()),

quote_literal_cstr(ds_path.c_str()));

pg::utils::spi_connector connector;

if (SPI_execute(buf.data, false, 0) != SPI_OK_INSERT) {

throw pg::exception("Failed to save table metadata");

}

return true;

});

{

if (tables_loaded_) {

return;

}

/// set this first to avoid reloading metadata

tables_loaded_ = true;

if (!pg::utils::check_table_exists("pg_deeplake_tables")) {

return;

}

const char* query = "SELECT table_oid, table_name, ds_path FROM public.pg_deeplake_tables";

pg::utils::spi_connector connector;

if (SPI_execute(query, true, 0) != SPI_OK_SELECT) {

base::log_warning(base::log_channel::generic, "Failed to query table metadata");

return;

}

const auto proc = SPI_processed;

const bool res = (proc > 0 && SPI_tuptable != nullptr);

if (!res) {

return;

}

TupleDesc tupdesc = SPI_tuptable->tupdesc;

SPITupleTable* tuptable = SPI_tuptable;

// Get credentials from current session

auto creds = session_credentials::get_credentials();

std::vector<Oid> invalid_table_oids;

for (auto i = 0; i < proc; ++i) {

HeapTuple tuple = tuptable->vals[i];

bool is_null = false;

Oid relid = InvalidOid;

Datum relid_datum = SPI_getbinval(tuple, tupdesc, 1, &is_null);

if (!is_null) {

relid = DatumGetUInt32(relid_datum);

}

const char* table_name = SPI_getvalue(tuple, tupdesc, 2);

const char* ds_path = SPI_getvalue(tuple, tupdesc, 3);

if (relid == InvalidOid || table_name == nullptr || ds_path == nullptr || tables_.contains(relid)) {

continue;

}

try {

// Get the relation and its tuple descriptor

Relation rel = try_relation_open(relid, NoLock);

if (rel == nullptr) {

elog(WARNING, "Could not open relation for table %s", table_name);

invalid_table_oids.push_back(relid);

continue;

}

{

pg::utils::memory_context_switcher context_switcher(TopMemoryContext);

table_data td(

relid, table_name, CreateTupleDescCopy(RelationGetDescr(rel)), std::string(ds_path), creds);

auto it2status = tables_.emplace(relid, std::move(td));

up_to_date_ = false;

ASSERT(it2status.second);

}

relation_close(rel, NoLock);

} catch (const base::exception& e) {

ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", e.what())));

}

}

for (Oid invalid_oid : invalid_table_oids) {

auto query = fmt::format("DELETE FROM public.pg_deeplake_tables WHERE table_oid = {}", invalid_oid);

if (SPI_execute(query.c_str(), false, 0) != SPI_OK_DELETE) {

base::log_warning(

base::log_channel::generic, "Failed to delete invalid table metadata for table_oid: {}", invalid_oid);

}

}

load_views();

load_schema_name();

{

if (!pg::utils::check_table_exists("pg_deeplake_views")) {

return;

}

const char* view_query = "SELECT view_name, query_string FROM public.pg_deeplake_views";

if (SPI_execute(view_query, true, 0) != SPI_OK_SELECT) {

base::log_warning(base::log_channel::generic, "Failed to query view metadata");

return;

}

const auto proc = SPI_processed;

const bool res = (proc > 0 && SPI_tuptable != nullptr);

if (!res) {

return;

}

const auto tupdesc = SPI_tuptable->tupdesc;

const auto tuptable = SPI_tuptable;

for (auto i = 0; i < proc; ++i) {

HeapTuple tuple = tuptable->vals[i];

char* view_name = SPI_getvalue(tuple, tupdesc, 1);

char* view_str = SPI_getvalue(tuple, tupdesc, 2);

if (view_name == nullptr || view_str == nullptr) {

continue;

}

Oid view_oid = RelnameGetRelid(view_name);

if (OidIsValid(view_oid) && !views_.contains(view_oid)) {

views_.emplace(view_oid, std::pair{view_name, view_str});

up_to_date_ = false;

}

}

{

const char* search_path = GetConfigOption("search_path", false, false);

std::string schema_name = "public";

std::string_view sv(search_path);

// Check if search_path contains 'public'

if (sv.find("public") == std::string_view::npos) {

// If 'public' is not in search_path, take the first schema from comma-separated list

auto comma_pos = sv.find(',');

if (comma_pos != std::string_view::npos) {

sv = sv.substr(0, comma_pos);

}

// Trim leading/trailing whitespace

while (!sv.empty() && std::isspace(sv.front())) {

sv.remove_prefix(1);

}

while (!sv.empty() && std::isspace(sv.back())) {

sv.remove_suffix(1);

}

if (!sv.empty()) {

schema_name = std::string(sv);

}

}

schema_name_ = std::move(schema_name);

{

pg::utils::memory_context_switcher context_switcher;

if (!pg::utils::check_table_exists("pg_deeplake_tables")) {

return;

}

StringInfoData buf;

initStringInfo(&buf);

appendStringInfo(

&buf, "DELETE FROM public.pg_deeplake_tables WHERE table_name = %s", quote_literal_cstr(table_name.c_str()));

pg::utils::spi_connector connector;

if (SPI_execute(buf.data, false, 0) != SPI_OK_DELETE) {

ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Failed to erase table metadata")));

}

{

// Acquire global DDL lock to prevent concurrent CREATE/DROP TABLE operations (also needed for deeplake)

pg::table_ddl_lock_guard ddl_lock;

pg::utils::memory_context_switcher context_switcher(TopMemoryContext);

auto options = pg::table_options::current();

pg::table_options::current().reset();

auto [schema_name, simple_table_name] = split_table_name(table_name);

if (table_exists(table_id) || table_exists(table_name)) {

return;

}

std::string dataset_path;

// Use provided dataset path or construct default path

if (!options.dataset_path().empty()) {

// Explicit path provided via WITH clause

dataset_path = options.dataset_path();

} else {

// Get session root path if set

auto session_root = session_credentials::get_root_path();

std::string root_dir;

if (!session_root.empty()) {

// Use session-level root path from deeplake.root_path GUC

root_dir = session_root;

} else {

// Fall back to DEEPLAKE_ROOT_PATH env var or PostgreSQL data directory

root_dir = pg::utils::get_deeplake_root_directory();

}

// Construct path: root_dir/schema_name/table_name

dataset_path = root_dir + "/" + schema_name + "/" + simple_table_name;

}

// Get credentials from current session

auto creds = session_credentials::get_credentials();

table_data td(table_id, table_name, CreateTupleDescCopy(tupdesc), dataset_path, creds);

PinTupleDesc(td.get_tuple_descriptor());

bool ds_exists = false;

try {

auto creds_for_exists = creds;

ds_exists = deeplake_api::exists(dataset_path, std::move(creds_for_exists)).get_future().get();

} catch (const base::exception& e) {

ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", e.what())));

}

try {

if (ds_exists) {

td.open_dataset(false);

base::log_info(base::log_channel::generic,

"Table dataset exists, url: {}, opening, num_rows: {}",

td.get_dataset_path().url(),

td.num_rows());

/// Validate columns

} else {

td.open_dataset(true);

// Create columns based on TupleDesc

for (auto i = 0; i < td.num_columns(); ++i) {

Form_pg_attribute attr = TupleDescAttr(td.get_tuple_descriptor(), i);

const char* column_name = NameStr(attr->attname);

// Resolve domain types to their base type

Oid base_typeid = td.get_base_atttypid(i);

// Map PostgreSQL types to DeepLake types

switch (base_typeid) {

case BOOLOID:

td.get_dataset()->add_column(column_name, nd::type::scalar(nd::dtype::boolean));

break;

case INT2OID:

td.get_dataset()->add_column(column_name, nd::type::scalar(nd::dtype::int16));

break;

case INT4OID:

case DATEOID:

td.get_dataset()->add_column(column_name, nd::type::scalar(nd::dtype::int32));

break;

case TIMEOID:

case TIMESTAMPOID:

case TIMESTAMPTZOID:

case INT8OID:

td.get_dataset()->add_column(column_name, nd::type::scalar(nd::dtype::int64));

break;

case FLOAT4OID:

td.get_dataset()->add_column(column_name, nd::type::scalar(nd::dtype::float32));

break;

case NUMERICOID: {

const int32_t typmod = attr->atttypmod;

if (typmod >= 0) {

const int32_t precision = ((typmod - VARHDRSZ) >> 16) & 0xFFFF;

if (precision > 15) {

const int32_t scale = (typmod - VARHDRSZ) & 0xFFFF;

elog(WARNING,

"Column '%s' has type NUMERIC(%d, %d), which may lose precision "

"as it is stored as FLOAT64.",

column_name,

precision,

scale);

}

}

}

case FLOAT8OID:

td.get_dataset()->add_column(column_name, nd::type::scalar(nd::dtype::float64));

break;

case CHAROID:

case BPCHAROID:

case VARCHAROID: {

const int32_t typmod = attr->atttypmod;

if (typmod == VARHDRSZ + 1) {

td.get_dataset()->add_column(column_name,

deeplake_core::type::generic(nd::type::scalar(nd::dtype::int8)));

} else {

td.get_dataset()->add_column(column_name, deeplake_core::type::text(codecs::compression::null));

}

break;

}

case UUIDOID:

case TEXTOID:

td.get_dataset()->add_column(column_name, deeplake_core::type::text(codecs::compression::null));

break;

case JSONOID:

case JSONBOID:

td.get_dataset()->add_column(column_name, deeplake_core::type::dict());

break;

case BYTEAOID:

td.get_dataset()->add_column(column_name,

deeplake_core::type::generic(nd::type::scalar(nd::dtype::byte)));

break;

case INT2ARRAYOID: {

int32_t ndims = (attr->attndims > 0) ? attr->attndims : 1;

if (ndims > 255) {

elog(ERROR,

"Column '%s' has unsupported type SMALLINT[] with %d dimensions (max 255)",

column_name,

ndims);

}

if (ndims == 1) {

td.get_dataset()->add_column(column_name, deeplake_core::type::embedding(0, nd::dtype::int16));

} else {

td.get_dataset()->add_column(

column_name, deeplake_core::type::generic(nd::type::array(nd::dtype::int16, ndims)));

}

break;

}

case INT4ARRAYOID: {

int32_t ndims = (attr->attndims > 0) ? attr->attndims : 1;

if (ndims > 255) {

elog(ERROR,

"Column '%s' has unsupported type INT[] with %d dimensions (max 255)",

column_name,

ndims);

}

if (ndims == 1) {

td.get_dataset()->add_column(column_name, deeplake_core::type::embedding(0, nd::dtype::int32));

} else {

td.get_dataset()->add_column(

column_name, deeplake_core::type::generic(nd::type::array(nd::dtype::int32, ndims)));

}

break;

}

case INT8ARRAYOID: {

int32_t ndims = (attr->attndims > 0) ? attr->attndims : 1;

if (ndims > 255) {

elog(ERROR,

"Column '%s' has unsupported type BIGINT[] with %d dimensions (max 255)",

column_name,

ndims);

}

if (ndims == 1) {

td.get_dataset()->add_column(column_name, deeplake_core::type::embedding(0, nd::dtype::int64));

} else {

td.get_dataset()->add_column(

column_name, deeplake_core::type::generic(nd::type::array(nd::dtype::int64, ndims)));

}

break;

}

case FLOAT4ARRAYOID: {

int32_t ndims = (attr->attndims > 0) ? attr->attndims : 1;

if (ndims > 255) {

elog(ERROR,

"Column '%s' has unsupported type REAL[] with %d dimensions (max 255)",

column_name,

ndims);

}

// Store FLOAT4[] as float32 (4 bytes)

if (ndims == 1) {

td.get_dataset()->add_column(column_name,

deeplake_core::type::embedding(0, nd::dtype::float32));

} else {

td.get_dataset()->add_column(

column_name, deeplake_core::type::generic(nd::type::array(nd::dtype::float32, ndims)));

}

break;

}

case FLOAT8ARRAYOID: {

int32_t ndims = (attr->attndims > 0) ? attr->attndims : 1;

if (ndims > 255) {

elog(ERROR,

"Column '%s' has unsupported type DOUBLE PRECISION[] with %d dimensions (max 255)",

column_name,

ndims);

}

if (ndims == 1) {

td.get_dataset()->add_column(column_name,

deeplake_core::type::embedding(0, nd::dtype::float64));

} else {

td.get_dataset()->add_column(

column_name, deeplake_core::type::generic(nd::type::array(nd::dtype::float64, ndims)));

}

break;

}

case BYTEAARRAYOID: {

if (attr->attndims > 1) {

elog(ERROR,

"Column '%s' has unsupported type BYTEA[] with %d dimensions",

column_name,

attr->attndims);

}

td.get_dataset()->add_column(

column_name, deeplake_core::type::generic(nd::type::array(nd::dtype::byte, attr->attndims)));

break;

}

case VARCHARARRAYOID:

case TEXTARRAYOID: {

if (attr->attndims > 1) {

elog(ERROR,

"Column '%s' has unsupported type TEXT[] with %d dimensions",

column_name,

attr->attndims);

}

td.get_dataset()->add_column(

column_name, deeplake_core::type::generic(nd::type::array(nd::dtype::string, attr->attndims)));

break;

}

default: {

auto creds_for_delete = creds;

deeplake_api::delete_dataset(td.get_dataset_path(), std::move(creds_for_delete)).get_future().get();

const char* tname = format_type_with_typemod(attr->atttypid, attr->atttypmod);

elog(ERROR,

"Create Table: Column '%s' has unsupported type '%s' (OID %u, base OID %u)",

column_name,

tname,

attr->atttypid,

base_typeid);

}

}

}

base::log_info(base::log_channel::generic,

"Table dataset initialized with {} columns, url: {}",

td.num_columns(),

td.get_dataset_path().url());

td.commit();

}

save_table_metadata(td);

} catch (const base::exception& e) {

ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", e.what())));

}

if (auto it = primary_keys_.find(simple_table_name); it != primary_keys_.end() && !it->second.empty()) {

try {

td.set_primary_keys(it->second);

} catch (const base::exception& e) {

elog(WARNING, "Failed to set primary keys for table %s: %s", table_name.c_str(), e.what());

}

}

tables_.emplace(table_id, std::move(td));

up_to_date_ = false;

{

pg::table_ddl_lock_guard ddl_lock;

if (!table_exists(table_name)) {

force_load_table_metadata();

}

if (table_exists(table_name)) {

auto& table_data = get_table_data(table_name);

auto creds = session_credentials::get_credentials();

try {

table_data.commit(); // Ensure all changes are committed before deletion

table_version_tracker::drop_table(table_data.get_table_oid());

deeplake_api::delete_dataset(table_data.get_dataset_path(), std::move(creds)).get_future().get();

} catch (const base::exception& e) {

ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Failed to delete dataset: %s", e.what())));

}

erase_table(table_name);

}

/// FreeTupleDesc(table_data.tuple_descriptor_);

erase_table_metadata(table_name);

{

auto& table_data = get_table_data(table_id);

table_data.add_insert_slots(nslots, slots);

if (nslots > 1) {

table_data.reset_streamers();

table_data.flush();

}

{

auto& table_data = get_table_data(table_id);

try {

const auto row_number = utils::tid_to_row_number(tid);

// Verify row exists

if (row_number >= table_data.num_rows()) {

ereport(

ERROR,

(errcode(ERRCODE_INTERNAL_ERROR),

errmsg("Row number %zu out of range for table %s", row_number, table_data.get_table_name().c_str())));

return false;

}

table_data.add_delete_row(row_number);

return true;

} catch (const base::exception& e) {

ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Failed to delete tuple: %s", e.what())));

return false;

}

{

auto& table_data = get_table_data(table_id);

TupleDesc tupdesc = table_data.get_tuple_descriptor();

try {

const auto row_number = utils::tid_to_row_number(tid);

// Verify row exists

if (row_number >= table_data.num_rows()) {

ereport(

ERROR,

(errcode(ERRCODE_INTERNAL_ERROR),

errmsg("Row number %zu out of range for table %s", row_number, table_data.get_table_name().c_str())));

return false;

}

// Extract values from HeapTuple

std::vector<Datum> values(tupdesc->natts, 0);

std::vector<uint8_t> nulls(tupdesc->natts, 0);

heap_deform_tuple(new_tuple, tupdesc, values.data(), reinterpret_cast<bool*>(nulls.data()));

icm::string_map<nd::array> row;

::convert_pg_to_nd(table_data, values, nulls, new_tuple->t_len, row);

table_data.add_update_row(row_number, std::move(row));

return true;

} catch (const base::exception& e) {

ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Failed to update tuple: %s", e.what())));

return false;

}

{

auto& table_data = get_table_data(table_id);

table_scan tscan(table_id, false, false);

try {

ExecClearTuple(slot);

const auto row_number = utils::tid_to_row_number(tid);

if (row_number >= table_data.num_rows()) {

return false;

}

Datum* values = slot->tts_values;

bool* nulls = slot->tts_isnull;

PG_TRY();

{

tscan.convert_nd_to_pg(row_number, values, nulls);

}

PG_CATCH();

{

PG_RE_THROW();

}

PG_END_TRY();

ExecStoreVirtualTuple(slot);

slot->tts_tid = *tid;

return true;

} catch (const base::exception& e) {

ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Failed to fetch tuple: %s", e.what())));

}

return false;

{

auto creds = session_credentials::get_credentials();

// Clean up all datasets and metadata

for (auto& [table_id, table_data] : tables_) {

try {

table_data.commit(); // Ensure all changes are committed before deletion

auto creds_for_delete = creds;

deeplake_api::delete_dataset(table_data.get_dataset_path(), std::move(creds_for_delete)).get_future().get();

FreeTupleDesc(table_data.get_tuple_descriptor());

} catch (const base::exception& e) {

// Log error but continue cleaning up other tables

ereport(

WARNING,

(errcode(ERRCODE_INTERNAL_ERROR),

errmsg("Failed to delete dataset for table %s: %s", table_data.get_table_name().c_str(), e.what())));

}

}

table_version_tracker::clear_all_versions();

tables_.clear();

views_.clear();

up_to_date_ = false;

if (!pg::utils::check_table_exists("pg_deeplake_tables") || !pg::utils::check_table_exists("pg_deeplake_views")) {

return;

}

// Clean up metadata table

pg::utils::memory_context_switcher context_switcher;

pg::utils::spi_connector connector;

if (SPI_execute("DELETE FROM public.pg_deeplake_tables", false, 0) != SPI_OK_DELETE) {

ereport(WARNING, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Failed to clear metadata table")));

}

if (SPI_execute("DELETE FROM public.pg_deeplake_views", false, 0) != SPI_OK_DELETE) {

ereport(WARNING, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Failed to clear views metadata table")));

}

{

if (view_exists(view_oid)) {

return;

}

std::string insert_str =

fmt::format("INSERT INTO public.pg_deeplake_views (view_name, query_string) VALUES ('{}', $${}$$) "

"ON CONFLICT (view_name) DO UPDATE SET query_string = EXCLUDED.query_string",

view_name,

view_str);

pg::utils::spi_connector connector;

if (SPI_execute(insert_str.c_str(), false, 0) != SPI_OK_INSERT) {

throw pg::exception("Failed to save view metadata");

}

views_.emplace(view_oid, std::pair{view_name, view_str});

up_to_date_ = false;

{

Oid view_oid = RelnameGetRelid(view_name.c_str());

if (views_.erase(view_oid) == 0) {

return;

}

up_to_date_ = false;

pg::utils::memory_context_switcher context_switcher;

if (!pg::utils::check_table_exists("pg_deeplake_views")) {

return;

}

std::string delete_str = fmt::format("DELETE FROM public.pg_deeplake_views WHERE view_name = '{}'", view_name);

pg::utils::spi_connector connector;

if (SPI_execute(delete_str.c_str(), false, 0) != SPI_OK_DELETE) {

ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Failed to erase view metadata")));

}