#include "QueryEngine/Execute.h"

#include <llvm/Transforms/Utils/BasicBlockUtils.h>

#include <boost/filesystem/operations.hpp>

#include <boost/filesystem/path.hpp>

#ifdef HAVE_CUDA

#include <cuda.h>

#endif // HAVE_CUDA

#include <RexVisitor.h>

#include <chrono>

#include <ctime>

#include <future>

#include <memory>

#include <mutex>

#include <numeric>

#include <set>

#include <thread>

#include <type_traits>

#include "Catalog/Catalog.h"

#include "CudaMgr/CudaMgr.h"

#include "DataMgr/BufferMgr/BufferMgr.h"

#include "DataMgr/ForeignStorage/FsiChunkUtils.h"

#include "OSDependent/heavyai_path.h"

#include "Parser/ParserNode.h"

#include "QueryEngine/AggregateUtils.h"

#include "QueryEngine/AggregatedColRange.h"

#include "QueryEngine/CodeGenerator.h"

#include "QueryEngine/ColumnFetcher.h"

#include "QueryEngine/Descriptors/QueryCompilationDescriptor.h"

#include "QueryEngine/Descriptors/QueryFragmentDescriptor.h"

#include "QueryEngine/ErrorHandling.h"

#include "QueryEngine/ExecutorResourceMgr/ExecutorResourceMgr.h"

#include "QueryEngine/ExpressionRewrite.h"

#include "QueryEngine/ExternalCacheInvalidators.h"

#include "QueryEngine/JoinHashTable/BaselineJoinHashTable.h"

#include "QueryEngine/OutputBufferInitialization.h"

#include "QueryEngine/QueryDispatchQueue.h"

#include "QueryEngine/QueryEngine.h"

#include "QueryEngine/QueryRewrite.h"

#include "QueryEngine/QueryTemplateGenerator.h"

#include "QueryEngine/RelAlgDag.h"

#include "QueryEngine/ResultSetReductionJIT.h"

#include "QueryEngine/RuntimeFunctions.h"

#include "QueryEngine/SpeculativeTopN.h"

#include "QueryEngine/StringDictionaryGenerations.h"

#include "QueryEngine/TableFunctions/TableFunctionCompilationContext.h"

#include "QueryEngine/TableFunctions/TableFunctionExecutionContext.h"

#include "QueryEngine/Utils/SerializeLiterals.h"

#include "QueryEngine/Visitors/TransientStringLiteralsVisitor.h"

#include "Shared/SystemParameters.h"

#include "Shared/TypedDataAccessors.h"

#include "Shared/measure.h"

#include "Shared/misc.h"

#include "Shared/scope.h"

#include "Shared/threading.h"

bool g_enable_watchdog{false};

bool g_enable_dynamic_watchdog{false};

size_t g_watchdog_none_encoded_string_translation_limit{1000000UL};

size_t g_watchdog_max_projected_rows_per_device{128000000};

size_t g_preflight_count_query_threshold{1000000};

size_t g_watchdog_in_clause_max_num_elem_non_bitmap{10000};

size_t g_watchdog_in_clause_max_num_elem_bitmap{1 << 25};

size_t g_watchdog_in_clause_max_num_input_rows{5000000};

size_t g_in_clause_num_elem_skip_bitmap{100};

bool g_enable_cpu_sub_tasks{false};

size_t g_cpu_sub_task_size{500'000};

bool g_enable_filter_function{true};

unsigned g_dynamic_watchdog_time_limit{10000};

bool g_allow_cpu_retry{true};

bool g_allow_query_step_cpu_retry{true};

bool g_null_div_by_zero{false};

unsigned g_trivial_loop_join_threshold{1000};

bool g_from_table_reordering{true};

bool g_inner_join_fragment_skipping{true};

extern bool g_enable_smem_group_by;

extern std::unique_ptr<llvm::Module> udf_gpu_module;

extern std::unique_ptr<llvm::Module> udf_cpu_module;

bool g_enable_filter_push_down{false};

float g_filter_push_down_max_selectivity{0.1f};

size_t g_filter_push_down_selectivity_override_max_passing_num_rows{4000000};

bool g_enable_columnar_output{false};

bool g_enable_left_join_filter_hoisting{true};

bool g_optimize_row_initialization{true};

bool g_enable_bbox_intersect_hashjoin{true};

size_t g_ratio_num_hash_entry_to_num_tuple_switch_to_baseline{100};

bool g_enable_distance_rangejoin{true};

bool g_enable_hashjoin_many_to_many{true};

size_t g_bbox_intersect_max_table_size_bytes{1024 * 1024 * 1024};

double g_bbox_intersect_target_entries_per_bin{1.3};

bool g_strip_join_covered_quals{false};

size_t g_constrained_by_in_threshold{10};

size_t g_default_max_groups_buffer_entry_guess{16384};

size_t g_big_group_threshold{g_default_max_groups_buffer_entry_guess};

size_t g_baseline_groupby_threshold{

1000000}; // if a perfect hash needs more entries, use baseline

bool g_enable_window_functions{true};

bool g_enable_table_functions{true};

bool g_enable_ml_functions{true};

bool g_restrict_ml_model_metadata_to_superusers{false};

bool g_enable_dev_table_functions{false};

bool g_enable_geo_ops_on_uncompressed_coords{true};

bool g_enable_rf_prop_table_functions{true};

bool g_allow_memory_status_log{true};

size_t g_max_memory_allocation_size{2000000000}; // set to max slab size

size_t g_min_memory_allocation_size{

256}; // minimum memory allocation required for projection query output buffer

// without pre-flight count

bool g_enable_bump_allocator{false};

double g_bump_allocator_step_reduction{0.75};

bool g_enable_direct_columnarization{true};

extern bool g_enable_string_functions;

bool g_enable_lazy_fetch{true};

bool g_enable_runtime_query_interrupt{true};

bool g_enable_non_kernel_time_query_interrupt{true};

bool g_use_estimator_result_cache{true};

unsigned g_pending_query_interrupt_freq{1000};

double g_running_query_interrupt_freq{0.1};

size_t g_gpu_smem_threshold{

0}; // GPU shared memory threshold (in bytes).

// If larger buffer sizes are required we do not use GPU shared

// memory optimizations. Setting this to 0 means unlimited

// (subject to other dynamically calculated caps).

bool g_enable_smem_grouped_non_count_agg{

true}; // enable use of shared memory when performing group-by with select non-count

// aggregates

bool g_enable_smem_non_grouped_agg{

true}; // enable optimizations for using GPU shared memory in implementation of

// non-grouped aggregates

bool g_is_test_env{false}; // operating under a unit test environment. Currently only

// limits the allocation for the output buffer arena

// and data recycler test

size_t g_enable_parallel_linearization{

10000}; // # rows that we are trying to linearize varlen col in parallel

bool g_enable_data_recycler{true};

bool g_use_hashtable_cache{true};

bool g_use_query_resultset_cache{true};

bool g_use_chunk_metadata_cache{true};

bool g_allow_auto_resultset_caching{false};

bool g_allow_query_step_skipping{true};

size_t g_hashtable_cache_total_bytes{size_t(1) << 32};

size_t g_max_cacheable_hashtable_size_bytes{size_t(1) << 31};

size_t g_query_resultset_cache_total_bytes{size_t(1) << 32};

size_t g_max_cacheable_query_resultset_size_bytes{size_t(1) << 31};

size_t g_auto_resultset_caching_threshold{size_t(1) << 20};

bool g_optimize_cuda_block_and_grid_sizes{false};

size_t g_approx_quantile_buffer{1000};

size_t g_approx_quantile_centroids{300};

bool g_enable_automatic_ir_metadata{true};

size_t g_max_log_length{500};

bool g_enable_executor_resource_mgr{true};

double g_executor_resource_mgr_cpu_result_mem_ratio{0.8};

size_t g_executor_resource_mgr_cpu_result_mem_bytes{Executor::auto_cpu_mem_bytes};

double g_executor_resource_mgr_per_query_max_cpu_slots_ratio{0.9};

double g_executor_resource_mgr_per_query_max_cpu_result_mem_ratio{0.8};

bool g_executor_resource_mgr_allow_cpu_kernel_concurrency{true};

bool g_executor_resource_mgr_allow_cpu_gpu_kernel_concurrency{true};

bool g_executor_resource_mgr_allow_cpu_slot_oversubscription_concurrency{false};

bool g_executor_resource_mgr_allow_cpu_result_mem_oversubscription_concurrency{false};

double g_executor_resource_mgr_max_available_resource_use_ratio{0.8};

bool g_executor_resource_mgr_allow_auto_shrink_num_cpu_slot_for_groupby_query{true};

bool g_use_cpu_mem_pool_for_output_buffers{true};

extern bool g_cache_string_hash;

extern bool g_allow_memory_status_log;

int const Executor::max_gpu_count;

int g_max_num_gpu_per_query{0};

int Executor::last_selected_device_id_{0};

std::mutex Executor::last_selected_device_id_mutex_;

std::map<Executor::ExtModuleKinds, std::string> Executor::extension_module_sources;

extern std::unique_ptr<llvm::Module> read_llvm_module_from_bc_file(

const std::string& udf_ir_filename,

llvm::LLVMContext& ctx);

extern std::unique_ptr<llvm::Module> read_llvm_module_from_ir_file(

const std::string& udf_ir_filename,

llvm::LLVMContext& ctx,

bool is_gpu = false);

extern std::unique_ptr<llvm::Module> read_llvm_module_from_ir_string(

const std::string& udf_ir_string,

llvm::LLVMContext& ctx,

bool is_gpu = false);

namespace {

void prepare_string_dictionaries(const std::unordered_set<PhysicalInput>& phys_inputs) {

}

bool is_empty_table(Fragmenter_Namespace::AbstractFragmenter* fragmenter) {

}

size_t determineMaxCpuSlabSize(DataMgr* data_mgr, size_t default_max_cpu_slab_size) {

}

} // namespace

namespace foreign_storage {

void populate_string_dictionary(int32_t table_id, int32_t col_id, int32_t db_id) {

}

} // namespace foreign_storage

Executor::Executor(const ExecutorId executor_id,

}

void Executor::initialize_extension_module_sources() {

}

void Executor::reset(bool discard_runtime_modules_only) {

}

void Executor::update_extension_modules(bool update_runtime_modules_only) {

}

Executor::CgenStateManager::CgenStateManager(Executor& executor)

: executor_(executor)

, lock_queue_clock_(timer_start())

, lock_(executor_.compilation_mutex_)

, cgen_state_(std::move(executor_.cgen_state_)) // store old CgenState instance

{

executor_.compilation_queue_time_ms_ += timer_stop(lock_queue_clock_);

executor_.cgen_state_.reset(new CgenState(0, false, &executor));

}

Executor::CgenStateManager::CgenStateManager(

Executor& executor,

const bool allow_lazy_fetch,

const std::vector<InputTableInfo>& query_infos,

const PlanState::DeletedColumnsMap& deleted_cols_map,

const RelAlgExecutionUnit* ra_exe_unit)

: executor_(executor)

, lock_queue_clock_(timer_start())

, lock_(executor_.compilation_mutex_)

, cgen_state_(std::move(executor_.cgen_state_)) // store old CgenState instance

{

executor_.compilation_queue_time_ms_ += timer_stop(lock_queue_clock_);

// nukeOldState creates new CgenState and PlanState instances for

// the subsequent code generation. It also resets

// kernel_queue_time_ms_ and compilation_queue_time_ms_ that we do

// not currently restore.. should we accumulate these timings?

executor_.nukeOldState(allow_lazy_fetch, query_infos, deleted_cols_map, ra_exe_unit);

}

Executor::CgenStateManager::~CgenStateManager() {

// prevent memory leak from hoisted literals

for (auto& p : executor_.cgen_state_->row_func_hoisted_literals_) {

auto inst = llvm::dyn_cast<llvm::LoadInst>(p.first);

if (inst && inst->getNumUses() == 0 && inst->getParent() == nullptr) {

// The llvm::Value instance stored in p.first is created by the

// CodeGenerator::codegenHoistedConstantsPlaceholders method.

p.first->deleteValue();

}

}

executor_.cgen_state_->row_func_hoisted_literals_.clear();

// move generated StringDictionaryTranslationMgrs and InValueBitmaps

// to the old CgenState instance as the execution of the generated

// code uses these bitmaps

for (auto& bm : executor_.cgen_state_->in_values_bitmaps_) {

cgen_state_->moveInValuesBitmap(bm);

}

executor_.cgen_state_->in_values_bitmaps_.clear();

for (auto& str_dict_translation_mgr :

executor_.cgen_state_->str_dict_translation_mgrs_) {

cgen_state_->moveStringDictionaryTranslationMgr(std::move(str_dict_translation_mgr));

}

executor_.cgen_state_->str_dict_translation_mgrs_.clear();

for (auto& tree_model_prediction_mgr :

executor_.cgen_state_->tree_model_prediction_mgrs_) {

cgen_state_->moveTreeModelPredictionMgr(std::move(tree_model_prediction_mgr));

}

executor_.cgen_state_->tree_model_prediction_mgrs_.clear();

// Delete worker module that may have been set by

// set_module_shallow_copy. If QueryMustRunOnCpu is thrown, the

// worker module is not instantiated, so the worker module needs to

// be deleted conditionally [see "Managing LLVM modules" comment in

// CgenState.h]:

if (executor_.cgen_state_->module_) {

delete executor_.cgen_state_->module_;

}

// restore the old CgenState instance

executor_.cgen_state_.reset(cgen_state_.release());

}

std::shared_ptr<Executor> Executor::getExecutor(

}

void Executor::clearMemory(const Data_Namespace::MemoryLevel memory_level) {

}

size_t Executor::getArenaBlockSize() {

return g_is_test_env ? 100000000 : (1UL << 32) + kArenaBlockOverhead;

}

StringDictionaryProxy* Executor::getStringDictionaryProxy(

const shared::StringDictKey& dict_id_in,

std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,

const bool with_generation) const {

CHECK(row_set_mem_owner);

std::lock_guard<std::mutex> lock(

str_dict_mutex_); // TODO: can we use RowSetMemOwner state mutex here?

return row_set_mem_owner->getOrAddStringDictProxy(dict_id_in, with_generation);

}

StringDictionaryProxy* RowSetMemoryOwner::getOrAddStringDictProxy(

const shared::StringDictKey& dict_key_in,

const bool with_generation) {

if (dict_key_in.db_id < 0) {

// Temporary dictionaries should already be created and stored in the

// str_dict_proxy_owned_ map when this method is called.

auto it = str_dict_proxy_owned_.find(dict_key_in);

CHECK(it != str_dict_proxy_owned_.end());

return it->second.get();

}

const int dict_id{dict_key_in.dict_id < 0 ? REGULAR_DICT(dict_key_in.dict_id)

: dict_key_in.dict_id};

const auto catalog =

Catalog_Namespace::SysCatalog::instance().getCatalog(dict_key_in.db_id);

if (catalog) {

const auto dd = catalog->getMetadataForDict(dict_id);

if (dd) {

auto dict_key = dict_key_in;

dict_key.dict_id = dict_id;

CHECK(dd->stringDict);

CHECK_LE(dd->dictNBits, 32);

const int64_t generation =

with_generation ? string_dictionary_generations_.getGeneration(dict_key) : -1;

return addStringDict(dd->stringDict, dict_key, generation);

}

}

CHECK_EQ(dict_id, DictRef::literalsDictId);

if (!lit_str_dict_proxy_) {

DictRef literal_dict_ref(dict_key_in.db_id, DictRef::literalsDictId);

std::shared_ptr<StringDictionary> tsd = std::make_shared<StringDictionary>(

literal_dict_ref, "", false, true, g_cache_string_hash);

lit_str_dict_proxy_ = std::make_shared<StringDictionaryProxy>(

tsd, shared::StringDictKey{literal_dict_ref.dbId, literal_dict_ref.dictId}, 0);

}

return lit_str_dict_proxy_.get();

}

const StringDictionaryProxy::IdMap* Executor::getStringProxyTranslationMap(

const shared::StringDictKey& source_dict_key,

const shared::StringDictKey& dest_dict_key,

const RowSetMemoryOwner::StringTranslationType translation_type,

const std::vector<StringOps_Namespace::StringOpInfo>& string_op_infos,

std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,

const bool with_generation) const {

CHECK(row_set_mem_owner);

std::lock_guard<std::mutex> lock(

str_dict_mutex_); // TODO: can we use RowSetMemOwner state mutex here?

return row_set_mem_owner->getOrAddStringProxyTranslationMap(

source_dict_key, dest_dict_key, with_generation, translation_type, string_op_infos);

}

const StringDictionaryProxy::IdMap*

Executor::getJoinIntersectionStringProxyTranslationMap(

const StringDictionaryProxy* source_proxy,

StringDictionaryProxy* dest_proxy,

const std::vector<StringOps_Namespace::StringOpInfo>& source_string_op_infos,

const std::vector<StringOps_Namespace::StringOpInfo>& dest_string_op_infos,

std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner) const {

CHECK(row_set_mem_owner);

std::lock_guard<std::mutex> lock(

str_dict_mutex_); // TODO: can we use RowSetMemOwner state mutex here?

// First translate lhs onto itself if there are string ops

if (!dest_string_op_infos.empty()) {

row_set_mem_owner->addStringProxyUnionTranslationMap(

dest_proxy, dest_proxy, dest_string_op_infos);

}

return row_set_mem_owner->addStringProxyIntersectionTranslationMap(

source_proxy, dest_proxy, source_string_op_infos);

}

const StringDictionaryProxy::TranslationMap<Datum>*

Executor::getStringProxyNumericTranslationMap(

const shared::StringDictKey& source_dict_key,

const std::vector<StringOps_Namespace::StringOpInfo>& string_op_infos,

std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,

const bool with_generation) const {

CHECK(row_set_mem_owner);

std::lock_guard<std::mutex> lock(

str_dict_mutex_); // TODO: can we use RowSetMemOwner state mutex here?

return row_set_mem_owner->getOrAddStringProxyNumericTranslationMap(

source_dict_key, with_generation, string_op_infos);

}

const StringDictionaryProxy::IdMap* RowSetMemoryOwner::getOrAddStringProxyTranslationMap(

const shared::StringDictKey& source_dict_key_in,

const shared::StringDictKey& dest_dict_key_in,

const bool with_generation,

const RowSetMemoryOwner::StringTranslationType translation_type,

const std::vector<StringOps_Namespace::StringOpInfo>& string_op_infos) {

const auto source_proxy = getOrAddStringDictProxy(source_dict_key_in, with_generation);

const auto dest_proxy = getOrAddStringDictProxy(dest_dict_key_in, with_generation);

if (translation_type == RowSetMemoryOwner::StringTranslationType::SOURCE_INTERSECTION) {

return addStringProxyIntersectionTranslationMap(

source_proxy, dest_proxy, string_op_infos);

} else {

return addStringProxyUnionTranslationMap(source_proxy, dest_proxy, string_op_infos);

}

}

const StringDictionaryProxy::TranslationMap<Datum>*

RowSetMemoryOwner::getOrAddStringProxyNumericTranslationMap(

const shared::StringDictKey& source_dict_key_in,

const bool with_generation,

const std::vector<StringOps_Namespace::StringOpInfo>& string_op_infos) {

const auto source_proxy = getOrAddStringDictProxy(source_dict_key_in, with_generation);

return addStringProxyNumericTranslationMap(source_proxy, string_op_infos);

}

quantile::TDigest* RowSetMemoryOwner::initTDigest(size_t const thread_idx,

ApproxQuantileDescriptor const desc,

double const q) {

static_assert(std::is_trivially_copyable_v<ApproxQuantileDescriptor>);

std::lock_guard<std::mutex> lock(state_mutex_);

auto t_digest = std::make_unique<quantile::TDigest>(

q, &t_digest_allocators_[thread_idx], desc.buffer_size, desc.centroids_size);

return t_digests_.emplace_back(std::move(t_digest)).get();

}

void RowSetMemoryOwner::reserveTDigestMemory(size_t thread_idx, size_t capacity) {

}

bool Executor::isCPUOnly() const {

CHECK(data_mgr_);

return !data_mgr_->getCudaMgr();

}

const ColumnDescriptor* Executor::getColumnDescriptor(

const Analyzer::ColumnVar* col_var) const {

return get_column_descriptor_maybe(col_var->getColumnKey());

}

const ColumnDescriptor* Executor::getPhysicalColumnDescriptor(

const Analyzer::ColumnVar* col_var,

int n) const {

const auto cd = getColumnDescriptor(col_var);

if (!cd || n > cd->columnType.get_physical_cols()) {

return nullptr;

}

auto column_key = col_var->getColumnKey();

column_key.column_id += n;

return get_column_descriptor_maybe(column_key);

}

const std::shared_ptr<RowSetMemoryOwner> Executor::getRowSetMemoryOwner() const {

return row_set_mem_owner_;

}

const TemporaryTables* Executor::getTemporaryTables() const {

return temporary_tables_;

}

Fragmenter_Namespace::TableInfo Executor::getTableInfo(

const shared::TableKey& table_key) const {

return input_table_info_cache_.getTableInfo(table_key);

}

const TableGeneration& Executor::getTableGeneration(

const shared::TableKey& table_key) const {

return table_generations_.getGeneration(table_key);

}

ExpressionRange Executor::getColRange(const PhysicalInput& phys_input) const {

return agg_col_range_cache_.getColRange(phys_input);

}

namespace {

void log_system_memory_info_impl(std::string const& mem_log,

}

} // namespace

void Executor::logSystemCPUMemoryStatus(std::string const& log_tag,

}

void Executor::logSystemGPUMemoryStatus(std::string const& log_tag,

}

namespace {

size_t get_col_byte_width(const shared::ColumnKey& column_key) {

}

} // anonymous namespace

std::map<shared::ColumnKey, size_t> Executor::getColumnByteWidthMap(

}

size_t Executor::getNumBytesForFetchedRow(

}

ExecutorResourceMgr_Namespace::ChunkRequestInfo Executor::getChunkRequestInfo(