{

int ret = OB_SUCCESS;

ObSQLSessionInfo *session = NULL;

if (OB_ISNULL(session = ctx_.get_my_session())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get session failed", K(ret));

} else {

// since we call get_next_row in inner_open, we have to set opened_ first in avoid to a infinite loop.

opened_ = true;

if (OB_FAIL(session->get_sql_select_limit(top_limit_cnt_))) {

LOG_WARN("fail tp get sql select limit", K(ret));

}

}

if (OB_SUCC(ret) && !MY_SPEC.external_properties_.str_.empty()) {

if (OB_FAIL(external_properties_.load_from_string(MY_SPEC.external_properties_.str_,

ctx_.get_allocator()))) {

LOG_WARN("failed to load external properties", K(ret));

} else {

format_type_ = external_properties_.format_type_;

}

}

if (OB_SUCC(ret)) {

switch (format_type_)

{

case ObExternalFileFormat::FormatType::CSV_FORMAT:

{

if (OB_FAIL(init_csv_env())) {

LOG_WARN("failed to init csv env", K(ret));

}

break;

}

case ObExternalFileFormat::FormatType::ODPS_FORMAT:

{

if (!GCONF._use_odps_jni_connector) {

ret = OB_NOT_SUPPORTED;

LOG_WARN("not support odps format", K(ret));

} else {

ret = OB_NOT_SUPPORTED;

LOG_WARN("not support odps format", K(ret));

}

break;

}

case ObExternalFileFormat::FormatType::PARQUET_FORMAT:

{

if (OB_FAIL(init_parquet_env())) {

LOG_WARN("failed to init csv env", K(ret));

}

break;

}

case ObExternalFileFormat::FormatType::ORC_FORMAT:

{

if (OB_FAIL(init_orc_env())) {

LOG_WARN("failed to init csv env", K(ret));

}

break;

}

default:

{

ret = OB_NOT_SUPPORTED;

LOG_WARN("not support select into type", K(format_type_));

}

}

}

return ret;

{

int ret = OB_SUCCESS;

ObSQLSessionInfo *session = NULL;

set_csv_format_options();

if (OB_ISNULL(session = ctx_.get_my_session())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get session failed", K(ret));

} else if (OB_FAIL(init_env_common())) {

LOG_WARN("failed to init env common", K(ret));

} else if (OB_FAIL(prepare_escape_printer())) {

LOG_WARN("failed to calc escape info", K(ret));

} else {

if (external_properties_.csv_format_.compression_algorithm_ != CsvCompressType::NONE) {

has_compress_ = true;

}

// setup binary output format for bit/binary

switch (external_properties_.csv_format_.binary_format_) {

case ObCSVGeneralFormat::ObCSVBinaryFormat::DEFAULT:

print_params_.binary_string_print_hex_ = false;

break;

case ObCSVGeneralFormat::ObCSVBinaryFormat::HEX:

print_params_.binary_string_print_hex_ = true;

break;

case ObCSVGeneralFormat::ObCSVBinaryFormat::BASE64:

print_params_.binary_string_print_base64_ = true;

break;

default:

ret = OB_ERR_UNEXPECTED;

LOG_WARN("failed to set csv binary output format", K(ret));

}

print_params_.tz_info_ = session->get_timezone_info();

print_params_.use_memcpy_ = true;

print_params_.cs_type_ = cs_type_;

}

//create buffer

if (OB_SUCC(ret) && T_INTO_OUTFILE == MY_SPEC.into_type_ && OB_FAIL(create_shared_buffer_for_data_writer())) {

LOG_WARN("failed to create buffer for data writer", K(ret));

}

return ret;

{

if (MY_SPEC.external_properties_.str_.empty()) {

field_str_ = MY_SPEC.field_str_;

line_str_ = MY_SPEC.line_str_;

has_enclose_ = MY_SPEC.closed_cht_.get_val_len() > 0;

char_enclose_ = has_enclose_ ? MY_SPEC.closed_cht_.get_char().ptr()[0] : 0;

is_optional_ = MY_SPEC.is_optional_;

has_escape_ = MY_SPEC.escaped_cht_.get_val_len() > 0;

char_escape_ = has_escape_ ? MY_SPEC.escaped_cht_.get_char().ptr()[0] : 0;

cs_type_ = MY_SPEC.cs_type_;

} else {

is_optional_ = external_properties_.csv_format_.is_optional_;

cs_type_ = ObCharset::get_default_collation(external_properties_.csv_format_.cs_type_);

field_str_.set_varchar(external_properties_.csv_format_.field_term_str_);

field_str_.set_collation_type(cs_type_);

line_str_.set_varchar(external_properties_.csv_format_.line_term_str_);

line_str_.set_collation_type(cs_type_);

if (external_properties_.csv_format_.field_enclosed_char_ == INT64_MAX) { // null

has_enclose_ = false;

char_enclose_ = 0;

} else {

has_enclose_ = true;

char_enclose_ = external_properties_.csv_format_.field_enclosed_char_;

}

if (external_properties_.csv_format_.field_escaped_char_ == INT64_MAX) { // null

has_escape_ = false;

char_escape_ = 0;

} else {

has_escape_ = true;

char_escape_ = external_properties_.csv_format_.field_escaped_char_;

}

}

{

int ret = OB_SUCCESS;

arrow_alloc_.init(MTL_ID());

if (OB_FAIL(setup_parquet_schema())) {

LOG_WARN("failed to set up parquet schema", K(ret));

} else if (OB_FAIL(init_env_common())) {

LOG_WARN("failed to init env common", K(ret));

}

return ret;

{

int ret = OB_SUCCESS;

orc_alloc_.init(MTL_ID());

if (OB_FAIL(setup_orc_schema())) {

LOG_WARN("failed to set up orc schema", K(ret));

} else if (OB_FAIL(init_env_common())) {

LOG_WARN("failed to init env common", K(ret));

} else if (external_properties_.orc_format_.compression_block_size_ < 100) { // parameter guard

ret = OB_INVALID_ARGUMENT;

LOG_WARN("compress block is too low or too high", K(external_properties_.orc_format_.compression_block_size_));

} else {

options_.setStripeSize(external_properties_.orc_format_.stripe_size_)

.setRowIndexStride(external_properties_.orc_format_.row_index_stride_)

.setCompressionBlockSize(external_properties_.orc_format_.compression_block_size_)

.setCompression(static_cast<orc::CompressionKind>(external_properties_.orc_format_.compress_type_index_))

.setMemoryPool(&orc_alloc_);

}

return ret;

{

int ret = OB_SUCCESS;

ObPhysicalPlanCtx *phy_plan_ctx = NULL;

bool need_check = false;

file_name_ = MY_SPEC.outfile_name_;

do_partition_ = MY_SPEC.file_partition_expr_ == NULL ? false : true;

if (OB_ISNULL(phy_plan_ctx = ctx_.get_physical_plan_ctx())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get phy_plan_ctx failed", K(ret));

} else if (OB_FAIL(ObSQLUtils::get_param_value(MY_SPEC.outfile_name_,

phy_plan_ctx->get_param_store(),

file_name_,

need_check))) {

LOG_WARN("get param value failed", K(ret));

} else if (OB_FAIL(calc_url_and_set_access_info())) {

LOG_WARN("failed to calc basic url and set device handle", K(ret));

} else if (OB_FAIL(check_has_lob_or_json())) {

LOG_WARN("failed to check has lob", K(ret));

} else if (has_coll_ && MY_SPEC.into_type_ == T_INTO_VARIABLES) {

ret = OB_NOT_SUPPORTED;

LOG_USER_ERROR(OB_NOT_SUPPORTED, "select array/map into variables");

} else if (do_partition_

&& OB_FAIL(partition_map_.create(128, ObLabel("SelectInto"), ObLabel("SelectInto"), MTL_ID()))) {

LOG_WARN("failed to create hashmap", K(ret));

} else if (MY_SPEC.select_exprs_.count() != MY_SPEC.alias_names_.strs_.count()) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected column count", K(MY_SPEC.select_exprs_.count()),

K(MY_SPEC.alias_names_.strs_.count()), K(ret));

}

return ret;

{

int ret = OB_SUCCESS;

const ObItemType into_type = MY_SPEC.into_type_;

ObString path = file_name_.get_varchar().trim();

if (path.prefix_match_ci(OB_S3_PREFIX)) {

file_location_ = IntoFileLocation::REMOTE_S3;

} else if (path.prefix_match_ci(OB_AZBLOB_PREFIX)) {

file_location_ = IntoFileLocation::REMOTE_AZBLOB;

} else {

file_location_ = IntoFileLocation::SERVER_DISK;

}

if (file_location_ == IntoFileLocation::SERVER_DISK && do_partition_) {

ret = OB_NOT_SUPPORTED;

LOG_WARN("not support partition option on server disk", K(ret));

LOG_USER_ERROR(OB_NOT_SUPPORTED, "partition option on server disk");

} else if (T_INTO_OUTFILE == into_type && !MY_SPEC.is_single_ && OB_FAIL(calc_first_file_path(path))) {

LOG_WARN("failed to calc first file path", K(ret));

} else if (file_location_ != IntoFileLocation::SERVER_DISK) {

ObString temp_url = path.split_on('?');

temp_url.trim();

ObString storage_info;

if (OB_FAIL(ob_write_string(ctx_.get_allocator(), temp_url, basic_url_, true))) {

LOG_WARN("failed to append string", K(ret));

} else if (OB_FAIL(ob_write_string(ctx_.get_allocator(), path, storage_info, true))) {

LOG_WARN("failed to append string", K(ret));

} else if (OB_FAIL(access_info_.set(basic_url_.ptr(), storage_info.ptr()))) {

LOG_WARN("failed to set access info", K(ret), K(path));

} else if (basic_url_.empty() || !access_info_.is_valid()) {

ret = OB_FILE_NOT_EXIST;

LOG_WARN("file path not exist", K(ret), K(basic_url_), K(access_info_));

}

} else { // IntoFileLocation::SERVER_DISK

if (OB_FAIL(ob_write_string(ctx_.get_allocator(), path, basic_url_, true))) {

LOG_WARN("failed to write string", K(ret));

}

}

if (OB_SUCC(ret) && (T_INTO_OUTFILE == into_type || T_INTO_DUMPFILE == into_type)

&& IntoFileLocation::SERVER_DISK == file_location_ && OB_FAIL(check_secure_file_path(basic_url_))) {

LOG_WARN("failed to check secure file path", K(ret));

}

return ret;

{

int ret = 0 == top_limit_cnt_ ? OB_ITER_END : OB_SUCCESS;

int64_t row_count = 0;

const ObItemType into_type = MY_SPEC.into_type_;

ObPhysicalPlanCtx *phy_plan_ctx = NULL;

ObExternalFileWriter *data_writer = NULL;

if (ObExternalFileFormat::FormatType::CSV_FORMAT != format_type_

&& ObExternalFileFormat::FormatType::ODPS_FORMAT != format_type_) {

ret = OB_NOT_SUPPORTED;

LOG_WARN("this type not supported in not batch interface", K(ret), K(format_type_));

LOG_USER_ERROR(OB_NOT_SUPPORTED, "this upload type");

} else if (OB_ISNULL(phy_plan_ctx = ctx_.get_physical_plan_ctx())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get phy_plan_ctx failed", K(ret));

}

//when do_partition is false, create the only data_writer here

if (OB_SUCC(ret) && ObExternalFileFormat::FormatType::CSV_FORMAT == format_type_

&& T_INTO_VARIABLES != into_type && !do_partition_

&& OB_FAIL(create_the_only_data_writer(data_writer))) {

LOG_WARN("failed to create the only data writer", K(ret));

}

while (OB_SUCC(ret) && row_count < top_limit_cnt_) {

clear_evaluated_flag();

if (OB_FAIL(child_->get_next_row())) {

if (OB_LIKELY(OB_ITER_END == ret)) {

} else {

LOG_WARN("get next row failed", K(ret));

}

} else {

++row_count;

if (ObExternalFileFormat::FormatType::ODPS_FORMAT == format_type_) {

if (is_odps_cpp_table_ == is_odps_java_table_) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("invalid table mode for odps table", K(ret),

K(is_odps_cpp_table_), K(is_odps_java_table_));

} else if (is_odps_cpp_table_) {

ret = OB_NOT_SUPPORTED;

LOG_USER_ERROR(OB_NOT_SUPPORTED, "external odps cpp table");

LOG_WARN("use supported version", K(ret));

} else {

ret = OB_NOT_SUPPORTED;

LOG_USER_ERROR(OB_NOT_SUPPORTED, "external odps table");

LOG_WARN("not support jni odps single write", K(ret));

}

} else if (T_INTO_VARIABLES == into_type) {

if (OB_FAIL(into_varlist())) {

LOG_WARN("into varlist failed", K(ret));

}

} else if (T_INTO_OUTFILE == into_type) {

if (OB_FAIL(into_outfile(data_writer))) {

LOG_WARN("into outfile failed", K(ret));

}

} else {

if (OB_FAIL(into_dumpfile(data_writer))) {

LOG_WARN("into dumpfile failed", K(ret));

}

}

}

if (OB_SUCC(ret) || OB_ITER_END == ret) { // if into user variables or into dumpfile, must be one row

if (ObExternalFileFormat::FormatType::CSV_FORMAT == format_type_

&& (T_INTO_VARIABLES == into_type || T_INTO_DUMPFILE == into_type) && row_count > 1) {

ret = OB_ERR_TOO_MANY_ROWS;

LOG_WARN("more than one row for into variables or into dumpfile", K(ret), K(row_count));

}

}

} //end while

if (OB_ITER_END == ret || OB_SUCC(ret)) { // set affected rows

phy_plan_ctx->set_affected_rows(row_count);

}

if (OB_FAIL(ret) && OB_ITER_END != ret) {

need_commit_ = false;

}

return ret;

{

int ret = OB_SUCCESS;

const ObBatchRows *child_brs = NULL;

int64_t batch_size = min(max_row_cnt, MY_SPEC.max_batch_size_);

int64_t row_count = 0;

const ObItemType into_type = MY_SPEC.into_type_;

ObPhysicalPlanCtx *phy_plan_ctx = NULL;

ObExternalFileWriter *data_writer = NULL;

bool stop_loop = false;

bool is_iter_end = false;

if (OB_ISNULL(phy_plan_ctx = ctx_.get_physical_plan_ctx())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get phy_plan_ctx failed", K(ret));

}

//when do_partition is false, create the only data_writer here

if (OB_SUCC(ret) && T_INTO_VARIABLES != into_type && !do_partition_

&& (ObExternalFileFormat::FormatType::CSV_FORMAT == format_type_

|| ObExternalFileFormat::FormatType::PARQUET_FORMAT == format_type_

|| ObExternalFileFormat::FormatType::ORC_FORMAT == format_type_)) {

if (OB_FAIL(create_the_only_data_writer(data_writer))) {

LOG_WARN("failed to create the only data writer", K(ret));

} else if (OB_ISNULL(data_writer)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null", K(ret));

}

}

if (0 == top_limit_cnt_) {

brs_.size_ = 0;

brs_.end_ = true;

stop_loop = true;

}

while (OB_SUCC(ret) && !stop_loop) {

clear_evaluated_flag();

int64_t rowkey_batch_size = min(batch_size, top_limit_cnt_ - row_count);

if (OB_FAIL(child_->get_next_batch(rowkey_batch_size, child_brs))) {

LOG_WARN("get next batch failed", K(ret));

} else {

brs_.size_ = child_brs->size_;

brs_.end_ = child_brs->end_;

is_iter_end = brs_.end_ && 0 == brs_.size_;

if (brs_.size_ > 0) {

brs_.skip_->deep_copy(*(child_brs->skip_), brs_.size_);

row_count += brs_.size_ - brs_.skip_->accumulate_bit_cnt(brs_.size_);

if (ObExternalFileFormat::FormatType::ODPS_FORMAT == format_type_) {

if (!GCONF._use_odps_jni_connector) {

ret = OB_NOT_SUPPORTED;

LOG_WARN("odps cpp connector is not supported", K(ret));

} else {

ret = OB_NOT_SUPPORTED;

LOG_WARN("odps jni connector is not supported", K(ret));

}

} else if (T_INTO_OUTFILE == into_type) {

if (ObExternalFileFormat::FormatType::CSV_FORMAT == format_type_) {

if (OB_FAIL(into_outfile_batch_csv(brs_, data_writer))) {

LOG_WARN("csv into outfile batch failed", K(ret));

}

} else if (ObExternalFileFormat::FormatType::PARQUET_FORMAT == format_type_) {

if (OB_FAIL(into_outfile_batch_parquet(brs_, data_writer))) {

LOG_WARN("parquet into outfile batch failed", K(ret));

}

} else if (ObExternalFileFormat::FormatType::ORC_FORMAT == format_type_) {

if (OB_FAIL(into_outfile_batch_orc(brs_, data_writer))) {

LOG_WARN("orc into outfile batch failed", K(ret));

}

} else {

ret = OB_NOT_SUPPORTED;

LOG_WARN("not support to write into outfile format.", K(ret), K(format_type_));

}

} else {

ObEvalCtx::BatchInfoScopeGuard guard(eval_ctx_);

guard.set_batch_size(brs_.size_);

for (int64_t i = 0; OB_SUCC(ret) && i < brs_.size_; i++) {

if (brs_.skip_->contain(i)) {

continue;

}

guard.set_batch_idx(i);

if (T_INTO_VARIABLES == into_type) {

if (OB_FAIL(into_varlist())) {

LOG_WARN("into varlist failed", K(ret));

}

} else {

if (OB_FAIL(into_dumpfile(data_writer))) {

LOG_WARN("into dumpfile failed", K(ret));

}

}

}

}

}

}

if (is_iter_end || row_count >= top_limit_cnt_) {

stop_loop = true;

}

if (OB_SUCC(ret) || is_iter_end) { // if into user variables or into dumpfile, must be one row

if ((T_INTO_VARIABLES == into_type || T_INTO_DUMPFILE == into_type) && row_count > 1) {

ret = OB_ERR_TOO_MANY_ROWS;

LOG_WARN("more than one row for into variables or into dumpfile", K(ret), K(row_count));

}

}

} //end while

if (OB_SUCC(ret)) { // set affected rows

phy_plan_ctx->set_affected_rows(row_count);

}

if (OB_FAIL(ret)) {

need_commit_ = false;

}

return ret;

{

int ret = OB_SUCCESS;

return ret;

{

int ret = OB_SUCCESS;

ObExternalFileWriter *data_writer = NULL;

int64_t estimated_bytes = 0;

if (ObExternalFileFormat::FormatType::ODPS_FORMAT == format_type_) {

if (!GCONF._use_odps_jni_connector) {

ret = OB_NOT_SUPPORTED;

LOG_WARN("odps jni connector is not supported", K(ret));

} else {

ret = OB_NOT_SUPPORTED;

LOG_WARN("odps jni connector is not supported", K(ret));

}

} else if (do_partition_) {

for (ObPartitionWriterMap::iterator iter = partition_map_.begin();

OB_SUCC(ret) && iter != partition_map_.end(); iter++) {

if (OB_ISNULL(data_writer = iter->second)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("data writer is unexpected null", K(ret));

} else if (OB_FAIL(data_writer->close_data_writer())) {

LOG_WARN("failed to close data writer", K(ret));

}

}

} else if (OB_NOT_NULL(data_writer_) && OB_FAIL(data_writer_->close_data_writer())) {

LOG_WARN("failed to close data writer", K(ret));

}

return ret;

bool is_first_row,

char *buf,

int64_t &pos)

{

int ret = OB_SUCCESS;

const ObObj &field_str = field_str_;

char closed_cht = char_enclose_;

//before 4_1 use output

//after 4_1 use select exprs

const ObIArray<ObExpr*> &select_exprs = (MY_SPEC.select_exprs_.empty()) ?

MY_SPEC.output_ : MY_SPEC.select_exprs_;

if (!is_first_row && line_str_.is_varying_len_char_type()) { // lines terminated by "a"

ret = databuff_printf(buf, buf_len, pos, "%.*s", line_str_.get_varchar().length(),

line_str_.get_varchar().ptr());

}

for (int i = 0 ; OB_SUCC(ret) && i < select_exprs.count() ; i++) {

const ObExpr *expr = select_exprs.at(i);

if (0 != closed_cht && (!is_optional_ || ob_is_string_type(expr->datum_meta_.type_))) {

// closed by "a" (for all cell) or optionally by "a" (for string cell)

if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%c", closed_cht))) {

LOG_WARN("print closed character failed", K(ret), K(closed_cht));

}

}

if (OB_SUCC(ret)) {

ObObj cell;

ObDatum *datum = NULL;

if (OB_FAIL(expr->eval(eval_ctx_, datum))) {

LOG_WARN("expr eval failed", K(ret));

} else if (OB_FAIL(datum->to_obj(cell, expr->obj_meta_))) {

LOG_WARN("to obj failed", K(ret));

} else if (OB_FAIL(cell.print_plain_str_literal(buf, buf_len, pos))) { // cell value

LOG_WARN("print sql failed", K(ret), K(cell));

} else if (0 != closed_cht && (!is_optional_ || ob_is_string_type(expr->datum_meta_.type_))) {

if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%c", closed_cht))) {

LOG_WARN("print closed character failed", K(ret), K(closed_cht));

}

}

// field terminated by "a"

if (OB_SUCC(ret) && i != select_exprs.count() - 1 && field_str.is_varying_len_char_type()) {

if (OB_FAIL(databuff_printf(buf, buf_len, pos, "%.*s", field_str.get_varchar().length(), field_str.get_varchar().ptr()))) {

LOG_WARN("print field str failed", K(ret), K(field_str));

}

}

}

}

return ret;

{

int ret = OB_SUCCESS;

ObSqlString file_name_with_suffix;

ObString file_extension;

ObSelectIntoOpInput *input = static_cast<ObSelectIntoOpInput*>(input_);

ObString input_file_name = file_location_ != IntoFileLocation::SERVER_DISK

? path.split_on('?').trim()

: path;

if (OB_ISNULL(input)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("op input is null", K(ret));

} else if (input_file_name.length() == 0 || path.length() == 0) {

ret = OB_INVALID_ARGUMENT;

LOG_USER_ERROR(OB_INVALID_ARGUMENT, "invalid outfile path");

LOG_WARN("invalid outfile path", K(ret));

} else {

if (input_file_name.ptr()[input_file_name.length() - 1] == '/'){

OZ(file_name_with_suffix.append_fmt("%.*sdata", input_file_name.length(), input_file_name.ptr()));

} else {

OZ(file_name_with_suffix.append_fmt("%.*s", input_file_name.length(), input_file_name.ptr()));

}

if (MY_SPEC.parallel_ > 1) {

OZ(file_name_with_suffix.append_fmt("_%ld_%ld_%d", input->sqc_id_, input->task_id_, 0));

} else {

OZ(file_name_with_suffix.append_fmt("_%d", 0));

}

OZ(external_properties_.get_format_file_extension(format_type_, file_extension));

if (!file_extension.empty() && file_extension.ptr()[0] != '.') {

OZ(file_name_with_suffix.append("."));

}

OZ(file_name_with_suffix.append(file_extension));

if (format_type_ == ObExternalFileFormat::FormatType::CSV_FORMAT) {

OZ(file_name_with_suffix.append(compression_algorithm_to_suffix(external_properties_.csv_format_.compression_algorithm_)));

}

if (file_location_ != IntoFileLocation::SERVER_DISK) {

OZ(file_name_with_suffix.append_fmt("?%.*s", path.length(), path.ptr()));

}

if (OB_SUCC(ret) && OB_FAIL(ob_write_string(ctx_.get_allocator(), file_name_with_suffix.string(), path))) {

LOG_WARN("failed to write string", K(ret));

}

}

return ret;

{

int ret = OB_SUCCESS;

ObSqlString url_with_suffix;

ObString file_path;

data_writer.split_file_id_++;

if (data_writer.split_file_id_ > 0) {

if (MY_SPEC.is_single_ && IntoFileLocation::SERVER_DISK != file_location_) {

file_path = (data_writer.split_file_id_ > 1)

? data_writer.url_.split_on(data_writer.url_.reverse_find('.'))

: data_writer.url_;

if (OB_FAIL(url_with_suffix.assign(file_path))) {

LOG_WARN("failed to assign string", K(ret));

} else if (OB_FAIL(url_with_suffix.append_fmt(".extend%ld", data_writer.split_file_id_))) {

LOG_WARN("failed to append string", K(ret));

}

} else if (!MY_SPEC.is_single_) {

file_path = data_writer.url_.split_on(data_writer.url_.reverse_find('_'));

if (OB_FAIL(url_with_suffix.assign(file_path))) {

LOG_WARN("failed to assign string", K(ret));

} else if (OB_FAIL(url_with_suffix.append_fmt("_%ld", data_writer.split_file_id_))) {

LOG_WARN("failed to append string", K(ret));

}

} else {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected single value", K(ret));

}

if (!MY_SPEC.is_single_) {

ObString file_extension;

OZ(external_properties_.get_format_file_extension(format_type_, file_extension));

if (!file_extension.empty() && file_extension.ptr()[0] != '.') {

OZ(url_with_suffix.append("."));

}

OZ(url_with_suffix.append(file_extension));

}

if (!MY_SPEC.is_single_

&& format_type_ == ObExternalFileFormat::FormatType::CSV_FORMAT) {

OZ(url_with_suffix.append(compression_algorithm_to_suffix(external_properties_.csv_format_.compression_algorithm_)));

}

if (OB_SUCC(ret) && OB_FAIL(ob_write_string(ctx_.get_allocator(),

url_with_suffix.string(),

data_writer.url_, true))) {

LOG_WARN("failed to write string", K(ret));

}

} else {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected split file id", K(ret));

}

return ret;

{

int ret = OB_SUCCESS;

ObSqlString url_with_partition;

ObString dir_path;

if (OB_FAIL(ob_write_string(ctx_.get_allocator(), basic_url_, data_writer.url_))) {

LOG_WARN("failed to write string", K(ret));

} else {

dir_path = data_writer.url_.split_on(data_writer.url_.reverse_find('/'));

if (OB_FAIL(url_with_partition.assign(dir_path))) {

LOG_WARN("failed to assign string", K(ret));

} else if (url_with_partition.length() != 0 && OB_FAIL(url_with_partition.append("/"))) {

LOG_WARN("failed to append string", K(ret));

} else if (partition.length() != 0 && OB_FAIL(url_with_partition.append_fmt("%.*s/",

partition.length(),

partition.ptr()))) {

LOG_WARN("failed to append string", K(ret));

} else if (partition.length() == 0 && OB_FAIL(url_with_partition.append("__NULL__/"))) {

LOG_WARN("failed to append string", K(ret));

} else if (OB_FAIL(url_with_partition.append_fmt("%.*s",

data_writer.url_.length(),

data_writer.url_.ptr()))) {

LOG_WARN("failed to append string", K(ret));

} else if (OB_FAIL(ob_write_string(ctx_.get_allocator(),

url_with_partition.string(),

data_writer.url_,

true))) {

LOG_WARN("failed to write string", K(ret));

}

}

return ret;

{

int ret = OB_SUCCESS;

if (ObExternalFileFormat::FormatType::CSV_FORMAT == format_type_) {

ObCsvFileWriter *csv_data_writer = static_cast<ObCsvFileWriter*>(&data_writer);

if (OB_ISNULL(csv_data_writer)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null data writer", K(ret));

} else if (!use_shared_buf_ && OB_FAIL(csv_data_writer->flush_buf())) {

LOG_WARN("failed to flush buffer", K(ret));

} else if (has_lob_ && use_shared_buf_ && OB_FAIL(csv_data_writer->flush_shared_buf(shared_buf_))) {

// To ensure the integrity of each line in the file, when there is a lob, the shared buffer may not contain a complete line

// Therefore the remaining content in the shared buffer also needs to be flushed to the current file, in this case, the max_file_size limit cannot be strictly enforced

LOG_WARN("failed to flush shared buffer", K(ret));

}

}

if (OB_FAIL(ret)) {

} else if (OB_FAIL(data_writer.close_file())) {

LOG_WARN("failed to close file", K(ret));

} else if (OB_FAIL(calc_next_file_path(data_writer))) {

LOG_WARN("failed to calculate new file path", K(ret));

}

return ret;

{

int ret = OB_SUCCESS;

int64_t curr_bytes = data_writer.get_file_size();

int64_t curr_bytes_exclude_curr_line = data_writer.get_curr_bytes_exclude_curr_line();

int64_t curr_line_len = curr_bytes - curr_bytes_exclude_curr_line;

bool has_split = false;

bool has_use_shared_buf = use_shared_buf_;

if (has_compress_ && OB_ISNULL(data_writer.get_compress_stream_writer())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null compress stream writer", K(ret));

} else if (!(has_lob_ && has_use_shared_buf) && curr_bytes_exclude_curr_line == 0) {

} else if (file_need_split(curr_bytes)) {

if (OB_FAIL(split_file(data_writer))) {

LOG_WARN("failed to split file", K(ret));

} else {

has_split = true;

}

}

if (OB_SUCC(ret)) {

if (has_lob_ && has_use_shared_buf) {

if (!has_compress_) {

data_writer.set_write_bytes(has_split ? 0 : curr_bytes);

}

data_writer.reset_curr_line_len();

} else {

if (!has_compress_) {

data_writer.set_write_bytes(has_split ? curr_line_len : curr_bytes);

}

}

if (has_compress_ && has_split) {

data_writer.get_compress_stream_writer()->reuse();

}

data_writer.update_last_line_pos();

}

return ret;

{

int ret = OB_SUCCESS;

buf = use_shared_buf_ ? get_shared_buf() : data_writer.get_buf();

buf_len = use_shared_buf_ ? get_shared_buf_len() : data_writer.get_buf_len();

pos = data_writer.get_curr_pos();

if (OB_ISNULL(buf) && !use_shared_buf_ && OB_FAIL(use_shared_buf(data_writer, buf, buf_len, pos))) {

LOG_WARN("failed to use shared buffer", K(ret));

} else if (OB_ISNULL(buf)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("buf should not be null", K(ret));

}

return ret;

char* &buf,

int64_t &buf_len,

int64_t &pos)

{

int ret = OB_SUCCESS;

int64_t curr_pos = data_writer.get_curr_pos();

if (!use_shared_buf_ && data_writer.get_last_line_pos() == 0) {

if (OB_NOT_NULL(data_writer.get_buf()) && curr_pos > 0) {

MEMCPY(shared_buf_, data_writer.get_buf(), curr_pos);

}

use_shared_buf_ = true;

buf = shared_buf_;

buf_len = shared_buf_len_;

pos = curr_pos;

} else {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("last line should be flushed before this line copied", K(ret));

}

return ret;

int64_t &buf_len,

int64_t &pos,

int64_t curr_pos,

bool is_json)

{

int ret = OB_SUCCESS;

int64_t new_buf_len = buf_len * 2;

char* new_buf = NULL;

if (OB_ISNULL(new_buf = static_cast<char*>(ctx_.get_allocator().alloc(new_buf_len)))) {

ret = OB_ALLOCATE_MEMORY_FAILED;

LOG_WARN("failed to allocate buffer", K(ret), K(new_buf_len));

} else if (!is_json) {

if (curr_pos > 0) {

MEMCPY(new_buf, shared_buf_, curr_pos);

}

shared_buf_ = new_buf;

shared_buf_len_ = new_buf_len;

} else {

json_buf_ = new_buf;

json_buf_len_ = new_buf_len;

}

if (OB_SUCC(ret)) {

buf = new_buf;

buf_len = new_buf_len;

pos = is_json ? 0 : curr_pos;

}

return ret;

char* &buf,

int64_t &buf_len,

int64_t &pos)

{

int ret = OB_SUCCESS;

if (!use_shared_buf_) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get invalid argument", K(use_shared_buf_), K(ret));

} else if (has_lob_ && data_writer.get_curr_pos() > 0) {

if (OB_FAIL(data_writer.flush_shared_buf(shared_buf_, true))) {

LOG_WARN("failed to flush shared buffer", K(ret));

} else {

pos = 0;

}

} else if (OB_FAIL(resize_buf(buf, buf_len, pos, data_writer.get_curr_pos()))) {

LOG_WARN("failed to resize shared buffer", K(ret));

}

return ret;

char* &buf,

int64_t &buf_len,

int64_t &pos,

int64_t str_len)

{

int ret = OB_SUCCESS;

if (buf_len < str_len * 1.1) {

if (OB_FAIL(data_writer.flush_buf())) {

LOG_WARN("failed to flush buffer", K(ret));

} else if (OB_FAIL(use_shared_buf(data_writer, buf, buf_len, pos))) {

LOG_WARN("failed to use shared buffer", K(ret));

}

}

return ret;

{

int ret = OB_SUCCESS;

// binary collation do not require to escape when encode with base64/hex

if (obj.get_collation_type() == CS_TYPE_BINARY &&

(print_params_.binary_string_print_hex_ || print_params_.binary_string_print_base64_)) {

need_escape = false;

}

if ((obj.is_string_type() || obj.is_json() || obj.is_collection_sql_type()) && need_escape) {

if (OB_FAIL(print_str_or_json_with_escape(obj, data_writer))) {

LOG_WARN("failed to print str or json with escape", K(ret));

}

} else if (OB_FAIL(print_normal_obj_without_escape(obj, data_writer))) {

LOG_WARN("failed to print normal obj without escape", K(ret));

}

return ret;

{

int ret = OB_SUCCESS;

char* buf = NULL;

int64_t buf_len = 0;

int64_t pos = 0;

ObCharsetType src_type = ObCharset::charset_type_by_coll(obj.get_collation_type());

ObCharsetType dst_type = ObCharset::charset_type_by_coll(cs_type_);

escape_printer_.do_encode_ = !(src_type == CHARSET_BINARY || src_type == dst_type

|| src_type == CHARSET_INVALID);

escape_printer_.need_enclose_ = has_enclose_ && !obj.is_null();

escape_printer_.do_escape_ = true;

escape_printer_.print_hex_ = obj.get_collation_type() == CS_TYPE_BINARY

&& print_params_.binary_string_print_hex_;

ObString str_to_escape;

ObEvalCtx::TempAllocGuard tmp_alloc_g(eval_ctx_);

common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator();

if (OB_FAIL(get_buf(escape_printer_.buf_, escape_printer_.buf_len_, escape_printer_.pos_, data_writer))) {

LOG_WARN("failed to get buffer", K(ret));

} else if (obj.is_json() || obj.is_collection_sql_type()) {

ObObj inrow_obj = obj;

if (obj.is_lob_storage()

&& OB_FAIL(ObTextStringIter::convert_outrow_lob_to_inrow_templob(obj, inrow_obj, NULL, &temp_allocator))) {

LOG_WARN("failed to convert outrow lobs", K(ret), K(obj));

} else if (obj.is_collection_sql_type()) {

ObSubSchemaValue sub_meta;

if (OB_FAIL((get_exec_ctx().get_sqludt_meta_by_subschema_id(obj.get_meta().get_subschema_id(), sub_meta)))) {

LOG_WARN("failed to get collection subschema", K(ret), K(obj.get_meta().get_subschema_id()));

} else {

print_params_.coll_meta_ = reinterpret_cast<ObSqlCollectionInfo *>(sub_meta.value_);

}

}

if (OB_FAIL(ret)) {

} else if (OB_FAIL(print_json_to_json_buf(inrow_obj, buf, buf_len, pos, data_writer))) {

LOG_WARN("failed to print normal obj without escape", K(ret));

} else {

str_to_escape.assign_ptr(buf, pos);

escape_printer_.do_encode_ = false;

}

} else {

str_to_escape = obj.get_varchar();

}

if (OB_SUCC(ret) && !use_shared_buf_ && OB_FAIL(check_buf_sufficient(data_writer,

escape_printer_.buf_,

escape_printer_.buf_len_,

escape_printer_.pos_,

str_to_escape.length()))) {

LOG_WARN("failed to check if buf is sufficient", K(ret));

}

if (OB_SUCC(ret) && !use_shared_buf_) {

if (OB_FAIL(ObFastStringScanner::foreach_char(str_to_escape,

src_type,

escape_printer_,

escape_printer_.do_encode_,

escape_printer_.ignore_convert_failed_))) {

if (OB_SIZE_OVERFLOW != ret) {

LOG_WARN("failed to print plain str", K(ret), K(src_type), K(escape_printer_.do_encode_));

} else if (OB_FAIL(data_writer.flush_buf())) {

LOG_WARN("failed to flush buffer", K(ret));

} else if (OB_FALSE_IT(escape_printer_.pos_ = data_writer.get_curr_pos())) {

} else if (OB_FAIL(ObFastStringScanner::foreach_char(str_to_escape,

src_type,

escape_printer_,

escape_printer_.do_encode_,

escape_printer_.ignore_convert_failed_))) {

if (OB_SIZE_OVERFLOW != ret) {

LOG_WARN("failed to print plain str", K(ret), K(src_type), K(escape_printer_.do_encode_));

} else if (OB_FAIL(use_shared_buf(data_writer,

escape_printer_.buf_,

escape_printer_.buf_len_,

escape_printer_.pos_))) {

LOG_WARN("failed to use shared buffer", K(ret));

}

}

}

}

if (OB_SUCC(ret) && use_shared_buf_) {

do {

if (OB_FAIL(ObFastStringScanner::foreach_char(str_to_escape,

src_type,

escape_printer_,

escape_printer_.do_encode_,

escape_printer_.ignore_convert_failed_))) {

LOG_WARN("failed to print plain str", K(ret), K(src_type), K(escape_printer_.do_encode_));

}

} while (OB_SIZE_OVERFLOW == ret && OB_SUCC(resize_or_flush_shared_buf(data_writer,

escape_printer_.buf_,

escape_printer_.buf_len_,

escape_printer_.pos_)));

if (OB_FAIL(ret)) {

LOG_WARN("failed to print plain str", K(ret));

}

}

if (OB_SUCC(ret)) {

data_writer.set_curr_pos(escape_printer_.pos_);

}

return ret;