const AzureBlobStorage::RequestSettings & settings, const ReadSettings & read_settings)

{

/// Only settings that backup Azure IO actually consumes are reported. Disk/list-only fields

/// (read_only, list_object_keys_size) are intentionally omitted, as are the http_keep_alive_*

/// fields (which come from server settings, not RequestSettings), so the map is not misleading

/// about what the backup engine effectively uses.

return {

{"use_native_copy", settings.use_native_copy ? "1" : "0"},

{"check_objects_after_upload", settings.check_objects_after_upload ? "1" : "0"},

{"max_single_part_upload_size", std::to_string(settings.max_single_part_upload_size)},

/// Backup reads use ReadBufferFromAzureBlobStorage, whose seek coalescing uses the read

/// setting (remote_read_min_bytes_for_seek), not RequestSettings::min_bytes_for_seek.

{"min_bytes_for_seek", std::to_string(read_settings.remote_fs_settings.min_bytes_for_seek)},

{"max_single_read_retries", std::to_string(settings.max_single_read_retries)},

{"max_single_download_retries", std::to_string(settings.max_single_download_retries)},

{"min_upload_part_size", std::to_string(settings.min_upload_part_size)},

{"max_upload_part_size", std::to_string(settings.max_upload_part_size)},

{"max_single_part_copy_size", std::to_string(settings.max_single_part_copy_size)},

{"max_unexpected_write_error_retries", std::to_string(settings.max_unexpected_write_error_retries)},

{"max_inflight_parts_for_one_file", std::to_string(settings.max_inflight_parts_for_one_file)},

{"max_blocks_in_multipart_upload", std::to_string(settings.max_blocks_in_multipart_upload)},

{"strict_upload_part_size", std::to_string(settings.strict_upload_part_size)},

{"upload_part_size_multiply_factor", std::to_string(settings.upload_part_size_multiply_factor)},

{"upload_part_size_multiply_parts_count_threshold", std::to_string(settings.upload_part_size_multiply_parts_count_threshold)},

{"sdk_max_retries", std::to_string(settings.sdk_max_retries)},

{"sdk_retry_initial_backoff_ms", std::to_string(settings.sdk_retry_initial_backoff_ms)},

{"sdk_retry_max_backoff_ms", std::to_string(settings.sdk_retry_max_backoff_ms)},

};

const AzureBlobStorage::ConnectionParams & connection_params_,

const String & blob_path_,

bool allow_azure_native_copy,

const ReadSettings & read_settings_,

const WriteSettings & write_settings_,

const ContextPtr & context_)

: BackupReaderDefault(read_settings_, write_settings_, getLogger("BackupReaderAzureBlobStorage"))

, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURL(), false, false, ""}

, connection_params(connection_params_)

, blob_path(blob_path_)

{

auto client_ptr = AzureBlobStorage::getContainerClient(connection_params, /*readonly=*/ false);

auto settings_ptr = AzureBlobStorage::getRequestSettingsForBackup(context_, connection_params.endpoint.storage_account_url, allow_azure_native_copy);

object_storage = std::make_unique<AzureObjectStorage>(

"BackupReaderAzureBlobStorage",

connection_params.auth_method,

std::move(client_ptr),

std::move(settings_ptr),

connection_params,

connection_params.getContainer(),

connection_params.getConnectionURL(),

/*common_key_prefix*/ "");

client = object_storage->getAzureBlobStorageClient();

settings = object_storage->getSettings();

{

String key = fs::path(blob_path) / file_name;

return object_storage->exists(StoredObject(key));

{

String key = fs::path(blob_path) / file_name;

ObjectMetadata object_metadata = object_storage->getObjectMetadata(key, /*with_tags=*/ false);

return object_metadata.size_bytes;

{

String key = fs::path(blob_path) / file_name;

return std::make_unique<ReadBufferFromAzureBlobStorage>(

client, key, read_settings, settings->max_single_read_retries,

settings->max_single_download_retries);

DiskPtr destination_disk, const String & destination_path, WriteMode write_mode)

{

auto destination_data_source_description = destination_disk->getDataSourceDescription();

LOG_TRACE(log, "Source description {}, destination description {}", data_source_description.description, destination_data_source_description.description);

if (destination_data_source_description.object_storage_type == ObjectStorageType::Azure

&& destination_data_source_description.is_encrypted == encrypted_in_backup)

{

LOG_TRACE(log, "Copying {} from AzureBlobStorage to disk {}", path_in_backup, destination_disk->getName());

auto write_blob_function = [&](const Strings & dst_blob_path, WriteMode mode, const std::optional<ObjectAttributes> &) -> size_t

{

/// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files.

if (dst_blob_path.size() != 2 || mode != WriteMode::Rewrite)

throw Exception(ErrorCodes::LOGICAL_ERROR,

"Blob writing function called with unexpected blob_path.size={} or mode={}",

dst_blob_path.size(), mode);

copyAzureBlobStorageFile(

client,

destination_disk->getObjectStorage()->getAzureBlobStorageClient(),

connection_params.getContainer(),

fs::path(blob_path) / path_in_backup,

file_size,

/* dest_container */ dst_blob_path[1],

/* dest_path */ dst_blob_path[0],

settings,

read_settings,

std::optional<ObjectAttributes>(),

threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), ThreadName::AZURE_BACKUP_READER));

return file_size;

};

destination_disk->writeFileUsingBlobWritingFunction(destination_path, write_mode, write_blob_function);

return; /// copied!

}

/// Fallback to copy through buffers.

BackupReaderDefault::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode);

const AzureBlobStorage::ConnectionParams & connection_params_,

const String & blob_path_,

bool allow_azure_native_copy,

const ReadSettings & read_settings_,

const WriteSettings & write_settings_,

const ContextPtr & context_,

bool attempt_to_create_container)

: BackupWriterDefault(read_settings_, write_settings_, getLogger("BackupWriterAzureBlobStorage"))

, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURL(), false, false, ""}

, connection_params(connection_params_)

, blob_path(blob_path_)

{

if (!attempt_to_create_container)

connection_params.endpoint.container_already_exists = true;

auto client_ptr = AzureBlobStorage::getContainerClient(connection_params, /*readonly=*/ false);

auto settings_ptr = AzureBlobStorage::getRequestSettingsForBackup(context_, connection_params.endpoint.storage_account_url, allow_azure_native_copy);

object_storage = std::make_unique<AzureObjectStorage>(

"BackupWriterAzureBlobStorage",

connection_params.auth_method,

std::move(client_ptr),

std::move(settings_ptr),

connection_params,

connection_params.getContainer(),

connection_params.getConnectionURL(),

/*common_key_prefix*/ "");

client = object_storage->getAzureBlobStorageClient();

settings = object_storage->getSettings();

const String & path_in_backup, DiskPtr src_disk, const String & src_path, bool copy_encrypted, UInt64 start_pos, UInt64 length)

{

/// Use the native copy as a more optimal way to copy a file from AzureBlobStorage to AzureBlobStorage if it's possible.

auto source_data_source_description = src_disk->getDataSourceDescription();

LOG_TRACE(log, "Source description {}, destination description {}", source_data_source_description.description, data_source_description.description);

if (source_data_source_description.object_storage_type == ObjectStorageType::Azure

&& source_data_source_description.is_encrypted == copy_encrypted)

{

/// getBlobPath() can return more than 2 elements if the file is stored as multiple objects in AzureBlobStorage container.

/// In this case we can't use the native copy.

if (auto src_blob_path = src_disk->getBlobPath(src_path); src_blob_path.size() == 2)

{

/// The Azure-to-Azure copy transfers the whole source blob, so it can be used only when the

/// request covers the entire source. `start_pos != 0` is not a sound test, because a prefix

/// [0, length) of a bigger file is a range too. `length` counts the bytes actually copied,

/// so an encrypted source must be measured with its encrypted size.

const size_t source_size

= copy_encrypted ? src_disk->getEncryptedFileSize(src_path) : src_disk->getFileSize(src_path);

if ((start_pos == 0) && (length == source_size))

{

LOG_TRACE(log, "Copying file {} from disk {} to AzureBlobStorage", src_path, src_disk->getName());

copyAzureBlobStorageFile(

src_disk->getObjectStorage()->getAzureBlobStorageClient(),

client,

/* src_container */ src_blob_path[1],

/* src_path */ src_blob_path[0],

length,

connection_params.getContainer(),

fs::path(blob_path) / path_in_backup,

settings,

read_settings,

std::optional<ObjectAttributes>(),

threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), ThreadName::AZURE_BACKUP_WRITER));

return; /// copied!

}

LOG_TRACE(

log,

"Copying the range [{}, {}) of file {} of size {} from disk {} through buffers: "

"an Azure-to-Azure copy cannot copy a part of a blob",

start_pos, start_pos + length, src_path, source_size, src_disk->getName());

}

}

/// Fallback to copy through buffers.

BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length);

{

LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination);

copyAzureBlobStorageFile(

client,

client,

connection_params.getContainer(),

fs::path(blob_path)/ source,

size,

/* dest_container */ connection_params.getContainer(),

/* dest_path */ fs::path(blob_path) / destination,

settings,

read_settings,

std::optional<ObjectAttributes>(),

threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), ThreadName::AZURE_BACKUP_WRITER));

const String & path_in_backup,

const CreateReadBufferFunction & create_read_buffer,

UInt64 start_pos,

UInt64 length)

{

copyDataToAzureBlobStorageFile(

create_read_buffer,

start_pos,

length,

client,

connection_params.getContainer(),

fs::path(blob_path) / path_in_backup,

settings,

threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(),

ThreadName::AZURE_BACKUP_WRITER));

{

String key = fs::path(blob_path) / file_name;

return object_storage->exists(StoredObject(key));

{

String key = fs::path(blob_path) / file_name;

ObjectMetadata object_metadata = object_storage->getObjectMetadata(key, /*with_tags=*/ false);

return object_metadata.size_bytes;

{

String key = fs::path(blob_path) / file_name;

return std::make_unique<ReadBufferFromAzureBlobStorage>(

client, key, read_settings, settings->max_single_read_retries,

settings->max_single_download_retries);

{

String key = fs::path(blob_path) / file_name;

return std::make_unique<WriteBufferFromAzureBlobStorage>(

client,

key,

DBMS_DEFAULT_BUFFER_SIZE,

write_settings,

settings,

connection_params.getContainer(),

/* blob_log */ nullptr,

threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), ThreadName::AZURE_BACKUP_WRITER));

{

String key = fs::path(blob_path) / file_name;

WriteSettings conditional_write_settings = write_settings;

conditional_write_settings.object_storage_write_if_none_match = "*";

return std::make_unique<WriteBufferFromAzureBlobStorage>(

client,

key,

DBMS_DEFAULT_BUFFER_SIZE,

conditional_write_settings,

settings,

connection_params.getContainer(),

/* blob_log */ nullptr,

threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), ThreadName::AZURE_BACKUP_WRITER));

{

String key = fs::path(blob_path) / file_name;

StoredObject object(key);

object_storage->removeObjectIfExists(object);

{

StoredObjects objects;

for (const auto & file_name : file_names)

objects.emplace_back(fs::path(blob_path) / file_name);

object_storage->removeObjectsIfExist(objects);