const databento::TimeoutConf& timeout_conf) {

databento::detail::TcpClient::RetryConf retry_conf{};

retry_conf.connect_timeout = timeout_conf.connect;

return retry_conf;

ILogReceiver* log_receiver, std::string key, std::string dataset, bool send_ts_out,

VersionUpgradePolicy upgrade_policy,

std::optional<std::chrono::seconds> heartbeat_interval, std::size_t buffer_size,

std::string user_agent_ext, databento::Compression compression,

std::optional<databento::SlowReaderBehavior> slow_reader_behavior,

databento::TimeoutConf timeout_conf)

: log_receiver_{log_receiver},

key_{std::move(key)},

dataset_{std::move(dataset)},

gateway_{DetermineGateway()},

user_agent_ext_{std::move(user_agent_ext)},

port_{13000},

send_ts_out_{send_ts_out},

upgrade_policy_{upgrade_policy},

heartbeat_interval_{heartbeat_interval},

compression_{compression},

slow_reader_behavior_{slow_reader_behavior},

timeout_conf_{timeout_conf},

connection_{log_receiver_, gateway_, port_, RetryConfFrom(timeout_conf_)},

buffer_{kTextBufSize},

fsm_{upgrade_policy, buffer_size},

ILogReceiver* log_receiver, std::string key, std::string dataset,

std::string gateway, std::uint16_t port, bool send_ts_out,

VersionUpgradePolicy upgrade_policy,

std::optional<std::chrono::seconds> heartbeat_interval, std::size_t buffer_size,

std::string user_agent_ext, databento::Compression compression,

std::optional<databento::SlowReaderBehavior> slow_reader_behavior,

databento::TimeoutConf timeout_conf)

: log_receiver_{log_receiver},

key_{std::move(key)},

dataset_{std::move(dataset)},

gateway_{std::move(gateway)},

user_agent_ext_{std::move(user_agent_ext)},

port_{port},

send_ts_out_{send_ts_out},

upgrade_policy_{upgrade_policy},

heartbeat_interval_{heartbeat_interval},

compression_{compression},

slow_reader_behavior_{slow_reader_behavior},

timeout_conf_{timeout_conf},

connection_{log_receiver_, gateway_, port_, RetryConfFrom(timeout_conf_)},

buffer_{kTextBufSize},

fsm_{upgrade_policy, buffer_size},

SType stype_in, UnixNanos start) {

IncrementSubCounter();

std::ostringstream sub_msg;

sub_msg << "schema=" << ToString(schema) << "|stype_in=" << ToString(stype_in)

<< "|start=" << start.time_since_epoch().count()

<< "|id=" << std::to_string(sub_counter_);

Subscribe(sub_msg.str(), symbols, false);

subscriptions_.emplace_back(

LiveSubscription{symbols, schema, stype_in, start, sub_counter_});

SType stype_in, const std::string& start) {

IncrementSubCounter();

std::ostringstream sub_msg;

sub_msg << "schema=" << ToString(schema) << "|stype_in=" << ToString(stype_in)

<< "|id=" << std::to_string(sub_counter_);

if (!start.empty()) {

sub_msg << "|start=" << start;

}

Subscribe(sub_msg.str(), symbols, false);

if (start.empty()) {

subscriptions_.emplace_back(LiveSubscription{

symbols, schema, stype_in, LiveSubscription::NoStart{}, sub_counter_});

} else {

subscriptions_.emplace_back(

LiveSubscription{symbols, schema, stype_in, start, sub_counter_});

}

Schema schema, SType stype_in) {

IncrementSubCounter();

std::ostringstream sub_msg;

sub_msg << "schema=" << ToString(schema) << "|stype_in=" << ToString(stype_in)

<< "|id=" << std::to_string(sub_counter_);

Subscribe(sub_msg.str(), symbols, true);

subscriptions_.emplace_back(LiveSubscription{

symbols, schema, stype_in, LiveSubscription::Snapshot{}, sub_counter_});

const std::vector<std::string>& symbols,

bool use_snapshot) {

static constexpr auto kMethodName = "LiveBlocking::Subscribe";

constexpr std::ptrdiff_t kSymbolMaxChunkSize = 500;

if (symbols.empty()) {

throw InvalidArgumentError{kMethodName, "symbols",

"must contain at least one symbol"};

}

auto symbols_it = symbols.begin();

while (symbols_it != symbols.end()) {

const auto distance_from_end = std::distance(symbols_it, symbols.end());

const auto chunk_size = std::min(kSymbolMaxChunkSize, distance_from_end);

std::ostringstream chunked_sub_msg;

chunked_sub_msg << sub_msg << "|symbols="

<< JoinSymbolStrings(kMethodName, symbols_it,

symbols_it + chunk_size)

<< "|snapshot=" << use_snapshot

<< "|is_last=" << (distance_from_end <= kSymbolMaxChunkSize)

<< '\n';

if (log_receiver_->ShouldLog(LogLevel::Debug)) {

std::ostringstream log_ss;

log_ss << '[' << kMethodName

<< "] Sending subscription request: " << chunked_sub_msg.str();

log_receiver_->Receive(LogLevel::Debug, log_ss.str());

}

connection_.WriteAll(chunked_sub_msg.str());

symbols_it += chunk_size;

}

log_receiver_->Receive(LogLevel::Info, "[LiveBlocking::Start] Starting session");

connection_.WriteAll("start_session\n");

connection_.SetCompression(compression_);

// Authentication may have read part of the DBN stream

fsm_.WriteAll(buffer_.ReadBegin(), buffer_.ReadCapacity());

buffer_.Clear();

while (true) {

switch (fsm_.Process()) {

case detail::DbnFsm::Status::Metadata: {

last_read_time_ = std::chrono::steady_clock::now();

return fsm_.TakeMetadata();

}

case detail::DbnFsm::Status::Record: {

throw LiveApiError{"Received a record before the metadata"};

}

case detail::DbnFsm::Status::ReadMore: {

std::size_t length{};

auto* space = fsm_.Space(&length);

const auto read_res = connection_.ReadSome(space, length);

if (read_res.read_size == 0) {

throw LiveApiError{"Gateway closed the session before sending metadata"};

}

fsm_.Fill(read_res.read_size);

}

}

}

if (log_receiver_->ShouldLog(LogLevel::Info)) {

std::ostringstream log_msg;

log_msg << "Reconnecting to " << gateway_ << ':' << port_;

log_receiver_->Receive(LogLevel::Info, log_msg.str());

}

connection_ = detail::LiveConnection{log_receiver_, gateway_, port_,

RetryConfFrom(timeout_conf_)};

buffer_.Clear();

fsm_.Reset();

sub_counter_ = 0;

session_id_ = this->Authenticate();

last_read_time_ = std::chrono::steady_clock::now();

for (auto& subscription : subscriptions_) {

if (std::holds_alternative<UnixNanos>(subscription.start) ||

std::holds_alternative<std::string>(subscription.start)) {

subscription.start = LiveSubscription::NoStart{};

}

sub_counter_ = std::max(sub_counter_, subscription.id);

std::ostringstream sub_msg;

sub_msg << "schema=" << ToString(subscription.schema)

<< "|stype_in=" << ToString(subscription.stype_in)

<< "|id=" << std::to_string(sub_counter_);

Subscribe(sub_msg.str(), subscription.symbols,

std::holds_alternative<LiveSubscription::Snapshot>(subscription.start));

}

static constexpr auto kMethodName = "LiveBlocking::DecodeChallenge";

const auto result =

connection_.ReadSome(buffer_.WriteBegin(), buffer_.WriteCapacity(), timeout);

if (result.status == Status::Timeout) {

throw TcpError{0, "Authentication timed out waiting for challenge"};

}

const auto read_size = result.read_size;

if (read_size == 0) {

throw LiveApiError{"Gateway closed socket during authentication"};

}

buffer_.Fill(read_size);

// first line is version

std::string response{reinterpret_cast<const char*>(buffer_.ReadBegin()),

buffer_.ReadCapacity()};

auto first_nl_pos = response.find('\n');

if (first_nl_pos == std::string::npos) {

throw LiveApiError::UnexpectedMsg("Received malformed initial message", response);

}

if (log_receiver_->ShouldLog(LogLevel::Debug)) {

std::ostringstream log_ss;

log_ss << '[' << kMethodName

<< "] Received greeting: " << response.substr(0, first_nl_pos);

log_receiver_->Receive(LogLevel::Debug, log_ss.str());

}

const auto find_start = first_nl_pos + 1;

auto next_nl_pos = find_start == response.length() ? std::string::npos

: response.find('\n', find_start);

while (next_nl_pos == std::string::npos) {

// read more

const auto loop_result =

connection_.ReadSome(buffer_.WriteBegin(), buffer_.WriteCapacity(), timeout);

if (loop_result.status == Status::Timeout) {

throw TcpError{0, "Authentication timed out waiting for challenge"};

}

buffer_.Fill(loop_result.read_size);

if (buffer_.ReadCapacity() == 0) {

throw LiveApiError{"Gateway closed socket during authentication"};

}

response = {reinterpret_cast<const char*>(buffer_.ReadBegin()),

buffer_.ReadCapacity()};

next_nl_pos = response.find('\n', find_start);

}

const auto challenge_line = response.substr(find_start, next_nl_pos - find_start);

if (log_receiver_->ShouldLog(LogLevel::Debug)) {

std::ostringstream log_ss;

log_ss << '[' << kMethodName << "] Received CRAM challenge: " << challenge_line;

log_receiver_->Receive(LogLevel::Debug, log_ss.str());

}

if (challenge_line.compare(0, 4, "cram") != 0) {

throw LiveApiError::UnexpectedMsg("Did not receive CRAM challenge when expected",

challenge_line);

}

const auto equal_pos = challenge_line.find('=');

if (equal_pos == std::string::npos || equal_pos + 1 > challenge_line.size()) {

throw LiveApiError::UnexpectedMsg("Received malformed CRAM challenge",

challenge_line);

}

return challenge_line.substr(equal_pos + 1);

std::ostringstream gateway;

for (const char c : dataset_) {

gateway << (c == '.' ? '-' : static_cast<char>(std::tolower(c)));

}

gateway << ".lsg.databento.com";

return gateway.str();

static constexpr auto kMethodName = "LiveBlocking::Authenticate";

const auto deadline = std::chrono::steady_clock::now() +

std::chrono::duration_cast<std::chrono::steady_clock::duration>(

timeout_conf_.auth);

auto remaining = [&deadline]() {

const auto now = std::chrono::steady_clock::now();

if (now >= deadline) {

throw TcpError{0, "Authentication timed out"};

}

return std::chrono::duration_cast<std::chrono::milliseconds>(deadline - now);

};

const std::string challenge_key = DecodeChallenge(remaining()) + '|' + key_;

const std::string auth = GenerateCramReply(challenge_key);

const std::string req = EncodeAuthReq(auth);

if (log_receiver_->ShouldLog(LogLevel::Debug)) {

std::ostringstream log_ss;

log_ss << '[' << kMethodName << "] Sending CRAM reply: " << req;

log_receiver_->Receive(LogLevel::Debug, log_ss.str());

}

connection_.WriteAll(req);

const std::uint64_t session_id = DecodeAuthResp(remaining());

if (log_receiver_->ShouldLog(LogLevel::Info)) {

std::ostringstream log_ss;

log_ss << '[' << kMethodName << "] Successfully authenticated with session_id "

<< session_id;

log_receiver_->Receive(LogLevel::Info, log_ss.str());

}

return session_id;

std::ostringstream auth_stream;

auth_stream << detail::Sha256Hash(challenge_key) << '-'

<< key_.substr(kApiKeyLength - kBucketIdLength);

return auth_stream.str();

std::ostringstream req_stream;

req_stream << "auth=" << auth << "|dataset=" << dataset_ << "|encoding=dbn|"

<< "ts_out=" << send_ts_out_ << "|compression=" << compression_

<< "|client=" << kUserAgent;

if (!user_agent_ext_.empty()) {

req_stream << ' ' << user_agent_ext_;

}

if (heartbeat_interval_.has_value()) {

req_stream << "|heartbeat_interval_s=" << heartbeat_interval_->count();

}

if (slow_reader_behavior_.has_value()) {

req_stream << "|slow_reader_behavior=" << *slow_reader_behavior_;

}

req_stream << '\n';

return req_stream.str();

// handle split packet read

const std::byte* newline_ptr;

buffer_.Clear();

do {

const auto result =

connection_.ReadSome(buffer_.WriteBegin(), buffer_.WriteCapacity(), timeout);

if (result.status == Status::Timeout) {

throw TcpError{0, "Authentication timed out waiting for auth response"};

}

if (result.read_size == 0) {

throw LiveApiError{

"Unexpected end of message received from server after replying to "

"CRAM"};

}

buffer_.Fill(result.read_size);

newline_ptr =

std::find(buffer_.ReadBegin(), buffer_.ReadEnd(), static_cast<std::byte>('\n'));

} while (newline_ptr == buffer_.ReadEnd());

const std::string response{

reinterpret_cast<const char*>(buffer_.ReadBegin()),

static_cast<std::size_t>(newline_ptr - buffer_.ReadBegin())};

{

std::ostringstream log_ss;

log_ss << "[LiveBlocking::DecodeAuthResp] Authentication response: " << response;

log_receiver_->Receive(LogLevel::Debug, log_ss.str());

}

// set in case Read call also read records. One beyond newline

buffer_.Consume(response.length() + 1);

std::size_t pos{};

bool found_success{};

bool is_error{};

std::uint64_t session_id = 0;

std::string err_details;

while (true) {

const size_t count = response.find('|', pos);

if (count == response.length() - 1) {

break;

}

// passing count = npos to substr will take remainder of string

const std::string kv_pair = response.substr(pos, count - pos);

const std::size_t eq_pos = kv_pair.find('=');

if (eq_pos == std::string::npos) {

throw LiveApiError::UnexpectedMsg("Malformed authentication response", response);

}

const std::string key = kv_pair.substr(0, eq_pos);

if (key == "success") {

found_success = true;

if (kv_pair.substr(eq_pos + 1) != "1") {

is_error = true;

}

} else if (key == "error") {

err_details = kv_pair.substr(eq_pos + 1);

} else if (key == "session_id") {

session_id = std::stoull(kv_pair.substr(eq_pos + 1));

}

// no more keys to parse

if (count == std::string::npos) {

break;

}

pos = count + 1;

}

if (!found_success) {

throw LiveApiError{"Did not receive success indicator from authentication attempt"};

}

if (is_error) {

throw LiveApiError{"Failed to authenticate: " +

(err_details.empty() ? response : err_details)};

}

return session_id;

if (sub_counter_ == std::numeric_limits<uint32_t>::max()) {

log_receiver_->Receive(LogLevel::Warning,

"[LiveBlocking::Subscribe] Exhausted all subscription IDs");

} else {

++sub_counter_;

}

std::chrono::milliseconds timeout) {

std::size_t length{};

auto* space = fsm_.Space(&length);

const auto read_res = connection_.ReadSome(space, length, timeout);

fsm_.Fill(read_res.read_size);

if (read_res.read_size > 0) {

last_read_time_ = std::chrono::steady_clock::now();

}

return read_res;

if (heartbeat_interval_) {

return std::chrono::milliseconds{*heartbeat_interval_ + kHeartbeatTimeoutMargin};

}

return std::chrono::milliseconds{kDefaultHeartbeatTimeout};

const auto timeout = heartbeat_interval_.has_value()

? *heartbeat_interval_ + kHeartbeatTimeoutMargin

: kDefaultHeartbeatTimeout;

const auto elapsed = std::chrono::steady_clock::now() - last_read_time_;

if (elapsed > timeout) {

throw HeartbeatTimeoutError{

std::chrono::duration_cast<std::chrono::seconds>(elapsed)};

}

if (fsm_.LastRecord().RType() == RType::System) {

LogSystemRecord();

} else if (fsm_.LastRecord().RType() == RType::Error) {

LogErrorRecord();

}

static constexpr auto kMethodName = "[LiveBlocking::LogSystemRecord]";

std::ostringstream ss;

const auto log_heartbeat = [this, &ss] {

if (log_receiver_->ShouldLog(LogLevel::Debug)) {

ss << kMethodName << " Received gateway heartbeat";

log_receiver_->Receive(LogLevel::Debug, ss.str());

}

};

if (fsm_.LastRecord().Size() >= sizeof(SystemMsg)) {

const auto& system = fsm_.LastRecord().Get<SystemMsg>();

switch (system.code) {

case SystemCode::Heartbeat: {

log_heartbeat();

break;

}

case SystemCode::EndOfInterval: {

if (log_receiver_->ShouldLog(LogLevel::Debug)) {

ss << kMethodName << " Received: " << system.Msg();

log_receiver_->Receive(LogLevel::Debug, ss.str());

}

break;

}

case SystemCode::SlowReaderWarning: {

ss << kMethodName << " Received: " << system.Msg();

log_receiver_->Receive(LogLevel::Warning, ss.str());

break;

}

default: {

if (log_receiver_->ShouldLog(LogLevel::Info)) {

ss << kMethodName << " Received system message with code: " << system.code

<< " and message: " << system.Msg();

log_receiver_->Receive(LogLevel::Info, ss.str());

}

break;

}

}

} else {

// v1 path with explicit UpgradePolicy::AsIs

const auto& system = fsm_.LastRecord().Get<v1::SystemMsg>();

if (system.IsHeartbeat()) {

log_heartbeat();

} else if (log_receiver_->ShouldLog(LogLevel::Info)) {

// Don't make effort to parse based on Msg as this path is rare

ss << kMethodName << " Received system message: " << system.Msg();

log_receiver_->Receive(LogLevel::Info, ss.str());

}

}

static constexpr auto kMethodName = "[LiveBlocking::LogErrorRecord]";

std::ostringstream ss;

ss << kMethodName << ' ';

if (fsm_.LastRecord().Size() >= sizeof(ErrorMsg)) {

const auto& error = fsm_.LastRecord().Get<ErrorMsg>();

ss << " Received error with code: " << error.code << ", message:" << error.Err()

<< ", and is_last: " << static_cast<std::uint16_t>(error.is_last);

} else {

const auto& error = fsm_.LastRecord().Get<v1::ErrorMsg>();

ss << " Received error with message:" << error.Err();

}

log_receiver_->Receive(LogLevel::Error, ss.str());