Skip to content

Commit 5c2cf35

Browse files
Sunrisepeakclaude
andcommitted
feat: add download_to_file with streaming progress callback
- Add HttpClient::download_to_file() for streaming downloads to disk with real-time progress reporting (total/downloaded bytes per 8KB block) - Supports Content-Length, chunked transfer, and connection-close modes - Follows redirects up to maxRedirects - Add DownloadToFileResult and DownloadProgressFn types - Add gtest-based download tests (httpbin.org) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 8bc7712 commit 5c2cf35

File tree

4 files changed

+437
-0
lines changed

4 files changed

+437
-0
lines changed

src/http.cppm

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,17 @@ export struct HttpClientConfig {
4141
int maxRedirects { 10 }; // 0 = don't follow redirects
4242
};
4343

44+
// Progress callback for streaming downloads: (totalBytes, downloadedBytes)
45+
// totalBytes is 0 when Content-Length is unknown (chunked/connection-close).
46+
export using DownloadProgressFn = std::function<void(std::int64_t total, std::int64_t downloaded)>;
47+
48+
export struct DownloadToFileResult {
49+
int statusCode { 0 };
50+
std::string error;
51+
std::int64_t bytesWritten { 0 };
52+
bool ok() const { return statusCode >= 200 && statusCode < 300 && error.empty(); }
53+
};
54+
4455
export template<typename F>
4556
concept SseCallback = std::invocable<F, const SseEvent&> &&
4657
std::same_as<std::invoke_result_t<F, const SseEvent&>, bool>;
@@ -749,10 +760,252 @@ public:
749760
return response;
750761
}
751762

763+
// Download URL to file with streaming progress.
764+
// Follows redirects. Calls onProgress periodically during body read.
765+
DownloadToFileResult download_to_file(
766+
const std::string& url,
767+
const std::filesystem::path& destFile,
768+
DownloadProgressFn onProgress = nullptr)
769+
{
770+
return download_to_file_impl(url, destFile, std::move(onProgress), 0);
771+
}
772+
752773
HttpClientConfig& config() { return config_; }
753774
const HttpClientConfig& config() const { return config_; }
754775

755776
private:
777+
DownloadToFileResult download_to_file_impl(
778+
const std::string& url,
779+
const std::filesystem::path& destFile,
780+
DownloadProgressFn onProgress,
781+
int redirectCount)
782+
{
783+
DownloadToFileResult result;
784+
785+
auto parsed = parse_url(url);
786+
if (parsed.scheme != "https") {
787+
result.error = "Only HTTPS is supported";
788+
return result;
789+
}
790+
791+
std::string poolKey = parsed.host + ":" + std::to_string(parsed.port);
792+
793+
// Get or create connection
794+
TlsSocket* sock = nullptr;
795+
auto it = pool_.find(poolKey);
796+
if (it != pool_.end() && it->second.is_valid()) {
797+
sock = &it->second;
798+
} else {
799+
if (it != pool_.end()) pool_.erase(it);
800+
auto [insertIt, ok] = pool_.emplace(poolKey, TlsSocket{});
801+
sock = &insertIt->second;
802+
bool connected = false;
803+
if (config_.proxy.has_value()) {
804+
auto proxyConf = parse_proxy_url(config_.proxy.value());
805+
auto tunnel = proxy_connect(proxyConf.host, proxyConf.port,
806+
parsed.host, parsed.port,
807+
config_.connectTimeoutMs);
808+
if (tunnel.is_valid()) {
809+
connected = sock->connect_over(std::move(tunnel),
810+
parsed.host.c_str(),
811+
config_.verifySsl);
812+
}
813+
} else {
814+
connected = sock->connect(parsed.host.c_str(), parsed.port,
815+
config_.connectTimeoutMs, config_.verifySsl);
816+
}
817+
if (!connected) {
818+
pool_.erase(poolKey);
819+
result.error = "Connection failed";
820+
return result;
821+
}
822+
}
823+
824+
// Build GET request
825+
std::string reqStr = "GET ";
826+
reqStr += parsed.path;
827+
reqStr += " HTTP/1.1\r\nHost: ";
828+
reqStr += parsed.host;
829+
if (parsed.port != 443) {
830+
reqStr += ":";
831+
reqStr += std::to_string(parsed.port);
832+
}
833+
reqStr += "\r\nUser-Agent: tinyhttps/1.0\r\nAccept: */*\r\n";
834+
reqStr += config_.keepAlive ? "Connection: keep-alive\r\n" : "Connection: close\r\n";
835+
reqStr += "\r\n";
836+
837+
if (!write_all(*sock, reqStr)) {
838+
pool_.erase(poolKey);
839+
result.error = "Write failed";
840+
return result;
841+
}
842+
843+
// Read status line
844+
std::string statusLine = read_line(*sock, config_.readTimeoutMs);
845+
if (statusLine.empty()) {
846+
pool_.erase(poolKey);
847+
result.error = "No response";
848+
return result;
849+
}
850+
851+
// Parse status code
852+
{
853+
auto sp = statusLine.find(' ');
854+
if (sp != std::string::npos) {
855+
auto rest = std::string_view(statusLine).substr(sp + 1);
856+
for (char c : rest) {
857+
if (c >= '0' && c <= '9')
858+
result.statusCode = result.statusCode * 10 + (c - '0');
859+
else break;
860+
}
861+
}
862+
}
863+
864+
// Read headers
865+
bool chunked = false;
866+
std::int64_t contentLength = -1;
867+
bool connectionClose = false;
868+
std::string location;
869+
870+
while (true) {
871+
std::string line = read_line(*sock, config_.readTimeoutMs);
872+
if (line.empty()) break;
873+
auto colon = line.find(':');
874+
if (colon == std::string::npos) continue;
875+
std::string key = line.substr(0, colon);
876+
std::string_view val = std::string_view(line).substr(colon + 1);
877+
while (!val.empty() && val[0] == ' ') val = val.substr(1);
878+
std::string valStr(val);
879+
880+
if (iequals(key, "Transfer-Encoding") && iequals(valStr, "chunked"))
881+
chunked = true;
882+
if (iequals(key, "Content-Length")) {
883+
contentLength = 0;
884+
for (char c : valStr) {
885+
if (c >= '0' && c <= '9')
886+
contentLength = contentLength * 10 + (c - '0');
887+
}
888+
}
889+
if (iequals(key, "Connection") && iequals(valStr, "close"))
890+
connectionClose = true;
891+
if (iequals(key, "Location"))
892+
location = valStr;
893+
}
894+
895+
// Follow redirects
896+
if (result.statusCode >= 300 && result.statusCode < 400 &&
897+
!location.empty() && redirectCount < config_.maxRedirects) {
898+
// Drain any body to keep connection clean
899+
if (connectionClose) {
900+
sock->close();
901+
pool_.erase(poolKey);
902+
}
903+
// Resolve relative URL
904+
if (location.starts_with("/")) {
905+
location = parsed.scheme + "://" + parsed.host +
906+
(parsed.port != 443 ? ":" + std::to_string(parsed.port) : "") +
907+
location;
908+
}
909+
return download_to_file_impl(location, destFile, std::move(onProgress),
910+
redirectCount + 1);
911+
}
912+
913+
if (result.statusCode < 200 || result.statusCode >= 300) {
914+
result.error = "HTTP " + std::to_string(result.statusCode);
915+
if (connectionClose) { sock->close(); pool_.erase(poolKey); }
916+
return result;
917+
}
918+
919+
// Open output file
920+
std::error_code ec;
921+
std::filesystem::create_directories(destFile.parent_path(), ec);
922+
std::ofstream ofs(destFile, std::ios::binary);
923+
if (!ofs) {
924+
result.error = "Cannot open file: " + destFile.string();
925+
if (connectionClose) { sock->close(); pool_.erase(poolKey); }
926+
return result;
927+
}
928+
929+
std::int64_t totalBytes = contentLength > 0 ? contentLength : 0;
930+
std::int64_t downloaded = 0;
931+
932+
// Read body and stream to file
933+
if (chunked) {
934+
while (true) {
935+
std::string sizeLine = read_line(*sock, config_.readTimeoutMs);
936+
auto semi = sizeLine.find(';');
937+
if (semi != std::string::npos) sizeLine = sizeLine.substr(0, semi);
938+
while (!sizeLine.empty() && (sizeLine.back() == ' ' || sizeLine.back() == '\t'))
939+
sizeLine.pop_back();
940+
941+
int chunkSize = parse_hex(sizeLine);
942+
if (chunkSize == 0) {
943+
read_line(*sock, config_.readTimeoutMs);
944+
break;
945+
}
946+
947+
// Read chunk in sub-blocks for progress
948+
int remaining = chunkSize;
949+
char buf[8192];
950+
while (remaining > 0) {
951+
int toRead = remaining > static_cast<int>(sizeof(buf))
952+
? static_cast<int>(sizeof(buf)) : remaining;
953+
if (!read_exact(*sock, buf, toRead, config_.readTimeoutMs)) {
954+
result.error = "Read error during chunked transfer";
955+
ofs.close();
956+
result.bytesWritten = downloaded;
957+
return result;
958+
}
959+
ofs.write(buf, toRead);
960+
downloaded += toRead;
961+
remaining -= toRead;
962+
if (onProgress) onProgress(totalBytes, downloaded);
963+
}
964+
read_line(*sock, config_.readTimeoutMs); // trailing \r\n
965+
}
966+
} else if (contentLength > 0) {
967+
char buf[8192];
968+
std::int64_t remaining = contentLength;
969+
while (remaining > 0) {
970+
int toRead = remaining > static_cast<std::int64_t>(sizeof(buf))
971+
? static_cast<int>(sizeof(buf))
972+
: static_cast<int>(remaining);
973+
if (!read_exact(*sock, buf, toRead, config_.readTimeoutMs)) {
974+
result.error = "Read error";
975+
ofs.close();
976+
result.bytesWritten = downloaded;
977+
return result;
978+
}
979+
ofs.write(buf, toRead);
980+
downloaded += toRead;
981+
remaining -= toRead;
982+
if (onProgress) onProgress(totalBytes, downloaded);
983+
}
984+
} else {
985+
// Read until connection close
986+
connectionClose = true;
987+
char buf[8192];
988+
while (true) {
989+
if (!sock->wait_readable(config_.readTimeoutMs)) break;
990+
int ret = sock->read(buf, sizeof(buf));
991+
if (ret <= 0) break;
992+
ofs.write(buf, ret);
993+
downloaded += ret;
994+
if (onProgress) onProgress(totalBytes, downloaded);
995+
}
996+
}
997+
998+
ofs.close();
999+
result.bytesWritten = downloaded;
1000+
1001+
if (connectionClose) {
1002+
sock->close();
1003+
pool_.erase(poolKey);
1004+
}
1005+
1006+
return result;
1007+
}
1008+
7561009
HttpClientConfig config_;
7571010
std::map<std::string, TlsSocket> pool_;
7581011
};

0 commit comments

Comments
 (0)