-
Notifications
You must be signed in to change notification settings - Fork 28
Expand file tree
/
Copy pathdbn_decoder.cpp
More file actions
104 lines (94 loc) · 3.5 KB
/
Copy pathdbn_decoder.cpp
File metadata and controls
104 lines (94 loc) · 3.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
#include "databento/dbn_decoder.hpp"
#include <array>
#include <cstring> // strncmp
#include <string>
#include "databento/detail/buffer.hpp"
#include "databento/detail/zstd_stream.hpp"
#include "databento/exceptions.hpp"
#include "dbn_constants.hpp"
using databento::DbnDecoder;
DbnDecoder::DbnDecoder(ILogReceiver* log_receiver, InFileStream file_stream)
: DbnDecoder(log_receiver, std::make_unique<InFileStream>(std::move(file_stream))) {
}
DbnDecoder::DbnDecoder(ILogReceiver* log_receiver, std::unique_ptr<IReadable> input)
: DbnDecoder(log_receiver, std::move(input), VersionUpgradePolicy::UpgradeToV3) {}
DbnDecoder::DbnDecoder(ILogReceiver* log_receiver, std::unique_ptr<IReadable> input,
VersionUpgradePolicy upgrade_policy)
: log_receiver_{log_receiver}, input_{std::move(input)}, fsm_{upgrade_policy} {
DetectCompression();
}
databento::Metadata DbnDecoder::DecodeMetadata() {
while (true) {
switch (fsm_.Process()) {
case detail::DbnFsm::Status::Metadata: {
return fsm_.TakeMetadata();
}
case detail::DbnFsm::Status::Record: {
throw DbnResponseError{"Found a record before the metadata"};
}
case detail::DbnFsm::Status::ReadMore: {
if (FillBuffer() == 0) {
throw DbnResponseError{"Unexpected end of input before the metadata"};
}
}
}
}
}
const databento::Record* DbnDecoder::DecodeRecord() {
while (true) {
switch (fsm_.Process()) {
case detail::DbnFsm::Status::Record: {
return &fsm_.LastRecord();
}
case detail::DbnFsm::Status::Metadata: {
break;
}
case detail::DbnFsm::Status::ReadMore: {
if (FillBuffer() == 0) {
if (fsm_.UnreadBytes() > 0) {
log_receiver_->Receive(LogLevel::Warning,
"Unexpected partial record remaining in stream: " +
std::to_string(fsm_.UnreadBytes()) + " bytes");
}
return nullptr;
}
}
}
}
}
std::size_t DbnDecoder::FillBuffer() {
std::size_t length{};
auto* space = fsm_.Space(&length);
const auto fill_size = input_->ReadSome(space, length);
fsm_.Fill(fill_size);
return fill_size;
}
void DbnDecoder::DetectCompression() {
std::array<std::byte, kMagicSize> magic{};
input_->ReadExact(magic.data(), magic.size());
if (std::strncmp(reinterpret_cast<const char*>(magic.data()), kDbnPrefix, 3) == 0) {
fsm_.WriteAll(magic.data(), magic.size());
return;
}
const auto first_word = *reinterpret_cast<const std::uint32_t*>(magic.data());
// Zstandard skippable frames begin with 0x184D2A5? where the last 8 bits
// can be set to any value
constexpr auto kZstdSkippableFrame = 0x184D2A50;
if (first_word != kZstdMagicNumber) {
if ((first_word & kZstdSkippableFrame) == kZstdSkippableFrame) {
throw DbnResponseError{
"Legacy DBZ encoding is not supported. Please use the dbn CLI tool "
"to convert it to DBN."};
}
throw DbnResponseError{
"Couldn't detect input type. It doesn't appear to be Zstd or DBN."};
}
detail::Buffer zstd_magic{kMagicSize};
zstd_magic.WriteAll(magic.data(), magic.size());
input_ = std::make_unique<detail::ZstdDecodeStream>(std::move(input_), zstd_magic);
input_->ReadExact(magic.data(), magic.size());
if (std::strncmp(reinterpret_cast<const char*>(magic.data()), kDbnPrefix, 3) != 0) {
throw DbnResponseError{"Found Zstd input, but not DBN prefix"};
}
fsm_.WriteAll(magic.data(), magic.size());
}