Skip to content

Commit fb6ca80

Browse files
knopers8sawenzel
knopers8
authored and
sawenzel
committed
[QC-725] CSV I/O for TRFCollections
1 parent 310e6dc commit fb6ca80

File tree

5 files changed

+200
-18
lines changed

5 files changed

+200
-18
lines changed

‎DataFormats/QualityControl/include/DataFormatsQualityControl/FlagReasons.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ namespace quality_control
3030
{
3131

3232
class FlagReasonFactory;
33+
class TimeRangeFlagCollection;
3334

3435
class FlagReason
3536
{
@@ -58,6 +59,7 @@ class FlagReason
5859

5960
friend std::ostream& operator<<(std::ostream& os, FlagReason const& me);
6061
friend class FlagReasonFactory;
62+
friend class TimeRangeFlagCollection;
6163

6264
ClassDefNV(FlagReason, 1);
6365
};
@@ -68,4 +70,4 @@ class FlagReason
6870
// TODO: remove once we include it in QualityControl
6971
#include "DataFormatsQualityControl/FlagReasonFactory.h"
7072

71-
#endif
73+
#endif

‎DataFormats/QualityControl/include/DataFormatsQualityControl/TimeRangeFlag.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,10 @@ class TimeRangeFlag
7575
friend std::ostream& operator<<(std::ostream& output, const TimeRangeFlag& data);
7676

7777
private:
78-
RangeInterval mInterval = {}; ///< time interval of the masked range
79-
flag_type mFlag = FlagReasonFactory::Invalid(); ///< flag reason
80-
std::string mComment = ""; ///< optional comment, which may extend the reason
81-
std::string mSource = "Unknown"; ///< optional (but encouraged) source of the flag (e.g. Qc Check name)
78+
RangeInterval mInterval = {}; ///< time interval of the masked range
79+
flag_type mFlag; ///< flag reason
80+
std::string mComment = ""; ///< optional comment, which may extend the reason
81+
std::string mSource = "Unknown"; ///< optional (but encouraged) source of the flag (e.g. Qc Check name)
8282

8383
ClassDefNV(TimeRangeFlag, 1);
8484
};

‎DataFormats/QualityControl/include/DataFormatsQualityControl/TimeRangeFlagCollection.h‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "Rtypes.h"
2424

2525
// O2 includes
26+
#include "MathUtils/detail/Bracket.h"
2627
#include "DataFormatsQualityControl/TimeRangeFlag.h"
2728

2829
// STL
@@ -39,8 +40,10 @@ class TimeRangeFlagCollection
3940
{
4041
public:
4142
using collection_t = std::set<TimeRangeFlag>;
43+
using time_type = uint64_t;
44+
using RangeInterval = o2::math_utils::detail::Bracket<time_type>;
4245

43-
explicit TimeRangeFlagCollection(std::string name, std::string detector = "TST");
46+
explicit TimeRangeFlagCollection(std::string name, std::string detector = "TST", RangeInterval validityRange = {});
4447

4548
void insert(TimeRangeFlag&&);
4649
void insert(const TimeRangeFlag&);
@@ -57,18 +60,28 @@ class TimeRangeFlagCollection
5760

5861
const std::string& getName() const;
5962
const std::string& getDetector() const;
63+
time_type getStart() const { return mValidityRange.getMin(); }
64+
time_type getEnd() const { return mValidityRange.getMax(); }
65+
RangeInterval& getInterval() { return mValidityRange; }
6066

67+
void setStart(time_type start) { mValidityRange.setMin(start); }
68+
void setEnd(time_type end) { mValidityRange.setMax(end); }
69+
void setInterval(RangeInterval interval) { mValidityRange = interval; }
6170
/// write data to ostream
6271
void streamTo(std::ostream& output) const;
72+
/// Read data from instream
73+
void streamFrom(std::istream& input);
6374

6475
/// overloading output stream operator
6576
friend std::ostream& operator<<(std::ostream& output, const TimeRangeFlagCollection& data);
6677

6778
private:
6879
std::string mDetID; // three letter detector code
69-
std::string mName = ""; // some description of the collection, e.g. "Raw data checks", "QA Expert masks"
80+
std::string mName; // some description of the collection, e.g. "Raw data checks", "QA Expert masks"
7081
// with std::set we can sort the flags in time and have merge() for granted.
7182
collection_t mTimeRangeFlags;
83+
RangeInterval mValidityRange; // we need a validity range to e.g. state that there are no TRFs for given time interval
84+
7285

7386
ClassDefNV(TimeRangeFlagCollection, 1);
7487
};

‎DataFormats/QualityControl/src/TimeRangeFlagCollection.cxx‎

Lines changed: 116 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,18 +12,28 @@
1212
// O2 include
1313
#include "DataFormatsQualityControl/TimeRangeFlagCollection.h"
1414
#include "DataFormatsQualityControl/FlagReasonFactory.h"
15+
#include "Framework/Logger.h"
16+
1517
#include <iostream>
18+
#include <boost/algorithm/string/replace.hpp>
19+
#include <boost/tokenizer.hpp>
20+
#include <utility>
1621

1722
namespace o2::quality_control
1823
{
1924

20-
TimeRangeFlagCollection::TimeRangeFlagCollection(std::string name, std::string detector)
21-
: mName(name), mDetID(detector) {}
25+
constexpr const char* csvHeader = "start,end,flag_id,flag_name,flag_bad,comment,source";
26+
constexpr size_t csvColumns = 7;
27+
28+
TimeRangeFlagCollection::TimeRangeFlagCollection(std::string name, std::string detector, RangeInterval validityRange)
29+
: mName(std::move(name)), mDetID(std::move(detector)), mValidityRange(validityRange)
30+
{}
2231

2332
void TimeRangeFlagCollection::insert(TimeRangeFlag&& trf)
2433
{
2534
mTimeRangeFlags.insert(std::move(trf));
2635
}
36+
2737
void TimeRangeFlagCollection::insert(const TimeRangeFlag& trf)
2838
{
2939
mTimeRangeFlags.insert(trf);
@@ -39,7 +49,8 @@ void TimeRangeFlagCollection::merge(TimeRangeFlagCollection& other)
3949
if (mDetID != other.mDetID) {
4050
// We assume that one TimeRangeFlagCollection should correspond to one detector at most.
4151
// However, if this becomes annoying, we can reconsider it.
42-
throw std::runtime_error("The detector ID of the target collection '" + mDetID + "' is different than the other '" + mDetID);
52+
throw std::runtime_error(
53+
"The detector ID of the target collection '" + mDetID + "' is different than the other '" + mDetID);
4354
}
4455
mTimeRangeFlags.merge(other.mTimeRangeFlags);
4556
}
@@ -54,17 +65,114 @@ TimeRangeFlagCollection::collection_t::const_iterator TimeRangeFlagCollection::b
5465
{
5566
return mTimeRangeFlags.begin();
5667
}
68+
5769
TimeRangeFlagCollection::collection_t::const_iterator TimeRangeFlagCollection::end() const
5870
{
5971
return mTimeRangeFlags.end();
6072
}
6173

6274
void TimeRangeFlagCollection::streamTo(std::ostream& output) const
6375
{
64-
output << "TimeRangeFlagCollection '" << mName << "' for detector '" << mDetID << "':"
65-
<< "\n";
66-
for (const auto& trf : mTimeRangeFlags) {
67-
output << trf << "\n";
76+
auto escapeComma = [](const std::string& str) {
77+
return boost::algorithm::replace_all_copy(str, ",", "\\,");
78+
};
79+
output << csvHeader << '\n';
80+
for (const auto& trf : *this) {
81+
output << fmt::format("{},{},{},\"{}\",{:d},\"{}\",\"{}\"\n",
82+
trf.getStart(), trf.getEnd(),
83+
trf.getFlag().getID(), escapeComma(trf.getFlag().getName()), trf.getFlag().getBad(),
84+
escapeComma(trf.getComment()), escapeComma(trf.getSource()));
85+
}
86+
}
87+
88+
void TimeRangeFlagCollection::streamFrom(std::istream& input)
89+
{
90+
std::string line;
91+
std::getline(input, line);
92+
if (line != csvHeader) {
93+
throw std::runtime_error(
94+
"Unsupported TRFCollection format, the first line is \"" + line + "\" instead of \"" + csvHeader + "\"");
95+
}
96+
97+
while (std::getline(input, line)) {
98+
boost::tokenizer<boost::escaped_list_separator<char>> tok(line);
99+
100+
TimeRangeFlag::time_type start = 0;
101+
TimeRangeFlag::time_type end = 0;
102+
FlagReason flag = FlagReasonFactory::Invalid();
103+
std::string comment;
104+
std::string source;
105+
auto it = tok.begin();
106+
bool valid = true;
107+
size_t pos = 0;
108+
for (; it != tok.end() && valid; pos++, it++) {
109+
switch (pos) {
110+
case 0: {
111+
if (it->empty()) {
112+
LOG(error) << "Invalid line, empty start time of a flag, skipping...";
113+
valid = false;
114+
} else {
115+
start = static_cast<TimeRangeFlag::time_type>(std::stoull(*it));
116+
}
117+
break;
118+
}
119+
case 1: {
120+
if (it->empty()) {
121+
LOG(error) << "Invalid line, empty end time of a flag, skipping...";
122+
valid = false;
123+
} else {
124+
end = static_cast<TimeRangeFlag::time_type>(std::stoull(*it));
125+
}
126+
break;
127+
}
128+
case 2: {
129+
if (it->empty()) {
130+
LOG(error) << "Invalid line, empty flag id, skipping...";
131+
valid = false;
132+
} else {
133+
flag.mId = std::stoul(*it);
134+
}
135+
break;
136+
}
137+
case 3: {
138+
if (it->empty()) {
139+
LOG(error) << "Invalid line, empty flag name, skipping...";
140+
valid = false;
141+
} else {
142+
flag.mName = *it;
143+
}
144+
break;
145+
}
146+
case 4: {
147+
if (it->empty()) {
148+
LOG(error) << "Invalid line, empty flag 'bad' field, skipping...";
149+
valid = false;
150+
} else {
151+
flag.mBad = static_cast<bool>(std::stoul(*it));
152+
}
153+
break;
154+
}
155+
case 5: {
156+
comment = *it;
157+
break;
158+
}
159+
case 6: {
160+
source = *it;
161+
break;
162+
}
163+
default: {
164+
LOG(error) << "More columns (" << pos + 1 << ") than expected (" << csvColumns
165+
<< ") in this line, skipping...";
166+
valid = false;
167+
break;
168+
}
169+
}
170+
}
171+
if (valid && pos < csvColumns) {
172+
LOG(error) << "Less columns (" << pos << ") than expected (" << csvColumns << ") in this line, skipping...";
173+
} else if (valid) {
174+
insert({start, end, flag, comment, source});
175+
}
68176
}
69177
}
70178

@@ -78,6 +186,7 @@ const std::string& TimeRangeFlagCollection::getName() const
78186
{
79187
return mName;
80188
}
189+
81190
const std::string& TimeRangeFlagCollection::getDetector() const
82191
{
83192
return mDetID;

‎DataFormats/QualityControl/test/testTimeRangeFlagCollection.cxx‎

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,23 +15,26 @@
1515

1616
// boost includes
1717
#include <boost/test/unit_test.hpp>
18-
18+
// STL
19+
#include <sstream>
1920
// o2 includes
2021
#include "DataFormatsQualityControl/TimeRangeFlagCollection.h"
2122
#include "DataFormatsQualityControl/TimeRangeFlag.h"
2223

2324
using namespace o2::quality_control;
2425

25-
BOOST_AUTO_TEST_CASE(test_TimeRangeFlagCollection)
26+
BOOST_AUTO_TEST_CASE(test_TimeRangeFlagCollection_Methods)
2627
{
2728
TimeRangeFlag trf1{12, 34, FlagReasonFactory::ProcessingError(), "comment", "source"};
2829
TimeRangeFlag trf2{10, 34, FlagReasonFactory::ProcessingError(), "comment", "source"};
2930

30-
TimeRangeFlagCollection trfc1{"Raw data checks", "TOF"};
31+
TimeRangeFlagCollection trfc1{"Raw data checks", "TOF", {10, 20000}};
3132
trfc1.insert(trf1); // by copy
3233
trfc1.insert(trf2);
3334
trfc1.insert({50, 77, FlagReasonFactory::Invalid()}); // by move
3435
BOOST_CHECK_EQUAL(trfc1.size(), 3);
36+
BOOST_CHECK_EQUAL(trfc1.getStart(), 10);
37+
BOOST_CHECK_EQUAL(trfc1.getEnd(), 20000);
3538

3639
TimeRangeFlagCollection trfc2{"Reco checks", "TOF"};
3740
trfc2.insert({50, 77, FlagReasonFactory::Invalid()}); // this is a duplicate to an entry in trfc1
@@ -68,4 +71,59 @@ BOOST_AUTO_TEST_CASE(test_TimeRangeFlagCollection)
6871
for (const auto& trf : trfc3) {
6972
(void)trf;
7073
}
71-
}
74+
}
75+
76+
BOOST_AUTO_TEST_CASE(test_TimeRangeFlagCollection_IO)
77+
{
78+
{
79+
TimeRangeFlagCollection trfc1{"xyz", "TST"};
80+
81+
std::stringstream store;
82+
trfc1.streamTo(store);
83+
84+
TimeRangeFlagCollection trfc2{"xyz", "TST"};
85+
trfc2.streamFrom(store);
86+
87+
BOOST_CHECK_EQUAL(trfc2.size(), 0);
88+
}
89+
{
90+
TimeRangeFlagCollection trfc1{"xyz", "TST"};
91+
trfc1.insert({50, 77, FlagReasonFactory::Invalid(), "a comment", "a source"});
92+
trfc1.insert({51, 77, FlagReasonFactory::Invalid()});
93+
trfc1.insert({1234, 3434, FlagReasonFactory::LimitedAcceptance()});
94+
trfc1.insert({50, 77, FlagReasonFactory::LimitedAcceptance()});
95+
trfc1.insert({43434, 63421, FlagReasonFactory::NotBadFlagExample()});
96+
97+
std::stringstream store;
98+
trfc1.streamTo(store);
99+
100+
TimeRangeFlagCollection trfc2{"xyz", "TST"};
101+
trfc2.streamFrom(store);
102+
103+
BOOST_REQUIRE_EQUAL(trfc1.size(), trfc2.size());
104+
for (auto it1 = trfc1.begin(), it2 = trfc2.begin(); it1 != trfc1.end() && it2 != trfc2.end(); ++it1, ++it2) {
105+
BOOST_CHECK_EQUAL(*it1, *it2);
106+
}
107+
}
108+
{
109+
std::stringstream store;
110+
store << "start,end,flag_id,invalid,header,format\n";
111+
store << R"(123,345,11,"fdsa",1,"comment","source")";
112+
TimeRangeFlagCollection trfc1{"A", "TST"};
113+
BOOST_CHECK_THROW(trfc1.streamFrom(store), std::runtime_error);
114+
}
115+
{
116+
std::stringstream store;
117+
store << "start,end,flag_id,flag_name,flag_bad,comment,source\n";
118+
store << R"(123,345,11,"fdsa",1,"comment","source","toomanycolumns")" << '\n';
119+
store << R"(123,345,11,"fdsa",1)" << '\n';
120+
store << R"(123,,11,"fdsa",1,"comment","source")" << '\n';
121+
store << R"(,345,11,"fdsa",1,"comment","source")" << '\n';
122+
store << R"(123,345,,"fdsa",1,"comment","source")" << '\n';
123+
store << R"(123,345,11,"",1,"comment","source")" << '\n';
124+
store << R"(123,345,11,"fdsa",,"comment","source")" << '\n';
125+
TimeRangeFlagCollection trfc1{"A", "TST"};
126+
BOOST_CHECK_NO_THROW(trfc1.streamFrom(store));
127+
BOOST_CHECK_EQUAL(trfc1.size(), 0);
128+
}
129+
}

0 commit comments

Comments
 (0)