Skip to content

Commit bcd4e05

Browse files
knopers8knopers8
committed
[QC-338] Facilitate creating a Dispatcher without configuration files
1 parent 7a4812b commit bcd4e05

File tree

8 files changed

+249
-110
lines changed

8 files changed

+249
-110
lines changed

‎Utilities/DataSampling/include/DataSampling/DataSamplingCondition.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
namespace o2::utilities
2525
{
2626

27-
/// A standarised data sampling condition, to decide if given data sample should be passed forward.
27+
/// A standardised data sampling condition, to decide if given data sample should be passed forward.
2828
class DataSamplingCondition
2929
{
3030
public:

‎Utilities/DataSampling/include/DataSampling/DataSamplingPolicy.h‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ class DataSamplingPolicy
3939
class PathMap : public PathVectorBase
4040
{
4141
public:
42-
~PathMap() = default;
4342
PathMap() = default;
43+
~PathMap() = default;
4444
const PathVectorBase::const_iterator find(const framework::ConcreteDataMatcher& input) const
4545
{
4646
return std::find_if(begin(), end(), [input](const auto& el) {
@@ -50,21 +50,33 @@ class DataSamplingPolicy
5050
};
5151

5252
public:
53+
/// \brief Configures a policy using structured configuration entry.
54+
static DataSamplingPolicy fromConfiguration(const boost::property_tree::ptree&);
55+
5356
/// \brief Constructor.
54-
DataSamplingPolicy();
55-
/// \brief Constructor.
56-
DataSamplingPolicy(const boost::property_tree::ptree&);
57+
DataSamplingPolicy(std::string name);
5758
/// \brief Destructor
58-
~DataSamplingPolicy();
59+
~DataSamplingPolicy() = default;
60+
/// \brief Move constructor
61+
DataSamplingPolicy(DataSamplingPolicy&&) = default;
62+
/// \brief Move assignment
63+
DataSamplingPolicy& operator=(DataSamplingPolicy&&) = default;
64+
65+
/// \brief Adds a new association between inputs and outputs.
66+
void registerPath(const framework::InputSpec&, const framework::OutputSpec&);
67+
/// \brief Adds a new association between inputs and outputs.
68+
// void registerPolicy(framework::InputSpec&&, framework::OutputSpec&&);
69+
/// \brief Adds a new sampling condition.
70+
void registerCondition(std::unique_ptr<DataSamplingCondition>&&);
71+
/// \brief Sets a raw FairMQChannel. Deprecated, do not use.
72+
void setFairMQOutputChannel(std::string);
5973

60-
/// \brief Configures a policy using structured configuration entry.
61-
void configure(const boost::property_tree::ptree&);
6274
/// \brief Returns true if this policy requires data with given InputSpec.
6375
bool match(const framework::ConcreteDataMatcher& input) const;
6476
/// \brief Returns true if user-defined conditions of sampling are fulfilled.
6577
bool decide(const o2::framework::DataRef&);
6678
/// \brief Returns Output for given InputSpec to pass data forward.
67-
const framework::Output prepareOutput(const framework::ConcreteDataMatcher& input, framework::Lifetime lifetime = framework::Lifetime::Timeframe) const;
79+
framework::Output prepareOutput(const framework::ConcreteDataMatcher& input, framework::Lifetime lifetime = framework::Lifetime::Timeframe) const;
6880

6981
const std::string& getName() const;
7082
const PathMap& getPathMap() const;

‎Utilities/DataSampling/include/DataSampling/Dispatcher.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,14 @@ class Dispatcher : public framework::Task
5050
/// \brief Dispatcher process callback
5151
void run(framework::ProcessingContext& ctx) override;
5252

53-
/// \brief Create appropriate inputSpecs and outputSpecs for sampled data during the workflow declaration phase.
54-
void registerPath(const std::pair<framework::InputSpec, framework::OutputSpec>&);
53+
/// \brief Register a Data Sampling Policy
54+
void registerPolicy(std::unique_ptr<DataSamplingPolicy>&&);
5555

5656
const std::string& getName();
57+
/// \brief Assembles InputSpecs of all registered policies in a single vector, removing overlapping entries.
5758
framework::Inputs getInputSpecs();
5859
framework::Outputs getOutputSpecs();
60+
framework::Options getOptions();
5961

6062
private:
6163
DataSamplingHeader prepareDataSamplingHeader(const DataSamplingPolicy& policy, const framework::DeviceSpec& spec);
@@ -67,8 +69,6 @@ class Dispatcher : public framework::Task
6769

6870
std::string mName;
6971
std::string mReconfigurationSource;
70-
framework::Inputs inputs;
71-
framework::Outputs outputs;
7272
// policies should be shared between all pipeline threads
7373
std::vector<std::shared_ptr<DataSamplingPolicy>> mPolicies;
7474
};

‎Utilities/DataSampling/src/DataSampling.cxx‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -56,15 +56,14 @@ void DataSampling::GenerateInfrastructure(WorkflowSpec& workflow, const boost::p
5656
void DataSampling::DoGenerateInfrastructure(Dispatcher& dispatcher, WorkflowSpec& workflow, const boost::property_tree::ptree& policiesTree, size_t threads)
5757
{
5858
LOG(DEBUG) << "Generating Data Sampling infrastructure...";
59-
Options options;
6059

6160
for (auto&& policyConfig : policiesTree) {
6261

6362
std::unique_ptr<DataSamplingPolicy> policy;
6463

6564
// We don't want the Dispatcher to exit due to one faulty Policy
6665
try {
67-
policy = std::make_unique<DataSamplingPolicy>(policyConfig.second);
66+
dispatcher.registerPolicy(std::make_unique<DataSamplingPolicy>(DataSamplingPolicy::fromConfiguration(policyConfig.second)));
6867
} catch (const std::exception& ex) {
6968
LOG(WARN) << "Could not load the Data Sampling Policy '"
7069
<< policyConfig.second.get_optional<std::string>("id").value_or("") << "', because: " << ex.what();
@@ -74,27 +73,17 @@ void DataSampling::DoGenerateInfrastructure(Dispatcher& dispatcher, WorkflowSpec
7473
<< policyConfig.second.get_optional<std::string>("id").value_or("") << "'";
7574
continue;
7675
}
77-
78-
for (const auto& path : policy->getPathMap()) {
79-
dispatcher.registerPath({path.first, path.second});
80-
}
81-
82-
if (!policy->getFairMQOutputChannel().empty()) {
83-
options.push_back({"channel-config", VariantType::String, policy->getFairMQOutputChannel().c_str(), {"Out-of-band channel config"}});
84-
LOG(DEBUG) << " - registering output FairMQ channel '" << policy->getFairMQOutputChannel() << "'";
85-
}
8676
}
87-
options.push_back({"period-timer-stats", framework::VariantType::Int, 10 * 1000000, {"Dispatcher's stats timer period"}});
8877

8978
if (dispatcher.getInputSpecs().size() > 0) {
9079
DataProcessorSpec spec;
9180
spec.name = dispatcher.getName();
9281
spec.inputs = dispatcher.getInputSpecs();
9382
spec.outputs = dispatcher.getOutputSpecs();
94-
spec.algorithm = adaptFromTask<Dispatcher>(std::move(dispatcher));
9583
spec.maxInputTimeslices = threads;
9684
spec.labels = {{"DataSampling"}, {"Dispatcher"}};
97-
spec.options = options;
85+
spec.options = dispatcher.getOptions();
86+
spec.algorithm = adaptFromTask<Dispatcher>(std::move(dispatcher));
9887

9988
workflow.emplace_back(std::move(spec));
10089
} else {
@@ -127,7 +116,7 @@ std::vector<InputSpec> DataSampling::InputSpecsForPolicy(ConfigurationInterface*
127116

128117
for (auto&& policyConfig : policiesTree) {
129118
if (policyConfig.second.get<std::string>("id") == policyName) {
130-
DataSamplingPolicy policy(policyConfig.second);
119+
auto policy = DataSamplingPolicy::fromConfiguration(policyConfig.second);
131120
for (const auto& path : policy.getPathMap()) {
132121
InputSpec input = DataSpecUtils::matchingInput(path.second);
133122
inputs.push_back(input);
@@ -169,7 +158,7 @@ std::vector<OutputSpec> DataSampling::OutputSpecsForPolicy(ConfigurationInterfac
169158

170159
for (auto&& policyConfig : policiesTree) {
171160
if (policyConfig.second.get<std::string>("id") == policyName) {
172-
DataSamplingPolicy policy(policyConfig.second);
161+
auto policy = DataSamplingPolicy::fromConfiguration(policyConfig.second);
173162
for (const auto& path : policy.getPathMap()) {
174163
outputs.push_back(path.second);
175164
}

‎Utilities/DataSampling/src/DataSamplingPolicy.cxx‎

Lines changed: 27 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -29,28 +29,30 @@ namespace o2::utilities
2929

3030
using boost::property_tree::ptree;
3131

32-
DataSamplingPolicy::DataSamplingPolicy() = default;
32+
DataSamplingPolicy::DataSamplingPolicy(std::string name) : mName(std::move(name))
33+
{
34+
}
3335

34-
DataSamplingPolicy::DataSamplingPolicy(const ptree& config)
36+
void DataSamplingPolicy::registerPath(const InputSpec& inputSpec, const OutputSpec& outputSpec)
3537
{
36-
configure(config);
38+
mPaths.emplace_back(inputSpec, outputSpec);
3739
}
3840

39-
DataSamplingPolicy::~DataSamplingPolicy() = default;
41+
void DataSamplingPolicy::registerCondition(std::unique_ptr<DataSamplingCondition>&& condition)
42+
{
43+
mConditions.emplace_back(std::move(condition));
44+
}
4045

41-
void DataSamplingPolicy::configure(const ptree& config)
46+
void DataSamplingPolicy::setFairMQOutputChannel(std::string channel)
4247
{
43-
mName = config.get<std::string>("id");
44-
if (mName.size() > 14) {
45-
LOG(WARNING) << "DataSamplingPolicy name '" << mName << "' is longer than 14 characters, we have to trim it. "
46-
<< "Use a shorter policy name to avoid potential output name conflicts.";
47-
mName.resize(14);
48-
}
48+
mFairMQOutputChannel = std::move(channel);
49+
}
4950

50-
auto subSpecString = config.get_optional<std::string>("subSpec").value_or("*");
51-
auto subSpec = subSpecString.find_first_of("-*") != std::string::npos ? -1 : std::strtoull(subSpecString.c_str(), nullptr, 10);
51+
DataSamplingPolicy DataSamplingPolicy::fromConfiguration(const ptree& config)
52+
{
53+
auto name = config.get<std::string>("id");
54+
DataSamplingPolicy policy(name);
5255

53-
mPaths.clear();
5456
size_t outputId = 0;
5557
std::vector<InputSpec> inputSpecs = DataDescriptorQueryBuilder::parse(config.get<std::string>("query").c_str());
5658

@@ -60,29 +62,31 @@ void DataSamplingPolicy::configure(const ptree& config)
6062
OutputSpec outputSpec{
6163
{inputSpec.binding},
6264
createPolicyDataOrigin(),
63-
createPolicyDataDescription(mName, outputId++),
65+
createPolicyDataDescription(name, outputId++),
6466
DataSpecUtils::getOptionalSubSpec(inputSpec).value(),
6567
inputSpec.lifetime};
6668

67-
mPaths.push_back({inputSpec, outputSpec});
69+
policy.registerPath(inputSpec, outputSpec);
6870

6971
} else {
7072
OutputSpec outputSpec{
7173
{inputSpec.binding},
72-
{createPolicyDataOrigin(), createPolicyDataDescription(mName, outputId++)},
74+
{createPolicyDataOrigin(), createPolicyDataDescription(name, outputId++)},
7375
inputSpec.lifetime};
7476

75-
mPaths.push_back({inputSpec, outputSpec});
77+
policy.registerPath(inputSpec, outputSpec);
7678
}
7779
}
7880

79-
mConditions.clear();
8081
for (const auto& conditionConfig : config.get_child("samplingConditions")) {
81-
mConditions.push_back(DataSamplingConditionFactory::create(conditionConfig.second.get<std::string>("condition")));
82-
mConditions.back()->configure(conditionConfig.second);
82+
auto condition = DataSamplingConditionFactory::create(conditionConfig.second.get<std::string>("condition"));
83+
condition->configure(conditionConfig.second);
84+
policy.registerCondition(std::move(condition));
8385
}
8486

85-
mFairMQOutputChannel = config.get_optional<std::string>("fairMQOutput").value_or("");
87+
policy.setFairMQOutputChannel(config.get_optional<std::string>("fairMQOutput").value_or(""));
88+
89+
return policy;
8690
}
8791

8892
bool DataSamplingPolicy::match(const ConcreteDataMatcher& input) const
@@ -103,7 +107,7 @@ bool DataSamplingPolicy::decide(const o2::framework::DataRef& dataRef)
103107
return decision;
104108
}
105109

106-
const Output DataSamplingPolicy::prepareOutput(const ConcreteDataMatcher& input, Lifetime lifetime) const
110+
Output DataSamplingPolicy::prepareOutput(const ConcreteDataMatcher& input, Lifetime lifetime) const
107111
{
108112
auto result = mPaths.find(input);
109113
if (result != mPaths.end()) {

‎Utilities/DataSampling/src/Dispatcher.cxx‎

Lines changed: 62 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,6 @@ namespace o2::utilities
3838
Dispatcher::Dispatcher(std::string name, const std::string reconfigurationSource)
3939
: mName(name), mReconfigurationSource(reconfigurationSource)
4040
{
41-
header::DataDescription timerDescription;
42-
timerDescription.runtimeInit(("TIMER-" + name).substr(0, 16).c_str());
43-
inputs.emplace_back(InputSpec{"timer-stats", "DS", timerDescription, 0, Lifetime::Timer});
4441
}
4542

4643
Dispatcher::~Dispatcher() = default;
@@ -55,15 +52,16 @@ void Dispatcher::init(InitContext& ctx)
5552
std::unique_ptr<ConfigurationInterface> cfg = ConfigurationFactory::getConfiguration(mReconfigurationSource);
5653
policiesTree = cfg->getRecursive("dataSamplingPolicies");
5754
mPolicies.clear();
58-
} else {
55+
} else if (ctx.options().isSet("sampling-config-ptree")) {
5956
policiesTree = ctx.options().get<boost::property_tree::ptree>("sampling-config-ptree");
6057
mPolicies.clear();
61-
}
58+
} else
59+
; // we use policies declared during workflow init.
6260

6361
for (auto&& policyConfig : policiesTree) {
6462
// we don't want the Dispatcher to exit due to one faulty Policy
6563
try {
66-
mPolicies.emplace_back(std::make_shared<DataSamplingPolicy>(policyConfig.second));
64+
mPolicies.emplace_back(std::make_shared<DataSamplingPolicy>(DataSamplingPolicy::fromConfiguration(policyConfig.second)));
6765
} catch (std::exception& ex) {
6866
LOG(WARN) << "Could not load the Data Sampling Policy '"
6967
<< policyConfig.second.get_optional<std::string>("id").value_or("") << "', because: " << ex.what();
@@ -188,22 +186,9 @@ void Dispatcher::sendFairMQ(FairMQDevice* device, const DataRef& inputData, cons
188186
int64_t bytesSent = device->Send(message, fairMQChannel);
189187
}
190188

191-
void Dispatcher::registerPath(const std::pair<InputSpec, OutputSpec>& path)
189+
void Dispatcher::registerPolicy(std::unique_ptr<DataSamplingPolicy>&& policy)
192190
{
193-
//todo: take care of inputs inclusive in others, when subSpec matchers are supported
194-
auto cmp = [a = path.first](const InputSpec b) {
195-
return a.matcher == b.matcher && a.lifetime == b.lifetime;
196-
};
197-
198-
if (std::find_if(inputs.begin(), inputs.end(), cmp) == inputs.end()) {
199-
inputs.push_back(path.first);
200-
LOG(DEBUG) << "Registering input " << DataSpecUtils::describe(path.first);
201-
} else {
202-
LOG(DEBUG) << "Input " << DataSpecUtils::describe(path.first)
203-
<< " already registered";
204-
}
205-
206-
outputs.push_back(path.second);
191+
mPolicies.emplace_back(std::move(policy));
207192
}
208193

209194
const std::string& Dispatcher::getName()
@@ -213,12 +198,66 @@ const std::string& Dispatcher::getName()
213198

214199
Inputs Dispatcher::getInputSpecs()
215200
{
216-
return inputs;
201+
Inputs declaredInputs;
202+
203+
// Add data inputs. Avoid duplicates and inputs which include others (e.g. "TST/DATA" includes "TST/DATA/1".
204+
for (const auto& policy : mPolicies) {
205+
for (const auto& [potentiallyNewInput, _policyOutput] : policy->getPathMap()) {
206+
(void)_policyOutput;
207+
208+
// The idea is that we remove all existing inputs which are covered by the potentially new input.
209+
// If there are none which are broader than the new one, then we add it.
210+
// I hope this is enough for all corner cases, but I am not 100% sure.
211+
auto newInputIsBroader = [&potentiallyNewInput](const InputSpec& other) {
212+
return DataSpecUtils::includes(potentiallyNewInput, other);
213+
};
214+
declaredInputs.erase(std::remove_if(declaredInputs.begin(), declaredInputs.end(), newInputIsBroader), declaredInputs.end());
215+
216+
auto declaredInputIsBroader = [&potentiallyNewInput](const InputSpec& other) {
217+
return DataSpecUtils::includes(other, potentiallyNewInput);
218+
};
219+
if (std::none_of(declaredInputs.begin(), declaredInputs.end(), declaredInputIsBroader)) {
220+
declaredInputs.push_back(potentiallyNewInput);
221+
}
222+
}
223+
}
224+
225+
// add timer input
226+
header::DataDescription timerDescription;
227+
timerDescription.runtimeInit(("TIMER-" + mName).substr(0, 16).c_str());
228+
declaredInputs.emplace_back(InputSpec{"timer-stats", "DS", timerDescription, 0, Lifetime::Timer});
229+
230+
return declaredInputs;
217231
}
218232

219233
Outputs Dispatcher::getOutputSpecs()
220234
{
221-
return outputs;
235+
Outputs declaredOutputs;
236+
for (const auto& policy : mPolicies) {
237+
for (const auto& [_policyInput, policyOutput] : policy->getPathMap()) {
238+
(void)_policyInput;
239+
// In principle Data Sampling Policies should have different outputs.
240+
// We may add a check to be very gentle with users.
241+
declaredOutputs.push_back(policyOutput);
242+
}
243+
}
244+
return declaredOutputs;
245+
}
246+
framework::Options Dispatcher::getOptions()
247+
{
248+
o2::framework::Options options;
249+
for (const auto& policy : mPolicies) {
250+
if (!policy->getFairMQOutputChannel().empty()) {
251+
if (!options.empty()) {
252+
throw std::runtime_error("Maximum one policy with raw FairMQ channel is allowed, more have been declared.");
253+
}
254+
options.push_back({"channel-config", VariantType::String, policy->getFairMQOutputChannel().c_str(), {"Out-of-band channel config"}});
255+
LOG(DEBUG) << " - registering output FairMQ channel '" << policy->getFairMQOutputChannel() << "'";
256+
}
257+
}
258+
options.push_back({"period-timer-stats", framework::VariantType::Int, 10 * 1000000, {"Dispatcher's stats timer period"}});
259+
260+
return options;
222261
}
223262

224263
} // namespace o2::utilities

0 commit comments

Comments
 (0)