Skip to content

Commit bdcc6a4

Browse files
ktfktf
authored
DPL: improve forwarding policy (#4609)
1 parent 233b460 commit bdcc6a4

File tree

4 files changed

+42
-14
lines changed

4 files changed

+42
-14
lines changed

‎Framework/Core/include/Framework/runDataProcessing.h‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,6 @@ int main(int argc, char** argv)
127127
UserCustomizationsHelper::userDefinedCustomization(workflowOptions, 0);
128128
workflowOptions.push_back(ConfigParamSpec{"readers", VariantType::Int64, 1ll, {"number of parallel readers to use"}});
129129
workflowOptions.push_back(ConfigParamSpec{"pipeline", VariantType::String, "", {"override default pipeline size"}});
130-
workflowOptions.push_back(ConfigParamSpec{"dangling-outputs-policy", VariantType::String, "file", {"what to do with dangling outputs. file: write to file, fairmq: send to output proxy"}});
131130

132131
// options for AOD rate limiting
133132
workflowOptions.push_back(ConfigParamSpec{"aod-memory-rate-limit", VariantType::Int64, 0LL, {"Rate limit AOD processing based on memory"}});
@@ -139,6 +138,18 @@ int main(int argc, char** argv)
139138
workflowOptions.push_back(ConfigParamSpec{"aod-writer-ntfmerge", VariantType::Int, -1, {"Number of time frames to merge into one file"}});
140139
workflowOptions.push_back(ConfigParamSpec{"aod-writer-keep", VariantType::String, "", {"Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename"}});
141140

141+
workflowOptions.push_back(ConfigParamSpec{"forwarding-policy",
142+
VariantType::String,
143+
"dangling",
144+
{"Which messages to forward."
145+
" dangling: dangling outputs,"
146+
" all: all messages"}});
147+
workflowOptions.push_back(ConfigParamSpec{"forwarding-destination",
148+
VariantType::String,
149+
"file",
150+
{"Destination for forwarded messages."
151+
" file: write to file,"
152+
" fairmq: send to output proxy"}});
142153
std::vector<ChannelConfigurationPolicy> channelPolicies;
143154
UserCustomizationsHelper::userDefinedCustomization(channelPolicies, 0);
144155
auto defaultChannelPolicies = ChannelConfigurationPolicy::createDefaultPolicies();

‎Framework/Core/src/CommonDataProcessors.cxx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -480,7 +480,7 @@ DataProcessorSpec CommonDataProcessors::getGlobalFairMQSink(std::vector<InputSpe
480480
externalChannelSpec.protocol = ChannelProtocol::IPC;
481481
std::string defaultChannelConfig = formatExternalChannelConfiguration(externalChannelSpec);
482482
// at some point the formatting tool might add the transport as well so we have to check
483-
return specifyFairMQDeviceOutputProxy("internal-dpl-output-proxy", danglingOutputInputs, defaultChannelConfig.c_str());
483+
return specifyFairMQDeviceOutputProxy("internal-dpl-injected-output-proxy", danglingOutputInputs, defaultChannelConfig.c_str());
484484
}
485485

486486
DataProcessorSpec CommonDataProcessors::getDummySink(std::vector<InputSpec> const& danglingOutputInputs)

‎Framework/Core/src/WorkflowHelpers.cxx‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -468,23 +468,32 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
468468
workflow.insert(workflow.end(), extraSpecs.begin(), extraSpecs.end());
469469
extraSpecs.clear();
470470

471-
// file sink for notAOD dangling outputs
472-
// select dangling outputs which are not of type AOD
473-
std::vector<InputSpec> outputsInputsDangling;
471+
// Select dangling outputs which are not of type AOD
472+
std::vector<InputSpec> redirectedOutputsInputs;
474473
for (auto ii = 0u; ii < outputsInputs.size(); ii++) {
475-
if ((outputTypes[ii] & DANGLING) == DANGLING && (outputTypes[ii] & ANALYSIS) == 0) {
476-
outputsInputsDangling.emplace_back(outputsInputs[ii]);
474+
if (ctx.options().get<std::string>("forwarding-policy") == "none") {
475+
continue;
476+
}
477+
// We forward to the output proxy all the inputs only if they are dangling
478+
// or if the forwarding policy is "proxy".
479+
if (!(outputTypes[ii] & DANGLING) && (ctx.options().get<std::string>("forwarding-policy") != "all")) {
480+
continue;
481+
}
482+
// AODs are skipped in any case.
483+
if ((outputTypes[ii] & ANALYSIS)) {
484+
continue;
477485
}
486+
redirectedOutputsInputs.emplace_back(outputsInputs[ii]);
478487
}
479488

480489
std::vector<InputSpec> unmatched;
481-
if (outputsInputsDangling.size() > 0 && ctx.options().get<std::string>("dangling-outputs-policy") == "file") {
482-
auto fileSink = CommonDataProcessors::getGlobalFileSink(outputsInputsDangling, unmatched);
483-
if (unmatched.size() != outputsInputsDangling.size()) {
490+
if (redirectedOutputsInputs.size() > 0 && ctx.options().get<std::string>("forwarding-destination") == "file") {
491+
auto fileSink = CommonDataProcessors::getGlobalFileSink(redirectedOutputsInputs, unmatched);
492+
if (unmatched.size() != redirectedOutputsInputs.size()) {
484493
extraSpecs.push_back(fileSink);
485494
}
486-
} else if (outputsInputsDangling.size() > 0 && ctx.options().get<std::string>("dangling-outputs-policy") == "fairmq") {
487-
auto fairMQSink = CommonDataProcessors::getGlobalFairMQSink(outputsInputsDangling);
495+
} else if (redirectedOutputsInputs.size() > 0 && ctx.options().get<std::string>("forwarding-destination") == "fairmq") {
496+
auto fairMQSink = CommonDataProcessors::getGlobalFairMQSink(redirectedOutputsInputs);
488497
extraSpecs.push_back(fairMQSink);
489498
}
490499
if (unmatched.size() > 0) {

‎Framework/Core/test/test_WorkflowHelpers.cxx‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,16 @@ std::unique_ptr<ConfigContext> makeEmptyConfigContext()
2929
// FIXME: Ugly... We need to fix ownership and make sure the ConfigContext
3030
// either owns or shares ownership of the registry.
3131
std::vector<std::unique_ptr<ParamRetriever>> retrievers;
32-
static std::vector<ConfigParamSpec> specs;
33-
specs.push_back(ConfigParamSpec{"dangling-outputs-policy", VariantType::String, "file", {"what to do with dangling outputs. file: write to file, fairmq: send to output proxy"}});
32+
static std::vector<ConfigParamSpec> specs = {
33+
ConfigParamSpec{"forwarding-policy",
34+
VariantType::String,
35+
"dangling",
36+
{""}},
37+
ConfigParamSpec{"forwarding-destination",
38+
VariantType::String,
39+
"file",
40+
{"what to do with dangling outputs. file: write to file, fairmq: send to output proxy"}},
41+
};
3442
specs.push_back(ConfigParamSpec{"aod-memory-rate-limit", VariantType::String, "0", {"rate"}});
3543
auto store = std::make_unique<ConfigParamStore>(specs, std::move(retrievers));
3644
store->preload();

0 commit comments

Comments
 (0)