forked from mcoquet642/AliceO2
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy patho2ParallelWorkflow.cxx
More file actions
121 lines (106 loc) · 5.53 KB
/
o2ParallelWorkflow.cxx
File metadata and controls
121 lines (106 loc) · 5.53 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
// All rights not expressly granted are reserved.
//
// This software is distributed under the terms of the GNU General Public
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
//
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
#include "Framework/ConfigParamSpec.h"
#include <chrono>
#include <thread>
#include <vector>
using namespace o2::framework;
// we need to add workflow options before including Framework/runDataProcessing
void customize(std::vector<ConfigParamSpec>& workflowOptions)
{
std::string spaceParallelHelp("Number of tpc processing lanes. A lane is a pipeline of algorithms.");
workflowOptions.push_back(
ConfigParamSpec{"2-layer-jobs", VariantType::Int, 1, {spaceParallelHelp}});
std::string timeHelp("Time pipelining happening in the second layer");
workflowOptions.push_back(
ConfigParamSpec{"3-layer-pipelining", VariantType::Int, 1, {timeHelp}});
}
#include "Framework/runDataProcessing.h"
#include "Framework/DataProcessorSpec.h"
#include "Framework/DataSpecUtils.h"
#include "Framework/ParallelContext.h"
#include "Framework/ControlService.h"
#include "Framework/Logger.h"
#include <vector>
using DataHeader = o2::header::DataHeader;
DataProcessorSpec templateProcessor()
{
return DataProcessorSpec{"some-processor", {
InputSpec{"x", "TST", "A", 0, Lifetime::Timeframe},
},
{
OutputSpec{"TST", "P", 0, Lifetime::Timeframe},
},
// The producer is stateful, we use a static for the state in this
// particular case, but a Singleton or a captured new object would
// work as well.
AlgorithmSpec{[](InitContext& setup) {
srand(setup.services().get<ParallelContext>().index1D());
return [](ProcessingContext& ctx) {
// Create a single output.
size_t index = ctx.services().get<ParallelContext>().index1D();
auto& aData = ctx.outputs().make<int>(
Output{"TST", "P", static_cast<o2::header::DataHeader::SubSpecificationType>(index)}, 1);
std::this_thread::sleep_for(std::chrono::seconds(rand() % 5));
};
}}};
}
// This is a simple consumer / producer workflow where both are
// stateful, i.e. they have context which comes from their initialization.
WorkflowSpec defineDataProcessing(ConfigContext const& config)
{
size_t jobs = config.options().get<int>("2-layer-jobs");
size_t stages = config.options().get<int>("3-layer-pipelining");
// This is an example of how we can parallelize by subSpec.
// templatedProducer will be instanciated 32 times and the lambda function
// passed to the parallel statement will be applied to each one of the
// instances in order to modify it. Parallel will also make sure the name of
// the instance is amended from "some-producer" to "some-producer-<index>".
WorkflowSpec workflow = parallel(templateProcessor(), jobs, [](DataProcessorSpec& spec, size_t index) {
DataSpecUtils::updateMatchingSubspec(spec.inputs[0], index);
DataSpecUtils::updateMatchingSubspec(spec.outputs[0], index);
});
std::vector<OutputSpec> outputSpecs;
for (size_t ssi = 0; ssi < jobs; ++ssi) {
outputSpecs.emplace_back("TST", "A", ssi);
}
workflow.push_back(DataProcessorSpec{"reader", {}, outputSpecs, AlgorithmSpec{[jobs](InitContext& initCtx) {
return [jobs](ProcessingContext& ctx) {
for (size_t ji = 0; ji < jobs; ++ji) {
ctx.outputs().make<int>(Output{"TST", "A", static_cast<o2::header::DataHeader::SubSpecificationType>(ji)},
1);
}
};
}}});
workflow.push_back(timePipeline(DataProcessorSpec{
"merger",
mergeInputs(InputSpec{"x", "TST", "P"},
jobs,
[](InputSpec& input, size_t index) {
DataSpecUtils::updateMatchingSubspec(input, index);
}),
{OutputSpec{{"out"}, "TST", "M"}},
AlgorithmSpec{[](InitContext& setup) {
return [](ProcessingContext& ctx) {
ctx.outputs().make<int>(OutputRef("out", 0), 1);
};
}}},
stages));
workflow.push_back(DataProcessorSpec{
"writer",
{InputSpec{"x", "TST", "M"}},
{},
AlgorithmSpec{[](InitContext& setup) {
return [](ProcessingContext& ctx) {
};
}}});
return workflow;
}