Skip to content

Commit 577f91b

Browse files
knopers8knopers8
authored
DPL: Reconstruct the full command and propagate it to workflow dump tools (#5778)
1 parent 90ecb11 commit 577f91b

File tree

10 files changed

+145
-24
lines changed

10 files changed

+145
-24
lines changed

‎Framework/Core/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ o2_add_library(Framework
2525
src/ChannelMatching.cxx
2626
src/ChannelConfigurationPolicyHelpers.cxx
2727
src/ChannelSpecHelpers.cxx
28+
src/CommandInfo.cxx
2829
src/CommonDataProcessors.cxx
2930
src/CommonServices.cxx
3031
src/CommonMessageBackends.cxx

‎Framework/Core/include/Framework/CommandInfo.h‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright CERN and copyright holders of ALICE O2. This software is
2+
// distributed under the terms of the GNU General Public License v3 (GPL
3+
// Version 3), copied verbatim in the file "COPYING".
4+
//
5+
// See http://alice-o2.web.cern.ch/license for full licensing information.
6+
//
7+
// In applying this license CERN does not waive the privileges and immunities
8+
// granted to it by virtue of its status as an Intergovernmental Organization
9+
// or submit itself to any jurisdiction.
10+
11+
#ifndef O2_FRAMEWORK_COMMANDINFO_H_
12+
#define O2_FRAMEWORK_COMMANDINFO_H_
13+
14+
#include <string>
15+
16+
namespace o2::framework
17+
{
18+
19+
struct CommandInfo {
20+
CommandInfo() = default;
21+
CommandInfo(std::string command) : command(std::move(command)) {}
22+
CommandInfo(int argc, char* const* argv);
23+
24+
void merge(CommandInfo const& other);
25+
26+
std::string command;
27+
};
28+
29+
} // namespace o2::framework
30+
31+
#endif //O2_FRAMEWORK_COMMANDINFO_H_

‎Framework/Core/include/Framework/DriverControl.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include <functional>
1414
#include <vector>
1515

16+
#include "Framework/CommandInfo.h"
1617
#include "Framework/DriverInfo.h"
1718
#include "Framework/DataProcessorSpec.h"
1819
#include "Framework/DeviceSpec.h"
@@ -37,7 +38,8 @@ struct DriverControl {
3738
using Callback = std::function<void(std::vector<DataProcessorSpec> const& workflow,
3839
std::vector<DeviceSpec> const&,
3940
std::vector<DeviceExecution> const&,
40-
std::vector<DataProcessorInfo>&)>;
41+
std::vector<DataProcessorInfo>&,
42+
CommandInfo const&)>;
4143
/// States to be added to the stack on next iteration
4244
/// of the state machine processing.
4345
std::vector<DriverState> forcedTransitions;

‎Framework/Core/src/CommandInfo.cxx‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright CERN and copyright holders of ALICE O2. This software is
2+
// distributed under the terms of the GNU General Public License v3 (GPL
3+
// Version 3), copied verbatim in the file "COPYING".
4+
//
5+
// See http://alice-o2.web.cern.ch/license for full licensing information.
6+
//
7+
// In applying this license CERN does not waive the privileges and immunities
8+
// granted to it by virtue of its status as an Intergovernmental Organization
9+
// or submit itself to any jurisdiction.
10+
11+
#include "Framework/CommandInfo.h"
12+
13+
#include <sstream>
14+
#include <cassert>
15+
#include <cstring>
16+
17+
namespace o2::framework
18+
{
19+
20+
CommandInfo::CommandInfo(int argc, char* const* argv)
21+
{
22+
assert(argc > 0);
23+
24+
std::stringstream commandStream;
25+
commandStream << argv[0];
26+
27+
for (size_t ai = 1; ai < argc; ++ai) {
28+
const char* arg = argv[ai];
29+
if (strpbrk(arg, "\" ;@") != nullptr || arg[0] == 0) {
30+
commandStream << " '" << arg << "'";
31+
} else if (strpbrk(arg, "'") != nullptr) {
32+
commandStream << " \"" << arg << "\"";
33+
} else {
34+
commandStream << " " << arg;
35+
}
36+
}
37+
command = commandStream.str();
38+
}
39+
40+
void CommandInfo::merge(CommandInfo const& other)
41+
{
42+
if (!command.empty()) {
43+
command += " | ";
44+
}
45+
command += other.command;
46+
}
47+
48+
} // namespace o2::framework

‎Framework/Core/src/O2ControlHelpers.cxx‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -292,13 +292,14 @@ void dumpWorkflow(std::ostream& dumpOut, const std::vector<DeviceSpec>& specs, c
292292
auto& execution = executions[di];
293293
dumpRole(dumpOut, taskName(workflowName, spec.id), spec, specs, execution, indLevel + indScheme);
294294
}
295-
};
295+
}
296296

297297
} // namespace implementation
298298

299299
void dumpDeviceSpec2O2Control(std::string workflowName,
300300
const std::vector<DeviceSpec>& specs,
301-
const std::vector<DeviceExecution>& executions)
301+
const std::vector<DeviceExecution>& executions,
302+
const CommandInfo&)
302303
{
303304
const char* tasksDirectory = "tasks";
304305
const char* workflowsDirectory = "workflows";

‎Framework/Core/src/O2ControlHelpers.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
#include "Framework/DeviceSpec.h"
1414
#include "Framework/DeviceExecution.h"
15+
#include "Framework/CommandInfo.h"
1516
#include <vector>
1617
#include <iosfwd>
1718

@@ -40,7 +41,8 @@ namespace framework
4041

4142
void dumpDeviceSpec2O2Control(std::string workflowName,
4243
std::vector<DeviceSpec> const& specs,
43-
std::vector<DeviceExecution> const& executions);
44+
std::vector<DeviceExecution> const& executions,
45+
CommandInfo const& commandInfo);
4446

4547
} // namespace framework
4648
} // namespace o2

‎Framework/Core/src/WorkflowSerializationHelpers.cxx‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
3535
IN_EXECUTION,
3636
IN_WORKFLOW,
3737
IN_METADATA,
38+
IN_COMMAND,
3839
IN_DATAPROCESSORS,
3940
IN_DATAPROCESSOR,
4041
IN_DATAPROCESSOR_NAME,
@@ -87,6 +88,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
8788
case State::IN_WORKFLOW:
8889
s << "IN_WORKFLOW";
8990
break;
91+
case State::IN_COMMAND:
92+
s << "IN_COMMAND";
93+
break;
9094
case State::IN_DATAPROCESSORS:
9195
s << "IN_DATAPROCESSORS";
9296
break;
@@ -209,10 +213,12 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
209213
}
210214

211215
WorkflowImporter(std::vector<DataProcessorSpec>& o,
212-
std::vector<DataProcessorInfo>& m)
216+
std::vector<DataProcessorInfo>& m,
217+
CommandInfo& c)
213218
: states{},
214219
dataProcessors{o},
215-
metadata{m}
220+
metadata{m},
221+
command{c}
216222
{
217223
push(State::IN_START);
218224
}
@@ -245,6 +251,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
245251
metadata.push_back(DataProcessorInfo{});
246252
} else if (in(State::IN_METADATUM)) {
247253
metadata.push_back(DataProcessorInfo{});
254+
} else if (in(State::IN_COMMAND)) {
255+
command = CommandInfo{};
248256
}
249257
return true;
250258
}
@@ -469,6 +477,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
469477
push(State::IN_WORKFLOW_OPTIONS);
470478
} else if (in(State::IN_METADATUM) && strncmp(str, "channels", length) == 0) {
471479
push(State::IN_METADATUM_CHANNELS);
480+
} else if (in(State::IN_EXECUTION) && strncmp(str, "command", length) == 0) {
481+
push(State::IN_COMMAND);
472482
}
473483
return true;
474484
}
@@ -519,6 +529,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
519529
// This is in an array, so we do not actually want to
520530
// exit from the state.
521531
push(State::IN_METADATUM_CHANNEL);
532+
} else if (in(State::IN_COMMAND)) {
533+
command.merge({s});
522534
}
523535
pop();
524536
return true;
@@ -605,6 +617,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
605617
std::string spec;
606618
std::vector<DataProcessorSpec>& dataProcessors;
607619
std::vector<DataProcessorInfo>& metadata;
620+
CommandInfo& command;
608621
std::vector<ConfigParamSpec> inputOptions;
609622
std::string binding;
610623
header::DataOrigin origin;
@@ -623,7 +636,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
623636

624637
void WorkflowSerializationHelpers::import(std::istream& s,
625638
std::vector<DataProcessorSpec>& workflow,
626-
std::vector<DataProcessorInfo>& metadata)
639+
std::vector<DataProcessorInfo>& metadata,
640+
CommandInfo& command)
627641
{
628642
// Skip any line which does not start with '{'
629643
// If we do not find a starting {, we simply assume that no workflow
@@ -642,7 +656,7 @@ void WorkflowSerializationHelpers::import(std::istream& s,
642656
}
643657
rapidjson::Reader reader;
644658
rapidjson::IStreamWrapper isw(s);
645-
WorkflowImporter importer{workflow, metadata};
659+
WorkflowImporter importer{workflow, metadata, command};
646660
bool ok = reader.Parse(isw, importer);
647661
if (ok == false) {
648662
throw std::runtime_error("Error while parsing serialised workflow");
@@ -651,7 +665,8 @@ void WorkflowSerializationHelpers::import(std::istream& s,
651665

652666
void WorkflowSerializationHelpers::dump(std::ostream& out,
653667
std::vector<DataProcessorSpec> const& workflow,
654-
std::vector<DataProcessorInfo> const& metadata)
668+
std::vector<DataProcessorInfo> const& metadata,
669+
CommandInfo const& commandInfo)
655670
{
656671
rapidjson::OStreamWrapper osw(out);
657672
rapidjson::PrettyWriter<rapidjson::OStreamWrapper> w(osw);
@@ -841,6 +856,10 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
841856
w.EndObject();
842857
}
843858
w.EndArray();
859+
860+
w.Key("command");
861+
w.String(commandInfo.command.c_str());
862+
844863
w.EndObject();
845864
}
846865

‎Framework/Core/src/WorkflowSerializationHelpers.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
#include "Framework/DataProcessorSpec.h"
1414
#include "Framework/DataProcessorInfo.h"
15+
#include "Framework/CommandInfo.h"
1516

1617
#include <iosfwd>
1718
#include <vector>
@@ -22,10 +23,12 @@ namespace o2::framework
2223
struct WorkflowSerializationHelpers {
2324
static void import(std::istream& s,
2425
std::vector<DataProcessorSpec>& workflow,
25-
std::vector<DataProcessorInfo>& metadata);
26+
std::vector<DataProcessorInfo>& metadata,
27+
CommandInfo& command);
2628
static void dump(std::ostream& o,
2729
std::vector<DataProcessorSpec> const& workflow,
28-
std::vector<DataProcessorInfo> const& metadata);
30+
std::vector<DataProcessorInfo> const& metadata,
31+
CommandInfo const& commandInfo);
2932
};
3033

3134
} // namespace o2::framework

‎Framework/Core/src/runDataProcessing.cxx‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
#include "Framework/DataProcessorInfo.h"
4141
#include "Framework/DriverInfo.h"
4242
#include "Framework/DriverControl.h"
43+
#include "Framework/CommandInfo.h"
4344
#include "DriverServerContext.h"
4445
#include "ControlServiceHelpers.h"
4546
#include "HTTPParser.h"
@@ -1097,6 +1098,7 @@ void single_step_callback(uv_timer_s* ctx)
10971098
int runStateMachine(DataProcessorSpecs const& workflow,
10981099
WorkflowInfo const& workflowInfo,
10991100
DataProcessorInfos const& previousDataProcessorInfos,
1101+
CommandInfo const& commandInfo,
11001102
DriverControl& driverControl,
11011103
DriverInfo& driverInfo,
11021104
std::vector<DeviceMetricsInfo>& metricsInfos,
@@ -1491,7 +1493,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
14911493
// restart the data processors which need to be restarted.
14921494
LOG(INFO) << "Redeployment of configuration asked.";
14931495
std::ostringstream forwardedStdin;
1494-
WorkflowSerializationHelpers::dump(forwardedStdin, workflow, dataProcessorInfos);
1496+
WorkflowSerializationHelpers::dump(forwardedStdin, workflow, dataProcessorInfos, commandInfo);
14951497
infos.reserve(deviceSpecs.size());
14961498

14971499
// This is guaranteed to be a single CPU.
@@ -1673,7 +1675,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
16731675
}
16741676
case DriverState::PERFORM_CALLBACKS:
16751677
for (auto& callback : driverControl.callbacks) {
1676-
callback(workflow, deviceSpecs, deviceExecutions, dataProcessorInfos);
1678+
callback(workflow, deviceSpecs, deviceExecutions, dataProcessorInfos, commandInfo);
16771679
}
16781680
driverControl.callbacks.clear();
16791681
break;
@@ -1844,7 +1846,8 @@ void initialiseDriverControl(bpo::variables_map const& varmap,
18441846
control.callbacks = {[](WorkflowSpec const& workflow,
18451847
DeviceSpecs const& specs,
18461848
DeviceExecutions const&,
1847-
DataProcessorInfos&) {
1849+
DataProcessorInfos&,
1850+
CommandInfo const&) {
18481851
GraphvizHelpers::dumpDeviceSpec2Graphviz(std::cout, specs);
18491852
}};
18501853
control.forcedTransitions = {
@@ -1862,7 +1865,8 @@ void initialiseDriverControl(bpo::variables_map const& varmap,
18621865
control.callbacks = {[](WorkflowSpec const& workflow,
18631866
DeviceSpecs const& specs,
18641867
DeviceExecutions const& executions,
1865-
DataProcessorInfos&) {
1868+
DataProcessorInfos&,
1869+
CommandInfo const&) {
18661870
dumpDeviceSpec2DDS(std::cout, specs, executions);
18671871
}};
18681872
control.forcedTransitions = {
@@ -1877,8 +1881,9 @@ void initialiseDriverControl(bpo::variables_map const& varmap,
18771881
(WorkflowSpec const& workflow,
18781882
DeviceSpecs const& specs,
18791883
DeviceExecutions const& executions,
1880-
DataProcessorInfos&) {
1881-
dumpDeviceSpec2O2Control(workflowName, specs, executions);
1884+
DataProcessorInfos&,
1885+
CommandInfo const& commandInfo) {
1886+
dumpDeviceSpec2O2Control(workflowName, specs, executions, commandInfo);
18821887
}};
18831888
control.forcedTransitions = {
18841889
DriverState::EXIT, //
@@ -1903,14 +1908,15 @@ void initialiseDriverControl(bpo::variables_map const& varmap,
19031908
control.callbacks = {[filename = varmap["dump-workflow-file"].as<std::string>()](WorkflowSpec const& workflow,
19041909
DeviceSpecs const devices,
19051910
DeviceExecutions const&,
1906-
DataProcessorInfos& dataProcessorInfos) {
1911+
DataProcessorInfos& dataProcessorInfos,
1912+
CommandInfo const& commandInfo) {
19071913
if (filename == "-") {
1908-
WorkflowSerializationHelpers::dump(std::cout, workflow, dataProcessorInfos);
1914+
WorkflowSerializationHelpers::dump(std::cout, workflow, dataProcessorInfos, commandInfo);
19091915
// FIXME: this is to avoid trailing garbage..
19101916
exit(0);
19111917
} else {
19121918
std::ofstream output(filename);
1913-
WorkflowSerializationHelpers::dump(output, workflow, dataProcessorInfos);
1919+
WorkflowSerializationHelpers::dump(output, workflow, dataProcessorInfos, commandInfo);
19141920
}
19151921
}};
19161922
control.forcedTransitions = {
@@ -2078,9 +2084,10 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
20782084
}
20792085

20802086
std::vector<DataProcessorInfo> dataProcessorInfos;
2087+
CommandInfo commandInfo{};
20812088
if (isatty(STDIN_FILENO) == false) {
20822089
std::vector<DataProcessorSpec> importedWorkflow;
2083-
WorkflowSerializationHelpers::import(std::cin, importedWorkflow, dataProcessorInfos);
2090+
WorkflowSerializationHelpers::import(std::cin, importedWorkflow, dataProcessorInfos, commandInfo);
20842091

20852092
size_t workflowHashB = 0;
20862093
for (auto& dp : importedWorkflow) {
@@ -2260,6 +2267,8 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
22602267
driverInfo.processorInfo = dataProcessorInfos;
22612268
driverInfo.configContext = &configContext;
22622269

2270+
commandInfo.merge(CommandInfo(argc, argv));
2271+
22632272
std::string frameworkId;
22642273
// If the id is set, this means this is a device,
22652274
// otherwise this is the driver.
@@ -2274,6 +2283,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
22742283
return runStateMachine(physicalWorkflow,
22752284
currentWorkflow,
22762285
dataProcessorInfos,
2286+
commandInfo,
22772287
driverControl,
22782288
driverInfo,
22792289
gDeviceMetricsInfos,

0 commit comments

Comments
 (0)