Skip to content

DPL: Reconstruct the full command and propagate it to workflow dump tools #5778

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ktf merged 1 commit into from
Mar 25, 2021
Merged

DPL: Reconstruct the full command and propagate it to workflow dump tools #5778

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Framework/Core/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ o2_add_library(Framework
src/ChannelMatching.cxx
src/ChannelConfigurationPolicyHelpers.cxx
src/ChannelSpecHelpers.cxx
src/CommandInfo.cxx
src/CommonDataProcessors.cxx
src/CommonServices.cxx
src/CommonMessageBackends.cxx
Expand Down
31 changes: 31 additions & 0 deletions Framework/Core/include/Framework/CommandInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright CERN and copyright holders of ALICE O2. This software is
// distributed under the terms of the GNU General Public License v3 (GPL
// Version 3), copied verbatim in the file "COPYING".
//
// See http://alice-o2.web.cern.ch/license for full licensing information.
//
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.

#ifndef O2_FRAMEWORK_COMMANDINFO_H_
#define O2_FRAMEWORK_COMMANDINFO_H_

#include <string>

namespace o2::framework
{

struct CommandInfo {
CommandInfo() = default;
CommandInfo(std::string command) : command(std::move(command)) {}
CommandInfo(int argc, char* const* argv);

void merge(CommandInfo const& other);

std::string command;
};

} // namespace o2::framework

#endif //O2_FRAMEWORK_COMMANDINFO_H_
4 changes: 3 additions & 1 deletion Framework/Core/include/Framework/DriverControl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include <functional>
#include <vector>

#include "Framework/CommandInfo.h"
#include "Framework/DriverInfo.h"
#include "Framework/DataProcessorSpec.h"
#include "Framework/DeviceSpec.h"
Expand All @@ -37,7 +38,8 @@ struct DriverControl {
using Callback = std::function<void(std::vector<DataProcessorSpec> const& workflow,
std::vector<DeviceSpec> const&,
std::vector<DeviceExecution> const&,
std::vector<DataProcessorInfo>&)>;
std::vector<DataProcessorInfo>&,
CommandInfo const&)>;
/// States to be added to the stack on next iteration
/// of the state machine processing.
std::vector<DriverState> forcedTransitions;
Expand Down
48 changes: 48 additions & 0 deletions Framework/Core/src/CommandInfo.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright CERN and copyright holders of ALICE O2. This software is
// distributed under the terms of the GNU General Public License v3 (GPL
// Version 3), copied verbatim in the file "COPYING".
//
// See http://alice-o2.web.cern.ch/license for full licensing information.
//
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.

#include "Framework/CommandInfo.h"

#include <sstream>
#include <cassert>
#include <cstring>

namespace o2::framework
{

CommandInfo::CommandInfo(int argc, char* const* argv)
{
assert(argc > 0);

std::stringstream commandStream;
commandStream << argv[0];

for (size_t ai = 1; ai < argc; ++ai) {
const char* arg = argv[ai];
if (strpbrk(arg, "\" ;@") != nullptr || arg[0] == 0) {
commandStream << " '" << arg << "'";
} else if (strpbrk(arg, "'") != nullptr) {
commandStream << " \"" << arg << "\"";
} else {
commandStream << " " << arg;
}
}
command = commandStream.str();
}

void CommandInfo::merge(CommandInfo const& other)
{
if (!command.empty()) {
command += " | ";
}
command += other.command;
}

} // namespace o2::framework
5 changes: 3 additions & 2 deletions Framework/Core/src/O2ControlHelpers.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,13 +292,14 @@ void dumpWorkflow(std::ostream& dumpOut, const std::vector<DeviceSpec>& specs, c
auto& execution = executions[di];
dumpRole(dumpOut, taskName(workflowName, spec.id), spec, specs, execution, indLevel + indScheme);
}
};
}

} // namespace implementation

void dumpDeviceSpec2O2Control(std::string workflowName,
const std::vector<DeviceSpec>& specs,
const std::vector<DeviceExecution>& executions)
const std::vector<DeviceExecution>& executions,
const CommandInfo&)
{
const char* tasksDirectory = "tasks";
const char* workflowsDirectory = "workflows";
Expand Down
4 changes: 3 additions & 1 deletion Framework/Core/src/O2ControlHelpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

#include "Framework/DeviceSpec.h"
#include "Framework/DeviceExecution.h"
#include "Framework/CommandInfo.h"
#include <vector>
#include <iosfwd>

Expand Down Expand Up @@ -40,7 +41,8 @@ namespace framework

void dumpDeviceSpec2O2Control(std::string workflowName,
std::vector<DeviceSpec> const& specs,
std::vector<DeviceExecution> const& executions);
std::vector<DeviceExecution> const& executions,
CommandInfo const& commandInfo);

} // namespace framework
} // namespace o2
Expand Down
29 changes: 24 additions & 5 deletions Framework/Core/src/WorkflowSerializationHelpers.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
IN_EXECUTION,
IN_WORKFLOW,
IN_METADATA,
IN_COMMAND,
IN_DATAPROCESSORS,
IN_DATAPROCESSOR,
IN_DATAPROCESSOR_NAME,
Expand Down Expand Up @@ -87,6 +88,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
case State::IN_WORKFLOW:
s << "IN_WORKFLOW";
break;
case State::IN_COMMAND:
s << "IN_COMMAND";
break;
case State::IN_DATAPROCESSORS:
s << "IN_DATAPROCESSORS";
break;
Expand Down Expand Up @@ -209,10 +213,12 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
}

WorkflowImporter(std::vector<DataProcessorSpec>& o,
std::vector<DataProcessorInfo>& m)
std::vector<DataProcessorInfo>& m,
CommandInfo& c)
: states{},
dataProcessors{o},
metadata{m}
metadata{m},
command{c}
{
push(State::IN_START);
}
Expand Down Expand Up @@ -245,6 +251,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
metadata.push_back(DataProcessorInfo{});
} else if (in(State::IN_METADATUM)) {
metadata.push_back(DataProcessorInfo{});
} else if (in(State::IN_COMMAND)) {
command = CommandInfo{};
}
return true;
}
Expand Down Expand Up @@ -469,6 +477,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
push(State::IN_WORKFLOW_OPTIONS);
} else if (in(State::IN_METADATUM) && strncmp(str, "channels", length) == 0) {
push(State::IN_METADATUM_CHANNELS);
} else if (in(State::IN_EXECUTION) && strncmp(str, "command", length) == 0) {
push(State::IN_COMMAND);
}
return true;
}
Expand Down Expand Up @@ -519,6 +529,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
// This is in an array, so we do not actually want to
// exit from the state.
push(State::IN_METADATUM_CHANNEL);
} else if (in(State::IN_COMMAND)) {
command.merge({s});
}
pop();
return true;
Expand Down Expand Up @@ -605,6 +617,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
std::string spec;
std::vector<DataProcessorSpec>& dataProcessors;
std::vector<DataProcessorInfo>& metadata;
CommandInfo& command;
std::vector<ConfigParamSpec> inputOptions;
std::string binding;
header::DataOrigin origin;
Expand All @@ -623,7 +636,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,

void WorkflowSerializationHelpers::import(std::istream& s,
std::vector<DataProcessorSpec>& workflow,
std::vector<DataProcessorInfo>& metadata)
std::vector<DataProcessorInfo>& metadata,
CommandInfo& command)
{
// Skip any line which does not start with '{'
// If we do not find a starting {, we simply assume that no workflow
Expand All @@ -642,7 +656,7 @@ void WorkflowSerializationHelpers::import(std::istream& s,
}
rapidjson::Reader reader;
rapidjson::IStreamWrapper isw(s);
WorkflowImporter importer{workflow, metadata};
WorkflowImporter importer{workflow, metadata, command};
bool ok = reader.Parse(isw, importer);
if (ok == false) {
throw std::runtime_error("Error while parsing serialised workflow");
Expand All @@ -651,7 +665,8 @@ void WorkflowSerializationHelpers::import(std::istream& s,

void WorkflowSerializationHelpers::dump(std::ostream& out,
std::vector<DataProcessorSpec> const& workflow,
std::vector<DataProcessorInfo> const& metadata)
std::vector<DataProcessorInfo> const& metadata,
CommandInfo const& commandInfo)
{
rapidjson::OStreamWrapper osw(out);
rapidjson::PrettyWriter<rapidjson::OStreamWrapper> w(osw);
Expand Down Expand Up @@ -841,6 +856,10 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
w.EndObject();
}
w.EndArray();

w.Key("command");
w.String(commandInfo.command.c_str());

w.EndObject();
}

Expand Down
7 changes: 5 additions & 2 deletions Framework/Core/src/WorkflowSerializationHelpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

#include "Framework/DataProcessorSpec.h"
#include "Framework/DataProcessorInfo.h"
#include "Framework/CommandInfo.h"

#include <iosfwd>
#include <vector>
Expand All @@ -22,10 +23,12 @@ namespace o2::framework
struct WorkflowSerializationHelpers {
static void import(std::istream& s,
std::vector<DataProcessorSpec>& workflow,
std::vector<DataProcessorInfo>& metadata);
std::vector<DataProcessorInfo>& metadata,
CommandInfo& command);
static void dump(std::ostream& o,
std::vector<DataProcessorSpec> const& workflow,
std::vector<DataProcessorInfo> const& metadata);
std::vector<DataProcessorInfo> const& metadata,
CommandInfo const& commandInfo);
};

} // namespace o2::framework
Expand Down
30 changes: 20 additions & 10 deletions Framework/Core/src/runDataProcessing.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
#include "Framework/DataProcessorInfo.h"
#include "Framework/DriverInfo.h"
#include "Framework/DriverControl.h"
#include "Framework/CommandInfo.h"
#include "DriverServerContext.h"
#include "ControlServiceHelpers.h"
#include "HTTPParser.h"
Expand Down Expand Up @@ -1097,6 +1098,7 @@ void single_step_callback(uv_timer_s* ctx)
int runStateMachine(DataProcessorSpecs const& workflow,
WorkflowInfo const& workflowInfo,
DataProcessorInfos const& previousDataProcessorInfos,
CommandInfo const& commandInfo,
DriverControl& driverControl,
DriverInfo& driverInfo,
std::vector<DeviceMetricsInfo>& metricsInfos,
Expand Down Expand Up @@ -1491,7 +1493,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
// restart the data processors which need to be restarted.
LOG(INFO) << "Redeployment of configuration asked.";
std::ostringstream forwardedStdin;
WorkflowSerializationHelpers::dump(forwardedStdin, workflow, dataProcessorInfos);
WorkflowSerializationHelpers::dump(forwardedStdin, workflow, dataProcessorInfos, commandInfo);
infos.reserve(deviceSpecs.size());

// This is guaranteed to be a single CPU.
Expand Down Expand Up @@ -1673,7 +1675,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
}
case DriverState::PERFORM_CALLBACKS:
for (auto& callback : driverControl.callbacks) {
callback(workflow, deviceSpecs, deviceExecutions, dataProcessorInfos);
callback(workflow, deviceSpecs, deviceExecutions, dataProcessorInfos, commandInfo);
}
driverControl.callbacks.clear();
break;
Expand Down Expand Up @@ -1844,7 +1846,8 @@ void initialiseDriverControl(bpo::variables_map const& varmap,
control.callbacks = {[](WorkflowSpec const& workflow,
DeviceSpecs const& specs,
DeviceExecutions const&,
DataProcessorInfos&) {
DataProcessorInfos&,
CommandInfo const&) {
GraphvizHelpers::dumpDeviceSpec2Graphviz(std::cout, specs);
}};
control.forcedTransitions = {
Expand All @@ -1862,7 +1865,8 @@ void initialiseDriverControl(bpo::variables_map const& varmap,
control.callbacks = {[](WorkflowSpec const& workflow,
DeviceSpecs const& specs,
DeviceExecutions const& executions,
DataProcessorInfos&) {
DataProcessorInfos&,
CommandInfo const&) {
dumpDeviceSpec2DDS(std::cout, specs, executions);
}};
control.forcedTransitions = {
Expand All @@ -1877,8 +1881,9 @@ void initialiseDriverControl(bpo::variables_map const& varmap,
(WorkflowSpec const& workflow,
DeviceSpecs const& specs,
DeviceExecutions const& executions,
DataProcessorInfos&) {
dumpDeviceSpec2O2Control(workflowName, specs, executions);
DataProcessorInfos&,
CommandInfo const& commandInfo) {
dumpDeviceSpec2O2Control(workflowName, specs, executions, commandInfo);
}};
control.forcedTransitions = {
DriverState::EXIT, //
Expand All @@ -1903,14 +1908,15 @@ void initialiseDriverControl(bpo::variables_map const& varmap,
control.callbacks = {[filename = varmap["dump-workflow-file"].as<std::string>()](WorkflowSpec const& workflow,
DeviceSpecs const devices,
DeviceExecutions const&,
DataProcessorInfos& dataProcessorInfos) {
DataProcessorInfos& dataProcessorInfos,
CommandInfo const& commandInfo) {
if (filename == "-") {
WorkflowSerializationHelpers::dump(std::cout, workflow, dataProcessorInfos);
WorkflowSerializationHelpers::dump(std::cout, workflow, dataProcessorInfos, commandInfo);
// FIXME: this is to avoid trailing garbage..
exit(0);
} else {
std::ofstream output(filename);
WorkflowSerializationHelpers::dump(output, workflow, dataProcessorInfos);
WorkflowSerializationHelpers::dump(output, workflow, dataProcessorInfos, commandInfo);
}
}};
control.forcedTransitions = {
Expand Down Expand Up @@ -2078,9 +2084,10 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
}

std::vector<DataProcessorInfo> dataProcessorInfos;
CommandInfo commandInfo{};
if (isatty(STDIN_FILENO) == false) {
std::vector<DataProcessorSpec> importedWorkflow;
WorkflowSerializationHelpers::import(std::cin, importedWorkflow, dataProcessorInfos);
WorkflowSerializationHelpers::import(std::cin, importedWorkflow, dataProcessorInfos, commandInfo);

size_t workflowHashB = 0;
for (auto& dp : importedWorkflow) {
Expand Down Expand Up @@ -2260,6 +2267,8 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
driverInfo.processorInfo = dataProcessorInfos;
driverInfo.configContext = &configContext;

commandInfo.merge(CommandInfo(argc, argv));

std::string frameworkId;
// If the id is set, this means this is a device,
// otherwise this is the driver.
Expand All @@ -2274,6 +2283,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
return runStateMachine(physicalWorkflow,
currentWorkflow,
dataProcessorInfos,
commandInfo,
driverControl,
driverInfo,
gDeviceMetricsInfos,
Expand Down
Loading