AliceO2Group · ktf · Oct 22, 2019 · Oct 21, 2019 · Oct 22, 2019
@@ -152,6 +152,7 @@ foreach(t
         CallbackRegistry
         ChannelSpecHelpers
         CompletionPolicy
+        ComputingResourceHelpers
         ConfigParamRegistry
         ContextRegistry
         DataDescriptorMatcher
@@ -273,7 +274,7 @@ foreach(w
               PUBLIC_LINK_LIBRARIES O2::Framework
               TIMEOUT 30
               NO_BOOST_TEST
-              COMMAND_LINE_ARGS ${DPL_WORKFLOW_TESTS_EXTRA_OPTIONS} --run)
+              COMMAND_LINE_ARGS ${DPL_WORKFLOW_TESTS_EXTRA_OPTIONS} --run --shm-segment-size 20000000)
 endforeach()
 
 # TODO: DanglingInput test not working for the moment [ERROR] Unable to relay

@@ -32,6 +32,12 @@ enum struct ChannelType {
   Pull,
 };
 
+/// The kind of backend to use for the channels
+enum struct ChannelProtocol {
+  Network,
+  IPC
+};
+
 /// This describes an input channel. Since they are point to
 /// point connections, there is not much to say about them.
 /// Notice that this should be considered read only once it
@@ -42,6 +48,7 @@ struct InputChannelSpec {
   enum ChannelMethod method;
   std::string hostname;
   unsigned short port;
+  ChannelProtocol protocol = ChannelProtocol::Network;
 };
 
 /// This describes an output channel. Output channels are semantically
@@ -56,6 +63,7 @@ struct OutputChannelSpec {
   std::string hostname;
   unsigned short port;
   size_t listeners;
+  ChannelProtocol protocol = ChannelProtocol::Network;
 };
 
 } // namespace framework

@@ -44,14 +44,24 @@ char const* ChannelSpecHelpers::methodAsString(enum ChannelMethod method)
 
 std::string ChannelSpecHelpers::channelUrl(OutputChannelSpec const& channel)
 {
-  return channel.method == ChannelMethod::Bind ? fmt::format("tcp://*:{}", channel.port)
-                                               : fmt::format("tcp://{}:{}", channel.hostname, channel.port);
+  switch (channel.protocol) {
+    case ChannelProtocol::IPC:
+      return fmt::format("ipc://{}_{}", channel.hostname, channel.port);
+    default:
+      return channel.method == ChannelMethod::Bind ? fmt::format("tcp://*:{}", channel.port)
+                                                   : fmt::format("tcp://{}:{}", channel.hostname, channel.port);
+  }
 }
 
 std::string ChannelSpecHelpers::channelUrl(InputChannelSpec const& channel)
 {
-  return channel.method == ChannelMethod::Bind ? fmt::format("tcp://*:{}", channel.port)
-                                               : fmt::format("tcp://{}:{}", channel.hostname, channel.port);
+  switch (channel.protocol) {
+    case ChannelProtocol::IPC:
+      return fmt::format("ipc://{}_{}", channel.hostname, channel.port);
+    default:
+      return channel.method == ChannelMethod::Bind ? fmt::format("tcp://*:{}", channel.port)
+                                                   : fmt::format("tcp://{}:{}", channel.hostname, channel.port);
+  }
 }
 
 /// Stream operators so that we can use ChannelType with Boost.Test

@@ -10,6 +10,7 @@
 #include "ComputingResourceHelpers.h"
 #include <thread>
 #include <unistd.h>
+#include <sstream>
 
 namespace o2::framework
 {
@@ -20,15 +21,34 @@ long getTotalNumberOfBytes()
   return pages * page_size;
 };
 
-ComputingResource ComputingResourceHelpers::getLocalhostResource(unsigned short startPort, unsigned short rangeSize)
+ComputingResource ComputingResourceHelpers::getLocalhostResource()
 {
   ComputingResource result;
   result.cpu = std::thread::hardware_concurrency(),
   result.memory = getTotalNumberOfBytes();
   result.hostname = "localhost";
-  result.startPort = startPort;
-  result.lastPort = startPort + rangeSize;
+  result.startPort = 22000;
+  result.lastPort = 23000;
   result.usedPorts = 0;
   return result;
 }
+
+std::vector<ComputingResource> ComputingResourceHelpers::parseResources(std::string const& resourceString)
+{
+  std::vector<ComputingResource> resources;
+  std::istringstream str{resourceString};
+  std::string result;
+  while (std::getline(str, result, ',')) {
+    std::istringstream in{result};
+    char colon;
+    ComputingResource resource;
+    std::getline(in, resource.hostname, ':');
+    in >> resource.cpu >> colon >> resource.memory >> colon >> resource.startPort >> colon >> resource.lastPort;
+    resource.memory = resource.memory * 1000000;
+    resource.usedPorts = 0;
+    resources.emplace_back(resource);
+  }
+  return resources;
+}
+
 } // namespace o2::framework
@@ -13,10 +13,21 @@
 
 #include "Framework/ComputingResource.h"
 
+#include <string>
+#include <vector>
+
 namespace o2::framework
 {
 struct ComputingResourceHelpers {
-  static ComputingResource getLocalhostResource(unsigned short startPort, unsigned short rangeSize);
+  /// This will create a ComputingResource which matches what offered by localhost.
+  /// Notice that the port range will always be [22000, 23000) since in any case we will
+  /// use ipc:// in place of tcp://
+  static ComputingResource getLocalhostResource();
+
+  /// Parse a string which contains resources specified in the following format:
+  ///
+  /// <hostname>:<cpu cores>:<memory in MB>:<start port>:<last port>
+  static std::vector<ComputingResource> parseResources(std::string const& resourceString);
 };
 } // namespace o2::framework
 

@@ -27,6 +27,7 @@
 #include "Framework/OutputRoute.h"
 #include "Framework/WorkflowSpec.h"
 #include "Framework/ComputingResource.h"
+#include "Framework/Logger.h"
 
 #include "WorkflowHelpers.h"
 
@@ -625,7 +626,8 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(WorkflowSpec const& workf
                                                        std::vector<CompletionPolicy> const& completionPolicies,
                                                        std::vector<DispatchPolicy> const& dispatchPolicies,
                                                        std::vector<DeviceSpec>& devices,
-                                                       ResourceManager& resourceManager)
+                                                       ResourceManager& resourceManager,
+                                                       std::string const& uniqueWorkflowId)
 {
 
   std::vector<LogicalForwardInfo> availableForwardsInfo;
@@ -701,6 +703,40 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(WorkflowSpec const& workf
       }
     }
   }
+
+  auto findDeviceIndex = [&deviceIndex](size_t processorIndex, size_t timeslice) {
+    for (auto& deviceEdge : deviceIndex) {
+      if (deviceEdge.processorIndex != processorIndex) {
+        continue;
+      }
+      if (deviceEdge.timeslice != timeslice) {
+        continue;
+      }
+      return deviceEdge.deviceIndex;
+    }
+    throw std::runtime_error("Unable to find device.");
+  };
+
+  // Optimize the topology when two devices are
+  // running on the same node.
+  for (auto& connection : connections) {
+    auto& device1 = devices[findDeviceIndex(connection.consumer, connection.timeIndex)];
+    auto& device2 = devices[findDeviceIndex(connection.producer, connection.producerTimeIndex)];
+    // No need to do anything if they are not on the same host
+    if (device1.resource.hostname != device2.resource.hostname) {
+      continue;
+    }
+    for (auto& input : device1.inputChannels) {
+      for (auto& output : device2.outputChannels) {
+        if (input.hostname == output.hostname && input.port == output.port) {
+          input.protocol = ChannelProtocol::IPC;
+          output.protocol = ChannelProtocol::IPC;
+          input.hostname += uniqueWorkflowId;
+          output.hostname += uniqueWorkflowId;
+        }
+      }
+    }
+  }
 }
 
 void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
@@ -756,8 +792,11 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
     // FIXME: add some checksum in framework id. We could use this
     //        to avoid redeploys when only a portion of the workflow is changed.
     // FIXME: this should probably be done in one go with char *, but I am lazy.
-    std::vector<std::string> tmpArgs = {argv[0], "--id", spec.id.c_str(), "--control", "static",
-                                        "--log-color", "false", "--color", "false"};
+    std::vector<std::string> tmpArgs = {argv[0],
+                                        "--id", spec.id.c_str(),
+                                        "--control", "static",
+                                        "--log-color", "false",
+                                        "--color", "false"};
     if (defaultStopped) {
       tmpArgs.push_back("-s");
     }
@@ -802,6 +841,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
         bpo::options_description realOdesc = odesc;
         realOdesc.add_options()("child-driver", bpo::value<std::string>());
         realOdesc.add_options()("rate", bpo::value<std::string>());
+        realOdesc.add_options()("shm-segment-size", bpo::value<std::string>());
         filterArgsFct(expansions.we_wordc, expansions.we_wordv, realOdesc);
         wordfree(&expansions);
         return;
@@ -871,6 +911,9 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
     // FIXME: this should probably be reflected in the GUI
     std::ostringstream str;
     for (size_t ai = 0; ai < execution.args.size() - 1; ai++) {
+      if (execution.args[ai] == nullptr) {
+        LOG(ERROR) << "Bad argument for " << execution.args[ai - 1];
+      }
       assert(execution.args[ai]);
       str << " " << execution.args[ai];
     }
@@ -889,6 +932,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("plugin-search-path,S", bpo::value<std::string>(), "FairMQ plugins search path")                           //
     ("control-port", bpo::value<std::string>(), "Utility port to be used by O2 Control")                        //
     ("rate", bpo::value<std::string>(), "rate for a data source device (Hz)")                                   //
+    ("shm-segment-size", bpo::value<std::string>(), "size of the shared memory segment in bytes")               //
     ("monitoring-backend", bpo::value<std::string>(), "monitoring connection string")                           //
     ("infologger-mode", bpo::value<std::string>(), "INFOLOGGER_MODE override")                                  //
     ("infologger-severity", bpo::value<std::string>(), "minimun FairLogger severity which goes to info logger") //

@@ -45,17 +45,19 @@ struct DeviceSpecHelpers {
     std::vector<CompletionPolicy> const& completionPolicies,
     std::vector<DispatchPolicy> const& dispatchPolicies,
     std::vector<DeviceSpec>& devices,
-    ResourceManager& resourceManager);
+    ResourceManager& resourceManager,
+    std::string const& uniqueWorkflowId);
 
   static void dataProcessorSpecs2DeviceSpecs(
     const WorkflowSpec& workflow,
     std::vector<ChannelConfigurationPolicy> const& channelPolicies,
     std::vector<CompletionPolicy> const& completionPolicies,
     std::vector<DeviceSpec>& devices,
-    ResourceManager& resources)
+    ResourceManager& resourceManager,
+    std::string const& uniqueWorkflowId)
   {
     std::vector<DispatchPolicy> dispatchPolicies = DispatchPolicy::createDefaultPolicies();
-    dataProcessorSpecs2DeviceSpecs(workflow, channelPolicies, completionPolicies, dispatchPolicies, devices, resources);
+    dataProcessorSpecs2DeviceSpecs(workflow, channelPolicies, completionPolicies, dispatchPolicies, devices, resourceManager, uniqueWorkflowId);
   }
 
   /// Helper to prepare the arguments which will be used to

@@ -119,10 +119,13 @@ struct DriverInfo {
   /// The optional timeout after which the driver will request
   /// all the children to quit.
   double timeout;
-  /// The start port to use when looking for a free range
-  unsigned short startPort;
-  /// The size of the port range to consider allocated
-  unsigned short portRange;
+  /// The hostname which needs to be deployed by this instance of
+  /// the driver. By default it will be localhost
+  std::string deployHostname;
+  /// resources which are allocated for the whole workflow by
+  /// an external resource manager. If the value is an empty string
+  /// resources are obtained from the localhost.
+  std::string resources;
   /// The current set of metadata associated to each DataProcessor being
   /// executed.
   std::vector<DataProcessorInfo> processorInfo;
@@ -139,6 +142,8 @@ struct DriverInfo {
   float frameCost;
   /// The time between one frame and the other.
   float frameLatency;
+  /// The unique id used for ipc communications
+  std::string uniqueWorkflowId = "";
 };
 
 } // namespace framework

@@ -77,7 +77,7 @@ struct DeviceConnectionId {
   size_t consumer;
   size_t timeIndex;
   size_t producerTimeIndex;
-  size_t port;
+  uint16_t port;
 
   bool operator<(const DeviceConnectionId& rhs) const
   {