{

// We simply wake up the event loop. Nothing to be done here.

auto* state = (DeviceState*)handle->data;

state->loopReason |= DeviceState::TIMER_EXPIRED;

state->loopReason |= DeviceState::DATA_INCOMING;

{

// We simply wake up the event loop. Nothing to be done here.

auto* state = (DeviceState*)handle->data;

state->loopReason |= DeviceState::SIGNAL_ARRIVED;

state->loopReason |= DeviceState::DATA_INCOMING;

static RouteConfigurator::CreationConfigurator dataDrivenConfigurator()

{

return [](DeviceState&, ServiceRegistry&, ConfigParamRegistry const&) { return LifetimeHelpers::dataDrivenCreation(); };

}

static RouteConfigurator::CreationConfigurator timeDrivenConfigurator(InputSpec const& matcher)

{

return [matcher](DeviceState& state, ServiceRegistry&, ConfigParamRegistry const& options) {

std::string rateName = std::string{"period-"} + matcher.binding;

auto period = options.get<int>(rateName.c_str());

// We create a timer to wake us up. Notice the actual

// timeslot creation and record expiration still happens

// in a synchronous way.

auto* timer = (uv_timer_t*)(malloc(sizeof(uv_timer_t)));

timer->data = &state;

uv_timer_init(state.loop, timer);

uv_timer_start(timer, detail::timer_callback, period / 1000, period / 1000);

state.activeTimers.push_back(timer);

return LifetimeHelpers::timeDrivenCreation(std::chrono::microseconds(period));

};

}

static RouteConfigurator::CreationConfigurator loopEventDrivenConfigurator(InputSpec const& matcher)

{

return [matcher](DeviceState& state, ServiceRegistry&, ConfigParamRegistry const&) {

return LifetimeHelpers::uvDrivenCreation(DeviceState::LoopReason::OOB_ACTIVITY, state);

};

}

static RouteConfigurator::CreationConfigurator signalDrivenConfigurator(InputSpec const& matcher, size_t inputTimeslice, size_t maxInputTimeslices)

{

return [matcher, inputTimeslice, maxInputTimeslices](DeviceState& state, ServiceRegistry&, ConfigParamRegistry const& options) {

std::string startName = std::string{"start-value-"} + matcher.binding;

std::string endName = std::string{"end-value-"} + matcher.binding;

std::string stepName = std::string{"step-value-"} + matcher.binding;

auto start = options.get<int64_t>(startName.c_str());

auto stop = options.get<int64_t>(endName.c_str());

auto step = options.get<int64_t>(stepName.c_str());

// We create a timer to wake us up. Notice the actual

// timeslot creation and record expiration still happens

// in a synchronous way.

auto* sh = (uv_signal_t*)(malloc(sizeof(uv_signal_t)));

uv_signal_init(state.loop, sh);

sh->data = &state;

uv_signal_start(sh, detail::signal_callback, SIGUSR1);

state.activeSignals.push_back(sh);

return LifetimeHelpers::enumDrivenCreation(start, stop, step, inputTimeslice, maxInputTimeslices, 1);

};

}

static RouteConfigurator::CreationConfigurator oobDrivenConfigurator()

{

return [](DeviceState& state, ServiceRegistry&, ConfigParamRegistry const&) {

return LifetimeHelpers::uvDrivenCreation(DeviceState::LoopReason::OOB_ACTIVITY, state);

};

}

static RouteConfigurator::CreationConfigurator enumDrivenConfigurator(InputSpec const& matcher, size_t inputTimeslice, size_t maxInputTimeslices)

{

return [matcher, inputTimeslice, maxInputTimeslices](DeviceState&, ServiceRegistry&, ConfigParamRegistry const& options) {

std::string startName = std::string{"start-value-"} + matcher.binding;

std::string endName = std::string{"end-value-"} + matcher.binding;

std::string stepName = std::string{"step-value-"} + matcher.binding;

auto start = options.get<int64_t>(startName.c_str());

auto stop = options.get<int64_t>(endName.c_str());

auto step = options.get<int64_t>(stepName.c_str());

auto repetitions = 1;

for (auto& meta : matcher.metadata) {

if (meta.name == "repetitions") {

repetitions = meta.defaultValue.get<int64_t>();

break;

}

}

return LifetimeHelpers::enumDrivenCreation(start, stop, step, inputTimeslice, maxInputTimeslices, repetitions);

};

}

static RouteConfigurator::DanglingConfigurator danglingTimeframeConfigurator()

{

return [](DeviceState&, ConfigParamRegistry const&) { return LifetimeHelpers::expireNever(); };

}

static RouteConfigurator::ExpirationConfigurator expiringTimeframeConfigurator()

{

return [](DeviceState&, ConfigParamRegistry const&) { return LifetimeHelpers::doNothing(); };

}

static RouteConfigurator::DanglingConfigurator danglingConditionConfigurator()

{

return [](DeviceState&, ConfigParamRegistry const& options) {

auto serverUrl = options.get<std::string>("condition-backend");

return LifetimeHelpers::expectCTP(serverUrl, true);

};

}

static RouteConfigurator::ExpirationConfigurator expiringConditionConfigurator(InputSpec const& spec, std::string const& sourceChannel)

{

return [spec, sourceChannel](DeviceState&, ConfigParamRegistry const& options) {

auto serverUrl = options.get<std::string>("condition-backend");

auto forceTimestamp = options.get<std::string>("condition-timestamp");

return LifetimeHelpers::fetchFromCCDBCache(spec, serverUrl, forceTimestamp, sourceChannel);

};

}

static RouteConfigurator::CreationConfigurator fairmqDrivenConfiguration(InputSpec const& spec, int inputTimeslice, int maxInputTimeslices)

{

return [spec, inputTimeslice, maxInputTimeslices](DeviceState& state, ServiceRegistry& services, ConfigParamRegistry const&) {

// std::string channelNameOption = std::string{"out-of-band-channel-name-"} + spec.binding;

// auto channelName = options.get<std::string>(channelNameOption.c_str());

std::string channelName = "upstream";

for (auto& meta : spec.metadata) {

if (meta.name != "channel-name") {

continue;

}

channelName = meta.defaultValue.get<std::string>();

}

auto device = services.get<RawDeviceService>().device();

auto& channel = device->fChannels[channelName];

// We assume there is always a ZeroMQ socket behind.

int zmq_fd = 0;

size_t zmq_fd_len = sizeof(zmq_fd);

auto* poller = (uv_poll_t*)malloc(sizeof(uv_poll_t));

channel[0].GetSocket().GetOption("fd", &zmq_fd, &zmq_fd_len);

if (zmq_fd == 0) {

throw runtime_error_f("Cannot get file descriptor for channel %s", channelName.c_str());

}

LOG(debug) << "Polling socket for " << channel[0].GetName();

state.activeOutOfBandPollers.push_back(poller);

// We always create entries whenever we get invoked.

// Notice this works only if we are the only input.

// Otherwise we should check the channel for new data,

// before we create an entry.

return LifetimeHelpers::enumDrivenCreation(0, -1, 1, inputTimeslice, maxInputTimeslices, 1);

};

}

static RouteConfigurator::DanglingConfigurator danglingOutOfBandConfigurator()

{

return [](DeviceState&, ConfigParamRegistry const&) {

return LifetimeHelpers::expireAlways();

};

}

static RouteConfigurator::ExpirationConfigurator expiringOutOfBandConfigurator(InputSpec const& spec)

{

return [spec](DeviceState&, ConfigParamRegistry const& options) {

std::string channelNameOption = std::string{"out-of-band-channel-name-"} + spec.binding;

auto channelName = options.get<std::string>(channelNameOption.c_str());

return LifetimeHelpers::fetchFromFairMQ(spec, channelName);

};

}

static RouteConfigurator::DanglingConfigurator danglingQAConfigurator()

{

// FIXME: this should really be expireAlways. However, since we do not have

// a proper backend for conditions yet, I keep it behaving like it was

// before.

return [](DeviceState&, ConfigParamRegistry const&) { return LifetimeHelpers::expireNever(); };

}

static RouteConfigurator::ExpirationConfigurator expiringQAConfigurator()

{

return [](DeviceState&, ConfigParamRegistry const&) { return LifetimeHelpers::fetchFromQARegistry(); };

}

static RouteConfigurator::DanglingConfigurator danglingTimerConfigurator(InputSpec const& matcher)

{

return [matcher](DeviceState&, ConfigParamRegistry const&) {

return LifetimeHelpers::expireAlways();

};

}

static RouteConfigurator::DanglingConfigurator danglingEnumerationConfigurator(InputSpec const& matcher)

{

return [matcher](DeviceState&, ConfigParamRegistry const&) {

return LifetimeHelpers::expireAlways();

};

}

static RouteConfigurator::ExpirationConfigurator expiringTimerConfigurator(InputSpec const& spec, std::string const& sourceChannel)

{

auto m = std::get_if<ConcreteDataMatcher>(&spec.matcher);

if (m == nullptr) {

throw runtime_error("InputSpec for Timers must be fully qualified");

}

// We copy the matcher to avoid lifetime issues.

return [matcher = *m, sourceChannel](DeviceState&, ConfigParamRegistry const&) {

// Timers do not have any orbit associated to them

return LifetimeHelpers::enumerate(matcher, sourceChannel, 0, 0);

};

}

static RouteConfigurator::ExpirationConfigurator expiringOOBConfigurator(InputSpec const& spec, std::string const& sourceChannel)

{

auto m = std::get_if<ConcreteDataMatcher>(&spec.matcher);

if (m == nullptr) {

throw runtime_error("InputSpec for OOB must be fully qualified");

}

// We copy the matcher to avoid lifetime issues.

return [matcher = *m, sourceChannel](DeviceState&, ConfigParamRegistry const&) {

// Timers do not have any orbit associated to them

return LifetimeHelpers::enumerate(matcher, sourceChannel, 0, 0);

};

}

static RouteConfigurator::ExpirationConfigurator expiringEnumerationConfigurator(InputSpec const& spec, std::string const& sourceChannel)

{

auto m = std::get_if<ConcreteDataMatcher>(&spec.matcher);

if (m == nullptr) {

throw runtime_error("InputSpec for Enumeration must be fully qualified");

}

// We copy the matcher to avoid lifetime issues.

return [matcher = *m, sourceChannel](DeviceState&, ConfigParamRegistry const& config) {

size_t orbitOffset = config.get<int64_t>("orbit-offset-enumeration");

size_t orbitMultiplier = config.get<int64_t>("orbit-multiplier-enumeration");

return LifetimeHelpers::enumerate(matcher, sourceChannel, orbitOffset, orbitMultiplier);

};

}

static RouteConfigurator::DanglingConfigurator danglingTransientConfigurator()

{

// FIXME: this should really be expireAlways. However, since we do not have

// a proper backend for conditions yet, I keep it behaving like it was

// before.

return [](DeviceState&, ConfigParamRegistry const&) { return LifetimeHelpers::expireNever(); };

}

static RouteConfigurator::ExpirationConfigurator expiringTransientConfigurator(InputSpec const&)

{

return [](DeviceState&, ConfigParamRegistry const&) { return LifetimeHelpers::fetchFromObjectRegistry(); };

}

/// This behaves as data. I.e. we never create it unless data arrives.

static RouteConfigurator::CreationConfigurator createOptionalConfigurator()

{

return [](DeviceState&, ServiceRegistry&, ConfigParamRegistry const&) { return LifetimeHelpers::dataDrivenCreation(); };

}

/// This will always exipire an optional record when no data is received.

static RouteConfigurator::DanglingConfigurator danglingOptionalConfigurator(std::vector<InputRoute> const& routes)

{

return [routes](DeviceState&, ConfigParamRegistry const&) { return LifetimeHelpers::expireIfPresent(routes, ConcreteDataMatcher{"FLP", "DISTSUBTIMEFRAME", 0}); };

}

/// When the record expires, simply create a dummy entry.

static RouteConfigurator::ExpirationConfigurator expiringOptionalConfigurator(InputSpec const& spec, std::string const& sourceChannel)

{

try {

ConcreteDataMatcher concrete = DataSpecUtils::asConcreteDataMatcher(spec);

return [concrete, sourceChannel](DeviceState&, ConfigParamRegistry const&) {

return LifetimeHelpers::dummy(concrete, sourceChannel);

};

} catch (...) {

ConcreteDataTypeMatcher dataType = DataSpecUtils::asConcreteDataTypeMatcher(spec);

ConcreteDataMatcher concrete{dataType.origin, dataType.description, 0xdeadbeef};

return [concrete, sourceChannel](DeviceState&, ConfigParamRegistry const&) {

return LifetimeHelpers::dummy(concrete, sourceChannel);

};

// We copy the matcher to avoid lifetime issues.

}

}

{

return fmt::format("{}type={},method={},address={},rateLogging={},rcvBufSize={},sndBufSize={}",

channel.name.empty() ? "" : "name=" + channel.name + ",",

ChannelSpecHelpers::typeAsString(channel.type),

ChannelSpecHelpers::methodAsString(channel.method),

ChannelSpecHelpers::channelUrl(channel),

channel.rateLogging,

channel.recvBufferSize,

channel.sendBufferSize);

{

return fmt::format("{}type={},method={},address={},rateLogging={},rcvBufSize={},sndBufSize={}",

channel.name.empty() ? "" : "name=" + channel.name + ",",

ChannelSpecHelpers::typeAsString(channel.type),

ChannelSpecHelpers::methodAsString(channel.method),

ChannelSpecHelpers::channelUrl(channel),

channel.rateLogging,

channel.recvBufferSize,

channel.sendBufferSize);

std::vector<DeviceId>& deviceIndex,

std::vector<DeviceConnectionId>& connections,

ResourceManager& resourceManager,

const std::vector<size_t>& outEdgeIndex,

const std::vector<DeviceConnectionEdge>& logicalEdges,

const std::vector<EdgeAction>& actions, const WorkflowSpec& workflow,

const std::vector<OutputSpec>& outputsMatchers,

const std::vector<ChannelConfigurationPolicy>& channelPolicies,

std::string const& channelPrefix,

ComputingOffer const& defaultOffer)

{

// The topology cannot be empty or not connected. If that is the case, than

// something before this went wrong.

// FIXME: is that really true???

assert(!workflow.empty());

// Edges are navigated in order for each device, so the device associaited to

// an edge is always the last one created.

auto deviceForEdge = [&actions, &workflow, &devices,

&logicalEdges, &resourceManager,

&defaultOffer, &channelPrefix](size_t ei, ComputingOffer& acceptedOffer) {

auto& edge = logicalEdges[ei];

auto& action = actions[ei];

if (action.requiresNewDevice == false) {

assert(devices.empty() == false);

return devices.size() - 1;

}

if (acceptedOffer.hostname != "") {

resourceManager.notifyAcceptedOffer(acceptedOffer);

}

auto processor = workflow[edge.producer];

acceptedOffer.cpu = defaultOffer.cpu;

acceptedOffer.memory = defaultOffer.memory;

for (auto offer : resourceManager.getAvailableOffers()) {

if (offer.cpu < acceptedOffer.cpu) {

continue;

}

if (offer.memory < acceptedOffer.memory) {

continue;

}

acceptedOffer.hostname = offer.hostname;

acceptedOffer.startPort = offer.startPort;

acceptedOffer.rangeSize = 0;

break;

}

DeviceSpec device;

device.name = processor.name;

device.id = processor.name;

device.channelPrefix = channelPrefix;

if (processor.maxInputTimeslices != 1) {

device.id = processor.name + "_t" + std::to_string(edge.producerTimeIndex);

}

device.algorithm = processor.algorithm;

device.services = processor.requiredServices;

device.options = processor.options;

device.rank = processor.rank;

device.nSlots = processor.nSlots;

device.inputTimesliceId = edge.producerTimeIndex;

device.maxInputTimeslices = processor.maxInputTimeslices;

device.resource = {acceptedOffer};

device.labels = processor.labels;

/// If any of the inputs or outputs are "Lifetime::OutOfBand"

/// create the associated channels.

//

// for (auto& input : processor.inputs) {

// if (input.lifetime != Lifetime::OutOfBand) {

// continue;

// }

// InputChannelSpec extraInputChannelSpec{

// .name = "upstream",

// .type = ChannelType::Pair,

// .method = ChannelMethod::Bind,

// .hostname = "localhost",

// .port = 33000,

// .protocol = ChannelProtocol::IPC,

// };

// for (auto& meta : input.metadata) {

// if (meta.name == "name") {

// extraInputChannelSpec.name = meta.defaultValue.get<std::string>();

// }

// if (meta.name == "port") {

// extraInputChannelSpec.port = meta.defaultValue.get<int32_t>();

// }

// if (meta.name == "address") {

// extraInputChannelSpec.hostname = meta.defaultValue.get<std::string>();

// }

// }

// device.inputChannels.push_back(extraInputChannelSpec);

//}

for (auto& output : processor.outputs) {

if (output.lifetime != Lifetime::OutOfBand) {

continue;

}

OutputChannelSpec extraOutputChannelSpec{

.name = "downstream",

.type = ChannelType::Pair,

.method = ChannelMethod::Connect,

.hostname = "localhost",

.port = 33000,

.protocol = ChannelProtocol::IPC};

for (auto& meta : output.metadata) {

if (meta.name == "channel-name") {

extraOutputChannelSpec.name = meta.defaultValue.get<std::string>();

}

if (meta.name == "port") {

extraOutputChannelSpec.port = meta.defaultValue.get<int32_t>();

}

if (meta.name == "address") {

extraOutputChannelSpec.hostname = meta.defaultValue.get<std::string>();

}

}

device.outputChannels.push_back(extraOutputChannelSpec);

}

devices.push_back(device);

return devices.size() - 1;

};

auto channelFromDeviceEdgeAndPort = [&connections, &workflow, &channelPolicies](const DeviceSpec& device,

ComputingResource& deviceResource,

ComputingOffer& acceptedOffer,

const DeviceConnectionEdge& edge) {

OutputChannelSpec channel;

auto& consumer = workflow[edge.consumer];

std::string consumerDeviceId = consumer.name;

if (consumer.maxInputTimeslices != 1) {

consumerDeviceId += "_t" + std::to_string(edge.timeIndex);

}

channel.name = device.channelPrefix + "from_" + device.id + "_to_" + consumerDeviceId;

channel.port = acceptedOffer.startPort + acceptedOffer.rangeSize;

channel.hostname = acceptedOffer.hostname;

deviceResource.usedPorts += 1;

acceptedOffer.rangeSize += 1;

for (auto& policy : channelPolicies) {

if (policy.match(device.id, consumerDeviceId)) {

policy.modifyOutput(channel);

break;

}

}

DeviceConnectionId id{edge.producer, edge.consumer, edge.timeIndex, edge.producerTimeIndex, channel.port};

connections.push_back(id);

return channel;

};

auto isDifferentDestinationDeviceReferredBy = [&actions](size_t ei) { return actions[ei].requiresNewChannel; };

// This creates a new channel for a given edge, if needed. Notice that we

// navigate edges in a per device fashion (creating those if they are not

// alredy there) and create a new channel only if it connects two new

// devices. Whether or not this is the case was previously computed

// in the action.requiresNewChannel field.

auto createChannelForDeviceEdge = [&devices, &logicalEdges, &channelFromDeviceEdgeAndPort,

&deviceIndex](size_t di, size_t ei, ComputingOffer& offer) {

auto& device = devices[di];

auto& edge = logicalEdges[ei];

deviceIndex.emplace_back(DeviceId{edge.producer, edge.producerTimeIndex, di});

OutputChannelSpec channel = channelFromDeviceEdgeAndPort(device, device.resource, offer, edge);

device.outputChannels.push_back(channel);

return device.outputChannels.size() - 1;

};

// Notice how we need to behave in two different ways depending

// whether this is a real OutputRoute or if it's a forward from

// a previous consumer device.

// FIXME: where do I find the InputSpec for the forward?

auto appendOutputRouteToSourceDeviceChannel = [&outputsMatchers, &workflow, &devices, &logicalEdges](

size_t ei, size_t di, size_t ci) {

assert(ei < logicalEdges.size());

assert(di < devices.size());

assert(ci < devices[di].outputChannels.size());

auto& edge = logicalEdges[ei];

auto& device = devices[di];

assert(edge.consumer < workflow.size());

auto& consumer = workflow[edge.consumer];

auto& channel = devices[di].outputChannels[ci];

assert(edge.outputGlobalIndex < outputsMatchers.size());

if (edge.isForward == false) {

OutputRoute route{

edge.timeIndex,

consumer.maxInputTimeslices,

outputsMatchers[edge.outputGlobalIndex],

channel.name};

device.outputs.emplace_back(route);

} else {

ForwardRoute route{

edge.timeIndex,

consumer.maxInputTimeslices,

workflow[edge.consumer].inputs[edge.consumerInputIndex],

channel.name};

device.forwards.emplace_back(route);

}

};

auto sortDeviceIndex = [&deviceIndex]() { std::sort(deviceIndex.begin(), deviceIndex.end()); };

auto lastChannelFor = [&devices](size_t di) {

assert(di < devices.size());

assert(devices[di].outputChannels.empty() == false);

return devices[di].outputChannels.size() - 1;

};

//

// OUTER LOOP

//

// We need to create all the channels going out of a device, and associate

// routes to them for this reason

// we iterate over all the edges (which are per-datatype exchanged) and

// whenever we need to connect to a new device we create the channel. `device`

// here refers to the source device. This loop will therefore not create the

// devices which acts as sink, which are done in the preocessInEdgeActions

// function.

ComputingOffer acceptedOffer;

for (auto edge : outEdgeIndex) {

auto device = deviceForEdge(edge, acceptedOffer);

size_t channel = -1;

if (isDifferentDestinationDeviceReferredBy(edge)) {

channel = createChannelForDeviceEdge(device, edge, acceptedOffer);

} else {

channel = lastChannelFor(device);

}

appendOutputRouteToSourceDeviceChannel(edge, device, channel);

}

if (std::string(acceptedOffer.hostname) != "") {

resourceManager.notifyAcceptedOffer(acceptedOffer);

}

sortDeviceIndex();

std::vector<DeviceId>& deviceIndex,

const std::vector<DeviceConnectionId>& connections,

ResourceManager& resourceManager,

const std::vector<size_t>& inEdgeIndex,

const std::vector<DeviceConnectionEdge>& logicalEdges,

const std::vector<EdgeAction>& actions, const WorkflowSpec& workflow,

std::vector<LogicalForwardInfo> const& availableForwardsInfo,

std::vector<ChannelConfigurationPolicy> const& channelPolicies,

std::string const& channelPrefix,

ComputingOffer const& defaultOffer)

{

auto const& constDeviceIndex = deviceIndex;

auto findProducerForEdge = [&logicalEdges, &constDeviceIndex](size_t ei) {

auto& edge = logicalEdges[ei];

DeviceId pid{edge.producer, edge.producerTimeIndex, 0};

auto deviceIt = std::lower_bound(constDeviceIndex.cbegin(), constDeviceIndex.cend(), pid);

// By construction producer should always be there

assert(deviceIt != constDeviceIndex.end());

assert(deviceIt->processorIndex == pid.processorIndex && deviceIt->timeslice == pid.timeslice);

return deviceIt->deviceIndex;

};

auto findConsumerForEdge = [&logicalEdges, &constDeviceIndex](size_t ei) {

auto& edge = logicalEdges[ei];

if (!std::is_sorted(constDeviceIndex.cbegin(), constDeviceIndex.cend())) {

throw o2::framework::runtime_error("Needs a sorted vector to be correct");

}

DeviceId pid{edge.consumer, edge.timeIndex, 0};

auto deviceIt = std::lower_bound(constDeviceIndex.cbegin(), constDeviceIndex.cend(), pid);

// We search for a consumer only if we know it's is already there.

assert(deviceIt != constDeviceIndex.end());

assert(deviceIt->processorIndex == pid.processorIndex && deviceIt->timeslice == pid.timeslice);

return deviceIt->deviceIndex;

};

// Notice that to start with, consumer exists only if they also are

// producers, so we need to create one if it does not exist. Given this is

// stateful, we keep an eye on what edge was last searched to make sure we

// are not screwing up.

//

// Notice this is not thread safe.

decltype(deviceIndex.begin()) lastConsumerSearch;

size_t lastConsumerSearchEdge;

auto hasConsumerForEdge = [&lastConsumerSearch, &lastConsumerSearchEdge, &deviceIndex,

&logicalEdges](size_t ei) -> int {

auto& edge = logicalEdges[ei];

DeviceId cid{edge.consumer, edge.timeIndex, 0};

lastConsumerSearchEdge = ei; // This will invalidate the cache

lastConsumerSearch = std::lower_bound(deviceIndex.begin(), deviceIndex.end(), cid);

return lastConsumerSearch != deviceIndex.end() && cid.processorIndex == lastConsumerSearch->processorIndex &&

cid.timeslice == lastConsumerSearch->timeslice;

};

// The passed argument is there just to check. We do know that the last searched

// is the one we want.

auto getConsumerForEdge = [&lastConsumerSearch, &lastConsumerSearchEdge](size_t ei) {

assert(ei == lastConsumerSearchEdge);

return lastConsumerSearch->deviceIndex;

};

auto createNewDeviceForEdge = [&workflow, &logicalEdges, &devices,

&deviceIndex, &resourceManager, &defaultOffer,

&channelPrefix](size_t ei, ComputingOffer& acceptedOffer) {

auto& edge = logicalEdges[ei];

if (acceptedOffer.hostname != "") {

resourceManager.notifyAcceptedOffer(acceptedOffer);

}

auto& processor = workflow[edge.consumer];

acceptedOffer.cpu = defaultOffer.cpu;

acceptedOffer.memory = defaultOffer.memory;

for (auto offer : resourceManager.getAvailableOffers()) {

if (offer.cpu < acceptedOffer.cpu) {

continue;

}

if (offer.memory < acceptedOffer.memory) {

continue;

}

acceptedOffer.hostname = offer.hostname;

acceptedOffer.startPort = offer.startPort;

acceptedOffer.rangeSize = 0;

break;

}

DeviceSpec device;

device.name = processor.name;

device.id = processor.name;

device.channelPrefix = channelPrefix;

if (processor.maxInputTimeslices != 1) {

device.id += "_t" + std::to_string(edge.timeIndex);

}

device.algorithm = processor.algorithm;

device.services = processor.requiredServices;

device.options = processor.options;

device.rank = processor.rank;

device.nSlots = processor.nSlots;

device.inputTimesliceId = edge.timeIndex;

device.maxInputTimeslices = processor.maxInputTimeslices;

device.resource = {acceptedOffer};

device.labels = processor.labels;

// FIXME: maybe I should use an std::map in the end

// but this is really not performance critical

auto id = DeviceId{edge.consumer, edge.timeIndex, devices.size()};

devices.push_back(device);

deviceIndex.push_back(id);

std::sort(deviceIndex.begin(), deviceIndex.end());

return devices.size() - 1;

};

// We search for a preexisting outgoing connection associated to this edge.

// This is to retrieve the port of the source.

// This has to exists, because we already created all the outgoing connections

// so it's just a matter of looking it up.

auto findMatchingOutgoingPortForEdge = [&logicalEdges, &connections](size_t ei) {

auto const& edge = logicalEdges[ei];

DeviceConnectionId connectionId{edge.producer, edge.consumer, edge.timeIndex, edge.producerTimeIndex, 0};

auto it = std::lower_bound(connections.begin(), connections.end(), connectionId);

assert(it != connections.end());

assert(it->producer == connectionId.producer);

assert(it->consumer == connectionId.consumer);

assert(it->timeIndex == connectionId.timeIndex);

assert(it->producerTimeIndex == connectionId.producerTimeIndex);

return it->port;

};

auto checkNoDuplicatesFor = [](std::vector<InputChannelSpec> const& channels, const std::string& name) {

for (auto const& channel : channels) {

if (channel.name == name) {

return false;

}

}

return true;

};

auto appendInputChannelForConsumerDevice = [&devices, &connections, &checkNoDuplicatesFor, &channelPolicies](

size_t pi, size_t ci, unsigned short port) {

auto const& producerDevice = devices[pi];

auto& consumerDevice = devices[ci];

InputChannelSpec channel;

channel.name = producerDevice.channelPrefix + "from_" + producerDevice.id + "_to_" + consumerDevice.id;

channel.hostname = producerDevice.resource.hostname;

channel.port = port;

for (auto& policy : channelPolicies) {

if (policy.match(producerDevice.id, consumerDevice.id)) {

policy.modifyInput(channel);

break;

}

}

assert(checkNoDuplicatesFor(consumerDevice.inputChannels, channel.name));

consumerDevice.inputChannels.push_back(channel);

return consumerDevice.inputChannels.size() - 1;

};

// I think this is trivial, since I think it should always be the last one,

// in case it's not actually the case, I should probably do an actual lookup

// here.

auto getChannelForEdge = [&devices](size_t pi, size_t ci) {

auto& consumerDevice = devices[ci];

return consumerDevice.inputChannels.size() - 1;

};

// This is always called when adding a new channel, so we can simply refer

// to back. Notice also that this is the place where it makes sense to

// assign the forwarding, given that the forwarded stuff comes from some

// input.

auto appendInputRouteToDestDeviceChannel = [&devices, &logicalEdges, &workflow](size_t ei, size_t di, size_t ci) {

auto const& edge = logicalEdges[ei];

auto const& consumer = workflow[edge.consumer];

auto& consumerDevice = devices[di];

auto const& inputSpec = consumer.inputs[edge.consumerInputIndex];

auto const& sourceChannel = consumerDevice.inputChannels[ci].name;

InputRoute route{

inputSpec,

edge.consumerInputIndex,

sourceChannel,

edge.producerTimeIndex,

std::nullopt};

// In case we have wildcards, we must make sure that some other edge

// produced the same route, i.e. has the same matcher. Without this,

// otherwise, we would end up with as many input routes as the outputs that

// can be matched by the wildcard.

for (size_t iri = 0; iri < consumerDevice.inputs.size(); ++iri) {

auto& existingRoute = consumerDevice.inputs[iri];

if (existingRoute.timeslice != edge.producerTimeIndex) {

continue;

}

if (existingRoute.inputSpecIndex == edge.consumerInputIndex) {

return;

}

}

consumerDevice.inputs.push_back(route);

};

// Outer loop. A new device is needed for each

// of the sink data processors.

// New InputChannels need to refer to preexisting OutputChannels we create

// previously.

ComputingOffer acceptedOffer;

for (size_t edge : inEdgeIndex) {

auto& action = actions[edge];

size_t consumerDevice = -1;

if (action.requiresNewDevice) {

if (hasConsumerForEdge(edge)) {

consumerDevice = getConsumerForEdge(edge);

} else {

consumerDevice = createNewDeviceForEdge(edge, acceptedOffer);

}

} else {

consumerDevice = findConsumerForEdge(edge);

}

size_t producerDevice = findProducerForEdge(edge);

size_t channel = -1;

if (action.requiresNewChannel) {

int16_t port = findMatchingOutgoingPortForEdge(edge);

channel = appendInputChannelForConsumerDevice(producerDevice, consumerDevice, port);

} else {

channel = getChannelForEdge(producerDevice, consumerDevice);

}

appendInputRouteToDestDeviceChannel(edge, consumerDevice, channel);

}

// Bind the expiration mechanism to the input routes

for (auto& device : devices) {

for (auto& route : device.inputs) {

switch (route.matcher.lifetime) {

case Lifetime::OutOfBand:

route.configurator = {

.name = "oob",

.creatorConfigurator = ExpirationHandlerHelpers::loopEventDrivenConfigurator(route.matcher),

.danglingConfigurator = ExpirationHandlerHelpers::danglingOutOfBandConfigurator(),

.expirationConfigurator = ExpirationHandlerHelpers::expiringOOBConfigurator(route.matcher, route.sourceChannel)};

break;

// case Lifetime::Condition:

// route.configurator = {

// ExpirationHandlerHelpers::dataDrivenConfigurator(),

// ExpirationHandlerHelpers::danglingConditionConfigurator(),

// ExpirationHandlerHelpers::expiringConditionConfigurator(inputSpec, sourceChannel)};

// break;

case Lifetime::QA:

route.configurator = {

.name = "qa",

.creatorConfigurator = ExpirationHandlerHelpers::dataDrivenConfigurator(),

.danglingConfigurator = ExpirationHandlerHelpers::danglingQAConfigurator(),

.expirationConfigurator = ExpirationHandlerHelpers::expiringQAConfigurator()};

break;

case Lifetime::Timer:

route.configurator = {

.name = "timer",

.creatorConfigurator = ExpirationHandlerHelpers::timeDrivenConfigurator(route.matcher),

.danglingConfigurator = ExpirationHandlerHelpers::danglingTimerConfigurator(route.matcher),

.expirationConfigurator = ExpirationHandlerHelpers::expiringTimerConfigurator(route.matcher, route.sourceChannel)};

break;

case Lifetime::Enumeration:

route.configurator = {

.name = "enumeration",

.creatorConfigurator = ExpirationHandlerHelpers::enumDrivenConfigurator(route.matcher, device.inputTimesliceId, device.maxInputTimeslices),

.danglingConfigurator = ExpirationHandlerHelpers::danglingEnumerationConfigurator(route.matcher),

.expirationConfigurator = ExpirationHandlerHelpers::expiringEnumerationConfigurator(route.matcher, route.sourceChannel)};

break;

case Lifetime::Signal:

route.configurator = {

.name = "signal",

.creatorConfigurator = ExpirationHandlerHelpers::signalDrivenConfigurator(route.matcher, device.inputTimesliceId, device.maxInputTimeslices),

.danglingConfigurator = ExpirationHandlerHelpers::danglingEnumerationConfigurator(route.matcher),

.expirationConfigurator = ExpirationHandlerHelpers::expiringEnumerationConfigurator(route.matcher, route.sourceChannel)};

break;

case Lifetime::Transient:

route.configurator = {

.name = "transient",

.creatorConfigurator = ExpirationHandlerHelpers::dataDrivenConfigurator(),

.danglingConfigurator = ExpirationHandlerHelpers::danglingTransientConfigurator(),

.expirationConfigurator = ExpirationHandlerHelpers::expiringTransientConfigurator(route.matcher)};

break;

case Lifetime::Optional:

route.configurator = {

.name = "optional",

.creatorConfigurator = ExpirationHandlerHelpers::createOptionalConfigurator(),

.danglingConfigurator = ExpirationHandlerHelpers::danglingOptionalConfigurator(device.inputs),

.expirationConfigurator = ExpirationHandlerHelpers::expiringOptionalConfigurator(route.matcher, route.sourceChannel)};

break;

default:

break;

}

}

}

if (acceptedOffer.hostname != "") {

resourceManager.notifyAcceptedOffer(acceptedOffer);

}

std::vector<ChannelConfigurationPolicy> const& channelPolicies,

std::vector<CompletionPolicy> const& completionPolicies,

std::vector<DispatchPolicy> const& dispatchPolicies,

std::vector<ResourcePolicy> const& resourcePolicies,

std::vector<CallbacksPolicy> const& callbacksPolicies,

std::vector<SendingPolicy> const& sendingPolicies,

std::vector<DeviceSpec>& devices,

ResourceManager& resourceManager,

std::string const& uniqueWorkflowId,

ConfigContext const& configContext,

bool optimizeTopology,

unsigned short resourcesMonitoringInterval,

std::string const& channelPrefix)

{

std::vector<LogicalForwardInfo> availableForwardsInfo;

std::vector<DeviceConnectionEdge> logicalEdges;

std::vector<DeviceConnectionId> connections;

std::vector<DeviceId> deviceIndex;

// This is a temporary store for inputs and outputs,

// including forwarded channels, so that we can construct

// them before assigning to a device.

std::vector<OutputSpec> outputs;

WorkflowHelpers::constructGraph(workflow, logicalEdges, outputs, availableForwardsInfo);

// We need to instanciate one device per (me, timeIndex) in the

// DeviceConnectionEdge. For each device we need one new binding

// server per (me, other) -> port Moreover for each (me, other,

// outputGlobalIndex) we need to insert either an output or a

// forward.

//

// We then sort by other. For each (other, me) we need to connect to

// port (me, other) and add an input.

// Fill an index to do the sorting

std::vector<size_t> inEdgeIndex;

std::vector<size_t> outEdgeIndex;

WorkflowHelpers::sortEdges(inEdgeIndex, outEdgeIndex, logicalEdges);

std::vector<EdgeAction> outActions = WorkflowHelpers::computeOutEdgeActions(logicalEdges, outEdgeIndex);

// Crete the connections on the inverse map for all of them

// lookup for port and add as input of the current device.

std::vector<EdgeAction> inActions = WorkflowHelpers::computeInEdgeActions(logicalEdges, inEdgeIndex);

size_t deviceCount = 0;

for (auto& action : outActions) {

deviceCount += action.requiresNewDevice ? 1 : 0;

}

for (auto& action : inActions) {

deviceCount += action.requiresNewDevice ? 1 : 0;

}

ComputingOffer defaultOffer;

for (auto& offer : resourceManager.getAvailableOffers()) {

defaultOffer.cpu += offer.cpu;

defaultOffer.memory += offer.memory;

}

/// For the moment lets play it safe and underestimate default needed resources.

defaultOffer.cpu /= deviceCount + 1;

defaultOffer.memory /= deviceCount + 1;

processOutEdgeActions(devices, deviceIndex, connections, resourceManager, outEdgeIndex, logicalEdges,

outActions, workflow, outputs, channelPolicies, channelPrefix, defaultOffer);

// FIXME: is this not the case???

std::sort(connections.begin(), connections.end());

processInEdgeActions(devices, deviceIndex, connections, resourceManager, inEdgeIndex, logicalEdges,

inActions, workflow, availableForwardsInfo, channelPolicies, channelPrefix, defaultOffer);

// We apply the completion policies here since this is where we have all the

// devices resolved.

for (auto& device : devices) {

for (auto& policy : completionPolicies) {

if (policy.matcher(device) == true) {

device.completionPolicy = policy;

break;

}

}

for (auto& policy : dispatchPolicies) {