Skip to content

initial support for pmr containers in DPL #1593

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ktf merged 2 commits into from
Sep 21, 2019
Merged

initial support for pmr containers in DPL #1593

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
17cc067
Initial direct support for pmr aware containers in DPL
mkrzewic Jan 14, 2019
6ab1557
clang-format
Sep 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions DataFormats/MemoryResources/include/MemoryResources/MemoryResources.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,9 +224,9 @@ auto adoptVector(size_t nelem, FairMQMessagePtr message)

//__________________________________________________________________________________________________
/// Get the allocator associated to a transport factory
inline static ChannelResource* getTransportAllocator(FairMQTransportFactory* factory)
inline static FairMQMemoryResource* getTransportAllocator(FairMQTransportFactory* factory)
{
return factory->GetMemoryResource();
return *factory;
}

}; //namespace pmr
Expand Down
31 changes: 31 additions & 0 deletions Framework/Core/include/Framework/DataAllocator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,37 @@ class DataAllocator
return adopt(getOutputByBind(std::move(ref)), obj);
}

//make a stl (pmr) vector
template <typename T, typename... Args>
o2::vector<T> makeVector(const Output& spec, Args&&... args)
{
std::string channel = matchDataHeader(spec, mTimingInfo->timeslice);
auto context = mContextRegistry->get<MessageContext>();
o2::pmr::FairMQMemoryResource* targetResource = *context->proxy().getTransport(channel);
return o2::vector<T>{targetResource, std::forward<Args>(args)...};
}

//adopt container (if PMR is used with the appropriate memory resource in container it is ZERO-copy)
template <typename ContainerT>
void adoptContainer(const Output& spec, ContainerT& container) = delete; //only bind to moved-from containers
template <typename ContainerT>
void adoptContainer(const Output& spec, ContainerT&& container)
{
// Find a matching channel, extract the message for it form the container
// and put it in the queue to be sent at the end of the processing
std::string channel = matchDataHeader(spec, mTimingInfo->timeslice);

auto context = mContextRegistry->get<MessageContext>();
FairMQMessagePtr payloadMessage = o2::pmr::getMessage(std::forward<ContainerT>(container), *context->proxy().getTransport(channel));

FairMQMessagePtr headerMessage = headerMessageFromOutput(spec, channel, //
o2::header::gSerializationMethodNone, //
payloadMessage->GetSize() //
);

context->add<MessageContext::TrivialObject>(std::move(headerMessage), std::move(payloadMessage), channel);
}

/// snapshot object and route to output specified by OutputRef
/// Framework makes a (serialized) copy of object content.
///
Expand Down
2 changes: 1 addition & 1 deletion Framework/Core/include/Framework/FairMQDeviceProxy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ class FairMQDeviceProxy

/// Looks like what we really need in the headers is just the transport.
FairMQTransportFactory* getTransport();
FairMQTransportFactory* getTransport(const std::string& channel, int index);
FairMQTransportFactory* getTransport(const std::string& channel, int index = 0);
std::unique_ptr<FairMQMessage> createMessage() const;
std::unique_ptr<FairMQMessage> createMessage(const size_t size) const;

Expand Down
2 changes: 1 addition & 1 deletion Framework/Core/src/FairMQDeviceProxy.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ FairMQTransportFactory* FairMQDeviceProxy::getTransport()
return mDevice->Transport();
}

FairMQTransportFactory* FairMQDeviceProxy::getTransport(const std::string& channel, const int index = 0)
FairMQTransportFactory* FairMQDeviceProxy::getTransport(const std::string& channel, const int index)
{
return mDevice->GetChannel(channel, index).Transport();
}
Expand Down
6 changes: 2 additions & 4 deletions Framework/Utils/include/DPLUtils/Utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,14 @@

#include "Framework/DataProcessorSpec.h"
#include <functional>
#include "MemoryResources/MemoryResources.h"

namespace o2f = o2::framework;

namespace o2
{
namespace workflows
{
//TODO: this is to make DPLmerger compile, but this code is (and always was) horribly broken - please remove.
inline void freefn(void* data, void* /*hint*/) { delete static_cast<char*>(data); };
Copy link
Copy Markdown
Collaborator

@davidrohr davidrohr Sep 17, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Difficult to judge for me, but if this is broken and should be removed, and it compiles after removal, then it is probably the way to go. I mean, why is it there anyway? Was it just forgotten to remove it?

Copy link
Copy Markdown
Contributor Author

@mkrzewic mkrzewic Sep 17, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably, a remnant after removal of the Stack::freefn long time ago.


//
o2f::Output getOutput(const o2f::OutputSpec outputSpec);
std::shared_ptr<std::vector<o2f::Output>> getOutputList(const o2f::Outputs outputSpecs);
Expand All @@ -39,7 +37,7 @@ o2f::DataProcessorSpec defineBroadcaster(std::string devName, o2f::InputSpec usr
size_t fixMsgSize);
o2f::DataProcessorSpec defineBroadcaster(std::string devName, o2f::InputSpec usrInput, o2f::Outputs usrOutputs);

using OutputBuffer = std::vector<char>;
using OutputBuffer = o2::vector<char>;
// Merger implementations
o2f::DataProcessorSpec defineMerger(std::string devName, o2f::Inputs usrInputs, o2f::OutputSpec usrOutput,
std::function<void(OutputBuffer, const o2f::DataRef)> const mergerFunc);
Expand Down
11 changes: 5 additions & 6 deletions Framework/Utils/src/DPLMerger.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ namespace workflows
// This is a possible implementation of a DPL compliant and generic gatherer
// Every other implementation should fall back to this one, after required translations.
o2f::DataProcessorSpec defineMerger(std::string devName, o2f::Inputs usrInputs, o2f::OutputSpec usrOutput,
std::function<void(OutputBuffer, const o2f::DataRef)> const mergerFunc)
std::function<void(OutputBuffer&, const o2f::DataRef)> const mergerFunc)
{
return {devName, // Device name from user
usrInputs, // User defined input as a vector of one InputSpec
Expand All @@ -36,18 +36,17 @@ o2f::DataProcessorSpec defineMerger(std::string devName, o2f::Inputs usrInputs,
o2f::AlgorithmSpec{[usrOutput, mergerFunc](o2f::InitContext&) {
// Creating shared ptrs to useful parameters
auto outputPtr = std::make_shared<o2f::Output>(getOutput(usrOutput));
auto mergerFuncPtr = std::make_shared<std::function<void(OutputBuffer, o2f::DataRef)> const>(mergerFunc);
auto mergerFuncPtr = std::make_shared<std::function<void(OutputBuffer&, o2f::DataRef)> const>(mergerFunc);

// Defining the ProcessCallback as returned object of InitCallback
return [outputPtr, mergerFuncPtr](o2f::ProcessingContext& ctx) {
OutputBuffer outputBuffer;
OutputBuffer outputBuffer = ctx.outputs().makeVector<char>(*outputPtr);
// Iterating over the InputSpecs to aggregate msgs from the connected devices
for (const auto& itInputs : ctx.inputs()) {
(*mergerFuncPtr)(outputBuffer, itInputs);
}
// Adopting the buffer as new chunk
ctx.outputs().adoptChunk((*outputPtr), &outputBuffer[0], outputBuffer.size(), &freefn,
nullptr);
ctx.outputs().adoptContainer((*outputPtr), std::move(outputBuffer));
};
}}};
}
Expand All @@ -56,7 +55,7 @@ o2f::DataProcessorSpec defineMerger(std::string devName, o2f::Inputs usrInputs,
o2f::DataProcessorSpec defineMerger(std::string devName, o2f::Inputs usrInputs, o2f::OutputSpec usrOutput)
{
// This lambda retrieves the payload size through the API and back-inserts it on the output buffer
auto funcMerge = [](OutputBuffer buf, const o2f::DataRef d) {
auto funcMerge = [](OutputBuffer& buf, const o2f::DataRef d) {
auto msgSize = (o2::header::get<o2::header::DataHeader*>(d.header))->payloadSize;
buf.resize(buf.size() + msgSize);
std::copy(&(d.payload[0]), &(d.payload[msgSize - 1]), std::back_inserter(buf));
Expand Down