-
Notifications
You must be signed in to change notification settings - Fork 494
Expand file tree
/
Copy pathDataProcessor.cxx
More file actions
120 lines (111 loc) · 4.44 KB
/
DataProcessor.cxx
File metadata and controls
120 lines (111 loc) · 4.44 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
// Copyright CERN and copyright holders of ALICE O2. This software is
// distributed under the terms of the GNU General Public License v3 (GPL
// Version 3), copied verbatim in the file "COPYING".
//
// See http://alice-o2.web.cern.ch/license for full licensing information.
//
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
#include "Framework/DataProcessor.h"
#include "Framework/MessageContext.h"
#include "Framework/StringContext.h"
#include "Framework/ArrowContext.h"
#include "Framework/RawBufferContext.h"
#include "Framework/TMessageSerializer.h"
#include "FairMQResizableBuffer.h"
#include "CommonUtils/BoostSerializer.h"
#include "Headers/DataHeader.h"
#include <fairmq/FairMQParts.h>
#include <fairmq/FairMQDevice.h>
#include <arrow/io/memory.h>
#include <arrow/ipc/writer.h>
#include <cstddef>
#include <unordered_map>
using namespace o2::framework;
using DataHeader = o2::header::DataHeader;
namespace o2
{
namespace framework
{
void DataProcessor::doSend(FairMQDevice& device, FairMQParts&& parts, const char* channel, unsigned int index)
{
device.Send(parts, channel, index);
}
void DataProcessor::doSend(FairMQDevice& device, MessageContext& context)
{
std::unordered_map<std::string const*, FairMQParts> outputs;
auto contextMessages = context.getMessagesForSending();
for (auto& message : contextMessages) {
// monitoringService.send({ message->parts.Size(), "outputs/total" });
FairMQParts parts = std::move(message->finalize());
assert(message->empty());
assert(parts.Size() == 2);
for (auto& part : parts) {
outputs[&(message->channel())].AddPart(std::move(part));
}
}
for (auto& [channel, parts] : outputs) {
device.Send(parts, *channel, 0);
}
}
void DataProcessor::doSend(FairMQDevice& device, StringContext& context)
{
for (auto& messageRef : context) {
FairMQParts parts;
FairMQMessagePtr payload(device.NewMessage());
auto a = messageRef.payload.get();
// Rebuild the message using the string as input. For now it involves a copy.
payload->Rebuild(reinterpret_cast<void*>(const_cast<char*>(strdup(a->data()))), a->size(), nullptr, nullptr);
const DataHeader* cdh = o2::header::get<DataHeader*>(messageRef.header->GetData());
// sigh... See if we can avoid having it const by not
// exposing it to the user in the first place.
DataHeader* dh = const_cast<DataHeader*>(cdh);
dh->payloadSize = payload->GetSize();
parts.AddPart(std::move(messageRef.header));
parts.AddPart(std::move(payload));
device.Send(parts, messageRef.channel, 0);
}
}
// FIXME: for the moment we simply send empty messages.
void DataProcessor::doSend(FairMQDevice& device, ArrowContext& context)
{
for (auto& messageRef : context) {
FairMQParts parts;
// Depending on how the arrow table is constructed, we finalize
// the writing here.
messageRef.finalize(messageRef.buffer);
std::unique_ptr<FairMQMessage> payload = messageRef.buffer->Finalise();
// FIXME: for the moment we simply send empty bodies.
const DataHeader* cdh = o2::header::get<DataHeader*>(messageRef.header->GetData());
// sigh... See if we can avoid having it const by not
// exposing it to the user in the first place.
DataHeader* dh = const_cast<DataHeader*>(cdh);
dh->payloadSize = payload->GetSize();
parts.AddPart(std::move(messageRef.header));
parts.AddPart(std::move(payload));
device.Send(parts, messageRef.channel, 0);
}
}
void DataProcessor::doSend(FairMQDevice& device, RawBufferContext& context)
{
for (auto& messageRef : context) {
FairMQParts parts;
FairMQMessagePtr payload(device.NewMessage());
auto buffer = messageRef.serializeMsg().str();
// Rebuild the message using the serialized ostringstream as input. For now it involves a copy.
size_t size = buffer.length();
payload->Rebuild(size);
std::memcpy(payload->GetData(), buffer.c_str(), size);
const DataHeader* cdh = o2::header::get<DataHeader*>(messageRef.header->GetData());
// sigh... See if we can avoid having it const by not
// exposing it to the user in the first place.
DataHeader* dh = const_cast<DataHeader*>(cdh);
dh->payloadSize = size;
parts.AddPart(std::move(messageRef.header));
parts.AddPart(std::move(payload));
device.Send(parts, messageRef.channel, 0);
}
}
} // namespace framework
} // namespace o2