AliceO2Group · sawenzel · Jun 21, 2017 · Jun 16, 2017 · Jun 16, 2017 · matthiasrichter
@@ -76,6 +76,7 @@ ForEach (_file RANGE 0 ${_length})
   )
 EndForEach (_file RANGE 0 ${_length})
 
+
 O2_GENERATE_EXECUTABLE(
   EXE_NAME TimeframeValidationTool
   SOURCES src/TimeframeValidationTool
@@ -85,6 +86,8 @@ O2_GENERATE_EXECUTABLE(
 
 set(TEST_SRCS
   test/test_TimeframeParser.cxx
+  test/test_SubframeUtils01.cxx
+  test/test_PayloadMerger01.cxx
 )
 
 O2_GENERATE_TESTS(
@@ -94,5 +97,4 @@ O2_GENERATE_TESTS(
 
 O2_GENERATE_MAN(NAME TimeframeReaderDevice)
 O2_GENERATE_MAN(NAME TimeframeWriterDevice)
-
-target_compile_options(test_TimeframeParser PUBLIC -O0 -g)
+O2_GENERATE_MAN(NAME SubframeBuilderDevice)
@@ -0,0 +1,48 @@
+.\" Manpage for SubframeBuilderDevice.
+.TH AliceO2 1 "12 May 2017" "1.0" "SubframeBuilderDevice man page"
+
+.SH NAME
+
+SubframeBuilderDevice - aggregate HBF in input as a single STF 
+
+.SH SYNOPSIS
+
+SubframeBuilderDevice [options]
+
+.SH DESCRIPTION
+
+SubframeBuilderDevice will take in input a number of HeartBeat Frames
+(HBF) and merge them in a single STF, with a policy defined by the
+passed options.
+
+.SH OPTIONS
+
+--self-triggered        Time frame duration
+
+.TP 5
+
+--in-chan-name [NAME]   Name of the input channel
+
+.TP 5
+
+--out-chan-name [NAME]  Name of the output channel
+
+.TP 5
+
+--detector-name [NAME]  Name of detector as data source
+
+.TP 5
+
+--flp-id arg [NAME]     ID of the FLP used as data source
+
+.TP 5
+
+--strip-hbf             Strip HeartBeatHeader (HBH) & HeartBeatTrailer (HBT) from each HBF
+
+.SH SEE ALSO
+
+FLPSenderDEvice(1), EPNReceiverDevice(1), HeartbeatSampler(1), TimeframeValidator(1)
+
+.SH BUGS
+
+Lots of bugs
@@ -0,0 +1,124 @@
+// Copyright CERN and copyright holders of ALICE O2. This software is
+// distributed under the terms of the GNU General Public License v3 (GPL
+// Version 3), copied verbatim in the file "COPYING".
+//
+// See https://alice-o2.web.cern.ch/ for full licensing information.
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef PAYLOAD_MERGER_H
+#define PAYLOAD_MERGER_H
+
+#include <map>
+#include <cstdint>
+#include <vector>
+#include <functional>
+#include <cstring>
+
+#include <fairmq/FairMQMessage.h>
+
+namespace o2 { namespace dataflow {
+/// Helper class that given a set of FairMQMessage, merges (part of) their
+/// payload into a separate memory area.
+///
+/// - Append multiple messages via the aggregate method 
+/// - Finalise buffer creation with the finalise call.
+template <typename ID>
+class PayloadMerger {
+public:
+  using MergeableId = ID;
+  using MessageMap = std::multimap<MergeableId, std::unique_ptr<FairMQMessage>>;
+  using PayloadExtractor = std::function<size_t(char **, char *, size_t)>;
+  using IdExtractor = std::function<MergeableId(std::unique_ptr<FairMQMessage>&)>;
+  using MergeCompletionCheker = std::function<bool(MergeableId, MessageMap &)>;
+
+  /// Helper class to merge FairMQMessages sharing a user defined class of equivalence,
+  /// specified by @makeId. Completeness of the class of equivalence can be asserted by 
+  /// the @checkIfComplete policy. It's also possible to specify a user defined way of 
+  /// extracting the parts of the payload to be merged via the extractPayload method.
+  PayloadMerger(IdExtractor makeId,
+                MergeCompletionCheker checkIfComplete,
+                PayloadExtractor extractPayload = fullPayloadExtractor)
+    :
+      mMakeId{makeId},
+      mCheckIfComplete{checkIfComplete},
+      mExtractPayload{extractPayload}
+  {
+  }
+
+  /// Aggregates @payload to all the ones with the same id.
+  /// @return the id extracted from the payload via the constructor
+  ///         specified id policy (mMakeId callback).
+  MergeableId aggregate(std::unique_ptr<FairMQMessage> &payload) {
+    auto id = mMakeId(payload);
+    mPartsMap.emplace(std::make_pair(id, std::move(payload)));
+    return id;
+  }
+
+  /// This merges a set of messages sharing the same id @id to a unique buffer
+  /// @out, so that it can be either consumed or sent as a message itself.
+  /// The decision on whether the merge must happen is done by the constructor
+  /// specified policy mCheckIfComplete which can, for example, decide
+  /// to merge when a certain number of subparts are reached.
+  /// Merging at the moment requires an extra copy, but in principle this could
+  /// be easily extended to support scatter - gather.
+  size_t finalise(char **out, MergeableId &id) {
+    *out = nullptr;
+    if (mCheckIfComplete(id, mPartsMap) == false) {
+      return 0;
+    }
+    // If we are here, it means we can send the messages that belong
+    // to some predefined class of equivalence, identified by the MERGEABLE_ID,
+    // to the receiver. This is done by the following process:
+    // 
+    // - Extract what we actually want to send (this might be data embedded inside the message itself)
+    // - Calculate the aggregate size of all the payloads.
+    // - Copy all the parts into a final payload
+    // - Create the header part
+    // - Create the payload part
+    // - Send
+    std::vector<std::pair<char *, size_t>> parts;
+
+    size_t sum = 0;
+    auto range = mPartsMap.equal_range(id);
+    for (auto hi = range.first, he = range.second; hi != he; ++hi) {
+      std::unique_ptr<FairMQMessage> &payload = hi->second;
+      std::pair<char *, size_t> part;
+      part.second = mExtractPayload(&part.first, reinterpret_cast<char *>(payload->GetData()), payload->GetSize());
+      parts.push_back(part);
+      sum += part.second;
+    }
+
+    auto *payload = new char[sum]();
+    size_t offset = 0;
+    for (auto &part : parts) {
+      // Right now this does a copy. In principle this could be done with some sort of
+      // vectorized I/O
+      memcpy(payload + offset, part.first, part.second);
+      offset += part.second;
+    }
+
+    mPartsMap.erase(id);
+    *out = payload;
+    return sum;
+  }
+
+  // Helper method which leaves the payload untouched
+  static int64_t fullPayloadExtractor(char **payload,
+                                      char *buffer,
+                                      size_t bufferSize) {
+    *payload = buffer;
+    return bufferSize;
+  }
+private:
+  IdExtractor mMakeId;
+  MergeCompletionCheker mCheckIfComplete;
+  PayloadExtractor mExtractPayload;
+
+  MessageMap mPartsMap;
+};
+} /* dataflow */
+} /* o2 */
+
+#endif // PAYLOAD_MERGER_H
@@ -21,7 +21,10 @@
 #include "Headers/DataHeader.h"
 #include "Headers/HeartbeatFrame.h"
 #include "O2Device/O2Device.h"
+#include "DataFlow/PayloadMerger.h"
+#include "DataFlow/SubframeUtils.h"
 #include <cstring>
+#include <map>
 
 class FairMQParts;
 
@@ -51,22 +54,26 @@ namespace DataFlow {
 class SubframeBuilderDevice : public Base::O2Device
 {
 public:
-  typedef o2::Base::O2Message O2Message;
+  using O2Message = o2::Base::O2Message;
+  using SubframeId = o2::dataflow::SubframeId;
+  using Merger = dataflow::PayloadMerger<SubframeId>;
 
   static constexpr const char* OptionKeyInputChannelName = "in-chan-name";
   static constexpr const char* OptionKeyOutputChannelName = "out-chan-name";
-  static constexpr const char* OptionKeyDuration = "duration";
-  static constexpr const char* OptionKeySelfTriggered = "self-triggered";
+  static constexpr const char* OptionKeyOrbitDuration = "orbit-duration";
+  static constexpr const char* OptionKeyOrbitsPerTimeframe = "orbits-per-timeframe";
   static constexpr const char* OptionKeyInDataFile = "indatafile-name";
   static constexpr const char* OptionKeyDetector = "detector-name";
+  static constexpr const char* OptionKeyFLPId = "flp-id";
+  static constexpr const char* OptionKeyStripHBF = "strip-hbf";
 
   // TODO: this is just a first mockup, remove it
-  // Default duration is for now harcoded to 22 milliseconds.
   // Default start time for all the producers is 8/4/1977
   // Timeframe start time will be ((N * duration) + start time) where
   // N is the incremental number of timeframes being sent out.
   // TODO: replace this with a unique Heartbeat from a common device.
-  static constexpr uint32_t DefaultDuration = 22000000;
+  static constexpr uint32_t DefaultOrbitDuration = 88924;
+  static constexpr uint32_t DefaultOrbitsPerTimeframe = 256;
   static constexpr uint64_t DefaultHeartbeatStart = 229314600000000000LL;
 
   /// Default constructor
@@ -93,13 +100,15 @@ class SubframeBuilderDevice : public Base::O2Device
   bool BuildAndSendFrame(FairMQParts &parts);
 
 private:
-  unsigned mFrameNumber = 0;
-  constexpr static uint32_t mOrbitsPerTimeframe = 1;
-  constexpr static uint32_t mOrbitDuration = 1000000000;
-  constexpr static uint32_t mDuration = mOrbitsPerTimeframe * mOrbitDuration;
+  uint32_t mOrbitsPerTimeframe;
+  // FIXME: lookup the actual value
+  uint32_t mOrbitDuration;
   std::string mInputChannelName = "";
   std::string mOutputChannelName = "";
-  bool mIsSelfTriggered = false;
+  size_t mFLPId = 0;
+  bool mStripHBF = false;
+  std::unique_ptr<Merger> mMerger;
+
   uint64_t mHeartbeatStart = DefaultHeartbeatStart;
 
   template <typename T>

@@ -0,0 +1,45 @@
+// Copyright CERN and copyright holders of ALICE O2. This software is
+// distributed under the terms of the GNU General Public License v3 (GPL
+// Version 3), copied verbatim in the file "COPYING".
+//
+// See https://alice-o2.web.cern.ch/ for full licensing information.
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef DATAFLOW_SUBFRAMEUTILS_H
+#define DATAFLOW_SUBFRAMEUTILS_H
+
+#include <tuple>
+#include <cstddef>
+#include "Headers/HeartbeatFrame.h"
+
+namespace o2 { namespace dataflow {
+
+int64_t extractDetectorPayloadStrip(char **payload, char *buffer, size_t bufferSize) {
+  *payload = buffer + sizeof(o2::Header::HeartbeatHeader);
+  return bufferSize - sizeof(o2::Header::HeartbeatHeader) - sizeof(o2::Header::HeartbeatTrailer);
+}
+
+
+struct SubframeId {
+  size_t timeframeId;
+  size_t socketId;
+
+  // operator needed for the equal_range algorithm/ multimap method
+  bool operator<(const SubframeId& rhs) const {
+    return std::tie(timeframeId, socketId) < std::tie(rhs.timeframeId, rhs.socketId);
+  }
+};
+
+SubframeId makeIdFromHeartbeatHeader(const Header::HeartbeatHeader &header, size_t socketId, size_t orbitsPerTimeframe) {
+  SubframeId id = {
+    .timeframeId = header.orbit / orbitsPerTimeframe,
+    .socketId = socketId
+  };
+  return id;
+}
+
+} /* namespace dataflow */ } /* namespace o2 */
+
+#endif // DATAFLOW_SUBFRAMEUTILS_H