AliceO2Group · shahor02 · Aug 20, 2020 · Aug 20, 2020
@@ -125,12 +125,18 @@ header  : what it wants to add right after the RDH of the new CRU page before th
           the payload (starting at ptr+actualSize) will be written
 ```
 
-The method mast return actual size of the bloc which can be written (`<=maxSize`).
+The method must return actual size of the bloc which can be written (`<=maxSize`).
 If this method populates the trailer, it must ensure that it returns the actual size such that
 `actualSize + trailer.size() <= maxSize`.
-In case returned `actualSize` is 0, current CRU page will be closed w/o adding anything, and new
+In case returned `actualSize` is 0, current CRU page will be closed just with user trailer added (if any) and new
 query of this method will be done on the new CRU page.
 
+By default, the carry-over callback is not called if remaining data fits to the free space of the 8KB page (or the super-page).
+In case the splitting affects the information written in the payload trailer, user may set `writer.setApplyCarryOverToLastPage(true)`.
+With this flag set to `ON`, if there was at least one splitting for the user payload provided to `addData` method, then the carry-over
+method will be called also for the very last chunk (which by definition does not need splitting) and the supplied trailer will overwrite
+the tail of the this chunk instead of adding it incrementally.
+
 Additionally, in case detector wants to add some information between `empty` HBF packet's opening and
 closing RDHs (they will be added automatically using the HBFUtils functionality for all HBFs w/o data
 between the orbit 0 and last orbit of the TF seen in the simulations), it may implement a callback method

@@ -341,9 +341,12 @@ class RawFileWriter
     }
   }
 
+  void setApplyCarryOverToLastPage(bool v) { mApplyCarryOverToLastPage = v; }
+
   bool isRORCDetector() const { return !mCRUDetector; }
   bool isCRUDetector() const { return mCRUDetector; }
   bool isRDHStopUsed() const { return mUseRDHStop; }
+  bool isCarryOverToLastPageApplied() const { return mApplyCarryOverToLastPage; }
 
  private:
   void fillFromCache();
@@ -369,6 +372,7 @@ class RawFileWriter
   bool mAddSeparateHBFStopPage = true; // HBF stop is added on a separate CRU page
   bool mUseRDHStop = true;             // detector uses STOP in RDH
   bool mCRUDetector = true;            // Detector readout via CRU ( RORC if false)
+  bool mApplyCarryOverToLastPage = false; // call CarryOver method also for last chunk and overwrite modified trailer
 
   //>> caching --------------
   bool mCachingStage = false; // signal that current data should be cached

@@ -308,7 +308,7 @@ void RawFileWriter::LinkData::addData(const IR& ir, const gsl::span<char> data,
   // in case particular detector CRU pages need to be self-consistent, when carrying-over
   // large payload to new CRU page we may need to write optional trailer and header before
   // and after the new RDH.
-  bool carryOver = false;
+  bool carryOver = false, wasSplit = false, lastSplitPart = false;
   int splitID = 0;
   std::vector<char> carryOverHeader;
   while (dataSize > 0) {
@@ -328,26 +328,46 @@ void RawFileWriter::LinkData::addData(const IR& ir, const gsl::span<char> data,
       addHBFPage();  // start new CRU page, if needed, the completed superpage is flushed
       continue;
     }
-    if (dataSize <= sizeLeft) { // add all remaining data
+
+    if (dataSize <= sizeLeft) {
+      if (wasSplit && writer->mApplyCarryOverToLastPage) {
+        lastSplitPart = true;
+        carryOver = true;
+      }
+    } else {
+      carryOver = true;
+      wasSplit = true;
+    }
+
+    if (!carryOver) { // add all remaining data
       LOG(DEBUG) << "Adding payload " << dataSize << " bytes in IR " << ir << " (carryover=" << carryOver << " ) to " << describe();
       pushBack(ptr, dataSize);
       dataSize = 0;
     } else { // need to carryOver payload, determine 1st wsize bytes to write starting from ptr
-      carryOver = true;
+      if (sizeLeft > dataSize) {
+        sizeLeft = dataSize;
+      }
       int sizeActual = sizeLeft;
       std::vector<char> carryOverTrailer;
       if (writer->carryOverFunc) {
         sizeActual = writer->carryOverFunc(&rdhCopy, data, ptr, sizeLeft, splitID++, carryOverTrailer, carryOverHeader);
       }
       LOG(DEBUG) << "Adding carry-over " << splitID - 1 << " fitted payload " << sizeActual << " bytes in IR " << ir << " to " << describe();
-      if (sizeActual < 0 || sizeActual + carryOverTrailer.size() > sizeLeft) {
+      if (sizeActual < 0 || (!lastSplitPart && (sizeActual + carryOverTrailer.size() > sizeLeft))) {
         throw std::runtime_error(std::string("wrong carry-over data size provided by carryOverMethod") + std::to_string(sizeActual));
       }
-      pushBack(ptr, sizeActual); // write payload fitting to this page
+      // if there is carry-over trailer at the very last chunk, it must overwrite existing trailer
+      int trailerOffset = 0;
+      if (lastSplitPart) {
+        trailerOffset = carryOverTrailer.size();
+        if (sizeActual - trailerOffset < 0) {
+          throw std::runtime_error("trailer size of last split chunk cannot exceed actual size as it overwrites the existing trailer");
+        }
+      }
+      pushBack(ptr, sizeActual - trailerOffset); // write payload fitting to this page
       dataSize -= sizeActual;
       ptr += sizeActual;
-      LOG(DEBUG) << "Adding carryOverTrailer " << carryOverTrailer.size() << " bytes in IR "
-                 << ir << " to " << describe();
+      LOG(DEBUG) << "Adding carryOverTrailer " << carryOverTrailer.size() << " bytes in IR " << ir << " to " << describe();
       pushBack(carryOverTrailer.data(), carryOverTrailer.size());
     }
   }

@@ -81,6 +81,8 @@ struct TestRawWriter { // simple class to create detector payload for multiple l
       writer.setEmptyPageCallBack(this); // we want the writer to ask the detector code what to put in empty HBFs
     }
     writer.setCarryOverCallBack(this); // we want that writer to ask the detector code how to split large payloads
+
+    writer.setApplyCarryOverToLastPage(true); // call CarryOver method also for the last chunk
   }
 
   //_________________________________________________________________
@@ -155,18 +157,28 @@ struct TestRawWriter { // simple class to create detector payload for multiple l
   {
     // how we want to split the large payloads. The data is the full payload which was sent for writing and
     // it is already equiped with header and trailer
+    static int verboseCount = 0;
+
     if (maxSize <= RDHUtils::GBTWord) { // do not carry over trailer or header only
       return 0;
     }
 
+    int bytesLeft = data.size() - (ptr - &data[0]);
+    bool lastPage = bytesLeft <= maxSize;
+    if (verboseCount++ < 100) {
+      LOG(INFO) << "Carry-over method for chunk of size " << bytesLeft << " is called, MaxSize = " << maxSize << (lastPage ? " : last chunk being processed!" : "");
+    }
     // here we simply copy the header/trailer of the payload to every CRU page of this payload
     header.resize(RDHUtils::GBTWord);
     std::memcpy(header.data(), &data[0], RDHUtils::GBTWord);
     trailer.resize(RDHUtils::GBTWord);
     std::memcpy(trailer.data(), &data[data.size() - RDHUtils::GBTWord], RDHUtils::GBTWord);
     // since we write an extra GBT word (trailer) in the end of the CRU page, we ask to write
     // not the block ptr : ptr+maxSize, but ptr : ptr+maxSize - GBTWord;
-    int sz = maxSize - RDHUtils::GBTWord;
+    int sz = maxSize; // if the method is called for the last page, then the trailer is overwritten !!!
+    if (!lastPage) {  // otherwise it is added incrementally, so its size must be accounted
+      sz -= trailer.size();
+    }
     return sz;
   }
 };