Skip to content

Commit f455e37

Browse files
wiechulashahor02
wiechula
authored and
shahor02
committed
Code splitting and cleanup
o move common raw data processing function for calibrations to separate file o clean up includes
1 parent b160fdd commit f455e37

File tree

6 files changed

+203
-104
lines changed

6 files changed

+203
-104
lines changed

‎Detectors/TPC/workflow/CMakeLists.txt‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,12 @@ o2_add_library(TPCWorkflow
2323
src/LinkZSToDigitsSpec.cxx
2424
src/TPCSectorCompletionPolicy.cxx
2525
src/ZSSpec.cxx
26+
src/CalibProcessingHelper.cxx
2627
TARGETVARNAME targetName
2728
PUBLIC_LINK_LIBRARIES O2::Framework O2::DataFormatsTPC
2829
O2::DPLUtils O2::TPCReconstruction
29-
O2::TPCCalibration O2::TPCSimulation)
30+
O2::TPCCalibration O2::TPCSimulation
31+
O2::DetectorsCalibration)
3032

3133
o2_add_executable(reco-workflow
3234
COMPONENT_NAME tpc
@@ -41,8 +43,7 @@ o2_add_executable(raw-to-digits-workflow
4143
o2_add_executable(calib-pedestal
4244
COMPONENT_NAME tpc
4345
SOURCES src/tpc-calib-pedestal.cxx
44-
PUBLIC_LINK_LIBRARIES O2::TPCWorkflow
45-
O2::DetectorsCalibration)
46+
PUBLIC_LINK_LIBRARIES O2::TPCWorkflow)
4647

4748
o2_add_test(workflow
4849
COMPONENT_NAME tpc

‎Detectors/TPC/workflow/README.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,3 +146,44 @@ bz= magnetic field
146146
* implement configuration from the GRP
147147

148148
## Open questions
149+
150+
## Calibration workflows
151+
152+
### Pedestal calibration
153+
#### Options
154+
```bash
155+
--direct-file-dump write final calibration to local root file
156+
--max-events maximum number of events to process
157+
--no-calib-output don't send the calibration data via DPL (required in case the calibration write is not attached)
158+
--use-old-subspec Subspecification is built from RDH like
159+
```
160+
161+
#### Runnin with data distribution
162+
In one shell start the data distribution playback, e.g.
163+
```bash
164+
StfBuilder --id builder --data-source-enable --data-source-dir=2020-11-11T14_18_25Z --data-source-rate=100 --dpl-channel-name=dpl-chan --channel-config "name=dpl-chan,type=pair,method=connect,address=ipc:///tmp/stf-builder-dpl-pipe-0,transport=zeromq,rateLogging=1"
165+
```
166+
167+
In another shell start the pedestal calibration
168+
```bash
169+
o2-dpl-raw-proxy -b --dataspec "A:TPC/RAWDATA" --channel-config "name=readout-proxy,type=pair,method=bind,address=ipc:///tmp/stf-builder-dpl-pipe-0,transport=zeromq,rateLogging=1" \
170+
| o2-tpc-calib-pedestal --max-events 5 --direct-file-dump --shm-segment-size $((8<<30)) --no-calib-output --use-old-subspec
171+
```
172+
173+
#### Running with raw file playback
174+
Create a raw-reader.cfg e.g.
175+
```bash
176+
i=0; echo "[defaults]"; echo "dataOrigin = TPC"; echo "dataDescription = RAWDATA"; echo; for file in *.raw; do echo "[input-$i]"; echo "dataOrigin = TPC"; echo "dataDescription = RAWDATA"; echo "filePath=$file"; echo; i=$((i+1)); done > raw-reader.cfg
177+
```
178+
179+
Then run
180+
```bash
181+
o2-raw-file-reader-workflow --input-conf raw-reader.cfg --nocheck-hbf-per-tf --nocheck-hbf-jump --shm-segment-size $((8<<30)) \
182+
| o2-tpc-calib-pedestal --max-events 5 --direct-file-dump --shm-segment-size $((8<<30)) --no-calib-output
183+
```
184+
185+
#### send data to CCDB
186+
Remove the `--no-calib-output` option and add
187+
```bash
188+
| o2-calibration-ccdb-populator-workflow
189+
```

‎Detectors/TPC/workflow/include/TPCWorkflow/CalibProcessingHelper.h‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Copyright CERN and copyright holders of ALICE O2. This software is
2+
// distributed under the terms of the GNU General Public License v3 (GPL
3+
// Version 3), copied verbatim in the file "COPYING".
4+
//
5+
// See http://alice-o2.web.cern.ch/license for full licensing information.
6+
//
7+
// In applying this license CERN does not waive the privileges and immunities
8+
// granted to it by virtue of its status as an Intergovernmental Organization
9+
// or submit itself to any jurisdiction.
10+
11+
#ifndef O2_TPC_CalibProcessingHelper_H
12+
#define O2_TPC_CalibProcessingHelper_H
13+
14+
#include <memory>
15+
16+
#include "Framework/InputRecord.h"
17+
18+
using o2::tpc::rawreader::RawReaderCRU;
19+
20+
namespace o2
21+
{
22+
namespace tpc
23+
{
24+
namespace rawreader
25+
{
26+
class RawReaderCRU;
27+
}
28+
namespace calib_processing_helper
29+
{
30+
31+
void processRawData(o2::framework::InputRecord& inputs, std::unique_ptr<RawReaderCRU>& reader, bool useOldSubspec = false);
32+
} // namespace raw_reader_helper
33+
} // namespace tpc
34+
} // namespace o2
35+
36+
#endif

‎Detectors/TPC/workflow/include/TPCWorkflow/TPCCalibPedestalSpec.h‎

Lines changed: 28 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -14,33 +14,26 @@
1414
/// @file TPCCalibPedestalSpec.h
1515
/// @brief TPC Pedestal calibration processor
1616

17-
#include "DetectorsCalibration/Utils.h"
17+
#include <vector>
18+
#include <string>
19+
#include <chrono>
20+
1821
#include "Framework/Task.h"
19-
#include "Framework/ConfigParamRegistry.h"
2022
#include "Framework/ControlService.h"
21-
#include "Framework/WorkflowSpec.h"
22-
2323
#include "Framework/Logger.h"
24+
#include "Framework/ConfigParamRegistry.h"
25+
2426
#include "Headers/DataHeader.h"
25-
#include "Framework/ConfigParamSpec.h"
26-
#include "Framework/CompletionPolicy.h"
27-
#include "Framework/CompletionPolicyHelpers.h"
27+
#include "CCDB/CcdbApi.h"
2828
#include "DetectorsCalibration/Utils.h"
29-
#include "DPLUtils/RawParser.h"
30-
#include "Headers/DataHeader.h"
31-
#include "CommonUtils/ConfigurableParam.h"
29+
30+
#include "TPCBase/CDBInterface.h"
3231
#include "TPCCalibration/CalibPedestal.h"
3332
#include "TPCReconstruction/RawReaderCRU.h"
34-
#include <vector>
35-
#include <string>
36-
#include "DetectorsRaw/RDHUtils.h"
37-
#include "TPCBase/RDHUtils.h"
38-
#include "TPCBase/CDBInterface.h"
39-
#include "CCDB/CcdbApi.h"
33+
#include "TPCWorkflow/CalibProcessingHelper.h"
4034

4135
using namespace o2::framework;
4236
using namespace o2::tpc;
43-
using RDHUtils = o2::raw::RDHUtils;
4437
using clbUtils = o2::calibration::Utils;
4538

4639
namespace o2
@@ -51,6 +44,8 @@ namespace calibration
5144
class TPCCalibPedestalDevice : public o2::framework::Task
5245
{
5346
public:
47+
TPCCalibPedestalDevice(bool skipCalib) : mSkipCalib(skipCalib) {}
48+
5449
void init(o2::framework::InitContext& ic) final
5550
{
5651
// set up ADC value filling
@@ -78,81 +73,8 @@ class TPCCalibPedestalDevice : public o2::framework::Task
7873
return;
7974
}
8075

81-
std::vector<InputSpec> filter = {{"check", ConcreteDataTypeMatcher{o2::header::gDataOriginTPC, "RAWDATA"}, Lifetime::Timeframe}};
82-
83-
for (auto const& ref : InputRecordWalker(pc.inputs(), filter)) {
84-
const auto* dh = DataRefUtils::getHeader<o2::header::DataHeader*>(ref);
85-
86-
// ---| extract hardware information to do the processing |---
87-
const auto subSpecification = dh->subSpecification;
88-
rdh_utils::FEEIDType feeID = (rdh_utils::FEEIDType)dh->subSpecification;
89-
rdh_utils::FEEIDType cruID, linkID, endPoint;
90-
91-
if (mUseOldSubspec) {
92-
//---| old definition by Gvozden |---
93-
cruID = (rdh_utils::FEEIDType)(subSpecification >> 16);
94-
linkID = (rdh_utils::FEEIDType)((subSpecification + (subSpecification >> 8)) & 0xFF) - 1;
95-
endPoint = (rdh_utils::FEEIDType)((subSpecification >> 8) & 0xFF) > 0;
96-
} else {
97-
//---| new definition by David |---
98-
rdh_utils::getMapping(feeID, cruID, endPoint, linkID);
99-
}
100-
101-
const auto globalLinkID = linkID + endPoint * 12;
102-
103-
// ---| update hardware information in the reader |---
104-
auto& reader = mRawReader.getReaders()[0];
105-
reader->forceCRU(cruID);
106-
reader->setLink(globalLinkID);
107-
108-
LOGP(info, "Specifier: {}/{}/{}", dh->dataOrigin.as<std::string>(), dh->dataDescription.as<std::string>(), subSpecification);
109-
LOGP(info, "Payload size: {}", dh->payloadSize);
110-
LOGP(info, "CRU: {}; linkID: {}; endPoint: {}; globalLinkID: {}", cruID, linkID, endPoint, globalLinkID);
111-
112-
// TODO: exception handling needed?
113-
const gsl::span<const char> raw = pc.inputs().get<gsl::span<char>>(ref);
114-
o2::framework::RawParser parser(raw.data(), raw.size());
115-
116-
// TODO: it would be better to have external event handling and then moving the event processing functionality to CalibRawBase and RawReader to not repeat it in other places
117-
rawreader::ADCRawData rawData;
118-
rawreader::GBTFrame gFrame;
119-
120-
for (auto it = parser.begin(), end = parser.end(); it != end; ++it) {
121-
// debugging stuff
122-
/*
123-
auto* rdhPtr = it.get_if<o2::header::RAWDataHeader>();
124-
if (!rdhPtr) {
125-
break;
126-
}
127-
const auto& rdh = *rdhPtr;
128-
auto feeID = RDHUtils::getFEEID(rdh);
129-
rdh_utils::FEEIDType cru, link, endPoint;
130-
rdh_utils::getMapping(feeID, cru, endPoint, link);
131-
LOGP(info, "feeID: {} -- CRU: {}; linkID: {}; endPoint: {}", feeID, cru, link, endPoint);
132-
*/
133-
134-
const auto size = it.size();
135-
auto data = it.data();
136-
//LOGP(info, "Data size: {}", size);
137-
138-
int iFrame = 0;
139-
for (int i = 0; i < size; i += 16) {
140-
gFrame.setFrameNumber(iFrame);
141-
gFrame.setPacketNumber(iFrame / 508);
142-
gFrame.readFromMemory(gsl::span<const o2::byte>(data + i, 16));
143-
144-
// extract the half words from the 4 32-bit words
145-
gFrame.getFrameHalfWords();
146-
147-
gFrame.getAdcValues(rawData);
148-
gFrame.updateSyncCheck(false);
149-
150-
++iFrame;
151-
}
152-
}
153-
154-
reader->runADCDataCallback(rawData);
155-
}
76+
auto& reader = mRawReader.getReaders()[0];
77+
calib_processing_helper::processRawData(pc.inputs(), reader, mUseOldSubspec);
15678

15779
mCalibPedestal.incrementNEvents();
15880
LOGP(info, "Number of processed events: {} ({})", mCalibPedestal.getNumberOfProcessedEvents(), mMaxEvents);
@@ -174,7 +96,9 @@ class TPCCalibPedestalDevice : public o2::framework::Task
17496
{
17597
LOGP(info, "endOfStream");
17698
dumpCalibData();
177-
sendOutput(ec.outputs());
99+
if (!mSkipCalib) {
100+
sendOutput(ec.outputs());
101+
}
178102
ec.services().get<ControlService>().readyToQuit(QuitRequest::Me);
179103
}
180104

@@ -187,12 +111,16 @@ class TPCCalibPedestalDevice : public o2::framework::Task
187111
bool mUseOldSubspec{false};
188112
bool mForceQuit{false};
189113
bool mDirectFileDump{false};
114+
bool mSkipCalib{false};
190115

191116
//____________________________________________________________________________
192117
void sendOutput(DataAllocator& output)
193118
{
194119
CDBStorage::MetaData_t md;
195-
long timeStart = 0;
120+
121+
// perhaps should be changed to time of the run
122+
const auto now = std::chrono::system_clock::now();
123+
long timeStart = std::chrono::duration_cast<std::chrono::seconds>(now.time_since_epoch()).count();
196124
long timeEnd = 99999999999999;
197125

198126
std::array<const CalDet<float>*, 2> data = {&mCalibPedestal.getPedestal(), &mCalibPedestal.getNoise()};
@@ -233,19 +161,21 @@ class TPCCalibPedestalDevice : public o2::framework::Task
233161
namespace framework
234162
{
235163

236-
DataProcessorSpec getTPCCalibPedestalSpec(const std::string inputSpec)
164+
DataProcessorSpec getTPCCalibPedestalSpec(const std::string inputSpec, bool skipCalib)
237165
{
238166
using device = o2::calibration::TPCCalibPedestalDevice;
239167

240168
std::vector<OutputSpec> outputs;
241-
outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCLB, clbUtils::gDataDescriptionCLBPayload});
242-
outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCLB, clbUtils::gDataDescriptionCLBInfo});
169+
if (!skipCalib) {
170+
outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCLB, clbUtils::gDataDescriptionCLBPayload});
171+
outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCLB, clbUtils::gDataDescriptionCLBInfo});
172+
}
243173

244174
return DataProcessorSpec{
245175
"calib-tpc-pedestal",
246176
select(inputSpec.data()),
247177
outputs,
248-
AlgorithmSpec{adaptFromTask<device>()},
178+
AlgorithmSpec{adaptFromTask<device>(skipCalib)},
249179
Options{
250180
{"max-events", VariantType::Int, 100, {"maximum number of events to process"}},
251181
{"use-old-subspec", VariantType::Bool, false, {"use old subsecifiation definition"}},

‎Detectors/TPC/workflow/src/CalibProcessingHelper.cxx‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
// Copyright CERN and copyright holders of ALICE O2. This software is
2+
// distributed under the terms of the GNU General Public License v3 (GPL
3+
// Version 3), copied verbatim in the file "COPYING".
4+
//
5+
// See http://alice-o2.web.cern.ch/license for full licensing information.
6+
//
7+
// In applying this license CERN does not waive the privileges and immunities
8+
// granted to it by virtue of its status as an Intergovernmental Organization
9+
// or submit itself to any jurisdiction.
10+
11+
#include "Framework/ConcreteDataMatcher.h"
12+
#include "Framework/InputRecordWalker.h"
13+
#include "Framework/Logger.h"
14+
15+
#include "DPLUtils/RawParser.h"
16+
#include "TPCBase/RDHUtils.h"
17+
#include "TPCReconstruction/RawReaderCRU.h"
18+
19+
#include "TPCWorkflow/CalibProcessingHelper.h"
20+
21+
using namespace o2::tpc;
22+
using namespace o2::framework;
23+
24+
void calib_processing_helper::processRawData(o2::framework::InputRecord& inputs, std::unique_ptr<RawReaderCRU>& reader, bool useOldSubspec)
25+
{
26+
std::vector<InputSpec> filter = {{"check", ConcreteDataTypeMatcher{o2::header::gDataOriginTPC, "RAWDATA"}, Lifetime::Timeframe}};
27+
28+
for (auto const& ref : InputRecordWalker(inputs, filter)) {
29+
const auto* dh = DataRefUtils::getHeader<o2::header::DataHeader*>(ref);
30+
31+
// ---| extract hardware information to do the processing |---
32+
const auto subSpecification = dh->subSpecification;
33+
rdh_utils::FEEIDType feeID = (rdh_utils::FEEIDType)dh->subSpecification;
34+
rdh_utils::FEEIDType cruID, linkID, endPoint;
35+
36+
if (useOldSubspec) {
37+
//---| old definition by Gvozden |---
38+
cruID = (rdh_utils::FEEIDType)(subSpecification >> 16);
39+
linkID = (rdh_utils::FEEIDType)((subSpecification + (subSpecification >> 8)) & 0xFF) - 1;
40+
endPoint = (rdh_utils::FEEIDType)((subSpecification >> 8) & 0xFF) > 0;
41+
} else {
42+
//---| new definition by David |---
43+
rdh_utils::getMapping(feeID, cruID, endPoint, linkID);
44+
}
45+
46+
const auto globalLinkID = linkID + endPoint * 12;
47+
48+
// ---| update hardware information in the reader |---
49+
reader->forceCRU(cruID);
50+
reader->setLink(globalLinkID);
51+
52+
LOGP(info, "Specifier: {}/{}/{}", dh->dataOrigin.as<std::string>(), dh->dataDescription.as<std::string>(), subSpecification);
53+
LOGP(info, "Payload size: {}", dh->payloadSize);
54+
LOGP(info, "CRU: {}; linkID: {}; endPoint: {}; globalLinkID: {}", cruID, linkID, endPoint, globalLinkID);
55+
56+
// TODO: exception handling needed?
57+
const gsl::span<const char> raw = inputs.get<gsl::span<char>>(ref);
58+
o2::framework::RawParser parser(raw.data(), raw.size());
59+
60+
// TODO: it would be better to have external event handling and then moving the event processing functionality to CalibRawBase and RawReader to not repeat it in other places
61+
rawreader::ADCRawData rawData;
62+
rawreader::GBTFrame gFrame;
63+
64+
for (auto it = parser.begin(), end = parser.end(); it != end; ++it) {
65+
66+
const auto size = it.size();
67+
auto data = it.data();
68+
//LOGP(info, "Data size: {}", size);
69+
70+
int iFrame = 0;
71+
for (int i = 0; i < size; i += 16) {
72+
gFrame.setFrameNumber(iFrame);
73+
gFrame.setPacketNumber(iFrame / 508);
74+
gFrame.readFromMemory(gsl::span<const o2::byte>(data + i, 16));
75+
76+
// extract the half words from the 4 32-bit words
77+
gFrame.getFrameHalfWords();
78+
79+
gFrame.getAdcValues(rawData);
80+
gFrame.updateSyncCheck(false);
81+
82+
++iFrame;
83+
}
84+
}
85+
86+
reader->runADCDataCallback(rawData);
87+
}
88+
}

0 commit comments

Comments
 (0)