Skip to content

Commit f09760f

Browse files
davidrohrdavidrohr
committed
GPU: Use ConfigurableParam to configure GPU Tracking workflow
1 parent 336df45 commit f09760f

16 files changed

+352
-292
lines changed

‎Detectors/TPC/workflow/src/CATrackerSpec.cxx‎

Lines changed: 24 additions & 167 deletions
Original file line numberDiff line numberDiff line change
@@ -95,178 +95,37 @@ DataProcessorSpec getCATrackerSpec(ca::Config const& specconfig, std::vector<int
9595

9696
auto processAttributes = std::make_shared<ProcessAttributes>();
9797
auto initFunction = [processAttributes, specconfig](InitContext& ic) {
98-
auto options = ic.options().get<std::string>("tracker-options");
9998
{
10099
auto& parser = processAttributes->parser;
101100
auto& tracker = processAttributes->tracker;
102101
parser = std::make_unique<ClusterGroupParser>();
103102
tracker = std::make_unique<GPUCATracking>();
104103

105-
// Prepare initialization of CATracker - we parse the deprecated option string here,
106-
// and create the proper configuration objects for compatibility.
107-
// This should go away eventually.
108-
109-
// Default Settings
110-
float solenoidBz = 5.00668; // B-field
111-
float refX = 83.; // transport tracks to this x after tracking, >500 for disabling
112-
bool continuous = false; // time frame data v.s. triggered events
113-
int nThreads = 1; // number of threads if we run on the CPU, 1 = default, 0 = auto-detect
114-
bool useGPU = false; // use a GPU for processing, if false uses GPU
115-
int debugLevel = 0; // Enable additional debug output
116-
int dump = 0; // create memory dump of processed events for standalone runs, 2 to dump only and skip processing
117-
char gpuType[1024] = "CUDA"; // Type of GPU device, if useGPU is set to true
118-
int gpuDevice = -1; // Select GPU device id (-1 = auto-detect fastest, -2 = use pipeline-slice)
119-
GPUDisplayBackend* display = nullptr; // Ptr to display backend (enables event display)
120-
bool qa = false; // Run the QA after tracking
121-
bool readTransformationFromFile = false; // Read the TPC transformation from the file
122-
bool allocateOutputOnTheFly = true; // Provide a callback to allocate output buffer on the fly instead of preallocating
123-
char tpcTransformationFileName[1024] = ""; // A file with the TPC transformation
124-
char matBudFileName[1024] = ""; // Material budget file name
125-
char dEdxSplinesFile[1024] = ""; // File containing dEdx splines
126-
int tpcRejectionMode = GPUSettings::RejectionStrategyA;
127-
size_t memoryPoolSize = 1;
128-
size_t hostMemoryPoolSize = 0;
104+
// Create configuration object and fill settings
129105

106+
GPUO2InterfaceConfiguration config;
130107
const auto grp = o2::parameters::GRPObject::loadFrom("o2sim_grp.root");
131108
if (grp) {
132-
solenoidBz *= grp->getL3Current() / 30000.;
133-
continuous = grp->isDetContinuousReadOut(o2::detectors::DetID::TPC);
134-
LOG(INFO) << "Initializing run paramerers from GRP bz=" << solenoidBz << " cont=" << continuous;
109+
config.configEvent.solenoidBz *= grp->getL3Current() / 30000.;
110+
config.configEvent.continuousMaxTimeBin = grp->isDetContinuousReadOut(o2::detectors::DetID::TPC) ? -1 : 0; // Number of timebins in timeframe if continuous, 0 otherwise
111+
LOG(INFO) << "Initializing run paramerers from GRP bz=" << config.configEvent.solenoidBz << " cont=" << grp->isDetContinuousReadOut(o2::detectors::DetID::TPC);
135112
} else {
136113
throw std::runtime_error("Failed to initialize run parameters from GRP");
137114
}
138-
139-
// Parse the config string
140-
const char* opt = options.c_str();
141-
if (opt && *opt) {
142-
printf("Received options %s\n", opt);
143-
const char* optPtr = opt;
144-
while (optPtr && *optPtr) {
145-
while (*optPtr == ' ') {
146-
optPtr++;
147-
}
148-
const char* nextPtr = strstr(optPtr, " ");
149-
const int optLen = nextPtr ? nextPtr - optPtr : strlen(optPtr);
150-
if (strncmp(optPtr, "cont", optLen) == 0) {
151-
continuous = true;
152-
printf("Continuous tracking mode enabled\n");
153-
} else if (strncmp(optPtr, "dump", optLen) == 0) {
154-
dump = 1;
155-
printf("Dumping of input events enabled\n");
156-
} else if (strncmp(optPtr, "dumponly", optLen) == 0) {
157-
dump = 2;
158-
printf("Dumping of input events enabled, processing disabled\n");
159-
} else if (strncmp(optPtr, "display", optLen) == 0) {
160-
#ifdef GPUCA_BUILD_EVENT_DISPLAY
161-
processAttributes->displayBackend.reset(new GPUDisplayBackendGlfw);
162-
display = processAttributes->displayBackend.get();
163-
printf("Event display enabled\n");
164-
#else
165-
printf("Standalone Event Display not enabled at build time!\n");
166-
#endif
167-
} else if (strncmp(optPtr, "qa", optLen) == 0) {
168-
qa = true;
169-
printf("Enabling TPC Standalone QA\n");
170-
} else if (optLen > 3 && strncmp(optPtr, "bz=", 3) == 0) {
171-
sscanf(optPtr + 3, "%f", &solenoidBz);
172-
printf("Using solenoid field %f\n", solenoidBz);
173-
} else if (optLen > 5 && strncmp(optPtr, "refX=", 5) == 0) {
174-
sscanf(optPtr + 5, "%f", &refX);
175-
printf("Propagating to reference X %f\n", refX);
176-
} else if (optLen > 5 && strncmp(optPtr, "debug=", 6) == 0) {
177-
sscanf(optPtr + 6, "%d", &debugLevel);
178-
printf("Debug level set to %d\n", debugLevel);
179-
} else if (optLen > 8 && strncmp(optPtr, "threads=", 8) == 0) {
180-
sscanf(optPtr + 8, "%d", &nThreads);
181-
printf("Using %d threads\n", nThreads);
182-
} else if (optLen > 21 && strncmp(optPtr, "tpcRejectionStrategy=", 21) == 0) {
183-
sscanf(optPtr + 21, "%d", &tpcRejectionMode);
184-
tpcRejectionMode = tpcRejectionMode == 0 ? GPUSettings::RejectionNone : tpcRejectionMode == 1 ? GPUSettings::RejectionStrategyA : GPUSettings::RejectionStrategyB;
185-
printf("TPC Rejection Mode: %d\n", tpcRejectionMode);
186-
} else if (optLen > 8 && strncmp(optPtr, "gpuType=", 8) == 0) {
187-
int len = std::min(optLen - 8, 1023);
188-
memcpy(gpuType, optPtr + 8, len);
189-
gpuType[len] = 0;
190-
useGPU = true;
191-
printf("Using GPU Type %s\n", gpuType);
192-
} else if (optLen > 8 && strncmp(optPtr, "matBudFile=", 8) == 0) {
193-
int len = std::min(optLen - 11, 1023);
194-
memcpy(matBudFileName, optPtr + 11, len);
195-
matBudFileName[len] = 0;
196-
} else if (optLen > 7 && strncmp(optPtr, "gpuNum=", 7) == 0) {
197-
sscanf(optPtr + 7, "%d", &gpuDevice);
198-
printf("Using GPU device %d\n", gpuDevice);
199-
} else if (optLen > 8 && strncmp(optPtr, "gpuMemorySize=", 14) == 0) {
200-
sscanf(optPtr + 14, "%llu", (unsigned long long int*)&memoryPoolSize);
201-
printf("GPU memory pool size set to %llu\n", (unsigned long long int)memoryPoolSize);
202-
} else if (optLen > 8 && strncmp(optPtr, "hostMemorySize=", 15) == 0) {
203-
sscanf(optPtr + 15, "%llu", (unsigned long long int*)&hostMemoryPoolSize);
204-
printf("Host memory pool size set to %llu\n", (unsigned long long int)hostMemoryPoolSize);
205-
} else if (optLen > 8 && strncmp(optPtr, "dEdxFile=", 9) == 0) {
206-
int len = std::min(optLen - 9, 1023);
207-
memcpy(dEdxSplinesFile, optPtr + 9, len);
208-
dEdxSplinesFile[len] = 0;
209-
} else if (optLen > 15 && strncmp(optPtr, "transformation=", 15) == 0) {
210-
int len = std::min(optLen - 15, 1023);
211-
memcpy(tpcTransformationFileName, optPtr + 15, len);
212-
tpcTransformationFileName[len] = 0;
213-
readTransformationFromFile = true;
214-
printf("Read TPC transformation from the file \"%s\"\n", tpcTransformationFileName);
215-
} else {
216-
printf("Unknown option: %s\n", optPtr);
217-
throw std::invalid_argument("Unknown config string option");
218-
}
219-
optPtr = nextPtr;
220-
}
115+
const GPUSettingsO2& confParam = config.ReadConfigurableParam();
116+
processAttributes->allocateOutputOnTheFly = confParam.allocateOutputOnTheFly;
117+
processAttributes->suppressOutput = (confParam.dump == 2);
118+
if (config.configEvent.continuousMaxTimeBin == -1) {
119+
config.configEvent.continuousMaxTimeBin = (o2::raw::HBFUtils::Instance().getNOrbitsPerTF() * o2::constants::lhc::LHCMaxBunches + 2 * Constants::LHCBCPERTIMEBIN - 2) / Constants::LHCBCPERTIMEBIN;
221120
}
222-
223-
// Create configuration object and fill settings
224-
processAttributes->allocateOutputOnTheFly = allocateOutputOnTheFly;
225-
processAttributes->suppressOutput = (dump == 2);
226-
GPUO2InterfaceConfiguration config;
227-
if (useGPU) {
228-
config.configDeviceBackend.deviceType = GPUDataTypes::GetDeviceType(gpuType);
229-
} else {
230-
config.configDeviceBackend.deviceType = GPUDataTypes::DeviceType::CPU;
231-
}
232-
config.configDeviceBackend.forceDeviceType = true; // If we request a GPU, we force that it is available - no CPU fallback
233-
234-
if (gpuDevice == -2) {
121+
if (config.configProcessing.deviceNum == -2) {
235122
int myId = ic.services().get<const o2::framework::DeviceSpec>().inputTimesliceId;
236123
int idMax = ic.services().get<const o2::framework::DeviceSpec>().maxInputTimeslices;
237-
gpuDevice = myId;
124+
config.configProcessing.deviceNum = myId;
238125
LOG(INFO) << "GPU device number selected from pipeline id: " << myId << " / " << idMax;
239126
}
240-
config.configProcessing.deviceNum = gpuDevice;
241-
config.configProcessing.ompThreads = nThreads;
242-
config.configProcessing.runQA = qa; // Run QA after tracking
243-
config.configProcessing.runMC = specconfig.processMC; // Propagate MC labels
244-
config.configProcessing.eventDisplay = display; // Ptr to event display backend, for running standalone OpenGL event display
245-
config.configProcessing.debugLevel = debugLevel; // Debug verbosity
246-
config.configProcessing.forceMemoryPoolSize = memoryPoolSize; // GPU / Host Memory pool size, default = 1 = auto-detect
247-
config.configProcessing.forceHostMemoryPoolSize = hostMemoryPoolSize; // Same for host, overrides the avove value for the host if set
248-
if (memoryPoolSize || hostMemoryPoolSize) {
249-
config.configProcessing.memoryAllocationStrategy = 2;
250-
}
251-
252-
config.configEvent.solenoidBz = solenoidBz;
253-
int maxContTimeBin = (o2::raw::HBFUtils::Instance().getNOrbitsPerTF() * o2::constants::lhc::LHCMaxBunches + 2 * Constants::LHCBCPERTIMEBIN - 2) / Constants::LHCBCPERTIMEBIN;
254-
config.configEvent.continuousMaxTimeBin = continuous ? maxContTimeBin : 0; // Number of timebins in timeframe if continuous, 0 otherwise
255-
256-
config.configReconstruction.NWays = 3; // Should always be 3!
257-
config.configReconstruction.NWaysOuter = true; // Will create outer param for TRD
258-
config.configReconstruction.SearchWindowDZDR = 2.5f; // Should always be 2.5 for looper-finding and/or continuous tracking
259-
config.configReconstruction.TrackReferenceX = refX;
260-
261-
// Settings for TPC Compression:
262-
config.configReconstruction.tpcRejectionMode = tpcRejectionMode; // Implement TPC Strategy A
263-
config.configReconstruction.tpcRejectQPt = 1.f / 0.05f; // Reject clusters of tracks < 50 MeV
264-
config.configReconstruction.tpcCompressionModes = GPUSettings::CompressionFull; // Activate all compression steps
265-
config.configReconstruction.tpcCompressionSortOrder = GPUSettings::SortTime; // Sort order for differences compression
266-
config.configReconstruction.tpcSigBitsCharge = 4; // Number of significant bits in TPC cluster chargs
267-
config.configReconstruction.tpcSigBitsWidth = 3; // Number of significant bits in TPC cluster width
268-
269-
config.configInterface.dumpEvents = dump;
127+
config.configProcessing.runMC = specconfig.processMC;
128+
config.configReconstruction.NWaysOuter = true;
270129
config.configInterface.outputToExternalBuffers = true;
271130

272131
// Configure the "GPU workflow" i.e. which steps we run on the GPU (or CPU) with this instance of GPUCATracking
@@ -294,24 +153,25 @@ DataProcessorSpec getCATrackerSpec(ca::Config const& specconfig, std::vector<int
294153
}
295154

296155
// Create and forward data objects for TPC transformation, material LUT, ...
297-
if (readTransformationFromFile) {
156+
if (confParam.transformationFile.size()) {
298157
processAttributes->fastTransform = nullptr;
299-
config.configCalib.fastTransform = TPCFastTransform::loadFromFile(tpcTransformationFileName);
158+
config.configCalib.fastTransform = TPCFastTransform::loadFromFile(confParam.transformationFile.c_str());
300159
} else {
301160
processAttributes->fastTransform = std::move(TPCFastTransformHelperO2::instance()->create(0));
302161
config.configCalib.fastTransform = processAttributes->fastTransform.get();
303162
}
304163
if (config.configCalib.fastTransform == nullptr) {
305164
throw std::invalid_argument("GPUCATracking: initialization of the TPC transformation failed");
306165
}
307-
if (strlen(matBudFileName)) {
308-
config.configCalib.matLUT = o2::base::MatLayerCylSet::loadFromFile(matBudFileName, "MatBud");
166+
if (confParam.matLUTFile.size()) {
167+
config.configCalib.matLUT = o2::base::MatLayerCylSet::loadFromFile(confParam.matLUTFile.c_str(), "MatBud");
309168
}
310-
processAttributes->dEdxSplines.reset(new TPCdEdxCalibrationSplines);
311-
if (strlen(dEdxSplinesFile)) {
312-
TFile dEdxFile(dEdxSplinesFile);
313-
processAttributes->dEdxSplines->setSplinesFromFile(dEdxFile);
169+
if (confParam.dEdxFile.size()) {
170+
processAttributes->dEdxSplines.reset(new TPCdEdxCalibrationSplines(confParam.dEdxFile.c_str()));
171+
} else {
172+
processAttributes->dEdxSplines.reset(new TPCdEdxCalibrationSplines);
314173
}
174+
315175
config.configCalib.dEdxSplines = processAttributes->dEdxSplines.get();
316176

317177
// Sample code what needs to be done for the TRD Geometry, when we extend this to TRD tracking.
@@ -877,10 +737,7 @@ DataProcessorSpec getCATrackerSpec(ca::Config const& specconfig, std::vector<int
877737
return DataProcessorSpec{"tpc-tracker", // process id
878738
{createInputSpecs()},
879739
{createOutputSpecs()},
880-
AlgorithmSpec(initFunction),
881-
Options{
882-
{"tracker-options", VariantType::String, "", {"Option string passed to tracker"}},
883-
}};
740+
AlgorithmSpec(initFunction)};
884741
}
885742

886743
} // namespace tpc

‎GPU/GPUTracking/Base/GPUReconstruction.cxx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,9 @@ int GPUReconstruction::InitPhaseBeforeDevice()
193193
mRecoStepsGPU.set((unsigned char)0);
194194
}
195195

196+
if (mProcessingSettings.forceMemoryPoolSize || mProcessingSettings.forceHostMemoryPoolSize) {
197+
mProcessingSettings.memoryAllocationStrategy = GPUMemoryResource::ALLOCATION_GLOBAL;
198+
}
196199
if (mProcessingSettings.memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_AUTO) {
197200
mProcessingSettings.memoryAllocationStrategy = IsGPU() ? GPUMemoryResource::ALLOCATION_GLOBAL : GPUMemoryResource::ALLOCATION_INDIVIDUAL;
198201
}

‎GPU/GPUTracking/Base/GPUReconstructionTimeframe.cxx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ namespace GPUCA_NAMESPACE::gpu
3535
{
3636
extern GPUSettingsStandalone configStandalone;
3737
}
38-
static auto& config = configStandalone.configTF;
38+
static auto& config = configStandalone.TF;
3939

4040
GPUReconstructionTimeframe::GPUReconstructionTimeframe(GPUChainTracking* chain, int (*read)(int), int nEvents) : mChain(chain), mReadEvent(read), mNEventsInDirectory(nEvents), mDisUniReal(0., 1.), mRndGen1(configStandalone.seed), mRndGen2(mDisUniInt(mRndGen1))
4141
{

‎GPU/GPUTracking/Base/GPUSettings.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include "GPUCommonDef.h"
1818
#ifndef __OPENCL__
1919
#include <vector>
20+
#include <string>
2021
#endif
2122

2223
namespace GPUCA_NAMESPACE

‎GPU/GPUTracking/CMakeLists.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,8 @@ set(HDRS_INSTALL
133133

134134
# Sources only for O2
135135
if(ALIGPU_BUILD_TYPE STREQUAL "O2")
136-
set(SRCS ${SRCS} Interface/GPUO2Interface.cxx)
137-
set(HDRS_CINT_O2 ${HDRS_CINT_O2} Interface/GPUO2Interface.h dEdx/TPCdEdxCalibrationSplines.h)
136+
set(SRCS ${SRCS} Interface/GPUO2Interface.cxx Interface/GPUO2InterfaceConfigurableParam.cxx)
137+
set(HDRS_CINT_O2 ${HDRS_CINT_O2} Interface/GPUO2Interface.h Interface/GPUO2InterfaceConfigurableParam.h dEdx/TPCdEdxCalibrationSplines.h)
138138
endif()
139139

140140
# Sources for O2 and for Standalone if requested in config file

‎GPU/GPUTracking/GPUTrackingLinkDef_O2.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,10 @@
1919

2020
#pragma link C++ class o2::gpu::GPUTPCO2Interface + ;
2121
#pragma link C++ class o2::gpu::TPCdEdxCalibrationSplines + ;
22+
#pragma link C++ class o2::gpu::GPUConfigurableParamGPUSettingsO2 + ;
23+
#pragma link C++ class o2::gpu::GPUConfigurableParamGPUSettingsRec + ;
24+
#pragma link C++ class o2::gpu::GPUConfigurableParamGPUSettingsProcessing + ;
25+
#pragma link C++ class o2::gpu::GPUConfigurableParamGPUSettingsDisplay + ;
26+
#pragma link C++ class o2::gpu::GPUConfigurableParamGPUSettingsQA + ;
2227

2328
#endif

0 commit comments

Comments
 (0)