Skip to content

Fixes for CCDB fetcher #8328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ktf merged 4 commits into from
Mar 12, 2022
Merged

Fixes for CCDB fetcher #8328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0773a86
Create dummy ETag in snapshot mode to profit from caching
shahor02 Mar 12, 2022
f7a57b4
fixes for caching in snapshot mode
shahor02 Mar 12, 2022
bd52552
Avoid multiple entries of same condition case of pipelenes
shahor02 Mar 12, 2022
e5d0971
Fix default timestamp
shahor02 Mar 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions CCDB/src/CcdbApi.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -953,8 +953,12 @@ void* CcdbApi::retrieveFromTFile(std::type_info const& tinfo, std::string const&
// many isolated processes, all querying the CCDB (for potentially the same objects and same timestamp).
// In addition, we can monitor exactly which objects are fetched and what is their content.
// One can also distribute so obtained caches to sites without network access.
auto cachedir = getenv("ALICEO2_CCDB_LOCALCACHE");
const char* cachedir = getenv("ALICEO2_CCDB_LOCALCACHE");
const char* cwd = ".";
if (cachedir) {
if (cachedir[0] == 0) {
cachedir = cwd;
}
// protect this sensitive section by a multi-process named semaphore
boost::interprocess::named_semaphore* sem = nullptr;
std::hash<std::string> hasher;
Expand Down Expand Up @@ -1418,8 +1422,12 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector<char>& dest, std::string const& p
// many isolated processes, all querying the CCDB (for potentially the same objects and same timestamp).
// In addition, we can monitor exactly which objects are fetched and what is their content.
// One can also distribute so obtained caches to sites without network access.
auto cachedir = getenv("ALICEO2_CCDB_LOCALCACHE");
const char* cachedir = getenv("ALICEO2_CCDB_LOCALCACHE");
const char* cwd = ".";
if (cachedir) {
if (cachedir[0] == 0) {
cachedir = cwd;
}
// protect this sensitive section by a multi-process named semaphore
boost::interprocess::named_semaphore* sem = nullptr;
std::hash<std::string> hasher;
Expand Down Expand Up @@ -1648,6 +1656,9 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector<char>& dest, const std::string& p
*localHeaders = *storedmeta; // do a simple deep copy
delete storedmeta;
}
if (isSnapshotMode()) { // generate dummy ETag to profit from the caching
(*localHeaders)["ETag"] = path;
}
}
return;
}
Expand Down
2 changes: 1 addition & 1 deletion Detectors/TRD/workflow/src/TRDTrackletTransformerWorkflow.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ void customize(std::vector<ConfigParamSpec>& workflowOptions)
{"disable-mc", o2::framework::VariantType::Bool, false, {"Disable MC labels"}},
{"disable-root-input", o2::framework::VariantType::Bool, false, {"disable root-files input reader"}},
{"disable-root-output", o2::framework::VariantType::Bool, false, {"disable root-files output writer"}},
{"timestamp", o2::framework::VariantType::Int, 555555, {"timestamp for CCDB calibration objects"}},
{"timestamp", o2::framework::VariantType::Int, 5555555, {"timestamp for CCDB calibration objects"}},
{"filter-trigrec", o2::framework::VariantType::Bool, false, {"ignore interaction records without ITS data"}},
{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings ..."}}};

Expand Down
20 changes: 14 additions & 6 deletions Framework/Core/src/CCDBHelpers.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@
#include "Framework/ConfigParamRegistry.h"
#include "Framework/DataTakingContext.h"
#include "Framework/RawDeviceService.h"
#include "Framework/DataSpecUtils.h"
#include "CCDB/CcdbApi.h"
#include "CommonConstants/LHCConstants.h"
#include <typeinfo>
#include <TError.h>
#include <TMemFile.h>
#include <functional>

namespace o2::framework
{
Expand Down Expand Up @@ -161,7 +163,6 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
{
// For Giulio: the dtc.orbitResetTime is wrong, it is assigned from the dph->creation, why?
std::string ccdbMetadataPrefix = "ccdb-metadata-";

for (auto& route : helper->routes) {
LOGP(info, "Fetching object for route {}", route.matcher);

Expand Down Expand Up @@ -211,20 +212,22 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
LOGP(info, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
// one could modify the adoptContainer to take optional old cacheID to clean:
// mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), true, mapURL2DPLCache[URL]);
continue;
}
// cached object is fine
auto cacheId = helper->mapURL2DPLCache[path];
LOGP(info, "Reusing {} for {}", cacheId.value, path);
allocator.adoptFromCache(output, cacheId, header::gSerializationMethodCCDB);
// the outputBuffer was not used, can we destroy it?
}
// cached object is fine
auto cacheId = helper->mapURL2DPLCache[path];
LOGP(info, "Reusing {} for {}", cacheId.value, path);
allocator.adoptFromCache(output, cacheId, header::gSerializationMethodCCDB);
// the outputBuffer was not used, can we destroy it?
}
};

AlgorithmSpec CCDBHelpers::fetchFromCCDB()
{
return adaptStateful([](ConfigParamRegistry const& options, DeviceSpec const& spec) {
std::shared_ptr<CCDBFetcherHelper> helper = std::make_shared<CCDBFetcherHelper>();
std::unordered_map<std::string, bool> accountedSpecs;
auto backend = options.get<std::string>("condition-backend");
LOGP(info, "CCDB Backend at: {}", backend);
const auto& defHost = options.get<std::string>("condition-backend");
Expand All @@ -251,6 +254,11 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
if (route.matcher.lifetime != Lifetime::Condition) {
continue;
}
auto specStr = DataSpecUtils::describe(route.matcher);
if (accountedSpecs.find(specStr) != accountedSpecs.end()) {
continue;
}
accountedSpecs[specStr] = true;
helper->routes.push_back(route);
LOGP(info, "The following route is a condition {}", route.matcher);
for (auto& metadata : route.matcher.metadata) {
Expand Down