Skip to content

Commit f31a96f

Browse files
sawenzelshahor02
sawenzel
authored and
shahor02
committed
ConfigParam - Ccdb-Api integration
Update ConfigurableParams when they are fetched from CCDB via CcdbApi. This guarantees the following: a) ConfigParams are updated with the CCDB instance b) However, members overriden by the user (kRT provenance) are not overriden by CCDB values (kRT takes higher priority)
1 parent 74af607 commit f31a96f

File tree

6 files changed

+239
-6
lines changed

6 files changed

+239
-6
lines changed

‎CCDB/CMakeLists.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,13 @@ o2_add_test(CcdbApi
5353
PUBLIC_LINK_LIBRARIES O2::CCDB
5454
LABELS ccdb)
5555

56+
o2_add_test(CcdbApiConfigParam
57+
SOURCES test/testCcdbApi_ConfigParam.cxx
58+
COMPONENT_NAME ccdb
59+
PUBLIC_LINK_LIBRARIES O2::CCDB O2::DetectorsVertexing
60+
LABELS ccdb)
61+
62+
5663
o2_add_test(CcdbApi-Alien
5764
SOURCES test/testCcdbApi_alien.cxx
5865
COMPONENT_NAME ccdb

‎CCDB/include/CCDB/CcdbApi.h‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
#include <TObject.h>
2525
#include <TMessage.h>
2626
#include "CCDB/CcdbObjectInfo.h"
27+
#include <CommonUtils/ConfigurableParam.h>
28+
#include <type_traits>
2729

2830
#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__)
2931
#include <TJAlienCredentials.h>
@@ -150,9 +152,16 @@ class CcdbApi //: public DatabaseInterface
150152
* @return the object, or nullptr if none were found or type does not match serialized type.
151153
*/
152154
template <typename T>
153-
T* retrieveFromTFileAny(std::string const& path, std::map<std::string, std::string> const& metadata,
154-
long timestamp = -1, std::map<std::string, std::string>* headers = nullptr, std::string const& etag = "",
155-
const std::string& createdNotAfter = "", const std::string& createdNotBefore = "") const;
155+
typename std::enable_if<!std::is_base_of<o2::conf::ConfigurableParam, T>::value, T*>::type
156+
retrieveFromTFileAny(std::string const& path, std::map<std::string, std::string> const& metadata,
157+
long timestamp = -1, std::map<std::string, std::string>* headers = nullptr, std::string const& etag = "",
158+
const std::string& createdNotAfter = "", const std::string& createdNotBefore = "") const;
159+
160+
template <typename T>
161+
typename std::enable_if<std::is_base_of<o2::conf::ConfigurableParam, T>::value, T*>::type
162+
retrieveFromTFileAny(std::string const& path, std::map<std::string, std::string> const& metadata,
163+
long timestamp = -1, std::map<std::string, std::string>* headers = nullptr, std::string const& etag = "",
164+
const std::string& createdNotAfter = "", const std::string& createdNotBefore = "") const;
156165

157166
/**
158167
* Delete all versions of the object at this path.
@@ -409,13 +418,26 @@ class CcdbApi //: public DatabaseInterface
409418
};
410419

411420
template <typename T>
412-
T* CcdbApi::retrieveFromTFileAny(std::string const& path, std::map<std::string, std::string> const& metadata,
413-
long timestamp, std::map<std::string, std::string>* headers, std::string const& etag,
414-
const std::string& createdNotAfter, const std::string& createdNotBefore) const
421+
typename std::enable_if<!std::is_base_of<o2::conf::ConfigurableParam, T>::value, T*>::type
422+
CcdbApi::retrieveFromTFileAny(std::string const& path, std::map<std::string, std::string> const& metadata,
423+
long timestamp, std::map<std::string, std::string>* headers, std::string const& etag,
424+
const std::string& createdNotAfter, const std::string& createdNotBefore) const
415425
{
416426
return static_cast<T*>(retrieveFromTFile(typeid(T), path, metadata, timestamp, headers, etag, createdNotAfter, createdNotBefore));
417427
}
418428

429+
template <typename T>
430+
typename std::enable_if<std::is_base_of<o2::conf::ConfigurableParam, T>::value, T*>::type
431+
CcdbApi::retrieveFromTFileAny(std::string const& path, std::map<std::string, std::string> const& metadata,
432+
long timestamp, std::map<std::string, std::string>* headers, std::string const& etag,
433+
const std::string& createdNotAfter, const std::string& createdNotBefore) const
434+
{
435+
auto obj = retrieveFromTFile(typeid(T), path, metadata, timestamp, headers, etag, createdNotAfter, createdNotBefore);
436+
auto& param = const_cast<typename std::remove_const<T&>::type>(T::Instance());
437+
param.syncCCDBandRegistry(obj);
438+
return &param;
439+
}
440+
419441
} // namespace ccdb
420442
} // namespace o2
421443

‎CCDB/test/testCcdbApi_ConfigParam.cxx‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
///
13+
/// \file testCcdbApi_ConfigParam.cxx
14+
/// \author Sandro Wenzel
15+
///
16+
17+
#define BOOST_TEST_MODULE CCDB
18+
#define BOOST_TEST_MAIN
19+
#define BOOST_TEST_DYN_LINK
20+
21+
#include "CCDB/CcdbApi.h"
22+
#include "CommonUtils/ConfigurableParam.h"
23+
#include "DetectorsVertexing/PVertexerParams.h"
24+
#include "CCDB/CCDBTimeStampUtils.h"
25+
#include <boost/test/unit_test.hpp>
26+
#include <filesystem>
27+
#include <cstdio>
28+
#include <cassert>
29+
#include <iostream>
30+
#include <cstdio>
31+
#include <curl/curl.h>
32+
#include <sys/stat.h>
33+
#include <fcntl.h>
34+
#include <chrono>
35+
#include <CommonUtils/StringUtils.h>
36+
#include <sys/types.h>
37+
#include <unistd.h>
38+
39+
#include <boost/property_tree/json_parser.hpp>
40+
#include <boost/property_tree/ptree.hpp>
41+
#include <boost/foreach.hpp>
42+
#include <boost/optional/optional.hpp>
43+
44+
using namespace std;
45+
using namespace o2::ccdb;
46+
namespace utf = boost::unit_test;
47+
namespace tt = boost::test_tools;
48+
49+
static string ccdbUrl;
50+
static string basePath;
51+
bool hostReachable = false;
52+
53+
/**
54+
* Global fixture, ie general setup and teardown
55+
*/
56+
struct Fixture {
57+
Fixture()
58+
{
59+
CcdbApi api;
60+
ccdbUrl = "http://ccdb-test.cern.ch:8080";
61+
api.init(ccdbUrl);
62+
cout << "ccdb url: " << ccdbUrl << endl;
63+
hostReachable = api.isHostReachable();
64+
cout << "Is host reachable ? --> " << hostReachable << endl;
65+
basePath = string("Test/pid") + getpid() + "/";
66+
cout << "Path we will use in this test suite : " + basePath << endl;
67+
}
68+
~Fixture()
69+
{
70+
if (hostReachable) {
71+
CcdbApi api;
72+
map<string, string> metadata;
73+
api.init(ccdbUrl);
74+
api.truncate(basePath + "*");
75+
cout << "Test data truncated (" << basePath << ")" << endl;
76+
}
77+
}
78+
};
79+
BOOST_GLOBAL_FIXTURE(Fixture);
80+
81+
/**
82+
* Just an accessor to the hostReachable variable to be used to determine whether tests can be ran or not.
83+
*/
84+
struct if_reachable {
85+
tt::assertion_result operator()(utf::test_unit_id)
86+
{
87+
return hostReachable;
88+
}
89+
};
90+
91+
/**
92+
* Fixture for the tests, i.e. code is ran in every test that uses it, i.e. it is like a setup and teardown for tests.
93+
*/
94+
struct test_fixture {
95+
test_fixture()
96+
{
97+
api.init(ccdbUrl);
98+
metadata["Hello"] = "World";
99+
std::cout << "*** " << boost::unit_test::framework::current_test_case().p_name << " ***" << std::endl;
100+
}
101+
~test_fixture() = default;
102+
103+
CcdbApi api;
104+
map<string, string> metadata;
105+
};
106+
107+
BOOST_AUTO_TEST_CASE(testConfigParamRetrieval, *utf::precondition(if_reachable()))
108+
{
109+
test_fixture f;
110+
111+
// We'd like to demonstrate the following:
112+
// GIVEN:
113+
// - user modifies field in a config param from command line (RT)
114+
// - user fetches config param from CCDB
115+
//
116+
// WE'D LIKE TO ARRIVE AT A STATE with
117+
// - the returned object from CCDB gets syncs with the config param registry
118+
// - everything is consistent
119+
120+
// fetch the default instance
121+
auto& p1 = o2::vertexing::PVertexerParams::Instance();
122+
123+
// update the config system with some runtime keys
124+
o2::conf::ConfigurableParam::updateFromString("pvertexer.dbscanDeltaT=-3.");
125+
126+
std::map<std::string, std::string> headers;
127+
std::map<std::string, std::string> meta;
128+
long from = o2::ccdb::getCurrentTimestamp();
129+
auto* object = f.api.retrieveFromTFileAny<o2::vertexing::PVertexerParams>("GLO/Config/PVertexer", meta, from + 1, &headers);
130+
BOOST_CHECK(object != nullptr);
131+
BOOST_CHECK(object->getMemberProvenance("dbscanDeltaT") == o2::conf::ConfigurableParam::EParamProvenance::kRT);
132+
BOOST_CHECK(object->getMemberProvenance("useMeanVertexConstraint") == o2::conf::ConfigurableParam::EParamProvenance::kCCDB);
133+
BOOST_CHECK(p1.getMemberProvenance("dbscanDeltaT") == o2::conf::ConfigurableParam::EParamProvenance::kRT);
134+
BOOST_CHECK(p1.getMemberProvenance("useMeanVertexConstraint") == o2::conf::ConfigurableParam::EParamProvenance::kCCDB);
135+
BOOST_CHECK(object == &p1);
136+
}

‎Common/Utils/include/CommonUtils/ConfigurableParam.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,10 @@ class ConfigurableParam
262262
// be updated, absence of data for any of requested params will lead to fatal
263263
static void updateFromFile(std::string const&, std::string const& paramsList = "", bool unchangedOnly = false);
264264

265+
// interface for use from the CCDB API; allows to sync objects read from CCDB with the information
266+
// stored in the registry; modifies given object as well as registry
267+
virtual void syncCCDBandRegistry(void* obj) = 0;
268+
265269
protected:
266270
// constructor is doing nothing else but
267271
// registering the concrete parameters
@@ -299,6 +303,7 @@ class ConfigurableParam
299303
void setRegisterMode(bool b) { sRegisterMode = b; }
300304
bool isInitialized() const { return sIsFullyInitialized; }
301305

306+
// friend class o2::ccdb::CcdbApi;
302307
private:
303308
// static registry for implementations of this type
304309
static std::vector<ConfigurableParam*>* sRegisteredParamClasses; //!

‎Common/Utils/include/CommonUtils/ConfigurableParamHelper.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ class _ParamHelper
5555

5656
static void assignmentImpl(std::string const& mainkey, TClass* cl, void* to, void* from,
5757
std::map<std::string, ConfigurableParam::EParamProvenance>* provmap);
58+
static void syncCCDBandRegistry(std::string const& mainkey, TClass* cl, void* to, void* from,
59+
std::map<std::string, ConfigurableParam::EParamProvenance>* provmap);
5860

5961
static void outputMembersImpl(std::ostream& out, std::string const& mainkey, std::vector<ParamDataMember> const* members, bool showProv);
6062
static void printMembersImpl(std::string const& mainkey, std::vector<ParamDataMember> const* members, bool showProv);
@@ -165,6 +167,22 @@ class ConfigurableParamHelper : virtual public ConfigurableParam
165167

166168
// ----------------------------------------------------------------
167169

170+
void syncCCDBandRegistry(void* externalobj) final
171+
{
172+
// We may be getting an external copy from CCDB which is passed as externalobj.
173+
// The task of this function is to
174+
// a) update the internal registry with fields coming from CCDB
175+
// but only if keys have not been modified via RT == command line / ini file
176+
// b) update the external object with with fields having RT provenance
177+
//
178+
setRegisterMode(false);
179+
_ParamHelper::syncCCDBandRegistry(getName(), TClass::GetClass(typeid(P)), (void*)this, (void*)externalobj,
180+
sValueProvenanceMap);
181+
setRegisterMode(true);
182+
}
183+
184+
// ----------------------------------------------------------------
185+
168186
void serializeTo(TFile* file) const final
169187
{
170188
file->WriteObjectAny((void*)this, TClass::GetClass(typeid(P)), getName().c_str());

‎Common/Utils/src/ConfigurableParamHelper.cxx‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -387,6 +387,51 @@ void _ParamHelper::assignmentImpl(std::string const& mainkey, TClass* cl, void*
387387

388388
// ----------------------------------------------------------------------
389389

390+
void _ParamHelper::syncCCDBandRegistry(const std::string& mainkey, TClass* cl, void* to, void* from,
391+
std::map<std::string, ConfigurableParam::EParamProvenance>* provmap)
392+
{
393+
auto sync = [to, from, &mainkey, provmap](const TDataMember* dm, int index, int size) {
394+
const auto name = getName(dm, index, size);
395+
auto dt = dm->GetDataType();
396+
auto TS = getSizeOfUnderlyingType(*dm);
397+
char* pointerto = ((char*)to) + dm->GetOffset() + index * TS;
398+
char* pointerfrom = ((char*)from) + dm->GetOffset() + index * TS;
399+
400+
// check current provenance
401+
auto key = mainkey + "." + name;
402+
auto proviter = provmap->find(key);
403+
bool isRT = proviter != provmap->end() && proviter->second == ConfigurableParam::EParamProvenance::kRT;
404+
if (isRT) {
405+
return;
406+
}
407+
// lambda to update the provenance
408+
auto updateProv = [&proviter]() {
409+
proviter->second = ConfigurableParam::EParamProvenance::kCCDB;
410+
};
411+
412+
// test if a complicated case
413+
if (isString(*dm)) {
414+
std::string& target = *(std::string*)pointerto;
415+
std::string const& origin = *(std::string*)pointerfrom;
416+
// if (target.compare(origin) != 0) {
417+
updateProv();
418+
target = origin;
419+
// }
420+
return;
421+
}
422+
423+
//
424+
// if (!isMemblockDifferent(pointerto, pointerfrom, TS)) {
425+
updateProv();
426+
// actually copy
427+
std::memcpy(pointerto, pointerfrom, getSizeOfUnderlyingType(*dm));
428+
// }
429+
};
430+
loopOverMembers(cl, to, sync);
431+
}
432+
433+
// ----------------------------------------------------------------------
434+
390435
void _ParamHelper::printWarning(std::type_info const& tinfo)
391436
{
392437
LOG(WARNING) << "Registered parameter class with name " << tinfo.name()

0 commit comments

Comments
 (0)