Provenance tracking of parameters

Introducing provenance tracking of parameters which
allows to query who gave the decisive modification/setting
of the value. (If CCDB does not change default value it will stay
default).

For the moment 3 categories: default code, CCDB,
runtime (such as command line)

Example output:

```
mCUTGAM : 100		[ CCDB ]
mCUTELE : 1		[ CODE ]
mCUTNEU : -100		[ RT ]
mCUTHAD : 0.001		[ CODE ]
```

This will allow to see which parameters have changed by whom and
the mechanism can be easily extented.

Author: sawenzel

Common/SimConfig/include/SimConfig/ConfigurableParam.h

@@ -29,17 +29,21 @@ namespace conf
 //
 // A configurable parameter is a simple class, defining
 // a few (pod) properties/members which are registered
-// in a global (boost) property tree.
+// in a global (boost) property tree / structure.
 //
 // The features that we provide here are:
-// a) Automatic translation from C++ data description to INI/JSON/XML
+// *) Automatic translation from C++ data description to INI/JSON/XML
 //    format via ROOT introspection and boost property trees and
 //    the possibility to readably save the configuration
-// b) Automatic integration of sub-classes into a common configuration
-// c) Be able to query properties from high level interfaces (just knowing
-// d) Be able to set properties from high-level interfaces (and modifying the underlying
+// *) Serialization/Deserialization into ROOT binary blobs (for the purpose
+//    of writing/retrieving parameters from CCDB)
+// *) Automatic integration of sub-classes into a common configuration
+// *) Be able to query properties from high level interfaces (just knowing
+// *) Be able to set properties from high-level interfaces (and modifying the underlying
 //    C++ object)
-// e) Automatic ability to modify parameters from the command-line
+// *) Automatic ability to modify parameters from the command-line
+// *) Keeping track of the provenance of individual parameter values; The user is able
+//    to see whether is parameter is defaulted-code-value/coming-from-CCDB/coming-from-comandline
 //
 // Note that concrete parameter sub-classes **must** be implemented
 // by inheriting from ConfigurableParamHelper and not from this class.
@@ -79,11 +83,23 @@ namespace conf
 class ConfigurableParam
 {
  public:
+  enum EParamProvenance {
+    kCODE /* from default code initialization */,
+    kCCDB /* overwritten from CCDB */,
+    kRT /* changed during runtime via API call setValue (for example command line) */
+    /* can add more modes here */
+  };
+  static std::string toString(EParamProvenance p)
+  {
+    static std::array<std::string, 3> names = { "CODE", "CCDB", "RT" };
+    return names[(int)p];
+  }
+
   //
   virtual std::string getName() const = 0; // print the name of the configurable Parameter
 
-  // print the current keys and values to screen
-  virtual void printKeyValues() = 0;
+  // print the current keys and values to screen (optionally with provenance information)
+  virtual void printKeyValues(bool showprov = true) const = 0;
 
   static void printAllRegisteredParamNames();
   static void printAllKeyValuePairs();
@@ -107,7 +123,10 @@ class ConfigurableParam
     auto key = mainkey + "." + subkey;
     if (sPtree->get_optional<std::string>(key).is_initialized()) {
       sPtree->put(key, x);
-      updateThroughStorageMap(mainkey, subkey, typeid(T), (void*)&x);
+      auto changed = updateThroughStorageMap(mainkey, subkey, typeid(T), (void*)&x);
+      if (changed) {
+        sValueProvenanceMap->find(key)->second = kRT; // set to runtime
+      }
     }
   }
 
@@ -117,7 +136,10 @@ class ConfigurableParam
   {
     if (sPtree->get_optional<std::string>(key).is_initialized()) {
       sPtree->put(key, valuestring);
-      updateThroughStorageMapWithConversion(key, valuestring);
+      auto changed = updateThroughStorageMapWithConversion(key, valuestring);
+      if (changed) {
+        sValueProvenanceMap->find(key)->second = kRT; // set to runtime
+      }
     }
   }
 
@@ -144,9 +166,9 @@ class ConfigurableParam
   // registering the concrete parameters
   ConfigurableParam();
 
-  static void updatePropertyTree();
-  static void updateThroughStorageMap(std::string, std::string, std::type_info const&, void*);
-  static void updateThroughStorageMapWithConversion(std::string, std::string);
+  static void initPropertyTree();
+  static bool updateThroughStorageMap(std::string, std::string, std::type_info const&, void*);
+  static bool updateThroughStorageMapWithConversion(std::string, std::string);
 
   virtual ~ConfigurableParam() = default;
 
@@ -157,6 +179,9 @@ class ConfigurableParam
   // (internal use to easily sync updates, this is ok since parameter classes are singletons)
   static std::map<std::string, std::pair<int, void*>>* sKeyToStorageMap;
 
+  // keep track of provenance of parameters and values
+  static std::map<std::string, ConfigurableParam::EParamProvenance>* sValueProvenanceMap;
+
   void setRegisterMode(bool b) { sRegisterMode = b; }
 
  private:
@@ -174,7 +199,7 @@ class ConfigurableParam
 // a helper macro for boilerplate code in parameter classes
 #define O2ParamDef(classname, key)               \
  public:                                         \
-  classname(TRootIOCtor *) {}                    \
+  classname(TRootIOCtor*) {}                     \
  private:                                        \
   static constexpr char const* const sKey = key; \
   static classname sInstance;                    \

Common/SimConfig/include/SimConfig/ConfigurableParamHelper.h

@@ -24,18 +24,22 @@ namespace o2
 namespace conf
 {
 
-// just a (non-templated) Helper with exclusively private functions
+// just a (non-templated) helper with exclusively private functions
 // used by ConfigurableParamHelper
 class _ParamHelper
 {
  private:
-  static void printParametersImpl(TClass* cl, void*);
+  static void printParametersImpl(std::string mainkey, TClass* cl, void*,
+                                  std::map<std::string, ConfigurableParam::EParamProvenance> const* provmap);
 
   static void fillKeyValuesImpl(std::string mainkey, TClass* cl, void*, boost::property_tree::ptree*,
                                 std::map<std::string, std::pair<int, void*>>*);
 
   static void printWarning(std::type_info const&);
 
+  static void assignmentImpl(std::string mainkey, TClass* cl, void* to, void* from,
+                             std::map<std::string, ConfigurableParam::EParamProvenance>* provmap);
+
   template <typename P>
   friend class ConfigurableParamHelper;
 };
@@ -57,7 +61,7 @@ class ConfigurableParamHelper : virtual public ConfigurableParam
   }
 
   // one of the key methods, using introspection to print itself
-  void printKeyValues() final
+  void printKeyValues(bool showprov) const final
   {
     // just a helper line to make sure P::sInstance is looked-up
     // and that compiler complains about missing static sInstance of type P
@@ -71,9 +75,10 @@ class ConfigurableParamHelper : virtual public ConfigurableParam
       _ParamHelper::printWarning(typeid(P));
       return;
     }
-    _ParamHelper::printParametersImpl(cl, (void*)this);
+    _ParamHelper::printParametersImpl(getName(), cl, (void*)this, showprov ? sValueProvenanceMap : nullptr);
   }
 
+  // fills the data structures with the initial default values
   void putKeyValues(boost::property_tree::ptree* tree) final
   {
     auto cl = TClass::GetClass(typeid(P));
@@ -92,9 +97,8 @@ class ConfigurableParamHelper : virtual public ConfigurableParam
     P* readback = nullptr;
     file->GetObject(getName().c_str(), readback);
     if (readback != nullptr) {
-      // ATTENTION: override existing singleton object
-      // It would be better to have a way of reading INTO an existing object
-      std::memcpy((void*)this, readback, sizeof(P));
+      _ParamHelper::assignmentImpl(getName(), TClass::GetClass(typeid(P)), (void*)this, (void*)readback,
+                                   sValueProvenanceMap);
       delete readback;
     }
     setRegisterMode(true);