Merge branch 'dev' into stable-sync

Author: martenole

.github/workflows/async-auto-label.yml

@@ -0,0 +1,17 @@
+---
+name: Apply requested async label
+
+'on':
+  issue_comment:
+    types:
+      - created
+      - edited
+
+permissions: {}
+
+jobs:
+  apply_async_labels:
+    name: Apply requested async label
+    uses: alisw/ali-bot/.github/workflows/async-auto-label.yml@master
+    permissions:
+      pull-requests: write   # to update labels

.github/workflows/async-list-label.yml

@@ -0,0 +1,19 @@
+---
+name: Collect and print async labels
+
+'on':
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+    branches:
+      - dev
+
+permissions: {}
+
+jobs:
+  list_async_labels:
+    name: Collect and print async labels
+    uses: alisw/ali-bot/.github/workflows/async-list-label.yml@master
+    permissions:
+      pull-requests: write   # to update labels

.github/workflows/pr-security-approval.yml

@@ -0,0 +1,18 @@
+---
+name: Security approval
+
+'on':
+  pull_request_review:
+    types:
+      - edited
+      - submitted
+
+permissions: {}
+
+jobs:
+  clean:
+    name: Security approval
+    uses: alisw/ali-bot/.github/workflows/pr-security-approval.yml@master
+    permissions:
+      pull-requests: read  # to get last commit for PR
+      statuses: write      # for set-github-status

.github/workflows/reports.yml

@@ -18,10 +18,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.7
+        python-version: '3.10'
     - uses: actions/cache@v2
       name: Configure pip caching
       with:
@@ -81,7 +81,7 @@ jobs:
       run: |
         set -x
         mkdir -p doc/data
-        # We create new files once per month, mostly so that 
+        # We create new files once per month, mostly so that
         # we can keep the query results small. It does not
         # matter if we get results from different months,
         # as what matters is how we merge them.
@@ -96,7 +96,6 @@ jobs:
         #        being published
         LAST_RELEASE="${{ github.event.inputs.LAST_RELEASE_DATE }}"
         MERGED_AFTER=${LAST_RELEASE:-$(date -v -14d +%Y-%m-%d)}
-        
         # Here we convert all the json files to per subsystem
         # logs, using the MERGED_AFTER date to further filter them.
         # Notice we can have duplicates in each file,
@@ -106,7 +105,7 @@ jobs:
         for f in doc/data/*_prs.json; do
           for x in Algorithm Analysis Common DataFormats Detectors EventVisualisation Examples Framework Generators Steer Testing Utilities; do
             cat $f | jq ".repository.pullRequests.edges[].node | select(.files.edges[].node.path | test(\"$x\")) | del(.files) | select(.state == \"MERGED\" and .mergedAt >= \"${MERGED_AFTER}\")" > /tmp/${x}_prs.json
-            if [ ! X`jq -s length /tmp/${x}_prs.json` = X0 ]; then 
+            if [ ! X`jq -s length /tmp/${x}_prs.json` = X0 ]; then
               cat /tmp/${x}_prs.json | jq -r '"- [#\(.number)](https://github.com/AliceO2Group/AliceO2/pull/\(.number)) \(.mergedAt | split("T")[0]): \(.title) by [@\(.author.login)](https://github.com/\(.author.login))"' | sort -u >> /tmp/${x}_prs.md
             fi
           done

CCDB/CMakeLists.txt

@@ -45,6 +45,11 @@ o2_add_executable(upload
             SOURCES src/UploadTool.cxx
             PUBLIC_LINK_LIBRARIES O2::CCDB)
 
+o2_add_executable(cleansemaphores
+            COMPONENT_NAME ccdb
+            SOURCES src/CleanCCDBSemaphores.cxx
+            PUBLIC_LINK_LIBRARIES O2::CCDB)
+
 o2_add_executable(downloadccdbfile
             COMPONENT_NAME ccdb
             SOURCES src/DownloadCCDBFile.cxx

CCDB/include/CCDB/BasicCCDBManager.h

@@ -18,6 +18,8 @@
 #include "CCDB/CcdbApi.h"
 #include "CCDB/CCDBTimeStampUtils.h"
 #include "CommonUtils/NameConf.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/DefaultsHelpers.h"
 #include <string>
 #include <chrono>
 #include <map>
@@ -48,10 +50,19 @@ class CCDBManagerInstance
     std::string uuid;
     long startvalidity = 0;
     long endvalidity = -1;
+    long cacheValidFrom = 0;   // time for which the object was cached
+    long cacheValidUntil = -1; // object is guaranteed to be valid till this time (modulo new updates)
+    size_t minSize = -1ULL;
+    size_t maxSize = 0;
     int queries = 0;
     int fetches = 0;
     int failures = 0;
-    bool isValid(long ts) { return ts < endvalidity && ts > startvalidity; }
+    bool isValid(long ts) { return ts < endvalidity && ts >= startvalidity; }
+    bool isCacheValid(long ts)
+    {
+      LOGP(debug, "isCacheValid : {} : {} : {} --> {}", cacheValidFrom, ts, cacheValidUntil, ts < cacheValidUntil && ts >= cacheValidFrom);
+      return ts < cacheValidUntil && ts >= cacheValidFrom;
+    }
     void clear()
     {
       noCleanupPtr = nullptr;
@@ -68,6 +79,7 @@ class CCDBManagerInstance
   CCDBManagerInstance(std::string const& path) : mCCDBAccessor{}
   {
     mCCDBAccessor.init(path);
+    mDeplMode = o2::framework::DefaultsHelpers::deploymentMode();
   }
   /// set a URL to query from
   void setURL(const std::string& url);
@@ -99,6 +111,9 @@ class CCDBManagerInstance
     return getForTimeStamp<T>(path, timestamp);
   }
 
+  /// detect online processing modes (i.e. CCDB objects may be updated in the lifetime of the manager)
+  bool isOnline() const { return mDeplMode == o2::framework::DeploymentMode::OnlineAUX || mDeplMode == o2::framework::DeploymentMode::OnlineDDS || mDeplMode == o2::framework::DeploymentMode::OnlineECS; }
+
   /// retrieve an object of type T from CCDB as stored under path; will use the timestamp member
   template <typename T>
   T* get(std::string const& path)
@@ -132,7 +147,7 @@ class CCDBManagerInstance
     if (!isCachingEnabled()) {
       return false;
     }
-    return mCache[path].isValid(timestamp);
+    return (mCheckObjValidityEnabled && mCache[path].isValid(timestamp)) || mCache[path].isCacheValid(timestamp); // use stricter check
   }
 
   /// check if checks of object validity before CCDB query is enabled
@@ -165,17 +180,19 @@ class CCDBManagerInstance
   void setFatalWhenNull(bool b) { mFatalWhenNull = b; }
 
   /// A convenience function for MC to fetch
-  /// valid start and end timestamps given an ALICE run number.
-  /// On error it fatals (if fatal == true) or else returns the pair -1, -1.
-  std::pair<int64_t, int64_t> getRunDuration(int runnumber, bool fatal = true) const;
-
-  /// A convenience function for MC to fetch
-  /// valid start and end timestamps given an ALICE run number.
+  /// valid start and end timestamps for recorded TF data given an ALICE run number.
+  /// In absence of STF/ETF fields in the RCT with fall back to CTP SOX/EOX then to
+  /// ECS SOR/EOR.
   /// On error it fatals (if fatal == true) or else returns the pair -1, -1.
+  std::pair<int64_t, int64_t> getRunDuration(int runnumber, bool fatal = true);
   static std::pair<int64_t, int64_t> getRunDuration(o2::ccdb::CcdbApi const& api, int runnumber, bool fatal = true);
 
   std::string getSummaryString() const;
 
+  size_t getFetchedSize() const { return mFetchedSize; }
+
+  void report(bool longrep = false);
+
   void endOfStream();
 
  private:
@@ -190,14 +207,15 @@ class CCDBManagerInstance
   bool mCanDefault = false;                             // whether default is ok --> useful for testing purposes done standalone/isolation
   bool mCachingEnabled = true;                          // whether caching is enabled
   bool mCheckObjValidityEnabled = false;                // wether the validity of cached object is checked before proceeding to a CCDB API query
+  bool mFatalWhenNull = true;                           // if nullptr blob replies should be treated as fatal (can be set by user)
   long mCreatedNotAfter = 0;                            // upper limit for object creation timestamp (TimeMachine mode) - If-Not-After HTTP header
   long mCreatedNotBefore = 0;                           // lower limit for object creation timestamp (TimeMachine mode) - If-Not-Before HTTP header
   long mTimerMS = 0;                                    // timer for queries
-  bool mFatalWhenNull = true;                           // if nullptr blob replies should be treated as fatal (can be set by user)
+  size_t mFetchedSize = 0;                              // total fetched size
   int mQueries = 0;                                     // total number of object queries
   int mFetches = 0;                                     // total number of succesful fetches from CCDB
   int mFailures = 0;                                    // total number of failed fetches
-
+  o2::framework::DeploymentMode mDeplMode;              // O2 deployment mode
   ClassDefNV(CCDBManagerInstance, 1);
 };
 
@@ -218,11 +236,16 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
       mFailures++;
     } else {
       mFetches++;
+      auto sh = mHeaders.find("fileSize");
+      if (sh != mHeaders.end()) {
+        size_t s = atol(sh->second.c_str());
+        mFetchedSize += s;
+      }
     }
   } else {
     auto& cached = mCache[path];
-    if (mCheckObjValidityEnabled && cached.isValid(timestamp)) {
-      cached.queries++;
+    cached.queries++;
+    if ((!isOnline() && cached.isCacheValid(timestamp)) || (mCheckObjValidityEnabled && cached.isValid(timestamp))) {
       return reinterpret_cast<T*>(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get());
     }
     ptr = mCCDBAccessor.retrieveFromTFileAny<T>(path, mMetaData, timestamp, &mHeaders, cached.uuid,
@@ -238,17 +261,41 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
         cached.objPtr.reset(ptr);
       }
       cached.uuid = mHeaders["ETag"];
-      try { // this conversion can throw, better to catch immediately
-        cached.startvalidity = std::stol(mHeaders["Valid-From"]);
-        cached.endvalidity = std::stol(mHeaders["Valid-Until"]);
+
+      try {
+        if (mHeaders.find("Valid-From") != mHeaders.end()) {
+          cached.startvalidity = std::stol(mHeaders["Valid-From"]);
+        } else {
+          // if meta-information missing assume infinit validity
+          // (should happen only for locally created objects)
+          cached.startvalidity = 0;
+        }
+        if (mHeaders.find("Valid-Until") != mHeaders.end()) {
+          cached.endvalidity = std::stol(mHeaders["Valid-Until"]);
+        } else {
+          cached.endvalidity = std::numeric_limits<long>::max();
+        }
+        cached.cacheValidFrom = timestamp;
       } catch (std::exception const& e) {
         reportFatal("Failed to read validity from CCDB response (Valid-From :  " + mHeaders["Valid-From"] + std::string(" Valid-Until: ") + mHeaders["Valid-Until"] + std::string(")"));
       }
+      auto sh = mHeaders.find("fileSize");
+      if (sh != mHeaders.end()) {
+        size_t s = atol(sh->second.c_str());
+        mFetchedSize += s;
+        cached.minSize = std::min(s, cached.minSize);
+        cached.maxSize = std::max(s, cached.minSize);
+      }
     } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set
       cached.failures++;
       cached.clear(); // in case of any error clear cache for this object
-    } else {          // the old object is valid
-      ptr = reinterpret_cast<T*>(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get());
+    }
+    // the old object is valid, fetch cache end of validity
+    ptr = reinterpret_cast<T*>(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get());
+    if (mHeaders.find("Cache-Valid-Until") != mHeaders.end()) {
+      cached.cacheValidUntil = std::stol(mHeaders["Cache-Valid-Until"]);
+    } else {
+      cached.cacheValidUntil = -1;
     }
     mHeaders.clear();
     mMetaData.clear();

CCDB/include/CCDB/CCDBDownloader.h

@@ -52,6 +52,7 @@ typedef struct DownloaderRequestData {
   HeaderObjectPair_t hoPair;
   std::map<std::string, std::string>* headers;
   std::string userAgent;
+  curl_slist* optionsList;
 
   std::function<bool(std::string)> localContentCallback;
 } DownloaderRequestData;
@@ -296,6 +297,7 @@ class CCDBDownloader
     int hostInd;
     int locInd;
     DownloaderRequestData* requestData;
+    curl_slist** options;
   } PerformData;
 #endif
 
@@ -421,6 +423,6 @@ typedef struct DataForClosingSocket {
   curl_socket_t socket;
 } DataForClosingSocket;
 
-} // namespace o2
+} // namespace o2::ccdb
 
 #endif // O2_CCDB_CCDBDOWNLOADER_H

CCDB/include/CCDB/CcdbApi.h

@@ -375,8 +375,14 @@ class CcdbApi //: public DatabaseInterface
   void getFromSnapshot(bool createSnapshot, std::string const& path,
                        long timestamp, std::map<std::string, std::string>& headers,
                        std::string& snapshotpath, o2::pmr::vector<char>& dest, int& fromSnapshot, std::string const& etag) const;
-  void releaseNamedSemaphore(boost::interprocess::named_semaphore* sem, std::string path) const;
-  boost::interprocess::named_semaphore* createNamedSempahore(std::string path) const;
+  void releaseNamedSemaphore(boost::interprocess::named_semaphore* sem, std::string const& path) const;
+  boost::interprocess::named_semaphore* createNamedSemaphore(std::string const& path) const;
+  static std::string determineSemaphoreName(std::string const& basedir, std::string const& objectpath);
+  // queries and optionally removes a named semaphore from the system
+  // returns true when successful (either found or found + removed)
+  static bool removeSemaphore(std::string const& name, bool remove = false);
+  static void removeLeakingSemaphores(std::string const& basedir, bool remove = false);
+
   void loadFileToMemory(o2::pmr::vector<char>& dest, const std::string& path, std::map<std::string, std::string>* localHeaders = nullptr) const;
   void loadFileToMemory(o2::pmr::vector<char>& dest, std::string const& path,
                         std::map<std::string, std::string> const& metadata, long timestamp,
@@ -542,7 +548,7 @@ class CcdbApi //: public DatabaseInterface
    * @param tcl The TClass object describing the serialized type
    * @return raw pointer to created object
    */
-  void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo) const;
+  void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo, std::map<string, string>* headers) const;
 
   // initialize the TGrid (Alien connection)
   bool initTGrid() const;