Merge branch 'dev' into stable-sync

Author: martenole

.github/workflows/clean-test.yml

@@ -39,10 +39,6 @@ name: Clean PR checks
         description: build/O2/o2-cs8
         type: boolean
         default: true
-      'check_build/O2/o2-dataflow':
-        description: build/O2/o2-dataflow
-        type: boolean
-        default: true
       'check_build/O2/o2-dataflow-cs8':
         description: build/O2/o2-dataflow-cs8
         type: boolean

Algorithm/include/Algorithm/PageParser.h

@@ -255,12 +255,12 @@ class PageParser
       return mElement;
     }
     // comparison
-    bool operator==(const SelfType& rh)
+    bool operator==(const SelfType& rh) const
     {
       return mPosition == rh.mPosition;
     }
     // comparison
-    bool operator!=(const SelfType& rh)
+    bool operator!=(const SelfType& rh) const
     {
       return mPosition != rh.mPosition;
     }

CCDB/include/CCDB/CCDBDownloader.h

@@ -51,9 +51,9 @@ typedef struct DownloaderRequestData {
   long timestamp;
   HeaderObjectPair_t hoPair;
   std::map<std::string, std::string>* headers;
+  std::string userAgent;
 
   std::function<bool(std::string)> localContentCallback;
-  bool errorflag = false;
 } DownloaderRequestData;
 #endif
 
@@ -208,12 +208,17 @@ class CCDBDownloader
    */
   void runLoop(bool noWait);
 
- private:
+  /**
+   * Returns a message describing the transfer an it's result.
+   */
+  std::string prepareLogMessage(std::string host_url, std::string userAgent, const std::string& path, long ts, const std::map<std::string, std::string>* headers, long httpCode) const;
+
   /**
    * Leaves only the protocol and host part of the url, discrading path and metadata.
    */
   std::string trimHostUrl(std::string full_host_url) const;
 
+ private:
   /**
    * Recognizes whether the address is a full url, or a partial one (like for example "/Task/Detector/1") and combines it with potentialHost if needed.
    */

CCDB/include/CCDB/CcdbApi.h

@@ -64,6 +64,10 @@ class CcdbApi //: public DatabaseInterface
   /// \brief Default destructor
   virtual ~CcdbApi();
 
+  // Delete copy and copy assignment constructor
+  CcdbApi(const CcdbApi&) = delete;
+  void operator=(const CcdbApi&) = delete;
+
   const std::string getUniqueAgentID() const { return mUniqueAgentID; }
 
   static bool checkAlienToken();
@@ -369,7 +373,7 @@ class CcdbApi //: public DatabaseInterface
   void scheduleDownload(RequestContext& requestContext, size_t* requestCounter) const;
 
   void getFromSnapshot(bool createSnapshot, std::string const& path,
-                       long timestamp, std::map<std::string, std::string> headers,
+                       long timestamp, std::map<std::string, std::string>& headers,
                        std::string& snapshotpath, o2::pmr::vector<char>& dest, int& fromSnapshot, std::string const& etag) const;
   void releaseNamedSemaphore(boost::interprocess::named_semaphore* sem, std::string path) const;
   boost::interprocess::named_semaphore* createNamedSempahore(std::string path) const;

CCDB/src/CCDBDownloader.cxx

@@ -403,30 +403,37 @@ std::string CCDBDownloader::trimHostUrl(std::string full_host_url) const
 {
   CURLU* host_url = curl_url();
   curl_url_set(host_url, CURLUPART_URL, full_host_url.c_str(), 0);
-  // Get host part
+
+  // Get host part (the only critical part)
   char* host;
   CURLUcode host_result = curl_url_get(host_url, CURLUPART_HOST, &host, 0);
-  std::string host_name;
-  if (host_result == CURLUE_OK) {
-    // Host part present
-    host_name = host;
-    curl_free(host);
-  } else {
-    LOG(error) << "CCDBDownloader: Malformed url detected when processing redirect, could not identify the host part: " << host;
+  if (host_result != CURLUE_OK) {
+    LOG(error) << "CCDBDownloader: Malformed url detected when processing redirect, could not identify the host part: " << full_host_url;
     curl_url_cleanup(host_url);
     return "";
   }
   // Get scheme (protocol) part
   char* scheme;
   CURLUcode scheme_result = curl_url_get(host_url, CURLUPART_SCHEME, &scheme, 0);
+  // Get port
+  char* port;
+  CURLUcode port_result = curl_url_get(host_url, CURLUPART_PORT, &port, 0);
+
   curl_url_cleanup(host_url);
+
+  // Assemble parts
+  std::string trimmed_url = "";
   if (scheme_result == CURLUE_OK) {
-    // If protocol present combine with host
-    curl_free(scheme);
-    return scheme + std::string("://") + host_name;
-  } else {
-    return host_name;
+    trimmed_url += scheme + std::string("://");
+    free(scheme);
+  }
+  trimmed_url += host;
+  free(host);
+  if (port_result == CURLUE_OK) {
+    trimmed_url += std::string(":") + port;
+    free(port);
   }
+  return trimmed_url;
 }
 
 std::string CCDBDownloader::prepareRedirectedURL(std::string address, std::string potentialHost) const
@@ -469,38 +476,36 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
     } break;
     case ASYNCHRONOUS: {
       DownloaderRequestData* requestData = performData->requestData;
-
       if (requestData->headers) {
         for (auto& p : requestData->hoPair.header) {
           (*requestData->headers)[p.first] = p.second;
         }
       }
-      if (requestData->errorflag && requestData->headers) {
-        (*requestData->headers)["Error"] = "An error occurred during retrieval";
-      }
-
       // Log that transfer finished
       long httpCode;
       curl_easy_getinfo(easy_handle, CURLINFO_RESPONSE_CODE, &httpCode);
       char* url;
       curl_easy_getinfo(easy_handle, CURLINFO_EFFECTIVE_URL, &url);
       LOG(debug) << "Transfer for " << url << " finished with code " << httpCode << "\n";
+      std::string currentHost = requestData->hosts[performData->hostInd];
+      std::string loggingMessage = prepareLogMessage(currentHost, requestData->userAgent, requestData->path, requestData->timestamp, requestData->headers, httpCode);
 
       // Get alternative locations for the same host
       auto locations = getLocations(&(requestData->hoPair.header));
 
       // React to received http code
-      if (404 == httpCode) {
-        LOG(error) << "Requested resource does not exist: " << url;
-      } else if (304 == httpCode) {
-        LOGP(debug, "Object exists but I am not serving it since it's already in your possession");
-        contentRetrieved = true;
-      } else if (300 <= httpCode && httpCode < 400 && performData->locInd < locations.size()) {
-        followRedirect(performData, easy_handle, locations, rescheduled, contentRetrieved);
-      } else if (200 <= httpCode && httpCode < 300) {
-        contentRetrieved = true;
+      if (200 <= httpCode && httpCode < 400) {
+        LOG(debug) << loggingMessage;
+        if (304 == httpCode) {
+          LOGP(debug, "Object exists but I am not serving it since it's already in your possession");
+          contentRetrieved = true;
+        } else if (300 <= httpCode && httpCode < 400 && performData->locInd < locations.size()) {
+          followRedirect(performData, easy_handle, locations, rescheduled, contentRetrieved);
+        } else if (200 <= httpCode && httpCode < 300) {
+          contentRetrieved = true;
+        }
       } else {
-        LOG(error) << "Error in fetching object " << url << ", curl response code:" << httpCode;
+        LOG(error) << loggingMessage;
       }
 
       // Check if content was retrieved, or scheduled to be retrieved
@@ -516,12 +521,18 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
 
       if (!rescheduled) {
         // No more transfers will be done for this request, do cleanup specific for ASYNCHRONOUS calls
-        --(*performData->requestsLeft);
-        delete requestData;
-        delete performData->codeDestination;
         if (!contentRetrieved) {
+          if (requestData->hoPair.object) {
+            requestData->hoPair.object->clear();
+          }
+          if (requestData->headers) {
+            (*requestData->headers)["Error"] = "An error occurred during retrieval";
+          }
           LOGP(alarm, "Curl request to {}, response code: {}", url, httpCode);
         }
+        --(*performData->requestsLeft);
+        delete requestData;
+        delete performData->codeDestination;
       }
     } break;
   }
@@ -687,4 +698,21 @@ void CCDBDownloader::asynchSchedule(CURL* handle, size_t* requestCounter)
   // return codeVector;
 }
 
+std::string CCDBDownloader::prepareLogMessage(std::string host_url, std::string userAgent, const std::string& path, long ts, const std::map<std::string, std::string>* headers, long httpCode) const
+{
+  std::string upath{path};
+  if (headers) {
+    auto ent = headers->find("Valid-From");
+    if (ent != headers->end()) {
+      upath += "/" + ent->second;
+    }
+    ent = headers->find("ETag");
+    if (ent != headers->end()) {
+      upath += "/" + ent->second;
+    }
+  }
+  upath.erase(remove(upath.begin(), upath.end(), '\"'), upath.end());
+  return fmt::format("CcdbDownloader finished transfer {}{}{} for {} (agent_id: {}) with http code: {}", host_url, (host_url.back() == '/') ? "" : "/", upath, (ts < 0) ? getCurrentTimestamp() : ts, userAgent, httpCode);
+}
+
 } // namespace o2

CCDB/src/CcdbApi.cxx

@@ -147,6 +147,10 @@ void CcdbApi::init(std::string const& host)
   // In addition, we can monitor exactly which objects are fetched and what is their content.
   // One can also distribute so obtained caches to sites without network access.
   //
+  // THE INFORMATION BELOW IS TEMPORARILY WRONG: the functionality of checking the validity if IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE
+  // is NOT set is broken. At the moment the code is modified to behave as if the IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE is always set
+  // whenever the ALICEO2_CCDB_LOCALCACHE is defined.
+  //
   // When used with the DPL CCDB fetcher (i.e. loadFileToMemory is called), in order to prefer the available snapshot w/o its validity
   // check an extra variable IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE must be defined, otherwhise the object will be fetched from the
   // server after the validity check and new snapshot will be created if needed
@@ -161,7 +165,7 @@ void CcdbApi::init(std::string const& host)
     }
     snapshotReport = fmt::format("(cache snapshots to dir={}", mSnapshotCachePath);
   }
-  if (getenv("IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE")) {
+  if (cachedir) { // || getenv("IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE")) {
     mPreferSnapshotCache = true;
     if (mSnapshotCachePath.empty()) {
       LOGP(fatal, "IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE is defined but the ALICEO2_CCDB_LOCALCACHE is not");
@@ -252,6 +256,10 @@ int CcdbApi::storeAsTFile_impl(const void* obj, std::type_info const& tinfo, std
                                std::vector<char>::size_type maxSize) const
 {
   // We need the TClass for this type; will verify if dictionary exists
+  if (!obj) {
+    LOGP(error, "nullptr is provided for object {}/{}/{}", path, startValidityTimestamp, endValidityTimestamp);
+    return -1;
+  }
   CcdbObjectInfo info;
   auto img = createObjectImage(obj, tinfo, &info);
   return storeAsBinaryFile(img->data(), img->size(), info.getFileName(), info.getObjectType(),
@@ -372,6 +380,10 @@ int CcdbApi::storeAsTFile(const TObject* rootObject, std::string const& path, st
                           long startValidityTimestamp, long endValidityTimestamp, std::vector<char>::size_type maxSize) const
 {
   // Prepare file
+  if (!rootObject) {
+    LOGP(error, "nullptr is provided for object {}/{}/{}", path, startValidityTimestamp, endValidityTimestamp);
+    return -1;
+  }
   CcdbObjectInfo info;
   auto img = createObjectImage(rootObject, &info);
   return storeAsBinaryFile(img->data(), img->size(), info.getFileName(), info.getObjectType(), path, metadata, startValidityTimestamp, endValidityTimestamp, maxSize);
@@ -1486,12 +1498,6 @@ void CcdbApi::scheduleDownload(RequestContext& requestContext, size_t* requestCo
   auto data = new DownloaderRequestData(); // Deleted in transferFinished of CCDBDownloader.cxx
   data->hoPair.object = &requestContext.dest;
 
-  auto signalError = [&chunk = requestContext.dest, &errorflag = data->errorflag]() {
-    chunk.clear();
-    chunk.reserve(1);
-    errorflag = true;
-  };
-
   std::function<bool(std::string)> localContentCallback = [this, &requestContext](std::string url) {
     return this->loadLocalContentToMemory(requestContext.dest, url);
   };
@@ -1533,6 +1539,7 @@ void CcdbApi::scheduleDownload(RequestContext& requestContext, size_t* requestCo
   data->path = requestContext.path;
   data->timestamp = requestContext.timestamp;
   data->localContentCallback = localContentCallback;
+  data->userAgent = mUniqueAgentID;
 
   curl_easy_setopt(curl_handle, CURLOPT_URL, fullUrl.c_str());
   initCurlOptionsForRetrieve(curl_handle, (void*)(&data->hoPair), writeCallback, false);
@@ -1570,7 +1577,7 @@ void CcdbApi::releaseNamedSemaphore(boost::interprocess::named_semaphore* sem, s
 }
 
 void CcdbApi::getFromSnapshot(bool createSnapshot, std::string const& path,
-                              long timestamp, std::map<std::string, std::string> headers,
+                              long timestamp, std::map<std::string, std::string>& headers,
                               std::string& snapshotpath, o2::pmr::vector<char>& dest, int& fromSnapshot, std::string const& etag) const
 {
   if (createSnapshot) { // create named semaphore
@@ -1580,9 +1587,10 @@ void CcdbApi::getFromSnapshot(bool createSnapshot, std::string const& path,
       logStream << "CCDB-access[" << getpid() << "] of " << mUniqueAgentID << " to " << path << " timestamp " << timestamp << " for load to memory\n";
     }
   }
-
   if (mInSnapshotMode) { // file must be there, otherwise a fatal will be produced;
-    loadFileToMemory(dest, getSnapshotFile(mSnapshotTopPath, path), &headers);
+    if (etag.empty()) {
+      loadFileToMemory(dest, getSnapshotFile(mSnapshotTopPath, path), &headers);
+    }
     fromSnapshot = 1;
   } else if (mPreferSnapshotCache && std::filesystem::exists(snapshotpath)) {
     // if file is available, use it, otherwise cache it below from the server. Do this only when etag is empty since otherwise the object was already fetched and cached
@@ -1671,8 +1679,6 @@ void CcdbApi::vectoredLoadFileToMemory(std::vector<RequestContext>& requestConte
     // navigateSourcesAndLoadFile either retrieves file from snapshot immediately, or schedules it to be downloaded when mDownloader->runLoop is ran at a later time
     auto& requestContext = requestContexts.at(i);
     navigateSourcesAndLoadFile(requestContext, fromSnapshots.at(i), &requestCounter);
-    logReading(requestContext.path, requestContext.timestamp, &requestContext.headers,
-               fmt::format("{}{}", requestContext.considerSnapshot ? "load to memory" : "retrieve", fromSnapshots.at(i) ? " from snapshot" : ""));
   }
 
   // Download the rest
@@ -1683,12 +1689,12 @@ void CcdbApi::vectoredLoadFileToMemory(std::vector<RequestContext>& requestConte
   // Save snapshots
   for (int i = 0; i < requestContexts.size(); i++) {
     auto& requestContext = requestContexts.at(i);
+    logReading(requestContext.path, requestContext.timestamp, &requestContext.headers,
+               fmt::format("{}{}", requestContext.considerSnapshot ? "load to memory" : "retrieve", fromSnapshots.at(i) ? " from snapshot" : ""));
     if (!requestContext.dest.empty()) {
       if (requestContext.considerSnapshot && fromSnapshots.at(i) != 2) {
         saveSnapshot(requestContext);
       }
-    } else {
-      LOG(warning) << "Did not receive content for " << requestContext.path << "\n"; // Temporarily demoted to warning, since it floods the infologger
     }
   }
 }

CCDB/test/testCcdbApiDownloader.cxx

@@ -381,5 +381,15 @@ BOOST_AUTO_TEST_CASE(external_loop_test)
   delete uvLoop;
 }
 
+BOOST_AUTO_TEST_CASE(trim_host_url_test)
+{
+  CCDBDownloader downloader;
+  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080") == "http://localhost:8080");
+  BOOST_CHECK(downloader.trimHostUrl("http://localhost") == "http://localhost");
+  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080/some/path") == "http://localhost:8080");
+  BOOST_CHECK(downloader.trimHostUrl("http://localhost/some/path") == "http://localhost");
+  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080/Task/Detector/1?HTTPOnly=true") == "http://localhost:8080");
+}
+
 } // namespace ccdb
 } // namespace o2

Common/Constants/include/CommonConstants/MathConstants.h

@@ -23,7 +23,7 @@ namespace constants
 namespace math
 {
 constexpr float Almost0 = 1.17549e-38;
-constexpr float Almost1 = 1.f - Almost0;
+constexpr float Almost1 = 1.f - 1.0e-6;
 constexpr float VeryBig = 1.f / Almost0;
 
 constexpr float PI = 3.14159274101257324e+00f;