GPU: Add cache folder and file lock options to RTC

Author: davidrohr

GPU/GPUTracking/Base/cuda/GPUReconstructionCUDAGenRTC.cxx

@@ -23,6 +23,9 @@
 #ifdef GPUCA_HAVE_O2HEADERS
 #include "Framework/SHA1.h"
 #endif
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <filesystem>
 
 using namespace GPUCA_NAMESPACE::gpu;
 
@@ -62,11 +65,27 @@ int GPUReconstructionCUDA::genRTC(std::string& filename, unsigned int& nCompile)
 
   nCompile = mProcessingSettings.rtc.compilePerKernel ? kernels.size() : 1;
   bool cacheLoaded = false;
+  int fd = 0;
   if (mProcessingSettings.rtc.cacheOutput) {
+    if (mProcessingSettings.RTCcacheFolder != ".") {
+      std::filesystem::create_directories(mProcessingSettings.RTCcacheFolder);
+    }
 #ifndef GPUCA_HAVE_O2HEADERS
     throw std::runtime_error("Cannot use RTC cache without O2 headers");
 #else
-    FILE* fp = fopen("rtc.cuda.cache", "rb");
+    if (mProcessingSettings.rtc.cacheMutex) {
+      mode_t mask = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
+      fd = open((mProcessingSettings.RTCcacheFolder + "/cache.lock").c_str(), O_RDWR | O_CREAT | O_CLOEXEC, mask);
+      if (fd == -1) {
+        throw std::runtime_error("Error opening rtc cache mutex lock file");
+      }
+      fchmod(fd, mask);
+      if (lockf(fd, F_LOCK, 0)) {
+        throw std::runtime_error("Error locking rtc cache mutex file");
+      }
+    }
+
+    FILE* fp = fopen((mProcessingSettings.RTCcacheFolder + "/rtc.cuda.cache").c_str(), "rb");
     char sharead[20];
     if (fp) {
       size_t len;
@@ -100,6 +119,7 @@ int GPUReconstructionCUDA::genRTC(std::string& filename, unsigned int& nCompile)
           break;
         }
         GPUSettingsProcessingRTC cachedSettings;
+        static_assert(std::is_trivially_copyable_v<GPUSettingsProcessingRTC> == true, "GPUSettingsProcessingRTC must be POD");
         if (fread(&cachedSettings, sizeof(cachedSettings), 1, fp) != 1) {
           throw std::runtime_error("Cache file corrupt");
         }
@@ -183,7 +203,7 @@ int GPUReconstructionCUDA::genRTC(std::string& filename, unsigned int& nCompile)
     }
 #ifdef GPUCA_HAVE_O2HEADERS
     if (mProcessingSettings.rtc.cacheOutput) {
-      FILE* fp = fopen("rtc.cuda.cache", "w+b");
+      FILE* fp = fopen((mProcessingSettings.RTCcacheFolder + "/rtc.cuda.cache").c_str(), "w+b");
       if (fp == nullptr) {
         throw std::runtime_error("Cannot open cache file for writing");
       }
@@ -218,9 +238,16 @@ int GPUReconstructionCUDA::genRTC(std::string& filename, unsigned int& nCompile)
         }
       }
       fclose(fp);
+
     }
 #endif
   }
+  if (mProcessingSettings.rtc.cacheOutput && mProcessingSettings.rtc.cacheMutex) {
+    if (lockf(fd, F_ULOCK, 0)) {
+      throw std::runtime_error("Error unlocking RTC cache mutex file");
+    }
+    close(fd);
+  }
 
 #endif
   return 0;

GPU/GPUTracking/Definitions/GPUSettingsList.h

@@ -191,13 +191,15 @@ AddSubConfig(GPUSettingsRecTRD, trd)
 AddHelp("help", 'h')
 EndConfig()
 
+#ifndef __OPENCL__
 // Settings steering the processing once the device was selected, only available on the host
 BeginSubConfig(GPUSettingsProcessingRTC, rtc, configStandalone.proc, "RTC", 0, "Processing settings", proc_rtc)
 AddOption(cacheOutput, bool, false, "", 0, "Cache RTC compilation results")
 AddOption(optConstexpr, bool, true, "", 0, "Replace constant variables by static constexpr expressions")
 AddOption(compilePerKernel, bool, true, "", 0, "Run one RTC compilation per kernel")
 AddOption(enable, bool, false, "", 0, "Use RTC to optimize GPU code")
 AddOption(runTest, int, 0, "", 0, "Do not run the actual benchmark, but just test RTC compilation (1 full test, 2 test only compilation)")
+AddOption(cacheMutex, bool, true, "", 0, "Use a file lock to serialize access to the cache folder")
 AddHelp("help", 'h')
 EndConfig()
 
@@ -276,11 +278,13 @@ AddOption(tpcSingleSector, int, -1, "", 0, "Restrict TPC processing to a single
 AddOption(tpcDownscaledEdx, unsigned char, 0, "", 0, "If != 0, downscale dEdx processing (if enabled) to x %")
 AddOption(tpcMaxAttachedClustersPerSectorRow, unsigned int, 51000, "", 0, "Maximum number of TPC attached clusters which can be decoded per SectorRow")
 AddOption(tpcUseOldCPUDecoding, bool, false, "", 0, "Enable old CPU-based TPC decoding")
+AddOption(RTCcacheFolder, std::string, "./rtccache/", "", 0, "Folder in which the cache file is stored")
 AddVariable(eventDisplay, GPUCA_NAMESPACE::gpu::GPUDisplayFrontendInterface*, nullptr)
 AddSubConfig(GPUSettingsProcessingRTC, rtc)
 AddSubConfig(GPUSettingsProcessingParam, param)
 AddHelp("help", 'h')
 EndConfig()
+#endif // __OPENCL__
 
 #ifndef GPUCA_GPUCODE_DEVICE
 // Light settings concerning the event display (can be changed without rebuilding vertices)

GPU/GPUTracking/Standalone/Benchmark/standalone.cxx

@@ -297,8 +297,8 @@ int SetupReconstruction()
   GPUSettingsGRP grp = rec->GetGRPSettings();
   GPUSettingsRec recSet;
   GPUSettingsProcessing procSet;
-  memcpy((void*)&recSet, (void*)&configStandalone.rec, sizeof(GPUSettingsRec));
-  memcpy((void*)&procSet, (void*)&configStandalone.proc, sizeof(GPUSettingsProcessing));
+  recSet = configStandalone.rec;
+  procSet = configStandalone.proc;
   GPURecoStepConfiguration steps;
 
   if (configStandalone.eventGenerator) {

GPU/Workflow/src/GPUWorkflowSpec.cxx

@@ -321,11 +321,11 @@ void GPURecoWorkflowSpec::init(InitContext& ic)
       mode_t mask = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
       fd = open("/tmp/o2_gpu_memlock_mutex.lock", O_RDWR | O_CREAT | O_CLOEXEC, mask);
       if (fd == -1) {
-        throw std::runtime_error("Error opening lock file");
+        throw std::runtime_error("Error opening memlock mutex lock file");
       }
       fchmod(fd, mask);
       if (lockf(fd, F_LOCK, 0)) {
-        throw std::runtime_error("Error locking file");
+        throw std::runtime_error("Error locking memlock mutex file");
       }
     }
     std::chrono::time_point<std::chrono::high_resolution_clock> start, end;
@@ -342,7 +342,7 @@ void GPURecoWorkflowSpec::init(InitContext& ic)
     }
     if (mConfParam->mutexMemReg) {
       if (lockf(fd, F_ULOCK, 0)) {
-        throw std::runtime_error("Error unlocking file");
+        throw std::runtime_error("Error unlocking memlock mutex file");
       }
       close(fd);
     }