QC Workflow: Add option to set merged QC JSON cache filename from gen_topo

Author: davidrohr

DATA/production/qc-workflow.sh

@@ -5,7 +5,9 @@ if [[ -z "$WORKFLOW" ]] || [[ -z "$MYDIR" ]]; then
   exit 1
 fi
 
-if [[ -z $QC_JSON_FROM_OUTSIDE ]]; then
+if [[ -z $QC_JSON_FROM_OUTSIDE && ! -z $GEN_TOPO_QC_JSON_FILE && -f $GEN_TOPO_QC_JSON_FILE ]]; then
+  QC_JSON_FROM_OUTSIDE=$GEN_TOPO_QC_JSON_FILE
+elif [[ -z $QC_JSON_FROM_OUTSIDE ]]; then
   if [[ $EPNSYNCMODE == 1 || "0$GEN_TOPO_LOAD_QC_JSON_FROM_CONSUL" == "01" ]]; then
     [[ -z "$QC_JSON_TPC" ]] && QC_JSON_TPC=consul://o2/components/qc/ANY/any/tpc-full-qcmn
     [[ -z "$QC_JSON_ITS" ]] && QC_JSON_ITS=consul://o2/components/qc/ANY/any/its-qcmn-epn-full
@@ -137,11 +139,15 @@ if [[ -z $QC_JSON_FROM_OUTSIDE ]]; then
   fi
 
   if [[ ! -z "$JSON_FILES" ]]; then
-    mkdir -p $GEN_TOPO_WORKDIR/json_cache
-    if [[ "0$GEN_TOPO_ONTHEFLY" == "01" ]]; then
-      find $GEN_TOPO_WORKDIR/json_cache/ -maxdepth 1 -type f -mtime +30 | xargs rm -f
+    if [[ -z "$GEN_TOPO_QC_JSON_FILE" ]]; then
+      mkdir -p $GEN_TOPO_WORKDIR/json_cache
+      if [[ "0$GEN_TOPO_ONTHEFLY" == "01" ]]; then
+        find $GEN_TOPO_WORKDIR/json_cache/ -maxdepth 1 -type f -mtime +30 | xargs rm -f
+      fi
+      MERGED_JSON_FILENAME=$GEN_TOPO_WORKDIR/json_cache/`date +%Y%m%d-%H%M%S`-$$-$RANDOM-$OUTPUT_SUFFIX.json
+    else
+      MERGED_JSON_FILENAME=$GEN_TOPO_QC_JSON_FILE
     fi
-    MERGED_JSON_FILENAME=$GEN_TOPO_WORKDIR/json_cache/`date +%Y%m%d-%H%M%S`-$$-$RANDOM-$OUTPUT_SUFFIX.json
     jq -n 'reduce inputs as $s (input; .qc.tasks += ($s.qc.tasks) | .qc.checks += ($s.qc.checks)  | .qc.externalTasks += ($s.qc.externalTasks) | .qc.postprocessing += ($s.qc.postprocessing)| .dataSamplingPolicies += ($s.dataSamplingPolicies))' $QC_JSON_GLOBAL $JSON_FILES > $MERGED_JSON_FILENAME
     if [[ $? != 0 ]]; then
       echo Merging QC workflow with JSON files $JSON_FILES failed 1>&2

DATA/tools/parse

@@ -90,6 +90,11 @@ for line in f:
             print('O2 not loaded')
             raise
         xml_postprocessing = ('GEN_TOPO_OOM_WORKAROUND' in os.environ and int(os.environ['GEN_TOPO_OOM_WORKAROUND'])) or ('GEN_TOPO_DDS_ASSETS' in os.environ and int(os.environ['GEN_TOPO_DDS_ASSETS']))
+        if xml_postprocessing:
+            json_cache_path = os.environ['GEN_TOPO_WORKDIR'] + '/json_cache'
+            if not os.path.exists(json_cache_path):
+                os.makedirs(json_cache_path, 0o770)
+            os.environ['GEN_TOPO_QC_JSON_FILE'] = json_cache_path + '/qc_tmp_' + datetime.datetime.now().strftime('%y-%m-%d-%H-%M-%S') + '_' + str(random.randrange(100000000)) + '.json'
         with tempfile.TemporaryDirectory(prefix='o2_workflow_') as tmpdir:
             if 'GEN_TOPO_OVERRIDE_TEMPDIR' in os.environ:
                 tmpdir = os.environ['GEN_TOPO_OVERRIDE_TEMPDIR']
@@ -123,13 +128,10 @@ for line in f:
                     command_log_filter = '"^\[INFO"'
                 command_preopt = 'GEN_TOPO_CALIB_WORKFLOW=' + str(is_calib_workflow)
                 if xml_postprocessing:
-                    json_cache_path = os.environ['GEN_TOPO_WORKDIR'] + '/json_cache'
                     filename_xml = filename
                     if not 'GEN_TOPO_WORKDIR' in os.environ:
                         print('$GEN_TOPO_WORKDIR not set')
                         raise
-                    if not os.path.exists(json_cache_path):
-                        os.makedirs(json_cache_path, 0o770)
                     filename = json_cache_path + '/dpl_tmp_' + datetime.datetime.now().strftime('%y-%m-%d-%H-%M-%S') + wf[0] + str(i) + '_' + str(random.randrange(100000000)) + '.json'
                     command_preopt += ' WORKFLOWMODE=dump'
                 command = command_preopt + ' GLOBALDPLOPT+=" -b --dds-workflow-suffix _' + wf[0] + str(i) + '" ' + wf[3] + ' | grep -v ' + command_log_filter + ' > ' + filename + ' && [ `grep "^\[" ' + filename + ' | wc -l` == 0 ]'