[Anchor] Add test for anchored MC (#1464)

Runs a very simple anchored production, 2TFs, 50 pp events each

Co-authored-by: Benedikt Volkel <benedikt.volkel@cern.ch>

Author: benedikt-voelkel

MC/run/ANCHOR/anchorMC.sh

@@ -241,6 +241,8 @@ if [[ -z "${DISABLE_QC}" && "${MCRC}" = "0" && "${remainingargs}" == *"--include
   # do QC tasks
   echo "Doing QC"
   ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --target-labels QC --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} -k
+  # NOTE that with the -k|--keep-going option, the runner will try to keep on executing even if some tasks fail.
+  # That means, even if there is a failing QC task, the return code will be 0
   MCRC=$?
 fi
 

MC/run/ANCHOR/tests/test_anchor_2023_apass2_PbPb.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+#
+# An example steering script for anchored MC simulations, PbPb
+#
+
+# example anchoring
+# taken from https://its.cern.ch/jira/browse/O2-4586
+export ALIEN_JDL_LPMANCHORPASSNAME=apass2
+export ALIEN_JDL_MCANCHOR=apass2
+export ALIEN_JDL_COLLISIONSYSTEM=Pb-Pb
+export ALIEN_JDL_CPULIMIT=8
+export ALIEN_JDL_LPMPASSNAME=apass2
+export ALIEN_JDL_LPMRUNNUMBER=544121
+export ALIEN_JDL_LPMPRODUCTIONTYPE=MC
+export ALIEN_JDL_LPMINTERACTIONTYPE=PbPb
+export ALIEN_JDL_LPMPRODUCTIONTAG=LHC24a1
+export ALIEN_JDL_LPMANCHORRUN=544121
+export ALIEN_JDL_LPMANCHORPRODUCTION=LHC23zzh
+export ALIEN_JDL_LPMANCHORYEAR=2023
+
+export NTIMEFRAMES=2
+export NSIGEVENTS=2
+export SPLITID=100
+export PRODSPLIT=153
+export CYCLE=0
+
+# on the GRID, this is set, for our use case, we can mimic any job ID
+export ALIEN_PROC_ID=2963436952
+
+# run the central anchor steering script; this includes
+# * derive timestamp
+# * derive interaction rate
+# * extract and prepare configurations (which detectors are contained in the run etc.)
+# * run the simulation (and QC)
+${O2DPG_ROOT}/MC/run/ANCHOR/anchorMC.sh

MC/run/ANCHOR/tests/test_anchor_2023_apass2_pp.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+#
+# An example steering script for anchored MC simulations, pp
+#
+
+# example anchoring
+# taken from https://its.cern.ch/jira/browse/O2-4586
+export ALIEN_JDL_LPMANCHORPASSNAME=apass2
+export ALIEN_JDL_MCANCHOR=apass2
+export ALIEN_JDL_COLLISIONSYSTEM=p-p
+export ALIEN_JDL_CPULIMIT=8
+export ALIEN_JDL_LPMPASSNAME=apass2
+export ALIEN_JDL_LPMRUNNUMBER=535069
+export ALIEN_JDL_LPMPRODUCTIONTYPE=MC
+export ALIEN_JDL_LPMINTERACTIONTYPE=pp
+export ALIEN_JDL_LPMPRODUCTIONTAG=LHC24a2
+export ALIEN_JDL_LPMANCHORRUN=535069
+export ALIEN_JDL_LPMANCHORPRODUCTION=LHC23f
+export ALIEN_JDL_LPMANCHORYEAR=2023
+
+export NTIMEFRAMES=2
+export NSIGEVENTS=50
+export SPLITID=100
+export PRODSPLIT=153
+export CYCLE=0
+
+# on the GRID, this is set, for our use case, we can mimic any job ID
+export ALIEN_PROC_ID=2963436952
+
+# for pp and 50 events per TF, we launch only 4 workers.
+export NWORKERS=4
+
+# run the central anchor steering script; this includes
+# * derive timestamp
+# * derive interaction rate
+# * extract and prepare configurations (which detectors are contained in the run etc.)
+# * run the simulation (and QC)
+${O2DPG_ROOT}/MC/run/ANCHOR/anchorMC.sh

test/common/utils/utils.sh

@@ -78,7 +78,7 @@ make_wf_creation_script()
 print_error_logs()
 {
     local search_dir=${1}
-    local search_pattern="TASK-EXIT-CODE: ([1-9][0-9]*)|[Ss]egmentation violation|[Ee]xception caught|\[FATAL\]|uncaught exception|\(int\) ([1-9][0-9]*)|fair::FatalException"
+    local search_pattern="TASK-EXIT-CODE: ([1-9][0-9]*)|[Ss]egmentation violation|[Ss]egmentation fault|Program crashed|[Ee]xception caught|\[FATAL\]|uncaught exception|\(int\) ([1-9][0-9]*)|fair::FatalException"
     local error_files=$(find ${search_dir} -maxdepth 4 -type f \( -name "*.log" -or -name "*serverlog*" -or -name "*workerlog*" -or -name "*mergerlog*" \) | xargs grep -l -E "${search_pattern}" | sort)
     for ef in ${error_files} ; do
         echo_red "Error found in log $(realpath ${ef})"

test/run_workflow_tests.sh

@@ -3,12 +3,14 @@
 # The test parent dir to be cretaed in current directory
 TEST_PARENT_DIR_PWG="o2dpg_tests/workflows_pwgs"
 TEST_PARENT_DIR_BIN="o2dpg_tests/workflows_bin"
+TEST_PARENT_DIR_ANCHORED="o2dpg_tests/anchored"
 
 # a global counter for tests
 TEST_COUNTER=0
 
 # unified names of log files
 LOG_FILE_WF="o2dpg-test-wf.log"
+LOG_FILE_ANCHORED="o2dpg-test-anchored.log"
 
 # Prepare some colored output
 SRED="\033[0;31m"
@@ -123,6 +125,26 @@ run_workflow_creation()
     return ${RET}
 }
 
+test_anchored()
+{
+    local to_run="${1:-${O2DPG_ROOT}/MC/run/ANCHOR/tests/test_anchor_2023_apass2_pp.sh}"
+    local RET=0
+    for anchored_script in ${to_run} ; do
+        [[ ! -f ${anchored_script} ]] && { echo "Desired test script ${anchored_script} does not exist. Skip." ; continue ; }
+        ((TEST_COUNTER++))
+        local test_dir=${TEST_COUNTER}_$(basename ${anchored_script})_dir
+        rm -rf ${test_dir} 2> /dev/null
+        mkdir ${test_dir}
+        pushd ${test_dir} > /dev/null
+            echo -n "Test ${TEST_COUNTER}: ${anchored_script}"
+            ${anchored_script} >> ${LOG_FILE_ANCHORED} 2>&1
+            local ret_this=${?}
+            [[ "${ret_this}" != "0" ]] && RET=${ret_this}
+        popd > /dev/null
+    done
+    return ${RET}
+}
+
 collect_changed_pwg_wf_files()
 {
     # Collect all INI files which have changed
@@ -188,8 +210,9 @@ source ${REPO_DIR}/test/common/utils/utils.sh
 pushd ${REPO_DIR} > /dev/null
 
 # flag if anything changed in the sim workflow bin dir
-changed_wf_bin=$(get_changed_files | grep "MC/bin")
+changed_wf_bin=$(get_changed_files | grep -E "MC/bin")
 changed_wf_bin_related=$(get_changed_files | grep -E "MC/analysis_testing|MC/config/analysis_testing/json|MC/config/QC/json")
+changed_anchored_related=$(get_changed_files | grep -E "MC/run/ANCHOR/anchorMC.sh|MC/run/ANCHOR/tests|MC/bin|UTILS/parse-async-WorkflowConfig.py")
 
 
 # collect what has changed for PWGs
@@ -215,6 +238,27 @@ REPO_DIR=$(realpath ${REPO_DIR})
 export O2DPG_ROOT=${REPO_DIR}
 
 
+###############
+# ANCHORED MC #
+###############
+# prepare our local test directory for PWG tests
+rm -rf ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null
+mkdir -p ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null
+pushd ${TEST_PARENT_DIR_ANCHORED} > /dev/null
+
+# global return code for PWGs
+ret_global_anchored=0
+if [[ "${changed_anchored_related}" != "" ]] ; then
+    echo "### Test anchored ###"
+    # Run an anchored test
+    test_anchored
+    ret_global_anchored=${?}
+    echo
+fi
+
+# return to where we came from
+popd > /dev/null
+
 ########
 # PWGs #
 ########
@@ -226,7 +270,7 @@ pushd ${TEST_PARENT_DIR_PWG} > /dev/null
 # global return code for PWGs
 ret_global_pwg=0
 if [[ "${changed_wf_bin}" != "" ]] ; then
-    # Run all the PWG related WF creations, hence overwrite what was collected by collect_changed_pwg_wf_files eal=rlier
+    # Run all the PWG related WF creations, hence overwrite what was collected by collect_changed_pwg_wf_files earlier
     WF_FILES=$(get_all_workflows "MC/run/.*/")
     echo
 fi
@@ -240,7 +284,6 @@ if [[ "${WF_FILES}" != "" ]] ; then
     echo
 fi
 
-
 # return to where we came from
 popd > /dev/null
 
@@ -285,9 +328,21 @@ if [[ "${ret_global_bin}" != "0" ]] ; then
     echo "###################################"
     echo
     print_error_logs ${TEST_PARENT_DIR_BIN}
-    exit ${ret_global_bin}
 fi
 
+# However, if a central test fails, exit code will be !=0
+if [[ "${ret_global_anchored}" != "0" ]] ; then
+    echo
+    echo "##########################"
+    echo "# ERROR for anchored MCs #"
+    echo "##########################"
+    echo
+    print_error_logs ${TEST_PARENT_DIR_ANCHORED}
+fi
+
+RET=$(( ret_global_bin + ret_global_anchored ))
+
 echo
-echo_green "All required workflow tests successful"
-echo
+[[ "${RET}" != "0" ]] && echo "There were errors, please check!" || echo_green "All required workflow tests successful"
+
+exit ${RET}