InfluxDB and other updates

* add conversion to files that can be uploaded to InlfuxDB

* revise description comment at the top of the script

* provide global summary in "tabular" JSON format when comparing 2
  simulation directories

* small updates
  * possible to inspect global summary from the comparison of 2 sim
    directories
  * skip non-existent files in hit comparison
    (before e.g. ZDC comparison was done although it was deactovated)

* couple of minor fixes

Author: Benedikt Volkel

RelVal/o2dpg_release_validation.py

@@ -1,30 +1,63 @@
 #!/usr/bin/env python3
 #
-# This is a short script to conveniently wrap the ROOT macro used for release validation comparisons
+# Basically, this script allows a user to compare
+# 1. 2 corresponding ROOT files containing either histograms or QC Monitoring objects
+# 2. 2 corresponding simulation directories
 #
-# Help message:
-# usage: o2dpg_release_validation.py [-h] -f INPUT_FILES INPUT_FILES -t {1,2,3,4,5,6,7} [--chi2-value CHI2_VALUE] [--rel-bc-diff REL_BC_DIFF] [--rel-entries-diff REL_ENTRIES_DIFF] [--select-critical]
-# 
-# Wrapping ReleaseValidation macro
+# The RelVal suite is run with
+# o2dpg_release_validation.py rel-val -i <file-or-sim-dir1> <file-or-sim-dir2>
+#
+# If 2 sim directories should be compared, it is possible to specify for which parts the RelVal should be done by adding the flags
+# --with-<whic-part>
+# (see full help message below to see available options)
+#
+# usage: o2dpg_release_validation.py rel-val [-h] -i INPUT INPUT -t
+#                                            {1,2,3,4,5,6,7}
+#                                            [--chi2-value CHI2_VALUE]
+#                                            [--rel-mean-diff REL_MEAN_DIFF]
+#                                            [--rel-entries-diff REL_ENTRIES_DIFF]
+#                                            [--select-critical]
+#                                            [--threshold THRESHOLD]
+#                                            [--with-hits]
+#                                            [--detectors [{ITS,TOF,EMC,TRD,PHS,FT0,HMP,MFT,FDD,FV0,MCH,MID,CPV,ZDC,TPC} [{ITS,TOF,EMC,TRD,PHS,FT0,HMP,MFT,FDD,FV0,MCH,MID,CPV,ZDC,TPC} ...]]]
+#                                            [--with-tpctracks] [--with-kine]
+#                                            [--with-analysis] [--with-qc]
+#                                            [--no-plots] [--output OUTPUT]
 # 
 # optional arguments:
 #   -h, --help            show this help message and exit
-#   -f INPUT_FILES INPUT_FILES, --input-files INPUT_FILES INPUT_FILES
-#                         input files for comparison
+#   -i INPUT INPUT, --input INPUT INPUT
+#                         2 input files for comparison OR 2 input directories
+#                         from simulation for comparison
 #   -t {1,2,3,4,5,6,7}, --test {1,2,3,4,5,6,7}
 #                         index of test case
 #   --chi2-value CHI2_VALUE
 #                         Chi2 threshold
-#   --rel-bc-diff REL_BC_DIFF
-#                         Threshold of relative difference in normalised bin content
+#   --rel-mean-diff REL_MEAN_DIFF
+#                         Threshold of relative difference in mean
 #   --rel-entries-diff REL_ENTRIES_DIFF
 #                         Threshold of relative difference in number of entries
 #   --select-critical     Select the critical histograms and dump to file
-
+#   --threshold THRESHOLD
+#                         threshold for how far file sizes are allowed to
+#                         diverge before warning
+#   --with-hits           include hit comparison when RelVal when run on
+#                         simulation directories
+#   --detectors [{ITS,TOF,EMC,TRD,PHS,FT0,HMP,MFT,FDD,FV0,MCH,MID,CPV,ZDC,TPC} [{ITS,TOF,EMC,TRD,PHS,FT0,HMP,MFT,FDD,FV0,MCH,MID,CPV,ZDC,TPC} ...]]
+#                         include these detectors for hit RelVal
+#   --with-tpctracks      include TPC tracks RelVal when run on simulation
+#                         directories
+#   --with-kine           include kine RelVal when run on simulation directories
+#   --with-analysis       include analysis RelVal when run on simulation
+#                         directories
+#   --with-qc             include QC RelVal when run on simulation directories
+#   --no-plots            disable plotting
+#   --output OUTPUT, -o OUTPUT
+#                         output directory
 import sys
 import argparse
 from os import environ, makedirs
-from os.path import join, abspath, exists, isfile, isdir
+from os.path import join, abspath, exists, isfile, isdir, dirname
 from glob import glob
 from subprocess import Popen
 from pathlib import Path
@@ -44,6 +77,9 @@
 from ROOT import TFile, gDirectory, gROOT, TChain
 
 DETECTORS_OF_INTEREST_HITS = ["ITS", "TOF", "EMC", "TRD", "PHS", "FT0", "HMP", "MFT", "FDD", "FV0", "MCH", "MID", "CPV", "ZDC", "TPC"]
+
+REL_VAL_SEVERITY_MAP = {"GOOD": 0, "WARNING": 1, "NONCRIT_NC": 2, "CRIT_NC": 3, "BAD": 4}
+
 gROOT.SetBatch()
 
 def is_sim_dir(path):
@@ -259,31 +295,44 @@ def has_severity(filename, severity=("BAD", "CRIT_NC")):
     """
     Check if any 2 histograms have a given severity level after RelVal
     """
+    def rel_val_summary(d):
+        ret = False
+        for s in severity:
+            names = d.get(s)
+            if not names:
+                continue
+            print(f"Histograms for severity {s}:")
+            for n in names:
+                print(f"    {n}")
+            ret = True
+        return ret
+
+    def rel_val_summary_global(d):
+        ret = False
+        to_print = {k: [] for k in severity}
+        for s in severity:
+            for h in d:
+                if h["test_summary"] in severity:
+                    to_print[s].append(h["name"])
+                    ret = True
+        for s, names in to_print.items():
+            if not names:
+                continue
+            print(f"Histograms for severity {s}:")
+            for n in names:
+                print(f"    {n}")
+        return ret
+
     res = None
-    ret = False
     with open(filename, "r") as f:
         # NOTE For now care about the summary. However, we have each test individually, so we could do a more detailed check in the future
-        res = json.load(f)["test_summary"]
-    for s in severity:
-        names = res.get(s)
-        if not names:
-            continue
-        print(f"Histograms for severity {s}:")
-        for n in names:
-            print(f"    {n}")
-        ret = True
-    return ret
+        res = json.load(f)
 
+    # decide whether that is an overall summary or from 2 files only
+    if "histograms" in res:
+        return rel_val_summary_global(res["histograms"])
+    return rel_val_summary(res["test_summary"])
 
-def add_to_summary(paths_to_summary, key, summary_dict):
-    """
-    Extend a list of paths with given severity
-    """
-    summary_list = []
-    for p in paths_to_summary:
-        if has_severity(p):
-            summary_list.append(p)
-    summary_dict[key] = summary_list
 
 def rel_val_ttree(dir1, dir2, files, output_dir, args, treename="o2sim", *, combine_patterns=None):
     """
@@ -298,15 +347,19 @@ def rel_val_ttree(dir1, dir2, files, output_dir, args, treename="o2sim", *, comb
     # possibly combine common files, for instance when they come from different timeframes
     if combine_patterns:
         for cp in combine_patterns:
-            to_be_chained1.append([join(dir1, hf) for hf in files if cp in hf])
-            to_be_chained2.append([join(dir2, hf) for hf in files if cp in hf])
+            chained1 = [join(dir1, hf) for hf in files if cp in hf]
+            chained2 = [join(dir2, hf) for hf in files if cp in hf]
+            if not chained1 or not chained2:
+                continue
+            to_be_chained1.append(chained1)
+            to_be_chained2.append(chained2)
             output_dirs.append(f"{cp}_dir")
     else:
         to_be_chained1 = []
         to_be_chained2 = []
         for hf in files:
             to_be_chained1.append(join(dir1, hf))
-            to_be_chained2.append(join(dir1, hf))
+            to_be_chained2.append(join(dir2, hf))
             output_dirs.append(f"{hf}_dir")
 
     # paths for chains prepared, output directory names specified, do RelVal
@@ -321,6 +374,39 @@ def rel_val_ttree(dir1, dir2, files, output_dir, args, treename="o2sim", *, comb
     return 0
 
 
+def make_summary(in_dir):
+    """
+    Make a summary per histogram (that should be able to be parsed by Grafana eventually)
+    """
+    file_paths = glob(f"{in_dir}/**/Summary.json", recursive=True)
+    summary = []
+
+    for path in file_paths:
+        # go through all we found
+        current_summary = None
+        print(path)
+        with open(path, "r") as f:
+            current_summary = json.load(f)
+        # remove the file name, used as the top key for this collection
+        rel_val_path = "/".join(path.split("/")[:-1])
+        type_global = path.split("/")[1]
+        type_specific = "/".join(path.split("/")[1:-1])
+        make_summary = {}
+        for which_test, flagged_histos in current_summary.items():
+            # loop over tests done
+            for flag, histos in flagged_histos.items():
+                # loop over flags per test
+                for h in histos:
+                    if h not in make_summary:
+                        # re-arrange to have histogram at the sop
+                        make_summary[h] = {"name": h, "type_global": type_global, "type_specific": type_specific}
+                    # add outcome of test
+                    make_summary[h][which_test] = flag
+        # re-arrange to list, now each summary["path"] basically contains "rows" and each batch represents the columns
+        summary.extend([batch for batch in make_summary.values()])
+    return {"histograms": summary}
+
+
 def rel_val_histograms(dir1, dir2, files, output_dir, args):
     """
     Simply another wrapper to combine multiple files where we expect them to contain histograms already
@@ -359,7 +445,6 @@ def rel_val_sim_dirs(args):
         if not exists(output_dir_hits):
             makedirs(output_dir_hits)
         rel_val_ttree(dir1, dir2, hit_files, output_dir_hits, args, combine_patterns=[f"Hits{d}" for d in args.detectors])
-        add_to_summary(glob(f"{output_dir_hits}/**/{look_for}", recursive=True), "hist", summary_dict)
 
     # TPC tracks
     if args.with_tpctracks:
@@ -368,7 +453,6 @@ def rel_val_sim_dirs(args):
         if not exists(output_dir_tpctracks):
             makedirs(output_dir_tpctracks)
         rel_val_ttree(dir1, dir2, tpctrack_files, output_dir_tpctracks, args, "tpcrec", combine_patterns=["tpctracks.root"])
-        add_to_summary(glob(f"{output_dir_tpctracks}/**/{look_for}", recursive=True), "tpctracks", summary_dict)
 
     # TPC tracks
     if args.with_kine:
@@ -377,7 +461,6 @@ def rel_val_sim_dirs(args):
         if not exists(output_dir_kine):
             makedirs(output_dir_kine)
         rel_val_ttree(dir1, dir2, kine_files, output_dir_kine, args, combine_patterns=["Kine.root"])
-        add_to_summary(glob(f"{output_dir_kine}/**/{look_for}", recursive=True), "kine", summary_dict)
 
     # Analysis
     if args.with_analysis:
@@ -389,7 +472,6 @@ def rel_val_sim_dirs(args):
         if not exists(output_dir_analysis):
             makedirs(output_dir_analysis)
         rel_val_histograms(dir_analysis1, dir_analysis2, analysis_files, output_dir_analysis, args)
-        add_to_summary(glob(f"{output_dir_analysis}/**/{look_for}", recursive=True), "analysis", summary_dict)
 
     # QC
     if args.with_qc:
@@ -400,10 +482,10 @@ def rel_val_sim_dirs(args):
         if not exists(output_dir_qc):
             makedirs(output_dir_qc)
         rel_val_histograms(dir_qc1, dir_qc2, qc_files, output_dir_qc, args)
-        add_to_summary(glob(f"{output_dir_qc}/**/{look_for}", recursive=True), "qc", summary_dict)
 
-    with open(join(output_dir, "SummaryToBeChecked.json"), "w") as f:
-        json.dump(summary_dict, f, indent=2)
+    with open(join(output_dir, "SummaryGlobal.json"), "w") as f:
+        json.dump(make_summary(output_dir), f, indent=2)
+
 
 def rel_val(args):
     """
@@ -429,6 +511,47 @@ def inspect(args):
     """
     return has_severity(args.file, args.severity)
 
+
+def influx(args):
+    """
+    Create an influxDB metrics file
+    """
+    output_dir = args.dir
+    json_in = join(output_dir, "SummaryGlobal.json")
+    if not exists(json_in):
+        print(f"Cannot find expected JSON summary {json_in}.")
+        return 1
+
+    table_name = f"{args.table_prefix}_ReleaseValidation"
+    tags_out = ""
+    if args.tags:
+        for t in args.tags:
+            t_split = t.split("=")
+            if len(t_split) != 2:
+                print(f"ERROR: Invalid format of tags {t} for InfluxDB")
+                return 1
+            # we take it apart and put it back together again to make sure there are no whitespaces etc
+            tags_out += f",{t_split[0].strip()}={t_split[1].strip()}"
+
+    # always the same
+    row_tags = table_name + tags_out
+
+    out_file = join(output_dir, "influxDB.dat")
+
+    in_list = None
+    with open(json_in, "r") as f:
+        in_list = json.load(f)["histograms"]
+    with open(out_file, "w") as f:
+        for h in in_list:
+            s = f"{row_tags},type_global={h['type_global']},type_specific={h['type_specific']} histogram_name={h['name']}"
+            for k, v in h.items():
+                # add all tests - do it dynamically because more might be added in the future
+                if "test_" not in k:
+                    continue
+                s += f",{k}={REL_VAL_SEVERITY_MAP[v]}"
+            f.write(f"{s}\n")
+
+
 def main():
     """entry point when run directly from command line"""
     parser = argparse.ArgumentParser(description='Wrapping ReleaseValidation macro')
@@ -438,7 +561,7 @@ def main():
 
     sub_parsers = parser.add_subparsers(dest="command")
     rel_val_parser = sub_parsers.add_parser("rel-val", parents=[common_file_parser])
-    rel_val_parser.add_argument("-t", "--test", type=int, help="index of test case", choices=list(range(1, 8)), required=True)
+    rel_val_parser.add_argument("-t", "--test", type=int, help="index of test case", choices=list(range(1, 8)), default=7)
     rel_val_parser.add_argument("--chi2-value", dest="chi2_value", type=float, help="Chi2 threshold", default=1.5)
     rel_val_parser.add_argument("--rel-mean-diff", dest="rel_mean_diff", type=float, help="Threshold of relative difference in mean", default=1.5)
     rel_val_parser.add_argument("--rel-entries-diff", dest="rel_entries_diff", type=float, help="Threshold of relative difference in number of entries", default=0.01)
@@ -451,14 +574,20 @@ def main():
     rel_val_parser.add_argument("--with-analysis", dest="with_analysis", action="store_true", help="include analysis RelVal when run on simulation directories")
     rel_val_parser.add_argument("--with-qc", dest="with_qc", action="store_true", help="include QC RelVal when run on simulation directories")
     rel_val_parser.add_argument("--no-plots", dest="no_plots", action="store_true", help="disable plotting")
-    rel_val_parser.add_argument("--output", "-o", help="output directory", default="./")
+    rel_val_parser.add_argument("--output", "-o", help="output directory", default="rel_val")
     rel_val_parser.set_defaults(func=rel_val)
 
     inspect_parser = sub_parsers.add_parser("inspect")
     inspect_parser.add_argument("file", help="pass a JSON produced from ReleaseValidation (rel-val)")
-    inspect_parser.add_argument("--severity", nargs="*", default=["BAD", "CRIT_NC"], choices=["GOOD", "WARNING", "BAD", "CRIT_NC", "NONCRIT_NC"], help="Choose severity levels to search for")
+    inspect_parser.add_argument("--severity", nargs="*", default=["BAD", "CRIT_NC"], choices=REL_VAL_SEVERITY_MAP.keys(), help="Choose severity levels to search for")
     inspect_parser.set_defaults(func=inspect)
 
+    influx_parser = sub_parsers.add_parser("influx")
+    influx_parser.add_argument("--dir", help="directory where ReleaseValidation was run", required=True)
+    influx_parser.add_argument("--tags", nargs="*", help="tags to be added for influx, list of key=value")
+    influx_parser.add_argument("--table-prefix", dest="table_prefix", help="prefix for table name", default="O2DPG_MC")
+    influx_parser.set_defaults(func=influx)
+
     args = parser.parse_args()
     return(args.func(args))