Better visibility for cluster warnings

Author: bbimber

cluster/src/org/labkey/cluster/pipeline/AbstractClusterExecutionEngine.java

@@ -501,6 +501,11 @@ protected void updateJobStatus(@Nullable String status, ClusterJob j, @Nullable
             {
                 statusChanged = true;
             }
+
+            if (!statusChanged && !sf.getStatus().equalsIgnoreCase(j.getStatus()))
+            {
+                _log.error("ClusterExecutionEngine reports status not changed, but was: " + sf.getStatus() + " / " + j.getStatus());
+            }
         }
 
         //no need to redundantly update PipelineJob

cluster/src/org/labkey/cluster/pipeline/SlurmExecutionEngine.java

@@ -4,6 +4,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 import org.labkey.api.cluster.ClusterResourceAllocator;
 import org.labkey.api.collections.CaseInsensitiveHashSet;
 import org.labkey.api.data.Container;
@@ -207,14 +208,15 @@ protected Pair<String, String> getStatusForJob(ClusterJob job, Container c)
         }
 
         String command = getConfig().getHistoryCommandExpr().eval(ctx);
+        String info = null;
         List<String> ret = execute(command);
         if (ret != null)
         {
             //verify success
             boolean headerFound = false;
             boolean foundJobLine = false;
             LinkedHashSet<String> statuses = new LinkedHashSet<>();
-            List<String> header = null;
+            List<String> header;
             int jobIdx = -1;
             int stateIdx = -1;
             int hostnameIdx = -1;
@@ -290,7 +292,7 @@ else if (headerFound)
                     _log.error("more than one status returned for job " + job.getClusterId() + ": " + StringUtils.join(statuses, ";") + ", using: " + status);
                 }
 
-                return translateSlurmStatusToTaskStatus(status);
+                return translateSlurmStatusToTaskStatus(status, info);
             }
         }
 
@@ -496,21 +498,26 @@ private File createSubmitScript(PipelineJob job) throws PipelineJobException
     }
 
     private Pair<String, String> translateSlurmStatusToTaskStatus(String status)
+    {
+        return translateSlurmStatusToTaskStatus(status, null);
+    }
+
+    private Pair<String, String> translateSlurmStatusToTaskStatus(String status, @Nullable String info)
     {
         if (status == null)
             return null;
 
         try
         {
             StatusType st = StatusType.parseValue(status);
-            return Pair.of(st.getLabkeyStatus().toUpperCase(), st.getInfo());
+            return Pair.of(st.getLabkeyStatus().toUpperCase(), st.getInfo(info));
         }
         catch (IllegalArgumentException e)
         {
             _log.error("Unknown status type: [" + status + "]");
         }
 
-        return Pair.of(status, null);
+        return Pair.of(status, info);
     }
 
     public static enum StatusType
@@ -562,9 +569,9 @@ public String getLabkeyStatus()
             return _taskStatus == null ? _labkeyStatus : _taskStatus.name();
         }
 
-        public String getInfo()
+        public String getInfo(@Nullable String info)
         {
-            return _info;
+            return _info == null ? info : _info + (info == null ? "" : " / " + info);
         }
 
         public static StatusType parseValue(String value)