Improve cluster error reporting

Author: bbimber

cluster/api-src/org/labkey/api/cluster/ClusterService.java

@@ -36,6 +36,8 @@ public static void setInstance(ClusterService instance)
      */
     abstract public PipelineJob createClusterRemotePipelineJob(Container c, User u, String jobName, RemoteExecutionEngine engine, ClusterRemoteTask task, File logFile) throws PipelineValidationException;
 
+    abstract public File getSerializedJobFile(File jobLogFile);
+
     public interface ClusterRemoteTask extends Serializable
     {
         public void run(Logger log);

cluster/src/org/labkey/cluster/ClusterController.java

@@ -29,7 +29,6 @@
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.pipeline.PipeRoot;
 import org.labkey.api.pipeline.PipelineJob;
-import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.pipeline.PipelineJobService;
 import org.labkey.api.pipeline.PipelineService;
 import org.labkey.api.pipeline.PipelineStatusFile;
@@ -39,7 +38,6 @@
 import org.labkey.api.security.RequiresSiteAdmin;
 import org.labkey.api.security.permissions.AdminPermission;
 import org.labkey.api.security.permissions.ReadPermission;
-import org.labkey.api.settings.AppProps;
 import org.labkey.api.settings.ResourceURL;
 import org.labkey.api.util.HtmlString;
 import org.labkey.api.util.PageFlowUtil;
@@ -51,7 +49,6 @@
 import org.springframework.web.servlet.ModelAndView;
 
 import java.io.File;
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -313,26 +310,39 @@ public boolean handlePost(JobIdsForm form, BindException errors) throws Exceptio
                 sfs.add(sf);
             }
 
-            sfs.forEach(sf -> {
+            for (PipelineStatusFile sf : sfs)
+            {
                 File log = new File(sf.getFilePath());
                 File json = AbstractClusterExecutionEngine.getSerializedJobFile(log);
                 if (!json.exists())
                 {
-                    return;
+                    errors.reject(ERROR_MSG, "Unable to find pipeline JSON, expected: " + json.getPath());
+                    return false;
                 }
 
+                PipelineJob job = null;
                 try
                 {
-                    PipelineJob job = PipelineJob.readFromFile(json);
+                    job = PipelineJob.readFromFile(json);
 
-                    _log.info("Submitting job: " + job.getJobGUID() + ": " + job.getActiveTaskStatus());
+                    job.getLogger().info("Submitting job from JSON: " + job.getJobGUID() + ": " + job.getActiveTaskStatus());
                     PipelineService.get().setPipelineJobStatus(job, job.getActiveTaskStatus());
                 }
-                catch (PipelineJobException | IOException e)
+                catch (Exception e)
                 {
-                    _log.error(e);
+                    if (job != null)
+                    {
+                        job.getLogger().error("Unable to requeue job", e);
+                    }
+                    else
+                    {
+                        _log.error("Unable to requeue pipeline job", e);
+                    }
+
+                    errors.reject(ERROR_MSG, "Unable to requeue pipeline job: " + e.getMessage());
+                    return false;
                 }
-            });
+            }
 
             return true;
         }

cluster/src/org/labkey/cluster/ClusterServiceImpl.java

@@ -8,6 +8,7 @@
 import org.labkey.api.pipeline.RemoteExecutionEngine;
 import org.labkey.api.pipeline.TaskId;
 import org.labkey.api.security.User;
+import org.labkey.cluster.pipeline.AbstractClusterExecutionEngine;
 import org.labkey.cluster.pipeline.ClusterPipelineJob;
 
 import java.io.File;
@@ -74,6 +75,12 @@ public String getClusterUser(Container c)
         return ClusterManager.get().getClusterUser(c);
     }
 
+    @Override
+    public File getSerializedJobFile(File jobLogFile)
+    {
+        return AbstractClusterExecutionEngine.getSerializedJobFile(jobLogFile);
+    }
+
     @Override
     public PipelineJob createClusterRemotePipelineJob(Container c, User u, String jobName, RemoteExecutionEngine engine, ClusterRemoteTask task, File logFile) throws PipelineValidationException
     {

cluster/src/org/labkey/cluster/query/ForceCancelJobsButton.java

@@ -1,5 +1,7 @@
 package org.labkey.cluster.query;
 
+import org.labkey.api.data.ContainerManager;
+import org.labkey.api.data.TableInfo;
 import org.labkey.api.ldk.table.SimpleButtonConfigFactory;
 import org.labkey.api.module.ModuleLoader;
 import org.labkey.api.security.permissions.AdminPermission;
@@ -19,4 +21,10 @@ public ForceCancelJobsButton()
 
         setPermission(AdminPermission.class);
     }
+
+    @Override
+    public boolean isAvailable(TableInfo ti)
+    {
+        return super.isAvailable(ti) || ContainerManager.getRoot().equals(ti.getUserSchema().getContainer());
+    }
 }

cluster/src/org/labkey/cluster/query/RecoverCompletedJobsButton.java

@@ -1,5 +1,7 @@
 package org.labkey.cluster.query;
 
+import org.labkey.api.data.ContainerManager;
+import org.labkey.api.data.TableInfo;
 import org.labkey.api.ldk.table.SimpleButtonConfigFactory;
 import org.labkey.api.module.ModuleLoader;
 import org.labkey.api.security.permissions.AdminPermission;
@@ -19,4 +21,10 @@ public RecoverCompletedJobsButton()
 
         setPermission(AdminPermission.class);
     }
+
+    @Override
+    public boolean isAvailable(TableInfo ti)
+    {
+        return super.isAvailable(ti) || ContainerManager.getRoot().equals(ti.getUserSchema().getContainer());
+    }
 }