Improve reporting for cluster errors

Author: bbimber

cluster/api-src/org/labkey/api/cluster/ClusterService.java

@@ -34,10 +34,12 @@ public static void setInstance(ClusterService instance)
      * This creates a barebones PipelineJob configured to run on the selected RemoteExecutionEngine.
      * You may wish to further configure this job.  This method does not submit the job, which you can do using PipelineService.queueJob()
      */
-    abstract public PipelineJob createClusterRemotePipelineJob(Container c, User u, String jobName, RemoteExecutionEngine engine, ClusterRemoteTask task, File logFile) throws PipelineValidationException;
+    abstract public PipelineJob createClusterRemotePipelineJob(Container c, User u, String jobName, RemoteExecutionEngine<?> engine, ClusterRemoteTask task, File logFile) throws PipelineValidationException;
 
     abstract public File getSerializedJobFile(File jobLogFile);
 
+    abstract public File getExpectedSubmitScript(PipelineJob job);
+
     public interface ClusterRemoteTask extends Serializable
     {
         public void run(Logger log);

cluster/src/org/labkey/cluster/ClusterModule.java

@@ -37,6 +37,7 @@
 import org.labkey.api.view.WebPartFactory;
 import org.labkey.cluster.pipeline.ClusterPipelineJobNotificationProvider;
 import org.labkey.cluster.pipeline.ClusterPipelineProvider;
+import org.labkey.cluster.pipeline.SlurmExecutionEngine;
 import org.labkey.cluster.pipeline.TestCase;
 import org.labkey.cluster.query.ForceCancelJobsButton;
 import org.labkey.cluster.query.RecoverCompletedJobsButton;
@@ -128,6 +129,14 @@ public Set<Class> getIntegrationTests()
         return testClasses;
     }
 
+    @Override
+    public @NotNull Set<Class> getUnitTests()
+    {
+        return new HashSet<>(Arrays.asList(
+                SlurmExecutionEngine.TestCase.class
+        ));
+    }
+
     @Override
     protected void registerSchemas()
     {

cluster/src/org/labkey/cluster/ClusterServiceImpl.java

@@ -10,6 +10,8 @@
 import org.labkey.api.security.User;
 import org.labkey.cluster.pipeline.AbstractClusterExecutionEngine;
 import org.labkey.cluster.pipeline.ClusterPipelineJob;
+import org.labkey.cluster.pipeline.HTCondorExecutionEngine;
+import org.labkey.cluster.pipeline.SlurmExecutionEngine;
 
 import java.io.File;
 import java.util.ArrayList;
@@ -82,7 +84,28 @@ public File getSerializedJobFile(File jobLogFile)
     }
 
     @Override
-    public PipelineJob createClusterRemotePipelineJob(Container c, User u, String jobName, RemoteExecutionEngine engine, ClusterRemoteTask task, File logFile) throws PipelineValidationException
+    public File getExpectedSubmitScript(PipelineJob job)
+    {
+        if (job.getActiveTaskFactory() != null)
+        {
+            String location = job.getActiveTaskFactory().getExecutionLocation();
+            if (SlurmExecutionEngine.TYPE.equals(location))
+            {
+                return SlurmExecutionEngine.getExpectedSubmitScript(job);
+            }
+            else if (HTCondorExecutionEngine.TYPE.equals(location))
+            {
+                return HTCondorExecutionEngine.getExpectedSubmitScript(job);
+            }
+        }
+
+        job.getLogger().error("Unable to find appropriate remote execution engine for job: " + job.getJobGUID() + ", with active task factory: " + (job.getActiveTaskFactory() != null ? job.getActiveTaskFactory().getId() : " null"));
+
+        return null;
+    }
+
+    @Override
+    public PipelineJob createClusterRemotePipelineJob(Container c, User u, String jobName, RemoteExecutionEngine<?> engine, ClusterRemoteTask task, File logFile) throws PipelineValidationException
     {
         return ClusterPipelineJob.createJob(c, u, jobName, task, engine, logFile);
     }

cluster/src/org/labkey/cluster/pipeline/HTCondorExecutionEngine.java

@@ -167,7 +167,7 @@ private File createSubmitScript(PipelineJob job) throws PipelineJobException
             //we want this unique for each task, but reused if submitted multiple times
             File outDir = job.getLogFile().getParentFile();
             String basename = FileUtil.getBaseName(job.getLogFile());
-            File submitScript = new File(outDir, basename + (job.getActiveTaskId() == null ? "" : "." + job.getActiveTaskId().getNamespaceClass().getSimpleName()) + ".submit");
+            File submitScript = getExpectedSubmitScript(job);
             if (ClusterManager.get().isRecreateSubmitScriptFile() && submitScript.exists())
             {
                 job.getLogger().info("Deleting existing submit script");
@@ -528,4 +528,10 @@ protected boolean removeJob(ClusterJob clusterJob)
 
         return success;
     }
+
+    public static File getExpectedSubmitScript(PipelineJob job)
+    {
+        String basename = FileUtil.getBaseName(job.getLogFile());
+        return new File(job.getLogFile().getParentFile(), basename + (job.getActiveTaskId() == null ? "" : "." + job.getActiveTaskId().getNamespaceClass().getSimpleName()) + ".submit");
+    }
 }

cluster/src/org/labkey/cluster/pipeline/SlurmExecutionEngine.java

@@ -5,6 +5,8 @@
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.junit.Assert;
+import org.junit.Test;
 import org.labkey.api.cluster.ClusterResourceAllocator;
 import org.labkey.api.collections.CaseInsensitiveHashSet;
 import org.labkey.api.data.Container;
@@ -281,10 +283,10 @@ else if (headerFound)
                         if (maxRssIdx > -1 && maxRssIdx < tokens.length)
                         {
                             long bytes = FileSizeFormatter.convertStringRepresentationToBytes(tokens[maxRssIdx]);
-                            long requestInBytes = FileSizeFormatter.convertBytesToUnit(getConfig().getRequestMemory(), 'G');
+                            long requestInBytes = FileSizeFormatter.convertStringRepresentationToBytes(getConfig().getRequestMemory() + "G"); //request is always GB
                             if (bytes > requestInBytes)
                             {
-                                info = "Job exceeded memory: " + FileSizeFormatter.convertBytesToUnit(bytes, 'G');
+                                info = "Job exceeded memory, max was: " + FileSizeFormatter.convertBytesToUnit(bytes, 'G') + "G";
                             }
                         }
                     }
@@ -361,6 +363,12 @@ protected boolean removeJob(ClusterJob clusterJob)
         return success;
     }
 
+    public static File getExpectedSubmitScript(PipelineJob job)
+    {
+        String basename = FileUtil.getBaseName(job.getLogFile());
+        return new File(job.getLogFile().getParentFile(), basename + (job.getActiveTaskId() == null ? "" : "." + job.getActiveTaskId().getNamespaceClass().getSimpleName()) + ".slurm.sh");
+    }
+
     private File createSubmitScript(PipelineJob job) throws PipelineJobException
     {
         try
@@ -370,7 +378,7 @@ private File createSubmitScript(PipelineJob job) throws PipelineJobException
             //we want this unique for each task, but reused if submitted multiple times
             File outDir = job.getLogFile().getParentFile();
             String basename = FileUtil.getBaseName(job.getLogFile());
-            File submitScript = new File(outDir, basename + (job.getActiveTaskId() == null ? "" : "." + job.getActiveTaskId().getNamespaceClass().getSimpleName()) + ".slurm.sh");
+            File submitScript = getExpectedSubmitScript(job);
             if (ClusterManager.get().isRecreateSubmitScriptFile() && submitScript.exists())
             {
                 job.getLogger().info("Deleting existing submit script");
@@ -745,4 +753,17 @@ private static long getSizeFactor(char unit)
                     };
         }
     }
+
+    public static class TestCase
+    {
+        @Test
+        public void testFileSizeFormatter()
+        {
+            long bytes = FileSizeFormatter.convertStringRepresentationToBytes("1362624K");
+            Assert.assertEquals("Incorrect byte value", 1395326976, bytes);
+
+            long val2 = FileSizeFormatter.convertBytesToUnit(bytes, 'K');
+            Assert.assertEquals("Incorrect string value", 1362624, val2);
+        }
+    }
 }