Merge pull request #177 from bimberlabinternal/24.11_fb_merge

Merge discvr-24.7 to discvr-24.11

Author: bbimber

primeseq/resources/external/comb-p/Dockerfile

@@ -72,8 +72,6 @@ protected void init()
     @Override
     protected void doStartupAfterSpringConfig(ModuleContext moduleContext)
     {
-        SequencePipelineService.get().registerResourceSettings(new ExacloudResourceSettings());
-
         SystemMaintenance.addTask(new ClusterMaintenanceTask());
 
         ClusterService.get().registerResourceAllocator(new BlastPipelineJobResourceAllocator.Factory());
@@ -109,6 +107,8 @@ public PipelineStartup()
             }
             else
             {
+                SequencePipelineService.get().registerResourceSettings(new ExacloudResourceSettings());
+
                 SequencePipelineService.get().registerPipelineStep(new BismarkWrapper.Provider());
                 SequencePipelineService.get().registerPipelineStep(new BismarkWrapper.MethylationExtractorProvider());
 

primeseq/resources/external/comb-p/combpWrapper.sh

@@ -29,18 +29,19 @@
 import org.labkey.api.security.User;
 import org.labkey.api.sequenceanalysis.SequenceOutputFile;
 import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
+import org.labkey.api.sequenceanalysis.pipeline.PipelineOutputTracker;
 import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
 import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
 import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper;
 import org.labkey.api.sequenceanalysis.run.CommandWrapper;
+import org.labkey.api.sequenceanalysis.run.DockerWrapper;
 import org.labkey.api.util.FileType;
 import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.Pair;
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.writer.PrintWriters;
 import org.labkey.primeseq.PrimeseqModule;
-import org.labkey.primeseq.pipeline.CombpRunner;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -487,8 +488,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
                         Double seed = ctx.getParams().optDouble("seed", 0.05);
                         Integer step = ctx.getParams().optInt("step", 100);
 
-                        CombpRunner combp = new CombpRunner(ctx.getLogger());
-                        File outBed = combp.runCompP(finalOut, ctx.getOutputDir(), dist, seed, step);
+                        File outBed = runCompP(finalOut, ctx.getOutputDir(), dist, seed, step, ctx, ctx.getFileManager());
                         SequenceOutputFile so2 = new SequenceOutputFile();
                         so2.setName("Comb-p: " + ctx.getJob().getDescription());
                         so2.setDescription("Comb-p: " + jobDescription);
@@ -726,4 +726,36 @@ else if (i2.compareTo(il) > 0)
             }
         }
     }
+
+    private File runCompP(File inputBed, File outputDir, int dist, double seed, int stepSize, PipelineContext ctx, PipelineOutputTracker tracker) throws PipelineJobException
+    {
+        // See: https://github.com/bbimber/combpdocker
+        DockerWrapper wrapper = new DockerWrapper("bbimber/combpdocker", ctx.getLogger(), ctx);
+        File outputPrefix = new File(outputDir, FileUtil.getBaseName(inputBed) + ".combp");
+
+        List<String> args = new ArrayList<>();
+        args.add("comb-p");
+        args.add("pipeline");
+        args.add("-c");
+        args.add("5");
+        args.add("--dist");
+        args.add(String.valueOf(dist));
+        args.add("--step");
+        args.add(String.valueOf(stepSize));
+        args.add("--seed");
+        args.add(String.valueOf(seed));
+        args.add("--p");
+        args.add(outputPrefix.getPath());
+        args.add(inputBed.getPath());
+
+        wrapper.executeWithDocker(args, ctx.getWorkingDirectory(), tracker, Arrays.asList(inputBed));
+
+        File outputBed = new File(outputPrefix.getPath() + ".regions.bed");
+        if (!outputBed.exists())
+        {
+            throw new PipelineJobException("Unable to find expected output: " + outputBed.getPath());
+        }
+
+        return outputBed;
+    }
 }

primeseq/src/org/labkey/primeseq/PrimeseqModule.java

@@ -1,5 +1,6 @@
 package org.labkey.primeseq.pipeline;
 
+import org.jetbrains.annotations.Nullable;
 import org.json.JSONObject;
 import org.labkey.api.data.Container;
 import org.labkey.api.module.ModuleLoader;
@@ -9,6 +10,7 @@
 import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
 import org.labkey.primeseq.PrimeseqModule;
 
+import java.io.File;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
@@ -55,30 +57,47 @@ public Collection<String> getDockerVolumes(Container c)
         Set<String> volumes = new HashSet<>();
         volumes.add("/home/groups/prime-seq");
         volumes.add("/home/exacloud/gscratch");
+        volumes.add("/mnt/scratch");
 
         PipeRoot pr = PipelineService.get().findPipelineRoot(c);
         if (pr != null && pr.getRootPath().exists())
         {
-            if (pr.getRootPath().getPath().startsWith("/home/groups/"))
-            {
-                String folderName = pr.getRootPath().getPath().replaceAll("^/home/groups/", "").split("/")[0];
-                volumes.add("/home/groups/" + folderName);
-            }
+            volumes.add(convertHomeGroups(pr.getRootPath()).getPath());
         }
 
         if (c.isWorkbook())
         {
             PipeRoot pr2 = PipelineService.get().findPipelineRoot(c.getParent());
             if (pr2 != null && pr2.getRootPath().exists())
             {
-                if (pr2.getRootPath().getPath().startsWith("/home/groups/"))
-                {
-                    String folderName = pr2.getRootPath().getPath().replaceAll("^/home/groups/", "").split("/")[0];
-                    volumes.add("/home/groups/" + folderName);
-                }
+                volumes.add(convertHomeGroups(pr2.getRootPath()).getPath());
             }
         }
 
         return volumes;
     }
+
+    private File convertHomeGroups(File input)
+    {
+        input = input.isDirectory() ? input : input.getParentFile();
+        if (input.getPath().startsWith("/home/groups/"))
+        {
+            String folderName = input.getPath().replaceAll("^/home/groups/", "").split("/")[0];
+            return new File("/home/groups/", folderName);
+        }
+
+        return input;
+    }
+
+    @Override
+    public @Nullable File inferDockerVolume(File input)
+    {
+        input = input.isDirectory() ? input : input.getParentFile();
+        if (input.getPath().startsWith("/home/exacloud/gscratch"))
+        {
+            return new File("/home/exacloud/gscratch");
+        }
+
+        return convertHomeGroups(input);
+    }
 }

primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java

@@ -595,7 +595,7 @@ private void possiblyAddQOS(PipelineJob job, RemoteExecutionEngine engine, List<
 
     private String getPartition(PipelineJob job)
     {
-        return needsGPUs(job) ? "gpu" : "exacloud";
+        return needsGPUs(job) ? "gpu" : "batch";
     }
 
     private Long getFileSize(PipelineJob job)