Signficiant refactor of docker in pipeline jobs to migrate everything to DockerWrapper and improve handling of input file locations

Author: bbimber

primeseq/resources/external/comb-p/Dockerfile

@@ -29,18 +29,19 @@
 import org.labkey.api.security.User;
 import org.labkey.api.sequenceanalysis.SequenceOutputFile;
 import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
+import org.labkey.api.sequenceanalysis.pipeline.PipelineOutputTracker;
 import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
 import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
 import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper;
 import org.labkey.api.sequenceanalysis.run.CommandWrapper;
+import org.labkey.api.sequenceanalysis.run.DockerWrapper;
 import org.labkey.api.util.FileType;
 import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.Pair;
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.writer.PrintWriters;
 import org.labkey.primeseq.PrimeseqModule;
-import org.labkey.primeseq.pipeline.CombpRunner;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -487,8 +488,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
                         Double seed = ctx.getParams().optDouble("seed", 0.05);
                         Integer step = ctx.getParams().optInt("step", 100);
 
-                        CombpRunner combp = new CombpRunner(ctx.getLogger());
-                        File outBed = combp.runCompP(finalOut, ctx.getOutputDir(), dist, seed, step);
+                        File outBed = runCompP(finalOut, ctx.getOutputDir(), dist, seed, step, ctx, ctx.getFileManager());
                         SequenceOutputFile so2 = new SequenceOutputFile();
                         so2.setName("Comb-p: " + ctx.getJob().getDescription());
                         so2.setDescription("Comb-p: " + jobDescription);
@@ -726,4 +726,36 @@ else if (i2.compareTo(il) > 0)
             }
         }
     }
+
+    private File runCompP(File inputBed, File outputDir, int dist, double seed, int stepSize, PipelineContext ctx, PipelineOutputTracker tracker) throws PipelineJobException
+    {
+        // See: https://github.com/bbimber/combpdocker
+        DockerWrapper wrapper = new DockerWrapper("bbimber/combpdocker", ctx.getLogger(), ctx);
+        File outputPrefix = new File(outputDir, FileUtil.getBaseName(inputBed) + ".combp");
+
+        List<String> args = new ArrayList<>();
+        args.add("comb-p");
+        args.add("pipeline");
+        args.add("-c");
+        args.add("5");
+        args.add("--dist");
+        args.add(String.valueOf(dist));
+        args.add("--step");
+        args.add(String.valueOf(stepSize));
+        args.add("--seed");
+        args.add(String.valueOf(seed));
+        args.add("--p");
+        args.add(outputPrefix.getPath());
+        args.add(inputBed.getPath());
+
+        wrapper.executeWithDocker(args, ctx.getWorkingDirectory(), tracker, Arrays.asList(inputBed));
+
+        File outputBed = new File(outputPrefix.getPath() + ".regions.bed");
+        if (!outputBed.exists())
+        {
+            throw new PipelineJobException("Unable to find expected output: " + outputBed.getPath());
+        }
+
+        return outputBed;
+    }
 }

primeseq/resources/external/comb-p/combpWrapper.sh

@@ -1,5 +1,6 @@
 package org.labkey.primeseq.pipeline;
 
+import org.jetbrains.annotations.Nullable;
 import org.json.JSONObject;
 import org.labkey.api.data.Container;
 import org.labkey.api.module.ModuleLoader;
@@ -9,6 +10,7 @@
 import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
 import org.labkey.primeseq.PrimeseqModule;
 
+import java.io.File;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
@@ -60,26 +62,41 @@ public Collection<String> getDockerVolumes(Container c)
         PipeRoot pr = PipelineService.get().findPipelineRoot(c);
         if (pr != null && pr.getRootPath().exists())
         {
-            if (pr.getRootPath().getPath().startsWith("/home/groups/"))
-            {
-                String folderName = pr.getRootPath().getPath().replaceAll("^/home/groups/", "").split("/")[0];
-                volumes.add("/home/groups/" + folderName);
-            }
+            volumes.add(convertHomeGroups(pr.getRootPath()).getPath());
         }
 
         if (c.isWorkbook())
         {
             PipeRoot pr2 = PipelineService.get().findPipelineRoot(c.getParent());
             if (pr2 != null && pr2.getRootPath().exists())
             {
-                if (pr2.getRootPath().getPath().startsWith("/home/groups/"))
-                {
-                    String folderName = pr2.getRootPath().getPath().replaceAll("^/home/groups/", "").split("/")[0];
-                    volumes.add("/home/groups/" + folderName);
-                }
+                volumes.add(convertHomeGroups(pr2.getRootPath()).getPath());
             }
         }
 
         return volumes;
     }
+
+    private File convertHomeGroups(File input)
+    {
+        input = input.isDirectory() ? input : input.getParentFile();
+        if (input.getPath().startsWith("/home/groups/"))
+        {
+            String folderName = input.getPath().replaceAll("^/home/groups/", "").split("/")[0];
+            return new File("/home/groups/" + folderName);
+        }
+
+        return input;
+    }
+
+    @Override
+    public @Nullable File inferDockerVolume(File input)
+    {
+        if (input.getPath().startsWith("/home/exacloud/gscratch"))
+        {
+            return new File("/home/exacloud/gscratch");
+        }
+
+        return convertHomeGroups(input);
+    }
 }