Skip to content

Commit dfae19f

Browse files
bbimberbbimber
committed
Refactor nimble to use DockerWrapper
1 parent bf2cb04 commit dfae19f

File tree

4 files changed

+81
-139
lines changed

4 files changed

+81
-139
lines changed

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ abstract public class AbstractCommandWrapper implements CommandWrapper
4848
private boolean _warnNonZeroExits = true;
4949
private boolean _throwNonZeroExits = true;
5050
private Integer _lastReturnCode = null;
51-
private final Map<String, String> _environment = new HashMap<>();
51+
protected final Map<String, String> _environment = new HashMap<>();
5252
private final List<String> _commandsExecuted = new ArrayList<>();
5353

5454
public AbstractCommandWrapper(@Nullable Logger logger)

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/DockerWrapper.java‎

Lines changed: 56 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,27 @@ public class DockerWrapper extends AbstractCommandWrapper
2020
private final String _containerName;
2121
private final PipelineContext _ctx;
2222
private File _tmpDir = null;
23+
private String _entryPoint = null;
2324

2425
public DockerWrapper(String containerName, Logger log, PipelineContext ctx)
2526
{
2627
super(log);
2728
_containerName = containerName;
2829
_ctx = ctx;
30+
31+
_environment.clear();
2932
}
3033

3134
public void setTmpDir(File tmpDir)
3235
{
3336
_tmpDir = tmpDir;
3437
}
3538

39+
public void setEntryPoint(String entryPoint)
40+
{
41+
_entryPoint = entryPoint;
42+
}
43+
3644
public void executeWithDocker(List<String> containerArgs, File workDir, PipelineOutputTracker tracker) throws PipelineJobException
3745
{
3846
File localBashScript = new File(workDir, "docker.sh");
@@ -46,37 +54,72 @@ public void executeWithDocker(List<String> containerArgs, File workDir, Pipeline
4654
writer.println("#!/bin/bash");
4755
writer.println("set -x");
4856
writer.println("WD=`pwd`");
49-
writer.println("HOME=`echo ~/`");
57+
5058
writer.println("DOCKER='" + SequencePipelineService.get().getDockerCommand() + "'");
5159
writer.println("$DOCKER pull " + _containerName);
5260
writer.println("$DOCKER run --rm=true \\");
53-
writer.println("\t-v \"${WD}:/work\" \\");
54-
writer.println("\t-v \"${HOME}:/homeDir\" \\");
61+
62+
// NOTE: getDockerVolumes() should be refactored to remove the -v and this logic should be updated accordingly:
63+
File homeDir = new File(System.getProperty("user.home"));
64+
if (homeDir.exists())
65+
{
66+
final String searchString = "-v '" + homeDir.getPath() + "'";
67+
if (_ctx.getDockerVolumes().stream().noneMatch(searchString::startsWith))
68+
{
69+
writer.println("\t-v \"" + homeDir.getPath() + ":/homeDir\" \\");
70+
}
71+
else
72+
{
73+
_ctx.getLogger().debug("homeDir already present in docker volumes, omitting");
74+
}
75+
76+
_environment.put("USER_HOME", homeDir.getPath());
77+
}
78+
5579
_ctx.getDockerVolumes().forEach(ln -> writer.println(ln + " \\"));
5680
if (_tmpDir != null)
5781
{
58-
writer.println("\t-v \"" + _tmpDir.getPath() + ":/tmp\" \\");
82+
// NOTE: getDockerVolumes() should be refactored to remove the -v and this logic should be updated accordingly:
83+
final String searchString = "-v '" + _tmpDir.getPath() + "'";
84+
if (_ctx.getDockerVolumes().stream().noneMatch(searchString::startsWith))
85+
{
86+
writer.println("\t-v \"" + _tmpDir.getPath() + ":/tmp\" \\");
87+
}
88+
else
89+
{
90+
_ctx.getLogger().debug("tmpDir already present in docker volumes, omitting");
91+
}
92+
}
93+
94+
if (_entryPoint != null)
95+
{
96+
writer.println("\t--entrypoint \"" + _entryPoint + "\"\\");
5997
}
60-
writer.println("\t--entrypoint /bin/bash \\");
61-
writer.println("\t-w /work \\");
98+
99+
writer.println("\t-w " + workDir.getPath() + " \\");
62100
Integer maxRam = SequencePipelineService.get().getMaxRam();
63101
if (maxRam != null)
64102
{
65103
writer.println("\t-e SEQUENCEANALYSIS_MAX_RAM=" + maxRam + " \\");
66104
writer.println("\t--memory='" + maxRam + "g' \\");
67105
}
106+
107+
for (String key : _environment.keySet())
108+
{
109+
writer.println("\t-e " + key + "=" + _environment.get(key) + " \\");
110+
}
68111
writer.println("\t" + _containerName + " \\");
69-
writer.println("\t/work/" + dockerBashScript.getName());
70-
writer.println("EXIT_CODE=$?");
71-
writer.println("echo 'Docker run exit code: '$EXIT_CODE");
72-
writer.println("exit $EXIT_CODE");
112+
writer.println("\t" + workDir.getPath() + "/" + dockerBashScript.getName());
113+
writer.println("DOCKER_EXIT_CODE=$?");
114+
writer.println("echo 'Docker run exit code: '$DOCKER_EXIT_CODE");
115+
writer.println("exit $DOCKER_EXIT_CODE");
73116

74117
dockerWriter.println("#!/bin/bash");
75118
dockerWriter.println("set -x");
76119
dockerWriter.println(StringUtils.join(containerArgs, " "));
77-
dockerWriter.println("EXIT_CODE=$?");
78-
dockerWriter.println("echo 'Exit code: '$?");
79-
dockerWriter.println("exit $EXIT_CODE");
120+
dockerWriter.println("BASH_EXIT_CODE=$?");
121+
dockerWriter.println("echo 'Bash exit code: '$BASH_EXIT_CODE");
122+
dockerWriter.println("exit $BASH_EXIT_CODE");
80123
}
81124
catch (IOException e)
82125
{

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/run/alignment/ParagraphStep.java‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ else if (!svVcf.exists())
219219
}
220220

221221
double readLength = json.getInt("read_length");
222-
writer.println(rgId + "\t" + "/work/" + so.getFile().getName() + "\t" + depth + "\t" + readLength);
222+
writer.println(rgId + "\t" + so.getFile().getPath() + "\t" + depth + "\t" + readLength);
223223
}
224224
catch (IOException e)
225225
{
@@ -232,12 +232,9 @@ else if (!svVcf.exists())
232232
List<String> paragraphArgs = new ArrayList<>();
233233
paragraphArgs.add("/opt/paragraph/bin/multigrmpy.py");
234234

235-
dockerWrapper.ensureLocalCopy(so.getFile(), ctx.getWorkingDirectory(), ctx.getFileManager());
236-
dockerWrapper.ensureLocalCopy(SequenceAnalysisService.get().getExpectedBamOrCramIndex(so.getFile()), ctx.getWorkingDirectory(), ctx.getFileManager());
237-
238235
File paragraphOutDir = new File(ctx.getWorkingDirectory(), FileUtil.getBaseName(so.getFile()));
239236
paragraphArgs.add("-o");
240-
paragraphArgs.add("/work/" + paragraphOutDir.getName());
237+
paragraphArgs.add(paragraphOutDir.getPath());
241238

242239
File scratchDir = new File(ctx.getOutputDir(), "pgScratch");
243240
if (scratchDir.exists())
@@ -253,17 +250,15 @@ else if (!svVcf.exists())
253250
}
254251

255252
paragraphArgs.add("--scratch-dir");
256-
paragraphArgs.add("/work/" + scratchDir.getName());
253+
paragraphArgs.add(scratchDir.getPath());
257254

258255
ctx.getFileManager().addIntermediateFile(scratchDir);
259256

260257
paragraphArgs.add("-i");
261-
dockerWrapper.ensureLocalCopy(svVcf, ctx.getWorkingDirectory(), ctx.getFileManager());
262-
dockerWrapper.ensureLocalCopy(new File(svVcf.getPath() + ".tbi"), ctx.getWorkingDirectory(), ctx.getFileManager());
263-
paragraphArgs.add("/work/" + svVcf.getName());
258+
paragraphArgs.add(svVcf.getPath());
264259

265260
paragraphArgs.add("-m");
266-
paragraphArgs.add("/work/" + coverageFile.getName());
261+
paragraphArgs.add(coverageFile.getPath());
267262

268263
if (ctx.getParams().optBoolean("verbose", false))
269264
{
@@ -272,9 +267,7 @@ else if (!svVcf.exists())
272267

273268
paragraphArgs.add("-r");
274269
File genomeFasta = ctx.getSequenceSupport().getCachedGenome(so.getLibrary_id()).getWorkingFastaFile();
275-
dockerWrapper.ensureLocalCopy(genomeFasta, ctx.getWorkingDirectory(), ctx.getFileManager());
276-
dockerWrapper.ensureLocalCopy(new File(genomeFasta.getPath() + ".fai"), ctx.getWorkingDirectory(), ctx.getFileManager());
277-
paragraphArgs.add("/work/" + genomeFasta.getName());
270+
paragraphArgs.add(genomeFasta.getPath());
278271

279272
if (ctx.getParams().optBoolean("retrieveReferenceSeq", false))
280273
{

0 commit comments

Comments
 (0)