Skip to content

Commit 609752a

Browse files
labkey-jeckelslabkey-jeckels
committed
NextFlow improvements: unique job names, full file list, skip template file
1 parent 0e85609 commit 609752a

File tree

3 files changed

+49
-17
lines changed

3 files changed

+49
-17
lines changed

‎nextflow/src/org/labkey/nextflow/NextFlowController.java‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.labkey.api.data.PropertyStore;
1515
import org.labkey.api.pipeline.PipeRoot;
1616
import org.labkey.api.pipeline.PipelineJob;
17+
import org.labkey.api.pipeline.PipelineProvider;
1718
import org.labkey.api.pipeline.PipelineService;
1819
import org.labkey.api.pipeline.PipelineStatusUrls;
1920
import org.labkey.api.pipeline.browse.PipelinePathForm;
@@ -38,11 +39,13 @@
3839
import org.labkey.api.view.UnauthorizedException;
3940
import org.labkey.api.view.ViewBackgroundInfo;
4041
import org.labkey.nextflow.pipeline.NextFlowPipelineJob;
42+
import org.labkey.nextflow.pipeline.NextFlowProtocol;
4143
import org.springframework.validation.BindException;
4244
import org.springframework.validation.Errors;
4345
import org.springframework.web.servlet.ModelAndView;
4446

4547
import java.io.File;
48+
import java.nio.file.Files;
4649
import java.util.Arrays;
4750
import java.util.List;
4851

@@ -262,24 +265,39 @@ public void validateCommand(AnalyzeForm o, Errors errors)
262265
@Override
263266
public ModelAndView getView(AnalyzeForm o, boolean b, BindException errors)
264267
{
268+
List<File> selectedFiles = o.getValidatedFiles(getContainer(), false);
269+
if (selectedFiles.isEmpty())
270+
{
271+
return new HtmlView(HtmlString.of("Couldn't find input file(s)"));
272+
}
273+
// NextFlow operates on the full directory so show the list to the user, regardless of what they selected
274+
// from the file listing
275+
File inputDir = selectedFiles.get(0).getParentFile();
276+
277+
File[] inputFiles = inputDir.listFiles(new PipelineProvider.FileTypesEntryFilter(NextFlowProtocol.INPUT_TYPES));
278+
if (inputFiles == null || inputFiles.length == 0)
279+
{
280+
return new HtmlView(HtmlString.of("Couldn't find input file(s)"));
281+
}
282+
265283
NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
266284
if (config.getNextFlowConfigFilePath() != null)
267285
{
268286
File configDir = new File(config.getNextFlowConfigFilePath());
269287
if (configDir.isDirectory())
270288
{
271-
File[] files = configDir.listFiles();
272-
if (files != null && files.length > 0)
289+
File[] configFiles = configDir.listFiles();
290+
if (configFiles != null && configFiles.length > 0)
273291
{
274-
List<File> configFiles = Arrays.asList(files);
275292
return new HtmlView("NextFlow Runner", DIV(
276293
FORM(at(method, "POST"),
277294
INPUT(at(hidden, true, name, "launch", value, true)),
278295
Arrays.stream(o.getFile()).map(f -> INPUT(at(hidden, true, name, "file", value, f))).toList(),
279296
"Files: ",
280-
UL(Arrays.stream(o.getFile()).map(DOM::LI)),
297+
UL(Arrays.stream(inputFiles).map(File::getName).map(DOM::LI)),
281298
"Config: ",
282-
new Select.SelectBuilder().name("configFile").addOptions(configFiles.stream().filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(),
299+
new Select.SelectBuilder().name("configFile").addOptions(Arrays.stream(configFiles).filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(),
300+
DOM.BR(),
283301
new Button.ButtonBuilder("Start NextFlow").submit(true).build())));
284302
}
285303
}

‎nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,24 +59,26 @@ public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, Path
5959
super(new NextFlowProtocol(), NextFlowPipelineProvider.NAME, info, root, config.getFileName().toString(), config, inputFiles, false, false);
6060
this.config = config;
6161
setLogFile(log);
62-
LOG.info("NextFlow job queued: {}", getJsonJobInfo());
62+
LOG.info("NextFlow job queued: {}", getJsonJobInfo(null));
6363
}
6464

65-
protected JSONObject getJsonJobInfo()
65+
protected JSONObject getJsonJobInfo(Long invocationCount)
6666
{
6767
JSONObject result = new JSONObject();
6868
result.put("user", getUser().getEmail());
6969
result.put("container", getContainer().getPath());
7070
result.put("filePath", getLogFilePath().getParent().toString());
71-
result.put("runName", getNextFlowRunName());
71+
result.put("runName", getNextFlowRunName(invocationCount));
7272
result.put("configFile", getConfig().getFileName().toString());
7373
return result;
7474
}
7575

76-
protected String getNextFlowRunName()
76+
protected String getNextFlowRunName(Long invocationCount)
7777
{
7878
PipelineStatusFile file = PipelineService.get().getStatusFile(getJobGUID());
79-
return file == null ? "Unknown" : ("LabKeyJob" + file.getRowId());
79+
String result = file == null ? "Unknown" : ("LabKeyJob" + file.getRowId());
80+
result += invocationCount == null ? "" : ("_" + invocationCount);
81+
return result;
8082
}
8183

8284
@Override

‎nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
import org.apache.logging.log4j.Logger;
44
import org.jetbrains.annotations.NotNull;
5+
import org.labkey.api.data.ContainerManager;
6+
import org.labkey.api.data.DbSequence;
7+
import org.labkey.api.data.DbSequenceManager;
58
import org.labkey.api.exp.XarFormatException;
69
import org.labkey.api.pipeline.AbstractTaskFactory;
710
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
@@ -37,6 +40,8 @@ public class NextFlowRunTask extends WorkDirectoryTask<NextFlowRunTask.Factory>
3740

3841
public static final String ACTION_NAME = "NextFlow";
3942

43+
private static final DbSequence INVOCATION_SEQUENCE = DbSequenceManager.get(ContainerManager.getRoot(), NextFlowRunTask.class.getName());
44+
4045
public NextFlowRunTask(Factory factory, PipelineJob job)
4146
{
4247
super(factory, job);
@@ -46,7 +51,12 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
4651
public @NotNull RecordedActionSet run() throws PipelineJobException
4752
{
4853
Logger log = getJob().getLogger();
49-
NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo());
54+
55+
// NextFlow requires a unique job name for every execution. Increment a counter to append as a suffix to
56+
// ensure uniqueness
57+
long invocationCount = INVOCATION_SEQUENCE.next();
58+
INVOCATION_SEQUENCE.sync();
59+
NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
5060

5161
SecurityManager.TransformSession session = null;
5262
boolean success = false;
@@ -73,10 +83,10 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
7383
File dir = getJob().getLogFile().getParentFile();
7484
getJob().runSubProcess(secretsPB, dir);
7585

76-
ProcessBuilder executionPB = new ProcessBuilder(getArgs());
86+
ProcessBuilder executionPB = new ProcessBuilder(getArgs(invocationCount));
7787
getJob().runSubProcess(executionPB, dir);
7888
log.info("Job Finished");
79-
NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo());
89+
NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
8090

8191
RecordedAction action = new RecordedAction(ACTION_NAME);
8292
for (Path inputFile : getJob().getInputFilePaths())
@@ -100,14 +110,16 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
100110
}
101111
if (!success)
102112
{
103-
NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo());
113+
NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
104114
}
105115
}
106116
}
107117

108118
private void addOutputs(RecordedAction action, Path path, Logger log) throws IOException
109119
{
110-
if (Files.isRegularFile(path))
120+
// Skip results.sky.zip files - it's the template document. We want the file output doc that includes
121+
// the replicate analysis
122+
if (Files.isRegularFile(path) && !path.endsWith("results.sky.zip"))
111123
{
112124
action.addOutput(path.toFile(), "Output", false);
113125
if (path.toString().toLowerCase().endsWith(".sky.zip"))
@@ -164,7 +176,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
164176
}
165177

166178

167-
private @NotNull List<String> getArgs() throws PipelineJobException
179+
private @NotNull List<String> getArgs(long invocationCount) throws PipelineJobException
168180
{
169181
NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
170182
Path configFile = getJob().getConfig();
@@ -189,7 +201,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
189201
args.add("-c");
190202
args.add(configFile.toAbsolutePath().toString());
191203
args.add("-name");
192-
args.add(getJob().getNextFlowRunName());
204+
args.add(getJob().getNextFlowRunName(invocationCount));
193205
return args;
194206
}
195207

0 commit comments

Comments
 (0)