Skip to content

Commit e8efa5a

Browse files
labkey-jeckelslabkey-jeckels
authored
NextFlow improvements: unique job names, full file list, skip template file (#508)
1 parent 0e85609 commit e8efa5a

File tree

3 files changed

+48
-21
lines changed

3 files changed

+48
-21
lines changed

‎nextflow/src/org/labkey/nextflow/NextFlowController.java‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import lombok.Getter;
44
import lombok.Setter;
55
import org.apache.commons.lang3.StringUtils;
6-
import org.apache.logging.log4j.Logger;
76
import org.labkey.api.action.ApiResponse;
87
import org.labkey.api.action.ApiSimpleResponse;
98
import org.labkey.api.action.FormViewAction;
@@ -14,6 +13,7 @@
1413
import org.labkey.api.data.PropertyStore;
1514
import org.labkey.api.pipeline.PipeRoot;
1615
import org.labkey.api.pipeline.PipelineJob;
16+
import org.labkey.api.pipeline.PipelineProvider;
1717
import org.labkey.api.pipeline.PipelineService;
1818
import org.labkey.api.pipeline.PipelineStatusUrls;
1919
import org.labkey.api.pipeline.browse.PipelinePathForm;
@@ -31,13 +31,13 @@
3131
import org.labkey.api.util.Path;
3232
import org.labkey.api.util.URLHelper;
3333
import org.labkey.api.util.element.Select;
34-
import org.labkey.api.util.logging.LogHelper;
3534
import org.labkey.api.view.HtmlView;
3635
import org.labkey.api.view.JspView;
3736
import org.labkey.api.view.NavTree;
3837
import org.labkey.api.view.UnauthorizedException;
3938
import org.labkey.api.view.ViewBackgroundInfo;
4039
import org.labkey.nextflow.pipeline.NextFlowPipelineJob;
40+
import org.labkey.nextflow.pipeline.NextFlowProtocol;
4141
import org.springframework.validation.BindException;
4242
import org.springframework.validation.Errors;
4343
import org.springframework.web.servlet.ModelAndView;
@@ -64,8 +64,6 @@ public class NextFlowController extends SpringActionController
6464
private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(NextFlowController.class);
6565
public static final String NAME = "nextflow";
6666

67-
private static final Logger LOG = LogHelper.getLogger(NextFlowController.class, NAME);
68-
6967
public NextFlowController()
7068
{
7169
setActionResolver(_actionResolver);
@@ -262,24 +260,39 @@ public void validateCommand(AnalyzeForm o, Errors errors)
262260
@Override
263261
public ModelAndView getView(AnalyzeForm o, boolean b, BindException errors)
264262
{
263+
List<File> selectedFiles = o.getValidatedFiles(getContainer(), false);
264+
if (selectedFiles.isEmpty())
265+
{
266+
return new HtmlView(HtmlString.of("Couldn't find input file(s)"));
267+
}
268+
// NextFlow operates on the full directory so show the list to the user, regardless of what they selected
269+
// from the file listing
270+
File inputDir = selectedFiles.get(0).getParentFile();
271+
272+
File[] inputFiles = inputDir.listFiles(new PipelineProvider.FileTypesEntryFilter(NextFlowProtocol.INPUT_TYPES));
273+
if (inputFiles == null || inputFiles.length == 0)
274+
{
275+
return new HtmlView(HtmlString.of("Couldn't find input file(s)"));
276+
}
277+
265278
NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
266279
if (config.getNextFlowConfigFilePath() != null)
267280
{
268281
File configDir = new File(config.getNextFlowConfigFilePath());
269282
if (configDir.isDirectory())
270283
{
271-
File[] files = configDir.listFiles();
272-
if (files != null && files.length > 0)
284+
File[] configFiles = configDir.listFiles();
285+
if (configFiles != null && configFiles.length > 0)
273286
{
274-
List<File> configFiles = Arrays.asList(files);
275287
return new HtmlView("NextFlow Runner", DIV(
276288
FORM(at(method, "POST"),
277289
INPUT(at(hidden, true, name, "launch", value, true)),
278290
Arrays.stream(o.getFile()).map(f -> INPUT(at(hidden, true, name, "file", value, f))).toList(),
279291
"Files: ",
280-
UL(Arrays.stream(o.getFile()).map(DOM::LI)),
292+
UL(Arrays.stream(inputFiles).map(File::getName).map(DOM::LI)),
281293
"Config: ",
282-
new Select.SelectBuilder().name("configFile").addOptions(configFiles.stream().filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(),
294+
new Select.SelectBuilder().name("configFile").addOptions(Arrays.stream(configFiles).filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(),
295+
DOM.BR(),
283296
new Button.ButtonBuilder("Start NextFlow").submit(true).build())));
284297
}
285298
}

‎nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,24 +59,26 @@ public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, Path
5959
super(new NextFlowProtocol(), NextFlowPipelineProvider.NAME, info, root, config.getFileName().toString(), config, inputFiles, false, false);
6060
this.config = config;
6161
setLogFile(log);
62-
LOG.info("NextFlow job queued: {}", getJsonJobInfo());
62+
LOG.info("NextFlow job queued: {}", getJsonJobInfo(null));
6363
}
6464

65-
protected JSONObject getJsonJobInfo()
65+
protected JSONObject getJsonJobInfo(Long invocationCount)
6666
{
6767
JSONObject result = new JSONObject();
6868
result.put("user", getUser().getEmail());
6969
result.put("container", getContainer().getPath());
7070
result.put("filePath", getLogFilePath().getParent().toString());
71-
result.put("runName", getNextFlowRunName());
71+
result.put("runName", getNextFlowRunName(invocationCount));
7272
result.put("configFile", getConfig().getFileName().toString());
7373
return result;
7474
}
7575

76-
protected String getNextFlowRunName()
76+
protected String getNextFlowRunName(Long invocationCount)
7777
{
7878
PipelineStatusFile file = PipelineService.get().getStatusFile(getJobGUID());
79-
return file == null ? "Unknown" : ("LabKeyJob" + file.getRowId());
79+
String result = file == null ? "Unknown" : ("LabKeyJob" + file.getRowId());
80+
result += invocationCount == null ? "" : ("_" + invocationCount);
81+
return result;
8082
}
8183

8284
@Override

‎nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
import org.apache.logging.log4j.Logger;
44
import org.jetbrains.annotations.NotNull;
5+
import org.labkey.api.data.ContainerManager;
6+
import org.labkey.api.data.DbSequence;
7+
import org.labkey.api.data.DbSequenceManager;
58
import org.labkey.api.exp.XarFormatException;
69
import org.labkey.api.pipeline.AbstractTaskFactory;
710
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
@@ -37,6 +40,8 @@ public class NextFlowRunTask extends WorkDirectoryTask<NextFlowRunTask.Factory>
3740

3841
public static final String ACTION_NAME = "NextFlow";
3942

43+
private static final DbSequence INVOCATION_SEQUENCE = DbSequenceManager.get(ContainerManager.getRoot(), NextFlowRunTask.class.getName());
44+
4045
public NextFlowRunTask(Factory factory, PipelineJob job)
4146
{
4247
super(factory, job);
@@ -46,7 +51,12 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
4651
public @NotNull RecordedActionSet run() throws PipelineJobException
4752
{
4853
Logger log = getJob().getLogger();
49-
NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo());
54+
55+
// NextFlow requires a unique job name for every execution. Increment a counter to append as a suffix to
56+
// ensure uniqueness
57+
long invocationCount = INVOCATION_SEQUENCE.next();
58+
INVOCATION_SEQUENCE.sync();
59+
NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
5060

5161
SecurityManager.TransformSession session = null;
5262
boolean success = false;
@@ -73,10 +83,10 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
7383
File dir = getJob().getLogFile().getParentFile();
7484
getJob().runSubProcess(secretsPB, dir);
7585

76-
ProcessBuilder executionPB = new ProcessBuilder(getArgs());
86+
ProcessBuilder executionPB = new ProcessBuilder(getArgs(invocationCount));
7787
getJob().runSubProcess(executionPB, dir);
7888
log.info("Job Finished");
79-
NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo());
89+
NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
8090

8191
RecordedAction action = new RecordedAction(ACTION_NAME);
8292
for (Path inputFile : getJob().getInputFilePaths())
@@ -100,14 +110,16 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
100110
}
101111
if (!success)
102112
{
103-
NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo());
113+
NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
104114
}
105115
}
106116
}
107117

108118
private void addOutputs(RecordedAction action, Path path, Logger log) throws IOException
109119
{
110-
if (Files.isRegularFile(path))
120+
// Skip results.sky.zip files - it's the template document. We want the file output doc that includes
121+
// the replicate analysis
122+
if (Files.isRegularFile(path) && !path.endsWith("results.sky.zip"))
111123
{
112124
action.addOutput(path.toFile(), "Output", false);
113125
if (path.toString().toLowerCase().endsWith(".sky.zip"))
@@ -164,7 +176,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
164176
}
165177

166178

167-
private @NotNull List<String> getArgs() throws PipelineJobException
179+
private @NotNull List<String> getArgs(long invocationCount) throws PipelineJobException
168180
{
169181
NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
170182
Path configFile = getJob().getConfig();
@@ -189,7 +201,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
189201
args.add("-c");
190202
args.add(configFile.toAbsolutePath().toString());
191203
args.add("-name");
192-
args.add(getJob().getNextFlowRunName());
204+
args.add(getJob().getNextFlowRunName(invocationCount));
193205
return args;
194206
}
195207

0 commit comments

Comments
 (0)