Let users launch NextFlow from file browser (#470)

labkey-jeckels · web-flow · commit 8b13c2a6cf6a · 2024-12-11T10:34:06.000-08:00
diff --git a/nextflow/build.gradle b/nextflow/build.gradle
@@ -6,5 +6,10 @@ plugins {
 
 dependencies {
     BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "pipeline"), depProjectConfig: "published", depExtension: "module")
-}
 
+    compileOnly "org.projectlombok:lombok:${lombokVersion}"
+    annotationProcessor "org.projectlombok:lombok:${lombokVersion}"
+
+    BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "experiment"), depProjectConfig: "published", depExtension: "module")
+    BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "pipeline"), depProjectConfig: "published", depExtension: "module")
+}
diff --git a/nextflow/src/org/labkey/nextflow/NextFlowController.java b/nextflow/src/org/labkey/nextflow/NextFlowController.java
@@ -1,5 +1,8 @@
 package org.labkey.nextflow;
 
+import lombok.Data;
+import lombok.Getter;
+import lombok.Setter;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.labkey.api.action.ApiResponse;
@@ -15,15 +18,21 @@
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineService;
 import org.labkey.api.pipeline.PipelineStatusUrls;
+import org.labkey.api.pipeline.browse.PipelinePathForm;
 import org.labkey.api.security.AdminConsoleAction;
 import org.labkey.api.security.RequiresPermission;
 import org.labkey.api.security.permissions.AdminOperationsPermission;
 import org.labkey.api.security.permissions.InsertPermission;
 import org.labkey.api.security.permissions.ReadPermission;
 import org.labkey.api.security.permissions.SiteAdminPermission;
 import org.labkey.api.util.Button;
+import org.labkey.api.util.DOM;
+import org.labkey.api.util.FileUtil;
+import org.labkey.api.util.HtmlString;
 import org.labkey.api.util.PageFlowUtil;
+import org.labkey.api.util.Path;
 import org.labkey.api.util.URLHelper;
+import org.labkey.api.util.element.Select;
 import org.labkey.api.util.logging.LogHelper;
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.view.HtmlView;
@@ -36,14 +45,24 @@
 import org.springframework.validation.Errors;
 import org.springframework.web.servlet.ModelAndView;
 
+import javax.swing.text.html.FormView;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+
 import static org.labkey.api.util.DOM.Attribute.checked;
+import static org.labkey.api.util.DOM.Attribute.hidden;
 import static org.labkey.api.util.DOM.Attribute.method;
 import static org.labkey.api.util.DOM.Attribute.name;
 import static org.labkey.api.util.DOM.Attribute.type;
 import static org.labkey.api.util.DOM.Attribute.value;
 import static org.labkey.api.util.DOM.DIV;
 import static org.labkey.api.util.DOM.INPUT;
+import static org.labkey.api.util.DOM.LI;
 import static org.labkey.api.util.DOM.LK.FORM;
+import static org.labkey.api.util.DOM.UL;
 import static org.labkey.api.util.DOM.at;
 import static org.labkey.nextflow.NextFlowManager.NEXTFLOW_CONFIG;
 
@@ -245,11 +264,18 @@ public URLHelper getSuccessURL(EnabledForm o)
         }
     }
 
+    @Getter @Setter
+    public static class AnalyzeForm extends PipelinePathForm
+    {
+        private boolean launch = false;
+        private String configFile;
+    }
+
     @RequiresPermission(AdminOperationsPermission.class)
-    public class NextFlowRunAction extends FormViewAction<Object>
+    public class NextFlowRunAction extends FormViewAction<AnalyzeForm>
     {
         @Override
-        public void validateCommand(Object o, Errors errors)
+        public void validateCommand(AnalyzeForm o, Errors errors)
         {
             if (!NextFlowManager.get().isEnabled(getContainer()))
             {
@@ -258,26 +284,69 @@ public void validateCommand(Object o, Errors errors)
         }
 
         @Override
-        public ModelAndView getView(Object o, boolean b, BindException errors)
+        public ModelAndView getView(AnalyzeForm o, boolean b, BindException errors)
         {
-            return new HtmlView("NextFlow Runner", DIV("Run NextFlow Pipeline",
-                    FORM(at(method, "POST"),
-                            new Button.ButtonBuilder("Start NextFlow").submit(true).build())));
+            NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
+            if (config.getNextFlowConfigFilePath() != null)
+            {
+                File configDir = new File(config.getNextFlowConfigFilePath());
+                if (configDir.isDirectory())
+                {
+                    File[] files = configDir.listFiles();
+                    if (files != null && files.length > 0)
+                    {
+                        List<File> configFiles = Arrays.asList(files);
+                        return new HtmlView("NextFlow Runner", DIV(
+                                FORM(at(method, "POST"),
+                                        INPUT(at(hidden, true, name, "launch", value, true)),
+                                        Arrays.stream(o.getFile()).map(f -> INPUT(at(hidden, true, name, "file", value, f))).toList(),
+                                        "Files: ",
+                                        UL(Arrays.stream(o.getFile()).map(DOM::LI)),
+                                        "Config: ",
+                                        new Select.SelectBuilder().name("configFile").addOptions(configFiles.stream().filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(),
+                                        new Button.ButtonBuilder("Start NextFlow").submit(true).build())));
+                    }
+                }
+            }
+            return new HtmlView(HtmlString.of("Couldn't find NextFlow config file(s)"));
         }
 
         @Override
-        public boolean handlePost(Object o, BindException errors) throws Exception
+        public boolean handlePost(AnalyzeForm form, BindException errors) throws Exception
         {
-            ViewBackgroundInfo info = getViewBackgroundInfo();
-            PipeRoot root = PipelineService.get().findPipelineRoot(info.getContainer());
-            PipelineJob job = new NextFlowPipelineJob(info, root);
-            PipelineService.get().queueJob(job);
+            if (!form.isLaunch())
+            {
+                return false;
+            }
+
+            NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
+            File configDir = new File(config.getNextFlowConfigFilePath());
+            File configFile = FileUtil.appendPath(configDir, Path.parse(form.getConfigFile()));
+            if (!configFile.exists())
+            {
+                errors.reject(ERROR_MSG, "Config file does not exist");
+            }
+            else
+            {
+                List<File> inputFiles = form.getValidatedFiles(getContainer());
+                if (inputFiles.isEmpty())
+                {
+                    errors.reject(ERROR_MSG, "No input files");
+                }
+                else
+                {
+                    ViewBackgroundInfo info = getViewBackgroundInfo();
+                    PipeRoot root = PipelineService.get().findPipelineRoot(info.getContainer());
+                    PipelineJob job = NextFlowPipelineJob.create(info, root, configFile.toPath(), inputFiles.stream().map(File::toPath).toList());
+                    PipelineService.get().queueJob(job);
+                }
+            }
 
             return !errors.hasErrors();
         }
 
         @Override
-        public URLHelper getSuccessURL(Object o)
+        public URLHelper getSuccessURL(AnalyzeForm o)
         {
             return PageFlowUtil.urlProvider(PipelineStatusUrls.class).urlBegin(getContainer());
         }
diff --git a/nextflow/src/org/labkey/nextflow/NextFlowManager.java b/nextflow/src/org/labkey/nextflow/NextFlowManager.java
@@ -7,6 +7,9 @@
 import org.labkey.api.data.PropertyManager;
 import org.springframework.validation.BindException;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -44,6 +47,12 @@ private void checkArgs(NextFlowConfiguration config, BindException errors)
         if (StringUtils.isEmpty(config.getNextFlowConfigFilePath()))
             errors.rejectValue("nextFlowConfigFilePath", ERROR_MSG, "NextFlow config file path is required");
 
+        Path configPath = Paths.get(config.getNextFlowConfigFilePath());
+        if (Files.isDirectory(configPath))
+        {
+            errors.rejectValue("nextFlowConfigFilePath", ERROR_MSG, "NextFlow config file path must be a directory");
+        }
+
         // Not yet used
 //        if (StringUtils.isEmpty(config.getAccountName()))
 //            errors.rejectValue("accountName", ERROR_MSG, "AWS account name is required");
diff --git a/nextflow/src/org/labkey/nextflow/NextFlowModule.java b/nextflow/src/org/labkey/nextflow/NextFlowModule.java
@@ -21,6 +21,8 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext)
     {
         ActionURL adminUrl = new ActionURL(NextFlowController.NextFlowConfigurationAction.class, ContainerManager.getRoot());
         AdminConsole.addLink(AdminConsole.SettingsLinkType.Configuration, "NextFlow Configuration", adminUrl, AdminPermission.class);
+
+        PipelineService.get().registerPipelineProvider(new NextFlowPipelineProvider(this));
     }
 
     @Override
diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java
@@ -1,44 +1,120 @@
 package org.labkey.nextflow.pipeline;
 
+import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
 import org.jetbrains.annotations.NotNull;
+import org.labkey.api.data.Container;
+import org.labkey.api.files.FileContentService;
+import org.labkey.api.pipeline.ParamParser;
 import org.labkey.api.pipeline.PipeRoot;
-import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobService;
 import org.labkey.api.pipeline.TaskId;
 import org.labkey.api.pipeline.TaskPipeline;
+import org.labkey.api.pipeline.file.AbstractFileAnalysisJob;
 import org.labkey.api.util.FileUtil;
-import org.labkey.api.util.URLHelper;
+import org.labkey.api.util.PageFlowUtil;
 import org.labkey.api.view.ViewBackgroundInfo;
 
+import java.io.BufferedWriter;
 import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
 
-public class NextFlowPipelineJob extends PipelineJob
+@Getter
+public class NextFlowPipelineJob extends AbstractFileAnalysisJob
 {
+    private Path config;
+
     @SuppressWarnings("unused") // For serialization
     protected NextFlowPipelineJob()
     {}
 
-    public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root)
+    public static NextFlowPipelineJob create(ViewBackgroundInfo info, @NotNull PipeRoot root, Path templateConfig, List<Path> inputFiles) throws IOException
     {
-        super(null, info, root);
-        setLogFile(new File(String.valueOf(root.getLogDirectory()), FileUtil.makeFileNameWithTimestamp("NextFlowPipelineJob", "log")).toPath());
+        Path parentDir = inputFiles.get(0).getParent();
+
+        String jobName = FileUtil.makeFileNameWithTimestamp("NextFlow");
+        Path jobDir = parentDir.resolve(jobName);
+        Path log = jobDir.resolve(jobName + ".log");
+        FileUtil.createDirectory(jobDir);
+
+        Path config = createConfig(templateConfig, log.getParent(), jobDir, info.getContainer());
+
+        return new NextFlowPipelineJob(info, root, config, inputFiles, log);
     }
 
-   @Override
-    public URLHelper getStatusHref()
+    public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, Path config, List<Path> inputFiles, Path log) throws IOException
     {
-        return null;
+        super(new NextFlowProtocol(), NextFlowPipelineProvider.NAME, info, root, config.getFileName().toString(), config, inputFiles, false, false);
+        this.config = config;
+        setLogFile(log);
+    }
+
+    @Override
+    public ParamParser getInputParameters()
+    {
+        return PipelineJobService.get().createParamParser();
+    }
+
+    /** Take the template config file and substitute in the values for this job */
+    private static Path createConfig(Path configTemplate, Path parentDir, Path jobDir, Container container) throws IOException
+    {
+        String template;
+        try (InputStream in = Files.newInputStream(configTemplate))
+        {
+            template = PageFlowUtil.getStreamContentsAsString(in);
+        }
+
+        String webdavUrl = FileContentService.get().getWebDavUrl(parentDir, container, FileContentService.PathType.full);
+        webdavUrl = StringUtils.stripEnd(webdavUrl, "/");
+
+        String substitutedContent = template.replace("${quant_spectra_dir}", "quant_spectra_dir = '" + webdavUrl + "'");
+
+        Path substitutedFile = jobDir.resolve(configTemplate.getFileName());
+        try (BufferedWriter writer = Files.newBufferedWriter(substitutedFile))
+        {
+            writer.write(substitutedContent);
+        }
+        return substitutedFile;
     }
 
     @Override
     public String getDescription()
     {
-        return "NextFlow Job";
+        return "NextFlow analysis using " + config.getFileName() + " of " + getInputFilePaths().size() + " files";
     }
 
     @Override
     public TaskPipeline<?> getTaskPipeline()
     {
-        return PipelineJobService.get().getTaskPipeline(new TaskId(NextFlowPipelineJob.class));
+        return PipelineJobService.get().getTaskPipeline(getTaskPipelineId());
     }
+
+    @Override
+    public TaskId getTaskPipelineId()
+    {
+        return new TaskId(NextFlowPipelineJob.class);
+    }
+
+    @Override
+    public AbstractFileAnalysisJob createSingleFileJob(File file)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public File findInputFile(String name)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public File findOutputFile(String name)
+    {
+        return null;
+    }
+
 }
diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineProvider.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineProvider.java
@@ -1,19 +1,22 @@
 package org.labkey.nextflow.pipeline;
 
-import org.labkey.api.module.Module;
 import org.labkey.api.pipeline.PipeRoot;
 import org.labkey.api.pipeline.PipelineDirectory;
 import org.labkey.api.pipeline.PipelineProvider;
 import org.labkey.api.security.permissions.InsertPermission;
 import org.labkey.api.view.ViewContext;
+import org.labkey.nextflow.NextFlowController;
 import org.labkey.nextflow.NextFlowManager;
 import org.labkey.nextflow.NextFlowModule;
 
 public class NextFlowPipelineProvider extends PipelineProvider
 {
+
+    public static final String NAME = "NextFlow";
+
     public NextFlowPipelineProvider(NextFlowModule owningModule)
     {
-        super("NextFlow", owningModule);
+        super(NAME, owningModule);
     }
 
     @Override
@@ -23,7 +26,15 @@ public void updateFileProperties(ViewContext context, PipeRoot pr, PipelineDirec
             return;
         if (!NextFlowManager.get().isEnabled(context.getContainer()))
             return;
-    }
-
 
+        String actionId = createActionId(NextFlowController.NextFlowRunAction.class, "Analyze with NextFlow");
+        addAction(actionId,
+                NextFlowController.NextFlowRunAction.class,
+                "Analyze with NextFlow",
+                directory,
+                directory.listPaths(new FileTypesEntryFilter(NextFlowProtocol.INPUT_TYPES)),
+                true,
+                true,
+                includeAll);
+    }
 }
diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowProtocol.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowProtocol.java
diff --git a/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java b/nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java
diff --git a/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml b/nextflow/webapp/WEB-INF/nextflow/nextflowContext.xml

Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,8 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext)`
`21`	`21`	`{`
`22`	`22`	`ActionURL adminUrl = new ActionURL(NextFlowController.NextFlowConfigurationAction.class, ContainerManager.getRoot());`
`23`	`23`	`AdminConsole.addLink(AdminConsole.SettingsLinkType.Configuration, "NextFlow Configuration", adminUrl, AdminPermission.class);`
	`24`	`+`
	`25`	`+ PipelineService.get().registerPipelineProvider(new NextFlowPipelineProvider(this));`
`24`	`26`	`}`
`25`	`27`
`26`	`28`	`@Override`