Create standalone lucene index step

Author: bbimber

jbrowse/src/org/labkey/jbrowse/JBrowseModule.java

@@ -28,6 +28,7 @@
 import org.labkey.api.query.DetailsURL;
 import org.labkey.api.security.permissions.AdminOperationsPermission;
 import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
+import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.settings.AdminConsole;
 import org.labkey.api.util.PageFlowUtil;
 import org.labkey.api.util.SystemMaintenance;
@@ -38,6 +39,7 @@
 import org.labkey.jbrowse.button.ModifyTrackConfigButton;
 import org.labkey.jbrowse.button.ReprocessResourcesButton;
 import org.labkey.jbrowse.button.ReprocessSessionsButton;
+import org.labkey.jbrowse.pipeline.IndexVariantsStep;
 import org.labkey.jbrowse.pipeline.JBrowseSessionPipelineProvider;
 import org.labkey.jbrowse.query.JBrowseUserSchema;
 
@@ -108,6 +110,8 @@ public void doStartupAfterSpringConfig(ModuleContext moduleContext)
 
         WebdavService.get().registerPreGzippedExtensions("jsonz");
         WebdavService.get().registerPreGzippedExtensions("txtz");
+
+        SequencePipelineService.get().registerPipelineStep(new IndexVariantsStep.Provider());
     }
 
     @Override

jbrowse/src/org/labkey/jbrowse/pipeline/IndexVariantsStep.java

@@ -0,0 +1,79 @@
+package org.labkey.jbrowse.pipeline;
+
+import htsjdk.samtools.util.Interval;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.SystemUtils;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.sequenceanalysis.pipeline.AbstractVariantProcessingStepProvider;
+import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
+import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider;
+import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
+import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
+import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStep;
+import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStepOutputImpl;
+import org.labkey.api.sequenceanalysis.run.AbstractCommandPipelineStep;
+import org.labkey.api.sequenceanalysis.run.SelectVariantsWrapper;
+import org.labkey.api.writer.PrintWriters;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.List;
+
+public class IndexVariantsStep extends AbstractCommandPipelineStep<SelectVariantsWrapper> implements VariantProcessingStep
+{
+    public static final String CATEGORY = "VCF Lucene Index";
+
+    public IndexVariantsStep(PipelineStepProvider<?> provider, PipelineContext ctx)
+    {
+        super(provider, ctx, new SelectVariantsWrapper(ctx.getLogger()));
+    }
+
+    public static class Provider extends AbstractVariantProcessingStepProvider<IndexVariantsStep> implements VariantProcessingStep.SupportsScatterGather
+    {
+        public Provider()
+        {
+            super("IndexVariantsStep", "Index Variants", "DISCVR-seq", "Create a lucene index for the selected fields", Arrays.asList(
+                    ToolParameterDescriptor.create("infoFieldsToIndex", "INFO fields to index", "A list of INFO fields to index", "sequenceanalysis-trimmingtextarea", null, null),
+                    ToolParameterDescriptor.create("allowLenientProcessing", "Allow Lenient Processing", "If selected, many error types will be logged but ignored.", "checkbox", null, false)
+            ), Arrays.asList("/sequenceanalysis/field/TrimmingTextArea.js"), "https://github.com/BimberLab/DISCVRSeq");
+        }
+
+        @Override
+        public IndexVariantsStep create(PipelineContext ctx)
+        {
+            return new IndexVariantsStep(this, ctx);
+        }
+    }
+
+    @Override
+    public Output processVariants(File inputVCF, File outputDirectory, ReferenceGenome genome, @Nullable List<Interval> intervals) throws PipelineJobException
+    {
+        VariantProcessingStepOutputImpl output = new VariantProcessingStepOutputImpl();
+
+        String infoFieldsRaw = StringUtils.trimToNull(getProvider().getParameterByName("infoFieldsToIndex").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class));
+        if (infoFieldsRaw == null)
+        {
+            throw new PipelineJobException("Missing info fields to index");
+        }
+
+        List<String> infoFields = Arrays.stream(infoFieldsRaw.split(";")).sorted().toList();
+        boolean allowLenientProcessing = getProvider().getParameterByName("allowLenientProcessing").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false);
+
+        File indexDir = new File(outputDirectory, "lucene");
+        JBrowseLucenePipelineJob.prepareLuceneIndex(inputVCF, indexDir, getPipelineCtx().getLogger(), infoFields, allowLenientProcessing);
+
+        File idx = new File(indexDir, "write.lock");
+        if (!idx.exists())
+        {
+            throw new PipelineJobException("Unable to find file: " + idx.getPath());
+        }
+
+        output.addSequenceOutput(idx, "Lucene index: " + inputVCF.getName(), CATEGORY, null, null, genome.getGenomeId(), "Fields indexed: " + infoFieldsRaw);
+
+        return output;
+    }
+}

jbrowse/src/org/labkey/jbrowse/pipeline/JBrowseLucenePipelineJob.java

@@ -1,6 +1,7 @@
 package org.labkey.jbrowse.pipeline;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.logging.log4j.Logger;
 import org.labkey.api.assay.AssayFileWriter;
 import org.labkey.api.data.Container;
@@ -22,10 +23,12 @@
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.view.ViewBackgroundInfo;
 import org.labkey.api.view.ViewContext;
+import org.labkey.api.writer.PrintWriters;
 import org.labkey.jbrowse.JBrowseManager;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.util.List;
 
 /**
@@ -168,6 +171,7 @@ public static void prepareLuceneIndex(File vcf, File indexDir, Logger log, List<
             args.add("LENIENT");
         }
 
+        infoFieldsForFullTextSearch = infoFieldsForFullTextSearch.stream().sorted().toList();
         for (String field : infoFieldsForFullTextSearch)
         {
             args.add("-IF");
@@ -193,10 +197,59 @@ public static void prepareLuceneIndex(File vcf, File indexDir, Logger log, List<
         }
 
         runner.execute(args);
+
+        if (!SystemUtils.IS_OS_WINDOWS)
+        {
+            try
+            {
+                log.debug("Updating file permissions");
+                recursivelyChangeDirectoryPermissions(indexDir);
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException(e);
+            }
+        }
+
+        File fieldFile = getFieldListFile(indexDir);
+        try (PrintWriter writer = PrintWriters.getPrintWriter(fieldFile))
+        {
+            infoFieldsForFullTextSearch.forEach(writer::println);
+        }
+        catch (IOException e)
+        {
+            throw new PipelineJobException(e);
+        }
     }
 
     public static File getExpectedLocationOfLuceneIndexStats(File indexDir)
     {
         return new File(indexDir.getPath() + ".stats.txt");
     }
+
+    private static void recursivelyChangeDirectoryPermissions(File f) throws IOException
+    {
+        if (f.isDirectory())
+        {
+            Runtime.getRuntime().exec(new String[]{"chmod", "775", f.getPath()});
+
+            File[] children = f.listFiles();
+            if (children != null)
+            {
+                for (File child : children)
+                {
+                    recursivelyChangeDirectoryPermissions(child);
+                }
+            }
+        }
+        else
+        {
+            Runtime.getRuntime().exec(new String[]{"chmod", "664", f.getPath()});
+        }
+    }
+
+    public static File getFieldListFile(File indexDir)
+    {
+        return new File(indexDir, "fieldList.txt");
+    }
 }