Merge pull request #270 from BimberLab/24.3_fb_merge

Merge discvr-23.11 to discvr-24.3

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceAnalysisService.java

@@ -19,6 +19,7 @@
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.data.Container;
+import org.labkey.api.laboratory.DemographicsProvider;
 import org.labkey.api.laboratory.NavItem;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobException;
@@ -76,6 +77,10 @@ static public void setInstance(SequenceAnalysisService instance)
 
     abstract public File ensureVcfIndex(File vcf, Logger log, boolean forceRecreate) throws IOException;
 
+    abstract public File ensureBamOrCramIdx(File bamOrCram, Logger log, boolean forceRecreate) throws PipelineJobException;
+
+    abstract public File getExpectedBamOrCramIndex(File bamOrCram);
+
     abstract public File bgzipFile(File input, Logger log) throws PipelineJobException;
 
     abstract public void ensureFastaIndex(File fasta, Logger log) throws PipelineJobException;
@@ -84,7 +89,7 @@ static public void setInstance(SequenceAnalysisService instance)
 
     abstract public Integer getExpRunIdForJob(PipelineJob job, boolean throwUnlessFound) throws PipelineJobException;
 
-    abstract public List<PedigreeRecord> generatePedigree(Collection<String> sampleNames, Container c, User u);
+    abstract public List<PedigreeRecord> generatePedigree(Collection<String> sampleNames, Container c, User u, DemographicsProvider d);
 
     abstract public String getVCFLineCount(File vcf, Logger log, boolean passOnly) throws PipelineJobException;
 

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AbstractAlignmentStepProvider.java

@@ -28,7 +28,7 @@
  * Date: 6/14/2014
  * Time: 2:44 PM
  */
-abstract public class AbstractAlignmentStepProvider<StepType extends AlignmentStep> extends AbstractPipelineStepProvider<StepType> implements AlignmentStepProvider
+abstract public class AbstractAlignmentStepProvider<StepType extends AlignmentStep> extends AbstractPipelineStepProvider<StepType> implements AlignmentStepProvider<StepType>
 {
     public static String ALIGNMENT_MODE_PARAM = "alignmentMode";
     public static String SUPPORT_MERGED_UNALIGNED = "supportsMergeUnaligned";

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AbstractPipelineStepProvider.java

@@ -31,7 +31,7 @@
  * Date: 6/14/2014
  * Time: 12:37 PM
  */
-abstract public class AbstractPipelineStepProvider<StepType extends PipelineStep> implements PipelineStepProvider
+abstract public class AbstractPipelineStepProvider<StepType extends PipelineStep> implements PipelineStepProvider<StepType>
 {
     private final String _name;
     private final String _label;
@@ -140,12 +140,11 @@ public JSONObject toJSON()
     @Override
     public Class<StepType> getStepClass()
     {
-        ParameterizedType parameterizedType = (ParameterizedType) getClass().getGenericSuperclass();
-        return (Class) parameterizedType.getActualTypeArguments()[0];
+        return (Class<StepType>) SequencePipelineService.get().findSuperClassParameterType(this);
     }
 
     @Override
-    public PipelineStepProvider<StepType> combineSteps(int existingStepIdx, PipelineStepCtx toCombine)
+    public PipelineStepProvider<StepType> combineSteps(int existingStepIdx, PipelineStepCtx<StepType> toCombine)
     {
         return null;
     }

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignmentStepProvider.java

@@ -1,6 +1,6 @@
 package org.labkey.api.sequenceanalysis.pipeline;
 
-public interface AlignmentStepProvider extends PipelineStepProvider
+public interface AlignmentStepProvider<StepType extends PipelineStep> extends PipelineStepProvider<StepType>
 {
     default boolean shouldRunIdxstats()
     {

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PedigreeToolParameterDescriptor.java

@@ -0,0 +1,20 @@
+package org.labkey.api.sequenceanalysis.pipeline;
+
+import org.json.JSONObject;
+
+public class PedigreeToolParameterDescriptor extends ToolParameterDescriptor
+{
+    public static String NAME = "pedigreeSource";
+
+    public PedigreeToolParameterDescriptor()
+    {
+        super(null, NAME, "Pedigree Source", "This is the table used for pedigree data", "laboratory-pedigreeselectorfield", "laboratory.subjects", new JSONObject(){{
+            put("allowBlank", false);
+        }});
+    }
+
+    public static String getClientDependencyPath()
+    {
+        return "/laboratory/field/PedigreeSelectorField.js";
+    }
+}

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PipelineStepProvider.java

@@ -94,5 +94,5 @@ default boolean hasParameter(String name)
      * Allows a given step to combine itself w/ a neighboring step to save compute time.  Should return a new provider, which will
      * replace both original provider.  Return null for no changes.
      */
-    PipelineStepProvider<StepType> combineSteps(int existingStepIdx, PipelineStepCtx toCombine);
+    PipelineStepProvider<StepType> combineSteps(int existingStepIdx, PipelineStepCtx<StepType> toCombine);
 }

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SamtoolsCramConverter.java

@@ -57,7 +57,7 @@ public File convert(File inputBam, File outputCram, File gzippedFasta, boolean d
         return outputCram;
     }
 
-    private void doIndex(File input, @Nullable Integer threads) throws PipelineJobException
+    public File doIndex(File input, @Nullable Integer threads) throws PipelineJobException
     {
         List<String> params = new ArrayList<>();
         params.add(getSamtoolsPath().getPath());
@@ -77,6 +77,8 @@ private void doIndex(File input, @Nullable Integer threads) throws PipelineJobEx
         {
             throw new PipelineJobException("Unable to find CRAM index: " + idx.getPath());
         }
+
+        return idx;
     }
 
     public static File getExpectedCramIndex(File input)

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java

@@ -53,9 +53,9 @@ static public void setInstance(SequencePipelineService instance)
 
     abstract public void registerPipelineStepType(Class<? extends PipelineStep> clazz, String paramName);
 
-    abstract public void registerPipelineStep(PipelineStepProvider provider);
+    abstract public void registerPipelineStep(PipelineStepProvider<?> provider);
 
-    abstract public Set<PipelineStepProvider> getAllProviders();
+    abstract public Set<PipelineStepProvider<?>> getAllProviders();
 
     abstract public <StepType extends PipelineStep> Set<PipelineStepProvider<StepType>> getProviders(Class<StepType> stepType);
 
@@ -146,4 +146,6 @@ static public void setInstance(SequencePipelineService instance)
     abstract public TaskFileManager getTaskFileManager();
 
     abstract public Set<JobResourceSettings> getResourceSettings();
+
+    abstract public Class<?> findSuperClassParameterType(Object instance);
 }

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ToolParameterDescriptor.java

@@ -187,6 +187,22 @@ public <ParamType> ParamType extractValue(PipelineJob job, PipelineStepProvider<
         return this.extractValue(job, provider, stepIdx, clazz, null);
     }
 
+    public boolean hasValueInJson(PipelineJob job, PipelineStepProvider<?> provider, int stepIdx)
+    {
+        String key = getJsonParamName(provider, stepIdx);
+        JSONObject jobParams;
+        if (job instanceof HasJobParams)
+        {
+            jobParams = ((HasJobParams)job).getParameterJson();
+        }
+        else
+        {
+            jobParams = new JSONObject(job.getParameters());
+        }
+
+        return jobParams.has(key);
+    }
+
     public <ParamType> ParamType extractValue(PipelineJob job, PipelineStepProvider<?> provider, int stepIdx, Class<ParamType> clazz, @Nullable ParamType defaultValue)
     {
         String key = getJsonParamName(provider, stepIdx);

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/VariantProcessingStep.java

@@ -17,8 +17,11 @@
 
 import htsjdk.samtools.util.Interval;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.laboratory.DemographicsProvider;
+import org.labkey.api.laboratory.LaboratoryService;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.pipeline.PipelineJobService;
 import org.labkey.api.sequenceanalysis.SequenceOutputFile;
 
 import java.io.File;
@@ -45,10 +48,22 @@ default void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<Seq
 
     enum ScatterGatherMethod
     {
-        none(),
-        contig(),
-        chunked(),
-        fixedJobs()
+        none(false),
+        contig(false),
+        chunked(true),
+        fixedJobs(false);
+
+        private final boolean _mayRequireSort;
+
+        ScatterGatherMethod(boolean mayRequireSort)
+        {
+            _mayRequireSort = mayRequireSort;
+        }
+
+        public boolean mayRequireSort()
+        {
+            return _mayRequireSort;
+        }
     }
 
     interface Output extends PipelineStepOutput
@@ -58,7 +73,20 @@ interface Output extends PipelineStepOutput
 
     interface RequiresPedigree
     {
+        default String getDemographicsProviderName(PipelineStepProvider<?> provider, PipelineJob job, int stepIdx)
+        {
+            return provider.getParameterByName(PedigreeToolParameterDescriptor.NAME).extractValue(job, provider, stepIdx, String.class);
+        }
 
+        default DemographicsProvider getDemographicsProvider(PipelineStepProvider<?> provider, PipelineJob job, int stepIdx)
+        {
+            if (PipelineJobService.get().getLocationType() != PipelineJobService.LocationType.WebServer)
+            {
+                throw new IllegalStateException("getDemographicsProvider() can only be run from the webserver");
+            }
+
+            return LaboratoryService.get().getDemographicsProviderByName(job.getContainer(), job.getUser(), getDemographicsProviderName(provider, job, stepIdx));
+        }
     }
 
     interface SupportsScatterGather