Merge discvr-21.3 to develop

Author: bbimber

.gitignore

@@ -8,3 +8,9 @@
 /jbrowse/jb_run.js
 /jbrowse/jb_setup.js
 /jbrowse/.gradle
+
+# Ignore labkey-npm build artifacts
+**/gen/**
+
+# Ignore MacOS .DS_Store artifacts
+**/.DS_Store

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenomeManager.java

@@ -117,7 +117,7 @@ public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws Pipeli
 
         //Note: neither source nor dest have trailing slashes, so the entire source (i.e '128', gets synced into a subdir of dest)
         new SimpleScriptWrapper(log).execute(Arrays.asList(
-                "rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", sourceDir.getPath(), localCacheDir.getPath()
+                "rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--chmod=D2770,F660", sourceDir.getPath(), localCacheDir.getPath()
         ));
 
         File lastUpdate = getLocalUpdateFile(genome);

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java

@@ -93,11 +93,16 @@ static public void setInstance(SequencePipelineService instance)
     abstract public CommandWrapper getCommandWrapper(Logger log);
 
     /**
-     * This allows instances to override the default docker exeutable. If DOCKER_EXE is provided in pipelineConfig.xml, this
-     * will be used. Otherise this defaults to 'docker'
+     * This allows instances to override the default docker executable. If DOCKER_EXE is provided in pipelineConfig.xml, this
+     * will be used. Otherwise this defaults to 'docker'
      */
     abstract public String getDockerCommand();
 
+    /**
+     * This allows instances to supply a user that will be passed to 'docker login'. This is rarely needed. It can be set using DOCKER_USER in pipelineConfig.xml
+     */
+    abstract public String getDockerUser();
+
     abstract public List<File> getSequenceJobInputFiles(PipelineJob job);
 
     /**

SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js

@@ -298,7 +298,7 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
 				additionalExtConfig: {
 					hidden: true,
 					isToolParam: false,
-					width: 500,
+					width: 600,
 					allowBlank: true //toggle value based on checkbox
 				},
 				defaultValue: null
@@ -329,7 +329,7 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
 					fieldLabel: 'Merge Priority Order',
 					style: 'padding: 10px;padding-bottom: 30px;',
 					border: true,
-					width: 500,
+					width: 600,
 					autoScroll: true,
 					helpPopup: 'This is the priority order in which files will be merged, top being highest priority',
 					displayField: 'name',

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java

@@ -134,11 +134,11 @@
 import org.labkey.sequenceanalysis.run.util.FastqcRunner;
 import org.labkey.sequenceanalysis.run.util.GenomicsDBAppendHandler;
 import org.labkey.sequenceanalysis.run.util.GenomicsDBImportHandler;
-import org.labkey.sequenceanalysis.run.variant.CombineVariantsHandler;
 import org.labkey.sequenceanalysis.run.variant.DepthOfCoverageHandler;
 import org.labkey.sequenceanalysis.run.variant.GenotypeConcordanceStep;
 import org.labkey.sequenceanalysis.run.variant.GenotypeFiltrationStep;
 import org.labkey.sequenceanalysis.run.variant.MendelianViolationReportStep;
+import org.labkey.sequenceanalysis.run.variant.MergeVcfsAndGenotypesHandler;
 import org.labkey.sequenceanalysis.run.variant.MultiAllelicPositionsHandler;
 import org.labkey.sequenceanalysis.run.variant.PlinkPcaStep;
 import org.labkey.sequenceanalysis.run.variant.SNPEffStep;
@@ -152,6 +152,7 @@
 import org.labkey.sequenceanalysis.run.variant.VariantFiltrationStep;
 import org.labkey.sequenceanalysis.run.variant.VariantQCStep;
 import org.labkey.sequenceanalysis.run.variant.VariantsToTableStep;
+import org.labkey.sequenceanalysis.run.variant.VcfComparisonStep;
 import org.labkey.sequenceanalysis.util.Barcoder;
 import org.labkey.sequenceanalysis.util.ChainFileValidator;
 import org.labkey.sequenceanalysis.util.ScatterGatherUtils;
@@ -326,6 +327,7 @@ public static void registerPipelineSteps()
         SequencePipelineService.get().registerPipelineStep(new VariantQCStep.Provider());
         SequencePipelineService.get().registerPipelineStep(new PlinkPcaStep.Provider());
         SequencePipelineService.get().registerPipelineStep(new MendelianViolationReportStep.Provider());
+        SequencePipelineService.get().registerPipelineStep(new VcfComparisonStep.Provider());
 
         //handlers
         SequenceAnalysisService.get().registerFileHandler(new LiftoverHandler());
@@ -343,7 +345,7 @@ public static void registerPipelineSteps()
         SequenceAnalysisService.get().registerFileHandler(new SbtGeneCountHandler());
         SequenceAnalysisService.get().registerFileHandler(new ProcessVariantsHandler());
         SequenceAnalysisService.get().registerFileHandler(new UnmappedReadExportHandler());
-        SequenceAnalysisService.get().registerFileHandler(new CombineVariantsHandler());
+        SequenceAnalysisService.get().registerFileHandler(new MergeVcfsAndGenotypesHandler());
         SequenceAnalysisService.get().registerFileHandler(new DepthOfCoverageHandler());
         SequenceAnalysisService.get().registerFileHandler(new MultiAllelicPositionsHandler());
         SequenceAnalysisService.get().registerFileHandler(new RecalculateSequenceMetricsHandler());

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java

@@ -403,6 +403,18 @@ public String getDockerCommand()
         return "docker";
     }
 
+    @Override
+    public String getDockerUser()
+    {
+        String val = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath("DOCKER_USER");
+        if (StringUtils.trimToNull(val) != null)
+        {
+            return val;
+        }
+
+        return null;
+    }
+
     @Override
     public List<File> getSequenceJobInputFiles(PipelineJob job)
     {

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java

@@ -43,7 +43,7 @@
 import org.labkey.api.writer.PrintWriters;
 import org.labkey.sequenceanalysis.SequenceAnalysisModule;
 import org.labkey.sequenceanalysis.run.util.AbstractGenomicsDBImportHandler;
-import org.labkey.sequenceanalysis.run.util.CombineVariantsWrapper;
+import org.labkey.sequenceanalysis.run.util.MergeVcfsAndGenotypesWrapper;
 import org.labkey.sequenceanalysis.util.SequenceUtil;
 
 import java.io.File;
@@ -524,7 +524,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
                 }
 
                 ReferenceGenome rg = ctx.getSequenceSupport().getCachedGenome(genomes.iterator().next());
-                CombineVariantsWrapper cv = new CombineVariantsWrapper(ctx.getLogger());
+                MergeVcfsAndGenotypesWrapper cv = new MergeVcfsAndGenotypesWrapper(ctx.getLogger());
 
                 Map<Integer, Integer> fileMap = new HashMap<>();
                 inputFiles.forEach(x -> fileMap.put(x.getRowid(), x.getDataId()));
@@ -559,7 +559,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
                 else
                 {
                     List<String> args = new ArrayList<>();
-                    args.add("-genotypeMergeOptions");
+                    args.add("-genotypeMergeOption");
                     args.add("PRIORITIZE");
 
                     List<Interval> intervals = getIntervals(ctx);

SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceAnalysisCustomizer.java

@@ -5,7 +5,6 @@
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.collections.CaseInsensitiveHashSet;
 import org.labkey.api.data.AbstractTableInfo;
-import org.labkey.api.data.BaseColumnInfo;
 import org.labkey.api.data.ColumnInfo;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.DataColumn;
@@ -162,15 +161,15 @@ public void customizeColumn(MutableColumnInfo col, AbstractTableInfo ti)
         COL_ENUM(Class dataType, @Nullable Collection<String> alternateNames){
             this.dataType = dataType;
             if (alternateNames != null)
-            this.alternateNames.addAll(alternateNames);
+                this.alternateNames.addAll(alternateNames);
         }
 
         public Collection<String> getAlternateNames()
         {
             return alternateNames;
         }
 
-        private static void setNonEditable(BaseColumnInfo col)
+        private static void setNonEditable(MutableColumnInfo col)
         {
             col.setUserEditable(false);
             col.setShownInInsertView(false);
@@ -182,8 +181,8 @@ private static void addFk(Container c, User u, MutableColumnInfo col, String sch
             if (col.getFk() == null)
             {
                 col.setFk(QueryForeignKey.from(DefaultSchema.get(u,c),null)
-                    .schema(schema, c)
-                    .to(query, pkCol, displayCol));
+                        .schema(schema, c)
+                        .to(query, pkCol, displayCol));
             }
         }
 
@@ -270,4 +269,4 @@ public void renderGridCellContents(RenderContext ctx, Writer out) throws IOExcep
             };
         }
     }
-}
+}