Skip to content

Commit 4e0b2eb

Browse files
jctrotterjctrotter
committed
Merge branch 'BimberLab:discvr-21.3' into dev
2 parents 2b8cddb + e81073c commit 4e0b2eb

29 files changed

+506
-122
lines changed

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenomeManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws Pipeli
117117

118118
//Note: neither source nor dest have trailing slashes, so the entire source (i.e '128', gets synced into a subdir of dest)
119119
new SimpleScriptWrapper(log).execute(Arrays.asList(
120-
"rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", sourceDir.getPath(), localCacheDir.getPath()
120+
"rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--chmod=D2770,F660", sourceDir.getPath(), localCacheDir.getPath()
121121
));
122122

123123
File lastUpdate = getLocalUpdateFile(genome);

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,11 +93,16 @@ static public void setInstance(SequencePipelineService instance)
9393
abstract public CommandWrapper getCommandWrapper(Logger log);
9494

9595
/**
96-
* This allows instances to override the default docker exeutable. If DOCKER_EXE is provided in pipelineConfig.xml, this
97-
* will be used. Otherise this defaults to 'docker'
96+
* This allows instances to override the default docker executable. If DOCKER_EXE is provided in pipelineConfig.xml, this
97+
* will be used. Otherwise this defaults to 'docker'
9898
*/
9999
abstract public String getDockerCommand();
100100

101+
/**
102+
* This allows instances to supply a user that will be passed to 'docker login'. This is rarely needed. It can be set using DOCKER_USER in pipelineConfig.xml
103+
*/
104+
abstract public String getDockerUser();
105+
101106
abstract public List<File> getSequenceJobInputFiles(PipelineJob job);
102107

103108
/**

‎SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,7 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
298298
additionalExtConfig: {
299299
hidden: true,
300300
isToolParam: false,
301-
width: 500,
301+
width: 600,
302302
allowBlank: true //toggle value based on checkbox
303303
},
304304
defaultValue: null
@@ -329,7 +329,7 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
329329
fieldLabel: 'Merge Priority Order',
330330
style: 'padding: 10px;padding-bottom: 30px;',
331331
border: true,
332-
width: 500,
332+
width: 600,
333333
autoScroll: true,
334334
helpPopup: 'This is the priority order in which files will be merged, top being highest priority',
335335
displayField: 'name',

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,11 +134,11 @@
134134
import org.labkey.sequenceanalysis.run.util.FastqcRunner;
135135
import org.labkey.sequenceanalysis.run.util.GenomicsDBAppendHandler;
136136
import org.labkey.sequenceanalysis.run.util.GenomicsDBImportHandler;
137-
import org.labkey.sequenceanalysis.run.variant.CombineVariantsHandler;
138137
import org.labkey.sequenceanalysis.run.variant.DepthOfCoverageHandler;
139138
import org.labkey.sequenceanalysis.run.variant.GenotypeConcordanceStep;
140139
import org.labkey.sequenceanalysis.run.variant.GenotypeFiltrationStep;
141140
import org.labkey.sequenceanalysis.run.variant.MendelianViolationReportStep;
141+
import org.labkey.sequenceanalysis.run.variant.MergeVcfsAndGenotypesHandler;
142142
import org.labkey.sequenceanalysis.run.variant.MultiAllelicPositionsHandler;
143143
import org.labkey.sequenceanalysis.run.variant.PlinkPcaStep;
144144
import org.labkey.sequenceanalysis.run.variant.SNPEffStep;
@@ -152,6 +152,7 @@
152152
import org.labkey.sequenceanalysis.run.variant.VariantFiltrationStep;
153153
import org.labkey.sequenceanalysis.run.variant.VariantQCStep;
154154
import org.labkey.sequenceanalysis.run.variant.VariantsToTableStep;
155+
import org.labkey.sequenceanalysis.run.variant.VcfComparisonStep;
155156
import org.labkey.sequenceanalysis.util.Barcoder;
156157
import org.labkey.sequenceanalysis.util.ChainFileValidator;
157158
import org.labkey.sequenceanalysis.util.ScatterGatherUtils;
@@ -326,6 +327,7 @@ public static void registerPipelineSteps()
326327
SequencePipelineService.get().registerPipelineStep(new VariantQCStep.Provider());
327328
SequencePipelineService.get().registerPipelineStep(new PlinkPcaStep.Provider());
328329
SequencePipelineService.get().registerPipelineStep(new MendelianViolationReportStep.Provider());
330+
SequencePipelineService.get().registerPipelineStep(new VcfComparisonStep.Provider());
329331

330332
//handlers
331333
SequenceAnalysisService.get().registerFileHandler(new LiftoverHandler());
@@ -343,7 +345,7 @@ public static void registerPipelineSteps()
343345
SequenceAnalysisService.get().registerFileHandler(new SbtGeneCountHandler());
344346
SequenceAnalysisService.get().registerFileHandler(new ProcessVariantsHandler());
345347
SequenceAnalysisService.get().registerFileHandler(new UnmappedReadExportHandler());
346-
SequenceAnalysisService.get().registerFileHandler(new CombineVariantsHandler());
348+
SequenceAnalysisService.get().registerFileHandler(new MergeVcfsAndGenotypesHandler());
347349
SequenceAnalysisService.get().registerFileHandler(new DepthOfCoverageHandler());
348350
SequenceAnalysisService.get().registerFileHandler(new MultiAllelicPositionsHandler());
349351
SequenceAnalysisService.get().registerFileHandler(new RecalculateSequenceMetricsHandler());

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -403,6 +403,18 @@ public String getDockerCommand()
403403
return "docker";
404404
}
405405

406+
@Override
407+
public String getDockerUser()
408+
{
409+
String val = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath("DOCKER_USER");
410+
if (StringUtils.trimToNull(val) != null)
411+
{
412+
return val;
413+
}
414+
415+
return null;
416+
}
417+
406418
@Override
407419
public List<File> getSequenceJobInputFiles(PipelineJob job)
408420
{

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
import org.labkey.api.writer.PrintWriters;
4444
import org.labkey.sequenceanalysis.SequenceAnalysisModule;
4545
import org.labkey.sequenceanalysis.run.util.AbstractGenomicsDBImportHandler;
46-
import org.labkey.sequenceanalysis.run.util.CombineVariantsWrapper;
46+
import org.labkey.sequenceanalysis.run.util.MergeVcfsAndGenotypesWrapper;
4747
import org.labkey.sequenceanalysis.util.SequenceUtil;
4848

4949
import java.io.File;
@@ -524,7 +524,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
524524
}
525525

526526
ReferenceGenome rg = ctx.getSequenceSupport().getCachedGenome(genomes.iterator().next());
527-
CombineVariantsWrapper cv = new CombineVariantsWrapper(ctx.getLogger());
527+
MergeVcfsAndGenotypesWrapper cv = new MergeVcfsAndGenotypesWrapper(ctx.getLogger());
528528

529529
Map<Integer, Integer> fileMap = new HashMap<>();
530530
inputFiles.forEach(x -> fileMap.put(x.getRowid(), x.getDataId()));
@@ -559,7 +559,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
559559
else
560560
{
561561
List<String> args = new ArrayList<>();
562-
args.add("-genotypeMergeOptions");
562+
args.add("-genotypeMergeOption");
563563
args.add("PRIORITIZE");
564564

565565
List<Interval> intervals = getIntervals(ctx);

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceAnalysisCustomizer.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,13 @@
55
import org.jetbrains.annotations.Nullable;
66
import org.labkey.api.collections.CaseInsensitiveHashSet;
77
import org.labkey.api.data.AbstractTableInfo;
8-
import org.labkey.api.data.BaseColumnInfo;
98
import org.labkey.api.data.ColumnInfo;
109
import org.labkey.api.data.Container;
1110
import org.labkey.api.data.DataColumn;
1211
import org.labkey.api.data.DisplayColumn;
1312
import org.labkey.api.data.DisplayColumnFactory;
1413
import org.labkey.api.data.JdbcType;
14+
import org.labkey.api.data.MutableColumnInfo;
1515
import org.labkey.api.data.RenderContext;
1616
import org.labkey.api.data.SQLFragment;
1717
import org.labkey.api.data.TableCustomizer;
@@ -99,21 +99,21 @@ private void customizeSharedCols(AbstractTableInfo ti)
9999
private enum COL_ENUM
100100
{
101101
refNtSequence(Integer.class, PageFlowUtil.set("sequenceid", "ref_nt_sequence")){
102-
public void customizeColumn(BaseColumnInfo col, AbstractTableInfo ti)
102+
public void customizeColumn(MutableColumnInfo col, AbstractTableInfo ti)
103103
{
104104
col.setLabel("Ref NT Sequence");
105105
addFk(ti.getUserSchema().getContainer(), ti.getUserSchema().getUser(), col, SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_REF_NT_SEQUENCES, "rowid", "name");
106106
}
107107
},
108108
libraryId(Integer.class, PageFlowUtil.set("genomeId", "genome_id", "library_id")){
109-
public void customizeColumn(BaseColumnInfo col, AbstractTableInfo ti)
109+
public void customizeColumn(MutableColumnInfo col, AbstractTableInfo ti)
110110
{
111111
col.setLabel("Reference Genome");
112112
addFk(ti.getUserSchema().getContainer(), ti.getUserSchema().getUser(), col, SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_REF_LIBRARIES, "rowid", "rowid");
113113
}
114114
},
115115
runid(Integer.class, PageFlowUtil.set("run_id")){
116-
public void customizeColumn(BaseColumnInfo col, AbstractTableInfo ti)
116+
public void customizeColumn(MutableColumnInfo col, AbstractTableInfo ti)
117117
{
118118
col.setLabel("Run");
119119
col.setShownInInsertView(false);
@@ -124,7 +124,7 @@ public void customizeColumn(BaseColumnInfo col, AbstractTableInfo ti)
124124
}
125125
},
126126
jobid(Integer.class, PageFlowUtil.set("job_id")){
127-
public void customizeColumn(BaseColumnInfo col, AbstractTableInfo ti)
127+
public void customizeColumn(MutableColumnInfo col, AbstractTableInfo ti)
128128
{
129129
col.setLabel("Job Id");
130130
col.setShownInInsertView(false);
@@ -134,21 +134,21 @@ public void customizeColumn(BaseColumnInfo col, AbstractTableInfo ti)
134134
}
135135
},
136136
dataId(Integer.class, PageFlowUtil.set("data_id")){
137-
public void customizeColumn(BaseColumnInfo col, AbstractTableInfo ti)
137+
public void customizeColumn(MutableColumnInfo col, AbstractTableInfo ti)
138138
{
139139
col.setLabel("File Id");
140140
addFk(ti.getUserSchema().getContainer(), ti.getUserSchema().getUser(), col, "exp", "data", "rowid", null);
141141
}
142142
},
143143
readset(Integer.class, PageFlowUtil.set("readsetId", "readset_id")){
144-
public void customizeColumn(BaseColumnInfo col, AbstractTableInfo ti)
144+
public void customizeColumn(MutableColumnInfo col, AbstractTableInfo ti)
145145
{
146146
col.setLabel("Readset");
147147
addFk(ti.getUserSchema().getContainer(), ti.getUserSchema().getUser(), col, SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_READSETS, "rowid", null);
148148
}
149149
},
150150
analysisId(Integer.class, PageFlowUtil.set("analysis_id")){
151-
public void customizeColumn(BaseColumnInfo col, AbstractTableInfo ti)
151+
public void customizeColumn(MutableColumnInfo col, AbstractTableInfo ti)
152152
{
153153
col.setLabel("Analysis Id");
154154
addFk(ti.getUserSchema().getContainer(), ti.getUserSchema().getUser(), col, SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_ANALYSES, "rowid", null);
@@ -169,14 +169,14 @@ public Collection<String> getAlternateNames()
169169
return alternateNames;
170170
}
171171

172-
private static void setNonEditable(BaseColumnInfo col)
172+
private static void setNonEditable(MutableColumnInfo col)
173173
{
174174
col.setUserEditable(false);
175175
col.setShownInInsertView(false);
176176
col.setShownInUpdateView(false);
177177
}
178178

179-
private static void addFk(Container c, User u, BaseColumnInfo col, String schema, String query, String pkCol, @Nullable String displayCol)
179+
private static void addFk(Container c, User u, MutableColumnInfo col, String schema, String query, String pkCol, @Nullable String displayCol)
180180
{
181181
if (col.getFk() == null)
182182
{
@@ -186,9 +186,9 @@ private static void addFk(Container c, User u, BaseColumnInfo col, String schema
186186
}
187187
}
188188

189-
abstract public void customizeColumn(BaseColumnInfo col, AbstractTableInfo ti);
189+
abstract public void customizeColumn(MutableColumnInfo col, AbstractTableInfo ti);
190190

191-
public static void processColumn(BaseColumnInfo col, AbstractTableInfo ti)
191+
public static void processColumn(MutableColumnInfo col, AbstractTableInfo ti)
192192
{
193193
for (COL_ENUM colEnum : COL_ENUM.values())
194194
{

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/LofreqAnalysis.java‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -246,25 +246,27 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc
246246

247247
File pindelOutput = PindelAnalysis.runPindel(output, getPipelineCtx(), rs, outputDir, inputBam, referenceGenome.getWorkingFastaFile(), minFraction, minDepth, true, coverageOut, minInsertSize);
248248
File pindelVcf = PindelAnalysis.createVcf(pindelOutput, new File(pindelOutput.getParentFile(), FileUtil.getBaseName(pindelOutput) + ".all.vcf.gz"), referenceGenome, settings);
249-
250-
try (VCFFileReader reader = new VCFFileReader(pindelVcf);CloseableIterator<VariantContext> it = reader.iterator())
249+
if (pindelVcf.exists())
251250
{
252-
while (it.hasNext())
251+
try (VCFFileReader reader = new VCFFileReader(pindelVcf); CloseableIterator<VariantContext> it = reader.iterator())
253252
{
254-
VariantContext vc = it.next();
255-
if (vc.hasAttribute("IN_CONSENSUS"))
253+
while (it.hasNext())
256254
{
257-
pindelConsensusVariants.add(vc);
258-
totalPindelConsensusVariants++;
259-
}
255+
VariantContext vc = it.next();
256+
if (vc.hasAttribute("IN_CONSENSUS"))
257+
{
258+
pindelConsensusVariants.add(vc);
259+
totalPindelConsensusVariants++;
260+
}
260261

261-
totalPindelVariants++;
262+
totalPindelVariants++;
263+
}
262264
}
263265
}
264266

265267
getPipelineCtx().getLogger().info("Total pindel variants: " + totalPindelVariants);
266268
getPipelineCtx().getLogger().info("Total consensus pindel variants: " + totalPindelConsensusVariants);
267-
if (totalPindelConsensusVariants == 0)
269+
if (totalPindelConsensusVariants == 0 && pindelVcf.exists())
268270
{
269271
getPipelineCtx().getLogger().info("deleting empty pindel VCF: " + pindelVcf.getPath());
270272
pindelVcf.delete();
@@ -714,7 +716,6 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc
714716
String[] pangolinData = null;
715717
if (runPangolinAndNextClade)
716718
{
717-
PangolinHandler.updatePangolinRefs(getPipelineCtx().getLogger());
718719
pangolinData = PangolinHandler.runPangolin(consensusFastaLoFreq, getPipelineCtx().getLogger(), output);
719720

720721
File json = NextCladeHandler.runNextClade(consensusFastaLoFreq, getPipelineCtx().getLogger(), output, outputDir);
@@ -740,11 +741,10 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc
740741

741742
if (pangolinData != null)
742743
{
743-
writer.writeNext(new String[]{"Pangolin", "PangolinLineage", pangolinData[1]});
744-
writer.writeNext(new String[]{"Pangolin", "PangolinConflicts", pangolinData[2]});
744+
writer.writeNext(new String[]{"Pangolin", "PangolinLineage", pangolinData[0]});
745+
writer.writeNext(new String[]{"Pangolin", "PangolinConflicts", pangolinData[1]});
746+
writer.writeNext(new String[]{"Pangolin", "PangolinAmbiguity", pangolinData[2]});
745747
writer.writeNext(new String[]{"Pangolin", "PangolinVersions", pangolinData[3]});
746-
//TODO: consider parsing
747-
writer.writeNext(new String[]{"Pangolin", "PangolinComment", pangolinData[4]});
748748
}
749749
else
750750
{
@@ -1223,7 +1223,7 @@ private File getExe()
12231223
}
12241224
}
12251225

1226-
private SortingCollection<VariantContext> getVariantSorter(VCFHeader outputHeader) {
1226+
public static SortingCollection<VariantContext> getVariantSorter(VCFHeader outputHeader) {
12271227
File tmpDir = IOUtil.getDefaultTmpDir();
12281228
if (!tmpDir.exists()) {
12291229
tmpDir.mkdirs();

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/NextCladeHandler.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,6 @@ public void complete(PipelineJob job, List<SequenceOutputFile> inputFiles, List<
124124
@Override
125125
public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException
126126
{
127-
PangolinHandler.updatePangolinRefs(ctx.getLogger());
128-
129127
for (SequenceOutputFile so : inputFiles)
130128
{
131129
File nextCladeJson = runNextClade(so.getFile(), ctx.getLogger(), ctx.getFileManager(), ctx.getOutputDir());

0 commit comments

Comments
 (0)