Skip to content

Commit 8496a9c

Browse files
bbimberbbimber
authored
Merge discvr-25.3 to develop (#246)
* Add checks for missing data * Add docker prune to maintenance * Prefer bcftools/liftover * Support tool to write 10x barcodes to file
2 parents cd18543 + fed156b commit 8496a9c

File tree

7 files changed

+45
-95
lines changed

7 files changed

+45
-95
lines changed

‎mGAP/resources/views/releaseNotes.html‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
<h4>Release 3.0:</h4>
2+
<ul>
3+
<li>This release involves a major change in the processing of variants. All prior releases omitted variants in complex or repetitive regions. We originally excluded these because genotypes can be less accurate; however, we made this change because some repetitive regions overlap coding regions and can contain valuable information. This also results in a significant increase in the total number of variants.</a></li>
4+
<li>This is the first release to include a second species. The dataset now contains both Rhesus macaques and Japanese macaques (which are separated into a separate track).</li>
5+
</ul>
6+
17
<h4>Release 2.5:</h4>
28
<ul>
39
<li>This release includes a major change in how we perform variant annotation, <a href="mgap-annotation.view">described in more detail here.</a></li>

‎mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStep;
3232
import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStepOutputImpl;
3333
import org.labkey.api.sequenceanalysis.run.AbstractCommandPipelineStep;
34+
import org.labkey.api.sequenceanalysis.run.LiftoverBcfToolsWrapper;
3435
import org.labkey.api.sequenceanalysis.run.SelectVariantsWrapper;
3536
import org.labkey.api.util.PageFlowUtil;
3637
import org.labkey.api.writer.PrintWriters;
@@ -327,8 +328,8 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
327328
File liftoverRejects = new File(outputDirectory, SequenceAnalysisService.get().getUnzippedBaseName(currentVcf.getName()) + ".liftoverReject" + grch37Genome.getGenomeId() + ".vcf.gz");
328329
if (!indexExists(liftoverRejects) || !indexExists(liftedToGRCh37))
329330
{
330-
LiftoverVcfRunner liftoverVcfRunner = new LiftoverVcfRunner(getPipelineCtx().getLogger());
331-
liftoverVcfRunner.doLiftover(currentVcf, chainFile, grch37Genome.getWorkingFastaFile(), liftoverRejects, liftedToGRCh37, 0.95);
331+
LiftoverBcfToolsWrapper liftoverVcfRunner = new LiftoverBcfToolsWrapper(getPipelineCtx().getLogger());
332+
liftoverVcfRunner.doLiftover(currentVcf, chainFile, genome.getWorkingFastaFile(), grch37Genome.getWorkingFastaFile(), liftoverRejects, liftedToGRCh37);
332333
}
333334
else
334335
{

‎mGAP/src/org/labkey/mgap/pipeline/LiftoverVcfRunner.java‎

Lines changed: 0 additions & 80 deletions
This file was deleted.

‎mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import org.labkey.api.data.TableSelector;
3232
import org.labkey.api.exp.api.ExpData;
3333
import org.labkey.api.exp.api.ExperimentService;
34-
import org.labkey.api.jbrowse.JBrowseService;
3534
import org.labkey.api.module.ModuleLoader;
3635
import org.labkey.api.pipeline.PipelineJob;
3736
import org.labkey.api.pipeline.PipelineJobException;
@@ -56,6 +55,7 @@
5655
import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
5756
import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
5857
import org.labkey.api.sequenceanalysis.run.GeneToNameTranslator;
58+
import org.labkey.api.sequenceanalysis.run.LiftoverBcfToolsWrapper;
5959
import org.labkey.api.sequenceanalysis.run.SelectVariantsWrapper;
6060
import org.labkey.api.util.FileType;
6161
import org.labkey.api.util.FileUtil;
@@ -77,7 +77,6 @@
7777
import java.util.Arrays;
7878
import java.util.Collection;
7979
import java.util.Collections;
80-
import java.util.Comparator;
8180
import java.util.Date;
8281
import java.util.HashMap;
8382
import java.util.HashSet;
@@ -723,7 +722,11 @@ private Double parseCadd(String cadd)
723722
{
724723
if (cadd.contains("|"))
725724
{
726-
return Arrays.stream(cadd.split("\\|")).map(Double::parseDouble).max(Double::compare).get();
725+
return Arrays.stream(cadd.split("\\|")).filter(x -> !".".equals(x)).map(Double::parseDouble).max(Double::compare).orElse(null);
726+
}
727+
else if (".".equals(cadd))
728+
{
729+
return null;
727730
}
728731

729732
return Double.parseDouble(cadd);
@@ -1020,7 +1023,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
10201023

10211024
File sitesOnlyVcf = getSitesOnlyVcf(ctx, primaryTrackVcf, genome);
10221025

1023-
File lifted = liftToHuman(ctx, primaryTrackVcf, sitesOnlyVcf, grch37Genome);
1026+
File lifted = liftToHuman(ctx, primaryTrackVcf, sitesOnlyVcf, genome, grch37Genome);
10241027
SequenceOutputFile output3 = new SequenceOutputFile();
10251028
output3.setFile(lifted);
10261029
output3.setName("mGAP Release: " + species + " " + releaseVersion + " Lifted to Human");
@@ -1091,7 +1094,7 @@ private File getSitesOnlyVcf(JobContext ctx, File primaryTrackVcf, ReferenceGeno
10911094
return noGenotypes;
10921095
}
10931096

1094-
private File liftToHuman(JobContext ctx, File primaryTrackVcf, File noGenotypes, ReferenceGenome grch37Genome) throws PipelineJobException
1097+
private File liftToHuman(JobContext ctx, File primaryTrackVcf, File noGenotypes, ReferenceGenome sourceGenome, ReferenceGenome grch37Genome) throws PipelineJobException
10951098
{
10961099
//lift to target genome
10971100
Integer chainFileId = ctx.getSequenceSupport().getCachedObject(AnnotationStep.CHAIN_FILE, Integer.class);
@@ -1104,8 +1107,8 @@ private File liftToHuman(JobContext ctx, File primaryTrackVcf, File noGenotypes,
11041107
File liftoverRejects = new File(ctx.getOutputDir(), SequenceAnalysisService.get().getUnzippedBaseName(primaryTrackVcf.getName()) + ".liftoverRejectGRCh37.vcf.gz");
11051108
if (!indexExists(liftoverRejects))
11061109
{
1107-
LiftoverVcfRunner liftoverVcfRunner = new LiftoverVcfRunner(ctx.getLogger());
1108-
liftoverVcfRunner.doLiftover(noGenotypes, chainFile, grch37Genome.getWorkingFastaFile(), liftoverRejects, liftedToGRCh37, 0.95);
1110+
LiftoverBcfToolsWrapper liftoverVcfRunner = new LiftoverBcfToolsWrapper(ctx.getLogger());
1111+
liftoverVcfRunner.doLiftover(noGenotypes, chainFile, sourceGenome.getWorkingFastaFile(), grch37Genome.getWorkingFastaFile(), liftoverRejects, liftedToGRCh37);
11091112
}
11101113
else
11111114
{
@@ -1388,12 +1391,12 @@ private void inspectAndSummarizeVcf(JobContext ctx, File vcfInput, GeneToNameTra
13881391
try
13891392
{
13901393
String as = StringUtils.trimToNull(polyphenScores.get(alleleIdx));
1391-
if (as == null)
1394+
if (as == null || ".".equals(as))
13921395
{
13931396
continue;
13941397
}
13951398

1396-
Double maxScore = Arrays.stream(as.split("\\|")).filter(x -> !x.isEmpty()).map(Double::parseDouble).max(Double::compare).orElse(-1.0);
1399+
double maxScore = Arrays.stream(as.split("\\|")).filter(x -> !x.isEmpty()).filter(x -> !".".equals(x)).map(Double::parseDouble).max(Double::compare).orElse(-1.0);
13971400
if (maxScore == 0.0)
13981401
{
13991402
ctx.getLogger().error("Suspicious values for Polyphen2_HVAR_S: " + maxScore + ", at position: " + vc.toStringWithoutGenotypes());
@@ -1581,7 +1584,7 @@ else if (af != null)
15811584
cadd = Arrays.stream(String.valueOf(cadd).split("\\|")).map(x -> {
15821585
try
15831586
{
1584-
double y = Double.parseDouble(x);
1587+
Double.parseDouble(x);
15851588
}
15861589
catch (Exception e)
15871590
{

‎primeseq/src/org/labkey/primeseq/analysis/CombineMethylationRatesHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public SequenceOutputProcessor getProcessor()
8383
return new Processor();
8484
}
8585

86-
public class Processor implements SequenceOutputProcessor
86+
public static class Processor implements SequenceOutputProcessor
8787
{
8888
@Override
8989
public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List<SequenceOutputFile> inputFiles, JSONObject params, File outputDir, List<RecordedAction> actions, List<SequenceOutputFile> outputsToCreate) throws UnsupportedOperationException, PipelineJobException

‎primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparison.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport
172172
Map<String, SequenceOutputFile> fileMap = new HashMap<>();
173173
for (SequenceOutputFile f : inputFiles)
174174
{
175-
action.addInputIfNotPresent(f.getFile(), "Methlylation Rates");
175+
action.addInputIfNotPresent(f.getFile(), "Methylation Rates");
176176
fileMap.put(f.getRowid().toString(), f);
177177
}
178178

‎primeseq/src/org/labkey/primeseq/pipeline/ClusterMaintenanceTask.java‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.labkey.api.pipeline.RemoteExecutionEngine;
2727
import org.labkey.api.query.FieldKey;
2828
import org.labkey.api.security.UserManager;
29+
import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
2930
import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
3031
import org.labkey.api.util.FileUtil;
3132
import org.labkey.api.util.Job;
@@ -84,7 +85,7 @@ public void run(Logger log)
8485
jobGuids.addAll(ts2.getArrayList(String.class));
8586

8687
JobRunner jr = JobRunner.getDefault();
87-
for (RemoteExecutionEngine engine : PipelineJobService.get().getRemoteExecutionEngines())
88+
for (RemoteExecutionEngine<?> engine : PipelineJobService.get().getRemoteExecutionEngines())
8889
{
8990
log.info("Starting maintenance task for: " + engine.getType());
9091

@@ -158,6 +159,8 @@ public void run(Logger log)
158159
//hacky, but this is only planned to be used by us
159160
inspectFolder(log, new File("/home/exacloud/gscratch/prime-seq/workDir/"));
160161
inspectFolder(log, new File("/home/exacloud/gscratch/prime-seq/cachedData/"));
162+
163+
runDockerPrune(log);
161164
}
162165

163166
private void deleteDirectory(File child, Logger log)
@@ -230,6 +233,23 @@ private void inspectFolder(Logger log, File workDirBase)
230233

231234
}
232235

236+
private static void runDockerPrune(Logger log)
237+
{
238+
try
239+
{
240+
new SimpleScriptWrapper(log).execute(Arrays.asList(
241+
SequencePipelineService.get().getDockerCommand(),
242+
"system",
243+
"prune",
244+
"-f"
245+
));
246+
}
247+
catch (PipelineJobException e)
248+
{
249+
_log.error("Error running docker prune", e);
250+
}
251+
}
252+
233253
public static class TestCase extends Assert
234254
{
235255
@Test

0 commit comments

Comments
 (0)