Skip to content

Commit 1031c27

Browse files
bbimberbbimber
authored
Merge pull request #118 from LabKey/fb_merge_21.11_to_develop
Merge discvr-21.11 to develop
2 parents a8ba40a + e5e1488 commit 1031c27

File tree

136 files changed

+24608
-5327
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

136 files changed

+24608
-5327
lines changed

‎.github/workflows/build.yml‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,11 @@ jobs:
2424

2525
- name: "Print default branch"
2626
run: |
27-
echo 'Default branch: ${{ fromJson(steps.get_default_branch.outputs.data).default_branch }}'
28-
echo "##[set-output name=branch;]$(echo '${{ fromJson(steps.get_default_branch.outputs.data).default_branch }}')"
27+
DEFAULT_BRANCH='${{ fromJson(steps.get_default_branch.outputs.data).default_branch }}'
28+
echo 'Default branch: '$DEFAULT_BRANCH
29+
echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
2930
id: default-branch
3031

31-
- name: Extract branch name
32-
shell: bash
33-
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
34-
id: extract-branch
35-
3632
- name: "Build DISCVR"
3733
uses: bimberlabinternal/DevOps/githubActions/discvr-build@master
3834
with:
@@ -42,14 +38,14 @@ jobs:
4238
github_token: ${{ secrets.PAT }}
4339
env:
4440
# Only generate the distribution if this is the default branch
45-
GENERATE_DIST: ${{ steps.default-branch.branch == steps.extract-branch.branch && '1' || '0' }}
41+
GENERATE_DIST: ${{ env.DEFAULT_BRANCH == github.ref_name && '1' || '0' }}
4642

4743
- name: Publish Release
48-
if: github.ref == '/refs/heads/${{ fromJson(steps.get_default_branch.outputs.data).default_branch }}' && github.event_name == 'push'
44+
if: ${{ env.DEFAULT_BRANCH == github.ref_name && github.event_name == 'push' && github.base_ref == '' }}
4945
uses: "marvinpinto/action-automatic-releases@latest"
5046
with:
5147
repo_token: "${{ secrets.PAT }}"
5248
automatic_release_tag: "latest"
5349
prerelease: true
54-
title: "Development Build: ${{ fromJson(steps.get_default_branch.outputs.data).default_branch }}"
50+
title: "Development Build: ${{ env.DEFAULT_BRANCH }}"
5551
files: /home/runner/work/_temp/_github_home/lkDist/discvr/DISCVR-*

‎.github/workflows/build_latest.yml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,14 @@ jobs:
1919
GITHUB_TOKEN: ${{ secrets.PAT }}
2020

2121
- name: "Print default branch"
22-
run: "echo 'Default branch: ${{ fromJson(steps.get_default_branch.outputs.data).default_branch }}'"
22+
run: |
23+
DEFAULT_BRANCH='${{ fromJson(steps.get_default_branch.outputs.data).default_branch }}'
24+
echo 'Default branch: '$DEFAULT_BRANCH
25+
echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
26+
id: default-branch
2327

2428
- name: "Build DISCVR"
25-
if: github.ref == '/refs/heads/${{ fromJson(steps.get_default_branch.outputs.data).default_branch }}'
29+
if: ${{ env.DEFAULT_BRANCH == github.ref_name }}
2630
uses: bimberlabinternal/DevOps/githubActions/discvr-build@master
2731
with:
2832
artifactory_user: ${{secrets.artifactory_user}}
@@ -34,7 +38,7 @@ jobs:
3438

3539
# See: https://github.com/marketplace/actions/automatic-releases
3640
- name: Deploy Build
37-
if: github.ref == '/refs/heads/${{ fromJson(steps.get_default_branch.outputs.data).default_branch }}'
41+
if: ${{ env.DEFAULT_BRANCH == github.ref_name }}
3842
uses: "marvinpinto/action-automatic-releases@latest"
3943
with:
4044
repo_token: "${{ secrets.PAT }}"

‎.github/workflows/release.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ name: Publish Release
22
on:
33
release:
44
types: [created]
5+
tags-ignore: [latest]
56
jobs:
67
generate:
78
name: Create release-artifacts
@@ -22,7 +23,6 @@ jobs:
2223
GENERATE_DIST: 1
2324

2425
- name: Upload Artifacts
25-
if: github.event_name == 'release' && github.event.action == 'created'
2626
uses: skx/github-action-publish-binaries@master
2727
env:
2828
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/BcftoolsRunner.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package org.labkey.api.sequenceanalysis.pipeline;
2+
3+
import org.apache.logging.log4j.Logger;
4+
import org.jetbrains.annotations.Nullable;
5+
import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper;
6+
7+
import java.io.File;
8+
9+
/**
10+
* User: bimber
11+
* Date: 12/15/12
12+
* Time: 9:11 PM
13+
*/
14+
public class BcftoolsRunner extends AbstractCommandWrapper
15+
{
16+
public BcftoolsRunner(@Nullable Logger logger)
17+
{
18+
super(logger);
19+
}
20+
21+
public File getBcfToolsPath()
22+
{
23+
return SequencePipelineService.get().getExeForPackage("BCFTOOLSPATH", "bcftools");
24+
}
25+
}

‎SequenceAnalysis/pipeline_code/sequence_tools_install.sh‎

Lines changed: 42 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -633,19 +633,50 @@ then
633633
rm -Rf $LKTOOLS_DIR/samtools
634634
rm -Rf $LKTOOLS_DIR/bcftools
635635

636-
wget $WGET_OPTS https://github.com/samtools/samtools/releases/download/1.10/samtools-1.10.tar.bz2
637-
bunzip2 samtools-1.10.tar.bz2
638-
tar -xf samtools-1.10.tar
636+
wget $WGET_OPTS https://github.com/samtools/samtools/releases/download/1.14/samtools-1.14.tar.bz2
637+
bunzip2 samtools-1.14.tar.bz2
638+
tar -xf samtools-1.14.tar
639639
echo "Compressing TAR"
640-
bzip2 samtools-1.10.tar
641-
cd samtools-1.10
640+
bzip2 samtools-1.14.tar
641+
cd samtools-1.14
642642
./configure
643643
make
644644
install ./samtools ${LKTOOLS_DIR}/samtools
645645
else
646646
echo "Already installed"
647647
fi
648648

649+
#
650+
#bcftools
651+
#
652+
echo ""
653+
echo ""
654+
echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
655+
echo "Install bcftools"
656+
echo ""
657+
cd $LKSRC_DIR
658+
659+
if [[ ! -e ${LKTOOLS_DIR}/bcftools || ! -z $FORCE_REINSTALL ]];
660+
then
661+
echo "Cleaning up previous installs"
662+
rm -Rf bcftools*
663+
rm -Rf $LKTOOLS_DIR/bcftools
664+
665+
wget $WGET_OPTS https://github.com/samtools/bcftools/releases/download/1.14/bcftools-1.14.tar.bz2
666+
bunzip2 bcftools-1.14.tar.bz2
667+
tar -xf bcftools-1.14.tar
668+
echo "Compressing TAR"
669+
bzip2 bcftools-1.14.tar
670+
chmod 755 bcftools-1.14
671+
cd bcftools-1.14
672+
./configure
673+
make
674+
675+
install ./bcftools $LKTOOLS_DIR
676+
else
677+
echo "Already installed"
678+
fi
679+
649680
#
650681
#tabix
651682
#
@@ -663,13 +694,13 @@ then
663694
rm -Rf $LKTOOLS_DIR/tabix
664695
rm -Rf $LKTOOLS_DIR/bgzip
665696

666-
wget $WGET_OPTS https://github.com/samtools/htslib/releases/download/1.10.2/htslib-1.10.2.tar.bz2
667-
bunzip2 htslib-1.10.2.tar.bz2
668-
tar -xf htslib-1.10.2.tar
697+
wget $WGET_OPTS https://github.com/samtools/htslib/releases/download/1.14/htslib-1.14.tar.bz2
698+
bunzip2 htslib-1.14.tar.bz2
699+
tar -xf htslib-1.14.tar
669700
echo "Compressing TAR"
670-
bzip2 htslib-1.10.2.tar
671-
chmod 755 htslib-1.10.2
672-
cd htslib-1.10.2
701+
bzip2 htslib-1.14.tar
702+
chmod 755 htslib-1.14
703+
cd htslib-1.14
673704
./configure
674705
make
675706

‎SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
216216
label: 'Max Alternate Alleles',
217217
description: 'Maximum number of alternate alleles to genotype',
218218
commandLineParam: '--max_alternate_alleles',
219-
defaultValue: 12
219+
defaultValue: null
220220
},{
221221
fieldXtype: 'checkbox',
222222
name: 'sharedPosixOptimizations',
@@ -249,6 +249,12 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
249249
label: 'Disable File Locking',
250250
description: 'This applies to GenomicsDB inputs only. Certain filesystems do not support file locking, including NFS and Lustre. If your data will be processed on a filesystem that does not support locking, check this.',
251251
defaultValue: true
252+
},{
253+
fieldXtype: 'ldk-expdatafield',
254+
name: 'exclude_intervals',
255+
label: 'Intervals to Exclude',
256+
description: 'A BED or similar file with intervals to skip',
257+
defaultValue: null
252258
},{
253259
fieldXtype: 'checkbox',
254260
name: 'doCopyInputs',

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisServiceImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,7 @@ private void appendParents(Set<String> distinctSubjects, PedigreeRecord p, List<
433433
@Override
434434
public String getVCFLineCount(File vcf, Logger log, boolean passOnly) throws PipelineJobException
435435
{
436-
return ProcessVariantsHandler.getVCFLineCount(vcf, log, passOnly);
436+
return passOnly ? ProcessVariantsHandler.getVCFLineCount(vcf, log, passOnly, false) : ProcessVariantsHandler.getVCFLineCount(vcf, log, passOnly, true);
437437
}
438438

439439
@Override

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/GenotypeGVCFHandler.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ public GenotypeGVCFHandler()
121121
{
122122
// ToolParameterDescriptor.create("fileBaseName", "Filename", "This is the basename that will be used for the output gzipped VCF", "textfield", null, "CombinedGenotypes"),
123123
// ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("-stand_call_conf"), "stand_call_conf", "Threshold For Calling Variants", "The minimum phred-scaled confidence threshold at which variants should be called", "ldk-numberfield", null, 30),
124-
// ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--max_alternate_alleles"), "max_alternate_alleles", "Max Alternate Alleles", "Maximum number of alternate alleles to genotype", "ldk-integerfield", null, 12),
124+
// ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--max_alternate_alleles"), "max_alternate_alleles", "Max Alternate Alleles", "Maximum number of alternate alleles to genotype", "ldk-integerfield", null, null),
125125
// ToolParameterDescriptor.createCommandLineParam(CommandLineParam.createSwitch("--includeNonVariantSites"), "includeNonVariantSites", "Include Non-Variant Sites", "If checked, all sites will be output into the VCF, instead of just those where variants are detected. This can dramatically increase the size of the VCF.", "checkbox", null, false)
126126
// ToolParameterDescriptor.create("sharedPosixOptimizations", "Use Shared Posix Optimizations", "This enabled optimizations for large shared filesystems, such as lustre.", "checkbox", new JSONObject(){{
127127
// put("checked", true);
@@ -349,6 +349,19 @@ private File runGenotypeGVCFs(PipelineJob job, JobContext ctx, ProcessVariantsHa
349349
toolParams.add(ctx.getParams().get("variantCalling.GenotypeGVCFs.stand_call_conf").toString());
350350
}
351351

352+
if (ctx.getParams().get("variantCalling.GenotypeGVCFs.exclude_intervals") != null)
353+
{
354+
toolParams.add("-XL");
355+
int dataId = Integer.parseInt(ctx.getParams().get("variantCalling.GenotypeGVCFs.exclude_intervals").toString());
356+
File bed = ctx.getSequenceSupport().getCachedData(dataId);
357+
if (bed == null)
358+
{
359+
throw new PipelineJobException("Unable to find ExpData: " + dataId);
360+
}
361+
362+
toolParams.add(bed.getPath());
363+
}
364+
352365
if (ctx.getParams().get("variantCalling.GenotypeGVCFs.max_alternate_alleles") != null)
353366
{
354367
toolParams.add("--max-alternate-alleles");

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/LiftoverHandler.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -321,20 +321,20 @@ public void liftOverVcf(JobContext ctx, ReferenceGenome targetGenome, ReferenceG
321321
Long mapped = null;
322322
if (output.exists())
323323
{
324-
String mappedStr = ProcessVariantsHandler.getVCFLineCount(output, job.getLogger(), false);
324+
String mappedStr = ProcessVariantsHandler.getVCFLineCount(output, job.getLogger(), false, true);
325325
mapped = StringUtils.trimToNull(mappedStr) == null ? 0L : Long.parseLong(mappedStr);
326326
job.getLogger().info("total variants mapped: " + mappedStr);
327-
job.getLogger().info("passing variants mapped: " + ProcessVariantsHandler.getVCFLineCount(output, job.getLogger(), true));
327+
job.getLogger().info("passing variants mapped: " + ProcessVariantsHandler.getVCFLineCount(output, job.getLogger(), true, false));
328328
SequenceAnalysisService.get().ensureVcfIndex(output, job.getLogger());
329329
}
330330

331331
Long unmapped = 0L;
332332
if (unmappedOutput != null && unmappedOutput.exists())
333333
{
334-
String unmappedStr = ProcessVariantsHandler.getVCFLineCount(unmappedOutput, job.getLogger(), false);
334+
String unmappedStr = ProcessVariantsHandler.getVCFLineCount(unmappedOutput, job.getLogger(), false, true);
335335
unmapped = StringUtils.trimToNull(unmappedStr) == null ? 0L : Long.parseLong(unmappedStr);
336336
job.getLogger().info("total unmapped variants: " + unmappedStr);
337-
job.getLogger().info("passing unmapped variants: " + ProcessVariantsHandler.getVCFLineCount(unmappedOutput, job.getLogger(), true));
337+
job.getLogger().info("passing unmapped variants: " + ProcessVariantsHandler.getVCFLineCount(unmappedOutput, job.getLogger(), true, false));
338338
SequenceAnalysisService.get().ensureVcfIndex(unmappedOutput, job.getLogger());
339339
}
340340

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
2828
import org.labkey.api.sequenceanalysis.SequenceOutputFile;
2929
import org.labkey.api.sequenceanalysis.pipeline.AbstractResumer;
30+
import org.labkey.api.sequenceanalysis.pipeline.BcftoolsRunner;
3031
import org.labkey.api.sequenceanalysis.pipeline.PipelineStepCtx;
3132
import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider;
3233
import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
@@ -430,8 +431,8 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
430431
{
431432
currentVCF = output.getVCF();
432433

433-
ctx.getJob().getLogger().info("total variants: " + getVCFLineCount(currentVCF, ctx.getJob().getLogger(), false));
434-
ctx.getJob().getLogger().info("passing variants: " + getVCFLineCount(currentVCF, ctx.getJob().getLogger(), true));
434+
ctx.getJob().getLogger().info("total variants: " + getVCFLineCount(currentVCF, ctx.getJob().getLogger(), false, true));
435+
ctx.getJob().getLogger().info("passing variants: " + getVCFLineCount(currentVCF, ctx.getJob().getLogger(), true, false));
435436
ctx.getJob().getLogger().debug("index exists: " + (new File(currentVCF.getPath() + ".tbi")).exists());
436437

437438
try
@@ -468,8 +469,25 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
468469
return null;
469470
}
470471

471-
public static String getVCFLineCount(File vcf, Logger log, boolean passOnly) throws PipelineJobException
472+
private static String countUsingBcfTools(File vcf, Logger log) throws PipelineJobException
472473
{
474+
BcftoolsRunner wrapper = new BcftoolsRunner(log);
475+
476+
return wrapper.executeWithOutput(Arrays.asList(wrapper.getBcfToolsPath().getPath(), "index", "-n", vcf.getPath()));
477+
}
478+
479+
public static String getVCFLineCount(File vcf, Logger log, boolean passOnly, boolean useBcfTools) throws PipelineJobException
480+
{
481+
if (useBcfTools)
482+
{
483+
if (passOnly)
484+
{
485+
throw new PipelineJobException("bcftools VCF count cannot be used with passOnly");
486+
}
487+
488+
return countUsingBcfTools(vcf, log);
489+
}
490+
473491
String cat = vcf.getName().endsWith(".gz") ? "zcat" : "cat";
474492
SimpleScriptWrapper wrapper = new SimpleScriptWrapper(null);
475493

0 commit comments

Comments
 (0)