Skip to content

Commit 391adde

Browse files
bbimberbbimber
authored
Merge pull request #217 from LabKey/fb_merge_23.3_to_develop
Merge discvr-23.3 to develop
2 parents eef1b6f + 91e82fb commit 391adde

File tree

38 files changed

+395
-304
lines changed

38 files changed

+395
-304
lines changed

‎OpenLdapSync/src/org/labkey/openldapsync/ldap/LdapScheduler.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
package org.labkey.openldapsync.ldap;
22

33
import org.apache.directory.api.ldap.model.exception.LdapException;
4-
import org.apache.logging.log4j.Logger;
54
import org.apache.logging.log4j.LogManager;
6-
import org.quartz.DailyTimeIntervalScheduleBuilder;
5+
import org.apache.logging.log4j.Logger;
76
import org.quartz.JobBuilder;
87
import org.quartz.JobDetail;
8+
import org.quartz.SimpleScheduleBuilder;
99
import org.quartz.Trigger;
1010
import org.quartz.TriggerBuilder;
1111
import org.quartz.impl.StdSchedulerFactory;
@@ -62,7 +62,7 @@ public synchronized void schedule()
6262

6363
Trigger trigger = TriggerBuilder.newTrigger()
6464
.withIdentity(LdapScheduler.class.getCanonicalName(), LdapScheduler.class.getCanonicalName())
65-
.withSchedule(DailyTimeIntervalScheduleBuilder.dailyTimeIntervalSchedule().withIntervalInHours(_frequency))
65+
.withSchedule(SimpleScheduleBuilder.simpleSchedule().withIntervalInHours(_frequency).repeatForever())
6666
.forJob(_job)
6767
.build();
6868

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public static File getIndexDir(ReferenceGenome genome, String name, boolean useW
6060
}
6161

6262
/**
63-
* If WorkDirectory is null, files will not be copied. Otherwise files be be copied to this destination.
63+
* If WorkDirectory is null, files will not be copied. Otherwise, files will be copied to this destination.
6464
*/
6565
private static boolean verifyOrCreateCachedIndex(PipelineContext ctx, @Nullable WorkDirectory wd, @Nullable AlignmentOutputImpl output, String localName, String webserverName, ReferenceGenome genome, boolean forceCopyLocal) throws PipelineJobException
6666
{
@@ -78,7 +78,7 @@ private static boolean verifyOrCreateCachedIndex(PipelineContext ctx, @Nullable
7878
File lockFile = new File(webserverIndexDir.getPath() + ".copyLock");
7979
if (lockFile.exists())
8080
{
81-
ctx.getLogger().error("Another job is actively saving this cached index. This job will skip that step; however, if this job tries to start alignment and use the index before copy is complete this might cause issues.");
81+
throw new PipelineJobException("Another job is actively saving this cached index. This error is being thrown as a precaution to avoid duplicate rsync jobs, and to prevent this job from progressing file that copy is in-progress. This job can be restarted after the copy is complete, and should resume normally.");
8282
}
8383

8484
hasCachedIndex = true;

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java‎

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -48,21 +48,6 @@ public void setMaxRamOverride(Integer maxRamOverride)
4848
_maxRamOverride = maxRamOverride;
4949
}
5050

51-
public void addJava8HomeToEnvironment()
52-
{
53-
//since GATK requires java8, set JAVA_HOME to match this:
54-
File java8 = new File(SequencePipelineService.get().getJava8FilePath()).getParentFile();
55-
if (java8.getParentFile() == null)
56-
{
57-
getLogger().debug("unexpected path to java8, cannot determine JAVA_HOME: " + java8.getPath());
58-
return;
59-
}
60-
61-
String javaDir = java8.getParentFile().getPath();
62-
getLogger().debug("setting JAVA_HOME to java8 location: " + javaDir);
63-
addToEnvironment("JAVA_HOME", javaDir);
64-
}
65-
6651
public boolean jarExists()
6752
{
6853
return getJAR(false) != null;
@@ -80,8 +65,9 @@ protected void ensureDictionary(File referenceFasta) throws PipelineJobException
8065
public String getVersionString() throws PipelineJobException
8166
{
8267
List<String> args = new ArrayList<>();
83-
args.add(SequencePipelineService.get().getJava8FilePath());
68+
args.add(SequencePipelineService.get().getJavaFilepath());
8469
args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride));
70+
args.add("-DGATK_STACKTRACE_ON_USER_EXCEPTION=true");
8571
args.add("-jar");
8672
args.add(getJAR().getPath());
8773
args.add("--version");
@@ -102,7 +88,7 @@ protected String getPackageName()
10288
public List<String> getBaseArgs(@Nullable String toolName)
10389
{
10490
List<String> args = new ArrayList<>();
105-
args.add(SequencePipelineService.get().getJava8FilePath());
91+
args.add(SequencePipelineService.get().getJavaFilepath());
10692
args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride));
10793
args.add("-jar");
10894
args.add(getJAR().getPath());

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatkWrapper.java‎

Lines changed: 0 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -54,38 +54,6 @@ public void setMaxRamOverride(Integer maxRamOverride)
5454
_maxRamOverride = maxRamOverride;
5555
}
5656

57-
protected void addJavaHomeToEnvironment()
58-
{
59-
//since GATK requires java8, set JAVA_HOME to match this:
60-
File java8 = new File(SequencePipelineService.get().getJava8FilePath()).getParentFile();
61-
if (java8.getParentFile() == null)
62-
{
63-
getLogger().debug("unexpected path to java8, cannot determine JAVA_HOME: " + java8.getPath());
64-
return;
65-
}
66-
67-
String javaDir = java8.getParentFile().getPath();
68-
getLogger().debug("setting JAVA_HOME to java8 location: " + javaDir);
69-
addToEnvironment("JAVA_HOME", javaDir);
70-
}
71-
72-
protected File getQueueJAR()
73-
{
74-
String path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath("GATKPATH");
75-
if (path != null)
76-
{
77-
return new File(path);
78-
}
79-
80-
path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(SequencePipelineService.SEQUENCE_TOOLS_PARAM);
81-
if (path == null)
82-
{
83-
path = PipelineJobService.get().getAppProperties().getToolsDirectory();
84-
}
85-
86-
return path == null ? new File("Queue.jar") : new File(path, "Queue.jar");
87-
}
88-
8957
public boolean jarExists()
9058
{
9159
return getJAR() == null || !getJAR().exists();
@@ -100,69 +68,6 @@ protected void ensureDictionary(File referenceFasta) throws PipelineJobException
10068
new CreateSequenceDictionaryWrapper(getLogger()).execute(referenceFasta, false);
10169
}
10270

103-
public String getVersionString() throws PipelineJobException
104-
{
105-
List<String> args = new ArrayList<>();
106-
args.add(SequencePipelineService.get().getJava8FilePath());
107-
args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride));
108-
args.add("-jar");
109-
args.add(getJAR().getPath());
110-
args.add("--version");
111-
112-
return StringUtils.trimToNull(executeWithOutput(args));
113-
}
114-
115-
public Integer getMinRamPerQueueJob()
116-
{
117-
return _minRamPerQueueJob;
118-
}
119-
120-
public void setMinRamPerQueueJob(Integer minRamPerQueueJob)
121-
{
122-
_minRamPerQueueJob = minRamPerQueueJob;
123-
}
124-
125-
protected Integer getScatterForQueueJob()
126-
{
127-
// NOTE: Queue will create n number of jobs, dividing memory evenly between them. Because it is possible
128-
// to submit a job w/ lower available RAM and comparably high CPUs, this could result in queue not having enough memory per job.
129-
// therefore do a quick check and potentially scale down scatter
130-
Integer maxThreads = SequencePipelineService.get().getMaxThreads(getLogger());
131-
if (maxThreads != null)
132-
{
133-
if (_minRamPerQueueJob != null && _minRamPerQueueJob > 0)
134-
{
135-
String maxRamSetting = StringUtils.trimToNull(System.getenv("SEQUENCEANALYSIS_MAX_RAM"));
136-
if (maxRamSetting != null)
137-
{
138-
try
139-
{
140-
Integer maxRamAllowed = ConvertHelper.convert(maxRamSetting, Integer.class);
141-
if (maxRamAllowed != null)
142-
{
143-
int adjusted = Math.max(maxRamAllowed / _minRamPerQueueJob, 1);
144-
if (adjusted < maxThreads)
145-
{
146-
getLogger().debug("lowering max threads to match available RAM. setting to: " + adjusted);
147-
maxThreads = adjusted;
148-
}
149-
}
150-
}
151-
catch (ConvergenceException e)
152-
{
153-
getLogger().warn("non-numeric value for SEQUENCEANALYSIS_MAX_RAM: [" + maxRamSetting + "]");
154-
}
155-
}
156-
}
157-
}
158-
else
159-
{
160-
maxThreads = 1;
161-
}
162-
163-
return maxThreads;
164-
}
165-
16671
protected List<String> getBaseArgs()
16772
{
16873
List<String> args = new ArrayList<>();

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/PicardWrapper.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public String getVersion() throws PipelineJobException
3636
}
3737

3838
List<String> params = new LinkedList<>();
39-
params.add(SequencePipelineService.get().getJava8FilePath());
39+
params.add(SequencePipelineService.get().getJavaFilepath());
4040
params.add("-jar");
4141
params.add(getJar().getPath());
4242
params.add(getToolName());
@@ -95,7 +95,7 @@ protected List<String> getBaseArgs(boolean basicArgsOnly) throws PipelineJobExce
9595
}
9696

9797
List<String> params = new LinkedList<>();
98-
params.add(SequencePipelineService.get().getJava8FilePath());
98+
params.add(SequencePipelineService.get().getJavaFilepath());
9999
params.addAll(SequencePipelineService.get().getJavaOpts());
100100
params.add("-jar");
101101
params.add(getJar().getPath());

‎SequenceAnalysis/pipeline_code/sequence_tools_install.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -276,10 +276,10 @@ then
276276
rm -Rf gatk-4*
277277
rm -Rf $LKTOOLS_DIR/GenomeAnalysisTK4.jar
278278

279-
wget $WGET_OPTS https://github.com/broadinstitute/gatk/releases/download/4.3.0.0/gatk-4.3.0.0.zip
280-
unzip gatk-4.3.0.0.zip
279+
wget $WGET_OPTS https://github.com/broadinstitute/gatk/releases/download/4.4.0.0/gatk-4.4.0.0.zip
280+
unzip gatk-4.4.0.0.zip
281281

282-
cp ./gatk-4.3.0.0/gatk-package-4.3.0.0-local.jar $LKTOOLS_DIR/GenomeAnalysisTK4.jar
282+
cp ./gatk-4.4.0.0/gatk-package-4.4.0.0-local.jar $LKTOOLS_DIR/GenomeAnalysisTK4.jar
283283
else
284284
echo "Already installed"
285285
fi
@@ -695,7 +695,7 @@ then
695695
rm -Rf $LKTOOLS_DIR/htsjdk-*
696696
rm -Rf $LKTOOLS_DIR/libIntelDeflater.so
697697

698-
wget $WGET_OPTS https://github.com/broadinstitute/picard/releases/download/2.27.4/picard.jar
698+
wget $WGET_OPTS https://github.com/broadinstitute/picard/releases/download/3.0.0/picard.jar
699699

700700
cp -R ./picard.jar $LKTOOLS_DIR/
701701
else

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -571,6 +571,10 @@ else if (_fastaFileType.isType(f))
571571
ret.add(f.getName() + ".gz.gzi");
572572
ret.add(f.getName() + ".gz.fai");
573573
}
574+
else if (new FileType("txt.gz").isType(f))
575+
{
576+
ret.add(f.getName() + ".tbi");
577+
}
574578

575579
// NOTE: this allows modules to register handlers for extra ancillary files, such as seurat metadata
576580
SequenceAnalysisServiceImpl.get().getAccessoryFileProviders().forEach(fn -> {

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheGenomePipelineJob.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.labkey.sequenceanalysis.pipeline;
22

3+
import org.apache.commons.io.FileUtils;
34
import org.apache.commons.lang3.StringUtils;
45
import org.jetbrains.annotations.NotNull;
56
import org.labkey.api.data.Container;
@@ -28,6 +29,7 @@
2829
import org.labkey.api.view.ViewContext;
2930

3031
import java.io.File;
32+
import java.io.IOException;
3133
import java.util.Arrays;
3234
import java.util.Collections;
3335
import java.util.List;
@@ -190,7 +192,17 @@ public boolean isJobComplete(PipelineJob job)
190192
if (toDelete != null && toDelete.length > 0)
191193
{
192194
getJob().getLogger().info("Folders will be deleted: " + StringUtils.join(toDelete, ", "));
193-
//TODO: verify
195+
for (File x : toDelete)
196+
{
197+
try
198+
{
199+
FileUtils.deleteDirectory(x);
200+
}
201+
catch (IOException e)
202+
{
203+
throw new PipelineJobException(e);
204+
}
205+
}
194206
}
195207
}
196208

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/run/bampostprocessing/IndelRealignerStep.java‎

Lines changed: 4 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
import java.io.File;
1818
import java.util.Arrays;
19+
import java.util.Collections;
1920

2021
/**
2122
* User: bimber
@@ -24,7 +25,7 @@
2425
*/
2526
public class IndelRealignerStep extends AbstractCommandPipelineStep<IndelRealignerWrapper> implements BamProcessingStep
2627
{
27-
public IndelRealignerStep(PipelineStepProvider provider, PipelineContext ctx)
28+
public IndelRealignerStep(PipelineStepProvider<?> provider, PipelineContext ctx)
2829
{
2930
super(provider, ctx, new IndelRealignerWrapper(ctx.getLogger()));
3031
}
@@ -33,17 +34,7 @@ public static class Provider extends AbstractPipelineStepProvider<IndelRealigner
3334
{
3435
public Provider()
3536
{
36-
super("IndelRealigner", "Indel Realigner", "GATK", "The step runs GATK's IndelRealigner tool. This tools performs local realignment to minmize the number of mismatching bases across all the reads.", Arrays.asList(
37-
ToolParameterDescriptor.create("useQueue", "Use Queue?", "If checked, this tool will attempt to run using GATK queue, allowing parallelization using scatter/gather.", "checkbox", new JSONObject()
38-
{{
39-
put("checked", false);
40-
}}, false),
41-
//TODO: consider supporting:
42-
//--maxReadsForRealignment
43-
//--maxReadsForConsensuses
44-
45-
ToolParameterDescriptor.create("minRamPerQueueJob", "Min RAM Per Queue Job", "This only applies if queue is checked. If provided, the scatter count (number of jobs) for queue will be adjusted to ensure at least this amount of RAM, in GB, is available for each job", "ldk-integerfield", null, null)
46-
), null, "http://www.broadinstitute.org/gatk/gatkdocs/org_broadinstitute_sting_gatk_walkers_indels_IndelRealigner.html");
37+
super("IndelRealigner", "Indel Realigner", "GATK", "The step runs GATK's IndelRealigner tool. This tools performs local realignment to minmize the number of mismatching bases across all the reads.", Collections.emptyList(), null, "http://www.broadinstitute.org/gatk/gatkdocs/org_broadinstitute_sting_gatk_walkers_indels_IndelRealigner.html");
4738
}
4839

4940
@Override
@@ -65,22 +56,7 @@ public Output processBam(Readset rs, File inputBam, ReferenceGenome referenceGen
6556
getPipelineCtx().getLogger().debug("dict exists: " + preExistingDictionary + ", " + dictionary.getPath());
6657

6758
File outputBam = new File(outputDirectory, FileUtil.getBaseName(inputBam) + ".realigned.bam");
68-
File created;
69-
if (getProvider().getParameterByName("useQueue").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false))
70-
{
71-
Integer minRamPerQueueJob = getProvider().getParameterByName("minRamPerQueueJob").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class);
72-
if (minRamPerQueueJob != null)
73-
{
74-
getWrapper().setMinRamPerQueueJob(minRamPerQueueJob);
75-
}
76-
77-
created = getWrapper().executeWithQueue(inputBam, outputBam, referenceGenome.getWorkingFastaFile(), null);
78-
}
79-
else
80-
{
81-
created = getWrapper().execute(inputBam, outputBam, referenceGenome.getWorkingFastaFile(), null);
82-
}
83-
59+
File created = getWrapper().execute(inputBam, outputBam, referenceGenome.getWorkingFastaFile(), null);
8460
if (created != null)
8561
{
8662
output.setBAM(created);

0 commit comments

Comments
 (0)