When running GenomicsDB, allow memory buffer for C++ layer

Author: bbimber

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java

@@ -21,6 +21,7 @@
 import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
 import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
 import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
+import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.sequenceanalysis.pipeline.TaskFileManager;
 import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
 import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStep;
@@ -581,6 +582,21 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
                     FileUtils.touch(startedFile);
 
                     List<Interval> intervals = getIntervals(ctx);
+
+                    Integer maxRam = SequencePipelineService.get().getMaxRam();
+                    Integer nativeMemoryBuffer = ctx.getParams().optInt("nativeMemoryBuffer", 0);
+                    if (maxRam != null && nativeMemoryBuffer > 0)
+                    {
+                        ctx.getLogger().info("Adjusting RAM based on memory buffer (" + nativeMemoryBuffer + ")");
+                        maxRam = maxRam - nativeMemoryBuffer;
+
+                        if (maxRam < 1)
+                        {
+                            throw new PipelineJobException("After adjusting for nativeMemoryBuffer, maxRam is less than 1: " + maxRam);
+                        }
+                        wrapper.setMaxRamOverride(maxRam);
+                    }
+
                     wrapper.execute(genome, vcfsToProcess, destinationWorkspaceFolder, intervals, options, _append);
 
                     FileUtils.touch(doneFile);

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/GenomicsDBAppendHandler.java

@@ -35,6 +35,9 @@ public GenomicsDBAppendHandler()
                 ToolParameterDescriptor.create("sharedPosixOptimizations", "Use Shared Posix Optimizations", "This enabled optimizations for large shared filesystems, such as lustre.", "checkbox", new JSONObject(){{
                     put("checked", true);
                 }}, true),
+                ToolParameterDescriptor.create("nativeMemoryBuffer", "C++ Memory Buffer", "By default, the pipeline java processes are allocated nearly all of the requested RAM.  GenomicsDB requires memory for the C++ layer - this value (in GB) will be reserved for this.  We recommend about 15-25% of the total job RAM", "checkbox", new JSONObject(){{
+                    put("minValue", 0);
+                }}, 24),
                 ToolParameterDescriptor.create("scatterGather", "Scatter/Gather Options", "If selected, this job will be divided to run job per chromosome.  The final step will take the VCF from each intermediate step and combined to make a final VCF file.", "sequenceanalysis-variantscattergatherpanel", new JSONObject(){{
                     put("defaultValue", "chunked");
                 }}, false)

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/GenomicsDBImportHandler.java

@@ -30,6 +30,9 @@ public GenomicsDBImportHandler()
                 ToolParameterDescriptor.create("sharedPosixOptimizations", "Use Shared Posix Optimizations", "This enabled optimizations for large shared filesystems, such as lustre.", "checkbox", new JSONObject(){{
                     put("checked", true);
                 }}, true),
+                ToolParameterDescriptor.create("nativeMemoryBuffer", "C++ Memory Buffer", "By default, the pipeline java processes are allocated nearly all of the requested RAM.  GenomicsDB requires memory for the C++ layer - this value (in GB) will be reserved for this.  We recommend about 15-25% of the total job RAM", "checkbox", new JSONObject(){{
+                    put("minValue", 0);
+                }}, 24),
                 ToolParameterDescriptor.create("scatterGather", "Scatter/Gather Options", "If selected, this job will be divided to run job per chromosome.  The final step will take the VCF from each intermediate step and combined to make a final VCF file.", "sequenceanalysis-variantscattergatherpanel", new JSONObject(){{
                     put("defaultValue", "chunked");
                 }}, false)