Merge pull request #196 from BimberLab/23.3_fb_merge

Merge discvr-22.11 into 23.3

Author: bbimber

OpenLdapSync/src/org/labkey/openldapsync/ldap/LdapScheduler.java

@@ -1,11 +1,11 @@
 package org.labkey.openldapsync.ldap;
 
 import org.apache.directory.api.ldap.model.exception.LdapException;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
-import org.quartz.DailyTimeIntervalScheduleBuilder;
+import org.apache.logging.log4j.Logger;
 import org.quartz.JobBuilder;
 import org.quartz.JobDetail;
+import org.quartz.SimpleScheduleBuilder;
 import org.quartz.Trigger;
 import org.quartz.TriggerBuilder;
 import org.quartz.impl.StdSchedulerFactory;
@@ -62,7 +62,7 @@ public synchronized void schedule()
 
             Trigger trigger = TriggerBuilder.newTrigger()
                     .withIdentity(LdapScheduler.class.getCanonicalName(), LdapScheduler.class.getCanonicalName())
-                    .withSchedule(DailyTimeIntervalScheduleBuilder.dailyTimeIntervalSchedule().withIntervalInHours(_frequency))
+                    .withSchedule(SimpleScheduleBuilder.simpleSchedule().withIntervalInHours(_frequency).repeatForever())
                     .forJob(_job)
                     .build();
 

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java

@@ -60,7 +60,7 @@ public static File getIndexDir(ReferenceGenome genome, String name, boolean useW
     }
 
     /**
-     * If WorkDirectory is null, files will not be copied.  Otherwise files be be copied to this destination.
+     * If WorkDirectory is null, files will not be copied.  Otherwise, files will be copied to this destination.
      */
     private static boolean verifyOrCreateCachedIndex(PipelineContext ctx, @Nullable WorkDirectory wd, @Nullable AlignmentOutputImpl output, String localName, String webserverName, ReferenceGenome genome, boolean forceCopyLocal) throws PipelineJobException
     {
@@ -78,7 +78,7 @@ private static boolean verifyOrCreateCachedIndex(PipelineContext ctx, @Nullable
                 File lockFile = new File(webserverIndexDir.getPath() + ".copyLock");
                 if (lockFile.exists())
                 {
-                    ctx.getLogger().error("Another job is actively saving this cached index.  This job will skip that step; however, if this job tries to start alignment and use the index before copy is complete this might cause issues.");
+                    throw new PipelineJobException("Another job is actively saving this cached index.  This error is being thrown as a precaution to avoid duplicate rsync jobs, and to prevent this job from progressing file that copy is in-progress. This job can be restarted after the copy is complete, and should resume normally.");
                 }
 
                 hasCachedIndex = true;

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheGenomePipelineJob.java

@@ -1,5 +1,6 @@
 package org.labkey.sequenceanalysis.pipeline;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.jetbrains.annotations.NotNull;
 import org.labkey.api.data.Container;
@@ -28,6 +29,7 @@
 import org.labkey.api.view.ViewContext;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -190,7 +192,17 @@ public boolean isJobComplete(PipelineJob job)
                 if (toDelete != null && toDelete.length > 0)
                 {
                     getJob().getLogger().info("Folders will be deleted: " + StringUtils.join(toDelete, ", "));
-                    //TODO: verify
+                    for (File x : toDelete)
+                    {
+                        try
+                        {
+                            FileUtils.deleteDirectory(x);
+                        }
+                        catch (IOException e)
+                        {
+                            throw new PipelineJobException(e);
+                        }
+                    }
                 }
             }
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/VariantAnnotatorStep.java

@@ -43,6 +43,9 @@ public Provider()
                     }}, null),
                     ToolParameterDescriptor.create("maf", "Minor Allele Frequency", "If selected, MAF will be annotated.", "checkbox", new JSONObject(){{
                         put("checked", true);
+                    }}, null),
+                    ToolParameterDescriptor.create("chromosomeCounts", "Chromosome Counts", "If selected, GATK ChromosomeCounts annotations, including AF and AC, will be annotated.", "checkbox", new JSONObject(){{
+                        put("checked", false);
                     }}, null)
             ), null, "");
         }
@@ -88,6 +91,12 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
             options.add("MinorAlleleFrequency");
         }
 
+        if (getProvider().getParameterByName("chromosomeCounts").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false))
+        {
+            options.add("-A");
+            options.add("ChromosomeCounts");
+        }
+
         if (intervals != null)
         {
             intervals.forEach(interval -> {

cluster/src/org/labkey/cluster/pipeline/AbstractClusterExecutionEngine.java

@@ -101,8 +101,8 @@ public void submitJob(PipelineJob job) throws PipelineJobException
                 }
                 else
                 {
-                    job.getLogger().error("duplicate submission attempt, skipping.  original cluster id: " + existingSubmission.getClusterId());
-                    _log.error("duplicate submission attempt, skipping.  original cluster id: " + existingSubmission.getClusterId() + ", job id: " + job.getJobGUID());
+                    job.getLogger().error("duplicate submission attempt, skipping.  original cluster id: " + existingSubmission.getClusterId() + ", with status: " + existingSubmission.getStatus());
+                    _log.error("duplicate submission attempt, skipping.  original cluster id: " + existingSubmission.getClusterId() + ", job id: " + job.getJobGUID() + ", with status: " + existingSubmission.getStatus());
                     return;
                 }
             }

jbrowse/resources/web/jbrowse/window/ModifyJsonConfigWindow.js

@@ -176,6 +176,52 @@ Ext4.define('JBrowse.window.ModifyJsonConfigWindow', {
         });
 
         this.callParent(arguments);
+
+        if (this.jsonFiles.length === 1) {
+            Ext4.Msg.wait('Loading...');
+            LABKEY.Query.selectRows({
+                schemaName: 'jbrowse',
+                queryName: 'jsonfiles',
+                filterArray: [
+                    LABKEY.Filter.create('objectid', this.jsonFiles.join(';'), LABKEY.Filter.Types.EQUALS_ONE_OF)
+                ],
+                columns: 'objectid,trackjson',
+                scope: this,
+                success: this.onSelectLoad,
+                failure: LDK.Utils.getErrorCallback()
+            });
+        }
+    },
+
+    onSelectLoad: function(results){
+        Ext4.Msg.hide();
+        if (results.rows && results.rows.length) {
+            if (!results.rows[0].trackJson) {
+                return;
+            }
+
+            var json = JSON.parse(results.rows[0].trackJson);
+            var store = this.down('ldk-gridpanel').store;
+            for (var attribute in json) {
+                var val = json[attribute];
+                var dataType = 'STRING';
+                if (typeof val === 'number' && !isNaN(val)){
+                    dataType = Number.isInteger(val) ? 'INT' : 'FLOAT';
+                }
+                else if (typeof val === 'string') {
+                    dataType = 'STRING';
+                }
+                else if (typeof val === 'boolean') {
+                    dataType = 'BOOLEAN';
+                }
+
+                store.add(store.createModel({
+                    attribute: attribute,
+                    value: val,
+                    dataType: dataType
+                }));
+            }
+        }
     },
 
     addAttribute: function(attribute, value, dataType){

jbrowse/src/org/labkey/jbrowse/model/JsonFile.java

@@ -680,6 +680,12 @@ private JSONObject getGxfTrack(Logger log, ExpData targetFile, ReferenceGenome r
 
     public String getTrackType()
     {
+        JSONObject extraConfig = getExtraTrackConfig();
+        if (extraConfig != null && extraConfig.has("type"))
+        {
+            return extraConfig.getString("type");
+        }
+
         ExpData targetFile = getExpData();
         if (TRACK_TYPES.vcf.getFileType().isType(targetFile.getFile()))
         {
@@ -831,7 +837,7 @@ public File prepareResource(Logger log, boolean throwIfNotPrepared, boolean forc
                 {
                     if (!targetFile.getParentFile().equals(finalLocation.getParentFile()))
                     {
-                        // Make local copy so we dont delete the original
+                        log.debug("Creating local copy of: " + targetFile.getPath());
                         File local = new File(finalLocation.getParentFile(), targetFile.getName());
                         if (local.exists())
                         {

jbrowse/src/org/labkey/jbrowse/pipeline/JBrowseSessionTask.java

@@ -256,6 +256,8 @@ private void createOrAddToSession() throws PipelineJobException
                     trackRecord.put("modifiedby", getJob().getUser().getUserId());
                     trackRecord = Table.insert(getJob().getUser(), databaseMembers, trackRecord);
                     databaseMemberRecordsCreated.add((Integer)trackRecord.get("rowid"));
+
+                    json.prepareResource(getJob().getLogger(), false, false);
                 }
             }
             else
@@ -305,6 +307,8 @@ private void createOrAddToSession() throws PipelineJobException
                     trackRecord.put("modifiedby", getJob().getUser().getUserId());
                     trackRecord = Table.insert(getJob().getUser(), databaseMembers, trackRecord);
                     databaseMemberRecordsCreated.add((Integer) trackRecord.get("rowid"));
+
+                    json.prepareResource(getJob().getLogger(), false, false);
                 }
             }
 
@@ -336,6 +340,7 @@ private void createOrAddToSession() throws PipelineJobException
                 }
             }
         }
+
     }
 
     private JBrowseSessionPipelineJob getPipelineJob()

singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java

@@ -479,7 +479,7 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List<Seur
 
         body.addAll(loadChunkFromFile("singlecell", "chunks/Functions.R"));
 
-        return new Chunk("parameters", null, null, body);
+        return new Chunk("parameters", null, null, body, "cache=FALSE");
     }
 
     protected String printInputFile(SeuratObjectWrapper so)

singlecell/resources/chunks/CellBarcodeFilter.R

@@ -0,0 +1,27 @@
+totalPassed <- 0
+for (datasetId in names(seuratObjects)) {
+  printName(datasetId)
+  seuratObj <- readRDS(seuratObjects[[datasetId]])
+
+  cellsToKeep <- colnames(seuratObj)
+  if (!all(is.null(cellbarcodesToDrop))) {
+    cellsToKeep <- cellsToKeep[!cellsToKeep %in% cellbarcodesToDrop]
+  }
+
+  if (length(cellsToKeep) == 0) {
+    print('There were no matching cells')
+  } else {
+    print(paste0('Total passing cells: ', length(cellsToKeep)))
+    seuratObj <- subset(seuratObj, cells = cellsToKeep)
+    saveData(seuratObj, datasetId)
+    totalPassed <- totalPassed + 1
+  }
+
+  # Cleanup
+  rm(seuratObj)
+  gc()
+}
+
+if (totalPassed == 0) {
+  addErrorMessage('No cells remained in any seurat objects after subsetting')
+}