Merge pull request #142 from LabKey/fb_merge_22.7_to_develop

Merge 22.7 to develop

Author: bbimber

mGAP/src/org/labkey/mgap/pipeline/RenameSamplesForMgapStep.java

@@ -2,6 +2,7 @@
 
 import au.com.bytecode.opencsv.CSVReader;
 import au.com.bytecode.opencsv.CSVWriter;
+import com.google.common.collect.Lists;
 import htsjdk.samtools.util.CloseableIterator;
 import htsjdk.samtools.util.Interval;
 import htsjdk.variant.utils.SAMSequenceDictionaryExtractor;
@@ -250,46 +251,22 @@ private Map<String, String> getSamplesToAlias(File input) throws PipelineJobExce
                 return Collections.emptyMap();
             }
 
-            TableInfo ti = QueryService.get().getUserSchema(getPipelineCtx().getJob().getUser(), (getPipelineCtx().getJob().getContainer().isWorkbook() ? getPipelineCtx().getJob().getContainer().getParent() : getPipelineCtx().getJob().getContainer()), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_ANIMAL_MAPPING);
-            TableSelector ts = new TableSelector(ti, PageFlowUtil.set("subjectname", "externalAlias", "otherNames"), new SimpleFilter(FieldKey.fromString("subjectname"), subjects, CompareType.IN), null);
-            ts.forEachResults(new Selector.ForEachBlock<Results>()
-            {
-                @Override
-                public void exec(Results rs) throws SQLException
-                {
-                    sampleNameMap.put(rs.getString(FieldKey.fromString("subjectname")), rs.getString(FieldKey.fromString("externalAlias")));
-
-                    if (rs.getObject(FieldKey.fromString("otherNames")) != null)
-                    {
-                        String val = StringUtils.trimToNull(rs.getString(FieldKey.fromString("otherNames")));
-                        if (val != null)
-                        {
-                            String[] tokens = val.split(",");
-                            for (String name : tokens)
-                            {
-                                name = StringUtils.trimToNull(name);
-                                if (name == null)
-                                {
-                                    continue;
-                                }
-
-                                if (sampleNameMap.containsKey(name) && !sampleNameMap.get(name).equals(rs.getString(FieldKey.fromString("externalAlias"))))
-                                {
-                                    throw new IllegalStateException("Improper data in mgap.aliases table. Dual/conflicting aliases: " + name + ": " + rs.getString(FieldKey.fromString("externalAlias")) + " / " + sampleNameMap.get(name));
-                                }
-
-                                sampleNameMap.put(name, rs.getString(FieldKey.fromString("externalAlias")));
-                            }
-                        }
-                    }
-                }
-            });
-
             Set<String> sampleNames = new HashSet<>(header.getSampleNamesInOrder());
             getPipelineCtx().getLogger().info("total samples in input VCF: " + sampleNames.size());
 
-            sampleNames.retainAll(subjects);
-            getPipelineCtx().getLogger().info("total samples to be written to any track: " + sampleNames.size());
+            // Pass 1: match on proper ID:
+            querySampleBatch(sampleNameMap, new SimpleFilter(FieldKey.fromString("subjectname"), subjects, CompareType.IN));
+
+            // Pass 2: add others using otherNames:
+            List<String> missingSamples = new ArrayList<>(sampleNames);
+            missingSamples.removeAll(sampleNameMap.keySet());
+            if (!missingSamples.isEmpty())
+            {
+                getPipelineCtx().getLogger().debug("Querying " + missingSamples.size() + " samples using otherNames field");
+                querySampleBatch(sampleNameMap, new SimpleFilter(FieldKey.fromString("otherNames"), missingSamples, CompareType.CONTAINS_ONE_OF));
+            }
+
+            getPipelineCtx().getLogger().info("total sample names to alias: " + sampleNameMap.size());
 
             sampleNames.removeAll(sampleNameMap.keySet());
             if (!sampleNames.isEmpty())
@@ -298,17 +275,54 @@ public void exec(Results rs) throws SQLException
             }
 
             //Now ensure we dont have duplicate mappings:
-            List<String> translated = new ArrayList<>(sampleNames.stream().map(sampleNameMap::get).collect(Collectors.toList()));
+            List<String> translated = new ArrayList<>(header.getSampleNamesInOrder().stream().map(sampleNameMap::get).toList());
             Set<String> unique = new HashSet<>();
-            List<String> duplicates = translated.stream().filter(o -> !unique.add(o)).collect(Collectors.toList());
+            List<String> duplicates = translated.stream().filter(o -> !unique.add(o)).toList();
             if (!duplicates.isEmpty())
             {
                 throw new PipelineJobException("There were duplicate mGAP IDs are translation. They were: " + StringUtils.join(duplicates, ","));
             }
         }
 
-        getPipelineCtx().getLogger().info("total sample names to alias: " + sampleNameMap.size());
-
         return sampleNameMap;
     }
+
+    private void querySampleBatch(Map<String, String> sampleNameMap, SimpleFilter filter)
+    {
+        TableInfo ti = QueryService.get().getUserSchema(getPipelineCtx().getJob().getUser(), (getPipelineCtx().getJob().getContainer().isWorkbook() ? getPipelineCtx().getJob().getContainer().getParent() : getPipelineCtx().getJob().getContainer()), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_ANIMAL_MAPPING);
+        TableSelector ts = new TableSelector(ti, PageFlowUtil.set("subjectname", "externalAlias", "otherNames"), new SimpleFilter(filter), null);
+        ts.forEachResults(new Selector.ForEachBlock<Results>()
+        {
+            @Override
+            public void exec(Results rs) throws SQLException
+            {
+                sampleNameMap.put(rs.getString(FieldKey.fromString("subjectname")), rs.getString(FieldKey.fromString("externalAlias")));
+
+                if (rs.getObject(FieldKey.fromString("otherNames")) != null)
+                {
+                    String val = StringUtils.trimToNull(rs.getString(FieldKey.fromString("otherNames")));
+                    if (val != null)
+                    {
+                        String[] tokens = val.split(",");
+                        for (String name : tokens)
+                        {
+                            name = StringUtils.trimToNull(name);
+                            if (name == null)
+                            {
+                                continue;
+                            }
+
+                            if (sampleNameMap.containsKey(name) && !sampleNameMap.get(name).equals(rs.getString(FieldKey.fromString("externalAlias"))))
+                            {
+                                throw new IllegalStateException("Improper data in mgap.aliases table. Dual/conflicting aliases: " + name + ": " + rs.getString(FieldKey.fromString("externalAlias")) + " / " + sampleNameMap.get(name));
+                            }
+
+                            getPipelineCtx().getLogger().debug("Adding otherName: " + name);
+                            sampleNameMap.put(name, rs.getString(FieldKey.fromString("externalAlias")));
+                        }
+                    }
+                }
+            }
+        });
+    }
 }

mGAP/src/org/labkey/mgap/pipeline/SampleSpecificGenotypeFiltrationStep.java

@@ -11,6 +11,7 @@
 import org.labkey.api.sequenceanalysis.SequenceOutputFile;
 import org.labkey.api.sequenceanalysis.model.Readset;
 import org.labkey.api.sequenceanalysis.pipeline.AbstractVariantProcessingStepProvider;
+import org.labkey.api.sequenceanalysis.pipeline.CommandLineParam;
 import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
 import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider;
 import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
@@ -64,8 +65,11 @@ public Provider()
                     }}, null),
                     ToolParameterDescriptor.create("wxsMinQual", "WXS Min Qual", "The min genotype qual for WXS samples.", "ldk-integerfield", new JSONObject(){{
                         put("minValue", 0);
-                    }}, 30)
-                    ), null, "");
+                    }}, 30),
+                    ToolParameterDescriptor.createCommandLineParam(CommandLineParam.createSwitch("--set-filtered-genotype-to-no-call"), "setFilteredGtToNocall", "Set Filtered Genotypes to No-Call", "If selected, any filtered genotypes will be converted to no-call.", "checkbox", new JSONObject(){{
+                        put("checked", true);
+                    }}, true)
+            ), null, "");
         }
 
         @Override

mcc/.npmrc

@@ -1 +1 @@
-@labkey:registry=https://labkey.jfrog.io/artifactory/api/npm/libs-client
+@labkey:registry=https://labkey.jfrog.io/artifactory/api/npm/libs-client