Improve handling of mGAP releases with or without lucene indexes

Author: bbimber

mGAP/resources/queries/mGAP/variantCatalogReleases/.qview.xml

@@ -9,5 +9,6 @@
     </columns>
     <sorts>
         <sort column="version" descending="true"/>
+        <sort column="releaseDate" descending="true"/>
     </sorts>
 </customView>

mGAP/resources/views/overview.html

@@ -7,6 +7,10 @@
     (function($){
         $( document ).ready(function() {
             mGAP.Utils.renderReleaseGraph('mgap-graph-outer', 300);
+
+            if (mGAP.Utils.releaseHasLuceneIndex()) {
+                $('li.mgap-lucene-link').removeAttr("hidden");
+            }
         });
     })(jQuery);
 </script>
@@ -31,7 +35,7 @@ <h4><span style="text-decoration: underline">Short Variant Catalog:</span> <img
         <br>
         <ul style="padding-top: 10px;">
             <li>Use the <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mGap-genomeBrowser.view?">Genome Browser</a> to view and search data. <i class="fa-solid fa-video mgap-video-icon" data-video="genome-browser" data-video-title="Genome Browser Overview"></i></li>
-<!--            <li>Use our new <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mGap-genomeBrowser.view?target=variantSearch">Full-text Search</a> tool to query variants based on gene, or using our <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mgap-annotation.view">extensive annotations</a> &lt;!&ndash;<i class="fa-solid fa-video mgap-video-icon" data-video="variant-search" data-video-title="Variant Full-text Search (BETA)"></i>&ndash;&gt;</li>-->
+            <li class="mgap-lucene-link" hidden="hidden">Use our new <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mGap-genomeBrowser.view?target=variantSearch">Full-text Search</a> tool to query variants based on gene, or using our <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mgap-annotation.view">extensive annotations</a> <!--<i class="fa-solid fa-video mgap-video-icon" data-video="variant-search" data-video-title="Variant Full-text Search (BETA)"></i>--></li>
             <li>View our list of <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mGap-variantList.view">Predicted Damaging Variants</a>, which is a list of predicted high-impact or disease associated variants, generated from each release <i class="fa-solid fa-video mgap-video-icon" data-video="predicted-damaging-variants" data-video-title="Predicted Damaging Variants"></i></li>
             <li>Unlike many datasets, mGAP has genotype-level data, <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/project-begin.view?pageId=clinical">often connected to living animals from pedigreed breeding colonies</a></li>
             <li>Download raw data, including <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/project-begin.view?pageId=datasets">sequence data</a> and <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/project-begin.view?pageId=variants">variant data</a></li>

mGAP/resources/views/quickLinks.html

@@ -10,14 +10,15 @@
         var releaseId = mGAP.Utils.getMGapReleaseId();
         var jbrowseId = mGAP.Utils.getMGapJBrowseSession();
         var humanJbrowseId = mGAP.Utils.getHumanMGapJBrowseSession();
+        var releaseHasLuceneIndex = mGAP.Utils.releaseHasLuceneIndex();
         if (releaseId){
             Ext4.fly(Ext4.query('#currentReleaseDiv')[0]).update(
                 '<span>Current Release:</span>' +
                 '<ul class="mgapList">' +
                     '<li><a href="' + LABKEY.ActionURL.buildURL('mgap', 'variantList', null) + '">Predicted Damaging Variants</a></li>' +
                     (jbrowseId ? '<li><a href="' + LABKEY.ActionURL.buildURL('jbrowse', 'browser', null, {database: jbrowseId}) + '">Genome Browser (Macaque)</a></li>' : '') +
                     (humanJbrowseId ? '<li><a href="' + LABKEY.ActionURL.buildURL('jbrowse', 'browser', null, {database: humanJbrowseId}) + '">Genome Browser (Human)</a></li>' : '') +
-                    // '<li><a href="' + LABKEY.ActionURL.buildURL('mgap', 'genomeBrowser', null, {target: 'variantSearch'}) + '">Full-text Variant Search (BETA)</a></li>' +
+                    (releaseHasLuceneIndex ? '<li><a href="' + LABKEY.ActionURL.buildURL('mgap', 'genomeBrowser', null, {target: 'variantSearch'}) + '">Full-text Variant Search (BETA)</a></li>' : '') +
                 '</ul>'
             );
         }

mGAP/resources/web/mGAP/DownloadWindow.js

@@ -39,7 +39,7 @@ Ext4.define('mGAP.window.DownloadWindow', {
             method: 'POST',
             schemaName: 'mgap',
             queryName: 'variantCatalogReleases',
-            columns: 'vcfId/dataid/Name,vcfId/library_id/fasta_file/Name,sitesOnlyVcfId/dataid/Name',
+            columns: 'objectId,vcfId/dataid/Name,vcfId/library_id/fasta_file/Name,sitesOnlyVcfId/dataid/Name',
             filterArray: [LABKEY.Filter.create('rowid', this.releaseId, LABKEY.Filter.Types.EQUAL)],
             scope: this,
             failure: LDK.Utils.getErrorCallback(),
@@ -52,9 +52,11 @@ Ext4.define('mGAP.window.DownloadWindow', {
                     return;
                 }
 
-                var releaseVcf = results.rows[0]['vcfId/dataid/Name'];
+                LDK.Assert.assertNotEmpty('Missing objectId variantCatalogReleases', results.rows[0].objectId);
+
+                var releaseVcf = results.rows[0].objectId + '/' + results.rows[0]['vcfId/dataid/Name'];
                 var urlFasta = results.rows[0]['vcfId/library_id/fasta_file/Name'];
-                var sitesOnlyVcf = results.rows[0]['sitesOnlyVcfId/dataid/Name'];
+                var sitesOnlyVcf = results.rows[0].objectId + '/' + results.rows[0]['sitesOnlyVcfId/dataid/Name'];
 
                 var toAdd = [{
                     html: 'Due to the large file size, the preferred option is to download using wget or curl on the command line, such as the exmaples below. Nonetheless, you also are able to paste the URLs into your browser and download through this way as well, although it will be slower and possibly not able to resume if your connection is disrupted.<br><br>' +

mGAP/resources/web/mGAP/Utils.js

@@ -221,6 +221,12 @@ mGAP.Utils = (function($){
             return ctx['mgapReleaseId'];
         },
 
+        releaseHasLuceneIndex: function(){
+            var ctx = LABKEY.getModuleContext('mgap') || {};
+
+            return !!ctx['luceneIndexId'];
+        },
+
         getMGapReleaseVersion: function(){
             var ctx = LABKEY.getModuleContext('mgap') || {};
 

mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java

@@ -270,7 +270,7 @@ private String getOrCreateJsonFile(Results rs, String fieldKey) throws SQLExcept
 
             if (isDefaultTrack)
             {
-                row.put("trackJson", getTrackJson());
+                row.put("trackJson", getTrackJson(rs));
             }
             else
             {
@@ -307,9 +307,24 @@ protected String getDatabaseName()
         return "mGAP Release: " + getInputValue("version");
     }
 
-    protected String getTrackJson()
+    protected String getTrackJson(Results rs)
     {
-        ArrayList<String> infoFields = new TableSelector(QueryService.get().getUserSchema(getContainerUser().getUser(), getContainerUser().getContainer(), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_ANNOTATIONS), PageFlowUtil.set("infoKey"), new SimpleFilter(FieldKey.fromString("isIndexed"), true), null).getArrayList(String.class);
-        return "{\"category\":\"mGAP Variant Catalog\",\"visibleByDefault\": true,\"ensemblId\":\"Macaca_mulatta\",\"additionalFeatureMsg\":\"<h2>**The annotations below are primarily derived from human data sources (not macaque), and must be viewed in that context.</h2>\", \"createFullTextIndex\": true,\"infoFieldsForFullTextSearch\":\"" + (infoFields.isEmpty() ? "null" : StringUtils.join(infoFields, ",")) + "\"}";
+        String indexString = "";
+
+        try
+        {
+            String luceneIndex = StringUtils.trimToNull(rs.getString(FieldKey.fromString("luceneIndex/dataid/DataFileUrl")));
+            if (luceneIndex != null)
+            {
+                ArrayList<String> infoFields = new TableSelector(QueryService.get().getUserSchema(getContainerUser().getUser(), getContainerUser().getContainer(), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_ANNOTATIONS), PageFlowUtil.set("infoKey"), new SimpleFilter(FieldKey.fromString("isIndexed"), true), null).getArrayList(String.class);
+                indexString = ", \"createFullTextIndex\": true,\"infoFieldsForFullTextSearch\":\"" + (infoFields.isEmpty() ? "null" : StringUtils.join(infoFields, ",")) + "\"";
+            }
+        }
+        catch (SQLException e)
+        {
+            getStatusLogger().error("Error extracting luceneIndex", e);
+        }
+
+        return "{\"category\":\"mGAP Variant Catalog\",\"visibleByDefault\": true,\"ensemblId\":\"Macaca_mulatta\",\"additionalFeatureMsg\":\"<h2>**The annotations below are primarily derived from human data sources (not macaque), and must be viewed in that context.</h2>\"" + indexString + "}";
     }
 }

mGAP/src/org/labkey/mgap/mGAPModule.java

@@ -150,7 +150,7 @@ public JSONObject getPageContextJson(ContainerUser context)
 
         SimpleFilter filter = new SimpleFilter();
         filter.addClause(ContainerFilter.current(context.getContainer()).createFilterClause(mGAPSchema.getInstance().getSchema(), FieldKey.fromString("container")));
-        TableSelector ts = new TableSelector(mGAPSchema.getInstance().getSchema().getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("rowid", "objectid", "version", "jbrowseId", "humanJbrowseId"), filter, new Sort("-releaseDate"));
+        TableSelector ts = new TableSelector(mGAPSchema.getInstance().getSchema().getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("rowid", "objectid", "version", "jbrowseId", "humanJbrowseId", "luceneIndex"), filter, new Sort("-releaseDate"));
         ts.setMaxRows(1);
         ts.forEachResults(rs -> {
             String jbrowseId = rs.getString(FieldKey.fromString("jbrowseId"));
@@ -166,11 +166,17 @@ public JSONObject getPageContextJson(ContainerUser context)
             }
 
             Integer rowId = rs.getInt(FieldKey.fromString("rowid"));
-            if (rowId != null)
+            if (rowId != null && rowId > 0)
             {
                 ret.put("mgapReleaseId", rowId);
             }
 
+            Integer luceneIndexId = rs.getInt(FieldKey.fromString("luceneIndex"));
+            if (luceneIndexId != null && luceneIndexId > 0)
+            {
+                ret.put("luceneIndexId", luceneIndexId);
+            }
+
             String releaseVersion = rs.getString(FieldKey.fromString("version"));
             if (releaseVersion != null)
             {

mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java

@@ -1,6 +1,7 @@
 package org.labkey.mgap;
 
 import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.Nullable;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerManager;
 import org.labkey.api.data.SimpleFilter;
@@ -19,8 +20,11 @@
 import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
 import org.labkey.api.util.PageFlowUtil;
 import org.labkey.api.util.SystemMaintenance;
+import org.labkey.api.writer.PrintWriters;
 
 import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -107,7 +111,8 @@ private void checkMGapFiles(Logger log, User u)
             log.error("Missing expected directory: " + dirName);
         }
 
-        releaseIds.forEach(f -> inspectReleaseFolder(f, baseDir, c, u, log, toDelete));
+        List<String> commandsToRun = new ArrayList<>();
+        releaseIds.forEach(f -> inspectReleaseFolder(f, baseDir, c, u, log, toDelete, commandsToRun));
 
         // Also verify genomes:
         Set<Integer> genomesIds = new HashSet<>(new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("genomeId")).getArrayList(Integer.class));
@@ -116,18 +121,35 @@ private void checkMGapFiles(Logger log, User u)
             try
             {
                 ReferenceGenome rg = SequenceAnalysisService.get().getReferenceGenome(genomeId, u);
-                checkSymlink(log, rg.getSourceFastaFile());
-                checkSymlink(log, new File(rg.getSourceFastaFile().getPath() + ".fai"));
-                checkSymlink(log, new File(rg.getSourceFastaFile().getPath().replace(".fasta", ".dict")));
+                checkSymlink(log, rg.getSourceFastaFile(), null, commandsToRun);
+                checkSymlink(log, new File(rg.getSourceFastaFile().getPath() + ".fai"), null, commandsToRun);
+                checkSymlink(log, new File(rg.getSourceFastaFile().getPath().replace(".fasta", ".dict")), null, commandsToRun);
             }
             catch (PipelineJobException e)
             {
                 log.error("Error validating genome: " + genomeId, e);
             }
         }
+
+        if (!commandsToRun.isEmpty())
+        {
+            log.error("There are missing symlinks. Please run makeSymlinks.sh");
+
+            try (PrintWriter writer = PrintWriters.getPrintWriter(new File(baseDir, "makeSymlinks.sh")))
+            {
+                writer.println("#!/bin/bash");
+                writer.println("set -e");
+                writer.println("set -x");
+                commandsToRun.forEach(writer::println);
+            }
+            catch (IOException e)
+            {
+                log.error("Error generating symlink script", e);
+            }
+        }
     }
 
-    private void inspectReleaseFolder(String releaseId, File baseDir, Container c, User u, final Logger log, final Set<File> toDelete)
+    private void inspectReleaseFolder(String releaseId, File baseDir, Container c, User u, final Logger log, final Set<File> toDelete, List<String> commandsToRun)
     {
         File releaseDir = new File(baseDir, releaseId);
         if (!releaseDir.exists())
@@ -154,9 +176,9 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U
             }
 
             expectedFiles.add(f);
-            checkSymlink(log, f);
+            checkSymlink(log, f, releaseId, commandsToRun);
             expectedFiles.add(new File(f.getPath() + ".tbi"));
-            checkSymlink(log, new File(f.getPath() + ".tbi"));
+            checkSymlink(log, new File(f.getPath() + ".tbi"), releaseId, commandsToRun);
         });
 
         final Set<String> fields = PageFlowUtil.set("vcfId", "variantTable", "liftedVcfId", "sitesOnlyVcfId", "novelSitesVcfId");
@@ -210,12 +232,23 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U
         }
     }
 
-    private void checkSymlink(Logger log, File f)
+    private void checkSymlink(Logger log, File f, @Nullable String dirName, List<String> commandsToRun)
     {
-        File expectedSymlink = new File("/var/www/html/", f.getName());
+        File expectedSymlink = new File("/var/www/html/", (dirName == null ? "" : dirName + "/") + f.getName());
+        if (dirName != null)
+        {
+            File expectedDir = new File("/var/www/html/", dirName);
+            if (!expectedDir.exists())
+            {
+                log.error("Missing expected subdir: " + expectedDir);
+                commandsToRun.add("mkdir -p " + expectedDir.getPath());
+            }
+        }
+
         if (!expectedSymlink.exists())
         {
-            log.error("Missing symlink, should run: ln -s " + f.getPath() + " " + expectedSymlink.getPath());
+            log.error("Missing symlink:  " + expectedSymlink.getPath());
+            commandsToRun.add("ln -s " + f.getPath() + " " + expectedSymlink.getPath());
         }
     }