Skip to content

Commit 95b0e50

Browse files
bbimberbbimber
committed
Merge discvr-23.11 to develop
2 parents 5302688 + 62cf397 commit 95b0e50

File tree

16 files changed

+299
-51
lines changed

16 files changed

+299
-51
lines changed

‎mGAP/resources/queries/mGAP/variantCatalogReleases/.qview.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,6 @@
99
</columns>
1010
<sorts>
1111
<sort column="version" descending="true"/>
12+
<sort column="releaseDate" descending="true"/>
1213
</sorts>
1314
</customView>

‎mGAP/resources/views/overview.html‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
(function($){
88
$( document ).ready(function() {
99
mGAP.Utils.renderReleaseGraph('mgap-graph-outer', 300);
10+
11+
if (mGAP.Utils.releaseHasLuceneIndex()) {
12+
$('li.mgap-lucene-link').removeAttr("hidden");
13+
}
1014
});
1115
})(jQuery);
1216
</script>
@@ -31,7 +35,7 @@ <h4><span style="text-decoration: underline">Short Variant Catalog:</span> <img
3135
<br>
3236
<ul style="padding-top: 10px;">
3337
<li>Use the <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mGap-genomeBrowser.view?">Genome Browser</a> to view and search data. <i class="fa-solid fa-video mgap-video-icon" data-video="genome-browser" data-video-title="Genome Browser Overview"></i></li>
34-
<!-- <li>Use our new <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mGap-genomeBrowser.view?target=variantSearch">Full-text Search</a> tool to query variants based on gene, or using our <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mgap-annotation.view">extensive annotations</a> &lt;!&ndash;<i class="fa-solid fa-video mgap-video-icon" data-video="variant-search" data-video-title="Variant Full-text Search (BETA)"></i>&ndash;&gt;</li>-->
38+
<li class="mgap-lucene-link" hidden="hidden">Use our new <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mGap-genomeBrowser.view?target=variantSearch">Full-text Search</a> tool to query variants based on gene, or using our <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mgap-annotation.view">extensive annotations</a> <!--<i class="fa-solid fa-video mgap-video-icon" data-video="variant-search" data-video-title="Variant Full-text Search (BETA)"></i>--></li>
3539
<li>View our list of <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mGap-variantList.view">Predicted Damaging Variants</a>, which is a list of predicted high-impact or disease associated variants, generated from each release <i class="fa-solid fa-video mgap-video-icon" data-video="predicted-damaging-variants" data-video-title="Predicted Damaging Variants"></i></li>
3640
<li>Unlike many datasets, mGAP has genotype-level data, <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/project-begin.view?pageId=clinical">often connected to living animals from pedigreed breeding colonies</a></li>
3741
<li>Download raw data, including <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/project-begin.view?pageId=datasets">sequence data</a> and <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/project-begin.view?pageId=variants">variant data</a></li>

‎mGAP/resources/views/quickLinks.html‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,15 @@
1010
var releaseId = mGAP.Utils.getMGapReleaseId();
1111
var jbrowseId = mGAP.Utils.getMGapJBrowseSession();
1212
var humanJbrowseId = mGAP.Utils.getHumanMGapJBrowseSession();
13+
var releaseHasLuceneIndex = mGAP.Utils.releaseHasLuceneIndex();
1314
if (releaseId){
1415
Ext4.fly(Ext4.query('#currentReleaseDiv')[0]).update(
1516
'<span>Current Release:</span>' +
1617
'<ul class="mgapList">' +
1718
'<li><a href="' + LABKEY.ActionURL.buildURL('mgap', 'variantList', null) + '">Predicted Damaging Variants</a></li>' +
1819
(jbrowseId ? '<li><a href="' + LABKEY.ActionURL.buildURL('jbrowse', 'browser', null, {database: jbrowseId}) + '">Genome Browser (Macaque)</a></li>' : '') +
1920
(humanJbrowseId ? '<li><a href="' + LABKEY.ActionURL.buildURL('jbrowse', 'browser', null, {database: humanJbrowseId}) + '">Genome Browser (Human)</a></li>' : '') +
20-
// '<li><a href="' + LABKEY.ActionURL.buildURL('mgap', 'genomeBrowser', null, {target: 'variantSearch'}) + '">Full-text Variant Search (BETA)</a></li>' +
21+
(releaseHasLuceneIndex ? '<li><a href="' + LABKEY.ActionURL.buildURL('mgap', 'genomeBrowser', null, {target: 'variantSearch'}) + '">Full-text Variant Search (BETA)</a></li>' : '') +
2122
'</ul>'
2223
);
2324
}

‎mGAP/resources/web/mGAP/DownloadWindow.js‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ Ext4.define('mGAP.window.DownloadWindow', {
3939
method: 'POST',
4040
schemaName: 'mgap',
4141
queryName: 'variantCatalogReleases',
42-
columns: 'vcfId/dataid/Name,vcfId/library_id/fasta_file/Name,sitesOnlyVcfId/dataid/Name',
42+
columns: 'objectId,vcfId/dataid/Name,vcfId/library_id/fasta_file/Name,sitesOnlyVcfId/dataid/Name',
4343
filterArray: [LABKEY.Filter.create('rowid', this.releaseId, LABKEY.Filter.Types.EQUAL)],
4444
scope: this,
4545
failure: LDK.Utils.getErrorCallback(),
@@ -52,9 +52,11 @@ Ext4.define('mGAP.window.DownloadWindow', {
5252
return;
5353
}
5454

55-
var releaseVcf = results.rows[0]['vcfId/dataid/Name'];
55+
LDK.Assert.assertNotEmpty('Missing objectId variantCatalogReleases', results.rows[0].objectId);
56+
57+
var releaseVcf = results.rows[0].objectId + '/' + results.rows[0]['vcfId/dataid/Name'];
5658
var urlFasta = results.rows[0]['vcfId/library_id/fasta_file/Name'];
57-
var sitesOnlyVcf = results.rows[0]['sitesOnlyVcfId/dataid/Name'];
59+
var sitesOnlyVcf = results.rows[0].objectId + '/' + results.rows[0]['sitesOnlyVcfId/dataid/Name'];
5860

5961
var toAdd = [{
6062
html: 'Due to the large file size, the preferred option is to download using wget or curl on the command line, such as the exmaples below. Nonetheless, you also are able to paste the URLs into your browser and download through this way as well, although it will be slower and possibly not able to resume if your connection is disrupted.<br><br>' +

‎mGAP/resources/web/mGAP/Utils.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,12 @@ mGAP.Utils = (function($){
221221
return ctx['mgapReleaseId'];
222222
},
223223

224+
releaseHasLuceneIndex: function(){
225+
var ctx = LABKEY.getModuleContext('mgap') || {};
226+
227+
return !!ctx['luceneIndexId'];
228+
},
229+
224230
getMGapReleaseVersion: function(){
225231
var ctx = LABKEY.getModuleContext('mgap') || {};
226232

‎mGAP/src/org/labkey/mgap/columnTransforms/JBrowseHumanSessionTransform.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
import org.labkey.api.util.GUID;
1010
import org.labkey.api.util.PageFlowUtil;
1111

12-
import java.util.Arrays;
1312
import java.util.Date;
1413
import java.util.HashMap;
1514
import java.util.List;
@@ -30,7 +29,7 @@ protected String getDatabaseName()
3029
}
3130

3231
@Override
33-
protected String getTrackJson()
32+
protected String getTrackJson(boolean hasLuceneIndex)
3433
{
3534
return "{\"category\":\"mGAP Variant Catalog\",\"visibleByDefault\": true,\"additionalFeatureMsg\":\"<h2>**These annotations are created by lifting the macaque variants to human coordinates, and must be viewed in that context.</h2>\"}";
3635
}
@@ -93,7 +92,7 @@ private String getOrCreateJsonFile()
9392
row.put("createdby", getContainerUser().getUser().getUserId());
9493
row.put("modified", new Date());
9594
row.put("modifiedby", getContainerUser().getUser().getUserId());
96-
row.put("trackJson", getTrackJson());
95+
row.put("trackJson", getTrackJson(false));
9796

9897
getStatusLogger().info("creating jsonfile for output: " + outputFileId);
9998
List<Map<String, Object>> rows = jsonFiles.getUpdateService().insertRows(getContainerUser().getUser(), getContainerUser().getContainer(), List.of(row), new BatchValidationException(), null, new HashMap<>());

‎mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,8 @@ private String getOrCreateJsonFile(Results rs, String fieldKey) throws SQLExcept
270270

271271
if (isDefaultTrack)
272272
{
273-
row.put("trackJson", getTrackJson());
273+
boolean hasLuceneIndex = StringUtils.trimToNull(rs.getString(FieldKey.fromString("luceneIndex/dataid/DataFileUrl"))) != null;
274+
row.put("trackJson", getTrackJson(hasLuceneIndex));
274275
}
275276
else
276277
{
@@ -307,9 +308,16 @@ protected String getDatabaseName()
307308
return "mGAP Release: " + getInputValue("version");
308309
}
309310

310-
protected String getTrackJson()
311+
protected String getTrackJson(boolean hasLuceneIndex)
311312
{
312-
ArrayList<String> infoFields = new TableSelector(QueryService.get().getUserSchema(getContainerUser().getUser(), getContainerUser().getContainer(), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_ANNOTATIONS), PageFlowUtil.set("infoKey"), new SimpleFilter(FieldKey.fromString("isIndexed"), true), null).getArrayList(String.class);
313-
return "{\"category\":\"mGAP Variant Catalog\",\"visibleByDefault\": true,\"ensemblId\":\"Macaca_mulatta\",\"additionalFeatureMsg\":\"<h2>**The annotations below are primarily derived from human data sources (not macaque), and must be viewed in that context.</h2>\", \"createFullTextIndex\": true,\"infoFieldsForFullTextSearch\":\"" + (infoFields.isEmpty() ? "null" : StringUtils.join(infoFields, ",")) + "\"}";
313+
String indexString = "";
314+
315+
if (hasLuceneIndex)
316+
{
317+
ArrayList<String> infoFields = new TableSelector(QueryService.get().getUserSchema(getContainerUser().getUser(), getContainerUser().getContainer(), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_ANNOTATIONS), PageFlowUtil.set("infoKey"), new SimpleFilter(FieldKey.fromString("isIndexed"), true), null).getArrayList(String.class);
318+
indexString = ", \"createFullTextIndex\": true,\"infoFieldsForFullTextSearch\":\"" + (infoFields.isEmpty() ? "null" : StringUtils.join(infoFields, ",")) + "\"";
319+
}
320+
321+
return "{\"category\":\"mGAP Variant Catalog\",\"visibleByDefault\": true,\"ensemblId\":\"Macaca_mulatta\",\"additionalFeatureMsg\":\"<h2>**The annotations below are primarily derived from human data sources (not macaque), and must be viewed in that context.</h2>\"" + indexString + "}";
314322
}
315323
}

‎mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
3333
"rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--chmod=D2770,F660", sourceDir.getPath(), targetDir.getPath()
3434
));
3535

36-
return new File(targetDir, f.getName());
36+
return new File(targetDir, sourceDir.getName() + "/" + f.getName());
3737
}
3838

3939
@Override

‎mGAP/src/org/labkey/mgap/mGAPModule.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import org.labkey.mgap.pipeline.RenameSamplesForMgapStep;
5454
import org.labkey.mgap.pipeline.SampleSpecificGenotypeFiltrationStep;
5555
import org.labkey.mgap.pipeline.VcfComparisonStep;
56+
import org.labkey.mgap.pipeline.mGapReleaseAlleleFreqStep;
5657
import org.labkey.mgap.pipeline.mGapReleaseAnnotateNovelSitesStep;
5758
import org.labkey.mgap.pipeline.mGapReleaseComparisonStep;
5859
import org.labkey.mgap.pipeline.mGapReleaseGenerator;
@@ -136,6 +137,7 @@ public PipelineStartup()
136137
SequencePipelineService.get().registerPipelineStep(new mGapReleaseAnnotateNovelSitesStep.Provider());
137138
SequencePipelineService.get().registerPipelineStep(new GenerateMgapTracksStep.Provider());
138139
SequencePipelineService.get().registerPipelineStep(new IndexVariantsForMgapStep.Provider());
140+
SequencePipelineService.get().registerPipelineStep(new mGapReleaseAlleleFreqStep.Provider());
139141

140142
_hasRegistered = true;
141143
}
@@ -150,7 +152,7 @@ public JSONObject getPageContextJson(ContainerUser context)
150152

151153
SimpleFilter filter = new SimpleFilter();
152154
filter.addClause(ContainerFilter.current(context.getContainer()).createFilterClause(mGAPSchema.getInstance().getSchema(), FieldKey.fromString("container")));
153-
TableSelector ts = new TableSelector(mGAPSchema.getInstance().getSchema().getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("rowid", "objectid", "version", "jbrowseId", "humanJbrowseId"), filter, new Sort("-releaseDate"));
155+
TableSelector ts = new TableSelector(mGAPSchema.getInstance().getSchema().getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("rowid", "objectid", "version", "jbrowseId", "humanJbrowseId", "luceneIndex"), filter, new Sort("-releaseDate"));
154156
ts.setMaxRows(1);
155157
ts.forEachResults(rs -> {
156158
String jbrowseId = rs.getString(FieldKey.fromString("jbrowseId"));
@@ -166,11 +168,17 @@ public JSONObject getPageContextJson(ContainerUser context)
166168
}
167169

168170
Integer rowId = rs.getInt(FieldKey.fromString("rowid"));
169-
if (rowId != null)
171+
if (rowId != null && rowId > 0)
170172
{
171173
ret.put("mgapReleaseId", rowId);
172174
}
173175

176+
Integer luceneIndexId = rs.getInt(FieldKey.fromString("luceneIndex"));
177+
if (luceneIndexId != null && luceneIndexId > 0)
178+
{
179+
ret.put("luceneIndexId", luceneIndexId);
180+
}
181+
174182
String releaseVersion = rs.getString(FieldKey.fromString("version"));
175183
if (releaseVersion != null)
176184
{

‎mGAP/src/org/labkey/mgap/mGapMaintenanceTask.java‎

Lines changed: 55 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.labkey.mgap;
22

33
import org.apache.logging.log4j.Logger;
4+
import org.jetbrains.annotations.Nullable;
45
import org.labkey.api.data.Container;
56
import org.labkey.api.data.ContainerManager;
67
import org.labkey.api.data.SimpleFilter;
@@ -19,8 +20,11 @@
1920
import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
2021
import org.labkey.api.util.PageFlowUtil;
2122
import org.labkey.api.util.SystemMaintenance;
23+
import org.labkey.api.writer.PrintWriters;
2224

2325
import java.io.File;
26+
import java.io.IOException;
27+
import java.io.PrintWriter;
2428
import java.util.ArrayList;
2529
import java.util.Arrays;
2630
import java.util.Collections;
@@ -107,7 +111,8 @@ private void checkMGapFiles(Logger log, User u)
107111
log.error("Missing expected directory: " + dirName);
108112
}
109113

110-
releaseIds.forEach(f -> inspectReleaseFolder(f, baseDir, c, u, log, toDelete));
114+
List<String> commandsToRun = new ArrayList<>();
115+
releaseIds.forEach(f -> inspectReleaseFolder(f, baseDir, c, u, log, toDelete, commandsToRun));
111116

112117
// Also verify genomes:
113118
Set<Integer> genomesIds = new HashSet<>(new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("genomeId")).getArrayList(Integer.class));
@@ -116,18 +121,35 @@ private void checkMGapFiles(Logger log, User u)
116121
try
117122
{
118123
ReferenceGenome rg = SequenceAnalysisService.get().getReferenceGenome(genomeId, u);
119-
checkSymlink(log, rg.getSourceFastaFile());
120-
checkSymlink(log, new File(rg.getSourceFastaFile().getPath() + ".fai"));
121-
checkSymlink(log, new File(rg.getSourceFastaFile().getPath().replace(".fasta", ".dict")));
124+
checkSymlink(log, rg.getSourceFastaFile(), null, commandsToRun);
125+
checkSymlink(log, new File(rg.getSourceFastaFile().getPath() + ".fai"), null, commandsToRun);
126+
checkSymlink(log, new File(rg.getSourceFastaFile().getPath().replace(".fasta", ".dict")), null, commandsToRun);
122127
}
123128
catch (PipelineJobException e)
124129
{
125130
log.error("Error validating genome: " + genomeId, e);
126131
}
127132
}
133+
134+
if (!commandsToRun.isEmpty())
135+
{
136+
log.error("There are missing symlinks. Please run makeSymlinks.sh");
137+
138+
try (PrintWriter writer = PrintWriters.getPrintWriter(new File(baseDir, "makeSymlinks.sh")))
139+
{
140+
writer.println("#!/bin/bash");
141+
writer.println("set -e");
142+
writer.println("set -x");
143+
commandsToRun.forEach(writer::println);
144+
}
145+
catch (IOException e)
146+
{
147+
log.error("Error generating symlink script", e);
148+
}
149+
}
128150
}
129151

130-
private void inspectReleaseFolder(String releaseId, File baseDir, Container c, User u, final Logger log, final Set<File> toDelete)
152+
private void inspectReleaseFolder(String releaseId, File baseDir, Container c, User u, final Logger log, final Set<File> toDelete, List<String> commandsToRun)
131153
{
132154
File releaseDir = new File(baseDir, releaseId);
133155
if (!releaseDir.exists())
@@ -154,12 +176,12 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U
154176
}
155177

156178
expectedFiles.add(f);
157-
checkSymlink(log, f);
179+
checkSymlink(log, f, releaseId, commandsToRun);
158180
expectedFiles.add(new File(f.getPath() + ".tbi"));
159-
checkSymlink(log, new File(f.getPath() + ".tbi"));
181+
checkSymlink(log, new File(f.getPath() + ".tbi"), releaseId, commandsToRun);
160182
});
161183

162-
final Set<String> fields = PageFlowUtil.set("vcfId", "variantTable", "liftedVcfId", "sitesOnlyVcfId", "novelSitesVcfId");
184+
final Set<String> fields = PageFlowUtil.set("vcfId", "variantTable", "liftedVcfId", "sitesOnlyVcfId", "novelSitesVcfId", "luceneIndex");
163185
new TableSelector(QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), fields, new SimpleFilter(FieldKey.fromString("objectid"), releaseId), null).forEachResults(rs -> {
164186
for (String field : fields)
165187
{
@@ -183,6 +205,12 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U
183205
continue;
184206
}
185207

208+
// NOTE: lucene points to a file, but the parent dir is what we need to manage:
209+
if ("write.lock".equals(f.getName()))
210+
{
211+
expectedFiles.add(f.getParentFile().getParentFile());
212+
}
213+
186214
expectedFiles.add(f);
187215
if (f.getPath().toLowerCase().endsWith("vcf.gz"))
188216
{
@@ -206,16 +234,31 @@ private void inspectReleaseFolder(String releaseId, File baseDir, Container c, U
206234
missingFiles.removeAll(filesPresent);
207235
for (File f : missingFiles)
208236
{
209-
log.error("Missing expected file: " + f.getPath());
237+
// NOTE: the write.lock file is technically one directory lower and not caught with the check above
238+
if (!f.exists())
239+
{
240+
log.error("Missing expected file: " + f.getPath());
241+
}
210242
}
211243
}
212244

213-
private void checkSymlink(Logger log, File f)
245+
private void checkSymlink(Logger log, File f, @Nullable String dirName, List<String> commandsToRun)
214246
{
215-
File expectedSymlink = new File("/var/www/html/", f.getName());
247+
File expectedSymlink = new File("/var/www/html/", (dirName == null ? "" : dirName + "/") + f.getName());
248+
if (dirName != null)
249+
{
250+
File expectedDir = new File("/var/www/html/", dirName);
251+
if (!expectedDir.exists())
252+
{
253+
log.error("Missing expected subdir: " + expectedDir);
254+
commandsToRun.add("mkdir -p " + expectedDir.getPath());
255+
}
256+
}
257+
216258
if (!expectedSymlink.exists())
217259
{
218-
log.error("Missing symlink, should run: ln -s " + f.getPath() + " " + expectedSymlink.getPath());
260+
log.error("Missing symlink: " + expectedSymlink.getPath());
261+
commandsToRun.add("ln -s " + f.getPath() + " " + expectedSymlink.getPath());
219262
}
220263
}
221264

0 commit comments

Comments
 (0)