Skip to content

Commit 350d81e

Browse files
hextrazaSebastian Benjamin
hextraza
and
Sebastian Benjamin
authored
New export endpoint (#321)
* WIP * Working export * Remove old const * Add CSV download test * Simplify try/catch --------- Co-authored-by: Sebastian Benjamin <sebastiancbenjamin@gmail.com>
1 parent a4f691d commit 350d81e

File tree

4 files changed

+215
-19
lines changed

4 files changed

+215
-19
lines changed

‎jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ import {
1313
GridToolbarExport
1414
} from '@mui/x-data-grid';
1515
import SearchIcon from '@mui/icons-material/Search';
16+
import LinkIcon from '@mui/icons-material/Link';
17+
import DownloadIcon from '@mui/icons-material/Download'
18+
import { ActionURL } from '@labkey/api';
1619
import React, { useEffect, useState } from 'react';
1720
import { getConf } from '@jbrowse/core/configuration';
1821
import { AppBar, Box, Button, Dialog, Paper, Popover, Toolbar, Tooltip, Typography } from '@mui/material';
@@ -92,6 +95,27 @@ const VariantTableWidget = observer(props => {
9295
fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, field, sort, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)});
9396
}
9497

98+
const handleExport = () => {
99+
const currentUrl = new URL(window.location.href);
100+
101+
const searchString = createEncodedFilterString(filters, true);
102+
const sortField = sortModel[0]?.field ?? 'genomicPosition';
103+
const sortDirection = sortModel[0]?.sort ?? false;
104+
105+
const sortReverse = (sortDirection === 'desc');
106+
107+
const rawUrl = ActionURL.buildURL('jbrowse', 'luceneCSVExport.api');
108+
const exportUrl = new URL(rawUrl, window.location.origin);
109+
110+
exportUrl.searchParams.set('sessionId', sessionId);
111+
exportUrl.searchParams.set('trackId', trackGUID);
112+
exportUrl.searchParams.set('searchString', searchString);
113+
exportUrl.searchParams.set('sortField', sortField);
114+
exportUrl.searchParams.set('sortReverse', sortReverse.toString());
115+
116+
window.location.href = exportUrl.toString();
117+
};
118+
95119
const TableCellWithPopover = (props: { value: any }) => {
96120
const { value } = props;
97121
const fullDisplayValue = value ? (Array.isArray(value) ? value.join(', ') : value) : ''
@@ -184,7 +208,7 @@ const VariantTableWidget = observer(props => {
184208
);
185209
}
186210

187-
function CustomToolbar({ setFilterModalOpen }) {
211+
function CustomToolbar({ setFilterModalOpen, handleExport }) {
188212
return (
189213
<GridToolbarContainer>
190214
<GridToolbarColumnsButton />
@@ -197,17 +221,25 @@ const VariantTableWidget = observer(props => {
197221
Search
198222
</Button>
199223
<GridToolbarDensitySelector />
200-
<GridToolbarExport csvOptions={{
201-
delimiter: ';',
202-
}} />
224+
225+
<Button
226+
startIcon={<DownloadIcon />}
227+
color="primary"
228+
onClick={handleExport}
229+
>
230+
Export CSV
231+
</Button>
203232

204233
<ShareButton />
205234
</GridToolbarContainer>
206235
);
207236
}
208237

209238
const ToolbarWithProps = () => (
210-
<CustomToolbar setFilterModalOpen={setFilterModalOpen} />
239+
<CustomToolbar
240+
setFilterModalOpen={setFilterModalOpen}
241+
handleExport={handleExport}
242+
/>
211243
);
212244

213245
const handleOffsetChange = (newOffset: number) => {

‎jbrowse/src/org/labkey/jbrowse/JBrowseController.java‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import htsjdk.variant.variantcontext.Genotype;
2121
import htsjdk.variant.variantcontext.VariantContext;
2222
import htsjdk.variant.vcf.VCFFileReader;
23+
import jakarta.servlet.http.HttpServletResponse;
2324
import org.apache.commons.lang3.StringUtils;
2425
import org.apache.logging.log4j.Logger;
2526
import org.jetbrains.annotations.NotNull;
@@ -78,6 +79,8 @@
7879
import java.io.File;
7980
import java.io.FileNotFoundException;
8081
import java.io.IOException;
82+
import java.time.LocalDateTime;
83+
import java.time.format.DateTimeFormatter;
8184
import java.util.ArrayList;
8285
import java.util.Arrays;
8386
import java.util.Collections;
@@ -891,6 +894,55 @@ public void validateForm(ResolveVcfFieldsForm form, Errors errors)
891894
}
892895
}
893896

897+
@RequiresPermission(ReadPermission.class)
898+
public static class LuceneCSVExportAction extends ReadOnlyApiAction<LuceneQueryForm>
899+
{
900+
@Override
901+
public ApiResponse execute(LuceneQueryForm form, BindException errors)
902+
{
903+
try
904+
{
905+
JBrowseLuceneSearch searcher = JBrowseLuceneSearch.create(form.getSessionId(), form.getTrackId(), getUser());
906+
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss");
907+
String timestamp = LocalDateTime.now().format(formatter);
908+
String filename = "mGAP_results_" + timestamp + ".csv";
909+
910+
HttpServletResponse response = getViewContext().getResponse();
911+
response.setContentType("text/csv");
912+
response.setHeader("Content-Disposition", "attachment; filename=\"" + filename + "\"");
913+
914+
searcher.doSearchCSV(
915+
getUser(),
916+
PageFlowUtil.decode(form.getSearchString()),
917+
form.getSortField(),
918+
form.getSortReverse(),
919+
response
920+
);
921+
922+
return null;
923+
}
924+
catch (Exception e)
925+
{
926+
_log.error("Error in JBrowse lucene query", e);
927+
errors.reject(ERROR_MSG, e.getMessage());
928+
return null;
929+
}
930+
}
931+
932+
@Override
933+
public void validateForm(LuceneQueryForm form, Errors errors)
934+
{
935+
if ((form.getSearchString() == null || form.getSessionId() == null || form.getTrackId() == null))
936+
{
937+
errors.reject(ERROR_MSG, "Must provide search string, track ID, and the JBrowse session ID");
938+
}
939+
else if (!isValidUUID(form.getTrackId()))
940+
{
941+
errors.reject(ERROR_MSG, "Invalid track ID: " + form.getTrackId());
942+
}
943+
}
944+
}
945+
894946
@RequiresPermission(ReadPermission.class)
895947
public static class LuceneQueryAction extends ReadOnlyApiAction<LuceneQueryForm>
896948
{
@@ -910,7 +962,14 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors)
910962

911963
try
912964
{
913-
return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset(), form.getSortField(), form.getSortReverse()));
965+
return new ApiSimpleResponse(searcher.doSearchJSON(
966+
getUser(),
967+
PageFlowUtil.decode(form.getSearchString()),
968+
form.getPageSize(),
969+
form.getOffset(),
970+
form.getSortField(),
971+
form.getSortReverse()
972+
));
914973
}
915974
catch (Exception e)
916975
{

‎jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java‎

Lines changed: 82 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.labkey.jbrowse;
22

3+
import jakarta.servlet.http.HttpServletResponse;
34
import org.apache.commons.lang3.StringUtils;
45
import org.apache.logging.log4j.Logger;
56
import org.apache.lucene.analysis.Analyzer;
@@ -20,6 +21,7 @@
2021
import org.apache.lucene.search.MatchAllDocsQuery;
2122
import org.apache.lucene.search.Query;
2223
import org.apache.lucene.search.QueryCachingPolicy;
24+
import org.apache.lucene.search.ScoreDoc;
2325
import org.apache.lucene.search.Sort;
2426
import org.apache.lucene.search.SortField;
2527
import org.apache.lucene.search.TopFieldDocs;
@@ -50,6 +52,7 @@
5052

5153
import java.io.File;
5254
import java.io.IOException;
55+
import java.io.PrintWriter;
5356
import java.net.URLDecoder;
5457
import java.nio.charset.StandardCharsets;
5558
import java.text.DecimalFormat;
@@ -189,7 +192,17 @@ public String extractFieldName(String queryString)
189192
return parts.length > 0 ? parts[0].trim() : null;
190193
}
191194

192-
public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException
195+
public JSONObject doSearchJSON(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException {
196+
SearchConfig searchConfig = createSearchConfig(u, searchString, pageSize, offset, sortField, sortReverse);
197+
return paginateJSON(searchConfig);
198+
}
199+
200+
public void doSearchCSV(User u, String searchString, String sortField, boolean sortReverse, HttpServletResponse response) throws IOException, ParseException {
201+
SearchConfig searchConfig = createSearchConfig(u, searchString, 0, 0, sortField, sortReverse);
202+
exportCSV(searchConfig, response);
203+
}
204+
205+
private SearchConfig createSearchConfig(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException
193206
{
194207
searchString = tryUrlDecode(searchString);
195208
File indexPath = _jsonFile.getExpectedLocationOfLuceneIndex(true);
@@ -199,6 +212,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
199212
Analyzer analyzer = new StandardAnalyzer();
200213

201214
List<String> stringQueryParserFields = new ArrayList<>();
215+
List<String> fieldsList = new ArrayList<>();
202216
Map<String, SortField.Type> numericQueryParserFields = new HashMap<>();
203217
PointsConfig intPointsConfig = new PointsConfig(new DecimalFormat(), Integer.class);
204218
PointsConfig doublePointsConfig = new PointsConfig(new DecimalFormat(), Double.class);
@@ -208,6 +222,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
208222
for (Map.Entry<String, JBrowseFieldDescriptor> entry : fields.entrySet())
209223
{
210224
String field = entry.getKey();
225+
fieldsList.add(field);
211226
JBrowseFieldDescriptor descriptor = entry.getValue();
212227

213228
switch(descriptor.getType())
@@ -267,14 +282,14 @@ else if (numericQueryParserFields.containsKey(fieldName))
267282
}
268283
catch (QueryNodeException e)
269284
{
270-
_log.error("Unable to parse query for field " + fieldName + ": " + queryString, e);
285+
_log.error("Unable to parse query for field {}: {}", fieldName, queryString, e);
271286

272287
throw new IllegalArgumentException("Unable to parse query: " + queryString + " for field: " + fieldName);
273288
}
274289
}
275290
else
276291
{
277-
_log.error("No such field(s), or malformed query: " + queryString + ", field: " + fieldName);
292+
_log.error("No such field(s), or malformed query: {}, field: {}", queryString, fieldName);
278293

279294
throw new IllegalArgumentException("No such field(s), or malformed query: " + queryString + ", field: " + fieldName);
280295
}
@@ -302,43 +317,79 @@ else if (numericQueryParserFields.containsKey(fieldName))
302317
sort = new Sort(new SortField(sortField + "_sort", fieldType, sortReverse));
303318
}
304319

320+
return new SearchConfig(cacheEntry, query, pageSize, offset, sort, fieldsList);
321+
}
322+
323+
private JSONObject paginateJSON(SearchConfig c) throws IOException, ParseException {
305324
// Get chunks of size {pageSize}. Default to 1 chunk -- add to the offset to get more.
306325
// We then iterate over the range of documents we want based on the offset. This does grow in memory
307326
// linearly with the number of documents, but my understanding is that these are just score,id pairs
308327
// rather than full documents, so mem usage *should* still be pretty low.
309328
// Perform the search with sorting
310-
TopFieldDocs topDocs = cacheEntry.indexSearcher.search(query, pageSize * (offset + 1), sort);
311-
329+
TopFieldDocs topDocs = c.cacheEntry.indexSearcher.search(c.query, c.pageSize * (c.offset + 1), c.sort);
312330
JSONObject results = new JSONObject();
313331

314332
// Iterate over the doc list, (either to the total end or until the page ends) grab the requested docs,
315333
// and add to returned results
316334
List<JSONObject> data = new ArrayList<>();
317-
for (int i = pageSize * offset; i < Math.min(pageSize * (offset + 1), topDocs.scoreDocs.length); i++)
335+
for (int i = c.pageSize * c.offset; i < Math.min(c.pageSize * (c.offset + 1), topDocs.scoreDocs.length); i++)
318336
{
319337
JSONObject elem = new JSONObject();
320-
Document doc = cacheEntry.indexSearcher.storedFields().document(topDocs.scoreDocs[i].doc);
338+
Document doc = c.cacheEntry.indexSearcher.storedFields().document(topDocs.scoreDocs[i].doc);
321339

322-
for (IndexableField field : doc.getFields()) {
340+
for (IndexableField field : doc.getFields())
341+
{
323342
String fieldName = field.name();
324343
String[] fieldValues = doc.getValues(fieldName);
325-
if (fieldValues.length > 1) {
344+
if (fieldValues.length > 1)
345+
{
326346
elem.put(fieldName, fieldValues);
327-
} else {
347+
}
348+
else
349+
{
328350
elem.put(fieldName, fieldValues[0]);
329351
}
330352
}
331-
332353
data.add(elem);
333354
}
334355

335356
results.put("data", data);
336357
results.put("totalHits", topDocs.totalHits.value);
337358

338-
//TODO: we should probably stream this
339359
return results;
340360
}
341361

362+
private void exportCSV(SearchConfig c, HttpServletResponse response) throws IOException
363+
{
364+
PrintWriter writer = response.getWriter();
365+
IndexSearcher searcher = c.cacheEntry.indexSearcher;
366+
TopFieldDocs topDocs = searcher.search(c.query, Integer.MAX_VALUE, c.sort);
367+
368+
writer.println(String.join(",", c.fields));
369+
370+
for (ScoreDoc scoreDoc : topDocs.scoreDocs)
371+
{
372+
Document doc = searcher.storedFields().document(scoreDoc.doc);
373+
List<String> rowValues = new ArrayList<>();
374+
375+
for (String fieldName : c.fields)
376+
{
377+
String[] values = doc.getValues(fieldName);
378+
String value = values.length > 0
379+
? String.join(",", values)
380+
: "";
381+
382+
// Escape strings
383+
value = "\"" + value.replace("\"", "\"\"") + "\"";
384+
rowValues.add(value);
385+
}
386+
387+
writer.println(String.join(",", rowValues));
388+
}
389+
390+
writer.flush();
391+
}
392+
342393
public static class DefaultJBrowseFieldCustomizer extends AbstractJBrowseFieldCustomizer
343394
{
344395
public DefaultJBrowseFieldCustomizer()
@@ -583,7 +634,7 @@ public void cacheDefaultQuery()
583634
try
584635
{
585636
JBrowseLuceneSearch.clearCache(_jsonFile.getObjectId());
586-
doSearch(_user, ALL_DOCS, 100, 0, GENOMIC_POSITION, false);
637+
doSearchJSON(_user, ALL_DOCS, 100, 0, GENOMIC_POSITION, false);
587638
}
588639
catch (ParseException | IOException e)
589640
{
@@ -641,4 +692,22 @@ public void shutdownStarted()
641692
JBrowseLuceneSearch.emptyCache();
642693
}
643694
}
695+
696+
private class SearchConfig {
697+
CacheEntry cacheEntry;
698+
Query query;
699+
int pageSize;
700+
int offset;
701+
Sort sort;
702+
List<String> fields;
703+
704+
public SearchConfig(CacheEntry cacheEntry, Query query, int pageSize, int offset, Sort sort, List<String> fields) {
705+
this.cacheEntry = cacheEntry;
706+
this.query = query;
707+
this.pageSize = pageSize;
708+
this.offset = offset;
709+
this.sort = sort;
710+
this.fields = fields;
711+
}
712+
}
644713
}

0 commit comments

Comments
 (0)