Custom server-side sorting (#264)

hextraza · Sebastian Benjamin · bbimber · web-flow · commit a20819325a91 · 2024-03-28T12:14:23.000-07:00
* Support custom sorting fields for JBrowse

---------

Co-authored-by: Sebastian Benjamin &lt;sebastiancbenjamin@gmail.com&gt;
Co-authored-by: bbimber &lt;bbimber@gmail.com&gt;
diff --git a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx
@@ -5,13 +5,16 @@ import {
     GridColumnVisibilityModel,
     GridPaginationModel,
     GridRenderCellParams,
+    GridSortDirection,
+    GridSortModel,
     GridToolbarColumnsButton,
     GridToolbarContainer,
     GridToolbarDensitySelector,
     GridToolbarExport
 } from '@mui/x-data-grid';
 import SearchIcon from '@mui/icons-material/Search';
-import React, { useEffect, useState, useMemo } from 'react';
+import LinkIcon from '@mui/icons-material/Link';
+import React, { useEffect, useState } from 'react';
 import { getConf } from '@jbrowse/core/configuration';
 import { AppBar, Box, Button, Dialog, Paper, Popover, Toolbar, Tooltip, Typography } from '@mui/material';
 import { FilterFormModal } from './FilterFormModal';
@@ -67,22 +70,25 @@ const VariantTableWidget = observer(props => {
         session.hideWidget(widget)
     }
 
-    function handleQuery(passedFilters, pushToHistory, pageQueryModel = pageSizeModel) {
+    function handleQuery(passedFilters, pushToHistory, pageQueryModel = pageSizeModel, sortQueryModel = sortModel) {
         const { page = pageSizeModel.page, pageSize = pageSizeModel.pageSize } = pageQueryModel;
+        const { field = "genomicPosition", sort = false } = sortQueryModel[0] ?? {};
 
         const encodedSearchString = createEncodedFilterString(passedFilters, false);
         const currentUrl = new URL(window.location.href);
         currentUrl.searchParams.set("searchString", encodedSearchString);
         currentUrl.searchParams.set("page", page.toString());
         currentUrl.searchParams.set("pageSize", pageSize.toString());
+        currentUrl.searchParams.set("sortField", field.toString());
+        currentUrl.searchParams.set("sortDirection", sort.toString());
 
         if (pushToHistory) {
           window.history.pushState(null, "", currentUrl.toString());
         }
 
         setFilters(passedFilters);
         setDataLoaded(false)
-        fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)});
+        fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, field, sort, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)});
     }
 
     const TableCellWithPopover = (props: { value: any }) => {
@@ -190,7 +196,29 @@ const VariantTableWidget = observer(props => {
                     Search
                 </Button>
                 <GridToolbarDensitySelector />
-                <GridToolbarExport />
+                <GridToolbarExport csvOptions={{
+                    delimiter: ';',
+                }} />
+
+                <Button
+                    startIcon={<LinkIcon />}
+                    size="small"
+                    color="primary"
+                    onClick={() => {
+                        navigator.clipboard.writeText(window.location.href)
+                        .then(() => {
+                            // Popup message for successful copy
+                            alert('URL copied to clipboard.');
+                        })
+                        .catch(err => {
+                            // Error handling
+                            console.error('Failed to copy the URL: ', err);
+                            alert('Failed to copy the URL.');
+                        });
+                    }}
+                >
+                Share
+                </Button>
             </GridToolbarContainer>
         );
     }
@@ -234,11 +262,15 @@ const VariantTableWidget = observer(props => {
     // False until initial data load or an error:
     const [dataLoaded, setDataLoaded] = useState(false)
 
-    const urlParams = new URLSearchParams(window.location.search);
-    const page = parseInt(urlParams.get('page') || '0');
-    const pageSize = parseInt(urlParams.get('pageSize') || '50');
+    const urlParams = new URLSearchParams(window.location.search)
+    const page = parseInt(urlParams.get('page') || '0')
+    const pageSize = parseInt(urlParams.get('pageSize') || '50')
     const [pageSizeModel, setPageSizeModel] = React.useState<GridPaginationModel>({ page, pageSize });
 
+    const sortField = urlParams.get('sortField') || 'genomicPosition'
+    const sortDirection = urlParams.get('sortDirection') || 'desc'
+    const [sortModel, setSortModel] = React.useState<GridSortModel>([{ field: sortField, sort: sortDirection as GridSortDirection }])
+
     const colVisURLComponent = urlParams.get("colVisModel") || "{}"
     const colVisModel = JSON.parse(decodeURIComponent(colVisURLComponent))
     const [columnVisibilityModel, setColumnVisibilityModel] = useState<GridColumnVisibilityModel>(colVisModel);
@@ -419,6 +451,11 @@ const VariantTableWidget = observer(props => {
                 currentUrl.searchParams.set("colVisModel", encodeURIComponent(JSON.stringify(trueValuesModel)));
                 window.history.pushState(null, "", currentUrl.toString());
             }}
+            sortingMode="server"
+            onSortModelChange={(newModel) => {
+                setSortModel(newModel)
+                handleQuery(filters, true, { page: 0, pageSize: pageSizeModel.pageSize }, newModel);
+            }}
         />
     )
 
@@ -440,7 +477,7 @@ const VariantTableWidget = observer(props => {
                 fieldTypeInfo: fieldTypeInfo,
                 allowedGroupNames: allowedGroupNames,
                 promotedFilters: promotedFilters,
-                handleQuery: (filters) => handleQuery(filters, true)
+                handleQuery: (filters) => handleQuery(filters, true, { page: 0, pageSize: pageSizeModel.pageSize}, sortModel)
             }}
         />
     );
diff --git a/jbrowse/src/client/JBrowse/utils.ts b/jbrowse/src/client/JBrowse/utils.ts
@@ -419,7 +419,7 @@ function generateLuceneString(field, operator, value) {
   return luceneQueryString;
 }
 
-export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, successCallback, failureCallback) {
+export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, sortField, sortReverseString, successCallback, failureCallback) {
     if (!offset) {
         offset = 0
     }
@@ -439,6 +439,13 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa
         return
     }
 
+    let sortReverse;
+    if(sortReverseString == "asc") {
+        sortReverse = true
+    } else {
+        sortReverse = false
+    }
+
     return Ajax.request({
         url: ActionURL.buildURL('jbrowse', 'luceneQuery.api'),
         method: 'GET',
@@ -449,7 +456,15 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa
         failure: function(res) {
             failureCallback("There was an error: " + res.status + "\n Status Body: " + res.responseText + "\n Session ID:" + sessionId)
         },
-        params: {"searchString": createEncodedFilterString(filters, true), "sessionId": sessionId, "trackId": trackGUID, "offset": offset, "pageSize": pageSize},
+        params: {
+            "searchString": createEncodedFilterString(filters, true),
+            "sessionId": sessionId,
+            "trackId": trackGUID,
+            "offset": offset,
+            "pageSize": pageSize,
+            "sortField": sortField ?? "genomicPosition",
+            "sortReverse": sortReverse 
+        },
     });
 }
 
diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java
@@ -910,7 +910,7 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors)
 
             try
             {
-                return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset()));
+                return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset(), form.getSortField(), form.getSortReverse()));
             }
             catch (Exception e)
             {
@@ -947,6 +947,10 @@ public static class LuceneQueryForm
 
         private int _offset = 0;
 
+        private String _sortField = "genomicPosition";
+
+        private boolean _sortReverse = false;
+
         public String getSearchString()
         {
             return _searchString;
@@ -987,6 +991,14 @@ public void setOffset(int offset)
             _offset = offset;
         }
 
+        public String getSortField() { return _sortField; }
+
+        public void setSortField(String sortField) { _sortField = sortField; }
+
+        public boolean getSortReverse() { return _sortReverse; }
+
+        public void setSortReverse(boolean sortReverse) { _sortReverse = sortReverse; }
+
         public String getTrackId()
         {
             return _trackId;
diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.NumericUtils;
 import org.jetbrains.annotations.Nullable;
 import org.json.JSONObject;
 import org.labkey.api.data.Container;
@@ -50,6 +51,7 @@
 import java.util.StringTokenizer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static org.labkey.jbrowse.JBrowseFieldUtils.VARIABLE_SAMPLES;
 import static org.labkey.jbrowse.JBrowseFieldUtils.getSession;
@@ -61,6 +63,8 @@ public class JBrowseLuceneSearch
     private final JsonFile _jsonFile;
     private final User _user;
     private final String[] specialStartPatterns = {"*:* -", "+", "-"};
+    private static final String ALL_DOCS = "all";
+    private static final String GENOMIC_POSITION = "genomicPosition";
 
     private JBrowseLuceneSearch(final JBrowseSession session, final JsonFile jsonFile, User u)
     {
@@ -130,7 +134,7 @@ public String extractFieldName(String queryString) {
         return parts.length > 0 ? parts[0].trim() : null;
     }
 
-    public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset) throws IOException, ParseException
+    public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException
     {
         searchString = tryUrlDecode(searchString);
         File indexPath = _jsonFile.getExpectedLocationOfLuceneIndex(true);
@@ -146,7 +150,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
             IndexSearcher indexSearcher = new IndexSearcher(indexReader);
 
             List<String> stringQueryParserFields = new ArrayList<>();
-            List<String> numericQueryParserFields = new ArrayList<>();
+            Map<String, SortField.Type> numericQueryParserFields = new HashMap<>();
             PointsConfig intPointsConfig = new PointsConfig(new DecimalFormat(), Integer.class);
             PointsConfig doublePointsConfig = new PointsConfig(new DecimalFormat(), Double.class);
             Map<String, PointsConfig> pointsConfigMap = new HashMap<>();
@@ -161,11 +165,11 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
                 {
                     case Flag, String, Character -> stringQueryParserFields.add(field);
                     case Float -> {
-                        numericQueryParserFields.add(field);
+                        numericQueryParserFields.put(field, SortField.Type.DOUBLE);
                         pointsConfigMap.put(field, doublePointsConfig);
                     }
                     case Integer -> {
-                        numericQueryParserFields.add(field);
+                        numericQueryParserFields.put(field, SortField.Type.INT);
                         pointsConfigMap.put(field, intPointsConfig);
                     }
                 }
@@ -182,14 +186,14 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
 
             BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder();
 
-            if (searchString.equals("all")) {
+            if (searchString.equals(ALL_DOCS)) {
                 booleanQueryBuilder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
             }
 
             // Split input into tokens, 1 token per query separated by &
             StringTokenizer tokenizer = new StringTokenizer(searchString, "&");
 
-            while (tokenizer.hasMoreTokens() && !searchString.equals("all"))
+            while (tokenizer.hasMoreTokens() && !searchString.equals(ALL_DOCS))
             {
                 String queryString = tokenizer.nextToken();
                 Query query = null;
@@ -205,7 +209,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
                 {
                     query = queryParser.parse(queryString);
                 }
-                else if (numericQueryParserFields.contains(fieldName))
+                else if (numericQueryParserFields.containsKey(fieldName))
                 {
                     try
                     {
@@ -226,16 +230,28 @@ else if (numericQueryParserFields.contains(fieldName))
 
             BooleanQuery query = booleanQueryBuilder.build();
 
+            // By default, sort in INDEXORDER, which is by genomicPosition
+            Sort sort = Sort.INDEXORDER;
+
+            // If the sort field is not genomicPosition, use the provided sorting data
+            if (!sortField.equals(GENOMIC_POSITION)) {
+                SortField.Type fieldType;
+
+                if (stringQueryParserFields.contains(sortField)) {
+                    fieldType = SortField.Type.STRING;
+                } else if (numericQueryParserFields.containsKey(sortField)) {
+                    fieldType = numericQueryParserFields.get(sortField);
+                } else {
+                    throw new IllegalArgumentException("Could not find type for sort field: " + sortField);
+                }
+
+                sort = new Sort(new SortField(sortField, fieldType, sortReverse));
+            }
+
             // Get chunks of size {pageSize}. Default to 1 chunk -- add to the offset to get more.
             // We then iterate over the range of documents we want based on the offset. This does grow in memory
             // linearly with the number of documents, but my understanding is that these are just score,id pairs
             // rather than full documents, so mem usage *should* still be pretty low.
-            //TopDocs topDocs = indexSearcher.search(query, pageSize * (offset + 1));
-
-            // Define sort field
-            SortField sortField = new SortField("pos", SortField.Type.INT, false);
-            Sort sort = new Sort(sortField);
-
             // Perform the search with sorting
             TopFieldDocs topDocs = indexSearcher.search(query, pageSize * (offset + 1), sort);
 
@@ -253,10 +269,8 @@ else if (numericQueryParserFields.contains(fieldName))
                     String fieldName = field.name();
                     String[] fieldValues = doc.getValues(fieldName);
                     if (fieldValues.length > 1) {
-                        // If there is more than one value, put the array of values into the JSON object.
                         elem.put(fieldName, fieldValues);
                     } else {
-                        // If there is only one value, just put this single value into the JSON object.
                         elem.put(fieldName, fieldValues[0]);
                     }
                 }
diff --git a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java

Original file line number	Diff line number	Diff line change
`@@ -910,7 +910,7 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors)`
`910`	`910`
`911`	`911`	`try`
`912`	`912`	`{`
`913`		`- return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset()));`
	`913`	`+ return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset(), form.getSortField(), form.getSortReverse()));`
`914`	`914`	`}`
`915`	`915`	`catch (Exception e)`
`916`	`916`	`{`
`@@ -947,6 +947,10 @@ public static class LuceneQueryForm`
`947`	`947`
`948`	`948`	`private int _offset = 0;`
`949`	`949`
	`950`	`+ private String _sortField = "genomicPosition";`
	`951`	`+`
	`952`	`+ private boolean _sortReverse = false;`
	`953`	`+`
`950`	`954`	`public String getSearchString()`
`951`	`955`	`{`
`952`	`956`	`return _searchString;`
`@@ -987,6 +991,14 @@ public void setOffset(int offset)`
`987`	`991`	`_offset = offset;`
`988`	`992`	`}`
`989`	`993`
	`994`	`+ public String getSortField() { return _sortField; }`
	`995`	`+`
	`996`	`+ public void setSortField(String sortField) { _sortField = sortField; }`
	`997`	`+`
	`998`	`+ public boolean getSortReverse() { return _sortReverse; }`
	`999`	`+`
	`1000`	`+ public void setSortReverse(boolean sortReverse) { _sortReverse = sortReverse; }`
	`1001`	`+`
`990`	`1002`	`public String getTrackId()`
`991`	`1003`	`{`
`992`	`1004`	`return _trackId;`