diff --git a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx index afe99496c..ac75ef4de 100644 --- a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx +++ b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx @@ -5,13 +5,16 @@ import { GridColumnVisibilityModel, GridPaginationModel, GridRenderCellParams, + GridSortDirection, + GridSortModel, GridToolbarColumnsButton, GridToolbarContainer, GridToolbarDensitySelector, GridToolbarExport } from '@mui/x-data-grid'; import SearchIcon from '@mui/icons-material/Search'; -import React, { useEffect, useState, useMemo } from 'react'; +import LinkIcon from '@mui/icons-material/Link'; +import React, { useEffect, useState } from 'react'; import { getConf } from '@jbrowse/core/configuration'; import { AppBar, Box, Button, Dialog, Paper, Popover, Toolbar, Tooltip, Typography } from '@mui/material'; import { FilterFormModal } from './FilterFormModal'; @@ -67,14 +70,17 @@ const VariantTableWidget = observer(props => { session.hideWidget(widget) } - function handleQuery(passedFilters, pushToHistory, pageQueryModel = pageSizeModel) { + function handleQuery(passedFilters, pushToHistory, pageQueryModel = pageSizeModel, sortQueryModel = sortModel) { const { page = pageSizeModel.page, pageSize = pageSizeModel.pageSize } = pageQueryModel; + const { field = "genomicPosition", sort = false } = sortQueryModel[0] ?? {}; const encodedSearchString = createEncodedFilterString(passedFilters, false); const currentUrl = new URL(window.location.href); currentUrl.searchParams.set("searchString", encodedSearchString); currentUrl.searchParams.set("page", page.toString()); currentUrl.searchParams.set("pageSize", pageSize.toString()); + currentUrl.searchParams.set("sortField", field.toString()); + currentUrl.searchParams.set("sortDirection", sort.toString()); if (pushToHistory) { window.history.pushState(null, "", currentUrl.toString()); @@ -82,7 +88,7 @@ const VariantTableWidget = observer(props => { setFilters(passedFilters); setDataLoaded(false) - fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)}); + fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, field, sort, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)}); } const TableCellWithPopover = (props: { value: any }) => { @@ -190,7 +196,29 @@ const VariantTableWidget = observer(props => { Search - + + + ); } @@ -234,11 +262,15 @@ const VariantTableWidget = observer(props => { // False until initial data load or an error: const [dataLoaded, setDataLoaded] = useState(false) - const urlParams = new URLSearchParams(window.location.search); - const page = parseInt(urlParams.get('page') || '0'); - const pageSize = parseInt(urlParams.get('pageSize') || '50'); + const urlParams = new URLSearchParams(window.location.search) + const page = parseInt(urlParams.get('page') || '0') + const pageSize = parseInt(urlParams.get('pageSize') || '50') const [pageSizeModel, setPageSizeModel] = React.useState({ page, pageSize }); + const sortField = urlParams.get('sortField') || 'genomicPosition' + const sortDirection = urlParams.get('sortDirection') || 'desc' + const [sortModel, setSortModel] = React.useState([{ field: sortField, sort: sortDirection as GridSortDirection }]) + const colVisURLComponent = urlParams.get("colVisModel") || "{}" const colVisModel = JSON.parse(decodeURIComponent(colVisURLComponent)) const [columnVisibilityModel, setColumnVisibilityModel] = useState(colVisModel); @@ -419,6 +451,11 @@ const VariantTableWidget = observer(props => { currentUrl.searchParams.set("colVisModel", encodeURIComponent(JSON.stringify(trueValuesModel))); window.history.pushState(null, "", currentUrl.toString()); }} + sortingMode="server" + onSortModelChange={(newModel) => { + setSortModel(newModel) + handleQuery(filters, true, { page: 0, pageSize: pageSizeModel.pageSize }, newModel); + }} /> ) @@ -440,7 +477,7 @@ const VariantTableWidget = observer(props => { fieldTypeInfo: fieldTypeInfo, allowedGroupNames: allowedGroupNames, promotedFilters: promotedFilters, - handleQuery: (filters) => handleQuery(filters, true) + handleQuery: (filters) => handleQuery(filters, true, { page: 0, pageSize: pageSizeModel.pageSize}, sortModel) }} /> ); diff --git a/jbrowse/src/client/JBrowse/utils.ts b/jbrowse/src/client/JBrowse/utils.ts index 1401ef791..32d514cab 100644 --- a/jbrowse/src/client/JBrowse/utils.ts +++ b/jbrowse/src/client/JBrowse/utils.ts @@ -419,7 +419,7 @@ function generateLuceneString(field, operator, value) { return luceneQueryString; } -export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, successCallback, failureCallback) { +export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, sortField, sortReverseString, successCallback, failureCallback) { if (!offset) { offset = 0 } @@ -439,6 +439,13 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa return } + let sortReverse; + if(sortReverseString == "asc") { + sortReverse = true + } else { + sortReverse = false + } + return Ajax.request({ url: ActionURL.buildURL('jbrowse', 'luceneQuery.api'), method: 'GET', @@ -449,7 +456,15 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa failure: function(res) { failureCallback("There was an error: " + res.status + "\n Status Body: " + res.responseText + "\n Session ID:" + sessionId) }, - params: {"searchString": createEncodedFilterString(filters, true), "sessionId": sessionId, "trackId": trackGUID, "offset": offset, "pageSize": pageSize}, + params: { + "searchString": createEncodedFilterString(filters, true), + "sessionId": sessionId, + "trackId": trackGUID, + "offset": offset, + "pageSize": pageSize, + "sortField": sortField ?? "genomicPosition", + "sortReverse": sortReverse + }, }); } diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java index ada6cbbd5..11a01c283 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java @@ -910,7 +910,7 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors) try { - return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset())); + return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset(), form.getSortField(), form.getSortReverse())); } catch (Exception e) { @@ -947,6 +947,10 @@ public static class LuceneQueryForm private int _offset = 0; + private String _sortField = "genomicPosition"; + + private boolean _sortReverse = false; + public String getSearchString() { return _searchString; @@ -987,6 +991,14 @@ public void setOffset(int offset) _offset = offset; } + public String getSortField() { return _sortField; } + + public void setSortField(String sortField) { _sortField = sortField; } + + public boolean getSortReverse() { return _sortReverse; } + + public void setSortReverse(boolean sortReverse) { _sortReverse = sortReverse; } + public String getTrackId() { return _trackId; diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java index c5e1889a9..14d0aa031 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java @@ -22,6 +22,7 @@ import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.NumericUtils; import org.jetbrains.annotations.Nullable; import org.json.JSONObject; import org.labkey.api.data.Container; @@ -50,6 +51,7 @@ import java.util.StringTokenizer; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.labkey.jbrowse.JBrowseFieldUtils.VARIABLE_SAMPLES; import static org.labkey.jbrowse.JBrowseFieldUtils.getSession; @@ -61,6 +63,8 @@ public class JBrowseLuceneSearch private final JsonFile _jsonFile; private final User _user; private final String[] specialStartPatterns = {"*:* -", "+", "-"}; + private static final String ALL_DOCS = "all"; + private static final String GENOMIC_POSITION = "genomicPosition"; private JBrowseLuceneSearch(final JBrowseSession session, final JsonFile jsonFile, User u) { @@ -130,7 +134,7 @@ public String extractFieldName(String queryString) { return parts.length > 0 ? parts[0].trim() : null; } - public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset) throws IOException, ParseException + public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException { searchString = tryUrlDecode(searchString); File indexPath = _jsonFile.getExpectedLocationOfLuceneIndex(true); @@ -146,7 +150,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina IndexSearcher indexSearcher = new IndexSearcher(indexReader); List stringQueryParserFields = new ArrayList<>(); - List numericQueryParserFields = new ArrayList<>(); + Map numericQueryParserFields = new HashMap<>(); PointsConfig intPointsConfig = new PointsConfig(new DecimalFormat(), Integer.class); PointsConfig doublePointsConfig = new PointsConfig(new DecimalFormat(), Double.class); Map pointsConfigMap = new HashMap<>(); @@ -161,11 +165,11 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina { case Flag, String, Character -> stringQueryParserFields.add(field); case Float -> { - numericQueryParserFields.add(field); + numericQueryParserFields.put(field, SortField.Type.DOUBLE); pointsConfigMap.put(field, doublePointsConfig); } case Integer -> { - numericQueryParserFields.add(field); + numericQueryParserFields.put(field, SortField.Type.INT); pointsConfigMap.put(field, intPointsConfig); } } @@ -182,14 +186,14 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); - if (searchString.equals("all")) { + if (searchString.equals(ALL_DOCS)) { booleanQueryBuilder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST); } // Split input into tokens, 1 token per query separated by & StringTokenizer tokenizer = new StringTokenizer(searchString, "&"); - while (tokenizer.hasMoreTokens() && !searchString.equals("all")) + while (tokenizer.hasMoreTokens() && !searchString.equals(ALL_DOCS)) { String queryString = tokenizer.nextToken(); Query query = null; @@ -205,7 +209,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina { query = queryParser.parse(queryString); } - else if (numericQueryParserFields.contains(fieldName)) + else if (numericQueryParserFields.containsKey(fieldName)) { try { @@ -226,16 +230,28 @@ else if (numericQueryParserFields.contains(fieldName)) BooleanQuery query = booleanQueryBuilder.build(); + // By default, sort in INDEXORDER, which is by genomicPosition + Sort sort = Sort.INDEXORDER; + + // If the sort field is not genomicPosition, use the provided sorting data + if (!sortField.equals(GENOMIC_POSITION)) { + SortField.Type fieldType; + + if (stringQueryParserFields.contains(sortField)) { + fieldType = SortField.Type.STRING; + } else if (numericQueryParserFields.containsKey(sortField)) { + fieldType = numericQueryParserFields.get(sortField); + } else { + throw new IllegalArgumentException("Could not find type for sort field: " + sortField); + } + + sort = new Sort(new SortField(sortField, fieldType, sortReverse)); + } + // Get chunks of size {pageSize}. Default to 1 chunk -- add to the offset to get more. // We then iterate over the range of documents we want based on the offset. This does grow in memory // linearly with the number of documents, but my understanding is that these are just score,id pairs // rather than full documents, so mem usage *should* still be pretty low. - //TopDocs topDocs = indexSearcher.search(query, pageSize * (offset + 1)); - - // Define sort field - SortField sortField = new SortField("pos", SortField.Type.INT, false); - Sort sort = new Sort(sortField); - // Perform the search with sorting TopFieldDocs topDocs = indexSearcher.search(query, pageSize * (offset + 1), sort); @@ -253,10 +269,8 @@ else if (numericQueryParserFields.contains(fieldName)) String fieldName = field.name(); String[] fieldValues = doc.getValues(fieldName); if (fieldValues.length > 1) { - // If there is more than one value, put the array of values into the JSON object. elem.put(fieldName, fieldValues); } else { - // If there is only one value, just put this single value into the JSON object. elem.put(fieldName, fieldValues[0]); } } diff --git a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java index 57006b138..ddbc580ac 100644 --- a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java +++ b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java @@ -569,6 +569,7 @@ private void testFullTextSearch() throws Exception String sessionId = info.getKey(); String trackId = info.getValue(); + // all // this should return 143 results. We can't make any other assumptions about the content String url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=143"; @@ -580,7 +581,6 @@ private void testFullTextSearch() throws Exception JSONArray jsonArray = mainJsonObject.getJSONArray("data"); Assert.assertEquals(143, jsonArray.length()); - // stringType: // ref equals A url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=ref%3AA"; @@ -1150,6 +1150,91 @@ private void testFullTextSearch() throws Exception Assert.assertEquals("A", jsonObject.getString("ref")); } + // Default genomic position sort (ascending) + url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100"; + beginAt(url); + waitForText("data"); + waitAndClick(Locator.tagWithId("a", "rawdata-tab")); + jsonString = getText(Locator.tagWithClass("pre", "data")); + mainJsonObject = new JSONObject(jsonString); + jsonArray = mainJsonObject.getJSONArray("data"); + + long previousGenomicPosition = Long.MIN_VALUE; + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + long currentGenomicPosition = jsonObject.getLong("genomicPosition"); + Assert.assertTrue(currentGenomicPosition >= previousGenomicPosition); + previousGenomicPosition = currentGenomicPosition; + } + + // Sort by alt, ascending + url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=alt"; + beginAt(url); + waitForText("data"); + waitAndClick(Locator.tagWithId("a", "rawdata-tab")); + jsonString = getText(Locator.tagWithClass("pre", "data")); + mainJsonObject = new JSONObject(jsonString); + jsonArray = mainJsonObject.getJSONArray("data"); + + String previousAlt = ""; + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + String currentAlt = jsonObject.getString("alt"); + Assert.assertTrue(currentAlt.compareTo(previousAlt) >= 0); + previousAlt = currentAlt; + } + + // Sort by alt, descending + url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=alt&sortReverse=true"; + beginAt(url); + waitForText("data"); + waitAndClick(Locator.tagWithId("a", "rawdata-tab")); + jsonString = getText(Locator.tagWithClass("pre", "data")); + mainJsonObject = new JSONObject(jsonString); + jsonArray = mainJsonObject.getJSONArray("data"); + + previousAlt = "ZZZZ"; // Assuming 'Z' is higher than any character in your data + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + String currentAlt = jsonObject.getString("alt"); + Assert.assertTrue(currentAlt.compareTo(previousAlt) <= 0); + previousAlt = currentAlt; + } + + // Sort by af, ascending + url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=AF"; + beginAt(url); + waitForText("data"); + waitAndClick(Locator.tagWithId("a", "rawdata-tab")); + jsonString = getText(Locator.tagWithClass("pre", "data")); + mainJsonObject = new JSONObject(jsonString); + jsonArray = mainJsonObject.getJSONArray("data"); + + double previousAf = -1.0; + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + double currentAf = jsonObject.getDouble("AF"); + Assert.assertTrue(currentAf >= previousAf); + previousAf = currentAf; + } + + // Sort by af, descending + url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=AF&sortReverse=true"; + beginAt(url); + waitForText("data"); + waitAndClick(Locator.tagWithId("a", "rawdata-tab")); + jsonString = getText(Locator.tagWithClass("pre", "data")); + mainJsonObject = new JSONObject(jsonString); + jsonArray = mainJsonObject.getJSONArray("data"); + + previousAf = 2.0; // Assuming 'af' is <= 1.0 + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + double currentAf = jsonObject.getDouble("AF"); + Assert.assertTrue(currentAf <= previousAf); + previousAf = currentAf; + } + testLuceneSearchUI(sessionId); }