Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Custom server-side sorting #264

Merged
merged 7 commits into from
Mar 28, 2024
Merged

Custom server-side sorting #264

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
adb5cc4
WIP custom sorting fields
Feb 12, 2024
e30600d
Fix passing sortReverse flag to API
Feb 23, 2024
9d5630c
Sort tests
Mar 16, 2024
d6290c5
Fix client-side react
Mar 27, 2024
8dac438
Fix AF tests
Mar 27, 2024
7535a3c
Expand cellsnp-like to export cell-level genotypes
bbimber Mar 28, 2024
12a9ea9
Merge branch 'discvr-23.11' into 23.11_fb_serverSort
bbimber Mar 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 45 additions & 8 deletions jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ import {
GridColumnVisibilityModel,
GridPaginationModel,
GridRenderCellParams,
GridSortDirection,
GridSortModel,
GridToolbarColumnsButton,
GridToolbarContainer,
GridToolbarDensitySelector,
GridToolbarExport
} from '@mui/x-data-grid';
import SearchIcon from '@mui/icons-material/Search';
import React, { useEffect, useState, useMemo } from 'react';
import LinkIcon from '@mui/icons-material/Link';
import React, { useEffect, useState } from 'react';
import { getConf } from '@jbrowse/core/configuration';
import { AppBar, Box, Button, Dialog, Paper, Popover, Toolbar, Tooltip, Typography } from '@mui/material';
import { FilterFormModal } from './FilterFormModal';
Expand Down Expand Up @@ -67,22 +70,25 @@ const VariantTableWidget = observer(props => {
session.hideWidget(widget)
}

function handleQuery(passedFilters, pushToHistory, pageQueryModel = pageSizeModel) {
function handleQuery(passedFilters, pushToHistory, pageQueryModel = pageSizeModel, sortQueryModel = sortModel) {
const { page = pageSizeModel.page, pageSize = pageSizeModel.pageSize } = pageQueryModel;
const { field = "genomicPosition", sort = false } = sortQueryModel[0] ?? {};

const encodedSearchString = createEncodedFilterString(passedFilters, false);
const currentUrl = new URL(window.location.href);
currentUrl.searchParams.set("searchString", encodedSearchString);
currentUrl.searchParams.set("page", page.toString());
currentUrl.searchParams.set("pageSize", pageSize.toString());
currentUrl.searchParams.set("sortField", field.toString());
currentUrl.searchParams.set("sortDirection", sort.toString());

if (pushToHistory) {
window.history.pushState(null, "", currentUrl.toString());
}

setFilters(passedFilters);
setDataLoaded(false)
fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)});
fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, field, sort, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)});
}

const TableCellWithPopover = (props: { value: any }) => {
Expand Down Expand Up @@ -190,7 +196,29 @@ const VariantTableWidget = observer(props => {
Search
</Button>
<GridToolbarDensitySelector />
<GridToolbarExport />
<GridToolbarExport csvOptions={{
delimiter: ';',
}} />

<Button
startIcon={<LinkIcon />}
size="small"
color="primary"
onClick={() => {
navigator.clipboard.writeText(window.location.href)
.then(() => {
// Popup message for successful copy
alert('URL copied to clipboard.');
})
.catch(err => {
// Error handling
console.error('Failed to copy the URL: ', err);
alert('Failed to copy the URL.');
});
}}
>
Share
</Button>
</GridToolbarContainer>
);
}
Expand Down Expand Up @@ -234,11 +262,15 @@ const VariantTableWidget = observer(props => {
// False until initial data load or an error:
const [dataLoaded, setDataLoaded] = useState(false)

const urlParams = new URLSearchParams(window.location.search);
const page = parseInt(urlParams.get('page') || '0');
const pageSize = parseInt(urlParams.get('pageSize') || '50');
const urlParams = new URLSearchParams(window.location.search)
const page = parseInt(urlParams.get('page') || '0')
const pageSize = parseInt(urlParams.get('pageSize') || '50')
const [pageSizeModel, setPageSizeModel] = React.useState<GridPaginationModel>({ page, pageSize });

const sortField = urlParams.get('sortField') || 'genomicPosition'
const sortDirection = urlParams.get('sortDirection') || 'desc'
const [sortModel, setSortModel] = React.useState<GridSortModel>([{ field: sortField, sort: sortDirection as GridSortDirection }])

const colVisURLComponent = urlParams.get("colVisModel") || "{}"
const colVisModel = JSON.parse(decodeURIComponent(colVisURLComponent))
const [columnVisibilityModel, setColumnVisibilityModel] = useState<GridColumnVisibilityModel>(colVisModel);
Expand Down Expand Up @@ -419,6 +451,11 @@ const VariantTableWidget = observer(props => {
currentUrl.searchParams.set("colVisModel", encodeURIComponent(JSON.stringify(trueValuesModel)));
window.history.pushState(null, "", currentUrl.toString());
}}
sortingMode="server"
onSortModelChange={(newModel) => {
setSortModel(newModel)
handleQuery(filters, true, { page: 0, pageSize: pageSizeModel.pageSize }, newModel);
}}
/>
)

Expand All @@ -440,7 +477,7 @@ const VariantTableWidget = observer(props => {
fieldTypeInfo: fieldTypeInfo,
allowedGroupNames: allowedGroupNames,
promotedFilters: promotedFilters,
handleQuery: (filters) => handleQuery(filters, true)
handleQuery: (filters) => handleQuery(filters, true, { page: 0, pageSize: pageSizeModel.pageSize}, sortModel)
}}
/>
);
Expand Down
19 changes: 17 additions & 2 deletions jbrowse/src/client/JBrowse/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,7 @@ function generateLuceneString(field, operator, value) {
return luceneQueryString;
}

export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, successCallback, failureCallback) {
export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, sortField, sortReverseString, successCallback, failureCallback) {
if (!offset) {
offset = 0
}
Expand All @@ -439,6 +439,13 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa
return
}

let sortReverse;
if(sortReverseString == "asc") {
sortReverse = true
} else {
sortReverse = false
}

return Ajax.request({
url: ActionURL.buildURL('jbrowse', 'luceneQuery.api'),
method: 'GET',
Expand All @@ -449,7 +456,15 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa
failure: function(res) {
failureCallback("There was an error: " + res.status + "\n Status Body: " + res.responseText + "\n Session ID:" + sessionId)
},
params: {"searchString": createEncodedFilterString(filters, true), "sessionId": sessionId, "trackId": trackGUID, "offset": offset, "pageSize": pageSize},
params: {
"searchString": createEncodedFilterString(filters, true),
"sessionId": sessionId,
"trackId": trackGUID,
"offset": offset,
"pageSize": pageSize,
"sortField": sortField ?? "genomicPosition",
"sortReverse": sortReverse
},
});
}

Expand Down
14 changes: 13 additions & 1 deletion jbrowse/src/org/labkey/jbrowse/JBrowseController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,7 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors)

try
{
return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset()));
return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset(), form.getSortField(), form.getSortReverse()));
}
catch (Exception e)
{
Expand Down Expand Up @@ -947,6 +947,10 @@ public static class LuceneQueryForm

private int _offset = 0;

private String _sortField = "genomicPosition";

private boolean _sortReverse = false;

public String getSearchString()
{
return _searchString;
Expand Down Expand Up @@ -987,6 +991,14 @@ public void setOffset(int offset)
_offset = offset;
}

public String getSortField() { return _sortField; }

public void setSortField(String sortField) { _sortField = sortField; }

public boolean getSortReverse() { return _sortReverse; }

public void setSortReverse(boolean sortReverse) { _sortReverse = sortReverse; }

public String getTrackId()
{
return _trackId;
Expand Down
44 changes: 29 additions & 15 deletions jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.NumericUtils;
import org.jetbrains.annotations.Nullable;
import org.json.JSONObject;
import org.labkey.api.data.Container;
Expand Down Expand Up @@ -50,6 +51,7 @@
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static org.labkey.jbrowse.JBrowseFieldUtils.VARIABLE_SAMPLES;
import static org.labkey.jbrowse.JBrowseFieldUtils.getSession;
Expand All @@ -61,6 +63,8 @@ public class JBrowseLuceneSearch
private final JsonFile _jsonFile;
private final User _user;
private final String[] specialStartPatterns = {"*:* -", "+", "-"};
private static final String ALL_DOCS = "all";
private static final String GENOMIC_POSITION = "genomicPosition";

private JBrowseLuceneSearch(final JBrowseSession session, final JsonFile jsonFile, User u)
{
Expand Down Expand Up @@ -130,7 +134,7 @@ public String extractFieldName(String queryString) {
return parts.length > 0 ? parts[0].trim() : null;
}

public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset) throws IOException, ParseException
public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException
{
searchString = tryUrlDecode(searchString);
File indexPath = _jsonFile.getExpectedLocationOfLuceneIndex(true);
Expand All @@ -146,7 +150,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
IndexSearcher indexSearcher = new IndexSearcher(indexReader);

List<String> stringQueryParserFields = new ArrayList<>();
List<String> numericQueryParserFields = new ArrayList<>();
Map<String, SortField.Type> numericQueryParserFields = new HashMap<>();
PointsConfig intPointsConfig = new PointsConfig(new DecimalFormat(), Integer.class);
PointsConfig doublePointsConfig = new PointsConfig(new DecimalFormat(), Double.class);
Map<String, PointsConfig> pointsConfigMap = new HashMap<>();
Expand All @@ -161,11 +165,11 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
{
case Flag, String, Character -> stringQueryParserFields.add(field);
case Float -> {
numericQueryParserFields.add(field);
numericQueryParserFields.put(field, SortField.Type.DOUBLE);
pointsConfigMap.put(field, doublePointsConfig);
}
case Integer -> {
numericQueryParserFields.add(field);
numericQueryParserFields.put(field, SortField.Type.INT);
pointsConfigMap.put(field, intPointsConfig);
}
}
Expand All @@ -182,14 +186,14 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina

BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder();

if (searchString.equals("all")) {
if (searchString.equals(ALL_DOCS)) {
booleanQueryBuilder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
}

// Split input into tokens, 1 token per query separated by &
StringTokenizer tokenizer = new StringTokenizer(searchString, "&");

while (tokenizer.hasMoreTokens() && !searchString.equals("all"))
while (tokenizer.hasMoreTokens() && !searchString.equals(ALL_DOCS))
{
String queryString = tokenizer.nextToken();
Query query = null;
Expand All @@ -205,7 +209,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
{
query = queryParser.parse(queryString);
}
else if (numericQueryParserFields.contains(fieldName))
else if (numericQueryParserFields.containsKey(fieldName))
{
try
{
Expand All @@ -226,16 +230,28 @@ else if (numericQueryParserFields.contains(fieldName))

BooleanQuery query = booleanQueryBuilder.build();

// By default, sort in INDEXORDER, which is by genomicPosition
Sort sort = Sort.INDEXORDER;

// If the sort field is not genomicPosition, use the provided sorting data
if (!sortField.equals(GENOMIC_POSITION)) {
SortField.Type fieldType;

if (stringQueryParserFields.contains(sortField)) {
fieldType = SortField.Type.STRING;
} else if (numericQueryParserFields.containsKey(sortField)) {
fieldType = numericQueryParserFields.get(sortField);
} else {
throw new IllegalArgumentException("Could not find type for sort field: " + sortField);
}

sort = new Sort(new SortField(sortField, fieldType, sortReverse));
}

// Get chunks of size {pageSize}. Default to 1 chunk -- add to the offset to get more.
// We then iterate over the range of documents we want based on the offset. This does grow in memory
// linearly with the number of documents, but my understanding is that these are just score,id pairs
// rather than full documents, so mem usage *should* still be pretty low.
//TopDocs topDocs = indexSearcher.search(query, pageSize * (offset + 1));

// Define sort field
SortField sortField = new SortField("pos", SortField.Type.INT, false);
Sort sort = new Sort(sortField);

// Perform the search with sorting
TopFieldDocs topDocs = indexSearcher.search(query, pageSize * (offset + 1), sort);

Expand All @@ -253,10 +269,8 @@ else if (numericQueryParserFields.contains(fieldName))
String fieldName = field.name();
String[] fieldValues = doc.getValues(fieldName);
if (fieldValues.length > 1) {
// If there is more than one value, put the array of values into the JSON object.
elem.put(fieldName, fieldValues);
} else {
// If there is only one value, just put this single value into the JSON object.
elem.put(fieldName, fieldValues[0]);
}
}
Expand Down
Loading