Skip to content
This repository was archived by the owner on Feb 26, 2025. It is now read-only.

Commit

Permalink
4083 // Run predicates in backend
Browse files Browse the repository at this point in the history 
Signed-off-by: Dinika Saxena <dinikasaxenas@gmail.com>
  • Loading branch information
@Dinika
Dinika committed Sep 29, 2023
1 parent 6f7fe77 commit 8b819ef
Show file tree
Hide file tree
Showing 7 changed files with 453 additions and 114 deletions.
7 changes: 5 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
"start": "NODE_ENV=development DEBUG=* webpack --mode development --config-name server && node dist/server.js",
"build": "NODE_ENV=production NODE_OPTIONS=--max_old_space_size=8192 webpack --mode production",
"test": "jest",
"test:watch": "jest --watch",
"test:watch": "jest --watch --maxWorkers=4",
"cy:open": "cypress open",
"cy:run": "cypress run",
"cy:ci": "echo | sudo docker exec --user=$UID -t cypress cypress run --config-file cypress.config.ts --browser chrome",
Expand Down Expand Up @@ -212,7 +212,10 @@
],
"globals": {
"FUSION_VERSION": "1.0.0",
"COMMIT_HASH": "9013fa343"
"COMMIT_HASH": "9013fa343",
"ts-jest": {
"isolatedModules": true
}
},
"watchPathIgnorePatterns": [
"node_modules"
Expand Down
27 changes: 25 additions & 2 deletions src/__mocks__/handlers/DataExplorer/handlers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,12 @@ export const graphAnalyticsTypeHandler = () => {
_name: 'propertyAlwaysThere',
},
{
'@id': 'https://neuroshapes.org/nr__number_stems',
'@id': 'https://neuroshapes.org/createdBy',
_count: 30,
_name: '_createdBy',
},
{
'@id': 'https://neuroshapes.org/nr__number_stems',
'@id': 'https://neuroshapes.org/edition',
_count: 30,
_name: 'edition',
},
Expand Down Expand Up @@ -147,6 +147,29 @@ export const filterByProjectHandler = (
});
};

export const elasticSearchQueryHandler = (ids: string[]) => {
return rest.post(
deltaPath('/graph-analytics/:org/:project/_search'),
(req, res, ctx) => {
const esResponse = {
hits: {
hits: ids.map(id => ({
_id: id,
_source: {
'@id': id,
},
})),
max_score: 0,
total: {
value: 479,
},
},
};
return res(ctx.status(200), ctx.json(esResponse));
}
);
};

const mockAggregationsResult = (
bucketForTypes: AggregatedBucket[] = defaultBucketForTypes
): AggregationsResult => {
Expand Down
91 changes: 76 additions & 15 deletions src/subapps/dataExplorer/DataExplorer.spec.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import userEvent from '@testing-library/user-event';
import { UserEvent } from '@testing-library/user-event/dist/types/setup/setup';
import {
dataExplorerPageHandler,
elasticSearchQueryHandler,
filterByProjectHandler,
getCompleteResources,
getMockResource,
Expand All @@ -24,6 +25,7 @@ import {
DOES_NOT_CONTAIN,
DOES_NOT_EXIST,
EXISTS,
FRONTEND_PREDICATE_WARNING,
getAllPaths,
} from './PredicateSelector';
import { createMemoryHistory } from 'history';
Expand Down Expand Up @@ -64,7 +66,12 @@ describe('DataExplorer', () => {
fetch,
uri: deltaPath(),
});
const store = configureStore(history, { nexus }, {});

const store = configureStore(
history,
{ nexus },
{ config: { apiEndpoint: 'https://localhost:3000' } }
);

dataExplorerPage = (
<Provider store={store}>
Expand Down Expand Up @@ -109,9 +116,6 @@ describe('DataExplorer', () => {
const expectRowCountToBe = async (expectedRowsCount: number) => {
return await waitFor(() => {
const rows = visibleTableRows();
rows.forEach(row => {
// console.log('MY ROW', row.innerHTML)
});
expect(rows.length).toEqual(expectedRowsCount);
return rows;
});
Expand Down Expand Up @@ -141,6 +145,7 @@ describe('DataExplorer', () => {
selector: 'th .ant-table-column-title',
exact: false,
});

expect(header).toBeInTheDocument();
return header;
};
Expand Down Expand Up @@ -355,6 +360,19 @@ describe('DataExplorer', () => {
server.use(filterByProjectHandler(mockResourcesForPage2));
};

const mockElasticSearchHits = (
path: string,
predicate: typeof EXISTS | typeof DOES_NOT_EXIST,
resources: Resource[]
) => {
const matchingResources = resources.filter(res =>
predicate === EXISTS ? res[path] : !res[path]
);
server.use(
elasticSearchQueryHandler(matchingResources.map(res => res['@id']))
);
};

const getResetProjectButton = async () => {
return await screen.getByTestId('reset-project-button');
};
Expand Down Expand Up @@ -673,12 +691,14 @@ describe('DataExplorer', () => {
it('shows resources that have path missing', async () => {
await updateResourcesShownInTable(mockResourcesForPage2);

mockElasticSearchHits('author', DOES_NOT_EXIST, mockResourcesForPage2);
await selectPath('author');
await selectOptionFromMenu(PredicateMenuLabel, DOES_NOT_EXIST);
await expectRowCountToBe(1);

await resetPredicate();

mockElasticSearchHits('edition', DOES_NOT_EXIST, mockResourcesForPage2);
await selectPath('edition');
await selectOptionFromMenu(PredicateMenuLabel, DOES_NOT_EXIST);
await expectRowCountToBe(2);
Expand Down Expand Up @@ -718,11 +738,8 @@ describe('DataExplorer', () => {
});

it('shows resources that have a path when user selects exists predicate', async () => {
await updateResourcesShownInTable([
getMockResource('self1', { author: 'piggy', edition: 1 }),
getMockResource('self2', { author: ['iggy', 'twinky'] }),
getMockResource('self3', { year: 2013 }),
]);
await updateResourcesShownInTable(mockResourcesForPage2);
mockElasticSearchHits('author', EXISTS, mockResourcesForPage2);

await selectPath('author');
await userEvent.click(container);
Expand Down Expand Up @@ -798,20 +815,61 @@ describe('DataExplorer', () => {
expect(totalFromFrontend).toEqual(null);
});

it('shows total filtered count if predicate is selected', async () => {
it('does not show total filtered count if backend predicate is selected', async () => {
await expectRowCountToBe(10);
await updateResourcesShownInTable(mockResourcesForPage2);
mockElasticSearchHits('author', EXISTS, mockResourcesForPage2);

await selectPath('author');
await userEvent.click(container);
await selectOptionFromMenu(PredicateMenuLabel, EXISTS);
await expectRowCountToBe(2);
const totalFromFrontendAfter = await getFilteredResultsCount(2);
expect(totalFromFrontendAfter).toBeVisible();
expect(await getFilteredResultsCount()).toBeFalsy();

mockElasticSearchHits('author', DOES_NOT_EXIST, mockResourcesForPage2);
await selectOptionFromMenu(PredicateMenuLabel, DOES_NOT_EXIST);
await expectRowCountToBe(1);
expect(await getFilteredResultsCount()).toBeFalsy();
});

it('shows filtered count if frontend predicate is selected', async () => {
await updateResourcesShownInTable(mockResourcesForPage2);

await selectPath('author');
await selectOptionFromMenu(PredicateMenuLabel, CONTAINS);
const valueInput = await screen.getByPlaceholderText('Search for...');
await userEvent.type(valueInput, 'twinky');
await expectRowCountToBe(1);

expect(await getFilteredResultsCount(1)).toBeVisible();
});

it('shows predicate warning if frontend predicate is selected', async () => {
await updateResourcesShownInTable(mockResourcesForPage2);

await selectPath('author');
await selectOptionFromMenu(PredicateMenuLabel, CONTAINS);

expect(await screen.getByText(FRONTEND_PREDICATE_WARNING)).toBeVisible();

await selectOptionFromMenu(PredicateMenuLabel, DOES_NOT_CONTAIN);
expect(await screen.getByText(FRONTEND_PREDICATE_WARNING)).toBeVisible();
});

it('does not show predicate warning if backend predicate is selected', async () => {
await updateResourcesShownInTable(mockResourcesForPage2);

await selectPath('author');
await selectOptionFromMenu(PredicateMenuLabel, EXISTS);

expect(await screen.queryByText(FRONTEND_PREDICATE_WARNING)).toBeFalsy();
await selectOptionFromMenu(PredicateMenuLabel, EXISTS);
expect(await screen.queryByText(FRONTEND_PREDICATE_WARNING)).toBeFalsy();
});

it('shows column for metadata path even if toggle for show metadata is off', async () => {
const metadataProperty = '_createdBy';
mockElasticSearchHits(metadataProperty, EXISTS, mockResourcesOnPage1);
await expectRowCountToBe(10);

await expectColumHeaderToNotExist(metadataProperty);
Expand All @@ -821,6 +879,8 @@ describe('DataExplorer', () => {
await selectPath(metadataProperty);
await selectOptionFromMenu(PredicateMenuLabel, EXISTS);

await expectRowCountToBe(10);

await expectColumHeaderToExist(metadataProperty);
expect(getTotalColumns().length).toEqual(originalColumns + 1);

Expand All @@ -830,6 +890,7 @@ describe('DataExplorer', () => {

it('resets predicate fields when reset predicate clicked', async () => {
await updateResourcesShownInTable(mockResourcesForPage2);
mockElasticSearchHits('author', EXISTS, mockResourcesForPage2);

await selectPath('author');
await selectPredicate(EXISTS);
Expand Down Expand Up @@ -985,17 +1046,17 @@ describe('DataExplorer', () => {
expect((valueInputAfter as HTMLInputElement).value).not.toEqual('iggy');
});

it('does not show predicate selector if multiple types are selected', async () => {
it('disables predicate selector if multiple types are selected', async () => {
await selectOptionFromMenu(ProjectMenuLabel, 'unhcr');
await selectOptionFromMenu(TypeMenuLabel, 'file', TypeOptionSelector);

expect(await getInputForLabel(PathMenuLabel)).toBeVisible();
expect(await getInputForLabel(PathMenuLabel)).toBeEnabled();

await selectOptionFromMenu(
TypeMenuLabel,
'StudioDashboard',
TypeOptionSelector
);
expect(getInputForLabel(PathMenuLabel)).rejects.toThrow();
expect(await getInputForLabel(PathMenuLabel)).toBeDisabled();
});
});
27 changes: 18 additions & 9 deletions src/subapps/dataExplorer/DataExplorer.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ export interface DataExplorerConfiguration {
offset: number;
orgAndProject?: [string, string];
types?: TType[];
predicate: ((resource: Resource) => boolean) | null;
frontendPredicate: ((resource: Resource) => boolean) | null;
backendPredicateQuery: Object | null;
selectedPath: string | null;
deprecated: boolean;
columns: TColumn[];
Expand Down Expand Up @@ -137,7 +138,11 @@ const DataExplorer: React.FC<{}> = () => {
pageSize,
offset,
orgAndProject,
predicate,
// NOTE: Right now, the `EXISTS` and `DOES_NOT_EXIST` predicates run on the backend and update the `backendPredicateQuery` parameter.
// `CONTAINS` and `DOES_NOT_CONTAIN` predicates on the other hand, only run on frontend and update `frontendPredicate` parameter.
// When we implement running all the predicates on backend, we should discard `frontendPredicate` parameter completely.
frontendPredicate,
backendPredicateQuery,
types,
selectedPath,
deprecated,
Expand All @@ -158,7 +163,8 @@ const DataExplorer: React.FC<{}> = () => {
offset: 0,
orgAndProject: undefined,
types: [],
predicate: null,
frontendPredicate: null,
backendPredicateQuery: null,
selectedPath: null,
deprecated: false,
columns: [],
Expand All @@ -173,12 +179,13 @@ const DataExplorer: React.FC<{}> = () => {
deprecated,
typeOperator,
types: types?.map(t => t.value),
predicateQuery: backendPredicateQuery,
});

const currentPageDataSource: Resource[] = resources?._results || [];

const displayedDataSource = predicate
? currentPageDataSource.filter(predicate)
const displayedDataSource = frontendPredicate
? currentPageDataSource.filter(frontendPredicate)
: currentPageDataSource;

const buildColumns = useMemo(() => {
Expand Down Expand Up @@ -224,6 +231,7 @@ const DataExplorer: React.FC<{}> = () => {
updateTableConfiguration({ columns: newColumns });
updateSelectedColumnsCached(newColumns);
};

useEffect(() => {
const selectedFilters = getSelectedFiltersCached();
if (selectedFilters) {
Expand Down Expand Up @@ -286,8 +294,7 @@ const DataExplorer: React.FC<{}> = () => {
return () => unlisten();
}, []);

const shouldShowPredicateSelector =
orgAndProject?.length === 2 && types?.length === 1;
const shouldShowPredicateSelector = orgAndProject?.length === 2;

return (
<div className="data-explorer-contents" ref={containerRef}>
Expand Down Expand Up @@ -359,7 +366,7 @@ const DataExplorer: React.FC<{}> = () => {
onResetCallback={onResetPredicateCallback}
org={orgAndProject![0]}
project={orgAndProject![1]}
types={types!.map(type => type.value)}
types={types}
/>
) : null}
</div>
Expand All @@ -380,7 +387,9 @@ const DataExplorer: React.FC<{}> = () => {
types={types}
nexusTotal={resources?._total ?? 0}
totalOnPage={resources?._results?.length ?? 0}
totalFiltered={predicate ? displayedDataSource.length : undefined}
totalFiltered={
frontendPredicate ? displayedDataSource.length : undefined
}
/>
<div className="data-explorer-toggles">
<Switch
Expand Down
Loading

0 comments on commit 8b819ef

Please sign in to comment.