diff --git a/src/__mocks__/handlers/DataExplorer/handlers.ts b/src/__mocks__/handlers/DataExplorer/handlers.ts index 6264b6079..5e1ea3c6b 100644 --- a/src/__mocks__/handlers/DataExplorer/handlers.ts +++ b/src/__mocks__/handlers/DataExplorer/handlers.ts @@ -4,6 +4,7 @@ import { Resource } from '@bbp/nexus-sdk'; import { AggregatedBucket, AggregationsResult, + NexusMultiFetchResponse, } from 'subapps/dataExplorer/DataExplorerUtils'; export const getCompleteResources = ( @@ -16,26 +17,40 @@ export const dataExplorerPageHandler = ( partialResources: Resource[] = defaultPartialResources, total: number = 300 ) => { - return rest.get(deltaPath(`/resources`), (req, res, ctx) => { - if (req.url.searchParams.has('aggregations')) { - return res(ctx.status(200), ctx.json(mockAggregationsResult())); - } - const passedType = req.url.searchParams.get('type'); - const mockResponse = { - '@context': [ - 'https://bluebrain.github.io/nexus/contexts/metadata.json', - 'https://bluebrain.github.io/nexus/contexts/search.json', - 'https://bluebrain.github.io/nexus/contexts/search-metadata.json', - ], - _total: total, - _results: passedType - ? partialResources.filter(res => res['@type'] === passedType) - : partialResources, - _next: - 'https://bbp.epfl.ch/nexus/v1/resources?size=50&sort=@id&after=%5B1687269183553,%22https://bbp.epfl.ch/neurosciencegraph/data/31e22529-2c36-44f0-9158-193eb50526cd%22%5D', - }; - return res(ctx.status(200), ctx.json(mockResponse)); - }); + return [ + rest.get(deltaPath(`/resources`), (req, res, ctx) => { + if (req.url.searchParams.has('aggregations')) { + return res(ctx.status(200), ctx.json(mockAggregationsResult())); + } + const passedType = req.url.searchParams.get('type'); + const mockResponse = { + '@context': [ + 'https://bluebrain.github.io/nexus/contexts/metadata.json', + 'https://bluebrain.github.io/nexus/contexts/search.json', + 'https://bluebrain.github.io/nexus/contexts/search-metadata.json', + ], + _total: total, + _results: passedType + ? partialResources.filter(res => res['@type'] === passedType) + : partialResources, + _next: + 'https://bbp.epfl.ch/nexus/v1/resources?size=50&sort=@id&after=%5B1687269183553,%22https://bbp.epfl.ch/neurosciencegraph/data/31e22529-2c36-44f0-9158-193eb50526cd%22%5D', + }; + return res(ctx.status(200), ctx.json(mockResponse)); + }), + rest.post(deltaPath('/multi-fetch/resources'), (req, res, ctx) => { + const requestedIds = ((req.body as any)?.resources).map( + (res: { id: string }) => res.id + ); + const response: NexusMultiFetchResponse = { + format: 'compacted', + resources: partialResources + .filter(res => requestedIds.includes(res['@id'])) + .map(r => ({ value: { ...r, ...propertiesOnlyInSource } })), + }; + return res(ctx.status(200), ctx.json(response)); + }), + ]; }; export const graphAnalyticsTypeHandler = () => { @@ -147,16 +162,17 @@ export const filterByProjectHandler = ( }); }; -export const elasticSearchQueryHandler = (ids: string[]) => { +export const elasticSearchQueryHandler = (resources: Resource[]) => { return rest.post( deltaPath('/graph-analytics/:org/:project/_search'), (req, res, ctx) => { const esResponse = { hits: { - hits: ids.map(id => ({ - _id: id, + hits: resources.map(resource => ({ + _id: resource['@id'], _source: { - '@id': id, + '@id': resource['@id'], + _project: resource._project, }, })), max_score: 0, diff --git a/src/subapps/dataExplorer/DataExplorer.spec.tsx b/src/subapps/dataExplorer/DataExplorer.spec.tsx index e3da2e493..1c6bec8f9 100644 --- a/src/subapps/dataExplorer/DataExplorer.spec.tsx +++ b/src/subapps/dataExplorer/DataExplorer.spec.tsx @@ -47,7 +47,7 @@ describe('DataExplorer', () => { ]; const server = setupServer( - dataExplorerPageHandler(undefined, defaultTotalResults), + ...dataExplorerPageHandler(undefined, defaultTotalResults), sourceResourceHandler(), filterByProjectHandler(), graphAnalyticsTypeHandler() @@ -241,7 +241,7 @@ describe('DataExplorer', () => { ) => { server.use( sourceResourceHandler(resources), - dataExplorerPageHandler(resources, total) + ...dataExplorerPageHandler(resources, total) ); const pageInput = await screen.getByRole('listitem', { name: '2' }); @@ -368,9 +368,7 @@ describe('DataExplorer', () => { const matchingResources = resources.filter(res => predicate === EXISTS ? res[path] : !res[path] ); - server.use( - elasticSearchQueryHandler(matchingResources.map(res => res['@id'])) - ); + server.use(elasticSearchQueryHandler(matchingResources)); }; const getResetProjectButton = async () => { @@ -531,7 +529,7 @@ describe('DataExplorer', () => { mock100Resources.push(getMockResource(`self${i}`, {})); } - server.use(dataExplorerPageHandler(mock100Resources)); + server.use(...dataExplorerPageHandler(mock100Resources)); const pageSizeChanger = await screen.getByRole('combobox', { name: 'Page Size', @@ -893,6 +891,7 @@ describe('DataExplorer', () => { mockElasticSearchHits('author', EXISTS, mockResourcesForPage2); await selectPath('author'); + await selectPredicate(EXISTS); const selectedPathBefore = await getSelectedValueInMenu(PathMenuLabel); diff --git a/src/subapps/dataExplorer/DataExplorer.tsx b/src/subapps/dataExplorer/DataExplorer.tsx index ee216b8af..35febbfe8 100644 --- a/src/subapps/dataExplorer/DataExplorer.tsx +++ b/src/subapps/dataExplorer/DataExplorer.tsx @@ -123,6 +123,7 @@ export const columnsFromDataSource = ( ) .sort(sortColumns); }; + const DataExplorer: React.FC<{}> = () => { const history = useHistory(); const [showMetadataColumns, setShowMetadataColumns] = useState(false); diff --git a/src/subapps/dataExplorer/DataExplorerUtils.tsx b/src/subapps/dataExplorer/DataExplorerUtils.tsx index 6c8c8df37..52803603e 100644 --- a/src/subapps/dataExplorer/DataExplorerUtils.tsx +++ b/src/subapps/dataExplorer/DataExplorerUtils.tsx @@ -7,7 +7,6 @@ import { isString } from 'lodash'; import PromisePool from '@supercharge/promise-pool'; import { makeOrgProjectTuple } from '../../shared/molecules/MyDataTable/MyDataTable'; import { TTypeOperator } from '../../shared/molecules/TypeSelector/types'; -import * as bodybuilder from 'bodybuilder'; import { useSelector } from 'react-redux'; import { RootState } from 'shared/store/reducers'; import { SearchResponse } from 'shared/types/search'; @@ -67,39 +66,14 @@ export const usePaginatedExpandedResources = ({ } ); - // If we failed to fetch the expanded source for some resources, we can use the compact/partial resource as a fallback. - const fallbackResources: Resource[] = []; - const { results: expandedResources } = await PromisePool.withConcurrency( - 4 - ) - .for(resultWithPartialResources._results) - .handleError(async (err, partialResource) => { - console.log( - `@@error in fetching resource with id: ${partialResource['@id']}`, - err - ); - fallbackResources.push(partialResource); - return; - }) - .process(async partialResource => { - if (partialResource._project) { - const { org, project } = makeOrgProjectTuple( - partialResource._project - ); - - return (await nexus.Resource.get( - org, - project, - encodeURIComponent(partialResource['@id']), - { annotate: true } - )) as Resource; - } - - return partialResource; - }); + const expandedResources = await fetchMultipleResources( + nexus, + apiEndpoint, + resultWithPartialResources._results + ); return { ...resultWithPartialResources, - _results: [...expandedResources, ...fallbackResources], + _results: expandedResources, }; }, onError: error => { @@ -121,6 +95,62 @@ export const usePaginatedExpandedResources = ({ }); }; +export type NexusResourceFormats = + | 'source' + | 'compacted' + | 'expanded' + | 'n-triples' + | 'dot'; +export type NexusMultiFetchResponse = { + format: NexusResourceFormats; + resources: { value: Resource }[]; +}; + +type PartialResource = Pick; + +export const fetchMultipleResources = async ( + nexus: ReturnType, + apiEndpoint: string, + partialResources: PartialResource[], + orgAndProject?: string +): Promise => { + const resourceData = partialResources + .filter(resource => resource._project) + .map(resource => { + if (orgAndProject) { + return { + id: resource['@id'], + project: orgAndProject, + }; + } + + const { org, project } = makeOrgProjectTuple(resource._project); + + return { + id: resource['@id'], + project: `${org}/${project}`, + }; + }); + console.log( + 'Going to multi fetch', + resourceData.map(r => r.id) + ); + const multipleResources: NexusMultiFetchResponse = await nexus + .httpPost({ + path: `${apiEndpoint}/multi-fetch/resources`, + headers: { Accept: 'application/json' }, + body: JSON.stringify({ + format: 'compacted', + resources: resourceData, + }), + }) + .catch(() => { + return { format: 'compacted', value: [] }; + }); + + return multipleResources.resources.map(({ value }) => ({ ...value })); +}; + export const useAggregations = ( bucketName: 'projects' | 'types', orgAndProject?: string[] @@ -190,38 +220,35 @@ const getResultsForPredicateQuery = async ( pageSize: number, offset: number ) => { - const searchResults: SearchResponse<{ '@id': string }> = await nexus.httpPost( - { - path: `${apiEndpoint}/graph-analytics/${org}/${project}/_search`, - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - }, - body: JSON.stringify({ - from: offset, - size: pageSize, - ...query, - }), - } - ); + const searchResults: SearchResponse<{ + '@id': string; + _project: string; + }> = await nexus.httpPost({ + path: `${apiEndpoint}/graph-analytics/${org}/${project}/_search`, + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + }, + body: JSON.stringify({ + from: offset, + size: pageSize, + ...query, + }), + }); - const { results: matchingResources } = await PromisePool.withConcurrency(4) - .for(searchResults.hits.hits) - .handleError((err, item) => { - console.error( - `There was an error retrieving matching resource ${item._source['@id']} for query.`, - query, - err - ); - }) - .process(async matchingHit => { - return (await nexus.Resource.get( - org, - project, - encodeURIComponent(matchingHit._source['@id']), - { annotate: true } - )) as Resource; - }); + const resourcesToFetch = searchResults.hits.hits.map(matching => ({ + '@id': matching._source['@id'], + _project: matching._source['_project'], + })); + console.log( + 'Requesting matching resources', + resourcesToFetch.map(r => r['@id']) + ); + const matchingResources = await fetchMultipleResources( + nexus, + apiEndpoint, + resourcesToFetch + ); return { _results: matchingResources,