4289 // Add utility function for multi-get

Signed-off-by: Dinika Saxena <dinikasaxenas@gmail.com>
BlueBrain · Sep 29, 2023 · 88f5a67 · 88f5a67
1 parent 8b819ef
commit 88f5a67
Show file tree

Hide file tree

Showing 4 changed files with 136 additions and 93 deletions.
diff --git a/src/__mocks__/handlers/DataExplorer/handlers.ts b/src/__mocks__/handlers/DataExplorer/handlers.ts
@@ -4,6 +4,7 @@ import { Resource } from '@bbp/nexus-sdk';
 import {
   AggregatedBucket,
   AggregationsResult,
+  NexusMultiFetchResponse,
 } from 'subapps/dataExplorer/DataExplorerUtils';
 
 export const getCompleteResources = (
@@ -16,26 +17,40 @@ export const dataExplorerPageHandler = (
   partialResources: Resource[] = defaultPartialResources,
   total: number = 300
 ) => {
-  return rest.get(deltaPath(`/resources`), (req, res, ctx) => {
-    if (req.url.searchParams.has('aggregations')) {
-      return res(ctx.status(200), ctx.json(mockAggregationsResult()));
-    }
-    const passedType = req.url.searchParams.get('type');
-    const mockResponse = {
-      '@context': [
-        'https://bluebrain.github.io/nexus/contexts/metadata.json',
-        'https://bluebrain.github.io/nexus/contexts/search.json',
-        'https://bluebrain.github.io/nexus/contexts/search-metadata.json',
-      ],
-      _total: total,
-      _results: passedType
-        ? partialResources.filter(res => res['@type'] === passedType)
-        : partialResources,
-      _next:
-        'https://bbp.epfl.ch/nexus/v1/resources?size=50&sort=@id&after=%5B1687269183553,%22https://bbp.epfl.ch/neurosciencegraph/data/31e22529-2c36-44f0-9158-193eb50526cd%22%5D',
-    };
-    return res(ctx.status(200), ctx.json(mockResponse));
-  });
+  return [
+    rest.get(deltaPath(`/resources`), (req, res, ctx) => {
+      if (req.url.searchParams.has('aggregations')) {
+        return res(ctx.status(200), ctx.json(mockAggregationsResult()));
+      }
+      const passedType = req.url.searchParams.get('type');
+      const mockResponse = {
+        '@context': [
+          'https://bluebrain.github.io/nexus/contexts/metadata.json',
+          'https://bluebrain.github.io/nexus/contexts/search.json',
+          'https://bluebrain.github.io/nexus/contexts/search-metadata.json',
+        ],
+        _total: total,
+        _results: passedType
+          ? partialResources.filter(res => res['@type'] === passedType)
+          : partialResources,
+        _next:
+          'https://bbp.epfl.ch/nexus/v1/resources?size=50&sort=@id&after=%5B1687269183553,%22https://bbp.epfl.ch/neurosciencegraph/data/31e22529-2c36-44f0-9158-193eb50526cd%22%5D',
+      };
+      return res(ctx.status(200), ctx.json(mockResponse));
+    }),
+    rest.post(deltaPath('/multi-fetch/resources'), (req, res, ctx) => {
+      const requestedIds = ((req.body as any)?.resources).map(
+        (res: { id: string }) => res.id
+      );
+      const response: NexusMultiFetchResponse = {
+        format: 'compacted',
+        resources: partialResources
+          .filter(res => requestedIds.includes(res['@id']))
+          .map(r => ({ value: { ...r, ...propertiesOnlyInSource } })),
+      };
+      return res(ctx.status(200), ctx.json(response));
+    }),
+  ];
 };
 
 export const graphAnalyticsTypeHandler = () => {
@@ -147,16 +162,17 @@ export const filterByProjectHandler = (
   });
 };
 
-export const elasticSearchQueryHandler = (ids: string[]) => {
+export const elasticSearchQueryHandler = (resources: Resource[]) => {
   return rest.post(
     deltaPath('/graph-analytics/:org/:project/_search'),
     (req, res, ctx) => {
       const esResponse = {
         hits: {
-          hits: ids.map(id => ({
-            _id: id,
+          hits: resources.map(resource => ({
+            _id: resource['@id'],
             _source: {
-              '@id': id,
+              '@id': resource['@id'],
+              _project: resource._project,
             },
           })),
           max_score: 0,

diff --git a/src/subapps/dataExplorer/DataExplorer.spec.tsx b/src/subapps/dataExplorer/DataExplorer.spec.tsx
@@ -47,7 +47,7 @@ describe('DataExplorer', () => {
   ];
 
   const server = setupServer(
-    dataExplorerPageHandler(undefined, defaultTotalResults),
+    ...dataExplorerPageHandler(undefined, defaultTotalResults),
     sourceResourceHandler(),
     filterByProjectHandler(),
     graphAnalyticsTypeHandler()
@@ -241,7 +241,7 @@ describe('DataExplorer', () => {
   ) => {
     server.use(
       sourceResourceHandler(resources),
-      dataExplorerPageHandler(resources, total)
+      ...dataExplorerPageHandler(resources, total)
     );
 
     const pageInput = await screen.getByRole('listitem', { name: '2' });
@@ -368,9 +368,7 @@ describe('DataExplorer', () => {
     const matchingResources = resources.filter(res =>
       predicate === EXISTS ? res[path] : !res[path]
     );
-    server.use(
-      elasticSearchQueryHandler(matchingResources.map(res => res['@id']))
-    );
+    server.use(elasticSearchQueryHandler(matchingResources));
   };
 
   const getResetProjectButton = async () => {
@@ -531,7 +529,7 @@ describe('DataExplorer', () => {
       mock100Resources.push(getMockResource(`self${i}`, {}));
     }
 
-    server.use(dataExplorerPageHandler(mock100Resources));
+    server.use(...dataExplorerPageHandler(mock100Resources));
 
     const pageSizeChanger = await screen.getByRole('combobox', {
       name: 'Page Size',
@@ -893,6 +891,7 @@ describe('DataExplorer', () => {
     mockElasticSearchHits('author', EXISTS, mockResourcesForPage2);
 
     await selectPath('author');
+
     await selectPredicate(EXISTS);
 
     const selectedPathBefore = await getSelectedValueInMenu(PathMenuLabel);

diff --git a/src/subapps/dataExplorer/DataExplorer.tsx b/src/subapps/dataExplorer/DataExplorer.tsx
@@ -123,6 +123,7 @@ export const columnsFromDataSource = (
     )
     .sort(sortColumns);
 };
+
 const DataExplorer: React.FC<{}> = () => {
   const history = useHistory();
   const [showMetadataColumns, setShowMetadataColumns] = useState(false);

diff --git a/src/subapps/dataExplorer/DataExplorerUtils.tsx b/src/subapps/dataExplorer/DataExplorerUtils.tsx
@@ -7,7 +7,6 @@ import { isString } from 'lodash';
 import PromisePool from '@supercharge/promise-pool';
 import { makeOrgProjectTuple } from '../../shared/molecules/MyDataTable/MyDataTable';
 import { TTypeOperator } from '../../shared/molecules/TypeSelector/types';
-import * as bodybuilder from 'bodybuilder';
 import { useSelector } from 'react-redux';
 import { RootState } from 'shared/store/reducers';
 import { SearchResponse } from 'shared/types/search';
@@ -67,39 +66,14 @@ export const usePaginatedExpandedResources = ({
         }
       );
 
-      // If we failed to fetch the expanded source for some resources, we can use the compact/partial resource as a fallback.
-      const fallbackResources: Resource[] = [];
-      const { results: expandedResources } = await PromisePool.withConcurrency(
-        4
-      )
-        .for(resultWithPartialResources._results)
-        .handleError(async (err, partialResource) => {
-          console.log(
-            `@@error in fetching resource with id: ${partialResource['@id']}`,
-            err
-          );
-          fallbackResources.push(partialResource);
-          return;
-        })
-        .process(async partialResource => {
-          if (partialResource._project) {
-            const { org, project } = makeOrgProjectTuple(
-              partialResource._project
-            );
-
-            return (await nexus.Resource.get(
-              org,
-              project,
-              encodeURIComponent(partialResource['@id']),
-              { annotate: true }
-            )) as Resource;
-          }
-
-          return partialResource;
-        });
+      const expandedResources = await fetchMultipleResources(
+        nexus,
+        apiEndpoint,
+        resultWithPartialResources._results
+      );
       return {
         ...resultWithPartialResources,
-        _results: [...expandedResources, ...fallbackResources],
+        _results: expandedResources,
       };
     },
     onError: error => {
@@ -121,6 +95,62 @@ export const usePaginatedExpandedResources = ({
   });
 };
 
+export type NexusResourceFormats =
+  | 'source'
+  | 'compacted'
+  | 'expanded'
+  | 'n-triples'
+  | 'dot';
+export type NexusMultiFetchResponse = {
+  format: NexusResourceFormats;
+  resources: { value: Resource }[];
+};
+
+type PartialResource = Pick<Resource, '@id' | '_project'>;
+
+export const fetchMultipleResources = async (
+  nexus: ReturnType<typeof useNexusContext>,
+  apiEndpoint: string,
+  partialResources: PartialResource[],
+  orgAndProject?: string
+): Promise<Resource[]> => {
+  const resourceData = partialResources
+    .filter(resource => resource._project)
+    .map(resource => {
+      if (orgAndProject) {
+        return {
+          id: resource['@id'],
+          project: orgAndProject,
+        };
+      }
+
+      const { org, project } = makeOrgProjectTuple(resource._project);
+
+      return {
+        id: resource['@id'],
+        project: `${org}/${project}`,
+      };
+    });
+  console.log(
+    'Going to multi fetch',
+    resourceData.map(r => r.id)
+  );
+  const multipleResources: NexusMultiFetchResponse = await nexus
+    .httpPost({
+      path: `${apiEndpoint}/multi-fetch/resources`,
+      headers: { Accept: 'application/json' },
+      body: JSON.stringify({
+        format: 'compacted',
+        resources: resourceData,
+      }),
+    })
+    .catch(() => {
+      return { format: 'compacted', value: [] };
+    });
+
+  return multipleResources.resources.map(({ value }) => ({ ...value }));
+};
+
 export const useAggregations = (
   bucketName: 'projects' | 'types',
   orgAndProject?: string[]
@@ -190,38 +220,35 @@ const getResultsForPredicateQuery = async (
   pageSize: number,
   offset: number
 ) => {
-  const searchResults: SearchResponse<{ '@id': string }> = await nexus.httpPost(
-    {
-      path: `${apiEndpoint}/graph-analytics/${org}/${project}/_search`,
-      headers: {
-        'Content-Type': 'application/json',
-        Accept: 'application/json',
-      },
-      body: JSON.stringify({
-        from: offset,
-        size: pageSize,
-        ...query,
-      }),
-    }
-  );
+  const searchResults: SearchResponse<{
+    '@id': string;
+    _project: string;
+  }> = await nexus.httpPost({
+    path: `${apiEndpoint}/graph-analytics/${org}/${project}/_search`,
+    headers: {
+      'Content-Type': 'application/json',
+      Accept: 'application/json',
+    },
+    body: JSON.stringify({
+      from: offset,
+      size: pageSize,
+      ...query,
+    }),
+  });
 
-  const { results: matchingResources } = await PromisePool.withConcurrency(4)
-    .for(searchResults.hits.hits)
-    .handleError((err, item) => {
-      console.error(
-        `There was an error retrieving matching resource ${item._source['@id']} for query.`,
-        query,
-        err
-      );
-    })
-    .process(async matchingHit => {
-      return (await nexus.Resource.get(
-        org,
-        project,
-        encodeURIComponent(matchingHit._source['@id']),
-        { annotate: true }
-      )) as Resource;
-    });
+  const resourcesToFetch = searchResults.hits.hits.map(matching => ({
+    '@id': matching._source['@id'],
+    _project: matching._source['_project'],
+  }));
+  console.log(
+    'Requesting matching resources',
+    resourcesToFetch.map(r => r['@id'])
+  );
+  const matchingResources = await fetchMultipleResources(
+    nexus,
+    apiEndpoint,
+    resourcesToFetch
+  );
 
   return {
     _results: matchingResources,