Skip to content

feat(bootstrap): Cache projects in IndexedDB #89139

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Apr 15, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,10 @@
"@sentry/status-page-list": "^0.6.0",
"@sentry/webpack-plugin": "^3.1.1",
"@spotlightjs/spotlight": "^2.0.0-alpha.1",
"@tanstack/react-query": "^5.64.1",
"@tanstack/react-query-devtools": "^5.64.1",
"@tanstack/query-async-storage-persister": "^5.72.1",
"@tanstack/react-query": "^5.72.1",
"@tanstack/react-query-devtools": "^5.72.1",
"@tanstack/react-query-persist-client": "^5.72.1",
"@tanstack/react-virtual": "^3.5.1",
"@types/color": "^3.0.3",
"@types/diff": "5.2.1",
Expand Down Expand Up @@ -117,6 +119,7 @@
"fuse.js": "^6.6.2",
"gettext-parser": "1.3.1",
"gl-matrix": "^3.4.3",
"idb-keyval": "^6.2.1",
"invariant": "^2.2.4",
"jed": "^1.1.0",
"jest-fetch-mock": "^3.0.3",
Expand Down
76 changes: 76 additions & 0 deletions static/app/appQueryClient.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import {createAsyncStoragePersister} from '@tanstack/query-async-storage-persister';
import {persistQueryClient} from '@tanstack/react-query-persist-client';
import {del as removeItem, get as getItem, set as setItem} from 'idb-keyval';

import {SENTRY_RELEASE_VERSION} from 'sentry/constants';
import {DEFAULT_QUERY_CLIENT_CONFIG, QueryClient} from 'sentry/utils/queryClient';

/**
* Named it appQueryClient because we already have a queryClient in sentry/utils/queryClient
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we have this comment explain the difference to try and help avoid people using this one if they don't need it?

* sentry/utils/queryClient is a small wrapper around react-query's functionality for our API.
*
* appQueryClient below is the app's react-query cache and should not be imported directly.
* Instead, use `const queryClient = useQueryClient()`.
* @link https://tanstack.com/query/v5/docs/reference/QueryClient
*/
export const appQueryClient = new QueryClient(DEFAULT_QUERY_CLIENT_CONFIG);
const cacheKey = 'sentry-react-query-cache';

const localStoragePersister = createAsyncStoragePersister({
// We're using indexedDB as our storage provider because projects cache can be large
storage: {getItem, setItem, removeItem},
// Reduce the frequency of writes to indexedDB
throttleTime: 10_000,
// The cache is stored entirely on one key
key: cacheKey,
});

const isProjectsCacheEnabled =
window.indexedDB &&
(window.__initialData?.features as unknown as string[])?.includes(
'organizations:cache-projects-ui'
);

/**
* Attach the persister to the query client
* @link https://tanstack.com/query/latest/docs/framework/react/plugins/persistQueryClient
*/
if (isProjectsCacheEnabled) {
persistQueryClient({
queryClient: appQueryClient,
persister: localStoragePersister,
/**
* Clear cache on release version change
* Locally this does nothing, if you need to clear cache locally you can clear indexdb
*/
buster: SENTRY_RELEASE_VERSION ?? 'local',
dehydrateOptions: {
// Persist a subset of queries to local storage
shouldDehydrateQuery(query) {
// This could be extended later to persist other queries
return (
// Query is not pending or failed
query.state.status === 'success' &&
!query.isStale() &&
Array.isArray(query.queryKey) &&
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this check is unnecessary - the queryKey is always an Array.

// Currently only bootstrap-projects is persisted
query.queryKey[0] === 'bootstrap-projects'
);
},
},
});
}

export function restoreQueryCache() {
if (isProjectsCacheEnabled) {
localStoragePersister.restoreClient();
}
}

export async function clearQueryCache() {
if (isProjectsCacheEnabled) {
// Mark queries as stale so they won't be recached
appQueryClient.invalidateQueries({queryKey: ['bootstrap-projects']});
Comment on lines +72 to +73
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this not only marks them as stale, but also refetches them if they are still active!

to avoid this, we need:

appQueryClient.invalidateQueries({
  queryKey: ['bootstrap-projects'],
  refetchType: 'none',
});

await removeItem(cacheKey);
}
}
2 changes: 2 additions & 0 deletions static/app/bootstrap/initializeApp.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import './legacyTwitterBootstrap';
import './exportGlobals';

import {restoreQueryCache} from 'sentry/appQueryClient';
import type {Config} from 'sentry/types/system';
import {metric} from 'sentry/utils/analytics';

Expand All @@ -11,6 +12,7 @@ import {renderMain} from './renderMain';
import {renderOnDomReady} from './renderOnDomReady';

export function initializeApp(config: Config) {
restoreQueryCache();
commonInitialization(config);
initializeSdk(config);

Expand Down
11 changes: 3 additions & 8 deletions static/app/main.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,16 @@ import {createBrowserRouter, RouterProvider} from 'react-router-dom';
import {wrapCreateBrowserRouterV6} from '@sentry/react';
import {ReactQueryDevtools} from '@tanstack/react-query-devtools';

import {appQueryClient} from 'sentry/appQueryClient';
import {OnboardingContextProvider} from 'sentry/components/onboarding/onboardingContext';
import {ThemeAndStyleProvider} from 'sentry/components/themeAndStyleProvider';
import {USE_REACT_QUERY_DEVTOOL} from 'sentry/constants';
import {routes} from 'sentry/routes';
import {DANGEROUS_SET_REACT_ROUTER_6_HISTORY} from 'sentry/utils/browserHistory';
import {
DEFAULT_QUERY_CLIENT_CONFIG,
QueryClient,
QueryClientProvider,
} from 'sentry/utils/queryClient';
import {QueryClientProvider} from 'sentry/utils/queryClient';

import {buildReactRouter6Routes} from './utils/reactRouter6Compat/router';

const queryClient = new QueryClient(DEFAULT_QUERY_CLIENT_CONFIG);

function buildRouter() {
const sentryCreateBrowserRouter = wrapCreateBrowserRouterV6(createBrowserRouter);
const router = sentryCreateBrowserRouter(buildReactRouter6Routes(routes()));
Expand All @@ -30,7 +25,7 @@ function Main() {
const [router] = useState(buildRouter);

return (
<QueryClientProvider client={queryClient}>
<QueryClientProvider client={appQueryClient}>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we really want to use the PersistQueryClientProvider here if we use persistence. It will handle restoring and subscriptions to the storage, and avoids race conditions between queries that mount while we are still restoring from indexDB.

<ThemeAndStyleProvider>
<OnboardingContextProvider>
<RouterProvider router={router} />
Expand Down
9 changes: 9 additions & 0 deletions static/app/stores/projectsStore.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {createStore} from 'reflux';

import {fetchOrganizationDetails} from 'sentry/actionCreators/organization';
import {Client} from 'sentry/api';
import {clearQueryCache} from 'sentry/appQueryClient';
import type {Team} from 'sentry/types/organization';
import type {Project} from 'sentry/types/project';

Expand Down Expand Up @@ -84,6 +85,7 @@ const storeConfig: ProjectsStoreDefinition = {
this.state = {...this.state, projects: newProjects};

this.trigger(new Set([prevProject.id]));
clearQueryCache();
},

onCreateSuccess(project: Project, orgSlug: string) {
Expand All @@ -94,6 +96,7 @@ const storeConfig: ProjectsStoreDefinition = {

// Reload organization details since we've created a new project
fetchOrganizationDetails(this.api, orgSlug);
clearQueryCache();

this.trigger(new Set([project.id]));
},
Expand All @@ -112,6 +115,7 @@ const storeConfig: ProjectsStoreDefinition = {
this.state = {...this.state, projects: newProjects};

this.trigger(new Set([data.id]));
clearQueryCache();
},

onStatsLoadSuccess(data) {
Expand All @@ -134,6 +138,7 @@ const storeConfig: ProjectsStoreDefinition = {
const newProjects = this.state.projects.filter(p => p.id !== project.id);
this.state = {...this.state, projects: newProjects};
this.trigger(new Set([project.id]));
clearQueryCache();
},

/**
Expand All @@ -151,6 +156,7 @@ const storeConfig: ProjectsStoreDefinition = {

const affectedProjectIds = projects.map(project => project.id);
this.trigger(new Set(affectedProjectIds));
clearQueryCache();
},

onRemoveTeam(teamSlug: string, projectSlug: string) {
Expand All @@ -162,6 +168,7 @@ const storeConfig: ProjectsStoreDefinition = {

this.removeTeamFromProject(teamSlug, project);
this.trigger(new Set([project.id]));
clearQueryCache();
},

onAddTeam(team: Team, projectSlug: string) {
Expand All @@ -179,6 +186,7 @@ const storeConfig: ProjectsStoreDefinition = {
this.state = {...this.state, projects: newProjects};

this.trigger(new Set([project.id]));
clearQueryCache();
},

// Internal method, does not trigger
Expand All @@ -189,6 +197,7 @@ const storeConfig: ProjectsStoreDefinition = {
p.id === project.id ? newProject : p
);
this.state = {...this.state, projects: newProjects};
clearQueryCache();
},

isLoading() {
Expand Down
62 changes: 44 additions & 18 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -3624,29 +3624,50 @@
dependencies:
"@typescript-eslint/utils" "^8.18.1"

"@tanstack/query-core@5.64.1":
version "5.64.1"
resolved "https://registry.yarnpkg.com/@tanstack/query-core/-/query-core-5.64.1.tgz#d56e26b3e29fc68a89d140f1fd92900bc8f3fc86"
integrity sha512-978Wx4Wl4UJZbmvU/rkaM9cQtXXrbhK0lsz/UZhYIbyKYA8E4LdomTwyh2GHZ4oU0BKKoDH4YlKk2VscCUgNmg==
"@tanstack/query-async-storage-persister@^5.72.1":
version "5.72.1"
resolved "https://registry.yarnpkg.com/@tanstack/query-async-storage-persister/-/query-async-storage-persister-5.72.1.tgz#2d4c230efbab7670fa03d483196232ff34a465c1"
integrity sha512-5l6NuV1GMz0Y2qGJ9B1JYgpDmLb92fs4c56/cxgNACwwi+bWyzEySQZocyXxiEW4ZG8fSQ/la57nAAw+OSsp0w==
dependencies:
"@tanstack/query-persist-client-core" "5.72.1"

"@tanstack/query-core@5.72.1":
version "5.72.1"
resolved "https://registry.yarnpkg.com/@tanstack/query-core/-/query-core-5.72.1.tgz#c46e3e5eecf1628c478965c0540cae98838e24fe"
integrity sha512-nOu0EEkZuJ0BZnYgeaEfo44+psq1jBO7/zp3KudixD4dvgOVerrhAhDEKsWx2N7MxB59mjO4r0ddP/VqWGPK+Q==

"@tanstack/query-devtools@5.72.1":
version "5.72.1"
resolved "https://registry.yarnpkg.com/@tanstack/query-devtools/-/query-devtools-5.72.1.tgz#3ef35de317a55a688072a13cb628931ddec0f097"
integrity sha512-D0vEoQaiVq9ayCqvvxA9XkDq7TIesyPpvgP69arRtt5FQF6n/Hrta4SlkfXC4m9BCvFLlhLDcKGYa2eMQ4ZIIA==

"@tanstack/query-persist-client-core@5.72.1":
version "5.72.1"
resolved "https://registry.yarnpkg.com/@tanstack/query-persist-client-core/-/query-persist-client-core-5.72.1.tgz#65ba35375677723fc51d28295cdaddfb20d65855"
integrity sha512-sceggk1lnJVNlvAUFoWZgaC1SVPgxyhoShHGczlAZHZIn7uLkEJcICXUQlvTSeJn4nr8XdFxBSW8ie/6YPmr6A==
dependencies:
"@tanstack/query-core" "5.72.1"

"@tanstack/query-devtools@5.62.16":
version "5.62.16"
resolved "https://registry.yarnpkg.com/@tanstack/query-devtools/-/query-devtools-5.62.16.tgz#a4b71c6b5bbf7575861437ef9a9f232333569255"
integrity sha512-3ff6UBJr0H3nIhfLSl9911rvKqXf0u4B58jl0uYdDWLqPk9pCvYIbxC35cGxK2+8INl4IaFVUHb/IdgWrNkg3Q==
"@tanstack/react-query-devtools@^5.72.1":
version "5.72.1"
resolved "https://registry.yarnpkg.com/@tanstack/react-query-devtools/-/react-query-devtools-5.72.1.tgz#9822a0799125dc67ff1bd846cf01a524b12fa998"
integrity sha512-ckNRgABst3MLjpM2nD/CzQToCiaT3jb3Xhtf+GP/0/9ij9SPT/SC+lc3wUDSkT0OupnHobBBF5E1/Xp6B+XZLg==
dependencies:
"@tanstack/query-devtools" "5.72.1"

"@tanstack/react-query-devtools@^5.64.1":
version "5.64.1"
resolved "https://registry.yarnpkg.com/@tanstack/react-query-devtools/-/react-query-devtools-5.64.1.tgz#e3ccf56b1b30453a4baed2b05d4fa717885ddf97"
integrity sha512-8ajcGE3wXYlb4KuJnkFYkJwJKc/qmPNTpQD7YTgLRMBPTGGp1xk7VMzxL87DoXuweO8luplUUblJJ3noVs/luQ==
"@tanstack/react-query-persist-client@^5.72.1":
version "5.72.1"
resolved "https://registry.yarnpkg.com/@tanstack/react-query-persist-client/-/react-query-persist-client-5.72.1.tgz#9564de508fb25ef1d63430b6ad3cea3f9727f28a"
integrity sha512-VJ1wqgfDblyjjxdrm910bQflHNC6IVcqKMN6/DIhC7AGw8jeSKWDc7AVZW4d9L57tblyYdQfUfwTOBfpGk646Q==
dependencies:
"@tanstack/query-devtools" "5.62.16"
"@tanstack/query-persist-client-core" "5.72.1"

"@tanstack/react-query@^5.64.1":
version "5.64.1"
resolved "https://registry.yarnpkg.com/@tanstack/react-query/-/react-query-5.64.1.tgz#46b4182f5b045299e4be8d0a91c549ac5dc0a20c"
integrity sha512-vW5ggHpIO2Yjj44b4sB+Fd3cdnlMJppXRBJkEHvld6FXh3j5dwWJoQo7mGtKI2RbSFyiyu/PhGAy0+Vv5ev9Eg==
"@tanstack/react-query@^5.72.1":
version "5.72.1"
resolved "https://registry.yarnpkg.com/@tanstack/react-query/-/react-query-5.72.1.tgz#7c8bcb94bfc64e57377bfa3c5b2a764c4ae83229"
integrity sha512-4UEMyRx54xj144D2nDvDIMiXSG5BrqyCJrmyNoGbymNS+VWODcBDFrmRk9p2fe12UGZ4JtKPTNuW2Jg0aisUgQ==
dependencies:
"@tanstack/query-core" "5.64.1"
"@tanstack/query-core" "5.72.1"

"@tanstack/react-virtual@^3.5.1":
version "3.5.1"
Expand Down Expand Up @@ -7719,6 +7740,11 @@ icss-utils@^5.0.0, icss-utils@^5.1.0:
resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae"
integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==

idb-keyval@^6.2.1:
version "6.2.1"
resolved "https://registry.yarnpkg.com/idb-keyval/-/idb-keyval-6.2.1.tgz#94516d625346d16f56f3b33855da11bfded2db33"
integrity sha512-8Sb3veuYCyrZL+VBt9LJfZjLUPWVvqn8tG28VqYNFCo43KHcKuq+b4EiXGeuaLAQWL2YmyDgMp2aSpH9JHsEQg==

ieee754@^1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
Expand Down
Loading