Skip to content

Commit dca43ba

Browse files
jeswankejeswanke
authored
ACM-15850-Polling-from-console-pods-adding-too-much-load-on-search (stolostron#4129)
* ACM-15850-Polling-from-console-pods-adding-too-much-load-on-search Signed-off-by: John Swanke <jswanke@redhat.com> * fix local-cluster Signed-off-by: John Swanke <jswanke@redhat.com> * backup Signed-off-by: John Swanke <jswanke@redhat.com> * backup Signed-off-by: John Swanke <jswanke@redhat.com> * backup Signed-off-by: John Swanke <jswanke@redhat.com> * fix test Signed-off-by: John Swanke <jswanke@redhat.com> * tweak Signed-off-by: John Swanke <jswanke@redhat.com> * tweak Signed-off-by: John Swanke <jswanke@redhat.com> * oops Signed-off-by: John Swanke <jswanke@redhat.com> * test coverage Signed-off-by: John Swanke <jswanke@redhat.com> * coverage Signed-off-by: John Swanke <jswanke@redhat.com> * fix issues Signed-off-by: John Swanke <jswanke@redhat.com> --------- Signed-off-by: John Swanke <jswanke@redhat.com>
1 parent ef6bab0 commit dca43ba

14 files changed

+917
-791
lines changed

‎backend/config/APP_ARGO_SEARCH_RESULT_LIMIT

-1
This file was deleted.

‎backend/config/APP_OCP_SEARCH_RESULT_LIMIT

-1
This file was deleted.

‎backend/config/APP_SEARCH_INTERVAL

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
60000

‎backend/config/APP_SEARCH_LIMIT

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
5000

‎backend/src/lib/clusters.ts

-17
This file was deleted.

‎backend/src/lib/config.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ export async function loadConfigSettings(): Promise<void> {
4343
}
4444
}
4545
for (const key in settings) {
46-
if (key.startsWith('LOG_')) {
46+
if (key.startsWith('LOG_') || key.startsWith('APP_SEARCH_')) {
4747
process.env[key] = settings[key]
4848
} else if (key === 'globalSearchFeatureFlag') {
4949
// Global search tech-preview requires feature flag toggle (2.11)

‎backend/src/lib/search.ts

+10-70
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ import { RequestOptions, request } from 'https'
44
import { URL } from 'url'
55
import { getMultiClusterHub } from '../lib/multi-cluster-hub'
66
import { getNamespace, getServiceAccountToken, getServiceCACertificate } from '../lib/serviceAccountToken'
7-
import { IResource } from '../resources/resource'
87
import { logger } from './logger'
8+
import { IQuery } from '../routes/aggregators/applications'
99

1010
export type ISearchResult = {
1111
data: {
@@ -57,74 +57,13 @@ export async function getSearchOptions(headers: OutgoingHttpHeaders): Promise<Re
5757
return options
5858
}
5959

60-
// search api does not provide paging but we want to break up our searches so as not to overtax the search api by querying for all apps at once
61-
// this is pseudo-paging: we grab all apps that begin with a letter, that way we don't have overlapping results
62-
// we don't want to do this letter by letter because that would take 36 searches
63-
// so we create 6 groupings of letters and we try to make each group search return about the same number of apps
64-
export const pagedSearchQueries: string[][] = [
65-
['a*', 'i*', 'n*'],
66-
['e*', 'r*', 'o*'],
67-
['s*', 't*', 'u*', 'l*', 'm*', 'c*'],
68-
['d*', 'b*', 'g*', '0*', '1*', '2*', '3*', '4*'],
69-
['h*', 'p*', 'k*', 'y*', 'v*', 'z*', 'w*', 'f*'],
70-
['j*', 'q*', 'x*', '5*', '6*', '7*', '8*', '9*'],
71-
]
72-
73-
export async function getPagedSearchResources(
74-
query: {
75-
operationName: string
76-
variables: { input: { filters: { property: string; values: string[] }[]; limit: number }[] }
77-
query: string
78-
},
79-
usePagedQuery: boolean,
80-
kind: string,
81-
pass: number
82-
) {
60+
export async function getSearchResults(query: IQuery) {
8361
const options = await getServiceAccountOptions()
84-
let resources: IResource[] = []
85-
for (let i = 0; i < pagedSearchQueries.length; ) {
86-
const _query = structuredClone(query)
87-
// should we limit the results by groupings of apps that
88-
// begin with certain letters?
89-
if (usePagedQuery) {
90-
const values = pagedSearchQueries[i]
91-
_query.variables.input[0].filters.push({
92-
property: 'name',
93-
values,
94-
})
95-
}
96-
let results: ISearchResult
97-
try {
98-
results = await getSearchResults(options, JSON.stringify(_query), kind, pass)
99-
} catch (e) {
100-
logger.error(`getPagedSearchResources ${kind} ${e}`)
101-
continue
102-
}
103-
const items = (results.data?.searchResult?.[0]?.items || []) as IResource[]
104-
resources = resources.concat(items)
105-
if (process.env.NODE_ENV !== 'test') {
106-
let timeout = 10000
107-
if (items.length < 1000) timeout = 2000
108-
await new Promise((r) => setTimeout(r, timeout))
109-
}
110-
if (!usePagedQuery) break
111-
i++
112-
}
113-
return resources
114-
}
115-
116-
export function getSearchResults(
117-
options: string | RequestOptions | URL,
118-
variables: string,
119-
kind: string,
120-
pass: number
121-
) {
122-
// if acm/mce are starting up, increase the timeout in case search hasn't started yet
123-
const requestTimeout = (pass <= 2 ? 10 : 2) * 60 * 1000
62+
const requestTimeout = 2 * 60 * 1000
12463
return new Promise<ISearchResult>((resolve, reject) => {
12564
let body = ''
12665
const id = setTimeout(() => {
127-
logger.error(`getSearchResults ${kind} request timeout`)
66+
logger.error(`getSearchResults request timeout`)
12867
reject(Error('request timeout'))
12968
}, requestTimeout)
13069
const req = request(options, (res) => {
@@ -136,14 +75,14 @@ export function getSearchResults(
13675
const result = JSON.parse(body) as ISearchResult
13776
const message = typeof result === 'string' ? result : result.message
13877
if (message) {
139-
logger.error(`getSearchResults ${kind} return error ${message}`)
78+
logger.error(`getSearchResults return error ${message}`)
14079
reject(Error(result.message))
14180
}
14281
resolve(result)
14382
} catch (e) {
14483
// search might be overwhelmed
14584
// pause before next request
146-
logger.error(`getSearchResults ${kind} parse error ${e} ${body}`)
85+
logger.error(`getSearchResults parse error ${e} ${body}`)
14786
setTimeout(() => {
14887
reject(Error(body))
14988
}, requestTimeout)
@@ -152,10 +91,10 @@ export function getSearchResults(
15291
})
15392
})
15493
req.on('error', (e) => {
155-
logger.error(`getSearchResults ${kind} request error ${e.message}`)
94+
logger.error(`getSearchResults request error ${e.message}`)
15695
reject(e)
15796
})
158-
req.write(variables)
97+
req.write(JSON.stringify(query))
15998
req.end()
16099
})
161100
}
@@ -206,7 +145,8 @@ export async function pingSearchAPI() {
206145
reject(new Error('no data'))
207146
}
208147
} catch (e) {
209-
reject(new Error(new String(e).valueOf()))
148+
logger.error(`pingSearchAPI parse error ${e} ${body}`)
149+
reject(new Error(String(e).valueOf()))
210150
}
211151
clearTimeout(id)
212152
})

‎backend/src/routes/aggregator.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import {
1212
import { requestAggregatedStatuses } from './aggregators/statuses'
1313

1414
export function startAggregating(): void {
15-
startAggregatingApplications()
15+
void startAggregatingApplications()
1616
}
1717

1818
export function stopAggregating(): void {

0 commit comments

Comments
 (0)