From ec20b2d4c4e744963d8d8f73cf388da28bc38330 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Mon, 3 Mar 2025 14:41:40 +0100 Subject: [PATCH] [scout] enable ES authc debug logs on ES start (#212866) ## Summary In #211055 we enabled Elasticsearch Authc debug logs, but we use the different entry function to start servers on CI. This PR moves the call into `runElasticsearch` so that logs are enabled for every Scout script (start-server or run-tests) (cherry picked from commit 043d780b2afbfc722c6f849bc7dfdb1098550447) --- .../kbn-scout/src/common/services/clients.ts | 9 ++---- .../kbn-scout/src/common/utils/index.ts | 14 +++++++++- .../src/servers/run_elasticsearch.ts | 28 ++++++++++++++++++- .../kbn-scout/src/servers/start_servers.ts | 10 +------ 4 files changed, 44 insertions(+), 17 deletions(-) diff --git a/src/platform/packages/shared/kbn-scout/src/common/services/clients.ts b/src/platform/packages/shared/kbn-scout/src/common/services/clients.ts index ed5efd8e77786..20018ec9584e4 100644 --- a/src/platform/packages/shared/kbn-scout/src/common/services/clients.ts +++ b/src/platform/packages/shared/kbn-scout/src/common/services/clients.ts @@ -8,7 +8,6 @@ */ import { createEsClientForTesting, KbnClient } from '@kbn/test'; -import { ToolingLog } from '@kbn/tooling-log'; import { ScoutLogger } from './logger'; import { ScoutTestConfig, EsClient } from '../../types'; @@ -17,7 +16,7 @@ interface ClientOptions { url: string; username: string; password: string; - log: ScoutLogger | ToolingLog; + log: ScoutLogger; } function createClientUrlWithAuth({ serviceName, url, username, password, log }: ClientOptions) { @@ -25,9 +24,7 @@ function createClientUrlWithAuth({ serviceName, url, username, password, log }: clientUrl.username = username; clientUrl.password = password; - if (log instanceof ScoutLogger) { - log.serviceLoaded(`${serviceName}Client`); - } + log.serviceLoaded(`${serviceName}Client`); return clientUrl.toString(); } @@ -35,7 +32,7 @@ function createClientUrlWithAuth({ serviceName, url, username, password, log }: let esClientInstance: EsClient | null = null; let kbnClientInstance: KbnClient | null = null; -export function getEsClient(config: ScoutTestConfig, log: ScoutLogger | ToolingLog) { +export function getEsClient(config: ScoutTestConfig, log: ScoutLogger) { if (!esClientInstance) { const { username, password } = config.auth; const elasticsearchUrl = createClientUrlWithAuth({ diff --git a/src/platform/packages/shared/kbn-scout/src/common/utils/index.ts b/src/platform/packages/shared/kbn-scout/src/common/utils/index.ts index ab112412b64a1..5829f7eeaf2e2 100644 --- a/src/platform/packages/shared/kbn-scout/src/common/utils/index.ts +++ b/src/platform/packages/shared/kbn-scout/src/common/utils/index.ts @@ -14,7 +14,19 @@ export async function silence(log: ToolingLog, milliseconds: number) { await Rx.firstValueFrom( log.getWritten$().pipe( Rx.startWith(null), - Rx.switchMap(() => Rx.timer(milliseconds)) + Rx.switchMap((message) => { + if ( + // TODO: remove workaround to ignore ES authc debug logs for stateful run + message?.args[0]?.includes( + 'Authentication of [kibana_system] using realm [reserved/reserved]' + ) || + message?.args[0]?.includes('realm [reserved] authenticated user [kibana_system]') + ) { + return Rx.of(null); + } else { + return Rx.timer(milliseconds); + } + }) ) ); } diff --git a/src/platform/packages/shared/kbn-scout/src/servers/run_elasticsearch.ts b/src/platform/packages/shared/kbn-scout/src/servers/run_elasticsearch.ts index 24c8a49da2d9a..1179cd090769c 100644 --- a/src/platform/packages/shared/kbn-scout/src/servers/run_elasticsearch.ts +++ b/src/platform/packages/shared/kbn-scout/src/servers/run_elasticsearch.ts @@ -13,7 +13,12 @@ import type { ToolingLog } from '@kbn/tooling-log'; import { REPO_ROOT } from '@kbn/repo-info'; import type { ArtifactLicense, ServerlessProjectType } from '@kbn/es'; import { isServerlessProjectType } from '@kbn/es/src/utils'; -import { createTestEsCluster, esTestConfig, cleanupElasticsearch } from '@kbn/test'; +import { + createTestEsCluster, + esTestConfig, + cleanupElasticsearch, + createEsClientForTesting, +} from '@kbn/test'; import { Config } from '../config'; interface RunElasticsearchOptions { @@ -81,6 +86,27 @@ export async function runElasticsearch( logsDir, config, }); + + // TODO: Remove this once we find out why SAML callback randomly fails with 401 + log.info('Enable authc debug logs for ES'); + const clientUrl = new URL( + Url.format({ + protocol: options.config.get('servers.elasticsearch.protocol'), + hostname: options.config.get('servers.elasticsearch.hostname'), + port: options.config.get('servers.elasticsearch.port'), + }) + ); + clientUrl.username = options.config.get('servers.kibana.username'); + clientUrl.password = options.config.get('servers.kibana.password'); + const esClient = createEsClientForTesting({ + esUrl: clientUrl.toString(), + }); + await esClient.cluster.putSettings({ + persistent: { + 'logger.org.elasticsearch.xpack.security.authc': 'debug', + }, + }); + return async () => { await cleanupElasticsearch(node, config.serverless, logsDir, log); }; diff --git a/src/platform/packages/shared/kbn-scout/src/servers/start_servers.ts b/src/platform/packages/shared/kbn-scout/src/servers/start_servers.ts index 876f60f18f02e..32eb2030c978d 100644 --- a/src/platform/packages/shared/kbn-scout/src/servers/start_servers.ts +++ b/src/platform/packages/shared/kbn-scout/src/servers/start_servers.ts @@ -16,7 +16,7 @@ import { runElasticsearch } from './run_elasticsearch'; import { getExtraKbnOpts, runKibanaServer } from './run_kibana_server'; import { StartServerOptions } from './flags'; import { loadServersConfig } from '../config'; -import { getEsClient, silence } from '../common'; +import { silence } from '../common'; export async function startServers(log: ToolingLog, options: StartServerOptions) { const runStartTime = Date.now(); @@ -32,14 +32,6 @@ export async function startServers(log: ToolingLog, options: StartServerOptions) logsDir: options.logsDir, }); - log.info('Enable authc debug logs for ES'); - const client = getEsClient(config.getScoutTestConfig(), log); - await client.cluster.putSettings({ - persistent: { - 'logger.org.elasticsearch.xpack.security.authc': 'debug', - }, - }); - await runKibanaServer({ procs, config,