[scout] enable ES authc debug logs on ES start (#212866)

## Summary

In #211055 we enabled Elasticsearch Authc debug logs, but we use the
different entry function to start servers on CI. This PR moves the call
into `runElasticsearch` so that logs are enabled for every Scout script
(start-server or run-tests)

(cherry picked from commit 043d780)

src/platform/packages/shared/kbn-scout/src/common/services/clients.ts

@@ -8,7 +8,6 @@
  */
 
 import { createEsClientForTesting, KbnClient } from '@kbn/test';
-import { ToolingLog } from '@kbn/tooling-log';
 import { ScoutLogger } from './logger';
 import { ScoutTestConfig, EsClient } from '../../types';
 
@@ -17,25 +16,23 @@ interface ClientOptions {
   url: string;
   username: string;
   password: string;
-  log: ScoutLogger | ToolingLog;
+  log: ScoutLogger;
 }
 
 function createClientUrlWithAuth({ serviceName, url, username, password, log }: ClientOptions) {
   const clientUrl = new URL(url);
   clientUrl.username = username;
   clientUrl.password = password;
 
-  if (log instanceof ScoutLogger) {
-    log.serviceLoaded(`${serviceName}Client`);
-  }
+  log.serviceLoaded(`${serviceName}Client`);
 
   return clientUrl.toString();
 }
 
 let esClientInstance: EsClient | null = null;
 let kbnClientInstance: KbnClient | null = null;
 
-export function getEsClient(config: ScoutTestConfig, log: ScoutLogger | ToolingLog) {
+export function getEsClient(config: ScoutTestConfig, log: ScoutLogger) {
   if (!esClientInstance) {
     const { username, password } = config.auth;
     const elasticsearchUrl = createClientUrlWithAuth({

src/platform/packages/shared/kbn-scout/src/common/utils/index.ts

@@ -14,7 +14,19 @@ export async function silence(log: ToolingLog, milliseconds: number) {
   await Rx.firstValueFrom(
     log.getWritten$().pipe(
       Rx.startWith(null),
-      Rx.switchMap(() => Rx.timer(milliseconds))
+      Rx.switchMap((message) => {
+        if (
+          // TODO: remove workaround to ignore ES authc debug logs for stateful run
+          message?.args[0]?.includes(
+            'Authentication of [kibana_system] using realm [reserved/reserved]'
+          ) ||
+          message?.args[0]?.includes('realm [reserved] authenticated user [kibana_system]')
+        ) {
+          return Rx.of(null);
+        } else {
+          return Rx.timer(milliseconds);
+        }
+      })
     )
   );
 }

src/platform/packages/shared/kbn-scout/src/servers/run_elasticsearch.ts

@@ -13,7 +13,12 @@ import type { ToolingLog } from '@kbn/tooling-log';
 import { REPO_ROOT } from '@kbn/repo-info';
 import type { ArtifactLicense, ServerlessProjectType } from '@kbn/es';
 import { isServerlessProjectType } from '@kbn/es/src/utils';
-import { createTestEsCluster, esTestConfig, cleanupElasticsearch } from '@kbn/test';
+import {
+  createTestEsCluster,
+  esTestConfig,
+  cleanupElasticsearch,
+  createEsClientForTesting,
+} from '@kbn/test';
 import { Config } from '../config';
 
 interface RunElasticsearchOptions {
@@ -81,6 +86,27 @@ export async function runElasticsearch(
     logsDir,
     config,
   });
+
+  // TODO: Remove this once we find out why SAML callback randomly fails with 401
+  log.info('Enable authc debug logs for ES');
+  const clientUrl = new URL(
+    Url.format({
+      protocol: options.config.get('servers.elasticsearch.protocol'),
+      hostname: options.config.get('servers.elasticsearch.hostname'),
+      port: options.config.get('servers.elasticsearch.port'),
+    })
+  );
+  clientUrl.username = options.config.get('servers.kibana.username');
+  clientUrl.password = options.config.get('servers.kibana.password');
+  const esClient = createEsClientForTesting({
+    esUrl: clientUrl.toString(),
+  });
+  await esClient.cluster.putSettings({
+    persistent: {
+      'logger.org.elasticsearch.xpack.security.authc': 'debug',
+    },
+  });
+
   return async () => {
     await cleanupElasticsearch(node, config.serverless, logsDir, log);
   };

src/platform/packages/shared/kbn-scout/src/servers/start_servers.ts

@@ -16,7 +16,7 @@ import { runElasticsearch } from './run_elasticsearch';
 import { getExtraKbnOpts, runKibanaServer } from './run_kibana_server';
 import { StartServerOptions } from './flags';
 import { loadServersConfig } from '../config';
-import { getEsClient, silence } from '../common';
+import { silence } from '../common';
 
 export async function startServers(log: ToolingLog, options: StartServerOptions) {
   const runStartTime = Date.now();
@@ -32,14 +32,6 @@ export async function startServers(log: ToolingLog, options: StartServerOptions)
       logsDir: options.logsDir,
     });
 
-    log.info('Enable authc debug logs for ES');
-    const client = getEsClient(config.getScoutTestConfig(), log);
-    await client.cluster.putSettings({
-      persistent: {
-        'logger.org.elasticsearch.xpack.security.authc': 'debug',
-      },
-    });
-
     await runKibanaServer({
       procs,
       config,