From f38f31a3097bb3f92c2dc374809b2cfdf27673e6 Mon Sep 17 00:00:00 2001 From: Mykola Harmash Date: Tue, 4 Mar 2025 14:14:01 +0100 Subject: [PATCH] [Onboarding][OTel Host] Use MOTel collector on Serverless --- .../kbn-es/src/serverless_resources/users | 2 + .../src/serverless_resources/users_roles | 1 + .../plugins/apm/server/plugin.ts | 2 +- .../observability/plugins/apm/server/types.ts | 1 + .../observability_onboarding/kibana.jsonc | 5 +- .../quickstart_flows/otel_logs/index.tsx | 40 ++++++------ .../otel_logs/use_encoded_api_key.ts | 65 +++++++++++++++++++ .../use_ingest_endpoint_url.test.tsx | 57 ++++++++++++++++ .../otel_logs/use_ingest_endpoint_url.ts | 34 ++++++++++ .../server/routes/logs/route.ts | 4 ++ .../observability_onboarding/server/types.ts | 2 + 11 files changed, 191 insertions(+), 22 deletions(-) create mode 100644 x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_encoded_api_key.ts create mode 100644 x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_ingest_endpoint_url.test.tsx create mode 100644 x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_ingest_endpoint_url.ts diff --git a/src/platform/packages/shared/kbn-es/src/serverless_resources/users b/src/platform/packages/shared/kbn-es/src/serverless_resources/users index 4ed7023a64b1f..fa87790c777e8 100644 --- a/src/platform/packages/shared/kbn-es/src/serverless_resources/users +++ b/src/platform/packages/shared/kbn-es/src/serverless_resources/users @@ -12,3 +12,5 @@ platform_engineer:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW endpoint_operations_analyst:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW endpoint_policy_manager:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW +elastic_viewer:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW + diff --git a/src/platform/packages/shared/kbn-es/src/serverless_resources/users_roles b/src/platform/packages/shared/kbn-es/src/serverless_resources/users_roles index a4d6c1987ce6d..9f72a87b6e637 100644 --- a/src/platform/packages/shared/kbn-es/src/serverless_resources/users_roles +++ b/src/platform/packages/shared/kbn-es/src/serverless_resources/users_roles @@ -10,3 +10,4 @@ detections_admin:detections_admin platform_engineer:platform_engineer endpoint_operations_analyst:endpoint_operations_analyst endpoint_policy_manager:endpoint_policy_manager +viewer:elastic_viewer diff --git a/x-pack/solutions/observability/plugins/apm/server/plugin.ts b/x-pack/solutions/observability/plugins/apm/server/plugin.ts index e06a8afc8510c..062e064749d57 100644 --- a/x-pack/solutions/observability/plugins/apm/server/plugin.ts +++ b/x-pack/solutions/observability/plugins/apm/server/plugin.ts @@ -245,7 +245,7 @@ export class APMPlugin }, }); - return { config$ }; + return { config$, config: currentConfig }; } public start(core: CoreStart, plugins: APMPluginStartDependencies) { diff --git a/x-pack/solutions/observability/plugins/apm/server/types.ts b/x-pack/solutions/observability/plugins/apm/server/types.ts index 5de9e10375308..f38b00843ced1 100644 --- a/x-pack/solutions/observability/plugins/apm/server/types.ts +++ b/x-pack/solutions/observability/plugins/apm/server/types.ts @@ -64,6 +64,7 @@ import type { APMConfig } from '.'; export interface APMPluginSetup { config$: Observable; + config: APMConfig; } export interface APMPluginSetupDependencies { diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/kibana.jsonc b/x-pack/solutions/observability/plugins/observability_onboarding/kibana.jsonc index 361451b1ce009..71a81d19a3bbb 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/kibana.jsonc +++ b/x-pack/solutions/observability/plugins/observability_onboarding/kibana.jsonc @@ -22,7 +22,8 @@ "discover", "share", "fleet", - "customIntegrations" + "customIntegrations", + "apm" ], "optionalPlugins": [ "cloud", @@ -35,4 +36,4 @@ "common" ] } -} \ No newline at end of file +} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx index 56a1b213d3fb1..a24bc405fb5e5 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx @@ -32,6 +32,8 @@ import { useFetcher } from '../../../hooks/use_fetcher'; import { MultiIntegrationInstallBanner } from './multi_integration_install_banner'; import { EmptyPrompt } from '../shared/empty_prompt'; import { FeedbackButtons } from '../shared/feedback_buttons'; +import { useEncodedApiKey } from './use_encoded_api_key'; +import { useIngestEndpointUrl } from './use_ingest_endpoint_url'; const HOST_COMMAND = i18n.translate( 'xpack.observability_onboarding.otelLogsPanel.p.runTheCommandOnYourHostLabel', @@ -43,25 +45,19 @@ const HOST_COMMAND = i18n.translate( export const OtelLogsPanel: React.FC = () => { const { - data: apiKeyData, - error, - refetch, - } = useFetcher( - (callApi) => { - return callApi('POST /internal/observability_onboarding/otel/api_key', {}); - }, - [], - { showToastOnError: false } - ); + services: { share, http }, + } = useKibana(); + + const { encodedApiKey, error } = useEncodedApiKey(); const { data: setup } = useFetcher((callApi) => { return callApi('GET /internal/observability_onboarding/logs/setup/environment'); }, []); - const { - services: { share, http }, - } = useKibana(); - + const ingestEndpointUrl = useIngestEndpointUrl({ + elasticsearchUrl: setup?.elasticsearchUrl[0], + managedServiceUrl: setup?.managedServiceUrl, + }); const AGENT_CDN_BASE_URL = 'artifacts.elastic.co/downloads/beats/elastic-agent'; const agentVersion = setup?.elasticAgentVersionInfo.agentVersion ?? ''; const urlEncodedAgentVersion = encodeURIComponent(agentVersion); @@ -89,7 +85,7 @@ export const OtelLogsPanel: React.FC = () => { curl --output elastic-distro-${agentVersion}-linux-$arch.tar.gz --url https://${AGENT_CDN_BASE_URL}/elastic-agent-${urlEncodedAgentVersion}-linux-$arch.tar.gz --proto '=https' --tlsv1.2 -fOL && mkdir -p elastic-distro-${agentVersion}-linux-$arch && tar -xvf elastic-distro-${agentVersion}-linux-$arch.tar.gz -C "elastic-distro-${agentVersion}-linux-$arch" --strip-components=1 && cd elastic-distro-${agentVersion}-linux-$arch -rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mkdir -p ./data/otelcol && sed -i 's#\\\${env:STORAGE_DIR}#'"$PWD"/data/otelcol'#g' ./otel.yml && sed -i 's#\\\${env:ELASTIC_ENDPOINT}#${setup?.elasticsearchUrl}#g' ./otel.yml && sed -i 's/\\\${env:ELASTIC_API_KEY}/${apiKeyData?.apiKeyEncoded}/g' ./otel.yml`, +rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mkdir -p ./data/otelcol && sed -i 's#\\\${env:STORAGE_DIR}#'"$PWD"/data/otelcol'#g' ./otel.yml && sed -i 's#\\\${env:ELASTIC_ENDPOINT}#${ingestEndpointUrl}#g' ./otel.yml && sed -i 's/\\\${env:ELASTIC_API_KEY}/${encodedApiKey}/g' ./otel.yml`, start: 'sudo ./otelcol --config otel.yml', type: 'copy', }, @@ -101,7 +97,7 @@ rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mk curl --output elastic-distro-${agentVersion}-darwin-$arch.tar.gz --url https://${AGENT_CDN_BASE_URL}/elastic-agent-${urlEncodedAgentVersion}-darwin-$arch.tar.gz --proto '=https' --tlsv1.2 -fOL && mkdir -p "elastic-distro-${agentVersion}-darwin-$arch" && tar -xvf elastic-distro-${agentVersion}-darwin-$arch.tar.gz -C "elastic-distro-${agentVersion}-darwin-$arch" --strip-components=1 && cd elastic-distro-${agentVersion}-darwin-$arch -rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mkdir -p ./data/otelcol && sed -i '' 's#\\\${env:STORAGE_DIR}#'"$PWD"/data/otelcol'#g' ./otel.yml && sed -i '' 's#\\\${env:ELASTIC_ENDPOINT}#${setup?.elasticsearchUrl}#g' ./otel.yml && sed -i '' 's/\\\${env:ELASTIC_API_KEY}/${apiKeyData?.apiKeyEncoded}/g' ./otel.yml`, +rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mkdir -p ./data/otelcol && sed -i '' 's#\\\${env:STORAGE_DIR}#'"$PWD"/data/otelcol'#g' ./otel.yml && sed -i '' 's#\\\${env:ELASTIC_ENDPOINT}#${ingestEndpointUrl}#g' ./otel.yml && sed -i '' 's/\\\${env:ELASTIC_API_KEY}/${encodedApiKey}/g' ./otel.yml`, start: './otelcol --config otel.yml', type: 'copy', }, @@ -112,7 +108,13 @@ rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mk const selectedContent = installTabContents.find((tab) => tab.id === selectedTab)!; if (error) { - return ; + return ( + window.location.reload()} + /> + ); } return ( @@ -144,9 +146,9 @@ rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mk }} /> - {(!setup || !apiKeyData) && } + {(!setup || !encodedApiKey) && } - {setup && apiKeyData && ( + {setup && encodedApiKey && ( <>

{selectedContent.firstStepTitle}

diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_encoded_api_key.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_encoded_api_key.ts new file mode 100644 index 0000000000000..46c88ed3825ff --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_encoded_api_key.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useEffect, useState } from 'react'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; +import type { ObservabilityOnboardingAppServices } from '../../..'; + +interface OnboardingApiKeyResponse { + apiKeyEncoded: string; +} +interface APMApiKeyResponse { + agentKey: { name: string; encoded: string }; +} + +export function useEncodedApiKey() { + const [encodedApiKey, setEncodedApiKey] = useState(null); + const [error, setError] = useState | null>(null); + + const { + services: { + http, + context: { isServerless }, + }, + } = useKibana(); + + useEffect(() => { + requestEncodedApiKey(isServerless, http) + .then((apiKey) => { + setEncodedApiKey(apiKey); + }) + .catch((err) => { + setError(err); + }); + }, [http, isServerless]); + + return { encodedApiKey, error }; +} + +function requestEncodedApiKey(isServerless: boolean, http: HttpSetup): Promise { + let requestPromise: Promise; + + if (isServerless) { + const timestamp = new Date().toISOString(); + + requestPromise = http + .post('/api/apm/agent_keys', { + body: JSON.stringify({ + name: `ingest-otel-host-${timestamp}`, + privileges: ['event:write'], + }), + }) + .then((res) => res.agentKey.encoded); + } else { + requestPromise = http + .post('/internal/observability_onboarding/otel/api_key') + .then((res) => res.apiKeyEncoded); + } + + return requestPromise; +} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_ingest_endpoint_url.test.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_ingest_endpoint_url.test.tsx new file mode 100644 index 0000000000000..4dcbb4954ac5e --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_ingest_endpoint_url.test.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { renderHook } from '@testing-library/react'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { useIngestEndpointUrl } from './use_ingest_endpoint_url'; + +const createWrapper = ({ isServerless }: { isServerless: boolean }) => { + return function Wrapper({ children }: React.PropsWithChildren) { + return ( + + {children} + + ); + }; +}; + +describe('useIngestEndpointUrl', () => { + it('returns ES endpoint when not on Serverless', () => { + const { result } = renderHook( + () => + useIngestEndpointUrl({ + elasticsearchUrl: 'http://elasticsearch', + managedServiceUrl: 'https://e2e-tests-c045db.apm.us-east-1.aws.elastic.cloud:443', + }), + { + wrapper: createWrapper({ isServerless: false }), + } + ); + + expect(result.current).toBe('http://elasticsearch'); + }); + + it('returns APM endpoint with replaced sub-domain when on Serverless', () => { + const { result } = renderHook( + () => + useIngestEndpointUrl({ + elasticsearchUrl: 'http://elasticsearch', + managedServiceUrl: 'https://e2e-tests-c045db.apm.us-east-1.aws.elastic.cloud:443', + }), + { + wrapper: createWrapper({ isServerless: true }), + } + ); + + expect(result.current).toBe('https://e2e-tests-c045db.ingest.us-east-1.aws.elastic.cloud:443'); + }); +}); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_ingest_endpoint_url.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_ingest_endpoint_url.ts new file mode 100644 index 0000000000000..b08f7d33a27d5 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/use_ingest_endpoint_url.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { ObservabilityOnboardingAppServices } from '../../..'; + +interface Props { + elasticsearchUrl?: string; + managedServiceUrl?: string; +} + +export function useIngestEndpointUrl({ elasticsearchUrl, managedServiceUrl }: Props): string { + const { + services: { + context: { isServerless }, + }, + } = useKibana(); + + if (!elasticsearchUrl || !managedServiceUrl) { + return ''; + } + + return isServerless ? convertApmUrlToOtelUrl(managedServiceUrl) : elasticsearchUrl; +} + +function convertApmUrlToOtelUrl(apmUrl: string): string { + const urlParts = apmUrl.split('.'); + + return `${urlParts[0]}.ingest.${urlParts.slice(2).join('.')}`; +} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/logs/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/logs/route.ts index 63770341ab1c8..0118cd991a6ac 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/logs/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/logs/route.ts @@ -51,6 +51,7 @@ const installShipperSetupRoute = createObservabilityOnboardingServerRoute({ scriptDownloadUrl: string; elasticAgentVersionInfo: ElasticAgentVersionInfo; elasticsearchUrl: string[]; + managedServiceUrl: string; }> { const { core, @@ -78,6 +79,9 @@ const installShipperSetupRoute = createObservabilityOnboardingServerRoute({ elasticsearchUrl, scriptDownloadUrl, elasticAgentVersionInfo, + managedServiceUrl: + plugins.apm.setup.config.managedServiceUrl || + 'https://e2e-tests-c045db.apm.us-east-1.aws.elastic.cloud:443', }; }, }); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/types.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/types.ts index 92f65cbe1b7d6..53bb49865fc36 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/types.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/types.ts @@ -12,6 +12,7 @@ import { PluginStart as DataPluginStart, } from '@kbn/data-plugin/server'; import { FleetSetupContract, FleetStartContract } from '@kbn/fleet-plugin/server'; +import { APMPluginSetup } from '@kbn/apm-plugin/server'; import { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; import { @@ -26,6 +27,7 @@ export interface ObservabilityOnboardingPluginSetupDependencies { usageCollection: UsageCollectionSetup; fleet: FleetSetupContract; customIntegrations: CustomIntegrationsPluginSetup; + apm: APMPluginSetup; } export interface ObservabilityOnboardingPluginStartDependencies {