Skip to content

Commit

Permalink
[Onboarding][OTel Host] Use MOTel collector on Serverless
Browse files Browse the repository at this point in the history
  • Loading branch information
mykolaharmash committed Mar 4, 2025
1 parent b26d85b commit f38f31a
Show file tree
Hide file tree
Showing 11 changed files with 191 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,5 @@ platform_engineer:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW
endpoint_operations_analyst:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW
endpoint_policy_manager:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW

elastic_viewer:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW

Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ detections_admin:detections_admin
platform_engineer:platform_engineer
endpoint_operations_analyst:endpoint_operations_analyst
endpoint_policy_manager:endpoint_policy_manager
viewer:elastic_viewer
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ export class APMPlugin
},
});

return { config$ };
return { config$, config: currentConfig };
}

public start(core: CoreStart, plugins: APMPluginStartDependencies) {
Expand Down
1 change: 1 addition & 0 deletions x-pack/solutions/observability/plugins/apm/server/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ import type { APMConfig } from '.';

export interface APMPluginSetup {
config$: Observable<APMConfig>;
config: APMConfig;
}

export interface APMPluginSetupDependencies {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
"discover",
"share",
"fleet",
"customIntegrations"
"customIntegrations",
"apm"
],
"optionalPlugins": [
"cloud",
Expand All @@ -35,4 +36,4 @@
"common"
]
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ import { useFetcher } from '../../../hooks/use_fetcher';
import { MultiIntegrationInstallBanner } from './multi_integration_install_banner';
import { EmptyPrompt } from '../shared/empty_prompt';
import { FeedbackButtons } from '../shared/feedback_buttons';
import { useEncodedApiKey } from './use_encoded_api_key';
import { useIngestEndpointUrl } from './use_ingest_endpoint_url';

const HOST_COMMAND = i18n.translate(
'xpack.observability_onboarding.otelLogsPanel.p.runTheCommandOnYourHostLabel',
Expand All @@ -43,25 +45,19 @@ const HOST_COMMAND = i18n.translate(

export const OtelLogsPanel: React.FC = () => {
const {
data: apiKeyData,
error,
refetch,
} = useFetcher(
(callApi) => {
return callApi('POST /internal/observability_onboarding/otel/api_key', {});
},
[],
{ showToastOnError: false }
);
services: { share, http },
} = useKibana<ObservabilityOnboardingAppServices>();

const { encodedApiKey, error } = useEncodedApiKey();

const { data: setup } = useFetcher((callApi) => {
return callApi('GET /internal/observability_onboarding/logs/setup/environment');
}, []);

const {
services: { share, http },
} = useKibana<ObservabilityOnboardingAppServices>();

const ingestEndpointUrl = useIngestEndpointUrl({
elasticsearchUrl: setup?.elasticsearchUrl[0],
managedServiceUrl: setup?.managedServiceUrl,
});
const AGENT_CDN_BASE_URL = 'artifacts.elastic.co/downloads/beats/elastic-agent';
const agentVersion = setup?.elasticAgentVersionInfo.agentVersion ?? '';
const urlEncodedAgentVersion = encodeURIComponent(agentVersion);
Expand Down Expand Up @@ -89,7 +85,7 @@ export const OtelLogsPanel: React.FC = () => {
curl --output elastic-distro-${agentVersion}-linux-$arch.tar.gz --url https://${AGENT_CDN_BASE_URL}/elastic-agent-${urlEncodedAgentVersion}-linux-$arch.tar.gz --proto '=https' --tlsv1.2 -fOL && mkdir -p elastic-distro-${agentVersion}-linux-$arch && tar -xvf elastic-distro-${agentVersion}-linux-$arch.tar.gz -C "elastic-distro-${agentVersion}-linux-$arch" --strip-components=1 && cd elastic-distro-${agentVersion}-linux-$arch
rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mkdir -p ./data/otelcol && sed -i 's#\\\${env:STORAGE_DIR}#'"$PWD"/data/otelcol'#g' ./otel.yml && sed -i 's#\\\${env:ELASTIC_ENDPOINT}#${setup?.elasticsearchUrl}#g' ./otel.yml && sed -i 's/\\\${env:ELASTIC_API_KEY}/${apiKeyData?.apiKeyEncoded}/g' ./otel.yml`,
rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mkdir -p ./data/otelcol && sed -i 's#\\\${env:STORAGE_DIR}#'"$PWD"/data/otelcol'#g' ./otel.yml && sed -i 's#\\\${env:ELASTIC_ENDPOINT}#${ingestEndpointUrl}#g' ./otel.yml && sed -i 's/\\\${env:ELASTIC_API_KEY}/${encodedApiKey}/g' ./otel.yml`,
start: 'sudo ./otelcol --config otel.yml',
type: 'copy',
},
Expand All @@ -101,7 +97,7 @@ rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mk
curl --output elastic-distro-${agentVersion}-darwin-$arch.tar.gz --url https://${AGENT_CDN_BASE_URL}/elastic-agent-${urlEncodedAgentVersion}-darwin-$arch.tar.gz --proto '=https' --tlsv1.2 -fOL && mkdir -p "elastic-distro-${agentVersion}-darwin-$arch" && tar -xvf elastic-distro-${agentVersion}-darwin-$arch.tar.gz -C "elastic-distro-${agentVersion}-darwin-$arch" --strip-components=1 && cd elastic-distro-${agentVersion}-darwin-$arch
rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mkdir -p ./data/otelcol && sed -i '' 's#\\\${env:STORAGE_DIR}#'"$PWD"/data/otelcol'#g' ./otel.yml && sed -i '' 's#\\\${env:ELASTIC_ENDPOINT}#${setup?.elasticsearchUrl}#g' ./otel.yml && sed -i '' 's/\\\${env:ELASTIC_API_KEY}/${apiKeyData?.apiKeyEncoded}/g' ./otel.yml`,
rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mkdir -p ./data/otelcol && sed -i '' 's#\\\${env:STORAGE_DIR}#'"$PWD"/data/otelcol'#g' ./otel.yml && sed -i '' 's#\\\${env:ELASTIC_ENDPOINT}#${ingestEndpointUrl}#g' ./otel.yml && sed -i '' 's/\\\${env:ELASTIC_API_KEY}/${encodedApiKey}/g' ./otel.yml`,
start: './otelcol --config otel.yml',
type: 'copy',
},
Expand All @@ -112,7 +108,13 @@ rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mk
const selectedContent = installTabContents.find((tab) => tab.id === selectedTab)!;

if (error) {
return <EmptyPrompt onboardingFlowType="otel_logs" error={error} onRetryClick={refetch} />;
return (
<EmptyPrompt
onboardingFlowType="otel_logs"
error={error}
onRetryClick={() => window.location.reload()}
/>
);
}

return (
Expand Down Expand Up @@ -144,9 +146,9 @@ rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mk
}}
/>

{(!setup || !apiKeyData) && <EuiSkeletonText lines={6} />}
{(!setup || !encodedApiKey) && <EuiSkeletonText lines={6} />}

{setup && apiKeyData && (
{setup && encodedApiKey && (
<>
<EuiText>
<p>{selectedContent.firstStepTitle}</p>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { useEffect, useState } from 'react';
import { useKibana } from '@kbn/kibana-react-plugin/public';
import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core/public';
import type { ObservabilityOnboardingAppServices } from '../../..';

interface OnboardingApiKeyResponse {
apiKeyEncoded: string;
}
interface APMApiKeyResponse {
agentKey: { name: string; encoded: string };
}

export function useEncodedApiKey() {
const [encodedApiKey, setEncodedApiKey] = useState<string | null>(null);
const [error, setError] = useState<IHttpFetchError<ResponseErrorBody> | null>(null);

const {
services: {
http,
context: { isServerless },
},
} = useKibana<ObservabilityOnboardingAppServices>();

useEffect(() => {
requestEncodedApiKey(isServerless, http)
.then((apiKey) => {
setEncodedApiKey(apiKey);
})
.catch((err) => {
setError(err);
});
}, [http, isServerless]);

return { encodedApiKey, error };
}

function requestEncodedApiKey(isServerless: boolean, http: HttpSetup): Promise<string> {
let requestPromise: Promise<string>;

if (isServerless) {
const timestamp = new Date().toISOString();

requestPromise = http
.post<APMApiKeyResponse>('/api/apm/agent_keys', {
body: JSON.stringify({
name: `ingest-otel-host-${timestamp}`,
privileges: ['event:write'],
}),
})
.then((res) => res.agentKey.encoded);
} else {
requestPromise = http
.post<OnboardingApiKeyResponse>('/internal/observability_onboarding/otel/api_key')
.then((res) => res.apiKeyEncoded);
}

return requestPromise;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import React from 'react';
import { renderHook } from '@testing-library/react';
import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
import { useIngestEndpointUrl } from './use_ingest_endpoint_url';

const createWrapper = ({ isServerless }: { isServerless: boolean }) => {
return function Wrapper({ children }: React.PropsWithChildren) {
return (
<KibanaContextProvider
services={{
context: { isServerless },
}}
>
{children}
</KibanaContextProvider>
);
};
};

describe('useIngestEndpointUrl', () => {
it('returns ES endpoint when not on Serverless', () => {
const { result } = renderHook(
() =>
useIngestEndpointUrl({
elasticsearchUrl: 'http://elasticsearch',
managedServiceUrl: 'https://e2e-tests-c045db.apm.us-east-1.aws.elastic.cloud:443',
}),
{
wrapper: createWrapper({ isServerless: false }),
}
);

expect(result.current).toBe('http://elasticsearch');
});

it('returns APM endpoint with replaced sub-domain when on Serverless', () => {
const { result } = renderHook(
() =>
useIngestEndpointUrl({
elasticsearchUrl: 'http://elasticsearch',
managedServiceUrl: 'https://e2e-tests-c045db.apm.us-east-1.aws.elastic.cloud:443',
}),
{
wrapper: createWrapper({ isServerless: true }),
}
);

expect(result.current).toBe('https://e2e-tests-c045db.ingest.us-east-1.aws.elastic.cloud:443');
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { useKibana } from '@kbn/kibana-react-plugin/public';
import { ObservabilityOnboardingAppServices } from '../../..';

interface Props {
elasticsearchUrl?: string;
managedServiceUrl?: string;
}

export function useIngestEndpointUrl({ elasticsearchUrl, managedServiceUrl }: Props): string {
const {
services: {
context: { isServerless },
},
} = useKibana<ObservabilityOnboardingAppServices>();

if (!elasticsearchUrl || !managedServiceUrl) {
return '';
}

return isServerless ? convertApmUrlToOtelUrl(managedServiceUrl) : elasticsearchUrl;
}

function convertApmUrlToOtelUrl(apmUrl: string): string {
const urlParts = apmUrl.split('.');

return `${urlParts[0]}.ingest.${urlParts.slice(2).join('.')}`;
}
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ const installShipperSetupRoute = createObservabilityOnboardingServerRoute({
scriptDownloadUrl: string;
elasticAgentVersionInfo: ElasticAgentVersionInfo;
elasticsearchUrl: string[];
managedServiceUrl: string;
}> {
const {
core,
Expand Down Expand Up @@ -78,6 +79,9 @@ const installShipperSetupRoute = createObservabilityOnboardingServerRoute({
elasticsearchUrl,
scriptDownloadUrl,
elasticAgentVersionInfo,
managedServiceUrl:
plugins.apm.setup.config.managedServiceUrl ||
'https://e2e-tests-c045db.apm.us-east-1.aws.elastic.cloud:443',
};
},
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
PluginStart as DataPluginStart,
} from '@kbn/data-plugin/server';
import { FleetSetupContract, FleetStartContract } from '@kbn/fleet-plugin/server';
import { APMPluginSetup } from '@kbn/apm-plugin/server';
import { ObservabilityPluginSetup } from '@kbn/observability-plugin/server';
import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
import {
Expand All @@ -26,6 +27,7 @@ export interface ObservabilityOnboardingPluginSetupDependencies {
usageCollection: UsageCollectionSetup;
fleet: FleetSetupContract;
customIntegrations: CustomIntegrationsPluginSetup;
apm: APMPluginSetup;
}

export interface ObservabilityOnboardingPluginStartDependencies {
Expand Down

0 comments on commit f38f31a

Please sign in to comment.