[Onboarding][Firehose] Switch to the latest CloudFormation template (e…

…lastic#204185) Closes elastic#203563 Updates the CF template to the latest available version and adjust the code to use a single Firehose stream created by the new template. You can use `Elastic Observability` AWS account (available through Okta) and the latest Kibana deployment from this PR to test the flow. ![CleanShot 2024-12-17 at 11 42 33@2x](https://github.com/user-attachments/assets/ac9ba9eb-1c9f-48fb-9fca-ed518970ac9b)
cqliu1 · Jan 2, 2025 · aa6d4d9 · aa6d4d9
1 parent b8773d0
commit aa6d4d9
Show file tree

Hide file tree

Showing 7 changed files with 25 additions and 36 deletions.
diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_firehose.ts b/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_firehose.ts
@@ -5,12 +5,11 @@
  * 2.0.
  */
 
-export const FIREHOSE_CLOUDFORMATION_STACK_NAME = 'Elastic-CloudwatchLogsAndMetricsToFirehose';
-export const FIREHOSE_LOGS_STREAM_NAME = 'Elastic-CloudwatchLogs';
-export const FIREHOSE_METRICS_STREAM_NAME = 'Elastic-CloudwatchMetrics';
+export const FIREHOSE_CLOUDFORMATION_STACK_NAME = 'Elastic-Firehose';
+export const FIREHOSE_STREAM_NAME = 'Elastic-Cloudwatch';
 
 export const FIREHOSE_CLOUDFORMATION_TEMPLATE_URL =
-  'https://elastic-cloudformation-templates.s3.amazonaws.com/v0.1.0/firehose_default_start.yml';
+  'https://elastic-cloudformation-templates.s3.amazonaws.com/v0.5.0/firehose_default_start.yml';
 
 export const AWS_INDEX_NAME_LIST = [
   // Logs

diff --git a/..._onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx b/..._onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx
@@ -18,8 +18,7 @@ import { FormattedMessage } from '@kbn/i18n-react';
 import React from 'react';
 import {
   FIREHOSE_CLOUDFORMATION_STACK_NAME,
-  FIREHOSE_LOGS_STREAM_NAME,
-  FIREHOSE_METRICS_STREAM_NAME,
+  FIREHOSE_STREAM_NAME,
 } from '../../../../common/aws_firehose';
 import { CopyToClipboardButton } from '../shared/copy_to_clipboard_button';
 import { DownloadTemplateCallout } from './download_template_callout';
@@ -42,8 +41,7 @@ export function CreateStackCommandSnippet({
   const createStackCommand = buildCreateStackCommand({
     templateUrl,
     stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME,
-    logsStreamName: FIREHOSE_LOGS_STREAM_NAME,
-    metricsStreamName: FIREHOSE_METRICS_STREAM_NAME,
+    streamName: FIREHOSE_STREAM_NAME,
     encodedApiKey,
     elasticsearchUrl,
   });

diff --git a/...y_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx b/...y_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx
@@ -11,8 +11,7 @@ import { FormattedMessage } from '@kbn/i18n-react';
 import React from 'react';
 import {
   FIREHOSE_CLOUDFORMATION_STACK_NAME,
-  FIREHOSE_LOGS_STREAM_NAME,
-  FIREHOSE_METRICS_STREAM_NAME,
+  FIREHOSE_STREAM_NAME,
 } from '../../../../common/aws_firehose';
 import { DownloadTemplateCallout } from './download_template_callout';
 import { buildCreateStackAWSConsoleURL } from './utils';
@@ -33,8 +32,7 @@ export function CreateStackInAWSConsole({
   const awsConsoleURL = buildCreateStackAWSConsoleURL({
     templateUrl,
     stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME,
-    logsStreamName: FIREHOSE_LOGS_STREAM_NAME,
-    metricsStreamName: FIREHOSE_METRICS_STREAM_NAME,
+    streamName: FIREHOSE_STREAM_NAME,
     elasticsearchUrl,
     encodedApiKey,
   });

diff --git a/...y_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts b/...y_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts
@@ -8,15 +8,15 @@
 import { useFetcher } from '../../../hooks/use_fetcher';
 import {
   FIREHOSE_CLOUDFORMATION_STACK_NAME,
-  FIREHOSE_LOGS_STREAM_NAME,
+  FIREHOSE_STREAM_NAME,
 } from '../../../../common/aws_firehose';
 
 export function usePopulatedAWSIndexList() {
   return useFetcher((callApi) => {
     return callApi('GET /internal/observability_onboarding/firehose/has-data', {
       params: {
         query: {
-          logsStreamName: FIREHOSE_LOGS_STREAM_NAME,
+          streamName: FIREHOSE_STREAM_NAME,
           stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME,
         },
       },

diff --git a/...ty/plugins/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts b/...ty/plugins/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts
@@ -10,15 +10,13 @@ export const HAS_DATA_FETCH_INTERVAL = 5000;
 export function buildCreateStackCommand({
   templateUrl,
   stackName,
-  logsStreamName,
-  metricsStreamName,
+  streamName,
   encodedApiKey,
   elasticsearchUrl,
 }: {
   templateUrl: string;
   stackName: string;
-  logsStreamName: string;
-  metricsStreamName: string;
+  streamName: string;
   encodedApiKey: string;
   elasticsearchUrl: string;
 }) {
@@ -29,8 +27,7 @@ export function buildCreateStackCommand({
     aws cloudformation create-stack
       --stack-name ${stackName}
       --template-url ${escapedTemplateUrl}
-      --parameters ParameterKey=FirehoseStreamNameForLogs,ParameterValue=${logsStreamName}
-                   ParameterKey=FirehoseStreamNameForMetrics,ParameterValue=${metricsStreamName}
+      --parameters ParameterKey=FirehoseStreamName,ParameterValue=${streamName}
                    ParameterKey=ElasticEndpointURL,ParameterValue=${escapedElasticsearchUrl}
                    ParameterKey=ElasticAPIKey,ParameterValue=${encodedApiKey}
       --capabilities CAPABILITY_IAM
@@ -54,15 +51,13 @@ export function buildStackStatusCommand({ stackName }: { stackName: string }) {
 export function buildCreateStackAWSConsoleURL({
   templateUrl,
   stackName,
-  logsStreamName,
-  metricsStreamName,
+  streamName,
   elasticsearchUrl,
   encodedApiKey,
 }: {
   templateUrl: string;
   stackName: string;
-  logsStreamName: string;
-  metricsStreamName: string;
+  streamName: string;
   elasticsearchUrl: string;
   encodedApiKey: string;
 }): string {
@@ -76,8 +71,7 @@ export function buildCreateStackAWSConsoleURL({
      * which triggers the eslint rule.
      */
     /* eslint-disable @typescript-eslint/naming-convention */
-    param_FirehoseStreamNameForLogs: logsStreamName,
-    param_FirehoseStreamNameForMetrics: metricsStreamName,
+    param_FirehoseStreamName: streamName,
     param_ElasticEndpointURL: elasticsearchUrl,
     param_ElasticAPIKey: encodedApiKey,
     /* eslint-enable @typescript-eslint/naming-convention */

diff --git a/.../solutions/observability/plugins/observability_onboarding/server/routes/firehose/route.ts b/.../solutions/observability/plugins/observability_onboarding/server/routes/firehose/route.ts
@@ -91,13 +91,13 @@ const hasFirehoseDataRoute = createObservabilityOnboardingServerRoute({
   endpoint: 'GET /internal/observability_onboarding/firehose/has-data',
   params: t.type({
     query: t.type({
-      logsStreamName: t.string,
+      streamName: t.string,
       stackName: t.string,
     }),
   }),
   options: { tags: [] },
   async handler(resources): Promise<HasFirehoseDataRouteResponse> {
-    const { logsStreamName, stackName } = resources.params.query;
+    const { streamName, stackName } = resources.params.query;
     const { elasticsearch } = await resources.context.core;
     const indexPatternList = AWS_INDEX_NAME_LIST.map((index) => `${index}-*`);
 
@@ -115,7 +115,7 @@ const hasFirehoseDataRoute = createObservabilityOnboardingServerRoute({
         query: {
           bool: {
             should: [
-              ...termQuery('aws.kinesis.name', logsStreamName),
+              ...termQuery('aws.kinesis.name', streamName),
               ...wildcardQuery('aws.exporter.arn', stackName),
             ],
           },

diff --git a/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts b/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts
@@ -11,7 +11,7 @@ import moment from 'moment';
 import { FtrProviderContext } from '../../../ftr_provider_context';
 
 const CF_COMMAND_REGEXP =
-  /aws cloudformation create-stack --stack-name (\S+) --template-url \S+ --parameters ParameterKey=FirehoseStreamNameForLogs,ParameterValue=(\S+) .+? --capabilities CAPABILITY_IAM/;
+  /aws cloudformation create-stack --stack-name (\S+) --template-url \S+ --parameters ParameterKey=FirehoseStreamName,ParameterValue=(\S+) .+? --capabilities CAPABILITY_IAM/;
 
 export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const PageObjects = getPageObjects(['common', 'svlCommonPage']);
@@ -60,9 +60,9 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       const AWS_SERVICE_ID = 'vpc-flow';
       await testSubjects.clickWhenNotDisabled('observabilityOnboardingCopyToClipboardButton');
       const copiedCommand = await browser.getClipboardValue();
-      const [, _stackName, logsStreamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? [];
+      const [, _stackName, streamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? [];
 
-      expect(logsStreamName).toBeDefined();
+      expect(streamName).toBeDefined();
 
       await browser.execute(`window.dispatchEvent(new Event("blur"))`);
 
@@ -75,7 +75,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
           .rate(1)
           .generator((timestamp) => {
             return log.create().dataset(DATASET).timestamp(timestamp).defaults({
-              'aws.kinesis.name': logsStreamName,
+              'aws.kinesis.name': streamName,
             });
           })
       );
@@ -89,11 +89,11 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       const AWS_SERVICE_ID = 'vpc-flow';
       await testSubjects.clickWhenNotDisabled('observabilityOnboardingCopyToClipboardButton');
       const copiedCommand = await browser.getClipboardValue();
-      const [, _stackName, logsStreamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? [];
+      const [, _stackName, streamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? [];
 
       await testSubjects.missingOrFail('observabilityOnboardingFirehosePanelExistingDataCallout');
 
-      expect(logsStreamName).toBeDefined();
+      expect(streamName).toBeDefined();
 
       // Simulate Firehose stream ingesting log files
       const to = new Date().toISOString();
@@ -104,7 +104,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
           .rate(1)
           .generator((timestamp) => {
             return log.create().dataset(DATASET).timestamp(timestamp).defaults({
-              'aws.kinesis.name': logsStreamName,
+              'aws.kinesis.name': streamName,
             });
           })
       );