diff --git a/src/platform/packages/shared/kbn-management/settings/setting_ids/index.ts b/src/platform/packages/shared/kbn-management/settings/setting_ids/index.ts index 0879cad21b804..f999629b46189 100644 --- a/src/platform/packages/shared/kbn-management/settings/setting_ids/index.ts +++ b/src/platform/packages/shared/kbn-management/settings/setting_ids/index.ts @@ -130,6 +130,7 @@ export const OBSERVABILITY_APM_ENABLE_CONTINUOUS_ROLLUPS_ID = export const OBSERVABILITY_APM_ENABLE_PROFILING_INTEGRATION_ID = 'observability:apmEnableProfilingIntegration'; export const OBSERVABILITY_APM_ENABLE_TABLE_SEARCH_BAR = 'observability:apmEnableTableSearchBar'; +export const OBSERVABILITY_APM_ENABLE_SERVICE_MAP_V2 = 'observability:apmEnableServiceMapV2'; export const OBSERVABILITY_APM_ENABLE_SERVICE_INVENTORY_TABLE_SEARCH_BAR = 'observability:apmEnableServiceInventoryTableSearchBar'; export const OBSERVABILITY_LOGS_SHARED_NEW_LOGS_OVERVIEW_ID = 'observability:newLogsOverview'; diff --git a/src/platform/packages/shared/serverless/settings/observability_project/index.ts b/src/platform/packages/shared/serverless/settings/observability_project/index.ts index 2b0ab1e3638b3..250f10cc21216 100644 --- a/src/platform/packages/shared/serverless/settings/observability_project/index.ts +++ b/src/platform/packages/shared/serverless/settings/observability_project/index.ts @@ -29,6 +29,7 @@ export const OBSERVABILITY_PROJECT_SETTINGS = [ settings.OBSERVABILITY_APM_ENABLE_CRITICAL_PATH_ID, settings.OBSERVABILITY_ENABLE_INFRASTRUCTURE_ASSET_CUSTOM_DASHBOARDS_ID, settings.OBSERVABILITY_APM_ENABLE_TABLE_SEARCH_BAR, + settings.OBSERVABILITY_APM_ENABLE_SERVICE_MAP_V2, settings.OBSERVABILITY_APM_ENABLE_SERVICE_INVENTORY_TABLE_SEARCH_BAR, settings.OBSERVABILITY_ENTITY_CENTRIC_EXPERIENCE, settings.OBSERVABILITY_AI_ASSISTANT_SIMULATED_FUNCTION_CALLING, diff --git a/src/platform/plugins/private/kibana_usage_collection/server/collectors/management/schema.ts b/src/platform/plugins/private/kibana_usage_collection/server/collectors/management/schema.ts index 5d988c516136b..08e530788237d 100644 --- a/src/platform/plugins/private/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/platform/plugins/private/kibana_usage_collection/server/collectors/management/schema.ts @@ -472,6 +472,10 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, }, + 'observability:apmEnableServiceMapV2': { + type: 'boolean', + _meta: { description: 'Non-default value of setting.' }, + }, 'observability:entityCentricExperience': { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, diff --git a/src/platform/plugins/private/kibana_usage_collection/server/collectors/management/types.ts b/src/platform/plugins/private/kibana_usage_collection/server/collectors/management/types.ts index ef5e5dfebf357..874d4db228c6d 100644 --- a/src/platform/plugins/private/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/platform/plugins/private/kibana_usage_collection/server/collectors/management/types.ts @@ -48,6 +48,7 @@ export interface UsageStats { 'observability:enableInfrastructureAssetCustomDashboards': boolean; 'observability:apmAgentExplorerView': boolean; 'observability:apmEnableTableSearchBar': boolean; + 'observability:apmEnableServiceMapV2': boolean; 'observability:apmEnableServiceInventoryTableSearchBar': boolean; 'observability:logSources': string[]; 'observability:newLogsOverview': boolean; diff --git a/x-pack/solutions/observability/plugins/apm/common/service_map/get_service_map_nodes.test.ts b/x-pack/solutions/observability/plugins/apm/common/service_map/get_service_map_nodes.test.ts index 7a1ddc76de26a..1b038462b7562 100644 --- a/x-pack/solutions/observability/plugins/apm/common/service_map/get_service_map_nodes.test.ts +++ b/x-pack/solutions/observability/plugins/apm/common/service_map/get_service_map_nodes.test.ts @@ -7,7 +7,7 @@ import { ServiceHealthStatus } from '../service_health_status'; import { SERVICE_NAME, SPAN_SUBTYPE, SPAN_TYPE } from '../es_fields/apm'; -import type { ServiceMapExitSpan, ServiceMapService, ServiceMapWithConnections } from './types'; +import type { ServiceMapExitSpan, ServiceMapService, ServiceMapConnections } from './types'; import { getServiceMapNodes } from './get_service_map_nodes'; import { getExternalConnectionNode, getServiceConnectionNode } from './utils'; @@ -75,7 +75,7 @@ const anomalies = { describe('getServiceMapNodes', () => { it('maps external destinations to internal services', () => { - const response: ServiceMapWithConnections = { + const response: ServiceMapConnections = { servicesData: [ getServiceConnectionNode(nodejsService), getServiceConnectionNode(javaService), @@ -107,7 +107,7 @@ describe('getServiceMapNodes', () => { }); it('adds connection for messaging-based external destinations', () => { - const response: ServiceMapWithConnections = { + const response: ServiceMapConnections = { servicesData: [ getServiceConnectionNode(nodejsService), getServiceConnectionNode(javaService), @@ -162,7 +162,7 @@ describe('getServiceMapNodes', () => { }); it('collapses external destinations based on span.destination.resource.name', () => { - const response: ServiceMapWithConnections = { + const response: ServiceMapConnections = { servicesData: [ getServiceConnectionNode(nodejsService), getServiceConnectionNode(javaService), @@ -197,7 +197,7 @@ describe('getServiceMapNodes', () => { }); it('picks the first span.type/subtype in an alphabetically sorted list', () => { - const response: ServiceMapWithConnections = { + const response: ServiceMapConnections = { servicesData: [getServiceConnectionNode(javaService)], exitSpanDestinations: [], connections: [ @@ -235,7 +235,7 @@ describe('getServiceMapNodes', () => { }); it('processes connections without a matching "service" aggregation', () => { - const response: ServiceMapWithConnections = { + const response: ServiceMapConnections = { servicesData: [getServiceConnectionNode(javaService)], exitSpanDestinations: [], connections: [ @@ -252,7 +252,7 @@ describe('getServiceMapNodes', () => { }); it('maps routing services child transasctions to their corresponding upstream service', () => { - const response: ServiceMapWithConnections = { + const response: ServiceMapConnections = { servicesData: [getServiceConnectionNode(javaService)], exitSpanDestinations: [ { diff --git a/x-pack/solutions/observability/plugins/apm/common/service_map/get_service_map_nodes.ts b/x-pack/solutions/observability/plugins/apm/common/service_map/get_service_map_nodes.ts index 08e44b26ab9c6..3f3730ef5e8cf 100644 --- a/x-pack/solutions/observability/plugins/apm/common/service_map/get_service_map_nodes.ts +++ b/x-pack/solutions/observability/plugins/apm/common/service_map/get_service_map_nodes.ts @@ -13,7 +13,7 @@ import { SPAN_TYPE, SPAN_SUBTYPE, } from '../es_fields/apm'; -import type { ExitSpanDestination } from './types'; +import type { ExitSpanDestination, ServicesResponse } from './types'; import type { Connection, ConnectionNode, @@ -21,9 +21,10 @@ import type { ExternalConnectionNode, ConnectionElement, ConnectionEdge, - ServiceMapWithConnections, + ServiceMapConnections, + GroupResourceNodesResponse, } from './types'; -import type { GroupResourceNodesResponse } from './group_resource_nodes'; + import { groupResourceNodes } from './group_resource_nodes'; import { getConnectionId, isExitSpan } from './utils'; @@ -55,10 +56,7 @@ function addMessagingConnections( return [...connections, ...messagingConnections]; } -function getAllNodes( - services: ServiceMapWithConnections['servicesData'], - connections: ServiceMapWithConnections['connections'] -) { +function getAllNodes(services: ServicesResponse[], connections: Connection[]) { const allNodesMap = new Map(); // Process connections in one pass @@ -257,7 +255,7 @@ export function getServiceMapNodes({ exitSpanDestinations, servicesData, anomalies, -}: ServiceMapWithConnections): GroupResourceNodesResponse { +}: ServiceMapConnections): GroupResourceNodesResponse { const allConnections = addMessagingConnections(connections, exitSpanDestinations); const allNodes = getAllNodes(servicesData, allConnections); const allServices = getAllServices(allNodes, exitSpanDestinations); diff --git a/x-pack/solutions/observability/plugins/apm/common/service_map/group_resource_nodes.test.ts b/x-pack/solutions/observability/plugins/apm/common/service_map/group_resource_nodes.test.ts index e208b1b036a2d..2c3485a0ee222 100644 --- a/x-pack/solutions/observability/plugins/apm/common/service_map/group_resource_nodes.test.ts +++ b/x-pack/solutions/observability/plugins/apm/common/service_map/group_resource_nodes.test.ts @@ -6,10 +6,8 @@ */ import type { ConnectionElement, ServiceMapExitSpan, ServiceMapService } from './types'; -import type { GroupedNode } from './group_resource_nodes'; +import type { GroupedNode } from './types'; import { groupResourceNodes } from './group_resource_nodes'; -import expectedGroupedData from '../../server/routes/service_map/mock_responses/group_resource_nodes_grouped.json'; -import preGroupedData from '../../server/routes/service_map/mock_responses/group_resource_nodes_pregrouped.json'; import { getEdgeId, getExternalConnectionNode, getServiceConnectionNode } from './utils'; describe('groupResourceNodes', () => { @@ -88,19 +86,6 @@ describe('groupResourceNodes', () => { describe('basic grouping', () => { it('should group external nodes', () => { - const responseWithGroups = groupResourceNodes( - preGroupedData as { elements: ConnectionElement[] } - ); - expect(responseWithGroups.elements).toHaveLength(expectedGroupedData.elements.length); - for (const element of responseWithGroups.elements) { - const expectedElement = expectedGroupedData.elements.find( - ({ data: { id } }: { data: { id: string } }) => id === element.data.id - )!; - expect(element).toMatchObject(expectedElement); - } - }); - - it('should group nodes when they meet minimum group size', () => { const elements: ConnectionElement[] = [ nodeJsServiceNode, nodeJsExitSpanQuora, diff --git a/x-pack/solutions/observability/plugins/apm/common/service_map/group_resource_nodes.ts b/x-pack/solutions/observability/plugins/apm/common/service_map/group_resource_nodes.ts index 9dd06170d0f94..18b1acf69d7e3 100644 --- a/x-pack/solutions/observability/plugins/apm/common/service_map/group_resource_nodes.ts +++ b/x-pack/solutions/observability/plugins/apm/common/service_map/group_resource_nodes.ts @@ -8,39 +8,22 @@ import { i18n } from '@kbn/i18n'; import { compact, groupBy } from 'lodash'; import { SPAN_TYPE, SPAN_SUBTYPE } from '../es_fields/apm'; -import type { ConnectionEdge, ConnectionElement, ConnectionNode } from './types'; +import type { + ConnectionEdge, + ConnectionElement, + ConnectionNode, + GroupResourceNodesResponse, + GroupedEdge, + GroupedNode, +} from './types'; import { getEdgeId, isSpanGroupingSupported } from './utils'; const MINIMUM_GROUP_SIZE = 4; -type GroupedConnection = ConnectionNode | ConnectionEdge; - -export interface GroupedNode { - data: { - id: string; - 'span.type': string; - label: string; - groupedConnections: GroupedConnection[]; - }; -} - -export interface GroupedEdge { - data: { - id: string; - source: string; - target: string; - }; -} - -export interface GroupResourceNodesResponse { - elements: Array; - nodesCount: number; -} - const isEdge = (el: ConnectionElement): el is { data: ConnectionEdge } => Boolean(el.data.source && el.data.target); const isNode = (el: ConnectionElement): el is { data: ConnectionNode } => !isEdge(el); -const isElligibleGroupNode = (el: ConnectionElement) => { +const isElligibleGroupNode = (el: ConnectionElement): el is { data: ConnectionNode } => { if (isNode(el) && SPAN_TYPE in el.data) { return isSpanGroupingSupported(el.data[SPAN_TYPE], el.data[SPAN_SUBTYPE]); } diff --git a/x-pack/solutions/observability/plugins/apm/common/service_map/index.ts b/x-pack/solutions/observability/plugins/apm/common/service_map/index.ts index 22d4fda95a289..000942ce3bdf3 100644 --- a/x-pack/solutions/observability/plugins/apm/common/service_map/index.ts +++ b/x-pack/solutions/observability/plugins/apm/common/service_map/index.ts @@ -12,15 +12,15 @@ import type { ExitSpanDestination, ConnectionElement, ExternalConnectionNode, + GroupResourceNodesResponse, ServiceConnectionNode, ServicesResponse, ServiceMapResponse, - ServiceMapWithConnections, + ServiceMapConnections, ServiceMapTelemetry, NodeStats, NodeItem, } from './types'; -import type { GroupResourceNodesResponse } from './group_resource_nodes'; export * from './utils'; export { getServiceMapNodes } from './get_service_map_nodes'; @@ -35,7 +35,7 @@ export { ExternalConnectionNode, ServiceConnectionNode, ServicesResponse, - ServiceMapWithConnections, + ServiceMapConnections as ServiceMapWithConnections, ServiceMapResponse, ServiceMapTelemetry, NodeStats, diff --git a/x-pack/solutions/observability/plugins/apm/common/service_map/types.ts b/x-pack/solutions/observability/plugins/apm/common/service_map/types.ts index 8d0a03118443f..bb7249995c8b9 100644 --- a/x-pack/solutions/observability/plugins/apm/common/service_map/types.ts +++ b/x-pack/solutions/observability/plugins/apm/common/service_map/types.ts @@ -7,50 +7,83 @@ import type cytoscape from 'cytoscape'; import type { AgentName } from '@kbn/apm-types/src/es_schemas/ui/fields'; +import type { AGENT_NAME, SERVICE_ENVIRONMENT, SERVICE_NAME } from '@kbn/apm-types'; +import type { SPAN_DESTINATION_SERVICE_RESOURCE, SPAN_SUBTYPE, SPAN_TYPE } from '@kbn/apm-types'; import type { ServiceAnomaliesResponse } from '../../server/routes/service_map/get_service_anomalies'; import type { Coordinate } from '../../typings/timeseries'; import type { ServiceAnomalyStats } from '../anomaly_detection'; export interface ServiceMapTelemetry { tracesCount: number; + nodesCount?: number; } -export interface ServiceMapWithConnections - extends Pick { +type GroupedConnection = ConnectionNode | ConnectionEdge; + +export interface GroupedNode { + data: { + id: string; + 'span.type': string; + label: string; + groupedConnections: GroupedConnection[]; + }; +} + +export interface GroupedEdge { + data: { + id: string; + source: string; + target: string; + }; +} + +export interface GroupResourceNodesResponse { + elements: Array; + nodesCount: number; +} + +export type ConnectionType = Connection | ConnectionLegacy; +export type DestinationType = ExitSpanDestination | ExitSpanDestinationLegacy; + +export interface ServiceMapConnections { + servicesData: ServicesResponse[]; + anomalies: ServiceAnomaliesResponse; connections: Connection[]; exitSpanDestinations: ExitSpanDestination[]; } -export type ServiceMapResponse = { - spans: ServiceMapNode[]; +export interface ServiceMapRawResponse { + spans: ServiceMapSpan[]; servicesData: ServicesResponse[]; anomalies: ServiceAnomaliesResponse; -} & ServiceMapTelemetry; +} +export type ServiceMapResponse = ServiceMapTelemetry & + (ServiceMapRawResponse | GroupResourceNodesResponse); export interface ServicesResponse { - 'service.name': string; - 'agent.name': string; - 'service.environment': string | null; + [SERVICE_NAME]: string; + [AGENT_NAME]: string; + [SERVICE_ENVIRONMENT]: string | null; } -export interface ServiceConnectionNode extends cytoscape.NodeDataDefinition { - id: string; - 'service.name': string; - 'service.environment': string | null; - 'agent.name': string; - 'service.node.name'?: string; - serviceAnomalyStats?: ServiceAnomalyStats; - label?: string; -} +export type ServiceConnectionNode = cytoscape.NodeDataDefinition & + ServicesResponse & { + id: string; + serviceAnomalyStats?: ServiceAnomalyStats; + label?: string; + }; export interface ExternalConnectionNode extends cytoscape.NodeDataDefinition { id: string; - 'span.destination.service.resource': string; - 'span.type': string; - 'span.subtype': string; + [SPAN_DESTINATION_SERVICE_RESOURCE]: string; + [SPAN_TYPE]: string; + [SPAN_SUBTYPE]: string; label?: string; } export type ConnectionNode = ServiceConnectionNode | ExternalConnectionNode; +export type ConnectionNodeLegacy = + | Omit + | Omit; export interface ConnectionEdge { id: string; @@ -73,6 +106,10 @@ export interface Connection { source: ConnectionNode; destination: ConnectionNode; } +export interface ConnectionLegacy { + source: ConnectionNodeLegacy; + destination: ConnectionNodeLegacy; +} export interface NodeStats { transactionStats?: { @@ -99,11 +136,17 @@ export interface NodeStats { }; } +export type ExitSpanDestinationType = ExitSpanDestination | ExitSpanDestinationLegacy; export interface ExitSpanDestination { from: ExternalConnectionNode; to: ServiceConnectionNode; } +export interface ExitSpanDestinationLegacy { + from: Omit; + to: Omit; +} + export interface ServiceMapService { serviceName: string; agentName: AgentName; @@ -117,6 +160,6 @@ export interface ServiceMapExitSpan extends ServiceMapService { spanSubtype: string; spanDestinationServiceResource: string; } -export type ServiceMapNode = ServiceMapExitSpan & { +export type ServiceMapSpan = ServiceMapExitSpan & { destinationService?: ServiceMapService; }; diff --git a/x-pack/solutions/observability/plugins/apm/common/service_map/utils.ts b/x-pack/solutions/observability/plugins/apm/common/service_map/utils.ts index 6973a470835a0..93aa94925348f 100644 --- a/x-pack/solutions/observability/plugins/apm/common/service_map/utils.ts +++ b/x-pack/solutions/observability/plugins/apm/common/service_map/utils.ts @@ -15,7 +15,12 @@ import { SPAN_SUBTYPE, SPAN_TYPE, } from '../es_fields/apm'; -import type { ConnectionEdge, ServiceMapExitSpan, ServiceMapService } from './types'; +import type { + ConnectionEdge, + ConnectionNodeLegacy, + ServiceMapExitSpan, + ServiceMapService, +} from './types'; export const invalidLicenseMessage = i18n.translate('xpack.apm.serviceMap.invalidLicenseMessage', { defaultMessage: @@ -47,7 +52,9 @@ export function isSpanGroupingSupported(type?: string, subtype?: string) { * 2.0. */ -export function getConnections(paths: ConnectionNode[][] | undefined): Connection[] { +export function getConnections( + paths: Array> | undefined +): Connection[] { if (!paths) { return []; } @@ -57,11 +64,18 @@ export function getConnections(paths: ConnectionNode[][] | undefined): Connectio paths.forEach((path) => { for (let i = 1; i < path.length; i++) { - const connectionId = getConnectionId({ source: path[i - 1], destination: path[i] }); + const sourceNode = ( + 'id' in path[i - 1] ? path[i - 1] : { ...path[i - 1], id: getLegacyNodeId(path[i - 1]) } + ) as ConnectionNode; + const destinationNode = ( + 'id' in path[i] ? path[i] : { ...path[i], id: getLegacyNodeId(path[i]) } + ) as ConnectionNode; + + const connectionId = getConnectionId({ source: sourceNode, destination: destinationNode }); if (!connectionsById.has(connectionId)) { connectionsById.add(connectionId); - connections.push({ source: path[i - 1], destination: path[i] }); + connections.push({ source: sourceNode, destination: destinationNode }); } } }); @@ -69,10 +83,20 @@ export function getConnections(paths: ConnectionNode[][] | undefined): Connectio return connections; } -export const isExitSpan = (node: ConnectionNode): node is ExternalConnectionNode => { +export const isExitSpan = ( + node: ConnectionNode | ConnectionNodeLegacy +): node is ExternalConnectionNode => { return !!(node as ExternalConnectionNode)[SPAN_DESTINATION_SERVICE_RESOURCE]; }; +// backward compatibility with scrited_metric versions +export const getLegacyNodeId = (node: ConnectionNodeLegacy) => { + if (isExitSpan(node)) { + return `>${node[SERVICE_NAME]}|${node[SPAN_DESTINATION_SERVICE_RESOURCE]}`; + } + return `${node[SERVICE_NAME]}`; +}; + export const getServiceConnectionNode = (event: ServiceMapService): ServiceConnectionNode => { return { id: event.serviceName, diff --git a/x-pack/solutions/observability/plugins/apm/public/components/app/service_map/use_service_map.ts b/x-pack/solutions/observability/plugins/apm/public/components/app/service_map/use_service_map.ts index d5cea1eb1df88..70db7249440ea 100644 --- a/x-pack/solutions/observability/plugins/apm/public/components/app/service_map/use_service_map.ts +++ b/x-pack/solutions/observability/plugins/apm/public/components/app/service_map/use_service_map.ts @@ -7,9 +7,9 @@ import { useEffect, useState } from 'react'; import type { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; import type { - ServiceMapNode, + ServiceMapSpan, ExitSpanDestination, - ServiceMapResponse, + ServiceMapRawResponse, } from '../../../../common/service_map/types'; import { useApmPluginContext } from '../../../context/apm_plugin/use_apm_plugin_context'; import { useLicenseContext } from '../../../context/license/use_license_context'; @@ -103,14 +103,21 @@ export const useServiceMap = ({ } if (data) { - try { - const transformedData = processServiceMapData(data); - setServiceMapNodes({ data: transformedData, status: FETCH_STATUS.SUCCESS }); - } catch (err) { + if ('spans' in data) { + try { + const transformedData = processServiceMapData(data); + setServiceMapNodes({ data: transformedData, status: FETCH_STATUS.SUCCESS }); + } catch (err) { + setServiceMapNodes({ + data: { elements: [], nodesCount: 0 }, + status: FETCH_STATUS.FAILURE, + error: err, + }); + } + } else { setServiceMapNodes({ - data: { elements: [], nodesCount: 0 }, - status: FETCH_STATUS.FAILURE, - error: err, + data, + status: FETCH_STATUS.SUCCESS, }); } } @@ -119,7 +126,7 @@ export const useServiceMap = ({ return serviceMapNodes; }; -const processServiceMapData = (data: ServiceMapResponse): GroupResourceNodesResponse => { +const processServiceMapData = (data: ServiceMapRawResponse): GroupResourceNodesResponse => { const paths = getPaths({ spans: data.spans }); return getServiceMapNodes({ connections: getConnections(paths.connections), @@ -129,7 +136,7 @@ const processServiceMapData = (data: ServiceMapResponse): GroupResourceNodesResp }); }; -const getPaths = ({ spans }: { spans: ServiceMapNode[] }) => { +const getPaths = ({ spans }: { spans: ServiceMapSpan[] }) => { const connections: ConnectionNode[][] = []; const exitSpanDestinations: ExitSpanDestination[] = []; diff --git a/x-pack/solutions/observability/plugins/apm/public/components/app/settings/general_settings/index.tsx b/x-pack/solutions/observability/plugins/apm/public/components/app/settings/general_settings/index.tsx index 053cba1b1f7a2..33c38796b5227 100644 --- a/x-pack/solutions/observability/plugins/apm/public/components/app/settings/general_settings/index.tsx +++ b/x-pack/solutions/observability/plugins/apm/public/components/app/settings/general_settings/index.tsx @@ -23,6 +23,7 @@ import { apmEnableTableSearchBar, apmEnableTransactionProfiling, apmEnableServiceInventoryTableSearchBar, + apmEnableServiceMapV2, } from '@kbn/observability-plugin/common'; import { isEmpty } from 'lodash'; import React from 'react'; @@ -56,6 +57,7 @@ function getApmSettingsKeys(isProfilingIntegrationEnabled: boolean) { enableAgentExplorerView, apmEnableTableSearchBar, apmEnableServiceInventoryTableSearchBar, + apmEnableServiceMapV2, ]; if (isProfilingIntegrationEnabled) { diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/apm_routes/register_apm_server_routes.ts b/x-pack/solutions/observability/plugins/apm/server/routes/apm_routes/register_apm_server_routes.ts index 0dc46d4d3e8b6..875d5813181f5 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/apm_routes/register_apm_server_routes.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/apm_routes/register_apm_server_routes.ts @@ -30,14 +30,9 @@ import { import { jsonRt, mergeRt } from '@kbn/io-ts-utils'; import type { InspectResponse } from '@kbn/observability-plugin/typings/common'; import apm from 'elastic-apm-node'; -import { type VersionedRouteRegistrar } from '@kbn/core-http-server'; +import type { VersionedRouteRegistrar } from '@kbn/core-http-server'; import type { IRuleDataClient } from '@kbn/rule-registry-plugin/server'; import type { APMIndices } from '@kbn/apm-data-access-plugin/server'; -import type { Observable } from 'rxjs'; -import { isObservable } from 'rxjs'; -import { observableIntoEventSourceStream } from '@kbn/sse-utils-server'; -import type { ServerSentEvent } from '@kbn/sse-utils'; -import { Stream } from 'stream'; import type { ApmFeatureFlags } from '../../../common/apm_feature_flags'; import type { APMCore, @@ -48,6 +43,7 @@ import type { import type { ApmPluginRequestHandlerContext } from '../typings'; import type { APMConfig } from '../..'; import type { APMPluginSetupDependencies, APMPluginStartDependencies } from '../../types'; + const inspectRt = t.exact( t.partial({ query: t.exact(t.partial({ _inspect: jsonRt.pipe(t.boolean) })), @@ -152,7 +148,7 @@ export function registerRoutes({ ), ruleDataClient, kibanaVersion, - }).then((value: Record | ReadableStream | undefined | null) => { + }).then((value: Record | undefined | null) => { return { aborted: false, data: value, @@ -174,42 +170,20 @@ export function registerRoutes({ throw new Error('Return type cannot be an array'); } - if (isObservable(data)) { - const controller = new AbortController(); - request.events.aborted$.subscribe(() => { - controller.abort(); - }); - return response.ok({ - body: observableIntoEventSourceStream(data as Observable, { - logger, - signal: controller.signal, - }), - }); - } else if (data instanceof Stream) { - const body = data || {}; - return response.custom({ - statusCode: 200, - headers: { - 'content-type': 'application/octet-stream', - 'transfer-encoding': 'chunked', - }, - body, + const body = validatedParams.query?._inspect + ? { + ...data, + _inspect: inspectableEsQueriesMap.get(request), + } + : { ...data }; + if (!options.disableTelemetry && telemetryUsageCounter) { + telemetryUsageCounter.incrementCounter({ + counterName: `${method.toUpperCase()} ${pathname}`, + counterType: 'success', }); - } else { - const body = validatedParams.query?._inspect - ? { - ...data, - _inspect: inspectableEsQueriesMap.get(request), - } - : { ...data }; - if (!options.disableTelemetry && telemetryUsageCounter) { - telemetryUsageCounter.incrementCounter({ - counterName: `${method.toUpperCase()} ${pathname}`, - counterType: 'success', - }); - } - return response.ok({ body }); } + + return response.ok({ body }); } catch (error) { logger.error(error); @@ -296,7 +270,6 @@ export type MinimalAPMRouteHandlerResources = Omit(); + + sampleExitSpans.aggregations?.exitSpans.buckets.forEach((bucket) => { + const { success, others } = bucket.eventOutcomeGroup.buckets; + const eventOutcomeGroup = + success.sample.top.length > 0 ? success : others.sample.top.length > 0 ? others : undefined; + + const sample = eventOutcomeGroup?.sample.top[0]?.metrics; + if (!sample) { + return; + } + + const spanId = sample[SPAN_ID] as string; + + destinationsBySpanId.set(spanId, { + spanId, + spanDestinationServiceResource: bucket.key.spanDestinationServiceResource as string, + spanType: sample[SPAN_TYPE] as string, + spanSubtype: sample[SPAN_SUBTYPE] as string, + agentName: sample[AGENT_NAME] as AgentName, + serviceName: bucket.key.serviceName as string, + serviceEnvironment: sample[SERVICE_ENVIRONMENT] as string, + }); + }); + + return destinationsBySpanId; +} + +async function fetchTransactionsFromExitSpans({ + apmEventClient, + exitSpansSample, + start, + end, +}: { + apmEventClient: APMEventClient; + exitSpansSample: Map; + start: number; + end: number; +}) { + const optionalFields = asMutableArray([SERVICE_ENVIRONMENT] as const); + const requiredFields = asMutableArray([SERVICE_NAME, AGENT_NAME, PARENT_ID] as const); + + const servicesResponse = await apmEventClient.search('get_transactions_for_exit_spans', { + apm: { + events: [ProcessorEvent.transaction], + }, + track_total_hits: false, + query: { + bool: { + filter: [...rangeQuery(start, end), ...termsQuery(PARENT_ID, ...exitSpansSample.keys())], + }, + }, + size: exitSpansSample.size, + fields: [...requiredFields, ...optionalFields], + }); + + const destinationsBySpanId = new Map(exitSpansSample); + + servicesResponse.hits.hits.forEach((hit) => { + const transaction = unflattenKnownApmEventFields(hit.fields, [...requiredFields]); + + const spanId = transaction.parent.id; + + const destination = destinationsBySpanId.get(spanId); + if (destination) { + destinationsBySpanId.set(spanId, { + ...destination, + destinationService: { + agentName: transaction.agent.name, + serviceEnvironment: transaction.service.environment, + serviceName: transaction.service.name, + }, + }); + } + }); + + return Array.from(destinationsBySpanId.values()); +} diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/fetch_service_paths_from_trace_ids.ts b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/fetch_service_paths_from_trace_ids.ts index 398f7873533ba..8599c0dc4d959 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/fetch_service_paths_from_trace_ids.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/fetch_service_paths_from_trace_ids.ts @@ -4,218 +4,337 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { Logger } from '@kbn/core/server'; -import { existsQuery, rangeQuery, termsQuery } from '@kbn/observability-plugin/server'; -import type { APMEventClient } from '@kbn/apm-data-access-plugin/server'; + +import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; -import { EventOutcome } from '../../../common/event_outcome'; -import type { ServiceMapNode } from '../../../common/service_map/types'; -import type { AgentName } from '../../../typings/es_schemas/ui/fields/agent'; -import { asMutableArray } from '../../../common/utils/as_mutable_array'; +import type { + ConnectionNodeLegacy, + ExitSpanDestinationLegacy, +} from '../../../common/service_map/types'; import { AGENT_NAME, PARENT_ID, + PROCESSOR_EVENT, SERVICE_ENVIRONMENT, SERVICE_NAME, SPAN_DESTINATION_SERVICE_RESOURCE, - SPAN_ID, SPAN_SUBTYPE, SPAN_TYPE, TRACE_ID, - EVENT_OUTCOME, } from '../../../common/es_fields/apm'; +import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; +import { calculateDocsPerShard } from './calculate_docs_per_shard'; -export async function fetchPathsFromTraceIds({ - apmEventClient, - traceIds, - start, - end, - logger, -}: { - apmEventClient: APMEventClient; - traceIds: string[]; - start: number; - end: number; - logger: Logger; -}) { - logger.debug(`Fetching spans (${traceIds.length} traces)`); - - const exitSpansSample = await fetchExitSpanIdsFromTraceIds({ - apmEventClient, - traceIds, - start, - end, - }); - - const transactionsFromExitSpans = await fetchTransactionsFromExitSpans({ - apmEventClient, - exitSpansSample, - start, - end, - }); +const SCRIPTED_METRICS_FIELDS_TO_COPY = [ + PARENT_ID, + SERVICE_NAME, + SERVICE_ENVIRONMENT, + SPAN_DESTINATION_SERVICE_RESOURCE, + TRACE_ID, + PROCESSOR_EVENT, + SPAN_TYPE, + SPAN_SUBTYPE, + AGENT_NAME, +]; - return transactionsFromExitSpans; -} +const AVG_BYTES_PER_FIELD = 55; -async function fetchExitSpanIdsFromTraceIds({ +export async function fetchServicePathsFromTraceIds({ apmEventClient, traceIds, start, end, + terminateAfter, + serviceMapMaxAllowableBytes, + numOfRequests, }: { apmEventClient: APMEventClient; traceIds: string[]; start: number; end: number; + terminateAfter: number; + serviceMapMaxAllowableBytes: number; + numOfRequests: number; }) { - const sampleExitSpans = await apmEventClient.search('get_service_map_exit_span_samples', { + // make sure there's a range so ES can skip shards + const dayInMs = 24 * 60 * 60 * 1000; + const startRange = start - dayInMs; + const endRange = end + dayInMs; + + const serviceMapParams = { apm: { - events: [ProcessorEvent.span], + events: [ProcessorEvent.span, ProcessorEvent.transaction], }, - + terminate_after: terminateAfter, track_total_hits: false, size: 0, query: { bool: { filter: [ - ...rangeQuery(start, end), - ...termsQuery(TRACE_ID, ...traceIds), - ...existsQuery(SPAN_DESTINATION_SERVICE_RESOURCE), - ], - }, - }, - aggs: { - exitSpans: { - composite: { - sources: asMutableArray([ - { serviceName: { terms: { field: SERVICE_NAME } } }, - { - spanDestinationServiceResource: { - terms: { field: SPAN_DESTINATION_SERVICE_RESOURCE }, - }, - }, - ] as const), - size: 10000, - }, - aggs: { - eventOutcomeGroup: { - filters: { - filters: { - success: { - term: { - [EVENT_OUTCOME]: EventOutcome.success as const, - }, - }, - others: { - bool: { - must_not: { - term: { - [EVENT_OUTCOME]: EventOutcome.success as const, - }, - }, - }, - }, - }, - }, - aggs: { - sample: { - top_metrics: { - size: 1, - sort: '_score', - metrics: asMutableArray([ - { field: SPAN_ID }, - { field: SPAN_TYPE }, - { field: SPAN_SUBTYPE }, - { field: SPAN_DESTINATION_SERVICE_RESOURCE }, - { field: SERVICE_NAME }, - { field: SERVICE_ENVIRONMENT }, - { field: AGENT_NAME }, - ] as const), - }, - }, + { + terms: { + [TRACE_ID]: traceIds, }, }, - }, + ...rangeQuery(startRange, endRange), + ], }, }, - }); + }; + // fetch without aggs to get shard count, first + const serviceMapQueryDataResponse = await apmEventClient.search( + 'get_trace_ids_shard_data', + serviceMapParams + ); + /* + * Calculate how many docs we can fetch per shard. + * Used in both terminate_after and tracking in the map script of the scripted_metric agg + * to ensure we don't fetch more than we can handle. + * + * 1. Use serviceMapMaxAllowableBytes setting, which represents our baseline request circuit breaker limit. + * 2. Divide by numOfRequests we fire off simultaneously to calculate bytesPerRequest. + * 3. Divide bytesPerRequest by the average doc size to get totalNumDocsAllowed. + * 4. Divide totalNumDocsAllowed by totalShards to get numDocsPerShardAllowed. + * 5. Use the lesser of numDocsPerShardAllowed or terminateAfter. + */ - const destinationsBySpanId = new Map(); - - sampleExitSpans.aggregations?.exitSpans.buckets.forEach((bucket) => { - const { success, others } = bucket.eventOutcomeGroup.buckets; - const eventOutcomeGroup = - success.sample.top.length > 0 ? success : others.sample.top.length > 0 ? others : undefined; - - const sample = eventOutcomeGroup?.sample.top[0]?.metrics; - if (!sample) { - return; - } - - const spanId = sample[SPAN_ID] as string; - - destinationsBySpanId.set(spanId, { - spanId, - spanDestinationServiceResource: bucket.key.spanDestinationServiceResource as string, - spanType: sample[SPAN_TYPE] as string, - spanSubtype: sample[SPAN_SUBTYPE] as string, - agentName: sample[AGENT_NAME] as AgentName, - serviceName: bucket.key.serviceName as string, - serviceEnvironment: sample[SERVICE_ENVIRONMENT] as string, - }); + const avgDocSizeInBytes = SCRIPTED_METRICS_FIELDS_TO_COPY.length * AVG_BYTES_PER_FIELD; // estimated doc size in bytes + const totalShards = serviceMapQueryDataResponse._shards.total; + + const calculatedDocs = calculateDocsPerShard({ + serviceMapMaxAllowableBytes, + avgDocSizeInBytes, + totalShards, + numOfRequests, }); - return destinationsBySpanId; -} + const numDocsPerShardAllowed = calculatedDocs > terminateAfter ? terminateAfter : calculatedDocs; -async function fetchTransactionsFromExitSpans({ - apmEventClient, - exitSpansSample, - start, - end, -}: { - apmEventClient: APMEventClient; - exitSpansSample: Map; - start: number; - end: number; -}) { - const optionalFields = asMutableArray([SERVICE_ENVIRONMENT] as const); - const requiredFields = asMutableArray([SERVICE_NAME, AGENT_NAME, PARENT_ID] as const); + /* + * Any changes to init_script, map_script, combine_script and reduce_script + * must be replicated on https://github.com/elastic/elasticsearch-serverless/blob/main/distribution/archives/src/serverless-default-settings.yml + */ + const serviceMapAggs = { + service_map: { + scripted_metric: { + params: { + limit: numDocsPerShardAllowed, + fieldsToCopy: SCRIPTED_METRICS_FIELDS_TO_COPY, + }, + init_script: { + lang: 'painless', + source: ` + state.docCount = 0; + state.limit = params.limit; + state.eventsById = new HashMap(); + state.fieldsToCopy = params.fieldsToCopy;`, + }, + map_script: { + lang: 'painless', + source: ` + if (state.docCount >= state.limit) { + // Stop processing if the document limit is reached + return; + } - const servicesResponse = await apmEventClient.search('get_transactions_for_exit_spans', { - apm: { - events: [ProcessorEvent.transaction], - }, - track_total_hits: false, - query: { - bool: { - filter: [...rangeQuery(start, end), ...termsQuery(PARENT_ID, ...exitSpansSample.keys())], - }, - }, - size: exitSpansSample.size, - fields: [...requiredFields, ...optionalFields], - }); + def id = $('span.id', null); + if (id == null) { + id = $('transaction.id', null); + } + + // Ensure same event isn't processed twice + if (id != null && !state.eventsById.containsKey(id)) { + def copy = new HashMap(); + copy.id = id; + + for(key in state.fieldsToCopy) { + def value = $(key, null); + if (value != null) { + copy[key] = value; + } + } + + state.eventsById[id] = copy; + state.docCount++; + } + `, + }, + combine_script: { + lang: 'painless', + source: `return state;`, + }, + reduce_script: { + lang: 'painless', + source: ` + def getDestination(def event) { + def destination = new HashMap(); + destination['span.destination.service.resource'] = event['span.destination.service.resource']; + destination['span.type'] = event['span.type']; + destination['span.subtype'] = event['span.subtype']; + return destination; + } + + def processAndReturnEvent(def context, def eventId) { + def stack = new Stack(); + def reprocessQueue = new LinkedList(); + + // Avoid reprocessing the same event + def visited = new HashSet(); + + stack.push(eventId); + + while (!stack.isEmpty()) { + def currentEventId = stack.pop(); + def event = context.eventsById.get(currentEventId); + + if (event == null || context.processedEvents.get(currentEventId) != null) { + continue; + } + visited.add(currentEventId); + + def service = new HashMap(); + service['service.name'] = event['service.name']; + service['service.environment'] = event['service.environment']; + service['agent.name'] = event['agent.name']; + + def basePath = new ArrayList(); + def parentId = event['parent.id']; + + if (parentId != null && !parentId.equals(currentEventId)) { + def parent = context.processedEvents.get(parentId); + + if (parent == null) { + + // Only adds the parentId to the stack if it hasn't been visited to prevent infinite loop scenarios + // if the parent is null, it means it hasn't been processed yet or it could also mean that the current event + // doesn't have a parent, in which case we should skip it + if (!visited.contains(parentId)) { + stack.push(parentId); + // Add currentEventId to be reprocessed once its parent is processed + reprocessQueue.add(currentEventId); + } + + + continue; + } - const destinationsBySpanId = new Map(exitSpansSample); + // copy the path from the parent + basePath.addAll(parent.path); + // flag parent path for removal, as it has children + context.locationsToRemove.add(parent.path); + + // if the parent has 'span.destination.service.resource' set, and the service is different, we've discovered a service + if (parent['span.destination.service.resource'] != null + && parent['span.destination.service.resource'] != "" + && (parent['service.name'] != event['service.name'] + || parent['service.environment'] != event['service.environment']) + ) { + def parentDestination = getDestination(parent); + context.externalToServiceMap.put(parentDestination, service); + } + } + + def lastLocation = basePath.size() > 0 ? basePath[basePath.size() - 1] : null; + def currentLocation = service; + + // only add the current location to the path if it's different from the last one + if (lastLocation == null || !lastLocation.equals(currentLocation)) { + basePath.add(currentLocation); + } + + // if there is an outgoing span, create a new path + if (event['span.destination.service.resource'] != null + && !event['span.destination.service.resource'].equals("")) { - servicesResponse.hits.hits.forEach((hit) => { - const transaction = unflattenKnownApmEventFields(hit.fields, [...requiredFields]); + def outgoingLocation = getDestination(event); + def outgoingPath = new ArrayList(basePath); + outgoingPath.add(outgoingLocation); + context.paths.add(outgoingPath); + } + + event.path = basePath; + context.processedEvents[currentEventId] = event; - const spanId = transaction.parent.id; + // reprocess events which were waiting for their parents to be processed + while (!reprocessQueue.isEmpty()) { + stack.push(reprocessQueue.remove()); + } + } - const destination = destinationsBySpanId.get(spanId); - if (destination) { - destinationsBySpanId.set(spanId, { - ...destination, - destinationService: { - agentName: transaction.agent.name, - serviceEnvironment: transaction.service.environment, - serviceName: transaction.service.name, + return null; + } + + def context = new HashMap(); + + context.processedEvents = new HashMap(); + context.eventsById = new HashMap(); + context.paths = new HashSet(); + context.externalToServiceMap = new HashMap(); + context.locationsToRemove = new HashSet(); + + for (state in states) { + context.eventsById.putAll(state.eventsById); + state.eventsById.clear(); + } + + states.clear(); + + for (entry in context.eventsById.entrySet()) { + processAndReturnEvent(context, entry.getKey()); + } + + context.processedEvents.clear(); + context.eventsById.clear(); + + def response = new HashMap(); + response.paths = new HashSet(); + response.discoveredServices = new HashSet(); + + for (foundPath in context.paths) { + if (!context.locationsToRemove.contains(foundPath)) { + response.paths.add(foundPath); + } + } + + context.locationsToRemove.clear(); + context.paths.clear(); + + for (entry in context.externalToServiceMap.entrySet()) { + def map = new HashMap(); + map.from = entry.getKey(); + map.to = entry.getValue(); + response.discoveredServices.add(map); + } + + context.externalToServiceMap.clear(); + + return response; + `, }, - }); - } - }); + }, + } as const, + }; + + const serviceMapParamsWithAggs = { + ...serviceMapParams, + size: 1, + terminate_after: numDocsPerShardAllowed, + aggs: serviceMapAggs, + }; + + const serviceMapFromTraceIdsScriptResponse = await apmEventClient.search( + 'get_service_paths_from_trace_ids', + serviceMapParamsWithAggs + ); - return Array.from(destinationsBySpanId.values()); + return serviceMapFromTraceIdsScriptResponse as { + aggregations?: { + service_map: { + value: { + paths: ConnectionNodeLegacy[][]; + discoveredServices: ExitSpanDestinationLegacy[]; + }; + }; + }; + }; } diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map.ts b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map.ts index 6d76c0545b464..3aa26f3349f39 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map.ts @@ -6,15 +6,22 @@ */ import type { Logger } from '@kbn/core/server'; -import type { ServiceMapResponse } from '../../../common/service_map'; +import { chunk } from 'lodash'; +import type { + Connection, + ExitSpanDestination, + ServiceMapSpan, +} from '../../../common/service_map/types'; +import { getServiceMapNodes, type ServiceMapResponse } from '../../../common/service_map'; import type { APMConfig } from '../..'; import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import type { MlClient } from '../../lib/helpers/get_ml_client'; import { withApmSpan } from '../../utils/with_apm_span'; import { getTraceSampleIds } from './get_trace_sample_ids'; -import { fetchPathsFromTraceIds } from './fetch_service_paths_from_trace_ids'; import { DEFAULT_ANOMALIES, getServiceAnomalies } from './get_service_anomalies'; import { getServiceStats } from './get_service_stats'; +import { getServiceMapFromTraceIds } from './get_service_map_from_trace_ids'; +import { fetchExitSpanSamplesFromTraceIds } from './fetch_exit_span_samples'; export interface IEnvOptions { mlClient?: MlClient; @@ -28,6 +35,7 @@ export interface IEnvOptions { end: number; serviceGroupKuery?: string; kuery?: string; + serviceMapV2Enabled?: boolean; } async function getConnectionData({ @@ -40,7 +48,13 @@ async function getConnectionData({ serviceGroupKuery, kuery, logger, -}: IEnvOptions) { + serviceMapV2Enabled = false, +}: IEnvOptions): Promise< + { tracesCount: number } & ( + | { connections: Connection[]; discoveredServices: ExitSpanDestination[] } + | { spans: ServiceMapSpan[] } + ) +> { return withApmSpan('get_service_map_connections', async () => { logger.debug('Getting trace sample IDs'); const { traceIds } = await getTraceSampleIds({ @@ -56,19 +70,57 @@ async function getConnectionData({ logger.debug(`Found ${traceIds.length} traces to inspect`); - const spans = await withApmSpan('get_service_map_spans_and_transactions_from_traces', () => - fetchPathsFromTraceIds({ - apmEventClient, - traceIds, - start, - end, - logger, - }) - ); + if (serviceMapV2Enabled) { + const spans = await withApmSpan( + 'get_service_map_exit_spans_and_transactions_from_traces', + () => + fetchExitSpanSamplesFromTraceIds({ + apmEventClient, + traceIds, + start, + end, + }) + ); + + return { + tracesCount: traceIds.length, + spans, + }; + } + + const chunkedResponses = await withApmSpan('get_service_paths_from_all_trace_ids', () => { + const chunks = chunk(traceIds, config.serviceMapMaxTracesPerRequest); + return Promise.all( + chunks.map((traceIdsChunk) => + getServiceMapFromTraceIds({ + apmEventClient, + traceIds: traceIdsChunk, + start, + end, + terminateAfter: config.serviceMapTerminateAfter, + serviceMapMaxAllowableBytes: config.serviceMapMaxAllowableBytes, + numOfRequests: chunks.length, + logger, + }) + ) + ); + }); + + logger.debug('Received chunk responses'); + + const mergedResponses = chunkedResponses.reduce((prev, current) => { + return { + connections: prev.connections.concat(current.connections), + discoveredServices: prev.discoveredServices.concat(current.discoveredServices), + }; + }); + + logger.debug('Merged responses'); return { + connections: mergedResponses.connections, + discoveredServices: mergedResponses.discoveredServices, tracesCount: traceIds.length, - spans, }; }); } @@ -96,11 +148,26 @@ export function getServiceMap( logger.debug('Received and parsed all responses'); - return { - spans: connectionData.spans, - tracesCount: connectionData.tracesCount, + if ('spans' in connectionData) { + return { + spans: connectionData.spans, + tracesCount: connectionData.tracesCount, + servicesData, + anomalies, + }; + } + + const serviceMapNodes = getServiceMapNodes({ + connections: connectionData.connections, + exitSpanDestinations: connectionData.discoveredServices, servicesData, anomalies, + }); + + return { + ...serviceMapNodes, + tracesCount: connectionData.tracesCount, + nodesCount: serviceMapNodes.nodesCount, }; }); } diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map_from_trace_ids.ts b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map_from_trace_ids.ts new file mode 100644 index 0000000000000..fd163f0fbf247 --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map_from_trace_ids.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; + +import type { ExitSpanDestination } from '../../../common/service_map/types'; +import { getConnections, getLegacyNodeId } from '../../../common/service_map'; +import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; +import { fetchServicePathsFromTraceIds } from './fetch_service_paths_from_trace_ids'; + +export async function getServiceMapFromTraceIds({ + apmEventClient, + traceIds, + start, + end, + terminateAfter, + serviceMapMaxAllowableBytes, + numOfRequests, + logger, +}: { + apmEventClient: APMEventClient; + traceIds: string[]; + start: number; + end: number; + terminateAfter: number; + serviceMapMaxAllowableBytes: number; + numOfRequests: number; + logger: Logger; +}) { + const serviceMapFromTraceIdsScriptResponse = await fetchServicePathsFromTraceIds({ + apmEventClient, + traceIds, + start, + end, + terminateAfter, + serviceMapMaxAllowableBytes, + numOfRequests, + }); + + logger.debug('Received scripted metric agg response'); + + const serviceMapScriptedAggValue = + serviceMapFromTraceIdsScriptResponse.aggregations?.service_map.value; + + return { + connections: getConnections(serviceMapScriptedAggValue?.paths), + discoveredServices: (serviceMapScriptedAggValue?.discoveredServices ?? []).map( + (service) => + ({ + from: { ...service.from, id: getLegacyNodeId(service.from) }, + to: { ...service.to, id: getLegacyNodeId(service.to) }, + } as ExitSpanDestination) + ), + }; +} diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/mock_responses/group_resource_nodes_grouped.json b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/mock_responses/group_resource_nodes_grouped.json deleted file mode 100644 index 94c508fe90230..0000000000000 --- a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/mock_responses/group_resource_nodes_grouped.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "elements": [ - { - "data": { - "id": "opbeans-rum", - "service.environment": "test", - "service.name": "opbeans-rum", - "agent.name": "rum-js" - } - }, - { - "data": { - "source": "opbeans-rum", - "target": "opbeans-node", - "id": "opbeans-rum~>opbeans-node" - } - }, - { - "data": { - "id": "opbeans-node", - "service.environment": "test", - "service.name": "opbeans-node", - "agent.name": "nodejs" - } - }, - { - "data": { - "source": "opbeans-node", - "target": "postgresql", - "id": "opbeans-node~>postgresql" - } - }, - { - "data": { - "id": "postgresql", - "span.subtype": "postgresql", - "span.destination.service.resource": "postgresql", - "span.type": "db", - "label": "postgresql" - } - }, - { - "data": { - "id": "elastic-co-rum-test", - "service.name": "elastic-co-rum-test", - "agent.name": "rum-js" - } - }, - { - "data": { - "id": "elastic-co-frontend", - "service.name": "elastic-co-frontend", - "agent.name": "rum-js" - } - }, - { - "data": { - "source": "elastic-co-frontend", - "target": "0.cdn.example.com:443", - "id": "elastic-co-frontend~>0.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-frontend", - "target": "resourceGroup{elastic-co-frontend;elastic-co-rum-test}", - "id": "elastic-co-frontend~>resourceGroup{elastic-co-frontend;elastic-co-rum-test}" - } - }, - { - "data": { - "source": "elastic-co-rum-test", - "target": "resourceGroup{elastic-co-frontend;elastic-co-rum-test}", - "id": "elastic-co-rum-test~>resourceGroup{elastic-co-frontend;elastic-co-rum-test}" - } - }, - { - "data": { - "source": "elastic-co-rum-test", - "target": "6.cdn.example.com:443", - "id": "elastic-co-rum-test~>6.cdn.example.com:443" - } - }, - { - "data": { - "id": "resourceGroup{elastic-co-frontend;elastic-co-rum-test}", - "span.type": "external", - "label": "5 resources", - "groupedConnections": [ - { - "label": "1.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "1.cdn.example.com:443" - }, - { - "label": "2.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "2.cdn.example.com:443" - }, - { - "label": "3.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "3.cdn.example.com:443" - }, - { - "label": "4.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "4.cdn.example.com:443" - }, - { - "label": "5.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "5.cdn.example.com:443" - } - ] - } - }, - { - "data": { - "id": "0.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "0.cdn.example.com:443" - } - }, - { - "data": { - "id": "6.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "6.cdn.example.com:443" - } - } - ] -} diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/mock_responses/group_resource_nodes_pregrouped.json b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/mock_responses/group_resource_nodes_pregrouped.json deleted file mode 100644 index 58469f607ac13..0000000000000 --- a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/mock_responses/group_resource_nodes_pregrouped.json +++ /dev/null @@ -1,204 +0,0 @@ -{ - "elements": [ - { - "data": { - "id": "opbeans-rum", - "service.environment": "test", - "service.name": "opbeans-rum", - "agent.name": "rum-js" - } - }, - { - "data": { - "source": "opbeans-rum", - "target": "opbeans-node", - "id": "opbeans-rum~>opbeans-node" - } - }, - { - "data": { - "id": "opbeans-node", - "service.environment": "test", - "service.name": "opbeans-node", - "agent.name": "nodejs" - } - }, - { - "data": { - "source": "opbeans-node", - "target": "postgresql", - "id": "opbeans-node~>postgresql" - } - }, - { - "data": { - "id": "postgresql", - "span.subtype": "postgresql", - "span.destination.service.resource": "postgresql", - "span.type": "db", - "label": "postgresql" - } - }, - { - "data": { - "id": "elastic-co-rum-test", - "service.name": "elastic-co-rum-test", - "agent.name": "rum-js" - } - }, - { - "data": { - "id": "elastic-co-frontend", - "service.name": "elastic-co-frontend", - "agent.name": "rum-js" - } - }, - { - "data": { - "source": "elastic-co-frontend", - "target": "0.cdn.example.com:443", - "id": "elastic-co-frontend~>0.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-frontend", - "target": "1.cdn.example.com:443", - "id": "elastic-co-frontend~>1.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-frontend", - "target": "2.cdn.example.com:443", - "id": "elastic-co-frontend~>2.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-frontend", - "target": "3.cdn.example.com:443", - "id": "elastic-co-frontend~>3.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-frontend", - "target": "4.cdn.example.com:443", - "id": "elastic-co-frontend~>4.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-frontend", - "target": "5.cdn.example.com:443", - "id": "elastic-co-frontend~>5.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-rum-test", - "target": "1.cdn.example.com:443", - "id": "elastic-co-rum-test~>1.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-rum-test", - "target": "2.cdn.example.com:443", - "id": "elastic-co-rum-test~>2.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-rum-test", - "target": "3.cdn.example.com:443", - "id": "elastic-co-rum-test~>3.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-rum-test", - "target": "4.cdn.example.com:443", - "id": "elastic-co-rum-test~>4.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-rum-test", - "target": "5.cdn.example.com:443", - "id": "elastic-co-rum-test~>5.cdn.example.com:443" - } - }, - { - "data": { - "source": "elastic-co-rum-test", - "target": "6.cdn.example.com:443", - "id": "elastic-co-rum-test~>6.cdn.example.com:443" - } - }, - { - "data": { - "id": "0.cdn.example.com:443", - "label": "0.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "0.cdn.example.com:443" - } - }, - { - "data": { - "id": "1.cdn.example.com:443", - "label": "1.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "1.cdn.example.com:443" - } - }, - { - "data": { - "id": "2.cdn.example.com:443", - "label": "2.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "2.cdn.example.com:443" - } - }, - { - "data": { - "id": "3.cdn.example.com:443", - "label": "3.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "3.cdn.example.com:443" - } - }, - { - "data": { - "id": "4.cdn.example.com:443", - "label": "4.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "4.cdn.example.com:443" - } - }, - { - "data": { - "id": "5.cdn.example.com:443", - "label": "5.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "5.cdn.example.com:443" - } - }, - { - "data": { - "id": "6.cdn.example.com:443", - "label": "6.cdn.example.com:443", - "span.type": "external", - "span.subtype": "http", - "span.destination.service.resource": "6.cdn.example.com:443" - } - } - ] -} diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/route.ts b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/route.ts index 646e42db946f8..1c0b237961100 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/route.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/route.ts @@ -7,7 +7,10 @@ import Boom from '@hapi/boom'; import * as t from 'io-ts'; -import { apmServiceGroupMaxNumberOfServices } from '@kbn/observability-plugin/common'; +import { + apmEnableServiceMapV2, + apmServiceGroupMaxNumberOfServices, +} from '@kbn/observability-plugin/common'; import type { ServiceMapResponse } from '../../../common/service_map'; import { isActivePlatinumLicense } from '../../../common/license_check'; import { invalidLicenseMessage } from '../../../common/service_map/utils'; @@ -64,17 +67,19 @@ const serviceMapRoute = createApmServerRoute({ uiSettings: { client: uiSettingsClient }, } = await context.core; - const [mlClient, apmEventClient, serviceGroup, maxNumberOfServices] = await Promise.all([ - getMlClient(resources), - getApmEventClient(resources), - serviceGroupId - ? getServiceGroup({ - savedObjectsClient, - serviceGroupId, - }) - : Promise.resolve(null), - uiSettingsClient.get(apmServiceGroupMaxNumberOfServices), - ]); + const [mlClient, apmEventClient, serviceGroup, maxNumberOfServices, serviceMapV2Enabled] = + await Promise.all([ + getMlClient(resources), + getApmEventClient(resources), + serviceGroupId + ? getServiceGroup({ + savedObjectsClient, + serviceGroupId, + }) + : Promise.resolve(null), + uiSettingsClient.get(apmServiceGroupMaxNumberOfServices), + uiSettingsClient.get(apmEnableServiceMapV2), + ]); const searchAggregatedTransactions = await getSearchTransactionsEvents({ apmEventClient, @@ -97,6 +102,7 @@ const serviceMapRoute = createApmServerRoute({ maxNumberOfServices, serviceGroupKuery: serviceGroup?.kuery, kuery, + serviceMapV2Enabled, }); }, }); diff --git a/x-pack/solutions/observability/plugins/observability/common/index.ts b/x-pack/solutions/observability/plugins/observability/common/index.ts index 327e6729426ce..7231dfde3e3c8 100644 --- a/x-pack/solutions/observability/plugins/observability/common/index.ts +++ b/x-pack/solutions/observability/plugins/observability/common/index.ts @@ -34,6 +34,7 @@ export { enableInfrastructureAssetCustomDashboards, enableAwsLambdaMetrics, enableAgentExplorerView, + apmEnableServiceMapV2, apmEnableTableSearchBar, entityCentricExperience, apmAWSLambdaPriceFactor, diff --git a/x-pack/solutions/observability/plugins/observability/common/ui_settings_keys.ts b/x-pack/solutions/observability/plugins/observability/common/ui_settings_keys.ts index 7025c120cae5d..5c29479c760c7 100644 --- a/x-pack/solutions/observability/plugins/observability/common/ui_settings_keys.ts +++ b/x-pack/solutions/observability/plugins/observability/common/ui_settings_keys.ts @@ -23,6 +23,7 @@ export const enableInfrastructureAssetCustomDashboards = export const enableAwsLambdaMetrics = 'observability:enableAwsLambdaMetrics'; export const enableAgentExplorerView = 'observability:apmAgentExplorerView'; export const apmEnableTableSearchBar = 'observability:apmEnableTableSearchBar'; +export const apmEnableServiceMapV2 = 'observability:apmEnableServiceMapV2'; export const entityCentricExperience = 'observability:entityCentricExperience'; export const apmEnableServiceInventoryTableSearchBar = 'observability:apmEnableServiceInventoryTableSearchBar'; diff --git a/x-pack/solutions/observability/plugins/observability/server/ui_settings.ts b/x-pack/solutions/observability/plugins/observability/server/ui_settings.ts index ed79e7e66787e..2ff8c36cd6338 100644 --- a/x-pack/solutions/observability/plugins/observability/server/ui_settings.ts +++ b/x-pack/solutions/observability/plugins/observability/server/ui_settings.ts @@ -45,6 +45,7 @@ import { apmEnableServiceInventoryTableSearchBar, profilingFetchTopNFunctionsFromStacktraces, searchExcludedDataTiers, + apmEnableServiceMapV2, } from '../common/ui_settings_keys'; const betaLabel = i18n.translate('xpack.observability.uiSettings.betaLabel', { @@ -365,6 +366,23 @@ export const uiSettings: Record = { type: 'boolean', solution: 'oblt', }, + [apmEnableServiceMapV2]: { + category: [observabilityFeatureId], + name: i18n.translate('xpack.observability.apmEnableServiceMapV2', { + defaultMessage: 'Service Map API V2', + }), + description: i18n.translate('xpack.observability.apmEnableServiceMapV2Description', { + defaultMessage: '{technicalPreviewLabel} Enables the new service map API.', + values: { + technicalPreviewLabel: `[${technicalPreviewLabel}]`, + }, + }), + schema: schema.boolean(), + value: false, + requiresPageReload: false, + type: 'boolean', + solution: 'oblt', + }, [apmAWSLambdaPriceFactor]: { category: [observabilityFeatureId], name: i18n.translate('xpack.observability.apmAWSLambdaPricePerGbSeconds', {