From 7240f57c941532a2083697e90e0d30907f5a0d71 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 20 Feb 2025 14:04:47 -0500 Subject: [PATCH 01/21] [Fleet] Use streaming for package install instead of huge assetsMap --- .../shared/fleet/common/types/models/epm.ts | 7 +- .../services/epm/archive/archive_iterator.ts | 9 +- .../elasticsearch/datastream_ilm/install.ts | 23 ++++- .../services/epm/elasticsearch/ilm/install.ts | 18 +++- .../ingest_pipeline/install.test.ts | 19 +++- .../elasticsearch/ingest_pipeline/install.ts | 17 +++- .../epm/elasticsearch/ml_model/install.ts | 16 +++- .../epm/elasticsearch/template/install.ts | 32 +++++-- .../epm/elasticsearch/transform/install.ts | 35 +++++-- .../transform/legacy_transforms.test.ts | 33 +++++++ .../transform/transforms.test.ts | 93 +++++++++++++++++++ 11 files changed, 270 insertions(+), 32 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts b/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts index 29b9c45e0e71d..b28677744cff8 100644 --- a/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts +++ b/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts @@ -132,7 +132,10 @@ export interface ArchiveEntry { } export interface ArchiveIterator { - traverseEntries: (onEntry: (entry: ArchiveEntry) => Promise) => Promise; + traverseEntries: ( + onEntry: (entry: ArchiveEntry) => Promise, + readBuffer?: (path: string) => boolean + ) => Promise; getPaths: () => Promise; } @@ -142,7 +145,7 @@ export interface PackageInstallContext { * @deprecated Use `archiveIterator` to access the package archive entries * without loading them all into memory at once. */ - assetsMap: AssetsMap; + assetsMap: AssetsMap; // TODO remove paths: string[]; archiveIterator: ArchiveIterator; } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/archive_iterator.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/archive_iterator.ts index bf7fda919e9df..4775add398392 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/archive_iterator.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/archive_iterator.ts @@ -74,10 +74,15 @@ export const createArchiveIterator = ( */ export const createArchiveIteratorFromMap = (assetsMap: AssetsMap): ArchiveIterator => { const traverseEntries = async ( - onEntry: (entry: ArchiveEntry) => Promise + onEntry: (entry: ArchiveEntry) => Promise, + readBuffer?: (path: string) => boolean ): Promise => { for (const [path, buffer] of assetsMap) { - await onEntry({ path, buffer }); + if (readBuffer && !readBuffer(path)) { + await onEntry({ path }); + } else { + await onEntry({ path, buffer }); + } } }; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts index eafda9a99e4fc..90f7201112381 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts @@ -12,7 +12,11 @@ import { ElasticsearchAssetType, type PackageInstallContext, } from '../../../../../common/types/models'; -import type { EsAssetReference, RegistryDataStream } from '../../../../../common/types/models'; +import type { + AssetsMap, + EsAssetReference, + RegistryDataStream, +} from '../../../../../common/types/models'; import { updateEsAssetReferences } from '../../packages/es_assets_reference'; import { getAssetFromAssetsMap } from '../../archive'; @@ -40,7 +44,7 @@ export const installIlmForDataStream = async ( logger: Logger, esReferences: EsAssetReference[] ) => { - const { packageInfo: registryPackage, paths, assetsMap } = packageInstallContext; + const { packageInfo: registryPackage, paths } = packageInstallContext; const previousInstalledIlmEsAssets = esReferences.filter( ({ type }) => type === ElasticsearchAssetType.dataStreamIlmPolicy ); @@ -72,6 +76,19 @@ export const installIlmForDataStream = async ( }; const dataStreamIlmPaths = paths.filter((path) => isDataStreamIlm(path)); + + const dataStreamIlmAssetsMap: AssetsMap = new Map(); + await packageInstallContext.archiveIterator.traverseEntries( + async (entry) => { + if (!entry.buffer) { + return; + } + + dataStreamIlmAssetsMap.set(entry.path, entry.buffer); + }, + (path) => dataStreamIlmPaths.includes(path) + ); + let installedIlms: EsAssetReference[] = []; if (dataStreamIlmPaths.length > 0) { const ilmPathDatasets = dataStreams.reduce((acc, dataStream) => { @@ -103,7 +120,7 @@ export const installIlmForDataStream = async ( const ilmInstallations: IlmInstallation[] = ilmPathDatasets.map( (ilmPathDataset: IlmPathDataset) => { const content = JSON.parse( - getAssetFromAssetsMap(assetsMap, ilmPathDataset.path).toString('utf-8') + getAssetFromAssetsMap(dataStreamIlmAssetsMap, ilmPathDataset.path).toString('utf-8') ); content.policy._meta = getESAssetMetadata({ packageName: registryPackage.name }); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ilm/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ilm/install.ts index 1203ec02ba9ac..eb40ae1590524 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ilm/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ilm/install.ts @@ -16,7 +16,7 @@ import { updateEsAssetReferences } from '../../packages/es_assets_reference'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; import { PackageInvalidArchiveError } from '../../../../errors'; -import type { PackageInstallContext } from '../../../../../common/types'; +import type { AssetsMap, PackageInstallContext } from '../../../../../common/types'; import { MAX_CONCURRENT_ILM_POLICIES_OPERATIONS } from '../../../../constants'; export async function installILMPolicy( @@ -30,10 +30,20 @@ export async function installILMPolicy( const ilmPaths = packageInstallContext.paths.filter((path) => isILMPolicy(path)); if (!ilmPaths.length) return esReferences; + const ilmAssetsMap: AssetsMap = new Map(); + await packageInstallContext.archiveIterator.traverseEntries( + async (entry) => { + if (!entry.buffer) { + return; + } + + ilmAssetsMap.set(entry.path, entry.buffer); + }, + (path) => ilmPaths.includes(path) + ); + const ilmPolicies = ilmPaths.map((path) => { - const body = JSON.parse( - getAssetFromAssetsMap(packageInstallContext.assetsMap, path).toString('utf-8') - ); + const body = JSON.parse(getAssetFromAssetsMap(ilmAssetsMap, path).toString('utf-8')); body.policy._meta = getESAssetMetadata({ packageName: packageInfo.name }); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts index fb12d58878d1d..7caeb2d35c225 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts @@ -7,6 +7,8 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { loggerMock } from '@kbn/logging-mocks'; +import { createArchiveIteratorFromMap } from '../../archive/archive_iterator'; + import { prepareToInstallPipelines } from './install'; jest.mock('../../archive/cache'); @@ -25,10 +27,11 @@ describe('Install pipeline tests', () => { path: '/datasettest', }, ], - }, + } as any, paths: [], assetsMap: new Map(), - } as any); + archiveIterator: createArchiveIteratorFromMap(new Map()), + }); expect(res.assetsToAdd).toEqual([{ id: 'logs-datasettest-1.0.0', type: 'ingest_pipeline' }]); const esClient = elasticsearchClientMock.createInternalClient(); @@ -74,6 +77,18 @@ describe('Install pipeline tests', () => { Buffer.from('description: test'), ], ]), + archiveIterator: createArchiveIteratorFromMap( + new Map([ + [ + 'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/default.yml', + Buffer.from('description: test'), + ], + [ + 'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/standard.yml', + Buffer.from('description: test'), + ], + ]) + ), } as any); expect(res.assetsToAdd).toEqual([ { id: 'logs-datasettest-1.0.0', type: 'ingest_pipeline' }, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts index 51162ac2c6335..47c060cf31caf 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -25,7 +25,7 @@ import { FLEET_EVENT_INGESTED_PIPELINE_CONTENT, } from '../../../../constants'; import { getPipelineNameForDatastream } from '../../../../../common/services'; -import type { ArchiveEntry, PackageInstallContext } from '../../../../../common/types'; +import type { ArchiveEntry, AssetsMap, PackageInstallContext } from '../../../../../common/types'; import { appendMetadataToIngestPipeline } from '../meta'; import { retryTransientEsErrors } from '../retry'; @@ -157,6 +157,19 @@ export async function installAllPipelines({ > = []; const substitutions: RewriteSubstitution[] = []; + const pipelineAssetsMap: AssetsMap = new Map(); + + await packageInstallContext.archiveIterator.traverseEntries( + async (entry) => { + if (!entry.buffer) { + return; + } + + pipelineAssetsMap.set(entry.path, entry.buffer); + }, + (path) => pipelinePaths.includes(path) + ); + let datastreamPipelineCreated = false; pipelinePaths.forEach((path) => { const { name, extension } = getNameAndExtension(path); @@ -169,7 +182,7 @@ export async function installAllPipelines({ dataStream, packageVersion: packageInstallContext.packageInfo.version, }); - const content = getAssetFromAssetsMap(packageInstallContext.assetsMap, path).toString('utf-8'); + const content = getAssetFromAssetsMap(pipelineAssetsMap, path).toString('utf-8'); pipelinesInfos.push({ nameForInstallation, shouldInstallCustomPipelines: dataStream && isMainPipeline, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ml_model/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ml_model/install.ts index 5057e5cfd68fe..ce0f3c61454c7 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ml_model/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ml_model/install.ts @@ -13,7 +13,7 @@ import { ElasticsearchAssetType, type PackageInstallContext, } from '../../../../../common/types/models'; -import type { EsAssetReference } from '../../../../../common/types/models'; +import type { AssetsMap, EsAssetReference } from '../../../../../common/types/models'; import { retryTransientEsErrors } from '../retry'; @@ -34,9 +34,19 @@ export const installMlModel = async ( const mlModelPath = packageInstallContext.paths.find((path) => isMlModel(path)); if (mlModelPath !== undefined) { - const content = getAssetFromAssetsMap(packageInstallContext.assetsMap, mlModelPath).toString( - 'utf-8' + const mlModelAssetsMap: AssetsMap = new Map(); + await packageInstallContext.archiveIterator.traverseEntries( + async (entry) => { + if (!entry.buffer) { + return; + } + + mlModelAssetsMap.set(entry.path, entry.buffer); + }, + (path) => path === mlModelPath ); + + const content = getAssetFromAssetsMap(mlModelAssetsMap, mlModelPath).toString('utf-8'); const pathParts = mlModelPath.split('/'); const modelId = pathParts[pathParts.length - 1].replace('.json', ''); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.ts index f17a05a6837aa..a85329ee68ca2 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.ts @@ -48,7 +48,7 @@ import { forEachMappings, } from '../../../experimental_datastream_features_helper'; import { appContextService } from '../../../app_context'; -import type { PackageInstallContext } from '../../../../../common/types'; +import type { AssetsMap, PackageInstallContext } from '../../../../../common/types'; import { generateMappings, @@ -131,14 +131,23 @@ const installPreBuiltTemplates = async ( ) => { const templatePaths = packageInstallContext.paths.filter((path) => isTemplate(path)); try { + const templateAssetsMap: AssetsMap = new Map(); + await packageInstallContext.archiveIterator.traverseEntries( + async (entry) => { + if (!entry.buffer) { + return; + } + + templateAssetsMap.set(entry.path, entry.buffer); + }, + (path) => templatePaths.includes(path) + ); await pMap( templatePaths, async (path) => { const { file } = getPathParts(path); const templateName = file.substr(0, file.lastIndexOf('.')); - const content = JSON.parse( - getAssetFromAssetsMap(packageInstallContext.assetsMap, path).toString('utf8') - ); + const content = JSON.parse(getAssetFromAssetsMap(templateAssetsMap, path).toString('utf8')); const esClientParams = { name: templateName, body: content }; const esClientRequestOptions = { ignore: [404] }; @@ -175,14 +184,23 @@ const installPreBuiltComponentTemplates = async ( ) => { const templatePaths = packageInstallContext.paths.filter((path) => isComponentTemplate(path)); try { + const templateAssetsMap: AssetsMap = new Map(); + await packageInstallContext.archiveIterator.traverseEntries( + async (entry) => { + if (!entry.buffer) { + return; + } + + templateAssetsMap.set(entry.path, entry.buffer); + }, + (path) => templatePaths.includes(path) + ); await pMap( templatePaths, async (path) => { const { file } = getPathParts(path); const templateName = file.substr(0, file.lastIndexOf('.')); - const content = JSON.parse( - getAssetFromAssetsMap(packageInstallContext.assetsMap, path).toString('utf8') - ); + const content = JSON.parse(getAssetFromAssetsMap(templateAssetsMap, path).toString('utf8')); const esClientParams = { name: templateName, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts index 85ae293455c9e..1d1e0c69f3d19 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -40,6 +40,7 @@ import type { ESAssetMetadata, IndexTemplate, RegistryElasticsearch, + AssetsMap, } from '../../../../../common/types/models'; import { getInstallation } from '../../packages'; import { retryTransientEsErrors } from '../retry'; @@ -93,6 +94,17 @@ const installLegacyTransformsAssets = async ( let installedTransforms: EsAssetReference[] = []; if (transformPaths.length > 0) { + const transformAssetsMap: AssetsMap = new Map(); + await packageInstallContext.archiveIterator.traverseEntries( + async (entry) => { + if (!entry.buffer) { + return; + } + + transformAssetsMap.set(entry.path, entry.buffer); + }, + (path) => transformPaths.includes(path) + ); const transformRefs = transformPaths.reduce((acc, path) => { acc.push({ id: getLegacyTransformNameForInstallation( @@ -117,9 +129,7 @@ const installLegacyTransformsAssets = async ( ); const transforms: TransformInstallation[] = transformPaths.map((path: string) => { - const content = JSON.parse( - getAssetFromAssetsMap(packageInstallContext.assetsMap, path).toString('utf-8') - ); + const content = JSON.parse(getAssetFromAssetsMap(transformAssetsMap, path).toString('utf-8')); content._meta = getESAssetMetadata({ packageName: packageInstallContext.packageInfo.name }); return { @@ -153,7 +163,7 @@ const installLegacyTransformsAssets = async ( return { installedTransforms, esReferences }; }; -const processTransformAssetsPerModule = ( +const processTransformAssetsPerModule = async ( packageInstallContext: PackageInstallContext, installNameSuffix: string, transformPaths: string[], @@ -161,7 +171,7 @@ const processTransformAssetsPerModule = ( force?: boolean, username?: string ) => { - const { assetsMap, packageInfo: installablePackage } = packageInstallContext; + const { packageInfo: installablePackage } = packageInstallContext; const transformsSpecifications = new Map(); const destinationIndexTemplates: DestinationIndexTemplateInstallation[] = []; const transforms: TransformInstallation[] = []; @@ -170,6 +180,17 @@ const processTransformAssetsPerModule = ( const transformsToRemoveWithDestIndex: EsAssetReference[] = []; const indicesToAddRefs: EsAssetReference[] = []; + const transformAssetsMap: AssetsMap = new Map(); + await packageInstallContext.archiveIterator.traverseEntries( + async (entry) => { + if (!entry.buffer) { + return; + } + + transformAssetsMap.set(entry.path, entry.buffer); + }, + (path) => transformPaths.includes(path) + ); transformPaths.forEach((path: string) => { const { transformModuleId, fileName } = getTransformFolderAndFileNames( installablePackage, @@ -182,7 +203,7 @@ const processTransformAssetsPerModule = ( } const packageAssets = transformsSpecifications.get(transformModuleId); - const content = load(getAssetFromAssetsMap(assetsMap, path).toString('utf-8')); + const content = load(getAssetFromAssetsMap(transformAssetsMap, path).toString('utf-8')); // Handling fields.yml and all other files within 'fields' folder if (fileName === TRANSFORM_SPECS_TYPES.FIELDS || isFields(path)) { @@ -441,7 +462,7 @@ const installTransformsAssets = async ( transformsSpecifications, transformsToRemove, transformsToRemoveWithDestIndex, - } = processTransformAssetsPerModule( + } = await processTransformAssetsPerModule( packageInstallContext, installNameSuffix, transformPaths, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/legacy_transforms.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/legacy_transforms.test.ts index 4b58dda7d0746..9e8aae71bb265 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/legacy_transforms.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/legacy_transforms.test.ts @@ -30,6 +30,7 @@ import type { PackageInstallContext } from '../../../../../common/types'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../constants'; import { installTransforms } from './install'; +import { createArchiveIteratorFromMap } from '../../archive/archive_iterator'; describe('test transform install with legacy schema', () => { let esClient: ReturnType; @@ -167,6 +168,22 @@ describe('test transform install with legacy schema', () => { Buffer.from('{"content": "data"}'), ], ]), + archiveIterator: createArchiveIteratorFromMap( + new Map([ + [ + 'endpoint-0.16.0-dev.0/data_stream/policy/elasticsearch/ingest_pipeline/default.json', + Buffer.from('{"content": "data"}'), + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata/default.json', + Buffer.from('{"content": "data"}'), + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json', + Buffer.from('{"content": "data"}'), + ], + ]) + ), } as unknown as PackageInstallContext, esClient, savedObjectsClient, @@ -337,6 +354,14 @@ describe('test transform install with legacy schema', () => { Buffer.from('{"content": "data"}'), ], ]), + archiveIterator: createArchiveIteratorFromMap( + new Map([ + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json', + Buffer.from('{"content": "data"}'), + ], + ]) + ), } as unknown as PackageInstallContext, esClient, savedObjectsClient, @@ -563,6 +588,14 @@ describe('test transform install with legacy schema', () => { Buffer.from('{"content": "data"}'), ], ]), + archiveIterator: createArchiveIteratorFromMap( + new Map([ + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json', + Buffer.from('{"content": "data"}'), + ], + ]) + ), } as unknown as PackageInstallContext, esClient, savedObjectsClient, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts index 4d927ca4cef69..43bed97901744 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts @@ -26,6 +26,7 @@ import { createAppContextStartContractMock } from '../../../../mocks'; import type { PackageInstallContext } from '../../../../../common/types'; import { installTransforms } from './install'; +import { createArchiveIteratorFromMap } from '../../archive/archive_iterator'; jest.mock('../../packages/get', () => { return { getInstallation: jest.fn(), getInstallationObject: jest.fn() }; @@ -292,6 +293,30 @@ _meta: sourceData.TRANSFORM, ], ]), + archiveIterator: createArchiveIteratorFromMap( + new Map([ + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/beats.yml', + Buffer.from(sourceData.BEATS_FIELDS), + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/agent.yml', + Buffer.from(sourceData.AGENT_FIELDS), + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', + Buffer.from(sourceData.FIELDS), + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', + Buffer.from(sourceData.MANIFEST), + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + Buffer.from(sourceData.TRANSFORM), + ], + ]) + ), } as unknown as PackageInstallContext, esClient, savedObjectsClient, @@ -586,6 +611,22 @@ _meta: Buffer.from(sourceData.TRANSFORM), ], ]), + archiveIterator: createArchiveIteratorFromMap( + new Map([ + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', + Buffer.from(sourceData.FIELDS), + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', + Buffer.from(sourceData.MANIFEST), + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + Buffer.from(sourceData.TRANSFORM), + ], + ]) + ), } as unknown as PackageInstallContext, esClient, savedObjectsClient, @@ -864,6 +905,18 @@ _meta: Buffer.from(sourceData.TRANSFORM), ], ]), + archiveIterator: createArchiveIteratorFromMap( + new Map([ + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', + Buffer.from(sourceData.FIELDS), + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + Buffer.from(sourceData.TRANSFORM), + ], + ]) as any + ), } as unknown as PackageInstallContext, esClient, savedObjectsClient, @@ -1092,6 +1145,18 @@ _meta: sourceData.TRANSFORM, ], ]), + archiveIterator: createArchiveIteratorFromMap( + new Map([ + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', + sourceData.MANIFEST, + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + sourceData.TRANSFORM, + ], + ]) as any + ), } as unknown as PackageInstallContext, esClient, savedObjectsClient, @@ -1196,6 +1261,18 @@ _meta: sourceData.TRANSFORM, ], ]), + archiveIterator: createArchiveIteratorFromMap( + new Map([ + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', + sourceData.MANIFEST, + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + sourceData.TRANSFORM, + ], + ]) as any + ), } as unknown as PackageInstallContext, esClient, savedObjectsClient, @@ -1295,6 +1372,22 @@ _meta: sourceData.TRANSFORM, ], ]), + archiveIterator: createArchiveIteratorFromMap( + new Map([ + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', + sourceData.FIELDS, + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', + sourceData.MANIFEST, + ], + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + sourceData.TRANSFORM, + ], + ]) as any + ), } as unknown as PackageInstallContext, esClient, savedObjectsClient, From 00df30bd3a97dc4f59cd87b2ecb1d80313082a3e Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 24 Feb 2025 09:41:15 -0500 Subject: [PATCH 02/21] fix tests --- .../transform/legacy_transforms.test.ts | 2 +- .../transform/transforms.test.ts | 2 +- .../services/epm/kibana/assets/install.ts | 154 ++++++------ .../epm/kibana/assets/tag_assets.test.ts | 226 ++++++++---------- .../services/epm/kibana/assets/tag_assets.ts | 41 ++-- 5 files changed, 192 insertions(+), 233 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/legacy_transforms.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/legacy_transforms.test.ts index 9e8aae71bb265..4e9f0598a0cc9 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/legacy_transforms.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/legacy_transforms.test.ts @@ -26,11 +26,11 @@ import { appContextService } from '../../../app_context'; import { getESAssetMetadata } from '../meta'; +import { createArchiveIteratorFromMap } from '../../archive/archive_iterator'; import type { PackageInstallContext } from '../../../../../common/types'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../constants'; import { installTransforms } from './install'; -import { createArchiveIteratorFromMap } from '../../archive/archive_iterator'; describe('test transform install with legacy schema', () => { let esClient: ReturnType; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts index 43bed97901744..7edf7f9c78c12 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts @@ -22,11 +22,11 @@ import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../constants'; import { getESAssetMetadata } from '../meta'; +import { createArchiveIteratorFromMap } from '../../archive/archive_iterator'; import { createAppContextStartContractMock } from '../../../../mocks'; import type { PackageInstallContext } from '../../../../../common/types'; import { installTransforms } from './install'; -import { createArchiveIteratorFromMap } from '../../archive/archive_iterator'; jest.mock('../../packages/get', () => { return { getInstallation: jest.fn(), getInstallationObject: jest.fn() }; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts index 5adf1708eb25a..4235b04d04983 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts @@ -19,7 +19,7 @@ import type { import { createListStream } from '@kbn/utils'; import { partition, chunk } from 'lodash'; -import { getAssetFromAssetsMap, getPathParts } from '../../archive'; +import { getPathParts } from '../../archive'; import { KibanaAssetType, KibanaSavedObjectType } from '../../../../types'; import type { AssetReference, Installation, PackageSpecTags } from '../../../../types'; import type { KibanaAssetReference, PackageInstallContext } from '../../../../../common/types'; @@ -28,7 +28,7 @@ import { getIndexPatternSavedObjects, makeManagedIndexPatternsGlobal, } from '../index_pattern/install'; -import { kibanaAssetsToAssetsRef, saveKibanaAssetsRefs } from '../../packages/install'; +import { saveKibanaAssetsRefs } from '../../packages/install'; import { deleteKibanaSavedObjectsAssets } from '../../packages/remove'; import { FleetError, KibanaSOReferenceError } from '../../../../errors'; import { withPackageSpan } from '../../packages/utils'; @@ -38,7 +38,7 @@ import { appContextService } from '../../..'; import { tagKibanaAssets } from './tag_assets'; import { getSpaceAwareSaveobjectsClients } from './saved_objects'; -const MAX_ASSETS_TO_INSTALL_IN_PARALLEL = 1000; +const MAX_ASSETS_TO_INSTALL_IN_PARALLEL = 100; type SavedObjectsImporterContract = Pick; const formatImportErrorsForLog = (errors: SavedObjectsImportFailure[]) => @@ -111,42 +111,47 @@ export async function installKibanaAssets(options: { savedObjectsImporter: SavedObjectsImporterContract; logger: Logger; pkgName: string; - kibanaAssets: Record; + kibanaAssetsArchiveIterator: ReturnType; }): Promise { - const { kibanaAssets, savedObjectsClient, savedObjectsImporter, logger } = options; + const { kibanaAssetsArchiveIterator, savedObjectsClient, savedObjectsImporter, logger } = options; - const assetsToInstall = Object.entries(kibanaAssets).flatMap(([assetType, assets]) => { - if (!validKibanaAssetTypes.has(assetType as KibanaAssetType)) { - return []; - } + // Todo check if we put this behind a condition + await installManagedIndexPattern({ + savedObjectsClient, + savedObjectsImporter, + }); - if (!assets.length) { - return []; - } + let assetsToInstall: ArchiveAsset[] = []; + let res: SavedObjectsImportSuccess[] = []; + async function flushAssetsToInstall() { + const installedAssets = await installKibanaSavedObjects({ + logger, + savedObjectsImporter, + kibanaAssets: assetsToInstall, + assetsChunkSize: MAX_ASSETS_TO_INSTALL_IN_PARALLEL, + }); + assetsToInstall = []; + res = [...res, ...installedAssets]; + } + await kibanaAssetsArchiveIterator(async ({ assetType, asset }) => { const assetFilter = AssetFilters[assetType]; if (assetFilter) { - return assetFilter(assets); + assetsToInstall = [...assetsToInstall, ...assetFilter([asset])]; + } else { + assetsToInstall.push(asset); } - return assets; + if (assetsToInstall.length >= MAX_ASSETS_TO_INSTALL_IN_PARALLEL) { + await flushAssetsToInstall(); + } }); - if (!assetsToInstall.length) { - return []; + if (assetsToInstall.length) { + await flushAssetsToInstall(); } - await installManagedIndexPattern({ - savedObjectsClient, - savedObjectsImporter, - }); - - return await installKibanaSavedObjects({ - logger, - savedObjectsImporter, - kibanaAssets: assetsToInstall, - assetsChunkSize: MAX_ASSETS_TO_INSTALL_IN_PARALLEL, - }); + return res; } export async function installManagedIndexPattern({ @@ -267,50 +272,44 @@ export async function installKibanaAssetsAndReferences({ const { savedObjectsImporter, savedObjectTagAssignmentService, savedObjectTagClient } = getSpaceAwareSaveobjectsClients(spaceId); // This is where the memory consumption is rising up in the first place - const kibanaAssets = getKibanaAssets(packageInstallContext); + // const kibanaAssets = getKibanaAssets(packageInstallContext); + const kibanaAssetsArchiveIterator = getKibanaAssetsArchiveIterator(packageInstallContext); + if (installedPkg) { + // TODO condition if security rule not delete await deleteKibanaSavedObjectsAssets({ installedPkg, spaceId }); } let installedKibanaAssetsRefs: KibanaAssetReference[] = []; - if (!installAsAdditionalSpace) { - // save new kibana refs before installing the assets - installedKibanaAssetsRefs = await saveKibanaAssetsRefs( - savedObjectsClient, - pkgName, - kibanaAssetsToAssetsRef(kibanaAssets) - ); - } const importedAssets = await installKibanaAssets({ savedObjectsClient, logger, savedObjectsImporter, pkgName, - kibanaAssets, + kibanaAssetsArchiveIterator, }); - if (installAsAdditionalSpace) { - const assets = importedAssets.map( - ({ id, type, destinationId }) => - ({ - id: destinationId ?? id, - originId: id, - type, - } as KibanaAssetReference) - ); - installedKibanaAssetsRefs = await saveKibanaAssetsRefs( - savedObjectsClient, - pkgName, - assets, - installedPkg && installedPkg.attributes.installed_kibana_space_id === spaceId - ? false - : installAsAdditionalSpace - ); - } + // if (installAsAdditionalSpace) { + const assets = importedAssets.map( + ({ id, type, destinationId }) => + ({ + id: destinationId ?? id, + originId: id, + type, + } as KibanaAssetReference) + ); + installedKibanaAssetsRefs = await saveKibanaAssetsRefs( + savedObjectsClient, + pkgName, + assets, + installedPkg && installedPkg.attributes.installed_kibana_space_id === spaceId + ? false + : installAsAdditionalSpace + ); + await withPackageSpan('Create and assign package tags', () => tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, pkgTitle, pkgName, spaceId, @@ -355,25 +354,32 @@ export const isKibanaAssetType = (path: string) => { return parts.service === 'kibana' && (kibanaAssetTypes as string[]).includes(parts.type); }; -export function getKibanaAssets( - packageInstallContext: PackageInstallContext -): Record { - const result = Object.fromEntries( - kibanaAssetTypes.map((type) => [type, []]) - ) as Record; - - packageInstallContext.paths.filter(isKibanaAssetType).forEach((path) => { - const buffer = getAssetFromAssetsMap(packageInstallContext.assetsMap, path); - const asset = JSON.parse(buffer.toString('utf8')); - - const assetType = getPathParts(path).type as KibanaAssetType; - const soType = KibanaSavedObjectTypeMapping[assetType]; - if (asset.type === soType) { - result[assetType].push(asset); - } - }); +function getKibanaAssetsArchiveIterator(packageInstallContext: PackageInstallContext) { + return ( + onEntry: (entry: { + path: string; + asset: ArchiveAsset; + assetType: KibanaAssetType; + }) => Promise + ) => { + return packageInstallContext.archiveIterator.traverseEntries(async (entry) => { + if (!entry.buffer) { + return; + } - return result; + const asset = JSON.parse(entry.buffer.toString('utf8')); + + const assetType = getPathParts(entry.path).type as KibanaAssetType; + const soType = KibanaSavedObjectTypeMapping[assetType]; + if (!validKibanaAssetTypes.has(assetType)) { + return; + } + + if (asset.type === soType) { + await onEntry({ path: entry.path, assetType, asset }); + } + }, isKibanaAssetType); + }; } const isImportConflictError = (e: SavedObjectsImportFailure) => e?.error?.type === 'conflict'; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/tag_assets.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/tag_assets.test.ts index 82989715be0e1..e5d646c3b85d6 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/tag_assets.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/tag_assets.test.ts @@ -42,16 +42,15 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any; + const importedAssets = [{ id: 'dashboard1', type: 'dashboard' }] as any; await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, pkgTitle: 'System', pkgName: 'system', spaceId: 'default', - importedAssets: [], + importedAssets, }); expect(savedObjectTagClient.create).toHaveBeenCalledWith( @@ -72,7 +71,7 @@ describe('tagKibanaAssets', () => { ); expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({ tags: ['fleet-managed-default', 'fleet-pkg-system-default'], - assign: kibanaAssets.dashboard, + assign: importedAssets, unassign: [], refresh: false, }); @@ -80,22 +79,21 @@ describe('tagKibanaAssets', () => { it('should only assign Managed and System tags when tags already exist', async () => { savedObjectTagClient.get.mockResolvedValue({ name: '', color: '', description: '' }); - const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any; + const importedAssets = [{ id: 'dashboard1', type: 'dashboard' }] as any; await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, + importedAssets, pkgTitle: 'System', pkgName: 'system', spaceId: 'default', - importedAssets: [], }); expect(savedObjectTagClient.create).not.toHaveBeenCalled(); expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({ tags: ['fleet-managed-default', 'fleet-pkg-system-default'], - assign: kibanaAssets.dashboard, + assign: importedAssets, unassign: [], refresh: false, }); @@ -103,16 +101,16 @@ describe('tagKibanaAssets', () => { it('should use destinationId instead of original SO id if imported asset has it', async () => { savedObjectTagClient.get.mockResolvedValue({ name: '', color: '', description: '' }); - const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any; await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, pkgTitle: 'System', pkgName: 'system', spaceId: 'default', - importedAssets: [{ id: 'dashboard1', destinationId: 'destination1' } as any], + importedAssets: [ + { id: 'dashboard1', destinationId: 'destination1', type: 'dashboard' } as any, + ], }); expect(savedObjectTagClient.create).not.toHaveBeenCalled(); @@ -129,33 +127,32 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { - dashboard: [{ id: 'dashboard1', type: 'dashboard' }], - search: [{ id: 's1', type: 'search' }], - config: [{ id: 'c1', type: 'config' }], - visualization: [{ id: 'v1', type: 'visualization' }], - osquery_pack_asset: [{ id: 'osquery-pack-asset1', type: 'osquery-pack-asset' }], - osquery_saved_query: [{ id: 'osquery_saved_query1', type: 'osquery_saved_query' }], - } as any; + const importedAssets = [ + { id: 'dashboard1', type: 'dashboard' }, + { id: 's1', type: 'search' }, + { id: 'c1', type: 'config' }, + { id: 'v1', type: 'visualization' }, + { id: 'osquery-pack-asset1', type: 'osquery-pack-asset' }, + { id: 'osquery_saved_query1', type: 'osquery-saved-query' }, + ] as any; await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, + importedAssets, pkgTitle: 'System', pkgName: 'system', spaceId: 'default', - importedAssets: [], }); expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({ tags: ['fleet-managed-default', 'fleet-pkg-system-default'], assign: [ - ...kibanaAssets.dashboard, - ...kibanaAssets.search, - ...kibanaAssets.visualization, - ...kibanaAssets.osquery_pack_asset, - ...kibanaAssets.osquery_saved_query, + { id: 'dashboard1', type: 'dashboard' }, + { id: 's1', type: 'search' }, + { id: 'v1', type: 'visualization' }, + { id: 'osquery-pack-asset1', type: 'osquery-pack-asset' }, + { id: 'osquery_saved_query1', type: 'osquery-saved-query' }, ], unassign: [], refresh: false, @@ -163,16 +160,15 @@ describe('tagKibanaAssets', () => { }); it('should do nothing if no taggable assets', async () => { - const kibanaAssets = { config: [{ id: 'c1', type: 'config' }] } as any; + const importedAssets = [{ id: 'c1', type: 'config' }] as any; await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, pkgTitle: 'System', pkgName: 'system', spaceId: 'default', - importedAssets: [], + importedAssets, }); expect(savedObjectTagAssignmentService.updateTagAssignments).not.toHaveBeenCalled(); @@ -188,16 +184,15 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any; + const importedAssets = [{ id: 'dashboard1', type: 'dashboard' }] as any; await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, + importedAssets, pkgTitle: 'System', pkgName: 'system', spaceId: 'default', - importedAssets: [], }); expect(savedObjectTagClient.create).not.toHaveBeenCalledWith( @@ -219,7 +214,7 @@ describe('tagKibanaAssets', () => { ); expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({ tags: ['managed', 'fleet-pkg-system-default'], - assign: kibanaAssets.dashboard, + assign: importedAssets, unassign: [], refresh: false, }); @@ -235,16 +230,15 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any; + const importedAssets = [{ id: 'dashboard1', type: 'dashboard' }] as any; await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, pkgTitle: 'System', pkgName: 'system', spaceId: 'default', - importedAssets: [], + importedAssets, }); expect(savedObjectTagClient.create).toHaveBeenCalledWith( @@ -266,7 +260,7 @@ describe('tagKibanaAssets', () => { ); expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({ tags: ['fleet-managed-default', 'system'], - assign: kibanaAssets.dashboard, + assign: importedAssets, unassign: [], refresh: false, }); @@ -283,22 +277,21 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any; + const importedAssets = [{ id: 'dashboard1', type: 'dashboard' }] as any; await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, + importedAssets, pkgTitle: 'System', pkgName: 'system', spaceId: 'default', - importedAssets: [], }); expect(savedObjectTagClient.create).not.toHaveBeenCalled(); expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({ tags: ['managed', 'system'], - assign: kibanaAssets.dashboard, + assign: importedAssets, unassign: [], refresh: false, }); @@ -309,14 +302,12 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { - dashboard: [ - { id: 'dashboard1', type: 'dashboard' }, - { id: 'dashboard2', type: 'dashboard' }, - { id: 'search_id1', type: 'search' }, - { id: 'search_id2', type: 'search' }, - ], - } as any; + const importedAssets = [ + { id: 'dashboard1', type: 'dashboard' }, + { id: 'dashboard2', type: 'dashboard' }, + { id: 'search_id1', type: 'search' }, + { id: 'search_id2', type: 'search' }, + ] as any; const assetTags = [ { text: 'Foo', @@ -326,11 +317,10 @@ describe('tagKibanaAssets', () => { await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, pkgTitle: 'TestPackage', pkgName: 'test-pkg', spaceId: 'default', - importedAssets: [], + importedAssets, assetTags, }); expect(savedObjectTagClient.create).toHaveBeenCalledTimes(3); @@ -412,23 +402,20 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { - dashboard: [ - { id: 'dashboard1', type: 'dashboard' }, - { id: 'dashboard2', type: 'dashboard' }, - { id: 'search_id1', type: 'search' }, - { id: 'search_id2', type: 'search' }, - ], - } as any; + const importedAssets = [ + { id: 'dashboard1', type: 'dashboard' }, + { id: 'dashboard2', type: 'dashboard' }, + { id: 'search_id1', type: 'search' }, + { id: 'search_id2', type: 'search' }, + ] as any; const assetTags = [{ text: 'Bar', asset_ids: ['dashboard1', 'search_id1'] }]; await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, + importedAssets, pkgTitle: 'TestPackage', pkgName: 'test-pkg', spaceId: 'default', - importedAssets: [], assetTags, }); expect(savedObjectTagClient.create).toHaveBeenCalledTimes(3); @@ -510,13 +497,11 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { - dashboard: [ - { id: 'dashboard1', type: 'dashboard' }, - { id: 'dashboard2', type: 'dashboard' }, - { id: 'search_id1', type: 'search' }, - ], - } as any; + const importedAssets = [ + { id: 'dashboard1', type: 'dashboard' }, + { id: 'dashboard2', type: 'dashboard' }, + { id: 'search_id1', type: 'search' }, + ] as any; const assetTags = [ { text: 'myCustomTag', @@ -527,11 +512,10 @@ describe('tagKibanaAssets', () => { await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, + importedAssets, pkgTitle: 'TestPackage', pkgName: 'test-pkg', spaceId: 'default', - importedAssets: [], assetTags, }); expect(savedObjectTagClient.create).toHaveBeenCalledTimes(3); @@ -610,14 +594,12 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { - dashboard: [ - { id: 'dashboard1', type: 'dashboard' }, - { id: 'dashboard2', type: 'dashboard' }, - { id: 'search_id1', type: 'search' }, - { id: 'search_id2', type: 'search' }, - ], - } as any; + const importedAssets = [ + { id: 'dashboard1', type: 'dashboard' }, + { id: 'dashboard2', type: 'dashboard' }, + { id: 'search_id1', type: 'search' }, + { id: 'search_id2', type: 'search' }, + ] as any; const assetTags = [ { text: 'Foo', @@ -633,11 +615,10 @@ describe('tagKibanaAssets', () => { await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, pkgTitle: 'TestPackage', pkgName: 'test-pkg', spaceId: 'default', - importedAssets: [], + importedAssets, assetTags, }); expect(savedObjectTagClient.create).not.toHaveBeenCalled(); @@ -657,14 +638,12 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { - dashboard: [ - { id: 'dashboard1', type: 'dashboard' }, - { id: 'dashboard2', type: 'dashboard' }, - { id: 'search_id1', type: 'search' }, - { id: 'search_id2', type: 'search' }, - ], - } as any; + const importedAssets = [ + { id: 'dashboard1', type: 'dashboard' }, + { id: 'dashboard2', type: 'dashboard' }, + { id: 'search_id1', type: 'search' }, + { id: 'search_id2', type: 'search' }, + ] as any; const assetTags = [ { text: 'foo', @@ -674,11 +653,10 @@ describe('tagKibanaAssets', () => { await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, + importedAssets, pkgTitle: 'TestPackage', pkgName: 'test-pkg', spaceId: 'default', - importedAssets: [], assetTags, }); expect(savedObjectTagClient.create).toHaveBeenCalledTimes(2); @@ -702,14 +680,12 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { - dashboard: [ - { id: 'dashboard1', type: 'dashboard' }, - { id: 'dashboard2', type: 'dashboard' }, - { id: 'search_id1', type: 'search' }, - { id: 'search_id2', type: 'search' }, - ], - } as any; + const importedAssets = [ + { id: 'dashboard1', type: 'dashboard' }, + { id: 'dashboard2', type: 'dashboard' }, + { id: 'search_id1', type: 'search' }, + { id: 'search_id2', type: 'search' }, + ] as any; const assetTags = [ { text: 'Security Solution', @@ -719,11 +695,10 @@ describe('tagKibanaAssets', () => { await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, + importedAssets, pkgTitle: 'TestPackage', pkgName: 'test-pkg', spaceId: 'default', - importedAssets: [], assetTags, }); expect(savedObjectTagClient.create).toHaveBeenCalledWith( @@ -753,14 +728,12 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { - dashboard: [ - { id: 'dashboard1', type: 'dashboard' }, - { id: 'dashboard2', type: 'dashboard' }, - { id: 'search_id1', type: 'search' }, - { id: 'search_id2', type: 'search' }, - ], - } as any; + const importedAssets = [ + { id: 'dashboard1', type: 'dashboard' }, + { id: 'dashboard2', type: 'dashboard' }, + { id: 'search_id1', type: 'search' }, + { id: 'search_id2', type: 'search' }, + ] as any; const assetTags = [ { text: 'Security Solution', @@ -770,11 +743,10 @@ describe('tagKibanaAssets', () => { await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, + importedAssets, pkgTitle: 'TestPackage', pkgName: 'test-pkg', spaceId: 'my-secondary-space', - importedAssets: [], assetTags, }); expect(savedObjectTagClient.create).toHaveBeenCalledWith(managedTagPayloadArg1, { @@ -815,23 +787,20 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { - dashboard: [ - { id: 'dashboard1', type: 'dashboard' }, - { id: 'dashboard2', type: 'dashboard' }, - { id: 'search_id1', type: 'search' }, - { id: 'search_id2', type: 'search' }, - ], - } as any; + const importedAssets = [ + { id: 'dashboard1', type: 'dashboard' }, + { id: 'dashboard2', type: 'dashboard' }, + { id: 'search_id1', type: 'search' }, + { id: 'search_id2', type: 'search' }, + ] as any; await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, + importedAssets, pkgTitle: 'TestPackage', pkgName: 'test-pkg', spaceId: 'default', - importedAssets: [], assetTags: [], }); expect(savedObjectTagClient.create).toHaveBeenCalledTimes(2); @@ -843,14 +812,12 @@ describe('tagKibanaAssets', () => { savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) => Promise.resolve({ id: name.toLowerCase(), name }) ); - const kibanaAssets = { - dashboard: [ - { id: 'dashboard1', type: 'dashboard' }, - { id: 'dashboard2', type: 'dashboard' }, - { id: 'search_id1', type: 'search' }, - { id: 'search_id2', type: 'search' }, - ], - } as any; + const importedAssets = [ + { id: 'dashboard1', type: 'dashboard' }, + { id: 'dashboard2', type: 'dashboard' }, + { id: 'search_id1', type: 'search' }, + { id: 'search_id2', type: 'search' }, + ] as any; const assetTags = [ { text: 'Foo', @@ -861,11 +828,10 @@ describe('tagKibanaAssets', () => { await tagKibanaAssets({ savedObjectTagAssignmentService, savedObjectTagClient, - kibanaAssets, + importedAssets, pkgTitle: 'TestPackage', pkgName: 'test-pkg', spaceId: 'default', - importedAssets: [], assetTags, }); expect(savedObjectTagClient.create).toHaveBeenCalledTimes(3); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/tag_assets.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/tag_assets.ts index 27c825f039e4a..bebc0ba9dc810 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/tag_assets.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/tag_assets.ts @@ -6,7 +6,7 @@ */ import { v5 as uuidv5 } from 'uuid'; -import { uniqBy } from 'lodash'; +import { omit, uniqBy } from 'lodash'; import pMap from 'p-map'; import type { SavedObjectsImportSuccess } from '@kbn/core-saved-objects-common'; import { taggableTypes } from '@kbn/saved-objects-tagging-plugin/common/constants'; @@ -14,14 +14,11 @@ import type { IAssignmentService } from '@kbn/saved-objects-tagging-plugin/serve import type { ITagsClient } from '@kbn/saved-objects-tagging-plugin/common/types'; import { MAX_CONCURRENT_PACKAGE_ASSETS } from '../../../../constants'; -import type { KibanaAssetType } from '../../../../../common'; + import type { PackageSpecTags } from '../../../../types'; import { appContextService } from '../../../app_context'; -import type { ArchiveAsset } from './install'; -import { KibanaSavedObjectTypeMapping } from './install'; - interface ObjectReference { type: string; id: string; @@ -85,7 +82,6 @@ const getRandomColor = () => { interface TagAssetsParams { savedObjectTagAssignmentService: IAssignmentService; savedObjectTagClient: ITagsClient; - kibanaAssets: Record; pkgTitle: string; pkgName: string; spaceId: string; @@ -93,14 +89,15 @@ interface TagAssetsParams { assetTags?: PackageSpecTags[]; } +const getNewId = (asset: SavedObjectsImportSuccess) => + asset?.destinationId ? asset.destinationId : asset.id; + export async function tagKibanaAssets(opts: TagAssetsParams) { - const { savedObjectTagAssignmentService, kibanaAssets, importedAssets } = opts; + const { savedObjectTagAssignmentService, importedAssets } = opts; - const getNewId = (assetId: string) => - importedAssets.find((imported) => imported.id === assetId)?.destinationId ?? assetId; - const taggableAssets = getTaggableAssets(kibanaAssets).map((asset) => ({ - ...asset, - id: getNewId(asset.id), + const taggableAssets = getTaggableAssets(importedAssets).map((asset) => ({ + ...omit(asset, 'destinationId'), + id: getNewId(asset), })); if (taggableAssets.length > 0) { const [managedTagId, packageTagId] = await Promise.all([ @@ -150,18 +147,8 @@ export async function tagKibanaAssets(opts: TagAssetsParams) { } } -function getTaggableAssets(kibanaAssets: TagAssetsParams['kibanaAssets']) { - return Object.entries(kibanaAssets).flatMap(([assetType, assets]) => { - if (!taggableTypes.includes(KibanaSavedObjectTypeMapping[assetType as KibanaAssetType])) { - return []; - } - - if (!assets.length) { - return []; - } - - return assets; - }); +function getTaggableAssets(importedAssets: SavedObjectsImportSuccess[]) { + return importedAssets.filter((asset) => taggableTypes.includes(asset.type)); } async function ensureManagedTag( @@ -219,7 +206,7 @@ async function ensurePackageTag( // Ensure that asset tags coming from the kibana/tags.yml file are correctly parsed and created async function getPackageSpecTags( - taggableAssets: ArchiveAsset[], + taggableAssets: SavedObjectsImportSuccess[], opts: Pick ): Promise { const { spaceId, savedObjectTagClient, pkgName, assetTags } = opts; @@ -254,7 +241,7 @@ async function getPackageSpecTags( // Get all the assets of types defined in tag.asset_types from taggable kibanaAssets const getAssetTypesObjectReferences = ( assetTypes: string[] | undefined, - taggableAssets: ArchiveAsset[] + taggableAssets: SavedObjectsImportSuccess[] ): ObjectReference[] => { if (!assetTypes || assetTypes.length === 0) return []; @@ -268,7 +255,7 @@ const getAssetTypesObjectReferences = ( // Get the references to ids defined in tag.asset_ids from taggable kibanaAssets const getAssetIdsObjectReferences = ( assetIds: string[] | undefined, - taggableAssets: ArchiveAsset[] + taggableAssets: SavedObjectsImportSuccess[] ): ObjectReference[] => { if (!assetIds || assetIds.length === 0) return []; From 047e933a215ad3e0f4d0b2a6d7c88570709dae78 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 24 Feb 2025 11:12:08 -0500 Subject: [PATCH 03/21] save archive assets --- .../steps/step_save_archive_entries.ts | 69 ++++++++++--------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts index f081d9a93e633..4b0160f853849 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts @@ -6,7 +6,7 @@ */ import { ASSETS_SAVED_OBJECT_TYPE } from '../../../../../constants'; -import type { PackageAssetReference } from '../../../../../types'; +import type { AssetsMap, KibanaAssetType, PackageAssetReference } from '../../../../../types'; import { removeArchiveEntries, saveArchiveEntriesFromAssetsMap } from '../../../archive/storage'; @@ -14,43 +14,50 @@ import { withPackageSpan } from '../../utils'; import type { InstallContext } from '../_state_machine_package_install'; import { INSTALL_STATES } from '../../../../../../common/types'; -import { isKibanaAssetType } from '../../../kibana/assets/install'; +import { getPathParts } from '../../../archive'; export async function stepSaveArchiveEntries(context: InstallContext) { - const { packageInstallContext, savedObjectsClient, installSource, useStreaming } = context; + const { packageInstallContext, savedObjectsClient, installSource } = context; const { packageInfo, archiveIterator } = packageInstallContext; - let assetsMap = packageInstallContext?.assetsMap; - let paths = packageInstallContext?.paths; - // For stream based installations, we don't want to save any assets but - // manifest.yaml due to the large number of assets in the package. - if (useStreaming) { - assetsMap = new Map(); - await archiveIterator.traverseEntries(async (entry) => { - // Skip only kibana assets type - if (!isKibanaAssetType(entry.path)) { - assetsMap.set(entry.path, entry.buffer); - } - }); - paths = Array.from(assetsMap.keys()); + let assetsToSaveMap: AssetsMap = new Map(); + + let packageAssetRefs: PackageAssetReference[] = []; + + async function flushAssets() { + const paths = Array.from(assetsToSaveMap.keys()); + const packageAssetResults = await withPackageSpan('Update archive entries', () => + saveArchiveEntriesFromAssetsMap({ + savedObjectsClient, + assetsMap: assetsToSaveMap, + paths, + packageInfo, + installSource, + }) + ); + packageAssetRefs = packageAssetRefs.concat( + packageAssetResults.saved_objects.map((result) => ({ + id: result.id, + type: ASSETS_SAVED_OBJECT_TYPE, + })) + ); + + assetsToSaveMap = new Map(); } - const packageAssetResults = await withPackageSpan('Update archive entries', () => - saveArchiveEntriesFromAssetsMap({ - savedObjectsClient, - assetsMap, - paths, - packageInfo, - installSource, - }) - ); - const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map( - (result) => ({ - id: result.id, - type: ASSETS_SAVED_OBJECT_TYPE, - }) - ); + await archiveIterator.traverseEntries(async (entry) => { + const assetType = getPathParts(entry.path).type as KibanaAssetType; + // Skip security rules to avoid storing to many things + if (assetType !== 'security_rule') { + assetsToSaveMap.set(entry.path, entry.buffer); + } + if (assetsToSaveMap.size > 100) { + await flushAssets(); + } + }); + + await flushAssets(); return { packageAssetRefs }; } From aa804a07b28ecfe359b67e5c166f0bafa648d8d3 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 24 Feb 2025 15:04:05 -0500 Subject: [PATCH 04/21] remove types --- .../shared/fleet/common/types/models/epm.ts | 2 +- .../routes/epm/kibana_assets_handler.ts | 1 - .../ingest_pipeline/install.test.ts | 1 - .../elasticsearch/template/install.test.ts | 36 +++++++++++++--- .../epm/elasticsearch/template/install.ts | 30 +++++++++++--- .../epm/elasticsearch/transform/install.ts | 12 +++++- .../elasticsearch/transform/mappings.test.ts | 41 ++++++++++--------- .../epm/elasticsearch/transform/mappings.ts | 9 +++- .../fleet/server/services/epm/fields/field.ts | 8 ++-- .../server/services/epm/package_service.ts | 1 - .../server/services/epm/packages/install.ts | 9 +--- .../install_index_template_pipeline.ts | 2 +- .../_state_machine_package_install.test.ts | 5 --- .../step_create_restart_installation.test.ts | 4 -- .../steps/step_install_ilm_policies.test.ts | 1 - .../steps/step_install_kibana_assets.test.ts | 4 -- .../steps/step_save_archive_entries.ts | 4 +- .../steps/step_save_system_object.test.ts | 2 - .../update_latest_executed_state.test.ts | 4 -- .../experimental_datastream_features.ts | 2 +- 20 files changed, 106 insertions(+), 72 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts b/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts index b28677744cff8..5ebbdf573dd07 100644 --- a/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts +++ b/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts @@ -145,7 +145,7 @@ export interface PackageInstallContext { * @deprecated Use `archiveIterator` to access the package archive entries * without loading them all into memory at once. */ - assetsMap: AssetsMap; // TODO remove + // assetsMap: AssetsMap; // TODO remove paths: string[]; archiveIterator: ArchiveIterator; } diff --git a/x-pack/platform/plugins/shared/fleet/server/routes/epm/kibana_assets_handler.ts b/x-pack/platform/plugins/shared/fleet/server/routes/epm/kibana_assets_handler.ts index 57880d5f08397..550377a058371 100644 --- a/x-pack/platform/plugins/shared/fleet/server/routes/epm/kibana_assets_handler.ts +++ b/x-pack/platform/plugins/shared/fleet/server/routes/epm/kibana_assets_handler.ts @@ -68,7 +68,6 @@ export const installPackageKibanaAssetsHandler: FleetRequestHandler< packageInstallContext: { packageInfo, paths: installedPkgWithAssets.paths, - assetsMap: installedPkgWithAssets.assetsMap, archiveIterator: createArchiveIteratorFromMap(installedPkgWithAssets.assetsMap), }, }); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts index 7caeb2d35c225..f2aa98ba44ed2 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts @@ -29,7 +29,6 @@ describe('Install pipeline tests', () => { ], } as any, paths: [], - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), }); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.test.ts index f735e8638b583..f3a8c955abbdd 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -10,6 +10,7 @@ import { createAppContextStartContractMock } from '../../../../mocks'; import { appContextService } from '../../..'; import { loadDatastreamsFieldsFromYaml } from '../../fields/field'; import type { PackageInstallContext, RegistryDataStream } from '../../../../../common/types'; +import { createArchiveIteratorFromMap } from '../../archive/archive_iterator'; import { prepareTemplate, prepareToInstallTemplates } from './install'; @@ -53,7 +54,11 @@ describe('EPM index template install', () => { const templatePriorityDatasetIsPrefixUnset = 200; const { indexTemplate: { indexTemplate }, - } = prepareTemplate({ packageInstallContext, dataStream: dataStreamDatasetIsPrefixUnset }); + } = await prepareTemplate({ + packageInstallContext, + fieldAssetsMap: new Map(), + dataStream: dataStreamDatasetIsPrefixUnset, + }); expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); }); @@ -74,7 +79,11 @@ describe('EPM index template install', () => { const templatePriorityDatasetIsPrefixFalse = 200; const { indexTemplate: { indexTemplate }, - } = prepareTemplate({ packageInstallContext, dataStream: dataStreamDatasetIsPrefixFalse }); + } = prepareTemplate({ + packageInstallContext, + fieldAssetsMap: new Map(), + dataStream: dataStreamDatasetIsPrefixFalse, + }); expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); @@ -96,7 +105,11 @@ describe('EPM index template install', () => { const templatePriorityDatasetIsPrefixTrue = 150; const { indexTemplate: { indexTemplate }, - } = prepareTemplate({ packageInstallContext, dataStream: dataStreamDatasetIsPrefixTrue }); + } = prepareTemplate({ + packageInstallContext, + fieldAssetsMap: new Map(), + dataStream: dataStreamDatasetIsPrefixTrue, + }); expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); @@ -119,6 +132,7 @@ describe('EPM index template install', () => { const { componentTemplates } = prepareTemplate({ packageInstallContext, + fieldAssetsMap: new Map(), dataStream: dataStreamDatasetIsPrefixTrue, }); @@ -150,6 +164,7 @@ describe('EPM index template install', () => { const { componentTemplates } = prepareTemplate({ packageInstallContext, + fieldAssetsMap: new Map(), dataStream: dataStreamDatasetIsPrefixTrue, }); @@ -182,6 +197,7 @@ describe('EPM index template install', () => { const { componentTemplates } = prepareTemplate({ packageInstallContext, dataStream: dataStreamDatasetIsPrefixTrue, + fieldAssetsMap: new Map(), experimentalDataStreamFeature: { data_stream: 'metrics-package.dataset', features: { @@ -220,6 +236,7 @@ describe('EPM index template install', () => { const { componentTemplates } = prepareTemplate({ packageInstallContext, dataStream: dataStreamDatasetIsPrefixTrue, + fieldAssetsMap: new Map(), experimentalDataStreamFeature: { data_stream: 'metrics-package.dataset', features: { @@ -257,6 +274,7 @@ describe('EPM index template install', () => { const { indexTemplate } = prepareTemplate({ packageInstallContext, + fieldAssetsMap: new Map(), dataStream: dataStreamDatasetIsPrefixTrue, }); @@ -288,6 +306,7 @@ describe('EPM index template install', () => { const { componentTemplates } = prepareTemplate({ packageInstallContext, dataStream, + fieldAssetsMap: new Map(), }); const packageTemplate = componentTemplates['logs-package.dataset@package'].template; @@ -315,6 +334,7 @@ describe('EPM index template install', () => { const { componentTemplates } = prepareTemplate({ packageInstallContext, dataStream, + fieldAssetsMap: new Map(), }); const packageTemplate = componentTemplates['logs-package.dataset@package'].template; @@ -352,6 +372,7 @@ describe('EPM index template install', () => { const { componentTemplates } = prepareTemplate({ packageInstallContext, + fieldAssetsMap: new Map(), dataStream, }); @@ -390,6 +411,7 @@ describe('EPM index template install', () => { const { componentTemplates } = prepareTemplate({ packageInstallContext, + fieldAssetsMap: new Map(), dataStream, }); @@ -430,6 +452,7 @@ describe('EPM index template install', () => { const { componentTemplates } = prepareTemplate({ packageInstallContext, + fieldAssetsMap: new Map(), dataStream, }); @@ -474,6 +497,7 @@ describe('EPM index template install', () => { const { componentTemplates } = prepareTemplate({ packageInstallContext, + fieldAssetsMap: new Map(), dataStream, }); @@ -509,6 +533,7 @@ describe('EPM index template install', () => { const { componentTemplates } = prepareTemplate({ packageInstallContext, + fieldAssetsMap: new Map(), dataStream, }); @@ -517,7 +542,7 @@ describe('EPM index template install', () => { expect(packageTemplate).not.toHaveProperty('lifecycle'); }); - test('test prepareToInstallTemplates does not include stack component templates in tracked assets', () => { + test('test prepareToInstallTemplates does not include stack component templates in tracked assets', async () => { const dataStreamDatasetIsPrefixUnset = { type: 'logs', dataset: 'package.dataset', @@ -528,13 +553,14 @@ describe('EPM index template install', () => { ingest_pipeline: 'default', } as RegistryDataStream; - const { assetsToAdd } = prepareToInstallTemplates( + const { assetsToAdd } = await prepareToInstallTemplates( { packageInfo: { name: 'package', version: '0.0.1', data_streams: [dataStreamDatasetIsPrefixUnset], }, + archiveIterator: createArchiveIteratorFromMap(new Map()), } as PackageInstallContext, [], [] diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.ts index a85329ee68ca2..829eed846ef00 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.ts @@ -32,7 +32,7 @@ import type { ExperimentalDataStreamFeature, } from '../../../../types'; import type { Fields } from '../../fields/field'; -import { loadDatastreamsFieldsFromYaml, processFields } from '../../fields/field'; +import { isFields, loadDatastreamsFieldsFromYaml, processFields } from '../../fields/field'; import { getAssetFromAssetsMap, getPathParts } from '../../archive'; import { FLEET_COMPONENT_TEMPLATES, @@ -62,16 +62,16 @@ import { isUserSettingsTemplate } from './utils'; const FLEET_COMPONENT_TEMPLATE_NAMES = FLEET_COMPONENT_TEMPLATES.map((tmpl) => tmpl.name); -export const prepareToInstallTemplates = ( +export const prepareToInstallTemplates = async ( packageInstallContext: PackageInstallContext, esReferences: EsAssetReference[], experimentalDataStreamFeatures: ExperimentalDataStreamFeature[] = [], onlyForDataStreams?: RegistryDataStream[] -): { +): Promise<{ assetsToAdd: EsAssetReference[]; assetsToRemove: EsAssetReference[]; install: (esClient: ElasticsearchClient, logger: Logger) => Promise; -} => { +}> => { const { packageInfo } = packageInstallContext; // remove package installation's references to index templates const assetsToRemove = esReferences.filter( @@ -80,6 +80,13 @@ export const prepareToInstallTemplates = ( type === ElasticsearchAssetType.componentTemplate ); + const fieldAssetsMap: AssetsMap = new Map(); + await packageInstallContext.archiveIterator.traverseEntries(async (entry) => { + if (entry.buffer) { + fieldAssetsMap.set(entry.path, entry.buffer); + } + }, isFields); + // build templates per data stream from yml files const dataStreams = onlyForDataStreams || packageInfo.data_streams; if (!dataStreams) return { assetsToAdd: [], assetsToRemove, install: () => Promise.resolve([]) }; @@ -90,7 +97,12 @@ export const prepareToInstallTemplates = ( datastreamFeature.data_stream === getRegistryDataStreamAssetBaseName(dataStream) ); - return prepareTemplate({ packageInstallContext, dataStream, experimentalDataStreamFeature }); + return prepareTemplate({ + packageInstallContext, + fieldAssetsMap, + dataStream, + experimentalDataStreamFeature, + }); }); const assetsToAdd = getAllTemplateRefs(templates.map((template) => template.indexTemplate)); @@ -590,15 +602,21 @@ function countFields(fields: Fields): number { export function prepareTemplate({ packageInstallContext, + fieldAssetsMap, dataStream, experimentalDataStreamFeature, }: { packageInstallContext: PackageInstallContext; + fieldAssetsMap: AssetsMap; dataStream: RegistryDataStream; experimentalDataStreamFeature?: ExperimentalDataStreamFeature; }): { componentTemplates: TemplateMap; indexTemplate: IndexTemplateEntry } { const { name: packageName, version: packageVersion } = packageInstallContext.packageInfo; - const fields = loadDatastreamsFieldsFromYaml(packageInstallContext, dataStream.path); + const fields = loadDatastreamsFieldsFromYaml( + packageInstallContext, + fieldAssetsMap, + dataStream.path + ); const isIndexModeTimeSeries = dataStream.elasticsearch?.index_mode === 'time_series' || diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts index 1d1e0c69f3d19..60aaf6f84bcbb 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -408,6 +408,12 @@ const processTransformAssetsPerModule = async ( version: t.transformVersion, })); + const fieldAssetsMap: AssetsMap = new Map(); + await packageInstallContext.archiveIterator.traverseEntries(async (entry) => { + if (entry.buffer) { + fieldAssetsMap.set(entry.path, entry.buffer); + } + }, isFields); // Load and generate mappings for (const destinationIndexTemplate of destinationIndexTemplates) { if (!destinationIndexTemplate.transformModuleId) { @@ -418,7 +424,11 @@ const processTransformAssetsPerModule = async ( .get(destinationIndexTemplate.transformModuleId) ?.set( 'mappings', - loadMappingForTransform(packageInstallContext, destinationIndexTemplate.transformModuleId) + loadMappingForTransform( + packageInstallContext, + fieldAssetsMap, + destinationIndexTemplate.transformModuleId + ) ); } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/mappings.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/mappings.test.ts index de962850fba8c..48cccc79f942a 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/mappings.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/mappings.test.ts @@ -14,10 +14,10 @@ describe('loadMappingForTransform', () => { const fields = loadMappingForTransform( { packageInfo: {} as any, - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], }, + new Map(), 'test' ); @@ -28,36 +28,37 @@ describe('loadMappingForTransform', () => { const fields = loadMappingForTransform( { packageInfo: {} as any, - assetsMap: new Map([ - [ - '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs.yml', - Buffer.from( - ` + + archiveIterator: createArchiveIteratorFromMap(new Map()), + paths: [ + '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs.yml', + '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs-extra.yml', + ], + }, + new Map([ + [ + '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs.yml', + Buffer.from( + ` - description: Description of the threat feed in a UI friendly format. name: threat.feed.description type: keyword - description: The name of the threat feed in UI friendly format. name: threat.feed.name type: keyword` - ), - ], - [ - '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs-extra.yml', - Buffer.from( - ` + ), + ], + [ + '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs-extra.yml', + Buffer.from( + ` - description: The display name indicator in an UI friendly format level: extended name: threat.indicator.name type: keyword` - ), - ], - ]), - archiveIterator: createArchiveIteratorFromMap(new Map()), - paths: [ - '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs.yml', - '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs-extra.yml', + ), ], - }, + ]), 'latest_ioc' ); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/mappings.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/mappings.ts index 130dae0ecca51..6fec40c435c42 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/mappings.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/mappings.ts @@ -4,15 +4,20 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { type PackageInstallContext } from '../../../../../common/types/models'; +import type { AssetsMap, PackageInstallContext } from '../../../../../common/types/models'; import { loadTransformFieldsFromYaml, processFields } from '../../fields/field'; import { generateMappings } from '../template/template'; export function loadMappingForTransform( packageInstallContext: PackageInstallContext, + fieldAssetsMap: AssetsMap, transformModuleId: string ) { - const fields = loadTransformFieldsFromYaml(packageInstallContext, transformModuleId); + const fields = loadTransformFieldsFromYaml( + packageInstallContext, + fieldAssetsMap, + transformModuleId + ); const validFields = processFields(fields); return generateMappings(validFields); } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.ts index a3ebf58d02e3b..30482541ff7a1 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.ts @@ -7,7 +7,7 @@ import { load } from 'js-yaml'; -import type { PackageInstallContext } from '../../../../common/types'; +import type { AssetsMap, PackageInstallContext } from '../../../../common/types'; import { getAssetsDataFromAssetsMap } from '../packages/assets'; // This should become a copy of https://github.com/elastic/beats/blob/d9a4c9c240a9820fab15002592e5bb6db318543b/libbeat/mapping/field.go#L39 @@ -310,12 +310,13 @@ function combineFilter(...filters: Array<(path: string) => boolean>) { export const loadDatastreamsFieldsFromYaml = ( packageInstallContext: PackageInstallContext, + fieldAssetsMap: AssetsMap, datasetName?: string ): Field[] => { // Fetch all field definition files const fieldDefinitionFiles = getAssetsDataFromAssetsMap( packageInstallContext.packageInfo, - packageInstallContext.assetsMap, + fieldAssetsMap, isFields, datasetName ); @@ -334,12 +335,13 @@ export const loadDatastreamsFieldsFromYaml = ( export const loadTransformFieldsFromYaml = ( packageInstallContext: PackageInstallContext, + fieldAssetsMap: AssetsMap, transformName: string ): Field[] => { // Fetch all field definition files const fieldDefinitionFiles = getAssetsDataFromAssetsMap( packageInstallContext.packageInfo, - packageInstallContext.assetsMap, + fieldAssetsMap, combineFilter(isFields, filterForTransformAssets(transformName)) ); return fieldDefinitionFiles.reduce((acc, file) => { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/package_service.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/package_service.ts index 782aeffb6a85c..f22c40e759994 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/package_service.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/package_service.ts @@ -405,7 +405,6 @@ class PackageClientImpl implements PackageClient { const { installedTransforms } = await installTransforms({ packageInstallContext: { - assetsMap, packageInfo, paths, archiveIterator, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts index 88608aa8e04f2..2d44b763595f6 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts @@ -485,7 +485,7 @@ async function installPackageFromRegistry({ } // get latest package version and requested version in parallel for performance - const [latestPackage, { paths, packageInfo, assetsMap, archiveIterator, verificationResult }] = + const [latestPackage, { paths, packageInfo, archiveIterator, verificationResult }] = await Promise.all([ latestPkg ? Promise.resolve(latestPkg) : queryLatest(), Registry.getPackage(pkgName, pkgVersion, { @@ -495,7 +495,6 @@ async function installPackageFromRegistry({ ]); const packageInstallContext: PackageInstallContext = { packageInfo, - assetsMap, paths, archiveIterator, }; @@ -820,7 +819,7 @@ async function installPackageByUpload({ packageInfo, }); - const { paths, assetsMap, archiveIterator } = await unpackBufferToAssetsMap({ + const { paths, archiveIterator } = await unpackBufferToAssetsMap({ archiveBuffer, contentType, useStreaming, @@ -828,7 +827,6 @@ async function installPackageByUpload({ const packageInstallContext: PackageInstallContext = { packageInfo: { ...packageInfo, version: pkgVersion }, - assetsMap, paths, archiveIterator, }; @@ -1023,7 +1021,6 @@ export async function installCustomPackage( const archiveIterator = createArchiveIteratorFromMap(assetsMap); const packageInstallContext: PackageInstallContext = { - assetsMap, paths, packageInfo, archiveIterator, @@ -1361,7 +1358,6 @@ export async function installAssetsForInputPackagePolicy(opts: { const archiveIterator = createArchiveIteratorFromMap(pkg.assetsMap); packageInstallContext = { - assetsMap: pkg.assetsMap, packageInfo: pkg.packageInfo, paths: pkg.paths, archiveIterator, @@ -1369,7 +1365,6 @@ export async function installAssetsForInputPackagePolicy(opts: { } else { const archiveIterator = createArchiveIteratorFromMap(installedPkgWithAssets.assetsMap); packageInstallContext = { - assetsMap: installedPkgWithAssets.assetsMap, packageInfo: installedPkgWithAssets.packageInfo, paths: installedPkgWithAssets.paths, archiveIterator, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_index_template_pipeline.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_index_template_pipeline.ts index ad738d1710fcc..b079f8ee86dbf 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_index_template_pipeline.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_index_template_pipeline.ts @@ -55,7 +55,7 @@ export async function installIndexTemplatesAndPipelines({ packageInstallContext, onlyForDataStreams ); - const preparedIndexTemplates = prepareToInstallTemplates( + const preparedIndexTemplates = await prepareToInstallTemplates( packageInstallContext, esReferences, experimentalDataStreamFeatures, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts index 73b78a6cc4aa0..dfd6cce6b91bc 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts @@ -111,7 +111,6 @@ describe('_stateMachineInstallPackage', () => { esClient, logger: loggerMock.create(), packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -174,7 +173,6 @@ describe('_stateMachineInstallPackage', () => { esClient, logger: loggerMock.create(), packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -211,7 +209,6 @@ describe('_stateMachineInstallPackage', () => { esClient, logger: loggerMock.create(), packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -261,7 +258,6 @@ describe('_stateMachineInstallPackage', () => { esClient, logger: loggerMock.create(), packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -341,7 +337,6 @@ describe('_stateMachineInstallPackage', () => { conditions: { kibana: { version: 'x.y.z' } }, owner: { github: 'elastic/fleet' }, } as any, - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], }, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts index e5a7fed55fe87..6cec95cd7c53c 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts @@ -85,7 +85,6 @@ describe('stepCreateRestartInstallation', () => { esClient, logger, packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -122,7 +121,6 @@ describe('stepCreateRestartInstallation', () => { esClient, logger, packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -167,7 +165,6 @@ describe('stepCreateRestartInstallation', () => { esClient, logger, packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -212,7 +209,6 @@ describe('stepCreateRestartInstallation', () => { esClient, logger, packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts index 4c106a0c68f15..7b78b7967abc7 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts @@ -241,7 +241,6 @@ describe('stepInstallILMPolicies', () => { conditions: { kibana: { version: 'x.y.z' } }, owner: { github: 'elastic/fleet' }, } as any, - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], }, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts index cf9d953868b6a..dcf7862fcfa53 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts @@ -74,7 +74,6 @@ const packageInstallContext = { owner: { github: 'elastic/fleet' }, } as any, paths: ['some/path/1', 'some/path/2'], - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), }; @@ -99,7 +98,6 @@ describe('stepInstallKibanaAssets', () => { esClient, logger: loggerMock.create(), packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -139,7 +137,6 @@ describe('stepInstallKibanaAssets', () => { esClient, logger: loggerMock.create(), packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -186,7 +183,6 @@ describe('stepInstallKibanaAssetsWithStreaming', () => { esClient, logger: loggerMock.create(), packageInstallContext: { - assetsMap, archiveIterator, paths: [], packageInfo: { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts index 4b0160f853849..017bfd2f6301f 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts @@ -17,7 +17,7 @@ import { INSTALL_STATES } from '../../../../../../common/types'; import { getPathParts } from '../../../archive'; export async function stepSaveArchiveEntries(context: InstallContext) { - const { packageInstallContext, savedObjectsClient, installSource } = context; + const { packageInstallContext, savedObjectsClient, installSource, useStreaming } = context; const { packageInfo, archiveIterator } = packageInstallContext; @@ -49,7 +49,7 @@ export async function stepSaveArchiveEntries(context: InstallContext) { await archiveIterator.traverseEntries(async (entry) => { const assetType = getPathParts(entry.path).type as KibanaAssetType; // Skip security rules to avoid storing to many things - if (assetType !== 'security_rule') { + if (assetType !== 'security_rule' && useStreaming) { assetsToSaveMap.set(entry.path, entry.buffer); } if (assetsToSaveMap.size > 100) { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts index c4ae211c58fc5..62414ad0e8089 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts @@ -66,7 +66,6 @@ describe('updateLatestExecutedState', () => { esClient, logger, packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -133,7 +132,6 @@ describe('updateLatestExecutedState', () => { esClient, logger, packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts index aea879aba5479..02ace384e760b 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts @@ -62,7 +62,6 @@ describe('updateLatestExecutedState', () => { esClient, logger, packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -118,7 +117,6 @@ describe('updateLatestExecutedState', () => { esClient, logger, packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -156,7 +154,6 @@ describe('updateLatestExecutedState', () => { esClient, logger, packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { @@ -202,7 +199,6 @@ describe('updateLatestExecutedState', () => { esClient, logger, packageInstallContext: { - assetsMap: new Map(), archiveIterator: createArchiveIteratorFromMap(new Map()), paths: [], packageInfo: { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/package_policies/experimental_datastream_features.ts b/x-pack/platform/plugins/shared/fleet/server/services/package_policies/experimental_datastream_features.ts index 0877f38293f4e..df5dd34ced2c7 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/package_policies/experimental_datastream_features.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/package_policies/experimental_datastream_features.ts @@ -75,11 +75,11 @@ export async function handleExperimentalDatastreamFeatureOptIn({ ); return prepareTemplate({ packageInstallContext: { - assetsMap, archiveIterator: createArchiveIteratorFromMap(assetsMap), packageInfo, paths, }, + fieldAssetsMap: assetsMap, dataStream, experimentalDataStreamFeature, }); From 09708da36735804df558ec8a90cdf86f83117897 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 24 Feb 2025 15:20:07 -0500 Subject: [PATCH 05/21] remove unused assets map fetching --- .../shared/fleet/server/services/epm/packages/install.ts | 2 +- .../shared/fleet/server/services/epm/registry/index.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts index 2d44b763595f6..b4aa9eb36e600 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts @@ -490,7 +490,7 @@ async function installPackageFromRegistry({ latestPkg ? Promise.resolve(latestPkg) : queryLatest(), Registry.getPackage(pkgName, pkgVersion, { ignoreUnverified: force && !neverIgnoreVerificationError, - useStreaming, + fetchFullAssetsMap: false, }), ]); const packageInstallContext: PackageInstallContext = { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts index 80d054f6238df..b32e0bcab15b6 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts @@ -355,7 +355,7 @@ async function getPackageInfoFromArchiveOrCache( export async function getPackage( name: string, version: string, - options?: { ignoreUnverified?: boolean; useStreaming?: boolean } + options?: { ignoreUnverified?: boolean; fetchFullAssetsMap?: boolean } ): Promise<{ paths: string[]; packageInfo: ArchivePackage; @@ -392,7 +392,7 @@ export async function getPackage( const { paths, assetsMap, archiveIterator } = await unpackBufferToAssetsMap({ archiveBuffer, contentType, - useStreaming: options?.useStreaming, + useStreaming: options?.fetchFullAssetsMap, }); if (!packageInfo) { From 95318934c1b324eda6ac0d8fce9aef0de7c306e8 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 25 Feb 2025 08:19:28 -0500 Subject: [PATCH 06/21] fix tests --- .../ingest_pipeline/install.test.ts | 10 --- .../transform/transforms.test.ts | 80 ------------------- .../services/epm/kibana/assets/install.ts | 6 +- 3 files changed, 2 insertions(+), 94 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts index f2aa98ba44ed2..9d12bd7a5e8da 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts @@ -66,16 +66,6 @@ describe('Install pipeline tests', () => { 'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/default.yml', 'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/standard.yml', ], - assetsMap: new Map([ - [ - 'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/default.yml', - Buffer.from('description: test'), - ], - [ - 'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/standard.yml', - Buffer.from('description: test'), - ], - ]), archiveIterator: createArchiveIteratorFromMap( new Map([ [ diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts index 7edf7f9c78c12..4b3529ff39aa8 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts @@ -271,28 +271,6 @@ _meta: 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', ], - assetsMap: new Map([ - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/beats.yml', - sourceData.BEATS_FIELDS, - ], - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/agent.yml', - sourceData.AGENT_FIELDS, - ], - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', - sourceData.FIELDS, - ], - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', - sourceData.MANIFEST, - ], - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', - sourceData.TRANSFORM, - ], - ]), archiveIterator: createArchiveIteratorFromMap( new Map([ [ @@ -597,20 +575,6 @@ _meta: 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', ], - assetsMap: new Map([ - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', - Buffer.from(sourceData.FIELDS), - ], - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', - Buffer.from(sourceData.MANIFEST), - ], - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', - Buffer.from(sourceData.TRANSFORM), - ], - ]), archiveIterator: createArchiveIteratorFromMap( new Map([ [ @@ -895,16 +859,6 @@ _meta: 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', ], - assetsMap: new Map([ - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', - Buffer.from(sourceData.FIELDS), - ], - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', - Buffer.from(sourceData.TRANSFORM), - ], - ]), archiveIterator: createArchiveIteratorFromMap( new Map([ [ @@ -1135,16 +1089,6 @@ _meta: 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', ], - assetsMap: new Map([ - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', - sourceData.MANIFEST, - ], - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', - sourceData.TRANSFORM, - ], - ]), archiveIterator: createArchiveIteratorFromMap( new Map([ [ @@ -1251,16 +1195,6 @@ _meta: 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', ], - assetsMap: new Map([ - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', - sourceData.MANIFEST, - ], - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', - sourceData.TRANSFORM, - ], - ]), archiveIterator: createArchiveIteratorFromMap( new Map([ [ @@ -1358,20 +1292,6 @@ _meta: 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', ], - assetsMap: new Map([ - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', - sourceData.FIELDS, - ], - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', - sourceData.MANIFEST, - ], - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', - sourceData.TRANSFORM, - ], - ]), archiveIterator: createArchiveIteratorFromMap( new Map([ [ diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts index 4235b04d04983..f55d3539e0c6c 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts @@ -38,7 +38,7 @@ import { appContextService } from '../../..'; import { tagKibanaAssets } from './tag_assets'; import { getSpaceAwareSaveobjectsClients } from './saved_objects'; -const MAX_ASSETS_TO_INSTALL_IN_PARALLEL = 100; +const MAX_ASSETS_TO_INSTALL_IN_PARALLEL = 200; type SavedObjectsImporterContract = Pick; const formatImportErrorsForLog = (errors: SavedObjectsImportFailure[]) => @@ -276,7 +276,6 @@ export async function installKibanaAssetsAndReferences({ const kibanaAssetsArchiveIterator = getKibanaAssetsArchiveIterator(packageInstallContext); if (installedPkg) { - // TODO condition if security rule not delete await deleteKibanaSavedObjectsAssets({ installedPkg, spaceId }); } let installedKibanaAssetsRefs: KibanaAssetReference[] = []; @@ -288,12 +287,11 @@ export async function installKibanaAssetsAndReferences({ pkgName, kibanaAssetsArchiveIterator, }); - // if (installAsAdditionalSpace) { const assets = importedAssets.map( ({ id, type, destinationId }) => ({ id: destinationId ?? id, - originId: id, + ...(id !== destinationId ? { originId: id } : {}), type, } as KibanaAssetReference) ); From b1930684918aafd247dc8e730f3b8bb20efd4e3b Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 25 Feb 2025 10:47:49 -0500 Subject: [PATCH 07/21] remove TODO --- .../shared/fleet/common/types/models/epm.ts | 5 ----- .../steps/step_save_archive_entries.ts | 17 +++++++---------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts b/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts index 5ebbdf573dd07..22469de4e7885 100644 --- a/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts +++ b/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts @@ -141,11 +141,6 @@ export interface ArchiveIterator { export interface PackageInstallContext { packageInfo: InstallablePackage; - /** - * @deprecated Use `archiveIterator` to access the package archive entries - * without loading them all into memory at once. - */ - // assetsMap: AssetsMap; // TODO remove paths: string[]; archiveIterator: ArchiveIterator; } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts index 017bfd2f6301f..5e772955d7ea1 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts @@ -36,22 +36,19 @@ export async function stepSaveArchiveEntries(context: InstallContext) { installSource, }) ); - packageAssetRefs = packageAssetRefs.concat( - packageAssetResults.saved_objects.map((result) => ({ + packageAssetRefs = [ + ...packageAssetRefs, + ...packageAssetResults.saved_objects.map((result) => ({ id: result.id, - type: ASSETS_SAVED_OBJECT_TYPE, - })) - ); + type: ASSETS_SAVED_OBJECT_TYPE as typeof ASSETS_SAVED_OBJECT_TYPE, + })), + ]; assetsToSaveMap = new Map(); } await archiveIterator.traverseEntries(async (entry) => { - const assetType = getPathParts(entry.path).type as KibanaAssetType; - // Skip security rules to avoid storing to many things - if (assetType !== 'security_rule' && useStreaming) { - assetsToSaveMap.set(entry.path, entry.buffer); - } + assetsToSaveMap.set(entry.path, entry.buffer); if (assetsToSaveMap.size > 100) { await flushAssets(); } From 33b7566f3f29a819e03a72a360ad7595457eef4c Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 25 Feb 2025 10:49:00 -0500 Subject: [PATCH 08/21] Update x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts --- .../shared/fleet/server/services/epm/kibana/assets/install.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts index f55d3539e0c6c..08f6e3657f40d 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts @@ -272,7 +272,6 @@ export async function installKibanaAssetsAndReferences({ const { savedObjectsImporter, savedObjectTagAssignmentService, savedObjectTagClient } = getSpaceAwareSaveobjectsClients(spaceId); // This is where the memory consumption is rising up in the first place - // const kibanaAssets = getKibanaAssets(packageInstallContext); const kibanaAssetsArchiveIterator = getKibanaAssetsArchiveIterator(packageInstallContext); if (installedPkg) { From 6e3f028088cd66525fe97a15c2f9515e9a704511 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 25 Feb 2025 11:40:41 -0500 Subject: [PATCH 09/21] fix typo --- .../install_state_machine/steps/step_save_archive_entries.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts index 5e772955d7ea1..e36bf4dbccda9 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts @@ -6,7 +6,7 @@ */ import { ASSETS_SAVED_OBJECT_TYPE } from '../../../../../constants'; -import type { AssetsMap, KibanaAssetType, PackageAssetReference } from '../../../../../types'; +import type { AssetsMap, PackageAssetReference } from '../../../../../types'; import { removeArchiveEntries, saveArchiveEntriesFromAssetsMap } from '../../../archive/storage'; @@ -14,10 +14,9 @@ import { withPackageSpan } from '../../utils'; import type { InstallContext } from '../_state_machine_package_install'; import { INSTALL_STATES } from '../../../../../../common/types'; -import { getPathParts } from '../../../archive'; export async function stepSaveArchiveEntries(context: InstallContext) { - const { packageInstallContext, savedObjectsClient, installSource, useStreaming } = context; + const { packageInstallContext, savedObjectsClient, installSource } = context; const { packageInfo, archiveIterator } = packageInstallContext; From 08e9862ec3d5f9000d9e601842fc37b41a577386 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 25 Feb 2025 13:38:41 -0500 Subject: [PATCH 10/21] fix tests --- .../steps/step_save_archive_entries.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts index e36bf4dbccda9..54221723cb28b 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts @@ -6,7 +6,7 @@ */ import { ASSETS_SAVED_OBJECT_TYPE } from '../../../../../constants'; -import type { AssetsMap, PackageAssetReference } from '../../../../../types'; +import type { AssetsMap, KibanaAssetType, PackageAssetReference } from '../../../../../types'; import { removeArchiveEntries, saveArchiveEntriesFromAssetsMap } from '../../../archive/storage'; @@ -14,9 +14,10 @@ import { withPackageSpan } from '../../utils'; import type { InstallContext } from '../_state_machine_package_install'; import { INSTALL_STATES } from '../../../../../../common/types'; +import { getPathParts } from '../../../archive'; export async function stepSaveArchiveEntries(context: InstallContext) { - const { packageInstallContext, savedObjectsClient, installSource } = context; + const { packageInstallContext, savedObjectsClient, installSource, useStreaming } = context; const { packageInfo, archiveIterator } = packageInstallContext; @@ -47,7 +48,12 @@ export async function stepSaveArchiveEntries(context: InstallContext) { } await archiveIterator.traverseEntries(async (entry) => { - assetsToSaveMap.set(entry.path, entry.buffer); + const assetType = getPathParts(entry.path).type as KibanaAssetType; + if (assetType === 'security_rule' && useStreaming) { + // Skip security rules to avoid storing to many things + } else { + assetsToSaveMap.set(entry.path, entry.buffer); + } if (assetsToSaveMap.size > 100) { await flushAssets(); } From 60fa794ff95fb02e1df88c79befbbd9cb34b36ae Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 25 Feb 2025 15:14:36 -0500 Subject: [PATCH 11/21] fix tests --- .../steps/step_install_kibana_assets.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts index 22c785f568402..399fffd430878 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts @@ -98,10 +98,11 @@ export async function cleanUpKibanaAssetsStep(context: InstallContext) { * */ export async function cleanUpUnusedKibanaAssetsStep(context: InstallContext) { - const { logger, installedPkg, packageInstallContext, spaceId, installedKibanaAssetsRefs } = - context; + const { logger, installedPkg, packageInstallContext, spaceId, kibanaAssetPromise } = context; const { packageInfo } = packageInstallContext; + const installedKibanaAssetsRefs = await kibanaAssetPromise; + if (!installedKibanaAssetsRefs) { return; } From 64e9345461bc253a6050e5dfa22f898ee3167a3e Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 26 Feb 2025 08:49:50 -0500 Subject: [PATCH 12/21] fix test --- .../shared/fleet/server/services/epm/kibana/assets/install.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts index 08f6e3657f40d..d9ca7975b3620 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts @@ -290,7 +290,7 @@ export async function installKibanaAssetsAndReferences({ ({ id, type, destinationId }) => ({ id: destinationId ?? id, - ...(id !== destinationId ? { originId: id } : {}), + ...(destinationId ? { originId: id } : {}), type, } as KibanaAssetReference) ); From 0c1e4b62309f03efef698f6d6adc75d6a210a8bb Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 26 Feb 2025 10:25:30 -0500 Subject: [PATCH 13/21] fix tests --- .../steps/step_install_kibana_assets.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts index 399fffd430878..22c785f568402 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts @@ -98,11 +98,10 @@ export async function cleanUpKibanaAssetsStep(context: InstallContext) { * */ export async function cleanUpUnusedKibanaAssetsStep(context: InstallContext) { - const { logger, installedPkg, packageInstallContext, spaceId, kibanaAssetPromise } = context; + const { logger, installedPkg, packageInstallContext, spaceId, installedKibanaAssetsRefs } = + context; const { packageInfo } = packageInstallContext; - const installedKibanaAssetsRefs = await kibanaAssetPromise; - if (!installedKibanaAssetsRefs) { return; } From 05ae82775db7d9d8cdf1a4845ecf49bccc025777 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Sun, 2 Mar 2025 18:50:16 -0500 Subject: [PATCH 14/21] fix tests --- .../server/services/epm/kibana/assets/install.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts index d9ca7975b3620..ba253d2b745c0 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts @@ -17,7 +17,7 @@ import type { Logger, } from '@kbn/core/server'; import { createListStream } from '@kbn/utils'; -import { partition, chunk } from 'lodash'; +import { partition, chunk, once } from 'lodash'; import { getPathParts } from '../../archive'; import { KibanaAssetType, KibanaSavedObjectType } from '../../../../types'; @@ -116,14 +116,20 @@ export async function installKibanaAssets(options: { const { kibanaAssetsArchiveIterator, savedObjectsClient, savedObjectsImporter, logger } = options; // Todo check if we put this behind a condition - await installManagedIndexPattern({ - savedObjectsClient, - savedObjectsImporter, - }); let assetsToInstall: ArchiveAsset[] = []; let res: SavedObjectsImportSuccess[] = []; + + const installManagedIndexPatternOnce = once(() => { + installManagedIndexPattern({ + savedObjectsClient, + savedObjectsImporter, + }); + }); + async function flushAssetsToInstall() { + await installManagedIndexPatternOnce(); + const installedAssets = await installKibanaSavedObjects({ logger, savedObjectsImporter, From a6897336b8c6fcf8c28603361d3b556017cea684 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Sun, 2 Mar 2025 20:04:27 -0500 Subject: [PATCH 15/21] fix linting --- .../fleet/server/services/epm/kibana/assets/install.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts index ba253d2b745c0..46356cac89551 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts @@ -120,12 +120,12 @@ export async function installKibanaAssets(options: { let assetsToInstall: ArchiveAsset[] = []; let res: SavedObjectsImportSuccess[] = []; - const installManagedIndexPatternOnce = once(() => { + const installManagedIndexPatternOnce = once(() => installManagedIndexPattern({ savedObjectsClient, savedObjectsImporter, - }); - }); + }) + ); async function flushAssetsToInstall() { await installManagedIndexPatternOnce(); From a872119786904e4f5315b1b9905ef74645d3dc7c Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Sun, 2 Mar 2025 21:06:17 -0500 Subject: [PATCH 16/21] missing commit --- .../plugins/shared/fleet/server/services/epm/registry/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts index f9a374fc3d5a0..6a2cb9d7b7143 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts @@ -396,7 +396,7 @@ export async function getPackage( const { paths, assetsMap, archiveIterator } = await unpackBufferToAssetsMap({ archiveBuffer, contentType, - useStreaming: options?.fetchFullAssetsMap, + useStreaming: options?.fetchFullAssetsMap === false, }); if (!packageInfo) { From e7d06f4addff04b0435675c615c6c7fc11ff0f86 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Sun, 2 Mar 2025 21:23:47 -0500 Subject: [PATCH 17/21] refacto --- .../shared/fleet/server/services/epm/packages/install.ts | 2 +- .../shared/fleet/server/services/epm/registry/index.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts index b4aa9eb36e600..24f96833bbee8 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts @@ -490,7 +490,7 @@ async function installPackageFromRegistry({ latestPkg ? Promise.resolve(latestPkg) : queryLatest(), Registry.getPackage(pkgName, pkgVersion, { ignoreUnverified: force && !neverIgnoreVerificationError, - fetchFullAssetsMap: false, + useStreaming: true, }), ]); const packageInstallContext: PackageInstallContext = { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts index 6a2cb9d7b7143..3276d14ed197f 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.ts @@ -359,7 +359,7 @@ async function getPackageInfoFromArchiveOrCache( export async function getPackage( name: string, version: string, - options?: { ignoreUnverified?: boolean; fetchFullAssetsMap?: boolean } + options?: { ignoreUnverified?: boolean; useStreaming?: boolean } ): Promise<{ paths: string[]; packageInfo: ArchivePackage; @@ -396,7 +396,7 @@ export async function getPackage( const { paths, assetsMap, archiveIterator } = await unpackBufferToAssetsMap({ archiveBuffer, contentType, - useStreaming: options?.fetchFullAssetsMap === false, + useStreaming: options?.useStreaming, }); if (!packageInfo) { From f881e57eb4685963fb048d2bb8e90e9300e32c17 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 3 Mar 2025 08:28:05 -0500 Subject: [PATCH 18/21] fix tests --- .../apis/epm/update_assets.ts | 93 ++++++++++--------- 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 17d54786245af..ce37a83924f17 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -10,6 +10,7 @@ import { FLEET_INSTALL_FORMAT_VERSION } from '@kbn/fleet-plugin/server/constants import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; +import { sortBy } from 'lodash'; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; @@ -336,50 +337,56 @@ export default function (providerContext: FtrProviderContext) { id: 'all_assets', }); - expect(res.attributes).eql({ + expect({ + ...res.attributes, + installed_kibana: sortBy(res.attributes.installed_kibana, ['id']), + }).eql({ installed_kibana_space_id: 'default', - installed_kibana: [ - { - id: 'sample_dashboard', - type: 'dashboard', - }, - { - id: 'sample_lens', - type: 'lens', - }, - { - id: 'sample_visualization', - type: 'visualization', - }, - { - id: 'sample_search2', - type: 'search', - }, - { - id: 'sample_ml_module', - type: 'ml-module', - }, - { - id: 'sample_security_rule', - type: 'security-rule', - }, - { - id: 'sample_csp_rule_template2', - type: 'csp-rule-template', - }, - { - id: 'sample_osquery_pack_asset', - type: 'osquery-pack-asset', - }, - { - id: 'sample_osquery_saved_query', - type: 'osquery-saved-query', - }, - { - id: 'sample_tag', - type: 'tag', - }, - ], + installed_kibana: sortBy( + [ + { + id: 'sample_dashboard', + type: 'dashboard', + }, + { + id: 'sample_lens', + type: 'lens', + }, + { + id: 'sample_visualization', + type: 'visualization', + }, + { + id: 'sample_search2', + type: 'search', + }, + { + id: 'sample_ml_module', + type: 'ml-module', + }, + { + id: 'sample_security_rule', + type: 'security-rule', + }, + { + id: 'sample_csp_rule_template2', + type: 'csp-rule-template', + }, + { + id: 'sample_osquery_pack_asset', + type: 'osquery-pack-asset', + }, + { + id: 'sample_osquery_saved_query', + type: 'osquery-saved-query', + }, + { + id: 'sample_tag', + type: 'tag', + }, + ], + 'id' + ), installed_es: [ { id: 'all_assets', From f94a365ca14a5ea3acc14b540af76324f9dcbd9d Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 3 Mar 2025 13:50:28 +0000 Subject: [PATCH 19/21] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- x-pack/test/fleet_api_integration/apis/epm/update_assets.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index ce37a83924f17..8611bcd0932e9 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -8,9 +8,9 @@ import expect from '@kbn/expect'; import { FLEET_INSTALL_FORMAT_VERSION } from '@kbn/fleet-plugin/server/constants'; +import { sortBy } from 'lodash'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; -import { sortBy } from 'lodash'; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; From 227a7565c7761755d2fee3a22793578a9e274b5b Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 3 Mar 2025 14:08:08 -0500 Subject: [PATCH 20/21] fix tests --- .../epm/__snapshots__/bulk_get_assets.snap | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/bulk_get_assets.snap b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/bulk_get_assets.snap index 37c7882fff203..3ff5a36d9147b 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/bulk_get_assets.snap +++ b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/bulk_get_assets.snap @@ -98,6 +98,12 @@ Array [ "id": "metrics-all_assets.test_metrics@custom", "type": "component_template", }, + Object { + "appLink": "", + "attributes": Object {}, + "id": "sample_csp_rule_template", + "type": "csp-rule-template", + }, Object { "appLink": "/app/dashboards#/view/sample_dashboard", "attributes": Object { @@ -117,39 +123,21 @@ Array [ "type": "dashboard", }, Object { - "appLink": "/app/lens#/edit/sample_lens", - "attributes": Object { - "description": "", - "title": "sample-lens", - }, - "id": "sample_lens", - "type": "lens", - }, - Object { - "appLink": "/app/visualize#/edit/sample_visualization", + "appLink": "/app/management/kibana/dataViews/dataView/test-*", "attributes": Object { - "description": "sample visualization update", - "title": "sample vis title", + "title": "test-*", }, - "id": "sample_visualization", - "type": "visualization", + "id": "test-*", + "type": "index-pattern", }, Object { - "appLink": "/app/discover#/view/sample_search", + "appLink": "/app/lens#/edit/sample_lens", "attributes": Object { "description": "", - "title": "All logs [Logs Kafka] ECS", - }, - "id": "sample_search", - "type": "search", - }, - Object { - "appLink": "/app/management/kibana/dataViews/dataView/test-*", - "attributes": Object { - "title": "test-*", + "title": "sample-lens", }, - "id": "test-*", - "type": "index-pattern", + "id": "sample_lens", + "type": "lens", }, Object { "appLink": "/app/ml/supplied_configurations/?_a=(supplied_configurations%3A(queryText%3A'Nginx%20access%20logs'))", @@ -160,20 +148,6 @@ Array [ "id": "sample_ml_module", "type": "ml-module", }, - Object { - "appLink": "", - "attributes": Object { - "description": "Identifies a suspicious parent child process relationship with cmd.exe descending from svchost.exe", - }, - "id": "sample_security_rule", - "type": "security-rule", - }, - Object { - "appLink": "", - "attributes": Object {}, - "id": "sample_csp_rule_template", - "type": "csp-rule-template", - }, Object { "appLink": "", "attributes": Object {}, @@ -188,6 +162,23 @@ Array [ "id": "sample_osquery_saved_query", "type": "osquery-saved-query", }, + Object { + "appLink": "/app/discover#/view/sample_search", + "attributes": Object { + "description": "", + "title": "All logs [Logs Kafka] ECS", + }, + "id": "sample_search", + "type": "search", + }, + Object { + "appLink": "", + "attributes": Object { + "description": "Identifies a suspicious parent child process relationship with cmd.exe descending from svchost.exe", + }, + "id": "sample_security_rule", + "type": "security-rule", + }, Object { "appLink": "", "attributes": Object { @@ -196,5 +187,14 @@ Array [ "id": "sample_tag", "type": "tag", }, + Object { + "appLink": "/app/visualize#/edit/sample_visualization", + "attributes": Object { + "description": "sample visualization update", + "title": "sample vis title", + }, + "id": "sample_visualization", + "type": "visualization", + }, ] `; From e8cd6f4ccd6d880557ba26a386586ad9ea33aec6 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 4 Mar 2025 09:23:14 -0500 Subject: [PATCH 21/21] clean comment --- .../shared/fleet/server/services/epm/kibana/assets/install.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts index 46356cac89551..0ca3e79f58616 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts @@ -115,8 +115,6 @@ export async function installKibanaAssets(options: { }): Promise { const { kibanaAssetsArchiveIterator, savedObjectsClient, savedObjectsImporter, logger } = options; - // Todo check if we put this behind a condition - let assetsToInstall: ArchiveAsset[] = []; let res: SavedObjectsImportSuccess[] = [];