From 5c670378d44d0af7f5c582fe06b5013008df56d4 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 24 Jan 2025 11:22:30 +0000 Subject: [PATCH] [Security Solution][Detection Engine] fixes siem-signal update when it was reindexed from v7 to v8 (#206119) ## Summary - addresses https://github.com/elastic/security-team/issues/11440 ### Testing 1. Create cloud env of 7.17 version, (East US 2 (Virginia) on Azurem where 8.18 snapshot available) 2. Create rule 3. Generate alerts 4. Create cloud env of 8.18 from existing 7.x snapshot (Restore snapshot data option) 5. Connect local Kibana of 8.18 from mirror branch of this one(https://github.com/elastic/kibana/pull/206120) 6. Add to Kibana dev config following options to enable Upgrade assistant(UA) showing outdated indices ```yml xpack.upgrade_assistant.featureSet: mlSnapshots: true migrateDataStreams: true migrateSystemIndices: true reindexCorrectiveActions: true ``` 7. When Kibana started DO NOT visit Detection rule or any Security page 8. Open KIbana Upgrade Assistant, 9. Got to step 3 - Review deprecated settings and resolve issues 11. Click Elasticsearch section 12. Find outdated .siem-signals-* index 13. Reindex it 14. Visit detection page to ensure index API updated mappings Visit to that page should initiate `POST /api/detection_engine/index`, which updates mappings Subsequent index status check should return: ```JSON GET kbn:/api/detection_engine/index // should return { "name": ".alerts-security.alerts-default", "index_mapping_outdated": false } ``` --- .../routes/index/create_index_route.ts | 35 +++++++++++++++++- .../reindexed_v8_siem_signals/data.json | 12 ++++++ .../reindexed_v8_siem_signals/mappings.json | 31 ++++++++++++++++ .../ess_specific_index_logic/create_index.ts | 37 +++++++++++++++++++ 4 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/data.json create mode 100644 x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/mappings.json diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts index 85fe5a2c29a1b..5257486dcb53a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts @@ -102,9 +102,20 @@ export const createDetectionIndex = async ( const aadIndexAliasName = ruleDataService.getResourceName(`security.alerts-${spaceId}`); if (await templateNeedsUpdate({ alias: index, esClient })) { + const reIndexedIndexPatterns = await getReIndexedV8IndexPatterns({ index, esClient }); + const template = getSignalsTemplate(index, aadIndexAliasName, spaceId) as Record< + string, + unknown + >; + + // addresses https://github.com/elastic/security-team/issues/11440 + if (reIndexedIndexPatterns.length > 0 && Array.isArray(template.index_patterns)) { + template.index_patterns.push(...reIndexedIndexPatterns); + } + await esClient.indices.putIndexTemplate({ name: index, - body: getSignalsTemplate(index, aadIndexAliasName, spaceId) as Record, + body: template, }); } // Check if the old legacy siem signals template exists and remove it @@ -209,3 +220,25 @@ const addIndexAliases = async ({ }; await esClient.indices.updateAliases({ body: aliasActions }); }; + +/** + * checks if indices under alias were reIndexed from v7 to v8(prefixed with '.reindexed-v8-') + * returns wildcard index patterns to include these indices and possible rollovers in index template + */ +const getReIndexedV8IndexPatterns = async ({ + esClient, + index, +}: { + esClient: ElasticsearchClient; + index: string; +}): Promise => { + const V8_PREFIX = '.reindexed-v8-'; + const indices = await esClient.indices.getAlias({ index: `${index}-*`, name: index }); + return Object.keys(indices).reduce((acc, concreteIndexName) => { + if (concreteIndexName.startsWith(V8_PREFIX)) { + acc.push(`${V8_PREFIX}${index.replace(/^\./, '')}-*`); + } + + return acc; + }, []); +}; diff --git a/x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/data.json b/x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/data.json new file mode 100644 index 0000000000000..55b98b0d4b59b --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/data.json @@ -0,0 +1,12 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": ".reindexed-v8-siem-signals-default-000001", + "source": { + "@timestamp": "2020-10-10T00:00:00.000Z", + "signal": {} + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/mappings.json b/x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/mappings.json new file mode 100644 index 0000000000000..43cd74e0838d0 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/mappings.json @@ -0,0 +1,31 @@ +{ + "type": "index", + "value": { + "aliases": { + ".siem-signals-default": { + "is_write_index": true + }, + ".siem-signals-default-000001": {} + }, + "index": ".reindexed-v8-siem-signals-default-000001", + "mappings": { + "_meta": { + "version": 1 + }, + "properties": { + "@timestamp": { + "type": "date" + }, + "signal": { "type": "object" } + } + }, + "settings": { + "index": { + "lifecycle": { + "name": ".siem-signals-default", + "rollover_alias": ".siem-signals-default" + } + } + } + } +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts index 8ed33f1b763f1..aa8abd2f33e50 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts @@ -82,6 +82,43 @@ export default ({ getService }: FtrProviderContext) => { ); }); }); + + describe('with reIndexed from 7.xto 8.x .siem-signals index', () => { + beforeEach(async () => { + await esArchiver.load( + 'x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals' + ); + }); + + afterEach(async () => { + await esArchiver.unload( + 'x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals' + ); + await es.indices.delete({ + index: '.reindexed-v8-siem-signals-default-000002', + ignore_unavailable: true, + }); + }); + + it('should report that alerts index is outdated', async () => { + const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(200); + expect(body).to.eql({ + index_mapping_outdated: true, + name: `${DEFAULT_ALERTS_INDEX}-default`, + }); + }); + + it('should update index mappings', async () => { + await supertest + .post(DETECTION_ENGINE_INDEX_URL) + .set('kbn-xsrf', 'true') + .send() + .expect({ acknowledged: true }); + + const { body: indexStatusBody } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send(); + expect(indexStatusBody.index_mapping_outdated).to.be(false); + }); + }); }); }); };