[Security Solution][Detection Engine] fixes siem-signal update when i…

…t was reindexed from v7 to v8 (elastic#206119)

## Summary

 - addresses elastic/security-team#11440


### Testing

1. Create cloud env of 7.17 version, (East US 2 (Virginia) on Azurem
where 8.18 snapshot available)
2. Create rule
3. Generate alerts
4. Create cloud env of 8.18 from existing 7.x snapshot (Restore snapshot
data option)
5. Connect local Kibana of 8.18 from mirror branch of this
one(elastic#206120)
6. Add to Kibana dev config following options to enable Upgrade
assistant(UA) showing outdated indices
    ```yml
    xpack.upgrade_assistant.featureSet:
      mlSnapshots: true
      migrateDataStreams: true
      migrateSystemIndices: true
      reindexCorrectiveActions: true
    ```  
7. When Kibana started DO NOT visit Detection rule or any Security  page
8. Open KIbana Upgrade Assistant, 
9. Got to step 3 - Review deprecated settings and resolve issues
11. Click Elasticsearch section
12. Find outdated .siem-signals-* index
13. Reindex it
14. Visit detection page to ensure index API updated mappings

Visit to that page should initiate `POST /api/detection_engine/index`,
which updates mappings

Subsequent index status check should return: 

```JSON
GET kbn:/api/detection_engine/index

// should return

{
  "name": ".alerts-security.alerts-default",
  "index_mapping_outdated": false
}
```

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts

@@ -102,9 +102,20 @@ export const createDetectionIndex = async (
   const aadIndexAliasName = ruleDataService.getResourceName(`security.alerts-${spaceId}`);
 
   if (await templateNeedsUpdate({ alias: index, esClient })) {
+    const reIndexedIndexPatterns = await getReIndexedV8IndexPatterns({ index, esClient });
+    const template = getSignalsTemplate(index, aadIndexAliasName, spaceId) as Record<
+      string,
+      unknown
+    >;
+
+    // addresses https://github.com/elastic/security-team/issues/11440
+    if (reIndexedIndexPatterns.length > 0 && Array.isArray(template.index_patterns)) {
+      template.index_patterns.push(...reIndexedIndexPatterns);
+    }
+
     await esClient.indices.putIndexTemplate({
       name: index,
-      body: getSignalsTemplate(index, aadIndexAliasName, spaceId) as Record<string, unknown>,
+      body: template,
     });
   }
   // Check if the old legacy siem signals template exists and remove it
@@ -209,3 +220,25 @@ const addIndexAliases = async ({
   };
   await esClient.indices.updateAliases({ body: aliasActions });
 };
+
+/**
+ * checks if indices under alias were reIndexed from v7 to v8(prefixed with '.reindexed-v8-')
+ * returns wildcard index patterns to include these indices and possible rollovers in index template
+ */
+const getReIndexedV8IndexPatterns = async ({
+  esClient,
+  index,
+}: {
+  esClient: ElasticsearchClient;
+  index: string;
+}): Promise<string[]> => {
+  const V8_PREFIX = '.reindexed-v8-';
+  const indices = await esClient.indices.getAlias({ index: `${index}-*`, name: index });
+  return Object.keys(indices).reduce<string[]>((acc, concreteIndexName) => {
+    if (concreteIndexName.startsWith(V8_PREFIX)) {
+      acc.push(`${V8_PREFIX}${index.replace(/^\./, '')}-*`);
+    }
+
+    return acc;
+  }, []);
+};

x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/data.json

@@ -0,0 +1,12 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "1",
+    "index": ".reindexed-v8-siem-signals-default-000001",
+    "source": {
+      "@timestamp": "2020-10-10T00:00:00.000Z",
+      "signal": {}
+    },
+    "type": "_doc"
+  }
+}

x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/mappings.json

@@ -0,0 +1,31 @@
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+      ".siem-signals-default": {
+        "is_write_index": true
+      },
+      ".siem-signals-default-000001": {}
+    },
+    "index": ".reindexed-v8-siem-signals-default-000001",
+    "mappings": {
+      "_meta": {
+        "version": 1
+      },
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "signal": { "type": "object" }
+      }
+    },
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": ".siem-signals-default",
+          "rollover_alias": ".siem-signals-default"
+        }
+      }
+    }
+  }
+}

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts

@@ -82,6 +82,43 @@ export default ({ getService }: FtrProviderContext) => {
           );
         });
       });
+
+      describe('with reIndexed from 7.xto 8.x .siem-signals index', () => {
+        beforeEach(async () => {
+          await esArchiver.load(
+            'x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals'
+          );
+        });
+
+        afterEach(async () => {
+          await esArchiver.unload(
+            'x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals'
+          );
+          await es.indices.delete({
+            index: '.reindexed-v8-siem-signals-default-000002',
+            ignore_unavailable: true,
+          });
+        });
+
+        it('should report that alerts index is outdated', async () => {
+          const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(200);
+          expect(body).to.eql({
+            index_mapping_outdated: true,
+            name: `${DEFAULT_ALERTS_INDEX}-default`,
+          });
+        });
+
+        it('should update index mappings', async () => {
+          await supertest
+            .post(DETECTION_ENGINE_INDEX_URL)
+            .set('kbn-xsrf', 'true')
+            .send()
+            .expect({ acknowledged: true });
+
+          const { body: indexStatusBody } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send();
+          expect(indexStatusBody.index_mapping_outdated).to.be(false);
+        });
+      });
     });
   });
 };