Skip to content

Commit

Permalink
Add related.entity to cloudtrail integration (#11115)
Browse files Browse the repository at this point in the history 
* Add  to cloudtrail integration

* Add another found key

* Update docs

* Fix tests

* Append entities based on event name

* Change structure of entity pipeline and add tests

* Remove keys

* Replace 2 brackets by 3 entity.yml

* Change implementation to painless

* Move processor to inline inside deafault.yml

* Add description to

* Update packages/aws/changelog.yml

Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>

* Add helper function to add multiple entities at the same time

* Use standard double quotes in strings

* Update docs

* Fix bracket spacing

---------

Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>
  • Loading branch information
@romulets @andrewkroh
romulets and andrewkroh authored Sep 27, 2024
1 parent e1fd1ea commit 83a5af6
Show file tree
Hide file tree
Showing 48 changed files with 1,357 additions and 2 deletions.
5 changes: 5 additions & 0 deletions packages/aws/changelog.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "2.25.0-preview01"
changes:
- description: Add related.entity field.
type: enhancement
link: https://github.com/elastic/integrations/pull/11115
- version: "2.25.0"
changes:
- description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
Expand Down
6 changes: 6 additions & 0 deletions ...s/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@
"name": "admin"
},
"related": {
"entity": [
"EX_PRINCIPAL_ID",
"arn:aws:iam::123456789012:user/Alice",
"Bob",
"Alice"
],
"user": [
"Alice",
"Bob"
Expand Down
23 changes: 23 additions & 0 deletions ...ges/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,18 @@
"info"
]
},
"related": {
"entity": [
"AROAIN5ATK5U7KEXAMPLE:JohnRole1",
"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1",
"Role2WithTags",
"JohnDoe",
"arn:aws:iam::111122223333:role/JohnRole2",
"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags",
"arn:aws:iam::111111111111:role/JohnRole1",
"arn:aws:iam::111111111111:role/JohnRole2"
]
},
"source": {
"address": "81.2.69.144",
"geo": {
Expand Down Expand Up @@ -228,6 +240,17 @@
"info"
]
},
"related": {
"entity": [
"AROAIN5ATK5U7KEXAMPLE:JohnRole1",
"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1",
"Role2WithTags",
"JohnDoe",
"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags",
"arn:aws:iam::111111111111:role/JohnRole1",
"arn:aws:iam::111111111111:role/JohnRole2"
]
},
"source": {
"address": "81.2.69.144",
"geo": {
Expand Down
10 changes: 10 additions & 0 deletions ...aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@
]
},
"related": {
"entity": [
"0123456789012",
"Alice",
"arn:aws:iam::0123456789012:user/Alice"
],
"user": [
"Alice"
]
Expand Down Expand Up @@ -110,6 +115,11 @@
]
},
"related": {
"entity": [
"0123456789012",
"Alice",
"arn:aws:iam::0123456789012:user/Alice"
],
"user": [
"Alice"
]
Expand Down
1 change: 1 addition & 0 deletions ...s/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@
"path": "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz"
},
"related": {
"entity": [],
"hash": [
"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987"
]
Expand Down
18 changes: 18 additions & 0 deletions ...s/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@
]
},
"related": {
"entity": [
"AIDACKCEVSQ6C2EXAMPLE",
"JohnDoe",
"arn:aws:iam::111122223333:user/JohnDoe"
],
"user": [
"JohnDoe"
]
Expand Down Expand Up @@ -158,6 +163,11 @@
]
},
"related": {
"entity": [
"AIDACKCEVSQ6C2EXAMPLE",
"JaneDoe",
"arn:aws:iam::111122223333:user/JaneDoe"
],
"user": [
"JaneDoe"
]
Expand Down Expand Up @@ -270,6 +280,14 @@
"info"
]
},
"related": {
"entity": [
"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName",
"arn:aws:iam::123456789012:role/RoleToBeAssumed",
"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName",
"RoleToBeAssumed"
]
},
"source": {
"address": "89.160.20.156",
"as": {
Expand Down
7 changes: 7 additions & 0 deletions ...s/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,13 @@
]
},
"related": {
"entity": [
"EXAMPLE_ID",
"Bob",
"EXAMPLE_KEY_ID",
"Alice",
"arn:aws:iam::0123456789012:user/Alice"
],
"user": [
"Alice",
"Bob"
Expand Down
1 change: 1 addition & 0 deletions packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-db-instance-json.log
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"awsRegion":"us-east-1","eventCategory":"Management","eventID":"d82a0cd1-6987-459a-b7bc-557a06bf16f2","eventName":"CreateDBInstance","eventSource":"rds.amazonaws.com","eventTime":"2024-09-11T09:29:51Z","eventType":"AwsApiCall","eventVersion":"1.08","readOnly":false,"recipientAccountId":"1010101010101","requestID":"b51e7190-610a-40c7-bb1c-a0895e3518f8","requestParameters":{"allocatedStorage":20,"dBInstanceClass":"db.t3.micro","dBInstanceIdentifier":"test-cloudtrail-event-instance-14340","engine":"mysql","masterUserPassword":"HIDDEN_DUE_TO_SECURITY_REASONS","masterUsername":"admin"},"responseElements":{"allocatedStorage":20,"associatedRoles":[],"autoMinorVersionUpgrade":true,"backupRetentionPeriod":1,"backupTarget":"region","cACertificateIdentifier":"rds-ca-rsa2048-g1","certificateDetails":{"cAIdentifier":"rds-ca-rsa2048-g1"},"copyTagsToSnapshot":false,"customerOwnedIpEnabled":false,"dBInstanceArn":"arn:aws:rds:us-east-1:1010101010101:db:test-cloudtrail-event-instance-14340","dBInstanceClass":"db.t3.micro","dBInstanceIdentifier":"test-cloudtrail-event-instance-14340","dBInstanceStatus":"creating","dBParameterGroups":[{"dBParameterGroupName":"default.mysql8.0","parameterApplyStatus":"in-sync"}],"dBSecurityGroups":[],"dBSubnetGroup":{"dBSubnetGroupDescription":"default","dBSubnetGroupName":"default","subnetGroupStatus":"Complete","subnets":[{"subnetAvailabilityZone":{"name":"us-east-1d"},"subnetIdentifier":"subnet-c4bf5e9b","subnetOutpost":{},"subnetStatus":"Active"},{"subnetAvailabilityZone":{"name":"us-east-1a"},"subnetIdentifier":"subnet-0a0bee6c","subnetOutpost":{},"subnetStatus":"Active"},{"subnetAvailabilityZone":{"name":"us-east-1e"},"subnetIdentifier":"subnet-37391109","subnetOutpost":{},"subnetStatus":"Active"},{"subnetAvailabilityZone":{"name":"us-east-1b"},"subnetIdentifier":"subnet-fee506df","subnetOutpost":{},"subnetStatus":"Active"},{"subnetAvailabilityZone":{"name":"us-east-1f"},"subnetIdentifier":"subnet-bf6ab5b1","subnetOutpost":{},"subnetStatus":"Active"},{"subnetAvailabilityZone":{"name":"us-east-1c"},"subnetIdentifier":"subnet-8bdf6bc6","subnetOutpost":{},"subnetStatus":"Active"}],"vpcId":"vpc-73d2e309"},"dbInstancePort":0,"dbiResourceId":"db-ANY6I3FNUJC7WQKYS5RFPU7ORM","dedicatedLogVolume":false,"deletionProtection":false,"domainMemberships":[],"engine":"mysql","engineLifecycleSupport":"open-source-rds-extended-support","engineVersion":"8.0.35","httpEndpointEnabled":false,"iAMDatabaseAuthenticationEnabled":false,"licenseModel":"general-public-license","masterUsername":"admin","monitoringInterval":0,"multiAZ":false,"networkType":"IPV4","optionGroupMemberships":[{"optionGroupName":"default:mysql-8-0","status":"in-sync"}],"pendingModifiedValues":{"masterUserPassword":"HIDDEN_DUE_TO_SECURITY_REASONS"},"performanceInsightsEnabled":false,"preferredBackupWindow":"09:23-09:53","preferredMaintenanceWindow":"sun:06:55-sun:07:25","publiclyAccessible":true,"readReplicaDBInstanceIdentifiers":[],"storageEncrypted":false,"storageThroughput":0,"storageType":"gp2","tagList":[],"vpcSecurityGroups":[{"status":"active","vpcSecurityGroupId":"sg-4e483165"}]},"sourceIPAddress":"216.160.83.56","tlsDetails":{"cipherSuite":"TLS_AES_128_GCM_SHA256","clientProvidedHostHeader":"rds.us-east-1.amazonaws.com","tlsVersion":"TLSv1.3"},"userAgent":"aws-cli/2.14.5 Python/3.11.6 Darwin/23.6.0 exec-env/grimoire_7ea17849-045c-4e81-8a81-d0822d5c4aaf exe/x86_64 prompt/off command/rds.create-db-instance","userIdentity":{"accessKeyId":"ACCESS_KEY_EXAMPLE","accountId":"1010101010101","arn":"arn:aws:iam::1010101010101:user/testcloudtrail@elastic.co","principalId":"AIDA2IBR2EZTJMPOR52WV","type":"IAMUser","userName":"testcloudtrail@elastic.co"}}
Loading

0 comments on commit 83a5af6

Please sign in to comment.