Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Rule Tuning] Update rules using NPC integration and non-ECS fields #3194

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
+31 −25
Open

[Rule Tuning] Update rules using NPC integration and non-ECS fields #3194

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e1abde7
[Rule Tuning] Update rules using NPC integration and non-ECS fields
brokensound77 Oct 16, 2023
4abb55a
Merge branch 'main' into update-npc-integration-ecs-fields
brokensound77 Oct 16, 2023
446e4d4
Merge branch 'main' into update-npc-integration-ecs-fields
brokensound77 Oct 16, 2023
e5f176d
switch from headers.content-type to mime_type
brokensound77 Oct 16, 2023
ce26596
Merge branch 'main' into update-npc-integration-ecs-fields
brokensound77 Nov 20, 2023
956d77b
update field to network_traffic.http.request.headers.authorization:*
brokensound77 Nov 21, 2023
d478741
update network_traffic integration schema
brokensound77 Dec 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions rules/network/command_and_control_cobalt_strike_beacon.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
creation_date = "2020/07/06"
integration = ["network_traffic"]
maturity = "production"
min_stack_comments = "New fields added: required_fields, related_integrations, setup"
min_stack_version = "8.3.0"
min_stack_comments = "Network integration fields updated for ECS compatibility (type -> network.protocol)"
min_stack_version = "8.10.0"
updated_date = "2023/10/16"

[rule]
Expand All @@ -21,7 +21,7 @@ false_positives = [
]
from = "now-9m"
index = ["packetbeat-*", "auditbeat-*", "filebeat-*", "logs-network_traffic.*"]
language = "lucene"
language = "eql"
license = "Elastic License v2"
name = "Cobalt Strike Command and Control Beacon"
note = """## Threat intel
Expand All @@ -37,12 +37,14 @@ rule_id = "cf53f532-9cc9-445a-9ae7-fced307ec53c"
severity = "high"
tags = ["Use Case: Threat Detection", "Tactic: Command and Control", "Domain: Endpoint"]
timestamp_override = "event.ingested"
type = "query"
type = "eql"

query = '''
((event.category: (network OR network_traffic) AND type: (tls OR http))
OR event.dataset: (network_traffic.tls OR network_traffic.http)
) AND destination.domain:/[a-z]{3}.stage.[0-9]{8}\..*/
any where
((event.category in ("network", "network_traffic") and network.protocol in ("tls", "http")) or
event.dataset in ("network_traffic.tls", "network_traffic.http")
) and
destination.domain regex~ """[a-z]{3}.stage.[0-9]{8}\..*"""
'''


Expand Down
17 changes: 10 additions & 7 deletions rules/network/command_and_control_fin7_c2_behavior.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
creation_date = "2020/07/06"
integration = ["network_traffic"]
maturity = "production"
min_stack_comments = "New fields added: required_fields, related_integrations, setup"
min_stack_version = "8.3.0"
min_stack_comments = "Network integration fields updated for ECS compatibility (type -> network.protocol)"
min_stack_version = "8.10.0"
updated_date = "2023/10/16"

[rule]
Expand All @@ -20,7 +20,7 @@ false_positives = [
]
from = "now-9m"
index = ["packetbeat-*", "auditbeat-*", "filebeat-*", "logs-network_traffic.*"]
language = "lucene"
language = "eql"
license = "Elastic License v2"
name = "Possible FIN7 DGA Command and Control Behavior"
note = """## Triage and analysis
Expand All @@ -34,12 +34,15 @@ rule_id = "4a4e23cf-78a2-449c-bac3-701924c269d3"
severity = "high"
tags = ["Use Case: Threat Detection", "Tactic: Command and Control", "Domain: Endpoint"]
timestamp_override = "event.ingested"
type = "query"
type = "eql"

query = '''
(event.dataset: (network_traffic.tls OR network_traffic.http) or
(event.category: (network OR network_traffic) AND type: (tls OR http) AND network.transport: tcp)) AND
destination.domain:/[a-zA-Z]{4,5}\.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us
any where
((event.category in ("network", "network_traffic") and network.protocol in ("tls", "http") and network.transport == "tcp") or
event.dataset in ("network_traffic.tls", "network_traffic.http")
) and
destination.domain regex~ """[a-zA-Z]{4,5}\.(pw|us|club|info|site|top)""" and
not destination.domain : "zoom.us"
'''


Expand Down
13 changes: 7 additions & 6 deletions rules/network/initial_access_unsecure_elasticsearch_node.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
creation_date = "2020/08/11"
integration = ["network_traffic"]
maturity = "production"
min_stack_comments = "New fields added: required_fields, related_integrations, setup"
min_stack_version = "8.3.0"
min_stack_comments = "Network integration fields updated for ECS compatibility (status -> http.response.status_phrase)"
min_stack_version = "8.10.0"
updated_date = "2023/10/16"

[rule]
Expand All @@ -20,7 +20,7 @@ false_positives = [
]
from = "now-9m"
index = ["packetbeat-*", "logs-network_traffic.*"]
language = "lucene"
language = "kuery"
license = "Elastic License v2"
name = "Inbound Connection to an Unsecure Elasticsearch Node"
note = """## Setup
Expand All @@ -38,9 +38,10 @@ timestamp_override = "event.ingested"
type = "query"

query = '''
(event.dataset: network_traffic.http OR (event.category: network_traffic AND network.protocol: http)) AND
status:OK AND destination.port:9200 AND network.direction:inbound AND NOT http.response.headers.content-type:"image/x-icon" AND NOT
_exists_:http.request.headers.authorization
(event.dataset:network_traffic.http or (event.category:network_traffic and network.protocol:http)) and
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because our query validation for integrations occurs separately from the base query validation (ECS + beats diff), there is a conflict where a rule cannot satisfy both validations. This is the first rule with an integration-specific field (that I am aware of), which results in it

  • failing base query validation because the fields do not exist in ECS or beats
  • passing integration validation

As a test, even if we added the fields to non-ecs-schema (as a bad hack), the rule still fails because the ECS/beats field is not known to the integration

Integration only fields:

  • http.response.status_phrase
  • network_traffic.http.request.headers.authorization

Unknown to integration

  • http.request.headers.authorization
  • status

With the existing process, the only way to rectify this is to split the rule, however that is not sustainable. We need to review this for potential refactor.

attempted cross-compatible query, which fails based on the above

(event.dataset:network_traffic.http or (event.category:network_traffic and network.protocol:http)) and
    (status:ok or http.response.status_phrase:ok) and
    destination.port:9200 and network.direction:inbound and
    not http.response.mime_type:"image/x-icon" and
    not (network_traffic.http.request.headers.authorization:* or http.request.headers.authorization:*)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reference the description above for further details and coexisting bugs

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks Justin for bringing this to our attention! I will have to dive a bit deeper, but I recall that we specifically allow it to validate ECS+beats OR integrations.

detection-rules/detection_rules/rule_validators.py

Lines 48 to 70 in a6c5cfc

def validate(self, data: QueryRuleData, meta: RuleMeta) -> None:
"""Validate the query, called from the parent which contains [metadata] information."""
if meta.query_schema_validation is False or meta.maturity == "deprecated":
# syntax only, which is done via self.ast
return
if isinstance(data, QueryRuleData) and data.language != 'lucene':
packages_manifest = load_integrations_manifests()
package_integrations = TOMLRuleContents.get_packaged_integrations(data, meta, packages_manifest)
validation_checks = {"stack": None, "integrations": None}
# validate the query against fields within beats
validation_checks["stack"] = self.validate_stack_combos(data, meta)
if package_integrations:
# validate the query against related integration fields
validation_checks["integrations"] = self.validate_integration(data, meta, package_integrations)
if (validation_checks["stack"] and not package_integrations):
raise validation_checks["stack"]
if (validation_checks["stack"] and validation_checks["integrations"]):
raise ValueError(f"Error in both stack and integrations checks: {validation_checks}")

If you review this code, validation_checks holds the state of ECS+Beats OR Integration validation checks. If both fail, then yes the rule should be reviewed because neither the integration's schema nor ECS, Non-ECS or Beats have the fields or the query is incorrect.

http.response.status_phrase:ok and destination.port:9200 and network.direction:inbound and
not http.response.mime_type:"image/x-icon" and
not network_traffic.http.request.headers.authorization:*
'''


Expand Down
10 changes: 5 additions & 5 deletions rules/network/lateral_movement_dns_server_overflow.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
creation_date = "2020/07/16"
integration = ["network_traffic"]
maturity = "production"
min_stack_comments = "New fields added: required_fields, related_integrations, setup"
min_stack_version = "8.3.0"
updated_date = "2023/08/01"
min_stack_comments = "Network integration fields updated for ECS compatibility (type -> network.protocol)"
min_stack_version = "8.10.0"
updated_date = "2023/10/16"

[rule]
author = ["Elastic"]
Expand Down Expand Up @@ -74,8 +74,8 @@ timestamp_override = "event.ingested"
type = "query"

query = '''
(event.dataset: network_traffic.dns or (event.category: (network or network_traffic) and destination.port: 53)) and
(event.dataset:zeek.dns or type:dns or event.type:connection) and network.bytes > 60000
(event.dataset:network_traffic.dns or (event.category:(network or network_traffic) and destination.port:53)) and
(event.dataset:zeek.dns or network.protocol:dns or event.type:connection) and network.bytes > 60000
'''


Expand Down