diff --git a/CLI.md b/CLI.md index 41a8ac98cbb..2e9421b9fea 100644 --- a/CLI.md +++ b/CLI.md @@ -85,12 +85,18 @@ Usage: detection_rules import-rules-to-repo [OPTIONS] [INPUT_FILE]... Import rules from json, toml, yaml, or Kibana exported rule file(s). Options: + -ac, --action-connector-import Include action connectors in export -e, --exceptions-import Include exceptions in export --required-only Only prompt for required fields -d, --directory DIRECTORY Load files from a directory -s, --save-directory DIRECTORY Save imported rules to a directory -se, --exceptions-directory DIRECTORY Save imported exceptions to a directory + -sa, --action-connectors-directory DIRECTORY + Save imported actions to a directory + -ske, --skip-errors Skip rule import errors + -da, --default-author TEXT Default author for rules missing one + -snv, --strip-none-values Strip None values from the rule -h, --help Show this message and exit. ``` @@ -290,13 +296,15 @@ Options: -id, --rule-id TEXT -o, --outfile PATH Name of file for exported rules -r, --replace-id Replace rule IDs with new IDs before export - --stack-version [7.10|7.11|7.12|7.13|7.14|7.15|7.16|7.8|7.9|8.0|8.1|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9|8.10|8.11|8.12|8.13|8.14|8.15] + --stack-version [7.8|7.9|7.10|7.11|7.12|7.13|7.14|7.15|7.16|8.0|8.1|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9|8.10|8.11|8.12|8.13|8.14] Downgrade a rule version to be compatible with older instances of Kibana -s, --skip-unsupported If `--stack-version` is passed, skip rule types which are unsupported (an error will be raised otherwise) --include-metadata Add metadata to the exported rules + -ac, --include-action-connectors + Include Action Connectors in export -e, --include-exceptions Include Exceptions Lists in export -h, --help Show this message and exit. ``` @@ -332,6 +340,7 @@ Options: --kibana-url TEXT -kp, --kibana-password TEXT -kc, --kibana-cookie TEXT Cookie from an authed session + --api-key TEXT --cloud-id TEXT ID of the cloud instance. Usage: detection_rules kibana import-rules [OPTIONS] @@ -344,7 +353,7 @@ Options: -id, --rule-id TEXT -o, --overwrite Overwrite existing rules -e, --overwrite-exceptions Overwrite exceptions in existing rules - -a, --overwrite-action-connectors + -ac, --overwrite-action-connectors Overwrite action connectors in existing rules -h, --help Show this message and exit. @@ -512,6 +521,7 @@ Options: --kibana-url TEXT -kp, --kibana-password TEXT -kc, --kibana-cookie TEXT Cookie from an authed session + --api-key TEXT --cloud-id TEXT ID of the cloud instance. Usage: detection_rules kibana export-rules [OPTIONS] @@ -520,15 +530,18 @@ Usage: detection_rules kibana export-rules [OPTIONS] Options: -d, --directory PATH Directory to export rules to [required] + -acd, --action-connectors-directory PATH + Directory to export action connectors to -ed, --exceptions-directory PATH Directory to export exceptions to -r, --rule-id TEXT Optional Rule IDs to restrict export to + -ac, --export-action-connectors + Include action connectors in export -e, --export-exceptions Include exceptions in export -s, --skip-errors Skip errors when exporting rules -sv, --strip-version Strip the version fields from all rules -h, --help Show this message and exit. - ``` Example of a rule exporting, with errors skipped diff --git a/detection_rules/action_connector.py b/detection_rules/action_connector.py new file mode 100644 index 00000000000..72cd948a1e0 --- /dev/null +++ b/detection_rules/action_connector.py @@ -0,0 +1,129 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. + +"""Dataclasses for Action.""" +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import List, Optional + +import pytoml +from marshmallow import EXCLUDE + +from .mixins import MarshmallowDataclassMixin +from .schemas import definitions +from .config import parse_rules_config + +RULES_CONFIG = parse_rules_config() + + +@dataclass(frozen=True) +class ActionConnectorMeta(MarshmallowDataclassMixin): + """Data stored in an Action Connector's [metadata] section of TOML.""" + + creation_date: definitions.Date + action_connector_name: str + rule_ids: List[definitions.UUIDString] + rule_names: List[str] + updated_date: definitions.Date + + # Optional fields + deprecation_date: Optional[definitions.Date] + comments: Optional[str] + maturity: Optional[definitions.Maturity] + + +@dataclass +class ActionConnector(MarshmallowDataclassMixin): + """Data object for rule Action Connector.""" + + id: str + attributes: dict + frequency: Optional[dict] + managed: Optional[bool] + type: Optional[str] + references: Optional[List] + + +@dataclass(frozen=True) +class TOMLActionConnectorContents(MarshmallowDataclassMixin): + """Object for action connector from TOML file.""" + + metadata: ActionConnectorMeta + action_connectors: List[ActionConnector] + + @classmethod + def from_action_connector_dict( + cls, + actions_dict: dict, + rule_list: dict, + ) -> "TOMLActionConnectorContents": + """Create a TOMLActionContents from a kibana rule resource.""" + rule_ids = [] + rule_names = [] + + for rule in rule_list: + rule_ids.append(rule["id"]) + rule_names.append(rule["name"]) + + # Format date to match schema + creation_date = datetime.strptime(actions_dict["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d") + updated_date = datetime.strptime(actions_dict["updated_at"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d") + metadata = { + "creation_date": creation_date, + "rule_ids": rule_ids, + "rule_names": rule_names, + "updated_date": updated_date, + "action_connector_name": f"Action Connector {actions_dict.get('id')}", + } + + return cls.from_dict({"metadata": metadata, "action_connectors": [actions_dict]}, unknown=EXCLUDE) + + def to_api_format(self) -> List[dict]: + """Convert the TOML Action Connector to the API format.""" + converted = [] + + for action in self.action_connectors: + converted.append(action.to_dict()) + return converted + + +@dataclass(frozen=True) +class TOMLActionConnector: + """Object for action connector from TOML file.""" + + contents: TOMLActionConnectorContents + path: Path + + @property + def name(self): + return self.contents.metadata.action_connector_name + + def save_toml(self): + """Save the action to a TOML file.""" + assert self.path is not None, f"Can't save action for {self.name} without a path" + # Check if self.path has a .toml extension + path = self.path + if path.suffix != ".toml": + # If it doesn't, add one + path = path.with_suffix(".toml") + with path.open("w") as f: + contents_dict = self.contents.to_dict() + # Sort the dictionary so that 'metadata' is at the top + sorted_dict = dict(sorted(contents_dict.items(), key=lambda item: item[0] != "metadata")) + pytoml.dump(sorted_dict, f) + + +def parse_action_connector_results_from_api(results: List[dict]) -> tuple[List[dict], List[dict]]: + """Filter Kibana export rule results for action connector dictionaries.""" + action_results = [] + non_action_results = [] + for result in results: + if result.get("type") != "action": + non_action_results.append(result) + else: + action_results.append(result) + + return action_results, non_action_results diff --git a/detection_rules/config.py b/detection_rules/config.py index a2348bfcae0..9416b9d535c 100644 --- a/detection_rules/config.py +++ b/detection_rules/config.py @@ -186,6 +186,7 @@ class RulesConfig: version_lock: Dict[str, dict] action_dir: Optional[Path] = None + action_connector_dir: Optional[Path] = None auto_gen_schema_file: Optional[Path] = None bbr_rules_dirs: Optional[List[Path]] = field(default_factory=list) bypass_version_lock: bool = False @@ -273,6 +274,8 @@ def parse_rules_config(path: Optional[Path] = None) -> RulesConfig: contents['exception_dir'] = base_dir.joinpath(directories.get('exception_dir')).resolve() if directories.get('action_dir'): contents['action_dir'] = base_dir.joinpath(directories.get('action_dir')).resolve() + if directories.get('action_connector_dir'): + contents['action_connector_dir'] = base_dir.joinpath(directories.get('action_connector_dir')).resolve() # version strategy contents['bypass_version_lock'] = loaded.get('bypass_version_lock', False) diff --git a/detection_rules/custom_rules.py b/detection_rules/custom_rules.py index 3b74be98790..1ffd0aded66 100644 --- a/detection_rules/custom_rules.py +++ b/detection_rules/custom_rules.py @@ -29,6 +29,11 @@ def create_config_content() -> str: config_content = { 'rule_dirs': ['rules'], 'bbr_rules_dirs': ['rules_building_block'], + 'directories': { + 'action_dir': 'actions', + 'action_connector_dir': 'action_connectors', + 'exception_dir': 'exceptions', + }, 'files': { 'deprecated_rules': 'etc/deprecated_rules.json', 'packages': 'etc/packages.yaml', @@ -98,6 +103,7 @@ def setup_config(directory: Path, kibana_version: str, overwrite: bool, enable_p ] directories = [ directory / 'actions', + directory / 'action_connectors', directory / 'exceptions', directory / 'rules', directory / 'rules_building_block', diff --git a/detection_rules/etc/_config.yaml b/detection_rules/etc/_config.yaml index 2c240f54104..cf63e90d233 100644 --- a/detection_rules/etc/_config.yaml +++ b/detection_rules/etc/_config.yaml @@ -19,6 +19,7 @@ normalize_kql_keywords: False # directories: # action_dir: actions # exception_dir: exceptions + # action_connector_dir: action_connectors # to set up a custom rules directory, copy this file to the root of the custom rules directory, which is set # using the environment variable CUSTOM_RULES_DIR diff --git a/detection_rules/generic_loader.py b/detection_rules/generic_loader.py index c316c28f71e..b9d3a1e73f2 100644 --- a/detection_rules/generic_loader.py +++ b/detection_rules/generic_loader.py @@ -11,6 +11,7 @@ import pytoml from .action import TOMLAction, TOMLActionContents +from .action_connector import TOMLActionConnector, TOMLActionConnectorContents from .config import parse_rules_config from .exception import TOMLException, TOMLExceptionContents from .rule_loader import dict_filter @@ -19,8 +20,8 @@ RULES_CONFIG = parse_rules_config() -GenericCollectionTypes = Union[TOMLAction, TOMLException] -GenericCollectionContentTypes = Union[TOMLActionContents, TOMLExceptionContents] +GenericCollectionTypes = Union[TOMLAction, TOMLActionConnector, TOMLException] +GenericCollectionContentTypes = Union[TOMLActionContents, TOMLActionConnectorContents, TOMLExceptionContents] def metadata_filter(**metadata) -> Callable[[GenericCollectionTypes], bool]: @@ -101,9 +102,9 @@ def _assert_new(self, item: GenericCollectionTypes) -> None: file_map = self.file_map name_map = self.name_map - assert not self.frozen, f"Unable to add item {item.name} {item.id} to a frozen collection" + assert not self.frozen, f"Unable to add item {item.name} to a frozen collection" assert item.name not in name_map, \ - f"Rule Name {item.name} for {item.id} collides with rule ID {name_map.get(item.name).id}" + f"Rule Name {item.name} collides with {name_map[item.name].name}" if item.path is not None: item_path = item.path.resolve() @@ -118,9 +119,18 @@ def add_item(self, item: GenericCollectionTypes) -> None: def load_dict(self, obj: dict, path: Optional[Path] = None) -> GenericCollectionTypes: """Load a dictionary into the collection.""" - is_exception = True if 'exceptions' in obj else False - contents = TOMLExceptionContents.from_dict(obj) if is_exception else TOMLActionContents.from_dict(obj) - item = TOMLException(path=path, contents=contents) + if 'exceptions' in obj: + contents = TOMLExceptionContents.from_dict(obj) + item = TOMLException(path=path, contents=contents) + elif 'actions' in obj: + contents = TOMLActionContents.from_dict(obj) + item = TOMLAction(path=path, contents=contents) + elif 'action_connectors' in obj: + contents = TOMLActionConnectorContents.from_dict(obj) + item = TOMLActionConnector(path=path, contents=contents) + else: + raise ValueError("Invalid object type") + self.add_item(item) return item @@ -178,6 +188,8 @@ def default(cls) -> 'GenericCollection': collection.load_directory(RULES_CONFIG.exception_dir) if RULES_CONFIG.action_dir: collection.load_directory(RULES_CONFIG.action_dir) + if RULES_CONFIG.action_connector_dir: + collection.load_directory(RULES_CONFIG.action_connector_dir) collection.freeze() cls.__default = collection diff --git a/detection_rules/kbwrap.py b/detection_rules/kbwrap.py index f10f9ff97d7..960752721c1 100644 --- a/detection_rules/kbwrap.py +++ b/detection_rules/kbwrap.py @@ -15,6 +15,8 @@ from .config import parse_rules_config from .cli_utils import multi_collection +from .action_connector import (TOMLActionConnector, TOMLActionConnectorContents, + parse_action_connector_results_from_api) from .exception import (TOMLException, TOMLExceptionContents, parse_exceptions_results_from_api) from .generic_loader import GenericCollection @@ -84,7 +86,7 @@ def upload_rule(ctx, rules: RuleCollection, replace_id): @multi_collection @click.option('--overwrite', '-o', is_flag=True, help='Overwrite existing rules') @click.option('--overwrite-exceptions', '-e', is_flag=True, help='Overwrite exceptions in existing rules') -@click.option('--overwrite-action-connectors', '-a', is_flag=True, +@click.option('--overwrite-action-connectors', '-ac', is_flag=True, help='Overwrite action connectors in existing rules') @click.pass_context def kibana_import_rules(ctx: click.Context, rules: RuleCollection, overwrite: Optional[bool] = False, @@ -95,10 +97,16 @@ def kibana_import_rules(ctx: click.Context, rules: RuleCollection, overwrite: Op rule_dicts = [r.contents.to_api_format() for r in rules] with kibana: cl = GenericCollection.default() - exception_dicts = [d.contents.to_api_format() for d in cl.items] + exception_dicts = [ + d.contents.to_api_format() for d in cl.items if isinstance(d.contents, TOMLExceptionContents) + ] + action_connectors_dicts = [ + d.contents.to_api_format() for d in cl.items if isinstance(d.contents, TOMLActionConnectorContents) + ] response, successful_rule_ids, results = RuleResource.import_rules( rule_dicts, exception_dicts, + action_connectors_dicts, overwrite=overwrite, overwrite_exceptions=overwrite_exceptions, overwrite_action_connectors=overwrite_action_connectors @@ -118,8 +126,12 @@ def kibana_import_rules(ctx: click.Context, rules: RuleCollection, overwrite: Op @kibana_group.command("export-rules") @click.option("--directory", "-d", required=True, type=Path, help="Directory to export rules to") +@click.option( + "--action-connectors-directory", "-acd", required=False, type=Path, help="Directory to export action connectors to" +) @click.option("--exceptions-directory", "-ed", required=False, type=Path, help="Directory to export exceptions to") @click.option("--rule-id", "-r", multiple=True, help="Optional Rule IDs to restrict export to") +@click.option("--export-action-connectors", "-ac", is_flag=True, help="Include action connectors in export") @click.option("--export-exceptions", "-e", is_flag=True, help="Include exceptions in export") @click.option("--skip-errors", "-s", is_flag=True, help="Skip errors when exporting rules") @click.option("--strip-version", "-sv", is_flag=True, help="Strip the version fields from all rules") @@ -127,8 +139,10 @@ def kibana_import_rules(ctx: click.Context, rules: RuleCollection, overwrite: Op def kibana_export_rules( ctx: click.Context, directory: Path, + action_connectors_directory: Optional[Path], exceptions_directory: Optional[Path], rule_id: Optional[Iterable[str]] = None, + export_action_connectors: bool = False, export_exceptions: bool = False, skip_errors: bool = False, strip_version: bool = False, @@ -145,6 +159,13 @@ def kibana_export_rules( if not exceptions_directory and export_exceptions: click.echo("Warning: Exceptions export requested, but no exceptions directory found") + # Handle Actions Connector Directory Location + if results and action_connectors_directory: + action_connectors_directory.mkdir(parents=True, exist_ok=True) + action_connectors_directory = action_connectors_directory or RULES_CONFIG.action_connector_dir + if not action_connectors_directory and export_action_connectors: + click.echo("Warning: Action Connector export requested, but no Action Connector directory found") + if results: directory.mkdir(parents=True, exist_ok=True) else: @@ -157,14 +178,20 @@ def kibana_export_rules( rules_count = results[-1]["exported_rules_count"] exception_list_count = results[-1]["exported_exception_list_count"] exception_list_item_count = results[-1]["exported_exception_list_item_count"] + action_connector_count = results[-1]["exported_action_connector_count"] # Parse rules results and exception results from API return rules_results = results[:rules_count] exception_results = results[rules_count:rules_count + exception_list_count + exception_list_item_count] + rules_and_exceptions_count = rules_count + exception_list_count + exception_list_item_count + action_connector_results = results[ + rules_and_exceptions_count: rules_and_exceptions_count + action_connector_count + ] errors = [] exported = [] exception_list_rule_table = {} + action_connector_rule_table = {} for rule_resource in rules_results: try: if strip_version: @@ -189,6 +216,14 @@ def kibana_export_rules( if exception_id not in exception_list_rule_table: exception_list_rule_table[exception_id] = [] exception_list_rule_table[exception_id].append({"id": rule.id, "name": rule.name}) + if rule.contents.data.actions: + # use connector ids as rule source + for action in rule.contents.data.actions: + action_id = action["id"] + if action_id not in action_connector_rule_table: + action_connector_rule_table[action_id] = [] + action_connector_rule_table[action_id].append({"id": rule.id, "name": rule.name}) + exported.append(rule) # Parse exceptions results from API return @@ -230,6 +265,39 @@ def kibana_export_rules( exceptions.append(exception) + # Parse action connector results from API return + action_connectors = [] + if export_action_connectors: + action_connector_results, _ = parse_action_connector_results_from_api(action_connector_results) + + # Build TOMLAction Objects + for action_connector_dict in action_connector_results: + try: + connector_id = action_connector_dict.get("id") + rule_list = action_connector_rule_table.get(connector_id) + if not rule_list: + click.echo(f"Warning action connector {connector_id} has no associated rules. Loading skipped.") + continue + else: + contents = TOMLActionConnectorContents.from_action_connector_dict( + action_connector_dict, + rule_list + ) + action_connector = TOMLActionConnector( + contents=contents, path=action_connectors_directory / f"{connector_id}_actions.toml" + ) + except Exception as e: + if skip_errors: + print(f"- skipping actions_connector export - {type(e).__name__}") + if not exceptions_directory: + errors.append(f"- no actions directory found - {e}") + else: + errors.append(f"- actions connector export - {e}") + continue + raise + + action_connectors.append(action_connector) + saved = [] for rule in exported: try: @@ -256,10 +324,26 @@ def kibana_export_rules( saved_exceptions.append(exception) + saved_action_connectors = [] + for action in action_connectors: + try: + action.save_toml() + except Exception as e: + if skip_errors: + print(f"- skipping {action.name} - {type(e).__name__}") + errors.append(f"- {action.name} - {e}") + continue + raise + + saved_action_connectors.append(action) + click.echo(f"{len(results)} results exported") click.echo(f"{len(exported)} rules converted") click.echo(f"{len(exceptions)} exceptions exported") - click.echo(f"{len(saved)} saved to {directory}") + click.echo(f"{len(action_connectors)} action connectors exported") + click.echo(f"{len(saved)} rules saved to {directory}") + click.echo(f"{len(saved_exceptions)} exception lists saved to {exceptions_directory}") + click.echo(f"{len(saved_action_connectors)} action connectors saved to {action_connectors_directory}") if errors: err_file = directory / "_errors.txt" err_file.write_text("\n".join(errors)) diff --git a/detection_rules/main.py b/detection_rules/main.py index af272d450b6..0faedd6c8db 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -19,6 +19,8 @@ from uuid import uuid4 import click +from .action_connector import (TOMLActionConnector, TOMLActionConnectorContents, + parse_action_connector_results_from_api) from .attack import build_threat_map_entry from .cli_utils import rule_prompt, multi_collection from .config import load_current_package_version, parse_rules_config @@ -99,6 +101,7 @@ def generate_rules_index(ctx: click.Context, query, overwrite, save_files=True): @root.command("import-rules-to-repo") @click.argument("input-file", type=click.Path(dir_okay=False, exists=True), nargs=-1, required=False) +@click.option("--action-connector-import", "-ac", is_flag=True, help="Include action connectors in export") @click.option("--exceptions-import", "-e", is_flag=True, help="Include exceptions in export") @click.option("--required-only", is_flag=True, help="Only prompt for required fields") @click.option("--directory", "-d", type=click.Path(file_okay=False, exists=True), help="Load files from a directory") @@ -111,15 +114,23 @@ def generate_rules_index(ctx: click.Context, query, overwrite, save_files=True): type=click.Path(file_okay=False, exists=True), help="Save imported exceptions to a directory", ) +@click.option( + "--action-connectors-directory", + "-sa", + type=click.Path(file_okay=False, exists=True), + help="Save imported actions to a directory", +) @click.option("--skip-errors", "-ske", is_flag=True, help="Skip rule import errors") @click.option("--default-author", "-da", type=str, required=False, help="Default author for rules missing one") @click.option("--strip-none-values", "-snv", is_flag=True, help="Strip None values from the rule") def import_rules_into_repo( input_file: click.Path, required_only: bool, + action_connector_import: bool, exceptions_import: bool, directory: click.Path, save_directory: click.Path, + action_connectors_directory: click.Path, exceptions_directory: click.Path, skip_errors: bool, default_author: str, @@ -144,9 +155,12 @@ def import_rules_into_repo( file_contents, skip_errors=True ) + action_connectors, unparsed_results = parse_action_connector_results_from_api(unparsed_results) + file_contents = unparsed_results exception_list_rule_table = {} + action_connector_rule_table = {} for contents in file_contents: # Don't load exceptions as rules if contents["type"] not in get_args(definitions.RuleType): @@ -191,6 +205,14 @@ def import_rules_into_repo( exception_list_rule_table[exception_id] = [] exception_list_rule_table[exception_id].append({"id": contents["id"], "name": contents["name"]}) + if contents.get("actions"): + # use connector ids as rule source + for action in contents["actions"]: + action_id = action["id"] + if action_id not in action_connector_rule_table: + action_connector_rule_table[action_id] = [] + action_connector_rule_table[action_id].append({"id": contents["id"], "name": contents["name"]}) + # Build TOMLException Objects if exceptions_import: for container in exceptions_containers.values(): @@ -222,10 +244,43 @@ def import_rules_into_repo( except Exception: raise + # Build TOMLAction Objects + if action_connector_import: + for actions_connector_dict in action_connectors: + try: + connector_id = actions_connector_dict.get("id") + rule_list = action_connector_rule_table.get(connector_id) + if not rule_list: + click.echo(f"Warning action connector {connector_id} has no associated rules. Loading skipped.") + continue + else: + contents = TOMLActionConnectorContents.from_action_connector_dict( + actions_connector_dict, + rule_list + ) + filename = f"{connector_id}_actions.toml" + if RULES_CONFIG.action_connector_dir is None and not action_connectors_directory: + raise FileNotFoundError( + "No Action Connector directory is specified. Please specify either in the config or CLI." + ) + actions_path = ( + Path(action_connectors_directory) / filename + if action_connectors_directory + else RULES_CONFIG.action_connector_dir / filename + ) + click.echo(f"[+] Building action connector(s) for {actions_path}") + TOMLActionConnector( + contents=contents, + path=actions_path, + ).save_toml() + except Exception: + raise + exceptions_count = 0 if not exceptions_import else len(exceptions_containers) + len(exceptions_items) click.echo(f"{len(file_contents) + exceptions_count} results exported") click.echo(f"{len(file_contents)} rules converted") click.echo(f"{exceptions_count} exceptions exported") + click.echo(f"{len(action_connectors)} actions connectors exported") if errors: err_file = save_directory if save_directory is not None else RULES_DIRS[0] / "_errors.txt" err_file.write_text("\n".join(errors)) @@ -354,6 +409,7 @@ def _export_rules( verbose=True, skip_unsupported=False, include_metadata: bool = False, + include_action_connectors: bool = False, include_exceptions: bool = False, ): """Export rules and exceptions into a consolidated ndjson file.""" @@ -384,14 +440,19 @@ def _export_rules( sort_keys=True) for r in rules] # Add exceptions to api format here and add to output_lines - if include_exceptions: + if include_exceptions or include_action_connectors: cl = GenericCollection.default() # Get exceptions in API format - exceptions = [d.contents.to_api_format() for d in cl.items] - # Flatten list of lists - exceptions = [e for sublist in exceptions for e in sublist] - # Append to Rules List - output_lines.extend(json.dumps(e, sort_keys=True) for e in exceptions) + if include_exceptions: + exceptions = [d.contents.to_api_format() for d in cl.items if isinstance(d.contents, TOMLExceptionContents)] + exceptions = [e for sublist in exceptions for e in sublist] + output_lines.extend(json.dumps(e, sort_keys=True) for e in exceptions) + if include_action_connectors: + action_connectors = [ + d.contents.to_api_format() for d in cl.items if isinstance(d.contents, TOMLActionConnectorContents) + ] + actions = [a for sublist in action_connectors for a in sublist] + output_lines.extend(json.dumps(a, sort_keys=True) for a in actions) outfile.write_text('\n'.join(output_lines) + '\n') @@ -425,11 +486,26 @@ def _export_rules( help="If `--stack-version` is passed, skip rule types which are unsupported " "(an error will be raised otherwise)", ) @click.option("--include-metadata", type=bool, is_flag=True, default=False, help="Add metadata to the exported rules") +@click.option( + "--include-action-connectors", + "-ac", + type=bool, + is_flag=True, + default=False, + help="Include Action Connectors in export", +) @click.option( "--include-exceptions", "-e", type=bool, is_flag=True, default=False, help="Include Exceptions Lists in export" ) def export_rules_from_repo( - rules, outfile: Path, replace_id, stack_version, skip_unsupported, include_metadata: bool, include_exceptions: bool + rules, + outfile: Path, + replace_id, + stack_version, + skip_unsupported, + include_metadata: bool, + include_action_connectors: bool, + include_exceptions: bool, ) -> RuleCollection: """Export rule(s) and exception(s) into an importable ndjson file.""" assert len(rules) > 0, "No rules found" @@ -452,6 +528,7 @@ def export_rules_from_repo( downgrade_version=stack_version, skip_unsupported=skip_unsupported, include_metadata=include_metadata, + include_action_connectors=include_action_connectors, include_exceptions=include_exceptions, ) diff --git a/detection_rules/rule.py b/detection_rules/rule.py index c86458fe30c..cc09be71d6f 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -612,7 +612,7 @@ def validate_note(self): f"field, use the environment variable `DR_BYPASS_NOTE_VALIDATION_AND_PARSE`") # raise if setup header is in note and in setup - if self.setup_in_note and self.setup: + if self.setup_in_note and (self.setup and self.setup != "None"): raise ValidationError("Setup header found in both note and setup fields.") diff --git a/detection_rules/rule_formatter.py b/detection_rules/rule_formatter.py index 044d40809b9..149f4d145de 100644 --- a/detection_rules/rule_formatter.py +++ b/detection_rules/rule_formatter.py @@ -231,6 +231,11 @@ def _do_write(_data, _contents): # This will ensure that the output file has the correct number of backslashes. v = v.replace("\\", "\\\\") + if k == 'setup' and isinstance(v, str): + # Transform instances of \ to \\ as calling write will convert \\ to \. + # This will ensure that the output file has the correct number of backslashes. + v = v.replace("\\", "\\\\") + if k == 'description' and isinstance(v, str): # Transform instances of \ to \\ as calling write will convert \\ to \. # This will ensure that the output file has the correct number of backslashes. diff --git a/detection_rules/schemas/__init__.py b/detection_rules/schemas/__init__.py index 195857d935a..edd79e39faa 100644 --- a/detection_rules/schemas/__init__.py +++ b/detection_rules/schemas/__init__.py @@ -36,7 +36,7 @@ def all_versions() -> List[str]: """Get all known stack versions.""" - return [str(v) for v in sorted(migrations)] + return [str(v) for v in sorted(migrations, key=lambda x: Version.parse(x, optional_minor_and_patch=True))] def migrate(version: str): diff --git a/docs/custom-rules.md b/docs/custom-rules.md index 67e52291fce..1260a81b841 100644 --- a/docs/custom-rules.md +++ b/docs/custom-rules.md @@ -29,9 +29,12 @@ custom-rules └── actions ├── action_1.toml ├── action_2.toml +└── action_connectors + ├── action_connector_1.toml + └── action_connectors_2.toml └── exceptions ├── exception_1.toml - └── exception_2.toml + └── exception_2.toml ``` This structure represents a portable set of custom rules. This is just an example, and the exact locations of the files @@ -58,6 +61,7 @@ files: version_lock: version.lock.json directories: action_dir: actions + action_connector_dir: action_connectors exception_dir: exceptions ``` @@ -69,7 +73,9 @@ Some notes: * To bypass using the version lock versioning strategy (version lock file) you can set the optional `bypass_version_lock` value to be `True` * To normalize the capitalization KQL keywords in KQL rule queries one can use the optional `normalize_kql_keywords` value set to `True` or `False` as desired. * To manage exceptions tied to rules one can set an exceptions directory using the optional `exception_dir` value (included above) set to be the desired path. If an exceptions directory is explicitly specified in a CLI command, the config value will be ignored. +* To manage action-connectors tied to rules one can set an action-connectors directory using the optional `action_connector_dir` value (included above) set to be the desired path. If an actions_connector directory is explicitly specified in a CLI command, the config value will be ignored. * To turn on automatic schema generation for non-ecs fields a custom schemas add `auto_gen_schema_file: `. This will generate a schema file in the specified location that will be used to add entries for each field and index combination that is not already in a known schema. This will also automatically add it to your stack-schema-map.yaml file when using a custom rules directory and config. +* For Kibana action items, currently these are included in the rule toml files themselves. At a later date, we may allow for bulk editing of rule action items through separate action toml files. The action_dir config key is left available for this later implementation. For now to bulk update, use the bulk actions add rule actions UI in Kibana. When using the repo, set the environment variable `CUSTOM_RULES_DIR=` @@ -186,6 +192,8 @@ Example: Note: the `custom` key can be any alpha numeric value except `beats`, `ecs`, or `endgame` as these are reserved terms. +Note: Remember if you want to turn on automatic schema generation for non-ecs fields a custom schemas add `auto_gen_schema_file: `. + Example schema json: ```json diff --git a/lib/kibana/kibana/resources.py b/lib/kibana/kibana/resources.py index 75ddc57049e..a46d2530f14 100644 --- a/lib/kibana/kibana/resources.py +++ b/lib/kibana/kibana/resources.py @@ -234,6 +234,7 @@ def import_rules( cls, rules: List[dict], exceptions: List[List[dict]] = [], + action_connectors: List[List[dict]] = [], overwrite: bool = False, overwrite_exceptions: bool = False, overwrite_action_connectors: bool = False, @@ -247,7 +248,10 @@ def import_rules( ) rule_ids = [r['rule_id'] for r in rules] flattened_exceptions = [e for sublist in exceptions for e in sublist] - headers, raw_data = Kibana.ndjson_file_data_prep(rules + flattened_exceptions, "import.ndjson") + flattened_actions_connectors = [a for sublist in action_connectors for a in sublist] + headers, raw_data = Kibana.ndjson_file_data_prep( + rules + flattened_exceptions + flattened_actions_connectors, "import.ndjson" + ) response = Kibana.current().post(url, headers=headers, params=params, raw_data=raw_data) errors = response.get("errors", []) error_rule_ids = [e['rule_id'] for e in errors] diff --git a/lib/kibana/pyproject.toml b/lib/kibana/pyproject.toml index 903b916a8a5..7e27bbdacb9 100644 --- a/lib/kibana/pyproject.toml +++ b/lib/kibana/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection-rules-kibana" -version = "0.4.0" +version = "0.5.0" description = "Kibana API utilities for Elastic Detection Rules" license = {text = "Elastic License v2"} keywords = ["Elastic", "Kibana", "Detection Rules", "Security", "Elasticsearch"]