From 3f184b9fa9fae821f3dbe950c7c7f7b370c88a44 Mon Sep 17 00:00:00 2001 From: Norrie Taylor <91171431+norrietaylor@users.noreply.github.com> Date: Fri, 5 Apr 2024 11:55:58 -0700 Subject: [PATCH 1/4] Special guidance for ECS contribtions during ECS donation (#2325) --- CHANGELOG.next.md | 2 ++ CONTRIBUTING.md | 57 ++++++++++++++++++++++++++++++++++++++++++----- README.md | 9 ++++++++ 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 4a9cd6c1fa..5996082bf8 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -34,6 +34,8 @@ Thanks, you're awesome :-) --> #### Added +* Documentation in README.md providing instruction on contributions to ECS during the OTel donation #2325 + #### Improvements #### Deprecated diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e9c40ddce..7424f80467 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,8 @@ ECS is an open source project and we love to receive contributions from our comm ## Table of Contents -- [How to Contribute](#how-to-contribute) +- [How to contribute](#how-to-contribute) +- - [Special guidance during OTel donation of ECS](#special-guidance-during-otel-donation-of-ecs) - [Dev Tools](#dev-tools) - [Submitting Changes](#submitting-changes) - [Git and Github Guidelines](#git-and-github-guidelines) @@ -22,12 +23,58 @@ ECS is an open source project and we love to receive contributions from our comm - [Schema Files](#schema-files) - [Additional Resources](#additional-resources) -## How to Contribute -There are two primary ways in which you can contribute to ECS. +## How to contribute -1. The [RFC process](./rfcs/README.md) is used for significant additions or breaking changes to the schema itself. -2. For bug fixes or incremental, non-controversial additions to ECS, changes can be made directly to the ECS project and submitted as pull request. +## ECS Donation to OpenTelemetry +In April 2023, OpenTelemetry and Elastic made an [important joint announcement](https://opentelemetry.io/blog/2023/ecs-otel-semconv-convergence/). In this announcement Elastic +shared its intention to achieve convergence of ECS and OTel Semantic Conventions into a single standard maintained +by OpenTelemetry. + +The stated plan has been to keep ECS in a frozen state during the transition. However, it is also apparent that these +things take time. It takes time for the OTel community to adopt donated fields, and it will take time for development +teams to build OTel native constructs in the Elastic stack. In the meantime, ECS users need to be able to develop +features for Elastic that rely on continued contributions to the schemas that drive our technology. + +For these reasons, we need a process and guidelines for contributing to these data schemas during this period that +allows us to avoid breaking changes. + +### How to contribute during OTel donation of ECS + +Bug fixes or minor field addition changes can be made directly to the ECS project and submitted as pull requests. + +Significant changes that add new use cases, top-level fieldsets, or could be considered controversial are +considered material. The general rule for contributing new material changes to schemas during the transition period is + +- First, merge a pull request to +[OTel Semantic Conventions](https://github.com/open-telemetry/semantic-conventions/blob/main/CONTRIBUTING.md) with new +fields, namespaces or schemas +- Second, to backport those changes to ECS at the starting point indicated in the table below +- Finally, once the Semantic Conventions changes are marked as stable, remove the Beta designation in ECS + +This will ensure that the latest changes are included in OTel Semantic Conventions, where schema evolution will continue +as the merger proceeds. It will also allow teams and users to continue using ECS while OTel migration tools and guidance +are being developed. Finally, this will reduce the risk of breaking changes if new fields are merged first to ECS, and +then require changes before being adopted in Semantic Conventions. + +_There are some exceptions to this rule._ + +1. My contribution to OTel Semantic Conventions is stalled. We are waiting for a sign-off from a second company. +In the meantime, our Elastic feature is blocked. +2. I want to build a workflow in Elastic, and the fields I need to proceed are already in OTel but not in ECS where I +need them today. + +In these cases, the recommendation is to make a contribution to ECS to unblock development. The appropriate ECS starting +point can be an [RFC](./rfcs/README.md) or pull request based on the maturity of the Otel changes. Please see the +following table. + +| OTel submission maturity | Breaking changes expected | ECS starting point | +|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------|------------------------------------------------------| +| OTel working groups accepts the premise of the addition and commits to considering this proposal as it advances. | Major | RFC Stage 1 | +| The initial field definitions comprehensively model the addition to the schema. Fundamental questions and concerns are resolved, though some less significant questions remain open. | Iterative | RFC Stage 2 | +| All requested changes from codeowners have been addressed, and there are no open discussions. | Iterative | Open an ECS pull request with new fields marked Beta | +| Fields, schema, namespace exists in OTel and are designated experimental | Iterative | Open an ECS pull request with new fields marked Beta | +| Fields, schema, namespace exists in OTel and are designated stable | None | Open an ECS pull request with new fields marked GA | ### Dev Tools diff --git a/README.md b/README.md index 16cf9e4125..5b7d78fc5c 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,18 @@ ingesting data into Elasticsearch. A common schema helps you correlate data from sources like logs and metrics or IT operations analytics and security analytics. +## ECS Donation to OpenTelemetry +In April 2023, OpenTelemetry and Elastic made an +[important joint announcement](https://opentelemetry.io/blog/2023/ecs-otel-semconv-convergence/). +In this announcement, we shared our intention to achieve convergence of ECS and OTel +Semantic Conventions into a single standard maintained by OpenTelemetry. + +Special guidance is provided during the donation period. Please review the [contribution guide](CONTRIBUTING.md). + ## Documentation The ECS reference is published on the main Elastic documentation website. + Visit [the official ECS Reference Documentation](https://www.elastic.co/guide/en/ecs/current/index.html). ## Getting Started From be84b7da61af81afc68377eb85310bbfe8f5b0fe Mon Sep 17 00:00:00 2001 From: Norrie Taylor <91171431+norrietaylor@users.noreply.github.com> Date: Fri, 5 Apr 2024 13:34:11 -0700 Subject: [PATCH 2/4] Clarification of release process during Otel donatrion period (#2332) --- CONTRIBUTING.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7424f80467..3d92ba5900 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,7 @@ ECS is an open source project and we love to receive contributions from our comm ## How to contribute -## ECS Donation to OpenTelemetry +## ECS donation to OpenTelemetry In April 2023, OpenTelemetry and Elastic made an [important joint announcement](https://opentelemetry.io/blog/2023/ecs-otel-semconv-convergence/). In this announcement Elastic shared its intention to achieve convergence of ECS and OTel Semantic Conventions into a single standard maintained by OpenTelemetry. @@ -39,6 +39,15 @@ features for Elastic that rely on continued contributions to the schemas that dr For these reasons, we need a process and guidelines for contributing to these data schemas during this period that allows us to avoid breaking changes. +### ECS releases during the donation to OpenTelemetry +Historically, ECS has shipped a new version with every minor release of the Elastic stack. While the schema is +effectively frozen during the Otel donation period, this approach has been halted. + +Moving forward, we will release ECS at the team's discretion as new material changes to the schema are adopted. + +While the decision to release will be discretionary, any release will still coincide with a minor Elastic stack release; +however, not every minor version will warrant a new release of the stack. + ### How to contribute during OTel donation of ECS Bug fixes or minor field addition changes can be made directly to the ECS project and submitted as pull requests. From 247d128bd73b53b7a7df66f75c6613f78d00286f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 May 2024 09:18:01 -0700 Subject: [PATCH 3/4] Bump jinja2 from 3.1.3 to 3.1.4 in /scripts (#2339) Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.3 to 3.1.4. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.3...3.1.4) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- scripts/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 30f66a6c8e..e703c6ad50 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -4,4 +4,4 @@ PyYAML==6.0.1 # License: BSD gitpython==3.1.41 # License: BSD -Jinja2==3.1.3 +Jinja2==3.1.4 From 4b2c7c6c4596b8d4acdd420907952a04e20c8c23 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 22 May 2024 13:27:18 -0700 Subject: [PATCH 4/4] Add `process.group` fields to generated schemas (#2335) Add process.group fields as the effective group for the process schema. process.group was (probably mistakenly) being excluded from the schema, as it wasn't part of schemas/subsets/main.yml. The other related process group fields are already included in the schema (real_group, saved_group), and process.group is defined with field re-use in group.yml. --------- Co-authored-by: Alexandra Konrad --- CHANGELOG.next.md | 1 + experimental/generated/beats/fields.ecs.yml | 12 ++++++++++ experimental/generated/csv/fields.csv | 2 ++ experimental/generated/ecs/ecs_flat.yml | 22 +++++++++++++++++++ experimental/generated/ecs/ecs_nested.yml | 22 +++++++++++++++++++ .../composable/component/process.json | 12 ++++++++++ .../elasticsearch/legacy/template.json | 12 ++++++++++ generated/beats/fields.ecs.yml | 12 ++++++++++ generated/csv/fields.csv | 2 ++ generated/ecs/ecs_flat.yml | 22 +++++++++++++++++++ generated/ecs/ecs_nested.yml | 22 +++++++++++++++++++ .../composable/component/process.json | 12 ++++++++++ generated/elasticsearch/legacy/template.json | 12 ++++++++++ schemas/subsets/main.yml | 4 ++++ 14 files changed, 169 insertions(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 5996082bf8..9613fb89e6 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -21,6 +21,7 @@ Thanks, you're awesome :-) --> * Advanced `process.env_vars` to GA. #2315 * Advanced `process.io` and `process.tty` fields to GA. #2317 * Added `threat.indicator.id`. #2324 +* Added `process.group` to generated schemas. #2335 #### Improvements diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 61e5088661..bc95a6db22 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -5496,6 +5496,18 @@ start).' example: 137 default_field: false + - name: group.id + level: extended + type: keyword + ignore_above: 1024 + description: Unique identifier for the group on the system/platform. + default_field: false + - name: group.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the group. + default_field: false - name: group_leader.args level: extended type: keyword diff --git a/experimental/generated/csv/fields.csv b/experimental/generated/csv/fields.csv index 360d885076..292ac5f917 100644 --- a/experimental/generated/csv/fields.csv +++ b/experimental/generated/csv/fields.csv @@ -690,6 +690,8 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.exit_code,long,extended,,137,The exit code of the process. +8.12.0-dev+exp,true,process,process.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. +8.12.0-dev+exp,true,process,process.group.name,keyword,extended,,,Name of the group. 8.12.0-dev+exp,true,process,process.group_leader.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. 8.12.0-dev+exp,true,process,process.group_leader.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev+exp,true,process,process.group_leader.command_line,wildcard,extended,,/usr/bin/ssh -l user 10.0.0.16,Full command line that started the process. diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 56716a240c..02b972886f 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -8930,6 +8930,28 @@ process.exit_code: normalize: [] short: The exit code of the process. type: long +process.group.id: + dashed_name: process-group-id + description: Unique identifier for the group on the system/platform. + flat_name: process.group.id + ignore_above: 1024 + level: extended + name: id + normalize: [] + original_fieldset: group + short: Unique identifier for the group on the system/platform. + type: keyword +process.group.name: + dashed_name: process-group-name + description: Name of the group. + flat_name: process.group.name + ignore_above: 1024 + level: extended + name: name + normalize: [] + original_fieldset: group + short: Name of the group. + type: keyword process.group_leader.args: dashed_name: process-group-leader-args description: 'Array of process arguments, starting with the absolute path to the diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index 312cf49b80..f600ab293a 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -11140,6 +11140,28 @@ process: normalize: [] short: The exit code of the process. type: long + process.group.id: + dashed_name: process-group-id + description: Unique identifier for the group on the system/platform. + flat_name: process.group.id + ignore_above: 1024 + level: extended + name: id + normalize: [] + original_fieldset: group + short: Unique identifier for the group on the system/platform. + type: keyword + process.group.name: + dashed_name: process-group-name + description: Name of the group. + flat_name: process.group.name + ignore_above: 1024 + level: extended + name: name + normalize: [] + original_fieldset: group + short: Name of the group. + type: keyword process.group_leader.args: dashed_name: process-group-leader-args description: 'Array of process arguments, starting with the absolute path to diff --git a/experimental/generated/elasticsearch/composable/component/process.json b/experimental/generated/elasticsearch/composable/component/process.json index 3f144db017..f4dd52c1ce 100644 --- a/experimental/generated/elasticsearch/composable/component/process.json +++ b/experimental/generated/elasticsearch/composable/component/process.json @@ -481,6 +481,18 @@ "exit_code": { "type": "long" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "group_leader": { "properties": { "args": { diff --git a/experimental/generated/elasticsearch/legacy/template.json b/experimental/generated/elasticsearch/legacy/template.json index 1dc48de290..18386e190c 100644 --- a/experimental/generated/elasticsearch/legacy/template.json +++ b/experimental/generated/elasticsearch/legacy/template.json @@ -3202,6 +3202,18 @@ "exit_code": { "type": "long" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "group_leader": { "properties": { "args": { diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index b88a755686..fa0007884b 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -5446,6 +5446,18 @@ start).' example: 137 default_field: false + - name: group.id + level: extended + type: keyword + ignore_above: 1024 + description: Unique identifier for the group on the system/platform. + default_field: false + - name: group.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the group. + default_field: false - name: group_leader.args level: extended type: keyword diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index 3ca25f1445..c31a8de31c 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -683,6 +683,8 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.exit_code,long,extended,,137,The exit code of the process. +8.12.0-dev,true,process,process.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. +8.12.0-dev,true,process,process.group.name,keyword,extended,,,Name of the group. 8.12.0-dev,true,process,process.group_leader.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. 8.12.0-dev,true,process,process.group_leader.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev,true,process,process.group_leader.command_line,wildcard,extended,,/usr/bin/ssh -l user 10.0.0.16,Full command line that started the process. diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 50e16f1826..2022bddaf4 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -8861,6 +8861,28 @@ process.exit_code: normalize: [] short: The exit code of the process. type: long +process.group.id: + dashed_name: process-group-id + description: Unique identifier for the group on the system/platform. + flat_name: process.group.id + ignore_above: 1024 + level: extended + name: id + normalize: [] + original_fieldset: group + short: Unique identifier for the group on the system/platform. + type: keyword +process.group.name: + dashed_name: process-group-name + description: Name of the group. + flat_name: process.group.name + ignore_above: 1024 + level: extended + name: name + normalize: [] + original_fieldset: group + short: Name of the group. + type: keyword process.group_leader.args: dashed_name: process-group-leader-args description: 'Array of process arguments, starting with the absolute path to the diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index fa5f354d5f..8057eeed15 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -11060,6 +11060,28 @@ process: normalize: [] short: The exit code of the process. type: long + process.group.id: + dashed_name: process-group-id + description: Unique identifier for the group on the system/platform. + flat_name: process.group.id + ignore_above: 1024 + level: extended + name: id + normalize: [] + original_fieldset: group + short: Unique identifier for the group on the system/platform. + type: keyword + process.group.name: + dashed_name: process-group-name + description: Name of the group. + flat_name: process.group.name + ignore_above: 1024 + level: extended + name: name + normalize: [] + original_fieldset: group + short: Name of the group. + type: keyword process.group_leader.args: dashed_name: process-group-leader-args description: 'Array of process arguments, starting with the absolute path to diff --git a/generated/elasticsearch/composable/component/process.json b/generated/elasticsearch/composable/component/process.json index c20dbd00f2..6cc1382d11 100644 --- a/generated/elasticsearch/composable/component/process.json +++ b/generated/elasticsearch/composable/component/process.json @@ -481,6 +481,18 @@ "exit_code": { "type": "long" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "group_leader": { "properties": { "args": { diff --git a/generated/elasticsearch/legacy/template.json b/generated/elasticsearch/legacy/template.json index 0e26f73020..a6b67033e2 100644 --- a/generated/elasticsearch/legacy/template.json +++ b/generated/elasticsearch/legacy/template.json @@ -3160,6 +3160,18 @@ "exit_code": { "type": "long" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "group_leader": { "properties": { "args": { diff --git a/schemas/subsets/main.yml b/schemas/subsets/main.yml index 69391cf839..ebefde9e2e 100644 --- a/schemas/subsets/main.yml +++ b/schemas/subsets/main.yml @@ -360,6 +360,10 @@ fields: args: {} args_count: {} executable: {} + group: + fields: + id: {} + name: {} real_group: fields: id: {}