Merge branch 'main' into renovate/actions-setup-python-digest

.github/workflows/comment-on-asciidoc-changes.yml

@@ -0,0 +1,21 @@
+---
+name: Comment on PR for .asciidoc changes
+
+on:
+  # We need to use pull_request_target to be able to comment on PRs from forks
+  pull_request_target:
+    types:
+      - synchronize
+      - opened
+      - reopened
+    branches:
+      - main
+      - master
+      - "9.0"
+
+jobs:
+  comment-on-asciidoc-change:
+    permissions:
+      contents: read
+      pull-requests: write
+    uses: elastic/docs-builder/.github/workflows/comment-on-asciidoc-changes.yml@main

.github/workflows/test.yml

@@ -4,7 +4,7 @@ on: [push, pull_request]
 
 jobs:
   tests:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     name: Unit Tests
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4

CHANGELOG.md

@@ -3,6 +3,19 @@
 # CHANGELOG
 All notable changes to this project will be documented in this file based on the [Keep a Changelog](http://keepachangelog.com/) Standard. This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [8.17.0](https://github.com/elastic/ecs/compare/v8.16.0...v8.17.0)
+
+### Schema Changes
+
+#### Bugfixes
+
+* Fix link rendering issues and usage of http in links. #2423
+
+#### Improvements
+
+* Increase ignore_above value for url.query. #2424
+* Set synthetic_source_keep = none on fields that represent sets. #2422
+
 ## [8.16.0](https://github.com/elastic/ecs/compare/v8.11.0...v8.16.0)
 
 ### Schema Changes

CHANGELOG.next.md

@@ -12,11 +12,14 @@ Thanks, you're awesome :-) -->
 
 #### Breaking changes
 
+* Remove deprecated fields from previous major release; `process.pgid`, `service.node.role`, and inherited users. #2410
+
 #### Bugfixes
 
 * Fix link rendering issues and usage of http in links. #2423
 
 #### Added
+* Add `origin_referrer_url` and `origin_url` fields, which indicate the origin information to the file, process and dll schemas #2441
 
 #### Improvements
 

docs/fields/field-details.asciidoc

@@ -1841,6 +1841,42 @@ example: `kernel32.dll`
 
 // ===============================================================
 
+|
+[[field-dll-origin-referrer-url]]
+<<field-dll-origin-referrer-url, dll.origin_referrer_url>>
+
+a| beta:[ This field is beta and subject to change. ]
+
+The URL of the webpage that linked to the dll file.
+
+type: keyword
+
+
+
+example: `http://example.com/article1.html`
+
+| extended
+
+// ===============================================================
+
+|
+[[field-dll-origin-url]]
+<<field-dll-origin-url, dll.origin_url>>
+
+a| beta:[ This field is beta and subject to change. ]
+
+The URL where the dll file is hosted.
+
+type: keyword
+
+
+
+example: `http://example.com/files/example.dll`
+
+| extended
+
+// ===============================================================
+
 |
 [[field-dll-path]]
 <<field-dll-path, dll.path>>
@@ -4447,6 +4483,42 @@ image:https://img.shields.io/badge/OpenTelemetry-4a5ca6?style=flat&logo=opentele
 
 // ===============================================================
 
+|
+[[field-file-origin-referrer-url]]
+<<field-file-origin-referrer-url, file.origin_referrer_url>>
+
+a| beta:[ This field is beta and subject to change. ]
+
+The URL of the webpage that linked to the file.
+
+type: keyword
+
+
+
+example: `http://example.com/article1.html`
+
+| extended
+
+// ===============================================================
+
+|
+[[field-file-origin-url]]
+<<field-file-origin-url, file.origin_url>>
+
+a| beta:[ This field is beta and subject to change. ]
+
+The URL where the file is hosted.
+
+type: keyword
+
+
+
+example: `http://example.com/imgs/article1_img1.jpg`
+
+| extended
+
+// ===============================================================
+
 |
 [[field-file-owner]]
 <<field-file-owner, file.owner>>
@@ -8624,24 +8696,6 @@ Multi-fields:
 
 example: `ssh`
 
-| extended
-
-// ===============================================================
-
-|
-[[field-process-pgid]]
-<<field-process-pgid, process.pgid>>
-
-a| Deprecated for removal in next major version release. This field is superseded by `process.group_leader.pid`.
-
-Identifier of the group of processes the process belongs to.
-
-type: long
-
-
-
-
-
 | extended
 
 // ===============================================================

docs/opentelemetry/otel-mapping-summary.asciidoc

@@ -311,7 +311,7 @@ h| Namespace
 
 
 | DLL
-^| <<ecs-dll,2>>
+^| <<ecs-dll,4>>
 ^| ·
 ^| ·
 ^| ·
@@ -443,7 +443,7 @@ h| Namespace
 
 
 | File
-^| <<ecs-file,22>>
+^| <<ecs-file,24>>
 ^| https://opentelemetry.io/docs/specs/semconv/attributes-registry/file[18]
 ^| 11
 ^| 7
@@ -815,7 +815,7 @@ h| Namespace
 
 
 | Process
-^| <<ecs-process,37>>
+^| <<ecs-process,36>>
 ^| https://opentelemetry.io/docs/specs/semconv/attributes-registry/process[33]
 ^| 15
 ^| 2

experimental/generated/beats/fields.ecs.yml

@@ -1399,6 +1399,20 @@
         This generally maps to the name of the file on disk.'
       example: kernel32.dll
       default_field: false
+    - name: origin_referrer_url
+      level: extended
+      type: keyword
+      ignore_above: 8192
+      description: The URL of the webpage that linked to the dll file.
+      example: http://example.com/article1.html
+      default_field: false
+    - name: origin_url
+      level: extended
+      type: keyword
+      ignore_above: 8192
+      description: The URL where the dll file is hosted.
+      example: http://example.com/files/example.dll
+      default_field: false
     - name: path
       level: extended
       type: keyword
@@ -3021,6 +3035,20 @@
       ignore_above: 1024
       description: Name of the file including the extension, without the directory.
       example: example.png
+    - name: origin_referrer_url
+      level: extended
+      type: keyword
+      ignore_above: 8192
+      description: The URL of the webpage that linked to the file.
+      example: http://example.com/article1.html
+      default_field: false
+    - name: origin_url
+      level: extended
+      type: keyword
+      ignore_above: 8192
+      description: The URL where the file is hosted.
+      example: http://example.com/imgs/article1_img1.jpg
+      default_field: false
     - name: owner
       level: extended
       type: keyword
@@ -6897,15 +6925,6 @@
       format: string
       description: PE Section List virtual size. This is always the same as `physical_size`.
       default_field: false
-    - name: parent.pgid
-      level: extended
-      type: long
-      format: string
-      description: 'Deprecated for removal in next major version release. This field
-        is superseded by `process.group_leader.pid`.
-
-        Identifier of the group of processes the process belongs to.'
-      default_field: false
     - name: parent.pid
       level: core
       type: long
@@ -7269,14 +7288,6 @@
       format: string
       description: PE Section List virtual size. This is always the same as `physical_size`.
       default_field: false
-    - name: pgid
-      level: extended
-      type: long
-      format: string
-      description: 'Deprecated for removal in next major version release. This field
-        is superseded by `process.group_leader.pid`.
-
-        Identifier of the group of processes the process belongs to.'
     - name: pid
       level: core
       type: long
@@ -9662,6 +9673,20 @@
       description: Name of the file including the extension, without the directory.
       example: example.png
       default_field: false
+    - name: enrichments.indicator.file.origin_referrer_url
+      level: extended
+      type: keyword
+      ignore_above: 8192
+      description: The URL of the webpage that linked to the file.
+      example: http://example.com/article1.html
+      default_field: false
+    - name: enrichments.indicator.file.origin_url
+      level: extended
+      type: keyword
+      ignore_above: 8192
+      description: The URL where the file is hosted.
+      example: http://example.com/imgs/article1_img1.jpg
+      default_field: false
     - name: enrichments.indicator.file.owner
       level: extended
       type: keyword
@@ -11284,6 +11309,20 @@
       description: Name of the file including the extension, without the directory.
       example: example.png
       default_field: false
+    - name: indicator.file.origin_referrer_url
+      level: extended
+      type: keyword
+      ignore_above: 8192
+      description: The URL of the webpage that linked to the file.
+      example: http://example.com/article1.html
+      default_field: false
+    - name: indicator.file.origin_url
+      level: extended
+      type: keyword
+      ignore_above: 8192
+      description: The URL where the file is hosted.
+      example: http://example.com/imgs/article1_img1.jpg
+      default_field: false
     - name: indicator.file.owner
       level: extended
       type: keyword

experimental/generated/csv/fields.csv

@@ -166,6 +166,8 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.0.0-dev+exp,true,dll,dll.hash.ssdeep,keyword,extended,,,SSDEEP hash.
 9.0.0-dev+exp,true,dll,dll.hash.tlsh,keyword,extended,,,TLSH hash.
 9.0.0-dev+exp,true,dll,dll.name,keyword,core,,kernel32.dll,Name of the library.
+9.0.0-dev+exp,true,dll,dll.origin_referrer_url,keyword,extended,,http://example.com/article1.html,The URL of the webpage that linked to the dll file.
+9.0.0-dev+exp,true,dll,dll.origin_url,keyword,extended,,http://example.com/files/example.dll,The URL where the dll file is hosted.
 9.0.0-dev+exp,true,dll,dll.path,keyword,extended,,C:\Windows\System32\kernel32.dll,Full file path of the library.
 9.0.0-dev+exp,true,dll,dll.pe.architecture,keyword,extended,,x64,CPU architecture target for the file.
 9.0.0-dev+exp,true,dll,dll.pe.company,keyword,extended,,Microsoft Corporation,"Internal company name of the file, provided at compile-time."
@@ -364,6 +366,8 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.0.0-dev+exp,true,file,file.mode,keyword,extended,,0640,Mode of the file in octal representation.
 9.0.0-dev+exp,true,file,file.mtime,date,extended,,,Last time the file content was modified.
 9.0.0-dev+exp,true,file,file.name,keyword,extended,,example.png,"Name of the file including the extension, without the directory."
+9.0.0-dev+exp,true,file,file.origin_referrer_url,keyword,extended,,http://example.com/article1.html,The URL of the webpage that linked to the file.
+9.0.0-dev+exp,true,file,file.origin_url,keyword,extended,,http://example.com/imgs/article1_img1.jpg,The URL where the file is hosted.
 9.0.0-dev+exp,true,file,file.owner,keyword,extended,,alice,File owner's username.
 9.0.0-dev+exp,true,file,file.path,keyword,extended,,/home/alice/example.png,"Full path to the file, including the file name."
 9.0.0-dev+exp,true,file,file.path.text,match_only_text,extended,,/home/alice/example.png,"Full path to the file, including the file name."
@@ -884,7 +888,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.0.0-dev+exp,true,process,process.parent.pe.sections.physical_size,long,extended,,,PE Section List physical size.
 9.0.0-dev+exp,true,process,process.parent.pe.sections.var_entropy,long,extended,,,Variance for Shannon entropy calculation from the section.
 9.0.0-dev+exp,true,process,process.parent.pe.sections.virtual_size,long,extended,,,PE Section List virtual size. This is always the same as `physical_size`.
-9.0.0-dev+exp,true,process,process.parent.pgid,long,extended,,,Deprecated identifier of the group of processes the process belongs to.
 9.0.0-dev+exp,true,process,process.parent.pid,long,core,,4242,Process id.
 9.0.0-dev+exp,true,process,process.parent.real_group.id,keyword,extended,,,Unique identifier for the group on the system/platform.
 9.0.0-dev+exp,true,process,process.parent.real_group.name,keyword,extended,,,Name of the group.
@@ -938,7 +941,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.0.0-dev+exp,true,process,process.pe.sections.physical_size,long,extended,,,PE Section List physical size.
 9.0.0-dev+exp,true,process,process.pe.sections.var_entropy,long,extended,,,Variance for Shannon entropy calculation from the section.
 9.0.0-dev+exp,true,process,process.pe.sections.virtual_size,long,extended,,,PE Section List virtual size. This is always the same as `physical_size`.
-9.0.0-dev+exp,true,process,process.pgid,long,extended,,,Deprecated identifier of the group of processes the process belongs to.
 9.0.0-dev+exp,true,process,process.pid,long,core,,4242,Process id.
 9.0.0-dev+exp,true,process,process.previous.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments.
 9.0.0-dev+exp,true,process,process.previous.args_count,long,extended,,4,Length of the process.args array.
@@ -1230,6 +1232,8 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.0.0-dev+exp,true,threat,threat.enrichments.indicator.file.mode,keyword,extended,,0640,Mode of the file in octal representation.
 9.0.0-dev+exp,true,threat,threat.enrichments.indicator.file.mtime,date,extended,,,Last time the file content was modified.
 9.0.0-dev+exp,true,threat,threat.enrichments.indicator.file.name,keyword,extended,,example.png,"Name of the file including the extension, without the directory."
+9.0.0-dev+exp,true,threat,threat.enrichments.indicator.file.origin_referrer_url,keyword,extended,,http://example.com/article1.html,The URL of the webpage that linked to the file.
+9.0.0-dev+exp,true,threat,threat.enrichments.indicator.file.origin_url,keyword,extended,,http://example.com/imgs/article1_img1.jpg,The URL where the file is hosted.
 9.0.0-dev+exp,true,threat,threat.enrichments.indicator.file.owner,keyword,extended,,alice,File owner's username.
 9.0.0-dev+exp,true,threat,threat.enrichments.indicator.file.path,keyword,extended,,/home/alice/example.png,"Full path to the file, including the file name."
 9.0.0-dev+exp,true,threat,threat.enrichments.indicator.file.path.text,match_only_text,extended,,/home/alice/example.png,"Full path to the file, including the file name."
@@ -1449,6 +1453,8 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.0.0-dev+exp,true,threat,threat.indicator.file.mode,keyword,extended,,0640,Mode of the file in octal representation.
 9.0.0-dev+exp,true,threat,threat.indicator.file.mtime,date,extended,,,Last time the file content was modified.
 9.0.0-dev+exp,true,threat,threat.indicator.file.name,keyword,extended,,example.png,"Name of the file including the extension, without the directory."
+9.0.0-dev+exp,true,threat,threat.indicator.file.origin_referrer_url,keyword,extended,,http://example.com/article1.html,The URL of the webpage that linked to the file.
+9.0.0-dev+exp,true,threat,threat.indicator.file.origin_url,keyword,extended,,http://example.com/imgs/article1_img1.jpg,The URL where the file is hosted.
 9.0.0-dev+exp,true,threat,threat.indicator.file.owner,keyword,extended,,alice,File owner's username.
 9.0.0-dev+exp,true,threat,threat.indicator.file.path,keyword,extended,,/home/alice/example.png,"Full path to the file, including the file name."
 9.0.0-dev+exp,true,threat,threat.indicator.file.path.text,match_only_text,extended,,/home/alice/example.png,"Full path to the file, including the file name."