Airflow task logs subpipeline

Author: aboitreaud

airflow/assets/logs/airflow.yaml

@@ -82,3 +82,29 @@ pipeline:
       enabled: true
       sources:
         - message
+    - type: pipeline
+      name: Task logs pipeline 
+      enabled: true
+      filter:
+        query: "dirname:*dag_id*"
+      processors:
+        - type: attribute-remapper
+          name: Map tag `dirname` to an attribute
+          enabled: true
+          sources:
+            - dirname
+          sourceType: tag
+          target: dirname
+          targetType: attribute
+          preserveSource: false
+          overrideOnConflict: false
+        - type: grok-parser
+          name: Task Attributes Parser
+          enabled: true
+          source: dirname
+          samples:
+            - path/to/airflow/logs/dag_id_airflow_dag_0/run_id_scheduled__2025-02-18T14_46_00+00_00/task_id_print_task_0/attempt_1.log
+          grok:
+            supportRules: ""
+            matchRules: default
+              (.*dag_id_%{word:airflow.dagRun.dag_id}/run_id_%{data:airflow.dagRun.run_id}/task_id_%{word:airflow.task.task_id}/attempt_%{word:airflow.task.attempt_id}.log)