[O11y][Kubernetes] Rally benchmark kubernetes.event (#9161)

elastic · Mar 13, 2024 · 56c25a9 · 56c25a9
1 parent 9b5410c
commit 56c25a9
Show file tree

Hide file tree

Showing 4 changed files with 231 additions and 0 deletions.
diff --git a/packages/kubernetes/_dev/benchmark/rally/event-benchmark.yml b/packages/kubernetes/_dev/benchmark/rally/event-benchmark.yml
@@ -0,0 +1,14 @@
+---
+description: Benchmark of 20000 kubernetes.event event ingested
+data_stream:
+  name: event
+corpora:
+  generator:
+    total_events: 20000
+    template:
+      type: gotext
+      path: ./event-benchmark/template.ndjson
+    config:
+      path: ./event-benchmark/config.yml
+    fields:
+      path: ./event-benchmark/fields.yml
diff --git a/packages/kubernetes/_dev/benchmark/rally/event-benchmark/config.yml b/packages/kubernetes/_dev/benchmark/rally/event-benchmark/config.yml
@@ -0,0 +1,40 @@
+fields:
+  - name: timestamp
+    period: 60m
+  - name: event_count
+    range:
+      min: 1
+      max: 10000
+  - name: event_duration
+    range:
+      min: 1
+      max: 10000000
+  - name: duration_start
+    enum: ["-1h", "-2h", "-3h", "-4h"]
+  - name: duration_end
+    enum: ["-30m", "-20m", "-10m", "-40m"]
+  - name: involved_object_kind
+    enum: ["Pod", "Node"]
+  - name: message_normal
+    value: "Node minikube status is now:"
+  - name: message_warning_backoff
+    value: "Back-off restarting failed container"
+  - name: message_warning_failed
+    enum:
+      - "Error: ErrImagePull"
+      - "Error: ImagePullBackOff"
+      - "Failed to pull image 'mylogin/posts:0.0.1': rpc error: code = Unknown desc = Error response from daemon: pull access denied for mylogin/posts, repository does not exist or may require 'docker login'"
+  - name: type
+    enum: ["Normal", "Warning"]
+  - name: reason_normal
+    enum: ["NodeHasSufficientDisk", "NodeHasSufficientMemory", "NodeHasNoDiskPressure", "NodeHasSufficientPID"]
+  - name: reason_warning
+    enum: ["Failed", "BackOff"]
+  - name: rangeofid
+    range:
+      min: 0
+      max: 10000
+  - name: metadata_resource_version
+    range:
+      min: 400000
+      max: 500000
diff --git a/packages/kubernetes/_dev/benchmark/rally/event-benchmark/fields.yml b/packages/kubernetes/_dev/benchmark/rally/event-benchmark/fields.yml
@@ -0,0 +1,38 @@
+- name: Ip
+  type: ip
+- name: timestamp
+  type: date
+- name: duration_start
+  type: keyword
+- name: duration_end
+  type: keyword
+- name: event_count
+  type: long
+- name: event_duration
+  type: long
+- name: involved_object_kind
+  type: keyword
+- name: involved_object_name
+  type: keyword
+- name: message_normal
+  type: keyword
+- name: message_warning_backoff
+  type: keyword
+- name: message_warning_failed
+  type: keyword
+- name: reason_normal
+  type: keyword
+- name: reason_warning
+  type: keyword
+- name: metadata_generate_name
+  type: keyword
+- name: metadata_timestamp_created
+  type: date
+- name: reason
+  type: keyword
+- name: type
+  type: keyword
+- name: rangeofid
+  type: integer
+- name: metadata_resource_version
+  type: long
diff --git a/packages/kubernetes/_dev/benchmark/rally/event-benchmark/template.ndjson b/packages/kubernetes/_dev/benchmark/rally/event-benchmark/template.ndjson
@@ -0,0 +1,139 @@
+{{- $timestamp := generate "timestamp" }}
+{{- $event_count := generate "event_count" }}
+{{- $event_duration := generate "event_duration" }}
+{{- $duration_start := generate "duration_start" }}
+{{- $duration_end := generate "duration_end" }}
+{{- $first_occurrence := generate "timestamp" | date_modify (print $duration_start) }}
+{{- $last_occurrence := generate "timestamp" | date_modify (print $duration_end) }}
+{{- $involved_object_kind := generate "involved_object_kind" }}
+{{- $involved_object_name := generate "involved_object_name" }}
+{{- $message_normal := generate "message_normal" }}
+{{- $message_warning_backoff := generate "message_warning_backoff" }}
+{{- $message_warning_failed := generate "message_warning_failed" }}
+{{- $reason_normal := generate "reason_normal" }}
+{{- $reason_warning := generate "reason_warning" }}
+{{- $metadata_generate_name := generate "metadata_generate_name" }}
+{{- $metadata_resource_version := generate "metadata_resource_version" }}
+{{- $reason := generate "reason" }}
+{{- $type := generate "type" }}
+{{- $rangeofid := generate "rangeofid" -}}
+{{- $eventid := div $rangeofid 110 -}}
+{
+    "@timestamp": "{{$timestamp.Format "2006-01-02T15:04:05.000Z07:00"}}",
+    "metricset": {
+        "name": "event"
+    },
+    "agent": {
+        "type": "metricbeat",
+        "version": "8.8.0",
+        "ephemeral_id": "22ed892c-43bd-408a-9121-65e2f5b6a56e",
+        "id": "de42127b-4db8-4471-824e-a7b14f478663",
+        "name": "kubernetes-scale-123456"
+    },
+     "elastic_agent": {
+        "id": "de42127b-4db8-4471-824e-a7b14f478663",
+        "version": "8.8.0",
+        "snapshot": true
+    },
+    "ecs": {
+        "version": "8.5.0"
+    },
+    "event": {
+        "duration": {{ $event_duration }},
+        "dataset": "kubernetes.event",
+        "module": "kubernetes"
+    },
+    "data_stream": {
+      "namespace": "ep",
+      "type": "metrics",
+      "dataset": "kubernetes.event"
+    },
+    "service": {
+        "type": "kubernetes"
+    },
+    "orchestrator":{
+        "cluster":{
+        "name":"kubernetes-scale",
+        "url":"https://{{ generate `Ip` }}"
+        }
+    },
+    "kubernetes": {
+        "event": {
+            "metadata": {
+                "uid": "host-{{ $eventid }}",
+                "resource_version": "{{ $metadata_resource_version }}",
+                "timestamp": {
+                    "created": "{{ $first_occurrence.Format "2006-01-02T15:04:05.000Z07:00" }}"
+                },
+                "name": "host-{{ $eventid }}",
+                "namespace": "demo-{{ $eventid }}",
+                "self_link": "/api/v1/namespaces/events/host-{{ $eventid }}",
+                "generate_name": "{{ $metadata_generate_name }}",
+                {{- if eq $type "Normal"}}
+                "annotations": {
+                    "kubectl_kubernetes_io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"involvedObject\":{\"kind\":\"{{ $involved_object_kind }}\"},\"kind\":\"Event\",\"message\":\"{{ $message_normal }} {{ $reason_normal }}\",\"metadata\":{\"annotations\":{},\"name\":\"host-{{ $eventid }}\",\"namespace\":\"demo-{{ $eventid }}\"},\"type\":\"{{ $type }}\"}\n"
+                }
+                {{- else if eq $type "Warning"}}
+                    {{- if eq $reason_warning "BackOff"}}
+                    "annotations": {
+                        "kubectl_kubernetes_io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"involvedObject\":{\"kind\":\"{{ $involved_object_kind }}\"},\"kind\":\"Event\",\"message\":\"{{ $message_warning_backoff }}\",\"metadata\":{\"annotations\":{},\"name\":\"host-{{ $eventid }}\",\"namespace\":\"demo-{{ $eventid }}\"},\"type\":\"{{ $type }}\"}\n"
+                    }
+                    {{- else if eq $reason_warning "Failed"}}
+                   "annotations": {
+                        "kubectl_kubernetes_io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"involvedObject\":{\"kind\":\"{{ $involved_object_kind }}\"},\"kind\":\"Event\",\"message\":\"{{ $message_warning_failed }}\",\"metadata\":{\"annotations\":{},\"name\":\"host-{{ $eventid }}\",\"namespace\":\"demo-{{ $eventid }}\"},\"type\":\"{{ $type }}\"}\n"
+                    }
+                    {{- end}}
+                {{- end}}
+            },
+            "timestamp": {
+                "first_occurrence": "{{ $first_occurrence.Format "2006-01-02T15:04:05.000Z07:00" }}",
+                "last_occurrence": "{{ $last_occurrence.Format "2006-01-02T15:04:05.000Z07:00" }}"
+            },
+            {{- if eq $type "Normal"}}
+            "message": "{{ $message_normal }} {{ $reason_normal }}",
+            "reason": "{{ $reason_normal }}",
+            "type": "{{ $type }}",
+            "count": {{ $event_count }},
+            {{- else if eq $type "Warning"}}
+                {{- if eq $reason_warning "BackOff"}}
+                "message": "{{ $message_warning_backoff }}",
+                "reason": "{{ $reason_warning }}",
+                "type": "{{ $type }}",
+                "count": {{ $event_count }},
+                {{- else if eq $reason_warning "Failed"}}
+                "message": "{{ $message_warning_failed }}",
+                "reason": "{{ $reason_warning }}",
+                "type": "{{ $type }}",
+                "count": {{ $event_count }},
+                {{- end}}
+            {{- end}}
+            "source": {
+                "host": "host-{{ $eventid }}",
+                "component": "demo-component-{{ $eventid }}"
+            },
+            "involved_object": {
+                "api_version": "v1",
+                "resource_version": "{{ $metadata_resource_version }}",
+                "name": "{{ $involved_object_name }}",
+                "kind": "{{ $involved_object_kind }}",
+                "uid": "host-{{ $eventid }}"
+            }
+        }    
+    },
+    "host": {
+        "hostname": "kubernetes-scale-123456",
+        "os": {
+            "kernel": "5.10.47-linuxkit",
+            "codename": "Core",
+            "name": "CentOS Linux",
+            "type": "linux",
+            "family": "redhat",
+            "version": "7 (Core)",
+            "platform": "centos"
+        },
+        "containerized": true,
+        "name": "kubernetes-scale-123456",
+        "id": "85e35c2b5e1b39ba72393a6baf6ee7cd",
+        "architecture": "x86_64"
+    }
+}