add filebeat for log streaming.

services/filebeat/.gitignore

@@ -0,0 +1,4 @@
+
+!**/.gitkeep
+**/filebeat
+**/server

services/filebeat/config/filebeat.yml

@@ -0,0 +1,98 @@
+setup.ilm.enabled: false
+setup.template.enabled: true
+setup.template.name: "nginx-log"
+setup.template.pattern: "nginx-log-*"
+setup.template.overwrite: true
+
+# filebeat.inputs:
+# - type: log
+#   enabled: true
+#   paths:
+#     - ${SERVICE_LOG_PATH}/*.log
+
+filebeat.inputs:
+  - type: filestream
+    enabled: true
+    id: all-logs-collector
+    scan_frequency: 1m
+    exclude_files: ['combined.log','servicemanager.log']
+    prospector.scanner.exclude_files: ['combined.log','servicemanager-console.log','openobserve-console.log','filebeat-error.log','filebeat-console.log']
+    paths:
+      - ${SERVICE_LOG_PATH}/*.log
+    parsers:
+      - multiline:
+          pattern: '^([0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T(2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9].[0-9]{3}Z )'
+          negate: true
+          match: after
+    processors:
+      # - grok:
+      #     field: "message"
+      #     patterns:
+      #       - '%{TIMESTAMP_ISO8601:timestamp}${SPACE}%{LOGLEVEL:level}${SPACE}%{GREEDYDATA:content}'
+      - dissect:
+          tokenizer: "%{timestamp} %{level} %{service} -- %{message}"
+          field: "message"
+          target_prefix: "log"
+      - timestamp:
+          field: "log_timestamp"
+          layouts:
+            - '2006-01-02T15:04:05Z'
+            - '2006-01-02T15:04:05.999Z'
+            - '2006-01-02T15:04:05.999-07:00'
+            - '2024-07-11T14:25:08.092898400+00:00'
+          test:
+            - '2024-07-11T13:25:15Z'
+            - '2024-07-11T13:25:15.140Z'
+            - '2024-07-11T13:25:15.140456+02:00'
+            - '2024-07-11T14:25:08.092898400+00:00'
+      - decode_json_fields:
+          fields: ["log_message"]
+          # target: "json"
+          # process_array: true
+          max_depth: 3
+          add_error_key: false
+      # - copy_fields:
+      #     fields:
+      #       - from: dissect_level
+      #         to: event.level
+      #       - from: dissect_message
+      #         to: event.message
+      #       - from: dissect_timestamp
+      #         to: event.timestamp
+      #       - from: dissect_service
+      #         to: event.service
+      #     fail_on_error: false
+      #     ignore_missing: true
+      - drop_fields:
+          fields: ["message"]
+
+  - type: filestream
+    enabled: false
+    id: json-combined-collector
+    paths:
+      - ${SERVICE_LOG_PATH}/combined.log
+    parsers:
+      - ndjson:
+          keys_under_root: true
+          message_key: message
+          target: ""
+          add_error_key: true
+    processors:
+    #   # - json:
+    #   #     keys_under_root: true
+    #   #     add_error_key: true
+    #   #     message_key: message
+    #   #     overwrite_keys: true
+      - decode_json_fields:
+          fields: ["message"]
+          # target: "json"
+          # process_array: true
+          # max_depth: 3
+
+output.elasticsearch:
+  hosts: ["${OPENOBSERVE_URL}"]
+  timeout: 10
+  path: "/api/default/"
+  index: "default"
+  username: "${OPENOBSERVE_ZO_ROOT_USER_EMAIL}"
+  password: "${OPENOBSERVE_ZO_ROOT_USER_PASSWORD}"

services/filebeat/service.json

@@ -0,0 +1,67 @@
+{
+  "id": "filebeat",
+  "name": "Filebeat Log Service",
+  "description": "Filebeat 8.14.2",
+  "version": "8.14.2",
+  "enabled": true,
+  "status": "30",
+  "logoutput": "",
+  "icon": "pi pi-globe",
+  "servicelocation": 10,
+  "execconfig": {
+    "debuglog": true,
+    "serviceorder": 3,
+    "executable": {
+      "win32": "filebeat\\filebeat.exe",
+      "darwin": "filebeat/filebeat",
+      "linux": "filebeat/filebeat",
+      "default": "filebeat\\filebeat"
+    },
+    "env": {
+      "PATH": [
+        "${ARCHIVE_HOME}",
+        "C:\\WINDOWS\\system32",
+        "C:\\WINDOWS"
+      ]
+    },
+    "globalenv": {
+      "FILEBEAT_HOME": "${SERVICE_HOME}"
+    },
+    "outputvarregex": {
+      "FILEBEAT_ENABLED_INPUTS": ".*Loading and starting Inputs completed. Enabled inputs: (\\d+).*"
+    },
+    "healthcheck": {
+      "type": "variable",
+      "variable": "FILEBEAT_ENABLED_INPUTS",
+      "retries": 180
+    },
+    "commandline": {
+      "win32": " run -e -v --path.config ${SERVICE_HOME}\\config --path.home \"${SERVICE_HOME}\\win32\\filebeat\" --path.logs \"${SERVICE_LOG_PATH}\" --path.data \"${SERVICE_DATA_PATH}\"",
+      "darwin": " run --path.config ${SERVICE_HOME}\\config --path.home \"${SERVICE_HOME}\\darwin\\filebeat\" --path.logs \"${SERVICE_LOG_PATH}\" --path.data \"${SERVICE_DATA_PATH}\" ",
+      "linux": " run --path.config ${SERVICE_HOME}\\config --path.home \"${SERVICE_HOME}\\linux\\filebeat\" --path.logs \"${SERVICE_LOG_PATH}\" --path.data \"${SERVICE_DATA_PATH}\" ",
+      "default": " run -e -v --path.config ${SERVICE_HOME}\\config --path.home \"${SERVICE_HOME}\\win32\\filebeat\" --path.logs \"${SERVICE_LOG_PATH}\" --path.data \"${SERVICE_DATA_PATH}\" "
+    },
+    "datapath": "/server/data",
+    "setuparchive": {
+      "win32": {
+        "name": "filebeat.7z",
+        "output": "filebeat"
+      },
+      "darwin": {
+        "name": "filebeat.7z",
+        "output": "filebeat"
+      },
+      "linux": {
+        "name": "filebeat.7z",
+        "output": "filebeat"
+      }
+    },
+    "setup": {
+      "win32": [
+        " run --help",
+        " test config ${SERVICE_HOME}\\config\\filebeat.yml",
+        " test output ${SERVICE_HOME}\\config\\filebeat.yml"
+      ]
+    }
+  }
+}