diff --git a/services/filebeat/.gitignore b/services/filebeat/.gitignore new file mode 100644 index 00000000..6f805618 --- /dev/null +++ b/services/filebeat/.gitignore @@ -0,0 +1,4 @@ + +!**/.gitkeep +**/filebeat +**/server diff --git a/services/filebeat/config/filebeat.yml b/services/filebeat/config/filebeat.yml new file mode 100644 index 00000000..dd0b797f --- /dev/null +++ b/services/filebeat/config/filebeat.yml @@ -0,0 +1,98 @@ +setup.ilm.enabled: false +setup.template.enabled: true +setup.template.name: "nginx-log" +setup.template.pattern: "nginx-log-*" +setup.template.overwrite: true + +# filebeat.inputs: +# - type: log +# enabled: true +# paths: +# - ${SERVICE_LOG_PATH}/*.log + +filebeat.inputs: + - type: filestream + enabled: true + id: all-logs-collector + scan_frequency: 1m + exclude_files: ['combined.log','servicemanager.log'] + prospector.scanner.exclude_files: ['combined.log','servicemanager-console.log','openobserve-console.log','filebeat-error.log','filebeat-console.log'] + paths: + - ${SERVICE_LOG_PATH}/*.log + parsers: + - multiline: + pattern: '^([0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T(2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9].[0-9]{3}Z )' + negate: true + match: after + processors: + # - grok: + # field: "message" + # patterns: + # - '%{TIMESTAMP_ISO8601:timestamp}${SPACE}%{LOGLEVEL:level}${SPACE}%{GREEDYDATA:content}' + - dissect: + tokenizer: "%{timestamp} %{level} %{service} -- %{message}" + field: "message" + target_prefix: "log" + - timestamp: + field: "log_timestamp" + layouts: + - '2006-01-02T15:04:05Z' + - '2006-01-02T15:04:05.999Z' + - '2006-01-02T15:04:05.999-07:00' + - '2024-07-11T14:25:08.092898400+00:00' + test: + - '2024-07-11T13:25:15Z' + - '2024-07-11T13:25:15.140Z' + - '2024-07-11T13:25:15.140456+02:00' + - '2024-07-11T14:25:08.092898400+00:00' + - decode_json_fields: + fields: ["log_message"] + # target: "json" + # process_array: true + max_depth: 3 + add_error_key: false + # - copy_fields: + # fields: + # - from: dissect_level + # to: event.level + # - from: dissect_message + # to: event.message + # - from: dissect_timestamp + # to: event.timestamp + # - from: dissect_service + # to: event.service + # fail_on_error: false + # ignore_missing: true + - drop_fields: + fields: ["message"] + + - type: filestream + enabled: false + id: json-combined-collector + paths: + - ${SERVICE_LOG_PATH}/combined.log + parsers: + - ndjson: + keys_under_root: true + message_key: message + target: "" + add_error_key: true + processors: + # # - json: + # # keys_under_root: true + # # add_error_key: true + # # message_key: message + # # overwrite_keys: true + - decode_json_fields: + fields: ["message"] + # target: "json" + # process_array: true + # max_depth: 3 + +output.elasticsearch: + hosts: ["${OPENOBSERVE_URL}"] + timeout: 10 + path: "/api/default/" + index: "default" + username: "${OPENOBSERVE_ZO_ROOT_USER_EMAIL}" + password: "${OPENOBSERVE_ZO_ROOT_USER_PASSWORD}" diff --git a/services/filebeat/service.json b/services/filebeat/service.json new file mode 100644 index 00000000..ebbf0000 --- /dev/null +++ b/services/filebeat/service.json @@ -0,0 +1,67 @@ +{ + "id": "filebeat", + "name": "Filebeat Log Service", + "description": "Filebeat 8.14.2", + "version": "8.14.2", + "enabled": true, + "status": "30", + "logoutput": "", + "icon": "pi pi-globe", + "servicelocation": 10, + "execconfig": { + "debuglog": true, + "serviceorder": 3, + "executable": { + "win32": "filebeat\\filebeat.exe", + "darwin": "filebeat/filebeat", + "linux": "filebeat/filebeat", + "default": "filebeat\\filebeat" + }, + "env": { + "PATH": [ + "${ARCHIVE_HOME}", + "C:\\WINDOWS\\system32", + "C:\\WINDOWS" + ] + }, + "globalenv": { + "FILEBEAT_HOME": "${SERVICE_HOME}" + }, + "outputvarregex": { + "FILEBEAT_ENABLED_INPUTS": ".*Loading and starting Inputs completed. Enabled inputs: (\\d+).*" + }, + "healthcheck": { + "type": "variable", + "variable": "FILEBEAT_ENABLED_INPUTS", + "retries": 180 + }, + "commandline": { + "win32": " run -e -v --path.config ${SERVICE_HOME}\\config --path.home \"${SERVICE_HOME}\\win32\\filebeat\" --path.logs \"${SERVICE_LOG_PATH}\" --path.data \"${SERVICE_DATA_PATH}\"", + "darwin": " run --path.config ${SERVICE_HOME}\\config --path.home \"${SERVICE_HOME}\\darwin\\filebeat\" --path.logs \"${SERVICE_LOG_PATH}\" --path.data \"${SERVICE_DATA_PATH}\" ", + "linux": " run --path.config ${SERVICE_HOME}\\config --path.home \"${SERVICE_HOME}\\linux\\filebeat\" --path.logs \"${SERVICE_LOG_PATH}\" --path.data \"${SERVICE_DATA_PATH}\" ", + "default": " run -e -v --path.config ${SERVICE_HOME}\\config --path.home \"${SERVICE_HOME}\\win32\\filebeat\" --path.logs \"${SERVICE_LOG_PATH}\" --path.data \"${SERVICE_DATA_PATH}\" " + }, + "datapath": "/server/data", + "setuparchive": { + "win32": { + "name": "filebeat.7z", + "output": "filebeat" + }, + "darwin": { + "name": "filebeat.7z", + "output": "filebeat" + }, + "linux": { + "name": "filebeat.7z", + "output": "filebeat" + } + }, + "setup": { + "win32": [ + " run --help", + " test config ${SERVICE_HOME}\\config\\filebeat.yml", + " test output ${SERVICE_HOME}\\config\\filebeat.yml" + ] + } + } +} diff --git a/services/filebeat/win32/filebeat.7z b/services/filebeat/win32/filebeat.7z new file mode 100644 index 00000000..c71beb72 Binary files /dev/null and b/services/filebeat/win32/filebeat.7z differ