From 5cc61cf114c31127f219f5b40acc9fd6072169be Mon Sep 17 00:00:00 2001 From: mallikabandaru <158026222+mallikabandaru@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:32:50 +0530 Subject: [PATCH 1/6] Update main.tf --- modules/runtime_container_engine_config/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/runtime_container_engine_config/main.tf b/modules/runtime_container_engine_config/main.tf index 145909e..2e5515b 100644 --- a/modules/runtime_container_engine_config/main.tf +++ b/modules/runtime_container_engine_config/main.tf @@ -38,6 +38,8 @@ locals { TFE_IACT_SUBNETS = var.iact_subnets TFE_IACT_TIME_LIMIT = var.iact_time_limit TFE_IACT_TRUSTED_PROXIES = join(",", var.trusted_proxies) + TFE_LOG_FORWARDING_CONFIG_PATH = var.tfe_log_forwarding_config_path + TFE_LOG_FORWARDING_ENABLED = var.tfe_log_forwarding_enabled } ) # compose files allow for $ deliminated variable injection. $$ is the appropriate escape. From f57267c589c8ea2e338b334b0e79a19507e1b1f9 Mon Sep 17 00:00:00 2001 From: mallikabandaru Date: Wed, 8 Jan 2025 17:40:27 +0530 Subject: [PATCH 2/6] added the log variables --- .../runtime_container_engine_config/variables.tf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/modules/runtime_container_engine_config/variables.tf b/modules/runtime_container_engine_config/variables.tf index 37f04d6..8087b9d 100644 --- a/modules/runtime_container_engine_config/variables.tf +++ b/modules/runtime_container_engine_config/variables.tf @@ -287,6 +287,21 @@ variable "tfe_license" { description = "The HashiCorp license. Defaults to \"\" if no value is given. Required when TFE_LICENSE_PATH is unset." } +# ------------------------------------------------------ +# Log Forwarding and Metrics +# ------------------------------------------------------ +variable "tfe_log_forwarding_enabled" { + default = false + type = bool + description = "(Optional) Whether or not to enable log forwarding for Terraform Enterprise. Defaults to false." +} + +variable "tfe_log_forwarding_config_path" { + default = null + type = string + description = "The path to a file containing valid Fluent Bit [OUTPUT] configuration" +} + variable "tls_ca_bundle_file" { default = null type = string From 673c8e2e0a04475271121efd6a29cba07813ff93 Mon Sep 17 00:00:00 2001 From: mallikabandaru Date: Tue, 21 Jan 2025 13:01:39 +0530 Subject: [PATCH 3/6] added the fluent-bit.conf in userdata and bind --- modules/runtime_container_engine_config/main.tf | 10 ++++++++++ .../templates/aws.ubuntu.docker.tfe.sh.tpl | 14 ++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/modules/runtime_container_engine_config/main.tf b/modules/runtime_container_engine_config/main.tf index 2e5515b..21a8e26 100644 --- a/modules/runtime_container_engine_config/main.tf +++ b/modules/runtime_container_engine_config/main.tf @@ -80,6 +80,16 @@ locals { source = "/var/run/docker.sock" target = "/run/docker.sock" }, + { + type = "bind" + source = "/var/run/docker.sock" + target = "/run/docker.sock" + }, + { + type = "bind" + source = "/etc/fluent-bit.conf" + target = "/etc/fluent-bit.conf" + }, { type = "bind" source = "/etc/tfe/ssl" diff --git a/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl b/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl index 2c3f6e2..a5e9458 100644 --- a/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl +++ b/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl @@ -32,6 +32,20 @@ https_proxy="${proxy_ip}:${proxy_port}" no_proxy="${no_proxy}" EOF +cat < /etc/fluent-bit.conf + +[OUTPUT] + Name datadog + Match * + Host http-intake.logs.datadoghq.com + TLS On + compress gzip + apikey 1234 + dd_service terraform-enterprise + dd_source docker + dd_tags environment:development,owner:scale-perf-team +EOF + /bin/cat </etc/apt/apt.conf Acquire::http::Proxy "http://${proxy_ip}:${proxy_port}"; Acquire::https::Proxy "http://${proxy_ip}:${proxy_port}"; From 736ebd239fbc495b8a89d500e82e15616aa814c5 Mon Sep 17 00:00:00 2001 From: mallikabandaru Date: Tue, 21 Jan 2025 14:21:32 +0530 Subject: [PATCH 4/6] added the fluent-bit.conf in userdata and bind --- modules/runtime_container_engine_config/main.tf | 4 ++-- modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/runtime_container_engine_config/main.tf b/modules/runtime_container_engine_config/main.tf index 21a8e26..d33f3ac 100644 --- a/modules/runtime_container_engine_config/main.tf +++ b/modules/runtime_container_engine_config/main.tf @@ -87,8 +87,8 @@ locals { }, { type = "bind" - source = "/etc/fluent-bit.conf" - target = "/etc/fluent-bit.conf" + source = "/etc/tfe/fluent-bit.conf" + target = "/etc/tfe/fluent-bit.conf" }, { type = "bind" diff --git a/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl b/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl index a5e9458..46f317a 100644 --- a/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl +++ b/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl @@ -32,7 +32,7 @@ https_proxy="${proxy_ip}:${proxy_port}" no_proxy="${no_proxy}" EOF -cat < /etc/fluent-bit.conf +cat < /etc/tfe/fluent-bit.conf [OUTPUT] Name datadog From 4ac079f8d61be886035b556183b8ad4817d0d371 Mon Sep 17 00:00:00 2001 From: mallikabandaru Date: Fri, 24 Jan 2025 10:28:38 +0530 Subject: [PATCH 5/6] added the fluent-bit.conf in userdata and bind --- .../runtime_container_engine_config/main.tf | 58 +++++++++---------- .../templates/aws.ubuntu.docker.tfe.sh.tpl | 3 +- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/modules/runtime_container_engine_config/main.tf b/modules/runtime_container_engine_config/main.tf index d33f3ac..43a2f31 100644 --- a/modules/runtime_container_engine_config/main.tf +++ b/modules/runtime_container_engine_config/main.tf @@ -11,35 +11,35 @@ locals { local.storage_configuration, local.vault_configuration, { - http_proxy = var.http_proxy != null ? "http://${var.http_proxy}" : null - HTTP_PROXY = var.http_proxy != null ? "http://${var.http_proxy}" : null - https_proxy = var.https_proxy != null ? "http://${var.https_proxy}" : null - HTTPS_PROXY = var.https_proxy != null ? "http://${var.https_proxy}" : null - no_proxy = var.no_proxy != null ? join(",", var.no_proxy) : null - NO_PROXY = var.no_proxy != null ? join(",", var.no_proxy) : null - TFE_HOSTNAME = var.hostname - TFE_HTTP_PORT = var.http_port - TFE_HTTPS_PORT = var.https_port - TFE_OPERATIONAL_MODE = var.operational_mode - TFE_ENCRYPTION_PASSWORD = random_password.enc_password.result - TFE_DISK_CACHE_VOLUME_NAME = "terraform-enterprise_terraform-enterprise-cache" - TFE_LICENSE_REPORTING_OPT_OUT = var.license_reporting_opt_out - TFE_USAGE_REPORTING_OPT_OUT = var.usage_reporting_opt_out - TFE_LICENSE = var.tfe_license - TFE_TLS_CA_BUNDLE_FILE = var.tls_ca_bundle_file != null ? var.tls_ca_bundle_file : null - TFE_TLS_CERT_FILE = var.cert_file - TFE_TLS_CIPHERS = var.tls_ciphers - TFE_TLS_KEY_FILE = var.key_file - TFE_TLS_VERSION = var.tls_version != null ? var.tls_version : "" - TFE_RUN_PIPELINE_IMAGE = var.run_pipeline_image - TFE_CAPACITY_CONCURRENCY = var.capacity_concurrency - TFE_CAPACITY_CPU = var.capacity_cpu - TFE_CAPACITY_MEMORY = var.capacity_memory - TFE_IACT_SUBNETS = var.iact_subnets - TFE_IACT_TIME_LIMIT = var.iact_time_limit - TFE_IACT_TRUSTED_PROXIES = join(",", var.trusted_proxies) + http_proxy = var.http_proxy != null ? "http://${var.http_proxy}" : null + HTTP_PROXY = var.http_proxy != null ? "http://${var.http_proxy}" : null + https_proxy = var.https_proxy != null ? "http://${var.https_proxy}" : null + HTTPS_PROXY = var.https_proxy != null ? "http://${var.https_proxy}" : null + no_proxy = var.no_proxy != null ? join(",", var.no_proxy) : null + NO_PROXY = var.no_proxy != null ? join(",", var.no_proxy) : null + TFE_HOSTNAME = var.hostname + TFE_HTTP_PORT = var.http_port + TFE_HTTPS_PORT = var.https_port + TFE_OPERATIONAL_MODE = var.operational_mode + TFE_ENCRYPTION_PASSWORD = random_password.enc_password.result + TFE_DISK_CACHE_VOLUME_NAME = "terraform-enterprise_terraform-enterprise-cache" + TFE_LICENSE_REPORTING_OPT_OUT = var.license_reporting_opt_out + TFE_USAGE_REPORTING_OPT_OUT = var.usage_reporting_opt_out + TFE_LICENSE = var.tfe_license + TFE_TLS_CA_BUNDLE_FILE = var.tls_ca_bundle_file != null ? var.tls_ca_bundle_file : null + TFE_TLS_CERT_FILE = var.cert_file + TFE_TLS_CIPHERS = var.tls_ciphers + TFE_TLS_KEY_FILE = var.key_file + TFE_TLS_VERSION = var.tls_version != null ? var.tls_version : "" + TFE_RUN_PIPELINE_IMAGE = var.run_pipeline_image + TFE_CAPACITY_CONCURRENCY = var.capacity_concurrency + TFE_CAPACITY_CPU = var.capacity_cpu + TFE_CAPACITY_MEMORY = var.capacity_memory + TFE_IACT_SUBNETS = var.iact_subnets + TFE_IACT_TIME_LIMIT = var.iact_time_limit + TFE_IACT_TRUSTED_PROXIES = join(",", var.trusted_proxies) TFE_LOG_FORWARDING_CONFIG_PATH = var.tfe_log_forwarding_config_path - TFE_LOG_FORWARDING_ENABLED = var.tfe_log_forwarding_enabled + TFE_LOG_FORWARDING_ENABLED = var.tfe_log_forwarding_enabled } ) # compose files allow for $ deliminated variable injection. $$ is the appropriate escape. @@ -88,7 +88,7 @@ locals { { type = "bind" source = "/etc/tfe/fluent-bit.conf" - target = "/etc/tfe/fluent-bit.conf" + target = "/etc/fluent-bit.conf" }, { type = "bind" diff --git a/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl b/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl index 46f317a..faf97d6 100644 --- a/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl +++ b/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl @@ -32,8 +32,7 @@ https_proxy="${proxy_ip}:${proxy_port}" no_proxy="${no_proxy}" EOF -cat < /etc/tfe/fluent-bit.conf - +/bin/cat < /etc/tfe/fluent-bit.conf [OUTPUT] Name datadog Match * From 4c8d7fc3facb2033fd0814ded6a06fc83aa7cbef Mon Sep 17 00:00:00 2001 From: mallikabandaru Date: Fri, 24 Jan 2025 14:59:11 +0530 Subject: [PATCH 6/6] added the fluent-bit.conf in userdata and bind --- modules/runtime_container_engine_config/main.tf | 2 +- modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/runtime_container_engine_config/main.tf b/modules/runtime_container_engine_config/main.tf index 43a2f31..8044296 100644 --- a/modules/runtime_container_engine_config/main.tf +++ b/modules/runtime_container_engine_config/main.tf @@ -87,7 +87,7 @@ locals { }, { type = "bind" - source = "/etc/tfe/fluent-bit.conf" + source = "/etc/fluent-bit/fluent-bit.conf" target = "/etc/fluent-bit.conf" }, { diff --git a/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl b/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl index faf97d6..1b4e26c 100644 --- a/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl +++ b/modules/tfe_init/templates/aws.ubuntu.docker.tfe.sh.tpl @@ -32,7 +32,8 @@ https_proxy="${proxy_ip}:${proxy_port}" no_proxy="${no_proxy}" EOF -/bin/cat < /etc/tfe/fluent-bit.conf +mkdir -p /etc/fluent-bit +/bin/cat < /etc/fluent-bit/fluent-bit.conf [OUTPUT] Name datadog Match * @@ -45,6 +46,8 @@ EOF dd_tags environment:development,owner:scale-perf-team EOF +chmod 644 /etc/fluent-bit/fluent-bit.conf + /bin/cat </etc/apt/apt.conf Acquire::http::Proxy "http://${proxy_ip}:${proxy_port}"; Acquire::https::Proxy "http://${proxy_ip}:${proxy_port}";