From 2d36a65052d276b8908bf9b9f90864f9007a6cd2 Mon Sep 17 00:00:00 2001 From: RMcVelia Date: Mon, 19 Feb 2024 16:00:49 +0000 Subject: [PATCH 1/3] Fix namespace dependency --- cluster/terraform_kubernetes/prometheus.tf | 2 +- cluster/terraform_kubernetes/thanos.tf | 10 +++++----- cluster/terraform_kubernetes/welcome_app.tf | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cluster/terraform_kubernetes/prometheus.tf b/cluster/terraform_kubernetes/prometheus.tf index 747e93ac..58fe0e7c 100644 --- a/cluster/terraform_kubernetes/prometheus.tf +++ b/cluster/terraform_kubernetes/prometheus.tf @@ -46,7 +46,7 @@ resource "kubernetes_config_map" "prometheus" { metadata { name = "prometheus-server-conf" - namespace = "monitoring" + namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name } data = { diff --git a/cluster/terraform_kubernetes/thanos.tf b/cluster/terraform_kubernetes/thanos.tf index 72386416..4154b2ce 100644 --- a/cluster/terraform_kubernetes/thanos.tf +++ b/cluster/terraform_kubernetes/thanos.tf @@ -34,7 +34,7 @@ resource "kubernetes_service" "thanos-store-gateway" { metadata { name = "thanos-store-gateway" - namespace = "monitoring" + namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name } spec { @@ -54,7 +54,7 @@ resource "kubernetes_deployment" "thanos-querier" { metadata { name = "thanos-querier" - namespace = "monitoring" + namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name } spec { @@ -132,7 +132,7 @@ resource "kubernetes_service" "thanos-querier" { metadata { name = "thanos-querier" - namespace = "monitoring" + namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name labels = { app = "thanos-querier" } @@ -155,7 +155,7 @@ resource "kubernetes_deployment" "thanos-store-gateway" { metadata { name = "thanos-store-gateway" - namespace = "monitoring" + namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name } spec { @@ -248,7 +248,7 @@ resource "kubernetes_deployment" "thanos-compactor" { metadata { name = "thanos-compactor" - namespace = "monitoring" + namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name } spec { diff --git a/cluster/terraform_kubernetes/welcome_app.tf b/cluster/terraform_kubernetes/welcome_app.tf index 09138b43..0e6f125f 100644 --- a/cluster/terraform_kubernetes/welcome_app.tf +++ b/cluster/terraform_kubernetes/welcome_app.tf @@ -3,7 +3,7 @@ resource "kubernetes_deployment" "welcome_app" { metadata { name = local.welcome_app_name - namespace = local.welcome_app_namespace + namespace = kubernetes_namespace.default_list["${local.welcome_app_namespace}"].metadata[0].name } spec { replicas = 2 @@ -73,7 +73,7 @@ resource "kubernetes_service" "welcome_app" { metadata { name = local.welcome_app_name - namespace = local.welcome_app_namespace + namespace = kubernetes_namespace.default_list["${local.welcome_app_namespace}"].metadata[0].name } spec { type = "ClusterIP" From 4c570024c135070ad46c3186e634284404cea643 Mon Sep 17 00:00:00 2001 From: RMcVelia Date: Mon, 19 Feb 2024 16:26:42 +0000 Subject: [PATCH 2/3] Fix dev cluster thanos storage account name --- cluster/terraform_kubernetes/thanos.tf | 2 +- cluster/terraform_kubernetes/variables.tf | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cluster/terraform_kubernetes/thanos.tf b/cluster/terraform_kubernetes/thanos.tf index 4154b2ce..9df4d211 100644 --- a/cluster/terraform_kubernetes/thanos.tf +++ b/cluster/terraform_kubernetes/thanos.tf @@ -1,6 +1,6 @@ resource "azurerm_storage_account" "thanos" { - name = "${var.resource_prefix}${var.cluster_short}thanossa" + name = "${var.resource_prefix}${local.cluster_sa_name}thanossa" location = data.azurerm_resource_group.resource_group.location resource_group_name = data.azurerm_resource_group.resource_group.name account_tier = "Standard" diff --git a/cluster/terraform_kubernetes/variables.tf b/cluster/terraform_kubernetes/variables.tf index 50bd3308..97c23b04 100644 --- a/cluster/terraform_kubernetes/variables.tf +++ b/cluster/terraform_kubernetes/variables.tf @@ -176,4 +176,9 @@ locals { storage-account-name = azurerm_storage_account.thanos.name storage-account-key = azurerm_storage_account.thanos.primary_access_key } + + cluster_sa_name = (var.environment == var.config ? + var.cluster_short : # pt,ts or pd + var.environment # cluster1, cluster2, etc + ) } From f28d71192fd49c4e10e86ca95b53f74f591022e1 Mon Sep 17 00:00:00 2001 From: RMcVelia Date: Tue, 20 Feb 2024 08:52:37 +0000 Subject: [PATCH 3/3] fix for thanos querier to see store correctly We don't need to label by replica as it's a single prometheus Prometheus service should be clusterip not nodeport Thanos store gateway is a headless service --- .../config/prometheus/development.prometheus.yml | 2 -- .../config/prometheus/platform-test.prometheus.yml | 2 -- .../config/prometheus/production.prometheus.yml | 2 -- .../config/prometheus/test.prometheus.yml | 2 -- cluster/terraform_kubernetes/prometheus.tf | 12 +----------- cluster/terraform_kubernetes/thanos.tf | 8 ++++---- 6 files changed, 5 insertions(+), 23 deletions(-) diff --git a/cluster/terraform_kubernetes/config/prometheus/development.prometheus.yml b/cluster/terraform_kubernetes/config/prometheus/development.prometheus.yml index 1aaf4914..0f542c20 100644 --- a/cluster/terraform_kubernetes/config/prometheus/development.prometheus.yml +++ b/cluster/terraform_kubernetes/config/prometheus/development.prometheus.yml @@ -13,8 +13,6 @@ global: keep_dropped_targets: 100 external_labels: cluster: prometheus - # Each Prometheus has to have unique labels. - replica: $(POD_NAME) rule_files: - /etc/prometheus/prometheus.rules alerting: diff --git a/cluster/terraform_kubernetes/config/prometheus/platform-test.prometheus.yml b/cluster/terraform_kubernetes/config/prometheus/platform-test.prometheus.yml index 3ede8793..335eb3a1 100644 --- a/cluster/terraform_kubernetes/config/prometheus/platform-test.prometheus.yml +++ b/cluster/terraform_kubernetes/config/prometheus/platform-test.prometheus.yml @@ -13,8 +13,6 @@ global: keep_dropped_targets: 100 external_labels: cluster: prometheus - # Each Prometheus has to have unique labels. - replica: $(POD_NAME) rule_files: - /etc/prometheus/prometheus.rules alerting: diff --git a/cluster/terraform_kubernetes/config/prometheus/production.prometheus.yml b/cluster/terraform_kubernetes/config/prometheus/production.prometheus.yml index 3ede8793..335eb3a1 100644 --- a/cluster/terraform_kubernetes/config/prometheus/production.prometheus.yml +++ b/cluster/terraform_kubernetes/config/prometheus/production.prometheus.yml @@ -13,8 +13,6 @@ global: keep_dropped_targets: 100 external_labels: cluster: prometheus - # Each Prometheus has to have unique labels. - replica: $(POD_NAME) rule_files: - /etc/prometheus/prometheus.rules alerting: diff --git a/cluster/terraform_kubernetes/config/prometheus/test.prometheus.yml b/cluster/terraform_kubernetes/config/prometheus/test.prometheus.yml index 3ede8793..335eb3a1 100644 --- a/cluster/terraform_kubernetes/config/prometheus/test.prometheus.yml +++ b/cluster/terraform_kubernetes/config/prometheus/test.prometheus.yml @@ -13,8 +13,6 @@ global: keep_dropped_targets: 100 external_labels: cluster: prometheus - # Each Prometheus has to have unique labels. - replica: $(POD_NAME) rule_files: - /etc/prometheus/prometheus.rules alerting: diff --git a/cluster/terraform_kubernetes/prometheus.tf b/cluster/terraform_kubernetes/prometheus.tf index 58fe0e7c..c98f0efb 100644 --- a/cluster/terraform_kubernetes/prometheus.tf +++ b/cluster/terraform_kubernetes/prometheus.tf @@ -149,15 +149,6 @@ resource "kubernetes_deployment" "prometheus" { # "--reloader.rule-dir=/etc/prometheus/rules/", ] - env { - name = "POD_NAME" - value_from { - field_ref { - field_path = "metadata.name" - } - } - } - liveness_probe { http_get { path = "/-/healthy" @@ -235,13 +226,12 @@ resource "kubernetes_service" "prometheus" { spec { port { - node_port = 30000 + name = "prometheus" port = 8080 target_port = kubernetes_deployment.prometheus.spec[0].template[0].spec[0].container[0].port[0].container_port } selector = { app = "prometheus" } - type = "NodePort" } } diff --git a/cluster/terraform_kubernetes/thanos.tf b/cluster/terraform_kubernetes/thanos.tf index 9df4d211..064f9de4 100644 --- a/cluster/terraform_kubernetes/thanos.tf +++ b/cluster/terraform_kubernetes/thanos.tf @@ -46,7 +46,8 @@ resource "kubernetes_service" "thanos-store-gateway" { selector = { thanos-store-api : "true" } - type = "ClusterIP" + type = "ClusterIP" + cluster_ip = "None" } } @@ -84,7 +85,7 @@ resource "kubernetes_deployment" "thanos-querier" { "query", "--log.level=debug", "--query.replica-label=replica", - "--store=dnssrv+thanos-store-gateway:10901", + "--store=dns+thanos-store-gateway:10901", ] liveness_probe { @@ -263,8 +264,7 @@ resource "kubernetes_deployment" "thanos-compactor" { template { metadata { labels = { - app = "thanos-compactor" - thanos-store-api = true + app = "thanos-compactor" } }