fix for thanos querier to see store correctly

We don't need to label by replica as it's a single prometheus Prometheus service should be clusterip not nodeport Thanos store gateway is a headless service
DFE-Digital · Feb 20, 2024 · f28d711 · f28d711
1 parent 4c57002
commit f28d711
Show file tree

Hide file tree

Showing 6 changed files with 5 additions and 23 deletions.
diff --git a/cluster/terraform_kubernetes/config/prometheus/development.prometheus.yml b/cluster/terraform_kubernetes/config/prometheus/development.prometheus.yml
@@ -13,8 +13,6 @@ global:
   keep_dropped_targets: 100
   external_labels:
     cluster: prometheus
-    # Each Prometheus has to have unique labels.
-    replica: $(POD_NAME)
 rule_files:
   - /etc/prometheus/prometheus.rules
 alerting:

diff --git a/cluster/terraform_kubernetes/config/prometheus/platform-test.prometheus.yml b/cluster/terraform_kubernetes/config/prometheus/platform-test.prometheus.yml
@@ -13,8 +13,6 @@ global:
   keep_dropped_targets: 100
   external_labels:
     cluster: prometheus
-    # Each Prometheus has to have unique labels.
-    replica: $(POD_NAME)
 rule_files:
   - /etc/prometheus/prometheus.rules
 alerting:

diff --git a/cluster/terraform_kubernetes/config/prometheus/production.prometheus.yml b/cluster/terraform_kubernetes/config/prometheus/production.prometheus.yml
@@ -13,8 +13,6 @@ global:
   keep_dropped_targets: 100
   external_labels:
     cluster: prometheus
-    # Each Prometheus has to have unique labels.
-    replica: $(POD_NAME)
 rule_files:
   - /etc/prometheus/prometheus.rules
 alerting:

diff --git a/cluster/terraform_kubernetes/config/prometheus/test.prometheus.yml b/cluster/terraform_kubernetes/config/prometheus/test.prometheus.yml
@@ -13,8 +13,6 @@ global:
   keep_dropped_targets: 100
   external_labels:
     cluster: prometheus
-    # Each Prometheus has to have unique labels.
-    replica: $(POD_NAME)
 rule_files:
   - /etc/prometheus/prometheus.rules
 alerting:

diff --git a/cluster/terraform_kubernetes/prometheus.tf b/cluster/terraform_kubernetes/prometheus.tf
@@ -149,15 +149,6 @@ resource "kubernetes_deployment" "prometheus" {
             # "--reloader.rule-dir=/etc/prometheus/rules/",
           ]
 
-          env {
-            name = "POD_NAME"
-            value_from {
-              field_ref {
-                field_path = "metadata.name"
-              }
-            }
-          }
-
           liveness_probe {
             http_get {
               path = "/-/healthy"
@@ -235,13 +226,12 @@ resource "kubernetes_service" "prometheus" {
 
   spec {
     port {
-      node_port   = 30000
+      name        = "prometheus"
       port        = 8080
       target_port = kubernetes_deployment.prometheus.spec[0].template[0].spec[0].container[0].port[0].container_port
     }
     selector = {
       app = "prometheus"
     }
-    type = "NodePort"
   }
 }
diff --git a/cluster/terraform_kubernetes/thanos.tf b/cluster/terraform_kubernetes/thanos.tf
@@ -46,7 +46,8 @@ resource "kubernetes_service" "thanos-store-gateway" {
     selector = {
       thanos-store-api : "true"
     }
-    type = "ClusterIP"
+    type       = "ClusterIP"
+    cluster_ip = "None"
   }
 }
 
@@ -84,7 +85,7 @@ resource "kubernetes_deployment" "thanos-querier" {
             "query",
             "--log.level=debug",
             "--query.replica-label=replica",
-            "--store=dnssrv+thanos-store-gateway:10901",
+            "--store=dns+thanos-store-gateway:10901",
           ]
 
           liveness_probe {
@@ -263,8 +264,7 @@ resource "kubernetes_deployment" "thanos-compactor" {
     template {
       metadata {
         labels = {
-          app              = "thanos-compactor"
-          thanos-store-api = true
+          app = "thanos-compactor"
         }
       }