Thanos fixes

cluster/terraform_kubernetes/config/prometheus/development.prometheus.yml

@@ -13,8 +13,6 @@ global:
   keep_dropped_targets: 100
   external_labels:
     cluster: prometheus
-    # Each Prometheus has to have unique labels.
-    replica: $(POD_NAME)
 rule_files:
   - /etc/prometheus/prometheus.rules
 alerting:

cluster/terraform_kubernetes/config/prometheus/platform-test.prometheus.yml

@@ -13,8 +13,6 @@ global:
   keep_dropped_targets: 100
   external_labels:
     cluster: prometheus
-    # Each Prometheus has to have unique labels.
-    replica: $(POD_NAME)
 rule_files:
   - /etc/prometheus/prometheus.rules
 alerting:

cluster/terraform_kubernetes/config/prometheus/production.prometheus.yml

@@ -13,8 +13,6 @@ global:
   keep_dropped_targets: 100
   external_labels:
     cluster: prometheus
-    # Each Prometheus has to have unique labels.
-    replica: $(POD_NAME)
 rule_files:
   - /etc/prometheus/prometheus.rules
 alerting:

cluster/terraform_kubernetes/config/prometheus/test.prometheus.yml

@@ -13,8 +13,6 @@ global:
   keep_dropped_targets: 100
   external_labels:
     cluster: prometheus
-    # Each Prometheus has to have unique labels.
-    replica: $(POD_NAME)
 rule_files:
   - /etc/prometheus/prometheus.rules
 alerting:

cluster/terraform_kubernetes/prometheus.tf

@@ -46,7 +46,7 @@ resource "kubernetes_config_map" "prometheus" {
 
   metadata {
     name      = "prometheus-server-conf"
-    namespace = "monitoring"
+    namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name
   }
 
   data = {
@@ -149,15 +149,6 @@ resource "kubernetes_deployment" "prometheus" {
             # "--reloader.rule-dir=/etc/prometheus/rules/",
           ]
 
-          env {
-            name = "POD_NAME"
-            value_from {
-              field_ref {
-                field_path = "metadata.name"
-              }
-            }
-          }
-
           liveness_probe {
             http_get {
               path = "/-/healthy"
@@ -235,13 +226,12 @@ resource "kubernetes_service" "prometheus" {
 
   spec {
     port {
-      node_port   = 30000
+      name        = "prometheus"
       port        = 8080
       target_port = kubernetes_deployment.prometheus.spec[0].template[0].spec[0].container[0].port[0].container_port
     }
     selector = {
       app = "prometheus"
     }
-    type = "NodePort"
   }
 }

cluster/terraform_kubernetes/thanos.tf

@@ -1,6 +1,6 @@
 resource "azurerm_storage_account" "thanos" {
 
-  name                            = "${var.resource_prefix}${var.cluster_short}thanossa"
+  name                            = "${var.resource_prefix}${local.cluster_sa_name}thanossa"
   location                        = data.azurerm_resource_group.resource_group.location
   resource_group_name             = data.azurerm_resource_group.resource_group.name
   account_tier                    = "Standard"
@@ -34,7 +34,7 @@ resource "kubernetes_service" "thanos-store-gateway" {
 
   metadata {
     name      = "thanos-store-gateway"
-    namespace = "monitoring"
+    namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name
   }
 
   spec {
@@ -46,15 +46,16 @@ resource "kubernetes_service" "thanos-store-gateway" {
     selector = {
       thanos-store-api : "true"
     }
-    type = "ClusterIP"
+    type       = "ClusterIP"
+    cluster_ip = "None"
   }
 }
 
 resource "kubernetes_deployment" "thanos-querier" {
 
   metadata {
     name      = "thanos-querier"
-    namespace = "monitoring"
+    namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name
   }
 
   spec {
@@ -84,7 +85,7 @@ resource "kubernetes_deployment" "thanos-querier" {
             "query",
             "--log.level=debug",
             "--query.replica-label=replica",
-            "--store=dnssrv+thanos-store-gateway:10901",
+            "--store=dns+thanos-store-gateway:10901",
           ]
 
           liveness_probe {
@@ -132,7 +133,7 @@ resource "kubernetes_service" "thanos-querier" {
 
   metadata {
     name      = "thanos-querier"
-    namespace = "monitoring"
+    namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name
     labels = {
       app = "thanos-querier"
     }
@@ -155,7 +156,7 @@ resource "kubernetes_deployment" "thanos-store-gateway" {
 
   metadata {
     name      = "thanos-store-gateway"
-    namespace = "monitoring"
+    namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name
   }
 
   spec {
@@ -248,7 +249,7 @@ resource "kubernetes_deployment" "thanos-compactor" {
 
   metadata {
     name      = "thanos-compactor"
-    namespace = "monitoring"
+    namespace = kubernetes_namespace.default_list["monitoring"].metadata[0].name
   }
 
   spec {
@@ -263,8 +264,7 @@ resource "kubernetes_deployment" "thanos-compactor" {
     template {
       metadata {
         labels = {
-          app              = "thanos-compactor"
-          thanos-store-api = true
+          app = "thanos-compactor"
         }
       }
 

cluster/terraform_kubernetes/variables.tf

@@ -176,4 +176,9 @@ locals {
     storage-account-name = azurerm_storage_account.thanos.name
     storage-account-key  = azurerm_storage_account.thanos.primary_access_key
   }
+
+  cluster_sa_name = (var.environment == var.config ?
+    var.cluster_short : # pt,ts or pd
+    var.environment     # cluster1, cluster2, etc
+  )
 }

cluster/terraform_kubernetes/welcome_app.tf

@@ -3,7 +3,7 @@ resource "kubernetes_deployment" "welcome_app" {
 
   metadata {
     name      = local.welcome_app_name
-    namespace = local.welcome_app_namespace
+    namespace = kubernetes_namespace.default_list["${local.welcome_app_namespace}"].metadata[0].name
   }
   spec {
     replicas = 2
@@ -73,7 +73,7 @@ resource "kubernetes_service" "welcome_app" {
 
   metadata {
     name      = local.welcome_app_name
-    namespace = local.welcome_app_namespace
+    namespace = kubernetes_namespace.default_list["${local.welcome_app_namespace}"].metadata[0].name
   }
   spec {
     type = "ClusterIP"