SNAT port exhaustion alerting

DFE-Digital · Jan 9, 2025 · d52ad9a · d52ad9a
1 parent 8b43922
commit d52ad9a
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 0 deletions.
diff --git a/Makefile b/Makefile
@@ -44,6 +44,7 @@ terraform-aks-cluster-init: set-azure-account
 
 	$(eval export TF_VAR_environment=${ENVIRONMENT})
 	$(eval export TF_VAR_resource_group_name=${RESOURCE_GROUP_NAME})
+	$(eval export TF_VAR_keyvault_name=${KEYVAULT_NAME})
 	$(eval export TF_VAR_resource_prefix=${RESOURCE_PREFIX})
 	$(eval export TF_VAR_config=${CONFIG})
 	$(eval export TF_VAR_azure_tags=${RG_TAGS})

diff --git a/cluster/terraform_aks_cluster/analytics.tf b/cluster/terraform_aks_cluster/analytics.tf
@@ -84,3 +84,83 @@ resource "azurerm_monitor_data_collection_rule_association" "aks" {
   target_resource_id      = azurerm_kubernetes_cluster.main.id
   data_collection_rule_id = azurerm_monitor_data_collection_rule.aks.id
 }
+
+resource "azurerm_monitor_action_group" "slack" {
+  name                = "${var.resource_prefix}-tsc-${var.environment}-slack"
+  resource_group_name = var.resource_group_name
+  short_name          = substr("tsc-${var.environment}", 0, 12)
+
+  webhook_receiver {
+    name        = "slack"
+    service_uri = data.azurerm_key_vault_secret.slack_webhook.value
+  }
+
+  tags = {
+    "Environment"      = "Test"
+    "Product"          = "Teacher services cloud"
+    "Service Offering" = ""
+  }
+}
+
+resource "azurerm_monitor_metric_alert" "port_exhaustion" {
+  name                = "${var.resource_prefix}-tsc-${var.environment}-port-exhaustion"
+  resource_group_name = "${var.resource_prefix}-tsc-aks-nodes-${var.environment}-rg"
+  scopes              = ["/subscriptions/${data.azurerm_subscription.current.subscription_id}/resourceGroups/${var.resource_prefix}-tsc-aks-nodes-${var.environment}-rg/providers/Microsoft.Network/loadBalancers/kubernetes"]
+  severity            = 1
+  criteria {
+    metric_namespace = "microsoft.network/loadbalancers"
+    metric_name      = "SnatConnectionCount"
+    aggregation      = "Total"
+    operator         = "GreaterThan"
+    threshold        = 0
+    dimension {
+      name     = "ConnectionState"
+      operator = "Include"
+      values   = ["failed"]
+    }
+  }
+
+  action {
+    action_group_id = azurerm_monitor_action_group.slack.id
+  }
+
+  tags = {
+    "Environment"      = "Test"
+    "Product"          = "Teacher services cloud"
+    "Service Offering" = ""
+  }
+}
+
+resource "azurerm_monitor_metric_alert" "high_port_usage" {
+  name                = "${var.resource_prefix}-tsc-${var.environment}-high-port-usage"
+  resource_group_name = "${var.resource_prefix}-tsc-aks-nodes-${var.environment}-rg"
+  scopes              = ["/subscriptions/${data.azurerm_subscription.current.subscription_id}/resourceGroups/${var.resource_prefix}-tsc-aks-nodes-${var.environment}-rg/providers/Microsoft.Network/loadBalancers/kubernetes"]
+  severity            = 2
+  criteria {
+    metric_namespace = "microsoft.network/loadbalancers"
+    metric_name      = "UsedSnatPorts"
+    aggregation      = "Average"
+    operator         = "GreaterThan"
+    threshold        = 900
+    dimension {
+      name     = "BackendIPAddress"
+      operator = "Include"
+      values   = ["*"]
+    }
+    dimension {
+      name     = "ProtocolType"
+      operator = "Include"
+      values   = ["TCP"]
+    }
+  }
+
+  action {
+    action_group_id = azurerm_monitor_action_group.slack.id
+  }
+
+  tags = {
+    "Environment"      = "Test"
+    "Product"          = "Teacher services cloud"
+    "Service Offering" = ""
+  }
+}
diff --git a/cluster/terraform_aks_cluster/data.tf b/cluster/terraform_aks_cluster/data.tf
@@ -0,0 +1,11 @@
+data "azurerm_key_vault" "key_vault" {
+  name                = var.keyvault_name
+  resource_group_name = var.resource_group_name
+}
+
+data "azurerm_key_vault_secret" "slack_webhook" {
+  name         = "SLACK-WEBHOOK-GENERIC" # or SLACK_SECRET ?
+  key_vault_id = data.azurerm_key_vault.key_vault.id
+}
+
+data "azurerm_subscription" "current" {}
diff --git a/cluster/terraform_aks_cluster/variables.tf b/cluster/terraform_aks_cluster/variables.tf
@@ -1,6 +1,7 @@
 # Set in config shell variables and used by Makefile
 variable "environment" { type = string }
 variable "resource_group_name" { type = string }
+variable "keyvault_name" { type = string }
 variable "resource_prefix" { type = string }
 variable "azure_tags" { type = string }
 variable "config" { type = string }