Merge pull request #3 from truefoundry/karpenter-sqs

dunefro · web-flow · commit 76f04a4c4abf · 2023-12-05T05:07:22.000Z
Added support for interruption handling in karpenter
diff --git a/README.md b/README.md
@@ -19,13 +19,20 @@ Truefoundry AWS Karpenter Module
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_karpenter_irsa_role"></a> [karpenter\_irsa\_role](#module\_karpenter\_irsa\_role) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | 5.27.0 |
+| <a name="module_karpenter_irsa_role"></a> [karpenter\_irsa\_role](#module\_karpenter\_irsa\_role) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | 5.32.0 |
 
 ## Resources
 
 | Name | Type |
 |------|------|
+| [aws_cloudwatch_event_rule.this](https://registry.terraform.io/providers/hashicorp/aws/5.17.0/docs/resources/cloudwatch_event_rule) | resource |
+| [aws_cloudwatch_event_target.this](https://registry.terraform.io/providers/hashicorp/aws/5.17.0/docs/resources/cloudwatch_event_target) | resource |
 | [aws_iam_instance_profile.karpenter](https://registry.terraform.io/providers/hashicorp/aws/5.17.0/docs/resources/iam_instance_profile) | resource |
+| [aws_iam_policy.sqs](https://registry.terraform.io/providers/hashicorp/aws/5.17.0/docs/resources/iam_policy) | resource |
+| [aws_sqs_queue.karpenter](https://registry.terraform.io/providers/hashicorp/aws/5.17.0/docs/resources/sqs_queue) | resource |
+| [aws_sqs_queue_policy.karpenter](https://registry.terraform.io/providers/hashicorp/aws/5.17.0/docs/resources/sqs_queue_policy) | resource |
+| [aws_iam_policy_document.node_termination_queue](https://registry.terraform.io/providers/hashicorp/aws/5.17.0/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.sqs](https://registry.terraform.io/providers/hashicorp/aws/5.17.0/docs/data-sources/iam_policy_document) | data source |
 
 ## Inputs
 
@@ -36,13 +43,16 @@ Truefoundry AWS Karpenter Module
 | <a name="input_controller_nodegroup_name"></a> [controller\_nodegroup\_name](#input\_controller\_nodegroup\_name) | The initial nodegroup name | `string` | n/a | yes |
 | <a name="input_k8s_service_account_name"></a> [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | The k8s karpenter service account name | `string` | n/a | yes |
 | <a name="input_k8s_service_account_namespace"></a> [k8s\_service\_account\_namespace](#input\_k8s\_service\_account\_namespace) | The k8s karpenter namespace | `string` | n/a | yes |
+| <a name="input_message_retention_seconds"></a> [message\_retention\_seconds](#input\_message\_retention\_seconds) | Message retention in seconds for SQS queue | `number` | `300` | no |
 | <a name="input_oidc_provider_arn"></a> [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The oidc provider  arn of the eks cluster | `string` | n/a | yes |
+| <a name="input_sqs_enable_encryption"></a> [sqs\_enable\_encryption](#input\_sqs\_enable\_encryption) | Enable Server side encryption for SQS | `bool` | `true` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | AWS Tags common to all the resources created | `map(string)` | `{}` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| <a name="output_karpenter_instance_profile_id"></a> [karpenter\_instance\_profile\_id](#output\_karpenter\_instance\_profile\_id) | n/a |
-| <a name="output_karpenter_role_arn"></a> [karpenter\_role\_arn](#output\_karpenter\_role\_arn) | n/a |
+| <a name="output_karpenter_instance_profile_id"></a> [karpenter\_instance\_profile\_id](#output\_karpenter\_instance\_profile\_id) | Karpenter instance profile ID |
+| <a name="output_karpenter_role_arn"></a> [karpenter\_role\_arn](#output\_karpenter\_role\_arn) | Karpenter role ARN |
+| <a name="output_karpenter_sqs_name"></a> [karpenter\_sqs\_name](#output\_karpenter\_sqs\_name) | Name of the SQS queue for interruption handling |
 <!-- END_TF_DOCS -->
diff --git a/locals.tf b/locals.tf
@@ -7,4 +7,38 @@ locals {
     },
     var.tags
   )
-}
+  events = {
+    health_event = {
+      name        = "HealthEvent"
+      description = "Karpenter interrupt - AWS health event"
+      event_pattern = {
+        source      = ["aws.health"]
+        detail-type = ["AWS Health Event"]
+      }
+    }
+    spot_interupt = {
+      name        = "SpotInterrupt"
+      description = "Karpenter interrupt - EC2 spot instance interruption warning"
+      event_pattern = {
+        source      = ["aws.ec2"]
+        detail-type = ["EC2 Spot Instance Interruption Warning"]
+      }
+    }
+    instance_rebalance = {
+      name        = "InstanceRebalance"
+      description = "Karpenter interrupt - EC2 instance rebalance recommendation"
+      event_pattern = {
+        source      = ["aws.ec2"]
+        detail-type = ["EC2 Instance Rebalance Recommendation"]
+      }
+    }
+    instance_state_change = {
+      name        = "InstanceStateChange"
+      description = "Karpenter interrupt - EC2 instance state-change notification"
+      event_pattern = {
+        source      = ["aws.ec2"]
+        detail-type = ["EC2 Instance State-change Notification"]
+      }
+    }
+  }
+}
diff --git a/main.tf b/main.tf
@@ -1,7 +1,8 @@
 # From https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/examples/irsa/irsa.tf
+
 module "karpenter_irsa_role" {
   source                             = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
-  version                            = "5.27.0"
+  version                            = "5.32.0"
   role_name                          = "${var.cluster_name}-karpenter"
   attach_karpenter_controller_policy = true
 
@@ -10,6 +11,10 @@ module "karpenter_irsa_role" {
 
   attach_vpc_cni_policy = true
   vpc_cni_enable_ipv4   = true
+  
+  role_policy_arns = {
+    "sqs_policy" = aws_iam_policy.sqs.arn
+  }
 
   oidc_providers = {
     main = {
diff --git a/output.tf b/output.tf
@@ -1,7 +1,14 @@
 output "karpenter_role_arn" {
   value = module.karpenter_irsa_role.iam_role_arn
+  description = "Karpenter role ARN"
 }
 
 output "karpenter_instance_profile_id" {
   value = aws_iam_instance_profile.karpenter.id
+  description = "Karpenter instance profile ID"
+}
+
+output "karpenter_sqs_name" {
+  value = aws_sqs_queue.karpenter.name
+  description = "Name of the SQS queue for interruption handling"
 }
diff --git a/sqs.tf b/sqs.tf
@@ -0,0 +1,56 @@
+resource "aws_sqs_queue" "karpenter" {
+  name                      = "${var.cluster_name}-karpenter"
+  message_retention_seconds = var.message_retention_seconds
+  sqs_managed_sse_enabled = var.sqs_enable_encryption
+  tags = local.tags
+}
+
+data "aws_iam_policy_document" "sqs" {
+  statement {
+    resources = [aws_sqs_queue.karpenter.arn]
+    actions   = ["sqs:DeleteMessage", "sqs:GetQueueUrl", "sqs:GetQueueAttributes", "sqs:ReceiveMessage"]
+    effect    = "Allow"
+  }
+}
+
+resource "aws_iam_policy" "sqs" {
+  name_prefix = "${var.cluster_name}-karpenter-access-to-sqs"
+  description = "Access policy for karpenter to access SQS for ${var.cluster_name}"
+  policy      = data.aws_iam_policy_document.sqs.json
+  tags        = local.tags
+}
+
+resource "aws_sqs_queue_policy" "karpenter" {
+  policy    = data.aws_iam_policy_document.node_termination_queue.json
+  queue_url = aws_sqs_queue.karpenter.url
+}
+
+data "aws_iam_policy_document" "node_termination_queue" {
+  statement {
+    resources = [aws_sqs_queue.karpenter.arn]
+    sid       = "SQSWrite"
+    actions   = ["sqs:SendMessage"]
+    principals {
+      type        = "Service"
+      identifiers = ["events.amazonaws.com", "sqs.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "this" {
+  for_each = { for k, v in local.events : k => v }
+
+  name_prefix   = substr("${var.cluster_name}-${each.value.name}-", 0, 37)
+  description   = each.value.description
+  event_pattern = jsonencode(each.value.event_pattern)
+
+  tags = local.tags
+}
+
+resource "aws_cloudwatch_event_target" "this" {
+  for_each = { for k, v in local.events : k => v }
+
+  rule      = aws_cloudwatch_event_rule.this[each.key].name
+  target_id = "KarpenterInterruptionQueueTarget"
+  arn       = aws_sqs_queue.karpenter.arn
+}
diff --git a/variables.tf b/variables.tf
@@ -27,6 +27,17 @@ variable "controller_nodegroup_name" {
   description = "The initial nodegroup name"
   type        = string
 }
+variable "sqs_enable_encryption" {
+  description = "Enable Server side encryption for SQS"
+  type = bool
+  default = true
+}
+
+variable "message_retention_seconds" {
+  description = "Message retention in seconds for SQS queue"
+  type = number
+  default = 300
+}
 
 variable "tags" {
   type        = map(string)

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,14 @@`
`1`	`1`	`output "karpenter_role_arn" {`
`2`	`2`	`value = module.karpenter_irsa_role.iam_role_arn`
	`3`	`+ description = "Karpenter role ARN"`
`3`	`4`	`}`
`4`	`5`
`5`	`6`	`output "karpenter_instance_profile_id" {`
`6`	`7`	`value = aws_iam_instance_profile.karpenter.id`
	`8`	`+ description = "Karpenter instance profile ID"`
	`9`	`+}`
	`10`	`+`
	`11`	`+output "karpenter_sqs_name" {`
	`12`	`+ value = aws_sqs_queue.karpenter.name`
	`13`	`+ description = "Name of the SQS queue for interruption handling"`
`7`	`14`	`}`