Merge branch '279-pod-pvc' into 264-execution-of-additional-sbg-workf…

…lows

.github/workflows/smoke_tests.yml

@@ -10,9 +10,9 @@ on:
       MCP_VENUE_DEV_AIRFLOW_ENDPOINT:
         description: "Base URL for the Airflow endpoint in MCP Venue Dev (i.e. http://abc.def.ghi:port-number)"
         type: string
-      # MCP_VENUE_TEST_AIRFLOW_ENDPOINT:
-      #   description: "Base URL for the Airflow endpoint in MCP Venue Test (i.e. http://abc.def.ghi:port-number)"
-      #   type: string
+      MCP_VENUE_SBG_DEV_AIRFLOW_ENDPOINT:
+        description: "Base URL for the Airflow endpoint in MCP Venue SBG Dev (i.e. http://abc.def.ghi:port-number)"
+        type: string
 jobs:
   smoke-tests:
     runs-on: ubuntu-latest
@@ -37,11 +37,11 @@ jobs:
         pytest -vv --gherkin-terminal-reporter \
         unity-test/system/smoke \
         --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_DEV_AIRFLOW_ENDPOINT || vars.MCP_VENUE_DEV_AIRFLOW_ENDPOINT }}
-    # - name: MCP Venue Test - Smoke tests
-    #   env:
-    #     AIRFLOW_WEBSERVER_PASSWORD: ${{ secrets.MCP_VENUE_TEST_AIRFLOW_WEBSERVER_PASSWORD }}
-    #   continue-on-error: true
-    #   run: |
-    #     pytest -vv --gherkin-terminal-reporter \
-    #     unity-test/system/smoke \
-    #     --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_TEST_AIRFLOW_ENDPOINT || vars.MCP_VENUE_TEST_AIRFLOW_ENDPOINT }}
+    - name: MCP Venue SBG Dev - Smoke tests
+      env:
+        AIRFLOW_WEBSERVER_PASSWORD: ${{ secrets.MCP_VENUE_SBG_DEV_AIRFLOW_WEBSERVER_PASSWORD }}
+      continue-on-error: true
+      run: |
+        pytest -vv --gherkin-terminal-reporter \
+        unity-test/system/smoke \
+        --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_SBG_DEV_AIRFLOW_ENDPOINT || vars.MCP_VENUE_SBG_DEV_AIRFLOW_ENDPOINT }}

airflow/helm/values.tmpl.yaml

@@ -131,3 +131,7 @@ extraEnv: |
     value: "1"
   - name: AIRFLOW__KUBERNETES__WORKER_PODS_CREATION_BATCH_SIZE
     value: "8"
+  - name: AIRFLOW__WEBSERVER__NAVBAR_COLOR
+    value: "${webserver_navbar_color}"
+  - name: AIRFLOW__WEBSERVER__INSTANCE_NAME
+    value: "${webserver_instance_name}"

terraform-unity/README.md

@@ -175,20 +175,20 @@ No resources.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_airflow_webserver_password"></a> [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | value | `string` | n/a | yes |
-| <a name="input_counter"></a> [counter](#input\_counter) | value | `string` | `""` | no |
-| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    }),<br>    ogc_processes_api = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | <pre>{<br>  "airflow": {<br>    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br>    "tag": "develop"<br>  },<br>  "ogc_processes_api": {<br>    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",<br>    "tag": "develop"<br>  }<br>}</pre> | no |
-| <a name="input_eks_cluster_name"></a> [eks\_cluster\_name](#input\_eks\_cluster\_name) | The name of the EKS cluster. | `string` | n/a | yes |
-| <a name="input_helm_charts"></a> [helm\_charts](#input\_helm\_charts) | Settings for the required Helm charts. | <pre>map(object({<br>    repository = string<br>    chart      = string<br>    version    = string<br>  }))</pre> | <pre>{<br>  "airflow": {<br>    "chart": "airflow",<br>    "repository": "https://airflow.apache.org",<br>    "version": "1.11.0"<br>  },<br>  "keda": {<br>    "chart": "keda",<br>    "repository": "https://kedacore.github.io/charts",<br>    "version": "v2.13.1"<br>  }<br>}</pre> | no |
-| <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | Path to the kubeconfig file for the Kubernetes cluster | `string` | `"../k8s/kubernetes.yml"` | no |
-| <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no |
-| <a name="input_release"></a> [release](#input\_release) | The SPS release version | `string` | n/a | yes |
-| <a name="input_service_area"></a> [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no |
-| <a name="input_venue"></a> [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | `null` | no |
+| <a name="input_airflow_webserver_password"></a> [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes |
+| <a name="input_counter"></a> [counter](#input\_counter) | Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead. | `string` | `""` | no |
+| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes |
+| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the associated services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    }),<br>    ogc_processes_api = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | <pre>{<br>  "airflow": {<br>    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br>    "tag": "develop"<br>  },<br>  "ogc_processes_api": {<br>    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",<br>    "tag": "develop"<br>  }<br>}</pre> | no |
+| <a name="input_helm_charts"></a> [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | <pre>map(object({<br>    repository = string<br>    chart      = string<br>    version    = string<br>  }))</pre> | <pre>{<br>  "airflow": {<br>    "chart": "airflow",<br>    "repository": "https://airflow.apache.org",<br>    "version": "1.11.0"<br>  },<br>  "keda": {<br>    "chart": "keda",<br>    "repository": "https://kedacore.github.io/charts",<br>    "version": "v2.13.1"<br>  }<br>}</pre> | no |
+| <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes |
+| <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS. | `string` | `"unity"` | no |
+| <a name="input_release"></a> [release](#input\_release) | The software release version. | `string` | n/a | yes |
+| <a name="input_service_area"></a> [service\_area](#input\_service\_area) | The service area owner of the resources being deployed. | `string` | `"sps"` | no |
+| <a name="input_venue"></a> [venue](#input\_venue) | The MCP venue in which the resources will be deployed. | `string` | n/a | yes |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| <a name="output_load_balancer_hostnames"></a> [load\_balancer\_hostnames](#output\_load\_balancer\_hostnames) | Load Balancer Ingress Hostnames |
+| <a name="output_resources"></a> [resources](#output\_resources) | SSM parameter IDs for pipeline resources. |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

terraform-unity/main.tf

@@ -3,9 +3,9 @@ module "unity-sps-airflow" {
   project                    = var.project
   venue                      = var.venue
   service_area               = var.service_area
+  deployment_name            = var.deployment_name
   counter                    = var.counter
   release                    = var.release
-  eks_cluster_name           = var.eks_cluster_name
   kubeconfig_filepath        = var.kubeconfig_filepath
   airflow_webserver_password = var.airflow_webserver_password
   docker_images              = var.docker_images

terraform-unity/modules/terraform-eks-cluster/README.md

@@ -8,12 +8,14 @@
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.7.2 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | 5.35.0 |
 | <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | 2.25.2 |
+| <a name="requirement_random"></a> [random](#requirement\_random) | 3.6.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
 | <a name="provider_aws"></a> [aws](#provider\_aws) | 5.35.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | 3.6.0 |
 
 ## Modules
 
@@ -26,6 +28,7 @@
 | Name | Type |
 |------|------|
 | [aws_iam_role_policy.sps_airflow_eks_inline_policy](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/iam_role_policy) | resource |
+| [random_id.counter](https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs/resources/id) | resource |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/caller_identity) | data source |
 | [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/eks_cluster) | data source |
 | [aws_eks_cluster_auth.auth](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/eks_cluster_auth) | data source |
@@ -34,8 +37,13 @@
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | n/a | `string` | n/a | yes |
+| <a name="input_counter"></a> [counter](#input\_counter) | Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead. | `string` | n/a | yes |
+| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes |
 | <a name="input_nodegroups"></a> [nodegroups](#input\_nodegroups) | A map of node group configurations | <pre>map(object({<br>    create_iam_role            = optional(bool)<br>    iam_role_arn               = optional(string)<br>    ami_id                     = optional(string)<br>    min_size                   = optional(number)<br>    max_size                   = optional(number)<br>    desired_size               = optional(number)<br>    instance_types             = optional(list(string))<br>    capacity_type              = optional(string)<br>    enable_bootstrap_user_data = optional(bool)<br>    metadata_options           = optional(map(any))<br>  }))</pre> | <pre>{<br>  "defaultGroup": {<br>    "desired_size": 1,<br>    "instance_types": [<br>      "m5.xlarge"<br>    ],<br>    "max_size": 1,<br>    "metadata_options": {<br>      "http_endpoint": "enabled",<br>      "http_put_response_hop_limit": 3<br>    },<br>    "min_size": 1<br>  }<br>}</pre> | no |
+| <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no |
+| <a name="input_release"></a> [release](#input\_release) | The software release version. | `string` | n/a | yes |
+| <a name="input_service_area"></a> [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no |
+| <a name="input_venue"></a> [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes |
 
 ## Outputs
 

terraform-unity/modules/terraform-eks-cluster/data.tf

@@ -1,12 +1,12 @@
 data "aws_caller_identity" "current" {}
 
 data "aws_eks_cluster" "cluster" {
-  name       = var.cluster_name
+  name       = local.cluster_name
   depends_on = [module.unity-eks]
 }
 
 data "aws_eks_cluster_auth" "auth" {
-  name       = var.cluster_name
+  name       = local.cluster_name
   depends_on = [module.unity-eks]
 }
 

terraform-unity/modules/terraform-eks-cluster/locals.tf

@@ -0,0 +1,17 @@
+locals {
+  counter              = var.counter != "" ? var.counter : random_id.counter.hex
+  resource_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s", var.deployment_name, local.counter]))
+  cluster_name         = format(local.resource_name_prefix, "eks")
+  common_tags = {
+    Name        = ""
+    Venue       = var.venue
+    Proj        = var.project
+    ServiceArea = var.service_area
+    CapVersion  = var.release
+    Component   = ""
+    CreatedBy   = var.service_area
+    Env         = var.venue
+    mission     = var.project
+    Stack       = ""
+  }
+}

terraform-unity/modules/terraform-eks-cluster/main.tf

@@ -1,6 +1,10 @@
+resource "random_id" "counter" {
+  byte_length = 2
+}
+
 module "unity-eks" {
   source          = "git@github.com:unity-sds/unity-cs-infra.git//terraform-unity-eks_module?ref=main"
-  deployment_name = var.cluster_name
+  deployment_name = local.cluster_name
 
   nodegroups = var.nodegroups
 

terraform-unity/modules/terraform-eks-cluster/variables.tf

@@ -1,10 +1,37 @@
-variable "cluster_name" {
-  type = string
+variable "project" {
+  description = "The project or mission deploying Unity SPS"
+  type        = string
+  default     = "unity"
+}
+
+variable "venue" {
+  description = "The MCP venue in which the cluster will be deployed (dev, test, prod)"
+  type        = string
+}
+
+variable "service_area" {
+  description = "The service area owner of the resources being deployed"
+  type        = string
+  default     = "sps"
+}
+
+variable "deployment_name" {
+  description = "The name of the deployment."
+  type        = string
+}
+
+variable "counter" {
+  description = "Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead."
+  type        = string
+}
+
+variable "release" {
+  description = "The software release version."
+  type        = string
 }
 
 variable "nodegroups" {
   description = "A map of node group configurations"
-
   type = map(object({
     create_iam_role            = optional(bool)
     iam_role_arn               = optional(string)
@@ -17,7 +44,6 @@ variable "nodegroups" {
     enable_bootstrap_user_data = optional(bool)
     metadata_options           = optional(map(any))
   }))
-
   default = {
     defaultGroup = {
       instance_types = ["m5.xlarge"]

terraform-unity/modules/terraform-eks-cluster/versions.tf

@@ -9,5 +9,9 @@ terraform {
       source  = "hashicorp/aws"
       version = "5.35.0"
     }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.6.0"
+    }
   }
 }