From 0ef61540d6a5dabc0826c2269068ec6dca2b5817 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Thu, 7 Mar 2024 16:04:38 -0800 Subject: [PATCH 1/3] Update smoke tests github action --- .github/workflows/smoke_tests.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/smoke_tests.yml b/.github/workflows/smoke_tests.yml index 9f6b4f42..5c024d96 100644 --- a/.github/workflows/smoke_tests.yml +++ b/.github/workflows/smoke_tests.yml @@ -10,9 +10,9 @@ on: MCP_VENUE_DEV_AIRFLOW_ENDPOINT: description: "Base URL for the Airflow endpoint in MCP Venue Dev (i.e. http://abc.def.ghi:port-number)" type: string - # MCP_VENUE_TEST_AIRFLOW_ENDPOINT: - # description: "Base URL for the Airflow endpoint in MCP Venue Test (i.e. http://abc.def.ghi:port-number)" - # type: string + MCP_VENUE_SBG_DEV_AIRFLOW_ENDPOINT: + description: "Base URL for the Airflow endpoint in MCP Venue SBG Dev (i.e. http://abc.def.ghi:port-number)" + type: string jobs: smoke-tests: runs-on: ubuntu-latest @@ -37,11 +37,11 @@ jobs: pytest -vv --gherkin-terminal-reporter \ unity-test/system/smoke \ --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_DEV_AIRFLOW_ENDPOINT || vars.MCP_VENUE_DEV_AIRFLOW_ENDPOINT }} - # - name: MCP Venue Test - Smoke tests - # env: - # AIRFLOW_WEBSERVER_PASSWORD: ${{ secrets.MCP_VENUE_TEST_AIRFLOW_WEBSERVER_PASSWORD }} - # continue-on-error: true - # run: | - # pytest -vv --gherkin-terminal-reporter \ - # unity-test/system/smoke \ - # --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_TEST_AIRFLOW_ENDPOINT || vars.MCP_VENUE_TEST_AIRFLOW_ENDPOINT }} + - name: MCP Venue SBG Dev - Smoke tests + env: + AIRFLOW_WEBSERVER_PASSWORD: ${{ secrets.MCP_VENUE_SBG_DEV_AIRFLOW_WEBSERVER_PASSWORD }} + continue-on-error: true + run: | + pytest -vv --gherkin-terminal-reporter \ + unity-test/system/smoke \ + --airflow-endpoint=${{ github.event.inputs.MCP_VENUE_SBG_DEV_AIRFLOW_ENDPOINT || vars.MCP_VENUE_SBG_DEV_AIRFLOW_ENDPOINT }} From b3221cd2563c1e093e0e7b85fd86fedc73e848c1 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Thu, 7 Mar 2024 16:21:10 -0800 Subject: [PATCH 2/3] Parameterize resource names --- .pre-commit-config.yaml | 2 +- airflow/helm/values.tmpl.yaml | 4 + terraform-unity/README.md | 22 ++--- terraform-unity/main.tf | 2 +- .../terraform-eks-cluster/.terraform.lock.hcl | 20 +++++ .../modules/terraform-eks-cluster/README.md | 10 ++- .../modules/terraform-eks-cluster/data.tf | 4 +- .../modules/terraform-eks-cluster/locals.tf | 17 ++++ .../modules/terraform-eks-cluster/main.tf | 6 +- .../terraform-eks-cluster/variables.tf | 34 +++++++- .../modules/terraform-eks-cluster/versions.tf | 4 + .../terraform-unity-sps-airflow/README.md | 23 +++-- .../terraform-unity-sps-airflow/data.tf | 4 +- .../terraform-unity-sps-airflow/locals.tf | 9 +- .../terraform-unity-sps-airflow/main.tf | 86 +++++++++++++++---- .../terraform-unity-sps-airflow/outputs.tf | 32 +++++-- .../terraform-unity-sps-airflow/variables.tf | 20 ++--- terraform-unity/outputs.tf | 11 ++- terraform-unity/variables.tf | 34 ++++---- 19 files changed, 258 insertions(+), 86 deletions(-) create mode 100644 terraform-unity/modules/terraform-eks-cluster/locals.tf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 109507c4..8c0ac886 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,7 +71,7 @@ repos: args: - --tf-init-args=-upgrade - id: terraform_fmt # Rewrites all Terraform configuration files to a canonical format. - - id: terraform_tflint # Validates all Terraform configuration files with TFLint. + # - id: terraform_tflint # Validates all Terraform configuration files with TFLint. - id: terraform_trivy # Static analysis of Terraform templates to spot potential security issues. args: - > diff --git a/airflow/helm/values.tmpl.yaml b/airflow/helm/values.tmpl.yaml index d17831e8..f8366be8 100644 --- a/airflow/helm/values.tmpl.yaml +++ b/airflow/helm/values.tmpl.yaml @@ -131,3 +131,7 @@ extraEnv: | value: "1" - name: AIRFLOW__KUBERNETES__WORKER_PODS_CREATION_BATCH_SIZE value: "8" + - name: AIRFLOW__WEBSERVER__NAVBAR_COLOR + value: "${webserver_navbar_color}" + - name: AIRFLOW__WEBSERVER__INSTANCE_NAME + value: "${webserver_instance_name}" diff --git a/terraform-unity/README.md b/terraform-unity/README.md index 9a067bab..21ab7bde 100644 --- a/terraform-unity/README.md +++ b/terraform-unity/README.md @@ -175,20 +175,20 @@ No resources. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | value | `string` | n/a | yes | -| [counter](#input\_counter) | value | `string` | `""` | no | -| [docker\_images](#input\_docker\_images) | Docker images for the services. | 
object({
    airflow = object({
      name = string
      tag  = string
    }),
    ogc_processes_api = object({
      name = string
      tag  = string
    })
  })
| 
{
  "airflow": {
    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",
    "tag": "develop"
  },
  "ogc_processes_api": {
    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",
    "tag": "develop"
  }
}
| no | -| [eks\_cluster\_name](#input\_eks\_cluster\_name) | The name of the EKS cluster. | `string` | n/a | yes | -| [helm\_charts](#input\_helm\_charts) | Settings for the required Helm charts. | 
map(object({
    repository = string
    chart      = string
    version    = string
  }))
| 
{
  "airflow": {
    "chart": "airflow",
    "repository": "https://airflow.apache.org",
    "version": "1.11.0"
  },
  "keda": {
    "chart": "keda",
    "repository": "https://kedacore.github.io/charts",
    "version": "v2.13.1"
  }
}
| no | -| [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | Path to the kubeconfig file for the Kubernetes cluster | `string` | `"../k8s/kubernetes.yml"` | no | -| [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no | -| [release](#input\_release) | The SPS release version | `string` | n/a | yes | -| [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no | -| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | `null` | no | +| [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes | +| [counter](#input\_counter) | Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead. | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes | +| [docker\_images](#input\_docker\_images) | Docker images for the associated services. | 
object({
    airflow = object({
      name = string
      tag  = string
    }),
    ogc_processes_api = object({
      name = string
      tag  = string
    })
  })
| 
{
  "airflow": {
    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",
    "tag": "develop"
  },
  "ogc_processes_api": {
    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",
    "tag": "develop"
  }
}
| no | +| [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | 
map(object({
    repository = string
    chart      = string
    version    = string
  }))
| 
{
  "airflow": {
    "chart": "airflow",
    "repository": "https://airflow.apache.org",
    "version": "1.11.0"
  },
  "keda": {
    "chart": "keda",
    "repository": "https://kedacore.github.io/charts",
    "version": "v2.13.1"
  }
}
| no | +| [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes | +| [project](#input\_project) | The project or mission deploying Unity SPS. | `string` | `"unity"` | no | +| [release](#input\_release) | The software release version. | `string` | n/a | yes | +| [service\_area](#input\_service\_area) | The service area owner of the resources being deployed. | `string` | `"sps"` | no | +| [venue](#input\_venue) | The MCP venue in which the resources will be deployed. | `string` | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| [load\_balancer\_hostnames](#output\_load\_balancer\_hostnames) | Load Balancer Ingress Hostnames | +| [resources](#output\_resources) | SSM parameter IDs for pipeline resources. | diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf index 49a5d045..23079f86 100644 --- a/terraform-unity/main.tf +++ b/terraform-unity/main.tf @@ -3,9 +3,9 @@ module "unity-sps-airflow" { project = var.project venue = var.venue service_area = var.service_area + deployment_name = var.deployment_name counter = var.counter release = var.release - eks_cluster_name = var.eks_cluster_name kubeconfig_filepath = var.kubeconfig_filepath airflow_webserver_password = var.airflow_webserver_password docker_images = var.docker_images diff --git a/terraform-unity/modules/terraform-eks-cluster/.terraform.lock.hcl b/terraform-unity/modules/terraform-eks-cluster/.terraform.lock.hcl index da3772cd..7d0331be 100644 --- a/terraform-unity/modules/terraform-eks-cluster/.terraform.lock.hcl +++ b/terraform-unity/modules/terraform-eks-cluster/.terraform.lock.hcl @@ -83,6 +83,26 @@ provider "registry.terraform.io/hashicorp/kubernetes" { ] } +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.0" + constraints = "3.6.0" + hashes = [ + "h1:I8MBeauYA8J8yheLJ8oSMWqB0kovn16dF/wKZ1QTdkk=", + "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", + "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", + "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", + "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", + "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", + "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", + "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", + "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", + "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", + "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", + ] +} + provider "registry.terraform.io/hashicorp/time" { version = "0.10.0" constraints = ">= 0.9.0" diff --git a/terraform-unity/modules/terraform-eks-cluster/README.md b/terraform-unity/modules/terraform-eks-cluster/README.md index 92728946..295f6262 100644 --- a/terraform-unity/modules/terraform-eks-cluster/README.md +++ b/terraform-unity/modules/terraform-eks-cluster/README.md @@ -8,12 +8,14 @@ | [terraform](#requirement\_terraform) | ~> 1.7.2 | | [aws](#requirement\_aws) | 5.35.0 | | [kubernetes](#requirement\_kubernetes) | 2.25.2 | +| [random](#requirement\_random) | 3.6.0 | ## Providers | Name | Version | |------|---------| | [aws](#provider\_aws) | 5.35.0 | +| [random](#provider\_random) | 3.6.0 | ## Modules @@ -26,6 +28,7 @@ | Name | Type | |------|------| | [aws_iam_role_policy.sps_airflow_eks_inline_policy](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/iam_role_policy) | resource | +| [random_id.counter](https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs/resources/id) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/eks_cluster) | data source | | [aws_eks_cluster_auth.auth](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/data-sources/eks_cluster_auth) | data source | @@ -34,8 +37,13 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [cluster\_name](#input\_cluster\_name) | n/a | `string` | n/a | yes | +| [counter](#input\_counter) | Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead. | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes | | [nodegroups](#input\_nodegroups) | A map of node group configurations | 
map(object({
    create_iam_role            = optional(bool)
    iam_role_arn               = optional(string)
    ami_id                     = optional(string)
    min_size                   = optional(number)
    max_size                   = optional(number)
    desired_size               = optional(number)
    instance_types             = optional(list(string))
    capacity_type              = optional(string)
    enable_bootstrap_user_data = optional(bool)
    metadata_options           = optional(map(any))
  }))
| 
{
  "defaultGroup": {
    "desired_size": 1,
    "instance_types": [
      "m5.xlarge"
    ],
    "max_size": 1,
    "metadata_options": {
      "http_endpoint": "enabled",
      "http_put_response_hop_limit": 3
    },
    "min_size": 1
  }
}
| no | +| [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no | +| [release](#input\_release) | The software release version. | `string` | n/a | yes | +| [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no | +| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | ## Outputs diff --git a/terraform-unity/modules/terraform-eks-cluster/data.tf b/terraform-unity/modules/terraform-eks-cluster/data.tf index ddfcf84a..49e5bba8 100644 --- a/terraform-unity/modules/terraform-eks-cluster/data.tf +++ b/terraform-unity/modules/terraform-eks-cluster/data.tf @@ -1,12 +1,12 @@ data "aws_caller_identity" "current" {} data "aws_eks_cluster" "cluster" { - name = var.cluster_name + name = local.cluster_name depends_on = [module.unity-eks] } data "aws_eks_cluster_auth" "auth" { - name = var.cluster_name + name = local.cluster_name depends_on = [module.unity-eks] } diff --git a/terraform-unity/modules/terraform-eks-cluster/locals.tf b/terraform-unity/modules/terraform-eks-cluster/locals.tf new file mode 100644 index 00000000..2b2cae34 --- /dev/null +++ b/terraform-unity/modules/terraform-eks-cluster/locals.tf @@ -0,0 +1,17 @@ +locals { + counter = var.counter != "" ? var.counter : random_id.counter.hex + resource_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s", var.deployment_name, local.counter])) + cluster_name = format(local.resource_name_prefix, "eks") + common_tags = { + Name = "" + Venue = var.venue + Proj = var.project + ServiceArea = var.service_area + CapVersion = var.release + Component = "" + CreatedBy = var.service_area + Env = var.venue + mission = var.project + Stack = "" + } +} diff --git a/terraform-unity/modules/terraform-eks-cluster/main.tf b/terraform-unity/modules/terraform-eks-cluster/main.tf index 09df65c6..aea4d140 100644 --- a/terraform-unity/modules/terraform-eks-cluster/main.tf +++ b/terraform-unity/modules/terraform-eks-cluster/main.tf @@ -1,6 +1,10 @@ +resource "random_id" "counter" { + byte_length = 2 +} + module "unity-eks" { source = "git@github.com:unity-sds/unity-cs-infra.git//terraform-unity-eks_module?ref=main" - deployment_name = var.cluster_name + deployment_name = local.cluster_name nodegroups = var.nodegroups diff --git a/terraform-unity/modules/terraform-eks-cluster/variables.tf b/terraform-unity/modules/terraform-eks-cluster/variables.tf index a47859f6..330bb477 100644 --- a/terraform-unity/modules/terraform-eks-cluster/variables.tf +++ b/terraform-unity/modules/terraform-eks-cluster/variables.tf @@ -1,10 +1,37 @@ -variable "cluster_name" { - type = string +variable "project" { + description = "The project or mission deploying Unity SPS" + type = string + default = "unity" +} + +variable "venue" { + description = "The MCP venue in which the cluster will be deployed (dev, test, prod)" + type = string +} + +variable "service_area" { + description = "The service area owner of the resources being deployed" + type = string + default = "sps" +} + +variable "deployment_name" { + description = "The name of the deployment." + type = string +} + +variable "counter" { + description = "Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead." + type = string +} + +variable "release" { + description = "The software release version." + type = string } variable "nodegroups" { description = "A map of node group configurations" - type = map(object({ create_iam_role = optional(bool) iam_role_arn = optional(string) @@ -17,7 +44,6 @@ variable "nodegroups" { enable_bootstrap_user_data = optional(bool) metadata_options = optional(map(any)) })) - default = { defaultGroup = { instance_types = ["m5.xlarge"] diff --git a/terraform-unity/modules/terraform-eks-cluster/versions.tf b/terraform-unity/modules/terraform-eks-cluster/versions.tf index 55298edb..8596e9b4 100644 --- a/terraform-unity/modules/terraform-eks-cluster/versions.tf +++ b/terraform-unity/modules/terraform-eks-cluster/versions.tf @@ -9,5 +9,9 @@ terraform { source = "hashicorp/aws" version = "5.35.0" } + random = { + source = "hashicorp/random" + version = "3.6.0" + } } } diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/README.md b/terraform-unity/modules/terraform-unity-sps-airflow/README.md index 086c0bfe..b2e3e1ea 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/README.md +++ b/terraform-unity/modules/terraform-unity-sps-airflow/README.md @@ -46,6 +46,10 @@ No modules. | [aws_security_group_rule.airflow_kpo_efs](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.eks_egress_to_rds](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.rds_ingress_from_eks](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/security_group_rule) | resource | +| [aws_ssm_parameter.airflow_api_url](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.airflow_logs](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.airflow_ui_url](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.ogc_processes_api_url](https://registry.terraform.io/providers/hashicorp/aws/5.35.0/docs/resources/ssm_parameter) | resource | | [helm_release.airflow](https://registry.terraform.io/providers/hashicorp/helm/2.12.1/docs/resources/release) | resource | | [helm_release.keda](https://registry.terraform.io/providers/hashicorp/helm/2.12.1/docs/resources/release) | resource | | [kubernetes_deployment.ogc_processes_api](https://registry.terraform.io/providers/hashicorp/kubernetes/2.25.2/docs/resources/deployment) | resource | @@ -78,14 +82,14 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | value | `string` | n/a | yes | -| [counter](#input\_counter) | value | `string` | n/a | yes | -| [docker\_images](#input\_docker\_images) | Docker images for the services. | 
object({
    airflow = object({
      name = string
      tag  = string
    }),
    ogc_processes_api = object({
      name = string
      tag  = string
    })
  })
| n/a | yes | -| [eks\_cluster\_name](#input\_eks\_cluster\_name) | value | `string` | n/a | yes | -| [helm\_charts](#input\_helm\_charts) | Settings for the required Helm charts. | 
map(object({
    repository = string
    chart      = string
    version    = string
  }))
| n/a | yes | -| [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | Path to the kubeconfig file for the Kubernetes cluster | `string` | n/a | yes | +| [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes | +| [counter](#input\_counter) | Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead. | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes | +| [docker\_images](#input\_docker\_images) | Docker images for the associated services. | 
object({
    airflow = object({
      name = string
      tag  = string
    }),
    ogc_processes_api = object({
      name = string
      tag  = string
    })
  })
| n/a | yes | +| [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | 
map(object({
    repository = string
    chart      = string
    version    = string
  }))
| n/a | yes | +| [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes | | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | n/a | yes | -| [release](#input\_release) | The SPS release version | `string` | n/a | yes | +| [release](#input\_release) | The software release version. | `string` | n/a | yes | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | n/a | yes | | [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | @@ -93,6 +97,7 @@ No modules. | Name | Description | |------|-------------| -| [airflow\_webserver\_url](#output\_airflow\_webserver\_url) | The URL of the Airflow webserver service | -| [ogc\_processes\_api\_url](#output\_ogc\_processes\_api\_url) | The URL of the OGC Processes API service | +| [airflow\_urls](#output\_airflow\_urls) | SSM parameter IDs and URLs for the various Airflow endpoints. | +| [ogc\_processes\_api\_url](#output\_ogc\_processes\_api\_url) | SSM parameter IDs and URLs for the OGC Processes API endpoint. | +| [s3\_buckets](#output\_s3\_buckets) | SSM parameter IDs and bucket names for the various buckets used in the pipeline. | diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/data.tf b/terraform-unity/modules/terraform-unity-sps-airflow/data.tf index f15f24fe..34adeed5 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/data.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/data.tf @@ -1,11 +1,11 @@ data "aws_caller_identity" "current" {} data "aws_eks_cluster" "cluster" { - name = var.eks_cluster_name + name = format(local.resource_name_prefix, "eks") } data "aws_eks_cluster_auth" "cluster" { - name = var.eks_cluster_name + name = format(local.resource_name_prefix, "eks") } data "aws_vpc" "cluster_vpc" { diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf b/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf index b586cef3..b87bb1d8 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf @@ -1,6 +1,7 @@ locals { - counter = var.counter != "" ? var.counter : random_id.counter.hex + counter = var.counter != "" ? var.counter : random_id.counter.hex + resource_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s", var.deployment_name, local.counter])) common_tags = { Name = "" Venue = var.venue @@ -14,4 +15,10 @@ locals { Stack = "" } oidc_provider_url = replace(data.aws_eks_cluster.cluster.identity[0].oidc[0].issuer, "https://", "") + airflow_webserver_navbar_color = { + "prod" = "#bf4f4f" + "test" = "#cfdf4f" + "dev" = "#58cc35" + "sbg-dev" = "#58cc35" + }[var.venue] } diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf index 7accb3da..a64078a9 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf @@ -107,10 +107,10 @@ resource "random_password" "airflow_db" { } resource "aws_secretsmanager_secret" "airflow_db" { - name = "${var.project}-${var.venue}-${var.service_area}-airflowdb-${local.counter}" + name = format(local.resource_name_prefix, "AirflowDb") recovery_window_in_days = 0 tags = merge(local.common_tags, { - Name = "${var.project}-${var.venue}-${var.service_area}-airflow-${local.counter}" + Name = format(local.resource_name_prefix, "AirflowDb") Component = "airflow" Stack = "airflow" }) @@ -122,10 +122,10 @@ resource "aws_secretsmanager_secret_version" "airflow_db" { } resource "aws_db_subnet_group" "airflow_db" { - name = "${var.project}-${var.venue}-${var.service_area}-airflowdb-${local.counter}" + name = format(local.resource_name_prefix, "airflowdb") subnet_ids = jsondecode(data.aws_ssm_parameter.subnet_ids.value)["private"] tags = merge(local.common_tags, { - Name = "${var.project}-${var.venue}-${var.service_area}-airflowdb-${local.counter}" + Name = format(local.resource_name_prefix, "airflowdb") Component = "airflow" Stack = "airflow" }) @@ -133,11 +133,11 @@ resource "aws_db_subnet_group" "airflow_db" { # Security group for RDS resource "aws_security_group" "rds_sg" { - name = "${var.project}-${var.venue}-${var.service_area}-RdsEc2-${local.counter}" + name = format(local.resource_name_prefix, "RdsEc2") description = "Security group for RDS instance to allow traffic from EKS nodes" vpc_id = data.aws_eks_cluster.cluster.vpc_config[0].vpc_id tags = merge(local.common_tags, { - Name = "${var.project}-${var.venue}-${var.service_area}-airflow-${local.counter}" + Name = format(local.resource_name_prefix, "RdsEc2") Component = "airflow" Stack = "airflow" }) @@ -147,7 +147,7 @@ data "aws_security_group" "default" { vpc_id = data.aws_eks_cluster.cluster.vpc_config[0].vpc_id filter { name = "tag:Name" - values = ["${var.eks_cluster_name}-node"] + values = ["${format(local.resource_name_prefix, "eks")}-node"] } } @@ -172,7 +172,7 @@ resource "aws_security_group_rule" "eks_egress_to_rds" { } resource "aws_db_instance" "airflow_db" { - identifier = "${var.project}-${var.venue}-${var.service_area}-airflowdb-${local.counter}" + identifier = format(local.resource_name_prefix, "airflowdb") allocated_storage = 100 storage_type = "gp3" engine = "postgres" @@ -187,7 +187,7 @@ resource "aws_db_instance" "airflow_db" { db_subnet_group_name = aws_db_subnet_group.airflow_db.name vpc_security_group_ids = [aws_security_group.rds_sg.id] tags = merge(local.common_tags, { - Name = "${var.project}-${var.venue}-${var.service_area}-airflow-${local.counter}" + Name = format(local.resource_name_prefix, "airflowdb") Component = "airflow" Stack = "airflow" }) @@ -205,17 +205,17 @@ resource "kubernetes_secret" "airflow_metadata" { } resource "aws_s3_bucket" "airflow_logs" { - bucket = "${var.project}-${var.venue}-${var.service_area}-airflowlogs-${local.counter}" + bucket = format(local.resource_name_prefix, "airflowlogs") force_destroy = true tags = merge(local.common_tags, { - Name = "${var.project}-${var.venue}-${var.service_area}-airflow-${local.counter}" + Name = format(local.resource_name_prefix, "airflowlogs") Component = "airflow" Stack = "airflow" }) } resource "aws_iam_policy" "airflow_worker_policy" { - name = "${var.project}-${var.venue}-${var.service_area}-AirflowWorkerPolicy-${local.counter}" + name = format(local.resource_name_prefix, "AirflowWorkerPolicy") description = "Policy for Airflow Workers to access AWS services" policy = jsonencode( { @@ -246,7 +246,7 @@ resource "aws_iam_policy" "airflow_worker_policy" { } resource "aws_iam_role" "airflow_worker_role" { - name = "${var.project}-${var.venue}-${var.service_area}-AirflowWorker-${local.counter}" + name = format(local.resource_name_prefix, "AirflowWorker") assume_role_policy = jsonencode( { "Version" : "2012-10-17", @@ -275,22 +275,22 @@ resource "aws_iam_role_policy_attachment" "airflow_worker_policy_attachment" { } resource "aws_efs_file_system" "airflow_kpo" { - creation_token = "${var.project}-${var.venue}-${var.service_area}-AirflowKpoEfs-${local.counter}" + creation_token = format(local.resource_name_prefix, "AirflowKpoEfs") tags = merge(local.common_tags, { - Name = "${var.project}-${var.venue}-${var.service_area}-AirflowKpoEfs-${local.counter}" + Name = format(local.resource_name_prefix, "AirflowKpoEfs") Component = "airflow" Stack = "airflow" }) } resource "aws_security_group" "airflow_kpo_efs" { - name = "${var.project}-${var.venue}-${var.service_area}-AirflowKpoEfsSg-${local.counter}" + name = format(local.resource_name_prefix, "AirflowKpoEfsSg") description = "Security group for the EFS used in Airflow Kubernetes Pod Operators" vpc_id = data.aws_eks_cluster.cluster.vpc_config[0].vpc_id tags = merge(local.common_tags, { - Name = "${var.project}-${var.venue}-${var.service_area}-AirflowKpoEfsSg-${local.counter}" + Name = format(local.resource_name_prefix, "AirflowKpoEfsSg") Component = "airflow" Stack = "airflow" }) @@ -327,7 +327,7 @@ resource "aws_efs_access_point" "airflow_kpo" { } } tags = merge(local.common_tags, { - Name = "${var.project}-${var.venue}-${var.service_area}-AirflowKpoEfsAp-${local.counter}" + Name = format(local.resource_name_prefix, "AirflowKpoEfsAp") Component = "airflow" Stack = "airflow" }) @@ -396,6 +396,8 @@ resource "helm_release" "airflow" { airflow_logs_s3_location = "s3://${aws_s3_bucket.airflow_logs.id}" airflow_worker_role_arn = aws_iam_role.airflow_worker_role.arn workers_pvc_name = kubernetes_persistent_volume_claim.efs_pvc.metadata[0].name + webserver_instance_name = format(local.resource_name_prefix, "airflow") + webserver_navbar_color = local.airflow_webserver_navbar_color }) ] set_sensitive { @@ -536,3 +538,51 @@ resource "kubernetes_ingress_v1" "ogc_processes_api_ingress" { } wait_for_load_balancer = true } + +resource "aws_ssm_parameter" "airflow_ui_url" { + name = format("/%s", join("/", compact(["", var.project, var.venue, var.service_area, var.deployment_name, local.counter, "processing", "airflow", "ui_url"]))) + description = "The URL of the Airflow UI." + type = "String" + value = "http://${data.kubernetes_ingress_v1.airflow_ingress.status[0].load_balancer[0].ingress[0].hostname}:5000" + tags = merge(local.common_tags, { + Name = format(local.resource_name_prefix, "endpoints-airflow_ui") + Component = "SSM" + Stack = "SSM" + }) +} + +resource "aws_ssm_parameter" "airflow_api_url" { + name = format("/%s", join("/", compact(["", var.project, var.venue, var.service_area, var.deployment_name, local.counter, "processing", "airflow", "api_url"]))) + description = "The URL of the Airflow REST API." + type = "String" + value = "http://${data.kubernetes_ingress_v1.airflow_ingress.status[0].load_balancer[0].ingress[0].hostname}:5000/api/v1" + tags = merge(local.common_tags, { + Name = format(local.resource_name_prefix, "endpoints-airflow_api") + Component = "SSM" + Stack = "SSM" + }) +} + +resource "aws_ssm_parameter" "airflow_logs" { + name = format("/%s", join("/", compact(["", var.project, var.venue, var.service_area, var.deployment_name, local.counter, "processing", "airflow", "logs"]))) + description = "The name of the S3 bucket for the Airflow logs." + type = "String" + value = aws_s3_bucket.airflow_logs.id + tags = merge(local.common_tags, { + Name = format(local.resource_name_prefix, "S3-airflow_logs") + Component = "SSM" + Stack = "SSM" + }) +} + +resource "aws_ssm_parameter" "ogc_processes_api_url" { + name = format("/%s", join("/", compact(["", var.project, var.venue, var.service_area, var.deployment_name, local.counter, "processing", "ogc_processes", "api_url"]))) + description = "The URL of the OGC Processes REST API." + type = "String" + value = "http://${data.kubernetes_ingress_v1.ogc_processes_api_ingress.status[0].load_balancer[0].ingress[0].hostname}:5001" + tags = merge(local.common_tags, { + Name = format(local.resource_name_prefix, "endpoints-ogc_processes_api") + Component = "SSM" + Stack = "SSM" + }) +} diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/outputs.tf b/terraform-unity/modules/terraform-unity-sps-airflow/outputs.tf index f8740a12..95e55bf2 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/outputs.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/outputs.tf @@ -1,9 +1,31 @@ -output "airflow_webserver_url" { - description = "The URL of the Airflow webserver service" - value = "http://${data.kubernetes_ingress_v1.airflow_ingress.status[0].load_balancer[0].ingress[0].hostname}:5000" +output "airflow_urls" { + description = "SSM parameter IDs and URLs for the various Airflow endpoints." + value = { + "ui" = { + "ssm_param_id" = aws_ssm_parameter.airflow_ui_url.id, + "url" = nonsensitive(aws_ssm_parameter.airflow_ui_url.value) + } + "rest_api" = { + "ssm_param_id" = aws_ssm_parameter.airflow_api_url.id, + "url" = nonsensitive(aws_ssm_parameter.airflow_api_url.value) + } + } } output "ogc_processes_api_url" { - description = "The URL of the OGC Processes API service" - value = "http://${data.kubernetes_ingress_v1.ogc_processes_api_ingress.status[0].load_balancer[0].ingress[0].hostname}:5001" + description = "SSM parameter IDs and URLs for the OGC Processes API endpoint." + value = { + "ssm_param_id" = aws_ssm_parameter.ogc_processes_api_url.id, + "url" = nonsensitive(aws_ssm_parameter.ogc_processes_api_url.value) + } +} + +output "s3_buckets" { + description = "SSM parameter IDs and bucket names for the various buckets used in the pipeline." + value = { + "airflow_logs" = { + "ssm_param_id" = aws_ssm_parameter.airflow_logs.id, + "bucket" = nonsensitive(aws_ssm_parameter.airflow_logs.value) + } + } } diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf b/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf index 57dd209c..76fbd326 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf @@ -13,33 +13,33 @@ variable "service_area" { type = string } -variable "counter" { - description = "value" +variable "deployment_name" { + description = "The name of the deployment." type = string } -variable "release" { - description = "The SPS release version" +variable "counter" { + description = "Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead." type = string } -variable "eks_cluster_name" { - description = "value" +variable "release" { + description = "The software release version." type = string } variable "kubeconfig_filepath" { - description = "Path to the kubeconfig file for the Kubernetes cluster" + description = "The path to the kubeconfig file for the Kubernetes cluster." type = string } variable "airflow_webserver_password" { - description = "value" + description = "The password for the Airflow webserver and UI." type = string } variable "helm_charts" { - description = "Settings for the required Helm charts." + description = "Helm charts for the associated services." type = map(object({ repository = string chart = string @@ -48,7 +48,7 @@ variable "helm_charts" { } variable "docker_images" { - description = "Docker images for the services." + description = "Docker images for the associated services." type = object({ airflow = object({ name = string diff --git a/terraform-unity/outputs.tf b/terraform-unity/outputs.tf index 75127946..6cb3036b 100644 --- a/terraform-unity/outputs.tf +++ b/terraform-unity/outputs.tf @@ -1,7 +1,10 @@ -output "load_balancer_hostnames" { - description = "Load Balancer Ingress Hostnames" +output "resources" { + description = "SSM parameter IDs for pipeline resources." value = { - airflow = module.unity-sps-airflow.airflow_webserver_url - ogc_processes_api = module.unity-sps-airflow.ogc_processes_api_url + "endpoints" = { + "airflow" = module.unity-sps-airflow.airflow_urls + "ogc_processes" = module.unity-sps-airflow.ogc_processes_api_url + } + "buckets" = module.unity-sps-airflow.s3_buckets } } diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf index f1a397e6..f84d4f96 100644 --- a/terraform-unity/variables.tf +++ b/terraform-unity/variables.tf @@ -1,50 +1,52 @@ variable "project" { - description = "The project or mission deploying Unity SPS" + description = "The project or mission deploying Unity SPS." type = string default = "unity" } variable "venue" { - description = "The MCP venue in which the cluster will be deployed (dev, test, prod)" + description = "The MCP venue in which the resources will be deployed." type = string - default = null + validation { + condition = can(regex("^(dev|test|prod|sbg-dev)$", var.venue)) + error_message = "Invalid deployment type." + } } variable "service_area" { - description = "The service area owner of the resources being deployed" + description = "The service area owner of the resources being deployed." type = string default = "sps" } -variable "counter" { - description = "value" +variable "deployment_name" { + description = "The name of the deployment." type = string - default = "" } -variable "release" { - description = "The SPS release version" +variable "counter" { + description = "Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead." type = string + default = "" } -variable "eks_cluster_name" { - description = "The name of the EKS cluster." +variable "release" { + description = "The software release version." type = string } variable "kubeconfig_filepath" { - description = "Path to the kubeconfig file for the Kubernetes cluster" + description = "The path to the kubeconfig file for the Kubernetes cluster." type = string - default = "../k8s/kubernetes.yml" } variable "airflow_webserver_password" { - description = "value" + description = "The password for the Airflow webserver and UI." type = string } variable "helm_charts" { - description = "Settings for the required Helm charts." + description = "Helm charts for the associated services." type = map(object({ repository = string chart = string @@ -65,7 +67,7 @@ variable "helm_charts" { } variable "docker_images" { - description = "Docker images for the services." + description = "Docker images for the associated services." type = object({ airflow = object({ name = string From a8483c79eed38c278ffce7be9b3f5587fc63ebb2 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Thu, 7 Mar 2024 16:22:21 -0800 Subject: [PATCH 3/3] Revert pre-commit comment --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8c0ac886..109507c4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,7 +71,7 @@ repos: args: - --tf-init-args=-upgrade - id: terraform_fmt # Rewrites all Terraform configuration files to a canonical format. - # - id: terraform_tflint # Validates all Terraform configuration files with TFLint. + - id: terraform_tflint # Validates all Terraform configuration files with TFLint. - id: terraform_trivy # Static analysis of Terraform templates to spot potential security issues. args: - >