From f123a0cdd004ef19fd24d6c2b46963f132bb121d Mon Sep 17 00:00:00 2001 From: Arthur de Lapertosa Lisboa Date: Tue, 18 Feb 2025 18:37:19 -0300 Subject: [PATCH 1/3] feat: add autopilot confidential nodes example --- build/int.cloudbuild.yaml | 15 ++++ .../confidential_autopilot_private/README.md | 34 ++++++++ .../confidential_autopilot_private/main.tf | 84 +++++++++++++++++++ .../confidential_autopilot_private/network.tf | 50 +++++++++++ .../confidential_autopilot_private/outputs.tf | 65 ++++++++++++++ .../variables.tf | 19 +++++ .../versions.tf | 27 ++++++ .../confidential_autopilot_private_test.go | 49 +++++++++++ 8 files changed, 343 insertions(+) create mode 100644 examples/confidential_autopilot_private/README.md create mode 100644 examples/confidential_autopilot_private/main.tf create mode 100644 examples/confidential_autopilot_private/network.tf create mode 100644 examples/confidential_autopilot_private/outputs.tf create mode 100644 examples/confidential_autopilot_private/variables.tf create mode 100644 examples/confidential_autopilot_private/versions.tf create mode 100644 test/integration/confidential_autopilot_private/confidential_autopilot_private_test.go diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 208943713d..a5b0a732a1 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -481,6 +481,21 @@ steps: - verify test-confidential-safer-cluster name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestConfidentialSaferCluster --stage teardown --verbose'] +- id: apply test-confidential-autopilot-private + waitFor: + - init-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestConfidentialAutopilotPrivate --stage apply --verbose'] +- id: verify test-confidential-autopilot-private + waitFor: + - apply test-confidential-autopilot-private + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestConfidentialAutopilotPrivate --stage verify --verbose'] +- id: teardown test-confidential-autopilot-private + waitFor: + - verify test-confidential-autopilot-private + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestConfidentialAutopilotPrivate --stage teardown --verbose'] tags: - 'ci' - 'integration' diff --git a/examples/confidential_autopilot_private/README.md b/examples/confidential_autopilot_private/README.md new file mode 100644 index 0000000000..7af8c4d40e --- /dev/null +++ b/examples/confidential_autopilot_private/README.md @@ -0,0 +1,34 @@ +# Confidential Autopilot Private Cluster + +This example illustrates how to create a autopilot cluster with beta features, +using Confidential Nodes and a Customer Managed Encryption Keys (CMEK). + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to host the cluster in | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| cluster\_name | Cluster name | +| kms\_key | CMEK used for disk and database encryption | +| kubernetes\_endpoint | The cluster endpoint | +| location | n/a | +| master\_kubernetes\_version | Kubernetes version of the master | +| network\_name | The name of the VPC being created | +| region | The region in which the cluster resides | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnet\_names | The names of the subnet being created | +| zones | List of zones in which the cluster resides | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/confidential_autopilot_private/main.tf b/examples/confidential_autopilot_private/main.tf new file mode 100644 index 0000000000..27a82bd528 --- /dev/null +++ b/examples/confidential_autopilot_private/main.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "confidential-autopilot-private" + network_name = "confidential-autopilot-private-network" + subnet_name = "confidential-autopilot-private-subnet" + master_auth_subnetwork = "confidential-autopilot-master-subnet" + pods_range_name = "ip-range-pods-confidential-autopilot" + svc_range_name = "ip-range-svc-confidential-autopilot" + subnet_names = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]] +} + +data "google_project" "main" { + project_id = var.project_id +} + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +module "kms" { + source = "terraform-google-modules/kms/google" + version = "~> 4.0" + + project_id = var.project_id + key_protection_level = "HSM" + location = "us-central1" + keyring = "keyring-${random_string.suffix.result}" + keys = ["key"] + prevent_destroy = false +} + +resource "google_kms_crypto_key_iam_member" "main" { + crypto_key_id = values(module.kms.keys)[0] + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:service-${data.google_project.main.number}@compute-system.iam.gserviceaccount.com" +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-private-cluster" + version = "~> 36.0" + + project_id = var.project_id + name = "${local.cluster_type}-cluster" + regional = true + region = "us-central1" + network = module.gcp-network.network_name + subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)] + ip_range_pods = local.pods_range_name + ip_range_services = local.svc_range_name + release_channel = "REGULAR" + enable_vertical_pod_autoscaling = true + enable_private_endpoint = true + enable_private_nodes = true + network_tags = [local.cluster_type] + deletion_protection = false + boot_disk_kms_key = values(module.kms.keys)[0] + enable_confidential_nodes = true + + database_encryption = [ + { + "key_name" : values(module.kms.keys)[0], + "state" : "ENCRYPTED" + } + ] + + depends_on = [google_kms_crypto_key_iam_member.main] +} diff --git a/examples/confidential_autopilot_private/network.tf b/examples/confidential_autopilot_private/network.tf new file mode 100644 index 0000000000..21967deacc --- /dev/null +++ b/examples/confidential_autopilot_private/network.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = ">= 7.5" + + project_id = var.project_id + network_name = local.network_name + + subnets = [ + { + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" + subnet_region = "us-central1" + subnet_private_access = true + }, + { + subnet_name = local.master_auth_subnetwork + subnet_ip = "10.60.0.0/17" + subnet_region = "us-central1" + }, + ] + + secondary_ranges = { + (local.subnet_name) = [ + { + range_name = local.pods_range_name + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = local.svc_range_name + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} diff --git a/examples/confidential_autopilot_private/outputs.tf b/examples/confidential_autopilot_private/outputs.tf new file mode 100644 index 0000000000..9a9f6ab829 --- /dev/null +++ b/examples/confidential_autopilot_private/outputs.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "location" { + value = module.gke.location +} + +output "master_kubernetes_version" { + description = "Kubernetes version of the master" + value = module.gke.master_version +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + +output "network_name" { + description = "The name of the VPC being created" + value = module.gcp-network.network_name +} + +output "subnet_names" { + description = "The names of the subnet being created" + value = module.gcp-network.subnets_names +} + +output "region" { + description = "The region in which the cluster resides" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "kms_key" { + description = "CMEK used for disk and database encryption" + value = values(module.kms.keys)[0] +} diff --git a/examples/confidential_autopilot_private/variables.tf b/examples/confidential_autopilot_private/variables.tf new file mode 100644 index 0000000000..d7b7d94925 --- /dev/null +++ b/examples/confidential_autopilot_private/variables.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} diff --git a/examples/confidential_autopilot_private/versions.tf b/examples/confidential_autopilot_private/versions.tf new file mode 100644 index 0000000000..b89def9937 --- /dev/null +++ b/examples/confidential_autopilot_private/versions.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } + required_version = ">= 1.3" +} diff --git a/test/integration/confidential_autopilot_private/confidential_autopilot_private_test.go b/test/integration/confidential_autopilot_private/confidential_autopilot_private_test.go new file mode 100644 index 0000000000..6e6582bc32 --- /dev/null +++ b/test/integration/confidential_autopilot_private/confidential_autopilot_private_test.go @@ -0,0 +1,49 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confidential_autopilot_private + +import ( + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" +) + +func TestConfidentialAutopilotPrivate(t *testing.T) { + projectID := testutils.GetTestProjectFromSetup(t, 1) + bpt := tft.NewTFBlueprintTest(t, + tft.WithVars(map[string]interface{}{"project_id": projectID}), + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + testutils.TGKEVerify(t, bpt, assert) + + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + key := bpt.GetStringOutput("kms_key") + + op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectID) + assert.True(op.Get("autopilot.enabled").Bool(), "should be autopilot") + assert.Equal(op.Get("autoscaling.autoprovisioningNodePoolDefaults.bootDiskKmsKey").String(), key, "should have CMEK configured in boot disk") + assert.True(op.Get("confidentialNodes.enabled").Bool(), "should have confidential nodes enabled") + assert.Equal(op.Get("databaseEncryption.state").String(), "ENCRYPTED", "should have database encryption") + assert.Equal(op.Get("databaseEncryption.keyName").String(), key, "should have CMEK configured in database") + }) + bpt.Test() +} From ccddbe6792d5ff209387dab4fe22dc15f7ecf17d Mon Sep 17 00:00:00 2001 From: Arthur de Lapertosa Lisboa Date: Tue, 25 Feb 2025 12:29:31 -0300 Subject: [PATCH 2/3] code review changes --- examples/confidential_autopilot_private/README.md | 6 +++--- examples/confidential_autopilot_private/main.tf | 4 ++-- examples/confidential_autopilot_private/network.tf | 6 +++--- examples/confidential_autopilot_private/outputs.tf | 3 ++- examples/confidential_autopilot_private/variables.tf | 7 +++++++ 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/examples/confidential_autopilot_private/README.md b/examples/confidential_autopilot_private/README.md index 7af8c4d40e..fbedc51322 100644 --- a/examples/confidential_autopilot_private/README.md +++ b/examples/confidential_autopilot_private/README.md @@ -1,7 +1,7 @@ # Confidential Autopilot Private Cluster -This example illustrates how to create a autopilot cluster with beta features, -using Confidential Nodes and a Customer Managed Encryption Keys (CMEK). +This example illustrates how to create an Autopilot cluster with beta features, +using Confidential GKE nodes and a Customer Managed Encryption Keys (CMEK). ## Inputs @@ -17,7 +17,7 @@ using Confidential Nodes and a Customer Managed Encryption Keys (CMEK). | cluster\_name | Cluster name | | kms\_key | CMEK used for disk and database encryption | | kubernetes\_endpoint | The cluster endpoint | -| location | n/a | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | | master\_kubernetes\_version | Kubernetes version of the master | | network\_name | The name of the VPC being created | | region | The region in which the cluster resides | diff --git a/examples/confidential_autopilot_private/main.tf b/examples/confidential_autopilot_private/main.tf index 27a82bd528..4002b54e65 100644 --- a/examples/confidential_autopilot_private/main.tf +++ b/examples/confidential_autopilot_private/main.tf @@ -40,7 +40,7 @@ module "kms" { project_id = var.project_id key_protection_level = "HSM" - location = "us-central1" + location = var.region keyring = "keyring-${random_string.suffix.result}" keys = ["key"] prevent_destroy = false @@ -59,7 +59,7 @@ module "gke" { project_id = var.project_id name = "${local.cluster_type}-cluster" regional = true - region = "us-central1" + region = var.region network = module.gcp-network.network_name subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)] ip_range_pods = local.pods_range_name diff --git a/examples/confidential_autopilot_private/network.tf b/examples/confidential_autopilot_private/network.tf index 21967deacc..af25d18573 100644 --- a/examples/confidential_autopilot_private/network.tf +++ b/examples/confidential_autopilot_private/network.tf @@ -16,7 +16,7 @@ module "gcp-network" { source = "terraform-google-modules/network/google" - version = ">= 7.5" + version = "~= 10.0" project_id = var.project_id network_name = local.network_name @@ -25,13 +25,13 @@ module "gcp-network" { { subnet_name = local.subnet_name subnet_ip = "10.0.0.0/17" - subnet_region = "us-central1" + subnet_region = var.region subnet_private_access = true }, { subnet_name = local.master_auth_subnetwork subnet_ip = "10.60.0.0/17" - subnet_region = "us-central1" + subnet_region = var.region }, ] diff --git a/examples/confidential_autopilot_private/outputs.tf b/examples/confidential_autopilot_private/outputs.tf index 9a9f6ab829..60078684a1 100644 --- a/examples/confidential_autopilot_private/outputs.tf +++ b/examples/confidential_autopilot_private/outputs.tf @@ -26,7 +26,8 @@ output "cluster_name" { } output "location" { - value = module.gke.location + description = "Cluster location (region if regional cluster, zone if zonal cluster)" + value = module.gke.location } output "master_kubernetes_version" { diff --git a/examples/confidential_autopilot_private/variables.tf b/examples/confidential_autopilot_private/variables.tf index d7b7d94925..ef80789110 100644 --- a/examples/confidential_autopilot_private/variables.tf +++ b/examples/confidential_autopilot_private/variables.tf @@ -16,4 +16,11 @@ variable "project_id" { description = "The project ID to host the cluster in" + type = string +} + +variable "region" { + description = "The region to host the cluster in" + type = string + default = "us-central1" } From e3cf2063a1d82b8927243e5abb3b1eb972d12bc5 Mon Sep 17 00:00:00 2001 From: Arthur de Lapertosa Lisboa Date: Tue, 25 Feb 2025 18:46:01 -0300 Subject: [PATCH 3/3] fix docs and build --- examples/confidential_autopilot_private/README.md | 3 ++- examples/confidential_autopilot_private/network.tf | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/confidential_autopilot_private/README.md b/examples/confidential_autopilot_private/README.md index fbedc51322..391469251c 100644 --- a/examples/confidential_autopilot_private/README.md +++ b/examples/confidential_autopilot_private/README.md @@ -8,7 +8,8 @@ using Confidential GKE nodes and a Customer Managed Encryption Keys (CMEK). | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| project\_id | The project ID to host the cluster in | `any` | n/a | yes | +| project\_id | The project ID to host the cluster in | `string` | n/a | yes | +| region | The region to host the cluster in | `string` | `"us-central1"` | no | ## Outputs diff --git a/examples/confidential_autopilot_private/network.tf b/examples/confidential_autopilot_private/network.tf index af25d18573..f32a4114c0 100644 --- a/examples/confidential_autopilot_private/network.tf +++ b/examples/confidential_autopilot_private/network.tf @@ -16,7 +16,7 @@ module "gcp-network" { source = "terraform-google-modules/network/google" - version = "~= 10.0" + version = "~> 10.0" project_id = var.project_id network_name = local.network_name