feat: add autopilot confidential nodes example

build/int.cloudbuild.yaml

@@ -481,6 +481,21 @@ steps:
     - verify test-confidential-safer-cluster
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestConfidentialSaferCluster --stage teardown --verbose']
+- id: apply test-confidential-autopilot-private
+  waitFor:
+    - init-all
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestConfidentialAutopilotPrivate --stage apply --verbose']
+- id: verify test-confidential-autopilot-private
+  waitFor:
+    - apply test-confidential-autopilot-private
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestConfidentialAutopilotPrivate --stage verify --verbose']
+- id: teardown test-confidential-autopilot-private
+  waitFor:
+    - verify test-confidential-autopilot-private
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestConfidentialAutopilotPrivate --stage teardown --verbose']
 tags:
 - 'ci'
 - 'integration'

examples/confidential_autopilot_private/README.md

@@ -0,0 +1,35 @@
+# Confidential Autopilot Private Cluster
+
+This example illustrates how to create an Autopilot cluster with beta features,
+using Confidential GKE nodes and a Customer Managed Encryption Keys (CMEK).
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| project\_id | The project ID to host the cluster in | `string` | n/a | yes |
+| region | The region to host the cluster in | `string` | `"us-central1"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cluster\_name | Cluster name |
+| kms\_key | CMEK used for disk and database encryption |
+| kubernetes\_endpoint | The cluster endpoint |
+| location | Cluster location (region if regional cluster, zone if zonal cluster) |
+| master\_kubernetes\_version | Kubernetes version of the master |
+| network\_name | The name of the VPC being created |
+| region | The region in which the cluster resides |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| subnet\_names | The names of the subnet being created |
+| zones | List of zones in which the cluster resides |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+To provision this example, run the following from within this directory:
+- `terraform init` to get the plugins
+- `terraform plan` to see the infrastructure plan
+- `terraform apply` to apply the infrastructure build
+- `terraform destroy` to destroy the built infrastructure

examples/confidential_autopilot_private/main.tf

@@ -0,0 +1,84 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  cluster_type           = "confidential-autopilot-private"
+  network_name           = "confidential-autopilot-private-network"
+  subnet_name            = "confidential-autopilot-private-subnet"
+  master_auth_subnetwork = "confidential-autopilot-master-subnet"
+  pods_range_name        = "ip-range-pods-confidential-autopilot"
+  svc_range_name         = "ip-range-svc-confidential-autopilot"
+  subnet_names           = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]]
+}
+
+data "google_project" "main" {
+  project_id = var.project_id
+}
+
+resource "random_string" "suffix" {
+  length  = 4
+  special = false
+  upper   = false
+}
+
+module "kms" {
+  source  = "terraform-google-modules/kms/google"
+  version = "~> 4.0"
+
+  project_id           = var.project_id
+  key_protection_level = "HSM"
+  location             = var.region
+  keyring              = "keyring-${random_string.suffix.result}"
+  keys                 = ["key"]
+  prevent_destroy      = false
+}
+
+resource "google_kms_crypto_key_iam_member" "main" {
+  crypto_key_id = values(module.kms.keys)[0]
+  role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+  member        = "serviceAccount:service-${data.google_project.main.number}@compute-system.iam.gserviceaccount.com"
+}
+
+module "gke" {
+  source  = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-private-cluster"
+  version = "~> 36.0"
+
+  project_id                      = var.project_id
+  name                            = "${local.cluster_type}-cluster"
+  regional                        = true
+  region                          = var.region
+  network                         = module.gcp-network.network_name
+  subnetwork                      = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)]
+  ip_range_pods                   = local.pods_range_name
+  ip_range_services               = local.svc_range_name
+  release_channel                 = "REGULAR"
+  enable_vertical_pod_autoscaling = true
+  enable_private_endpoint         = true
+  enable_private_nodes            = true
+  network_tags                    = [local.cluster_type]
+  deletion_protection             = false
+  boot_disk_kms_key               = values(module.kms.keys)[0]
+  enable_confidential_nodes       = true
+
+  database_encryption = [
+    {
+      "key_name" : values(module.kms.keys)[0],
+      "state" : "ENCRYPTED"
+    }
+  ]
+
+  depends_on = [google_kms_crypto_key_iam_member.main]
+}

examples/confidential_autopilot_private/network.tf

@@ -0,0 +1,50 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "gcp-network" {
+  source  = "terraform-google-modules/network/google"
+  version = "~> 10.0"
+
+  project_id   = var.project_id
+  network_name = local.network_name
+
+  subnets = [
+    {
+      subnet_name           = local.subnet_name
+      subnet_ip             = "10.0.0.0/17"
+      subnet_region         = var.region
+      subnet_private_access = true
+    },
+    {
+      subnet_name   = local.master_auth_subnetwork
+      subnet_ip     = "10.60.0.0/17"
+      subnet_region = var.region
+    },
+  ]
+
+  secondary_ranges = {
+    (local.subnet_name) = [
+      {
+        range_name    = local.pods_range_name
+        ip_cidr_range = "192.168.0.0/18"
+      },
+      {
+        range_name    = local.svc_range_name
+        ip_cidr_range = "192.168.64.0/18"
+      },
+    ]
+  }
+}

examples/confidential_autopilot_private/outputs.tf

@@ -0,0 +1,66 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "kubernetes_endpoint" {
+  description = "The cluster endpoint"
+  sensitive   = true
+  value       = module.gke.endpoint
+}
+
+output "cluster_name" {
+  description = "Cluster name"
+  value       = module.gke.name
+}
+
+output "location" {
+  description = "Cluster location (region if regional cluster, zone if zonal cluster)"
+  value       = module.gke.location
+}
+
+output "master_kubernetes_version" {
+  description = "Kubernetes version of the master"
+  value       = module.gke.master_version
+}
+
+output "service_account" {
+  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  value       = module.gke.service_account
+}
+
+output "network_name" {
+  description = "The name of the VPC being created"
+  value       = module.gcp-network.network_name
+}
+
+output "subnet_names" {
+  description = "The names of the subnet being created"
+  value       = module.gcp-network.subnets_names
+}
+
+output "region" {
+  description = "The region in which the cluster resides"
+  value       = module.gke.region
+}
+
+output "zones" {
+  description = "List of zones in which the cluster resides"
+  value       = module.gke.zones
+}
+
+output "kms_key" {
+  description = "CMEK used for disk and database encryption"
+  value       = values(module.kms.keys)[0]
+}

examples/confidential_autopilot_private/variables.tf

@@ -0,0 +1,26 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to host the cluster in"
+  type        = string
+}
+
+variable "region" {
+  description = "The region to host the cluster in"
+  type        = string
+  default     = "us-central1"
+}

examples/confidential_autopilot_private/versions.tf

@@ -0,0 +1,27 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_providers {
+    google = {
+      source = "hashicorp/google"
+    }
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+    }
+  }
+  required_version = ">= 1.3"
+}

test/integration/confidential_autopilot_private/confidential_autopilot_private_test.go

@@ -0,0 +1,49 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package confidential_autopilot_private
+
+import (
+	"testing"
+	"time"
+
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
+	"github.com/stretchr/testify/assert"
+	"github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils"
+)
+
+func TestConfidentialAutopilotPrivate(t *testing.T) {
+	projectID := testutils.GetTestProjectFromSetup(t, 1)
+	bpt := tft.NewTFBlueprintTest(t,
+		tft.WithVars(map[string]interface{}{"project_id": projectID}),
+		tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute),
+	)
+
+	bpt.DefineVerify(func(assert *assert.Assertions) {
+		testutils.TGKEVerify(t, bpt, assert)
+
+		location := bpt.GetStringOutput("location")
+		clusterName := bpt.GetStringOutput("cluster_name")
+		key := bpt.GetStringOutput("kms_key")
+
+		op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectID)
+		assert.True(op.Get("autopilot.enabled").Bool(), "should be autopilot")
+		assert.Equal(op.Get("autoscaling.autoprovisioningNodePoolDefaults.bootDiskKmsKey").String(), key, "should have CMEK configured in boot disk")
+		assert.True(op.Get("confidentialNodes.enabled").Bool(), "should have confidential nodes enabled")
+		assert.Equal(op.Get("databaseEncryption.state").String(), "ENCRYPTED", "should have database encryption")
+		assert.Equal(op.Get("databaseEncryption.keyName").String(), key, "should have CMEK configured in database")
+	})
+	bpt.Test()
+}