Skip to content

Commit 7f7d3a3

Browse files
roger2hkroger2hk
committed
Support conformance env lifecycle in GCP CI Cloud Build
1 parent f181ce4 commit 7f7d3a3

File tree

12 files changed

+287
-47
lines changed

12 files changed

+287
-47
lines changed

‎deployment/live/gcp/cloudbuild/README.md

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ is responsible for:
77

88
1. Building the `cmd/gcp` and `cmd/gcp/ci` docker images from the `main` branch,
99
1. Deploying the `cmd/gcp/ci` image to Cloud Run,
10-
1. TODO: Creating a fresh conformance testing environment,
11-
1. TODO: Running the conformance test against the newly build conformance docker image,
12-
1. TODO: Turning-down the conformance testing environment.
10+
1. Creating a fresh conformance testing environment,
11+
1. Running the conformance test against the newly build conformance docker image,
12+
1. Turning-down the conformance testing environment.
1313

1414
## Initial setup
1515

@@ -22,3 +22,13 @@ Error: Error creating Trigger: googleapi: Error 400: Repository mapping does not
2222

2323
This is a manual one-time step that needs to be followed to integrate GCP Cloud Build
2424
and the GitHub repository.
25+
26+
## Externally managed IAM
27+
28+
In case your GCP organization manages the IAM externally, execute the following command before executing `terragrunt apply`.
29+
30+
```sh
31+
export SKIP_IAM=true
32+
```
33+
34+
Note that the `SKIP_IAM` value in Cloud Build is propagated to the conformance testing environment.

‎deployment/modules/gcp/artifactregistry/outputs.tf

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
output "docker" {
2+
description = "The artifact registry repository for Docker container images"
3+
value = google_artifact_registry_repository.docker
4+
}

‎deployment/modules/gcp/cloudbuild/iam/main.tf

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
resource "google_service_account" "cloudbuild_service_account" {
2+
account_id = "cloudbuild-${var.env}-sa"
3+
display_name = "Service Account for Cloud Build (${var.env})"
4+
}
5+
6+
resource "google_project_iam_member" "logging_log_writer" {
7+
project = var.project_id
8+
role = "roles/logging.logWriter"
9+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
10+
}
11+
12+
resource "google_project_iam_member" "service_usage_viewer" {
13+
project = var.project_id
14+
role = "roles/serviceusage.serviceUsageViewer"
15+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
16+
}
17+
18+
resource "google_project_iam_member" "storage_admin" {
19+
project = var.project_id
20+
role = "roles/storage.admin"
21+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
22+
}
23+
24+
resource "google_project_iam_member" "spanner_admin" {
25+
project = var.project_id
26+
role = "roles/spanner.admin"
27+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
28+
}
29+
30+
resource "google_project_iam_member" "secretmanager_admin" {
31+
project = var.project_id
32+
role = "roles/secretmanager.admin"
33+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
34+
}
35+
36+
resource "google_project_iam_member" "iam_service_account_user" {
37+
project = var.project_id
38+
role = "roles/iam.serviceAccountUser"
39+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
40+
}
41+
42+
resource "google_project_iam_member" "iam_service_account_open_id_token_creator" {
43+
project = var.project_id
44+
role = "roles/iam.serviceAccountOpenIdTokenCreator"
45+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
46+
}
47+
48+
resource "google_project_iam_member" "iam_service_account_viewer" {
49+
project = var.project_id
50+
role = "roles/iam.serviceAccountViewer"
51+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
52+
}
53+
54+
resource "google_project_iam_member" "iam_service_account_admin" {
55+
project = var.project_id
56+
role = "roles/iam.serviceAccountAdmin"
57+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
58+
}
59+
60+
resource "google_project_iam_member" "resourcemanager_project_iam_admin" {
61+
project = var.project_id
62+
role = "roles/resourcemanager.projectIamAdmin"
63+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
64+
}
65+
66+
resource "google_project_iam_member" "artifactregistry_writer" {
67+
project = var.project_id
68+
role = "roles/artifactregistry.writer"
69+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
70+
}
71+
72+
resource "google_project_iam_member" "run_admin" {
73+
project = var.project_id
74+
role = "roles/run.admin"
75+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
76+
}

‎deployment/modules/gcp/cloudbuild/iam/variables.tf

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
variable "project_id" {
2+
description = "GCP project ID where the log is hosted"
3+
type = string
4+
}
5+
6+
variable "env" {
7+
description = "Unique identifier for the env, e.g. dev or ci or prod"
8+
type = string
9+
}

‎deployment/modules/gcp/cloudbuild/main.tf

Lines changed: 85 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ module "artifactregistry" {
2121
# Cloud Build
2222

2323
locals {
24-
artifact_repo = "${var.location}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.docker.name}"
24+
artifact_repo = "${var.location}-docker.pkg.dev/${var.project_id}/${module.artifactregistry.docker.name}"
2525
conformance_gcp_docker_image = "${local.artifact_repo}/conformance-gcp"
2626
}
2727

@@ -30,36 +30,18 @@ resource "google_project_service" "cloudbuild_api" {
3030
disable_on_destroy = false
3131
}
3232

33-
resource "google_service_account" "cloudbuild_service_account" {
34-
account_id = "cloudbuild-${var.env}-sa"
35-
display_name = "Service Account for Cloud Build (${var.env})"
33+
resource "google_project_service" "serviceusage_api" {
34+
service = "serviceusage.googleapis.com"
35+
disable_on_destroy = false
3636
}
3737

38-
resource "google_project_iam_member" "logging_log_writer" {
39-
project = var.project_id
40-
role = "roles/logging.logWriter"
41-
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
42-
}
38+
module "cloudbuild_iam" {
39+
source = "./iam"
4340

44-
resource "google_artifact_registry_repository_iam_member" "artifactregistry_writer" {
45-
project = google_artifact_registry_repository.docker.project
46-
location = google_artifact_registry_repository.docker.location
47-
repository = google_artifact_registry_repository.docker.name
48-
role = "roles/artifactregistry.writer"
49-
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
50-
}
51-
52-
# TODO: Use google_cloud_run_service_iam_member to limit the service scope.
53-
resource "google_project_iam_member" "run_developer" {
54-
project = var.project_id
55-
role = "roles/run.developer"
56-
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
57-
}
41+
project_id = var.project_id
42+
env = var.env
5843

59-
resource "google_project_iam_member" "iam_service_account_user" {
60-
project = var.project_id
61-
role = "roles/iam.serviceAccountUser"
62-
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
44+
count = var.skip_iam ? 0 : 1
6345
}
6446

6547
resource "google_cloudbuild_trigger" "build_trigger" {
@@ -76,8 +58,23 @@ resource "google_cloudbuild_trigger" "build_trigger" {
7658
}
7759

7860
build {
79-
## TODO: Destroy any pre-existing deployment/live/gcp/ci environment.
61+
## Destroy any pre-existing deployment/live/gcp/ci environment.
8062
## This might happen if a previous cloud build failed for some reason.
63+
step {
64+
id = "preclean_env"
65+
name = "alpine/terragrunt"
66+
script = <<EOT
67+
terragrunt --terragrunt-non-interactive --terragrunt-no-color destroy -auto-approve -no-color 2>&1
68+
EOT
69+
dir = "deployment/live/gcp/ci"
70+
env = [
71+
"GOOGLE_PROJECT=${var.project_id}",
72+
"SKIP_IAM=${var.skip_iam}",
73+
"TF_IN_AUTOMATION=1",
74+
"TF_INPUT=false",
75+
"TF_VAR_project_id=${var.project_id}"
76+
]
77+
}
8178

8279
## Build the SCTFE GCP Docker image.
8380
## This will be used by the building the conformance Docker image which includes
@@ -119,25 +116,70 @@ resource "google_cloudbuild_trigger" "build_trigger" {
119116
wait_for = ["docker_build_conformance_gcp"]
120117
}
121118

122-
## Deploy container image to Cloud Run.
123-
## TODO: Remove this as the `terragrunt apply` will bring up the Cloud Run.
119+
## Apply the deployment/live/gcp/ci terragrunt config.
120+
## This will bring up the conformance infrastructure, including a service
121+
## running the conformance server docker image built above.
124122
step {
125-
id = "cloud_run_deploy"
126-
name = "gcr.io/google.com/cloudsdktool/cloud-sdk"
127-
entrypoint = "gcloud"
128-
args = [
129-
"run",
130-
"deploy",
131-
"${var.docker_env}-static-ct",
132-
"--image",
133-
"${local.conformance_gcp_docker_image}:$SHORT_SHA",
134-
"--region",
135-
var.location
123+
id = "terraform_apply_conformance_ci"
124+
name = "alpine/terragrunt"
125+
script = <<EOT
126+
terragrunt --terragrunt-non-interactive --terragrunt-no-color apply -auto-approve -no-color 2>&1
127+
terragrunt --terragrunt-no-color output --raw conformance_url -no-color > /workspace/conformance_url
128+
EOT
129+
dir = "deployment/live/gcp/ci"
130+
env = [
131+
"GOOGLE_PROJECT=${var.project_id}",
132+
"SKIP_IAM=${var.skip_iam}",
133+
"TF_IN_AUTOMATION=1",
134+
"TF_INPUT=false",
135+
"TF_VAR_project_id=${var.project_id}"
136136
]
137137
wait_for = ["docker_push_conformance_gcp"]
138138
}
139139

140-
## TODO: Apply the terragrunt configuration to create the CI environment.
140+
## Since the conformance infrastructure is not publicly accessible, we need to use
141+
## bearer tokens for the test to access them.
142+
## This step creates those, and stores them for later use.
143+
step {
144+
id = "access_token"
145+
name = "gcr.io/cloud-builders/gcloud"
146+
script = <<EOT
147+
gcloud auth print-access-token > /workspace/cb_access_token
148+
curl -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/${google_service_account.cloudbuild_service_account.id}/identity?audience=$(cat /workspace/conformance_url)" > /workspace/cb_identity
149+
EOT
150+
wait_for = ["terraform_apply_conformance_ci"]
151+
}
152+
153+
## Test against the conformance server.
154+
## TODO: Replace this with CT Hammer when it is ready.
155+
step {
156+
id = "curl_test"
157+
name = "curlimages/curl"
158+
script = <<EOT
159+
curl -X POST $(cat /workspace/conformance_url)/ct/v1/add-pre-chain -H "Authorization: Bearer $(cat /workspace/cb_identity)"
160+
EOT
161+
wait_for = ["access_token"]
162+
}
163+
164+
## Destroy the deployment/live/gcp/ci terragrunt config.
165+
## This will tear down the conformance infrastructure we brought up
166+
## above.
167+
step {
168+
id = "terraform_destroy_conformance_ci"
169+
name = "alpine/terragrunt"
170+
script = <<EOT
171+
terragrunt --terragrunt-non-interactive --terragrunt-no-color destroy -auto-approve -no-color 2>&1
172+
EOT
173+
dir = "deployment/live/gcp/ci"
174+
env = [
175+
"GOOGLE_PROJECT=${var.project_id}",
176+
"SKIP_IAM=${var.skip_iam}",
177+
"TF_IN_AUTOMATION=1",
178+
"TF_INPUT=false",
179+
"TF_VAR_project_id=${var.project_id}"
180+
]
181+
wait_for = ["curl_test"]
182+
}
141183

142184
options {
143185
logging = "CLOUD_LOGGING_ONLY"
@@ -146,6 +188,6 @@ resource "google_cloudbuild_trigger" "build_trigger" {
146188
}
147189

148190
depends_on = [
149-
google_artifact_registry_repository.docker
191+
module.artifactregistry
150192
]
151193
}

‎deployment/modules/gcp/cloudbuild/variables.tf

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,8 @@ variable "github_owner" {
2222
description = "GitHub owner used in Cloud Build trigger repository mapping"
2323
type = string
2424
}
25+
26+
variable "skip_iam" {
27+
description = "Skip for GCP projects with externally managed IAM"
28+
type = bool
29+
}

‎deployment/modules/gcp/cloudrun/iam/main.tf

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
resource "google_service_account" "cloudrun_service_account" {
2+
account_id = var.cloudrun_service_account_id
3+
display_name = "Service Account for Cloud Run (${var.env})"
4+
}
5+
6+
resource "google_project_iam_member" "run_service_agent" {
7+
project = var.project_id
8+
role = "roles/run.serviceAgent"
9+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
10+
}
11+
12+
resource "google_project_iam_member" "monitoring_metric_writer" {
13+
project = var.project_id
14+
role = "roles/monitoring.metricWriter"
15+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
16+
}
17+
18+
resource "google_storage_bucket_iam_member" "member" {
19+
bucket = var.bucket
20+
role = "roles/storage.objectUser"
21+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
22+
}
23+
24+
resource "google_project_iam_member" "iam_secret_accessor" {
25+
project = var.project_id
26+
role = "roles/secretmanager.secretAccessor"
27+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
28+
}
29+
30+
resource "google_spanner_database_iam_member" "iam_log_spanner_database_user" {
31+
instance = var.log_spanner_instance
32+
database = var.log_spanner_db
33+
role = "roles/spanner.databaseUser"
34+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
35+
}
36+
37+
resource "google_spanner_database_iam_member" "iam_dedup_spanner_database_user" {
38+
instance = var.log_spanner_instance
39+
database = var.dedup_spanner_db
40+
role = "roles/spanner.databaseUser"
41+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
42+
}

‎deployment/modules/gcp/cloudrun/iam/variables.tf

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
variable "project_id" {
2+
description = "GCP project ID where the log is hosted"
3+
type = string
4+
}
5+
6+
variable "env" {
7+
description = "Unique identifier for the env, e.g. dev or ci or prod"
8+
type = string
9+
}
10+
11+
variable "cloudrun_service_account_id" {
12+
description = "The Clour Run service account ID to be created"
13+
type = string
14+
}
15+
16+
variable "bucket" {
17+
description = "Log GCS bucket"
18+
type = string
19+
}
20+
21+
variable "log_spanner_instance" {
22+
description = "Log Spanner instance"
23+
type = string
24+
}
25+
26+
variable "log_spanner_db" {
27+
description = "Log Spanner database"
28+
type = string
29+
}
30+
31+
variable "dedup_spanner_db" {
32+
description = "Dedup Spanner database"
33+
type = string
34+
}

‎deployment/modules/gcp/cloudrun/main.tf

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@ terraform {
99

1010
# Cloud Run
1111

12+
locals {
13+
cloudrun_service_account_id = "cloudrun-${var.env}-sa"
14+
spanner_log_db_path = "projects/${var.project_id}/instances/${var.log_spanner_instance}/databases/${var.log_spanner_db}"
15+
spanner_dedup_db_path = "projects/${var.project_id}/instances/${var.log_spanner_instance}/databases/${var.dedup_spanner_db}"
16+
}
17+
1218
resource "google_project_service" "cloudrun_api" {
1319
service = "run.googleapis.com"
1420
disable_on_destroy = false

‎deployment/modules/gcp/cloudrun/variables.tf

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,8 @@ variable "signer_private_key_secret_name" {
5252
description = "Private key secret name for checkpoints and SCTs signer. Format: projects/{projectId}/secrets/{secretName}/versions/{secretVersion}."
5353
type = string
5454
}
55+
56+
variable "skip_iam" {
57+
description = "Skip for GCP projects with externally managed IAM"
58+
type = bool
59+
}

0 commit comments

Comments
 (0)