Skip to content

Commit 5d62286

Browse files
roger2hkroger2hk
committed
Support conformance env lifecycle in GCP CI Cloud Build
1 parent 8ebb41f commit 5d62286

File tree

10 files changed

+271
-53
lines changed

10 files changed

+271
-53
lines changed

‎deployment/live/gcp/cloudbuild/README.md

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ is responsible for:
77

88
1. Building the `cmd/gcp` and `cmd/gcp/ci` docker images from the `main` branch,
99
1. Deploying the `cmd/gcp/ci` image to Cloud Run,
10-
1. TODO: Creating a fresh conformance testing environment,
11-
1. TODO: Running the conformance test against the newly build conformance docker image,
12-
1. TODO: Turning-down the conformance testing environment.
10+
1. Creating a fresh conformance testing environment,
11+
1. Running the conformance test against the newly build conformance docker image,
12+
1. Turning-down the conformance testing environment.
1313

1414
## Initial setup
1515

@@ -22,3 +22,13 @@ Error: Error creating Trigger: googleapi: Error 400: Repository mapping does not
2222

2323
This is a manual one-time step that needs to be followed to integrate GCP Cloud Build
2424
and the GitHub repository.
25+
26+
## Externally managed IAM
27+
28+
In case your GCP organization manages the IAM externally, execute the following command before executing `terragrunt apply`.
29+
30+
```sh
31+
export SKIP_IAM=true
32+
```
33+
34+
Note that the `SKIP_IAM` value in Cloud Build is propagated to the conformance testing environment.

‎deployment/modules/gcp/artifactregistry/outputs.tf

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
output "docker" {
2+
description = "The artifact registry repository for Docker container images"
3+
value = google_artifact_registry_repository.docker
4+
}

‎deployment/modules/gcp/cloudbuild/iam/main.tf

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
resource "google_service_account" "cloudbuild_service_account" {
2+
account_id = "cloudbuild-${var.env}-sa"
3+
display_name = "Service Account for Cloud Build (${var.env})"
4+
}
5+
6+
resource "google_project_iam_member" "logging_log_writer" {
7+
project = var.project_id
8+
role = "roles/logging.logWriter"
9+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
10+
}
11+
12+
resource "google_project_iam_member" "service_usage_viewer" {
13+
project = var.project_id
14+
role = "roles/serviceusage.serviceUsageViewer"
15+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
16+
}
17+
18+
resource "google_project_iam_member" "storage_admin" {
19+
project = var.project_id
20+
role = "roles/storage.admin"
21+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
22+
}
23+
24+
resource "google_project_iam_member" "spanner_admin" {
25+
project = var.project_id
26+
role = "roles/spanner.admin"
27+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
28+
}
29+
30+
resource "google_project_iam_member" "secretmanager_admin" {
31+
project = var.project_id
32+
role = "roles/secretmanager.admin"
33+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
34+
}
35+
36+
resource "google_project_iam_member" "iam_service_account_user" {
37+
project = var.project_id
38+
role = "roles/iam.serviceAccountUser"
39+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
40+
}
41+
42+
resource "google_project_iam_member" "iam_service_account_open_id_token_creator" {
43+
project = var.project_id
44+
role = "roles/iam.serviceAccountOpenIdTokenCreator"
45+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
46+
}
47+
48+
resource "google_project_iam_member" "iam_service_account_viewer" {
49+
project = var.project_id
50+
role = "roles/iam.serviceAccountViewer"
51+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
52+
}
53+
54+
resource "google_project_iam_member" "iam_service_account_admin" {
55+
project = var.project_id
56+
role = "roles/iam.serviceAccountAdmin"
57+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
58+
}
59+
60+
resource "google_project_iam_member" "resourcemanager_project_iam_admin" {
61+
project = var.project_id
62+
role = "roles/resourcemanager.projectIamAdmin"
63+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
64+
}
65+
66+
resource "google_project_iam_member" "artifactregistry_writer" {
67+
project = var.project_id
68+
role = "roles/artifactregistry.writer"
69+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
70+
}
71+
72+
resource "google_project_iam_member" "run_admin" {
73+
project = var.project_id
74+
role = "roles/run.admin"
75+
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
76+
}

‎deployment/modules/gcp/cloudbuild/iam/variables.tf

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
variable "project_id" {
2+
description = "GCP project ID where the log is hosted"
3+
type = string
4+
}
5+
6+
variable "env" {
7+
description = "Unique identifier for the env, e.g. dev or ci or prod"
8+
type = string
9+
}

‎deployment/modules/gcp/cloudbuild/main.tf

Lines changed: 79 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -21,50 +21,24 @@ module "artifactregistry" {
2121
# Cloud Build
2222

2323
locals {
24-
artifact_repo = "${var.location}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.docker.name}"
25-
conformance_gcp_docker_image = "${local.artifact_repo}/conformance-gcp"
24+
cloudbuild_service_account = "cloudbuild-${var.env}-sa@${var.project_id}.iam.gserviceaccount.com"
25+
artifact_repo = "${var.location}-docker.pkg.dev/${var.project_id}/${module.artifactregistry.docker.name}"
26+
conformance_gcp_docker_image = "${local.artifact_repo}/conformance-gcp"
2627
}
2728

2829
resource "google_project_service" "cloudbuild_api" {
2930
service = "cloudbuild.googleapis.com"
3031
disable_on_destroy = false
3132
}
3233

33-
resource "google_service_account" "cloudbuild_service_account" {
34-
account_id = "cloudbuild-${var.env}-sa"
35-
display_name = "Service Account for Cloud Build (${var.env})"
36-
}
37-
38-
resource "google_project_iam_member" "logging_log_writer" {
39-
project = var.project_id
40-
role = "roles/logging.logWriter"
41-
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
42-
}
43-
44-
resource "google_artifact_registry_repository_iam_member" "artifactregistry_writer" {
45-
project = google_artifact_registry_repository.docker.project
46-
location = google_artifact_registry_repository.docker.location
47-
repository = google_artifact_registry_repository.docker.name
48-
role = "roles/artifactregistry.writer"
49-
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
50-
}
51-
52-
# TODO: Use google_cloud_run_service_iam_member to limit the service scope.
53-
resource "google_project_iam_member" "run_developer" {
54-
project = var.project_id
55-
role = "roles/run.developer"
56-
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
57-
}
58-
59-
resource "google_project_iam_member" "iam_service_account_user" {
60-
project = var.project_id
61-
role = "roles/iam.serviceAccountUser"
62-
member = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
34+
resource "google_project_service" "serviceusage_api" {
35+
service = "serviceusage.googleapis.com"
36+
disable_on_destroy = false
6337
}
6438

6539
resource "google_cloudbuild_trigger" "build_trigger" {
6640
name = "build-docker-${var.docker_env}"
67-
service_account = google_service_account.cloudbuild_service_account.id
41+
service_account = "projects/${var.project_id}/serviceAccounts/${local.cloudbuild_service_account}"
6842
location = var.location
6943

7044
github {
@@ -76,8 +50,22 @@ resource "google_cloudbuild_trigger" "build_trigger" {
7650
}
7751

7852
build {
79-
## TODO: Destroy any pre-existing deployment/live/gcp/ci environment.
53+
## Destroy any pre-existing deployment/live/gcp/ci environment.
8054
## This might happen if a previous cloud build failed for some reason.
55+
step {
56+
id = "preclean_env"
57+
name = "alpine/terragrunt"
58+
script = <<EOT
59+
terragrunt --terragrunt-non-interactive --terragrunt-no-color destroy -auto-approve -no-color 2>&1
60+
EOT
61+
dir = "deployment/live/gcp/ci"
62+
env = [
63+
"GOOGLE_PROJECT=${var.project_id}",
64+
"TF_IN_AUTOMATION=1",
65+
"TF_INPUT=false",
66+
"TF_VAR_project_id=${var.project_id}"
67+
]
68+
}
8169

8270
## Build the SCTFE GCP Docker image.
8371
## This will be used by the building the conformance Docker image which includes
@@ -119,25 +107,67 @@ resource "google_cloudbuild_trigger" "build_trigger" {
119107
wait_for = ["docker_build_conformance_gcp"]
120108
}
121109

122-
## Deploy container image to Cloud Run.
123-
## TODO: Remove this as the `terragrunt apply` will bring up the Cloud Run.
110+
## Apply the deployment/live/gcp/ci terragrunt config.
111+
## This will bring up the conformance infrastructure, including a service
112+
## running the conformance server docker image built above.
124113
step {
125-
id = "cloud_run_deploy"
126-
name = "gcr.io/google.com/cloudsdktool/cloud-sdk"
127-
entrypoint = "gcloud"
128-
args = [
129-
"run",
130-
"deploy",
131-
"${var.docker_env}-static-ct",
132-
"--image",
133-
"${local.conformance_gcp_docker_image}:$SHORT_SHA",
134-
"--region",
135-
var.location
114+
id = "terraform_apply_conformance_ci"
115+
name = "alpine/terragrunt"
116+
script = <<EOT
117+
terragrunt --terragrunt-non-interactive --terragrunt-no-color apply -auto-approve -no-color 2>&1
118+
terragrunt --terragrunt-no-color output --raw conformance_url -no-color > /workspace/conformance_url
119+
EOT
120+
dir = "deployment/live/gcp/ci"
121+
env = [
122+
"GOOGLE_PROJECT=${var.project_id}",
123+
"TF_IN_AUTOMATION=1",
124+
"TF_INPUT=false",
125+
"TF_VAR_project_id=${var.project_id}"
136126
]
137127
wait_for = ["docker_push_conformance_gcp"]
138128
}
139129

140-
## TODO: Apply the terragrunt configuration to create the CI environment.
130+
## Since the conformance infrastructure is not publicly accessible, we need to use
131+
## bearer tokens for the test to access them.
132+
## This step creates those, and stores them for later use.
133+
step {
134+
id = "bearer_token"
135+
name = "gcr.io/cloud-builders/gcloud"
136+
script = <<EOT
137+
curl -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/${local.cloudbuild_service_account}/identity?audience=$(cat /workspace/conformance_url)" > /workspace/cb_identity
138+
EOT
139+
wait_for = ["terraform_apply_conformance_ci"]
140+
}
141+
142+
## Test against the conformance server.
143+
## TODO: Replace this with CT Hammer when it is ready.
144+
step {
145+
id = "curl_test"
146+
name = "curlimages/curl"
147+
script = <<EOT
148+
curl -iX POST $(cat /workspace/conformance_url)/ci-conformance/ct/v1/add-pre-chain -H "Authorization: Bearer $(cat /workspace/cb_identity)"
149+
EOT
150+
wait_for = ["bearer_token"]
151+
}
152+
153+
## Destroy the deployment/live/gcp/ci terragrunt config.
154+
## This will tear down the conformance infrastructure we brought up
155+
## above.
156+
step {
157+
id = "terraform_destroy_conformance_ci"
158+
name = "alpine/terragrunt"
159+
script = <<EOT
160+
terragrunt --terragrunt-non-interactive --terragrunt-no-color destroy -auto-approve -no-color 2>&1
161+
EOT
162+
dir = "deployment/live/gcp/ci"
163+
env = [
164+
"GOOGLE_PROJECT=${var.project_id}",
165+
"TF_IN_AUTOMATION=1",
166+
"TF_INPUT=false",
167+
"TF_VAR_project_id=${var.project_id}"
168+
]
169+
wait_for = ["curl_test"]
170+
}
141171

142172
options {
143173
logging = "CLOUD_LOGGING_ONLY"
@@ -146,6 +176,6 @@ resource "google_cloudbuild_trigger" "build_trigger" {
146176
}
147177

148178
depends_on = [
149-
google_artifact_registry_repository.docker
179+
module.artifactregistry
150180
]
151181
}

‎deployment/modules/gcp/cloudrun/iam/main.tf

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
resource "google_service_account" "cloudrun_service_account" {
2+
account_id = var.cloudrun_service_account_id
3+
display_name = "Service Account for Cloud Run (${var.env})"
4+
}
5+
6+
resource "google_project_iam_member" "run_service_agent" {
7+
project = var.project_id
8+
role = "roles/run.serviceAgent"
9+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
10+
}
11+
12+
resource "google_project_iam_member" "monitoring_metric_writer" {
13+
project = var.project_id
14+
role = "roles/monitoring.metricWriter"
15+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
16+
}
17+
18+
resource "google_storage_bucket_iam_member" "member" {
19+
bucket = var.bucket
20+
role = "roles/storage.objectUser"
21+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
22+
}
23+
24+
resource "google_project_iam_member" "iam_secret_accessor" {
25+
project = var.project_id
26+
role = "roles/secretmanager.secretAccessor"
27+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
28+
}
29+
30+
resource "google_spanner_database_iam_member" "iam_log_spanner_database_user" {
31+
instance = var.log_spanner_instance
32+
database = var.log_spanner_db
33+
role = "roles/spanner.databaseUser"
34+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
35+
}
36+
37+
resource "google_spanner_database_iam_member" "iam_dedup_spanner_database_user" {
38+
instance = var.log_spanner_instance
39+
database = var.dedup_spanner_db
40+
role = "roles/spanner.databaseUser"
41+
member = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
42+
}

‎deployment/modules/gcp/cloudrun/iam/variables.tf

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
variable "project_id" {
2+
description = "GCP project ID where the log is hosted"
3+
type = string
4+
}
5+
6+
variable "env" {
7+
description = "Unique identifier for the env, e.g. dev or ci or prod"
8+
type = string
9+
}
10+
11+
variable "cloudrun_service_account_id" {
12+
description = "The Clour Run service account ID to be created"
13+
type = string
14+
}
15+
16+
variable "bucket" {
17+
description = "Log GCS bucket"
18+
type = string
19+
}
20+
21+
variable "log_spanner_instance" {
22+
description = "Log Spanner instance"
23+
type = string
24+
}
25+
26+
variable "log_spanner_db" {
27+
description = "Log Spanner database"
28+
type = string
29+
}
30+
31+
variable "dedup_spanner_db" {
32+
description = "Dedup Spanner database"
33+
type = string
34+
}

‎deployment/modules/gcp/cloudrun/main.tf

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@ terraform {
99

1010
# Cloud Run
1111

12+
locals {
13+
cloudrun_service_account_id = "cloudrun-${var.env}-sa"
14+
spanner_log_db_path = "projects/${var.project_id}/instances/${var.log_spanner_instance}/databases/${var.log_spanner_db}"
15+
spanner_dedup_db_path = "projects/${var.project_id}/instances/${var.log_spanner_instance}/databases/${var.dedup_spanner_db}"
16+
}
17+
1218
resource "google_project_service" "cloudrun_api" {
1319
service = "run.googleapis.com"
1420
disable_on_destroy = false

0 commit comments

Comments
 (0)