Skip to content

Commit 8932c92

Browse files
phbnfphbnf
authored
Preloader config (#288)
* PRELOADER: add preloader * get current log size * trigger via URL * increase load * put preloader in its own module # Conflicts: # deployment/live/gcp/static-ct-staging/cloudbuild/staging/terragrunt.hcl * source_log_uri as parameter and end index # Conflicts: # deployment/modules/gcp/cloudbuild/preloader/main.tf # Conflicts: # deployment/live/gcp/static-ct-staging/cloudbuild/staging/terragrunt.hcl * don't chain preloader * increase load * nits * spaces
1 parent d9484a6 commit 8932c92

File tree

4 files changed

+219
-0
lines changed

4 files changed

+219
-0
lines changed

‎deployment/live/gcp/static-ct-staging/cloudbuild/preloader/.terraform.lock.hcl

Lines changed: 22 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎deployment/live/gcp/static-ct-staging/cloudbuild/preloader/terragrunt.hcl

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
terraform {
2+
source = "${get_repo_root()}/deployment/modules/gcp//cloudbuild/preloader"
3+
}
4+
5+
locals {
6+
env = get_env("GOOGLE_ENV", "staging")
7+
docker_env = get_env("GOOGLE_ENV", "staging")
8+
project_id = get_env("GOOGLE_PROJECT", "static-ct-staging")
9+
location = get_env("GOOGLE_REGION", "us-central1")
10+
github_owner = get_env("GITHUB_OWNER", "transparency-dev")
11+
source_log_uri = get_env("SOURCE_LOG_URI", "https://ct.googleapis.com/logs/us1/argon2025h1")
12+
submission_url = get_env("SUBMISSION_URL", "https://arche2025h1-64t3hlisgq-uc.a.run.app/arche2025h1.ct.transparency.dev")
13+
monitoring_url = get_env("MONITORING_URL", " https://storage.googleapis.com/static-ct-staging-arche2025h1-bucket")
14+
}
15+
16+
inputs = local
17+
18+
remote_state {
19+
backend = "gcs"
20+
21+
config = {
22+
project = local.project_id
23+
location = local.location
24+
bucket = "${local.project_id}-cloudbuild-preloader-terraform-state"
25+
prefix = "terraform.tfstate"
26+
27+
gcs_bucket_labels = {
28+
name = "terraform_state"
29+
env = "${local.env}"
30+
}
31+
}
32+
}

‎deployment/modules/gcp/cloudbuild/preloader/main.tf

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
terraform {
2+
backend "gcs" {}
3+
4+
required_providers {
5+
google = {
6+
source = "registry.terraform.io/hashicorp/google"
7+
version = "6.12.0"
8+
}
9+
}
10+
}
11+
12+
# Cloud Build
13+
14+
locals {
15+
cloudbuild_service_account = "cloudbuild-${var.env}-sa@${var.project_id}.iam.gserviceaccount.com"
16+
scheduler_service_account = "scheduler-${var.env}-sa@${var.project_id}.iam.gserviceaccount.com"
17+
}
18+
19+
resource "google_project_service" "cloudbuild_api" {
20+
service = "cloudbuild.googleapis.com"
21+
disable_on_destroy = false
22+
}
23+
24+
## Service usage API is required on the project to enable APIs.
25+
## https://cloud.google.com/apis/docs/getting-started#enabling_apis
26+
## serviceusage.googleapis.com acts as a central point for managing the API
27+
## lifecycle within your project. By ensuring the required APIs are enabled
28+
## and accessible, it allows Cloud Build to function seamlessly and interact
29+
## with other Google Cloud services as needed.
30+
##
31+
## The Cloud Build service account also needs roles/serviceusage.serviceUsageViewer.
32+
resource "google_project_service" "serviceusage_api" {
33+
service = "serviceusage.googleapis.com"
34+
disable_on_destroy = false
35+
}
36+
37+
resource "google_cloudbuild_trigger" "preloader_trigger" {
38+
name = "preloader-${var.env}"
39+
service_account = "projects/${var.project_id}/serviceAccounts/${local.cloudbuild_service_account}"
40+
location = var.location
41+
42+
# TODO(phboneff): use a better mechanism to trigger releases that re-uses Docker containters, or based on branches rather.
43+
# This is a temporary mechanism to speed up development.
44+
github {
45+
owner = var.github_owner
46+
name = "static-ct"
47+
push {
48+
tag = "^staging-deploy-(.+)$"
49+
}
50+
}
51+
52+
build {
53+
## Since TesseraCT's infrastructure is not publicly accessible, we need to use
54+
## bearer tokens for the test to access them.
55+
## This step creates those, and stores them for later use.
56+
step {
57+
id = "bearer_token"
58+
name = "gcr.io/cloud-builders/gcloud"
59+
script = <<EOT
60+
gcloud auth print-access-token > /workspace/cb_access
61+
curl -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/${local.cloudbuild_service_account}/identity?audience=${var.submission_url}" > /workspace/cb_identity
62+
EOT
63+
}
64+
65+
## TODO(phboneff): move to its own container / cloudrun / batch job.
66+
## Preload entries.
67+
## Leave enough time for the preloader to run, until the token expires.
68+
## Stop after 360k entries, this is what gets copied within 60 minutes.
69+
timeout = "4200s" // 60 minutes
70+
step {
71+
id = "ct_preloader"
72+
name = "golang"
73+
script = <<EOT
74+
START_INDEX=$(curl -H "Authorization: Bearer $(cat /workspace/cb_access)" ${var.monitoring_url}/checkpoint | head -2 | tail -1)
75+
END_INDEX=$(($START_INDEX+360000))
76+
echo "Will run preloader between $START_INDEX and $END_INDEX"
77+
go run github.com/google/certificate-transparency-go/preload/preloader@master \
78+
--target_log_uri=${var.submission_url}/ \
79+
--target_bearer_token="$(cat /workspace/cb_identity)" \
80+
--source_log_uri=${var.source_log_uri} \
81+
--start_index=$START_INDEX \
82+
--end_index=$END_INDEX \
83+
--num_workers=20 \
84+
--parallel_fetch=20 \
85+
--parallel_submit=20
86+
EOT
87+
wait_for = ["bearer_token"]
88+
timeout = "3420s" // 57 minutes, since token validity if of 60 min.
89+
}
90+
91+
options {
92+
logging = "CLOUD_LOGGING_ONLY"
93+
machine_type = "E2_HIGHCPU_8"
94+
}
95+
}
96+
}
97+
98+
// TODO(phboneff): replace with a long running job once the log is public.
99+
resource "google_cloud_scheduler_job" "deploy_cron" {
100+
paused = false
101+
project = var.project_id
102+
region = var.location
103+
name = "deploy-cron"
104+
105+
schedule = "50 * * * *"
106+
time_zone = "America/Los_Angeles"
107+
108+
attempt_deadline = "120s"
109+
110+
http_target {
111+
http_method = "POST"
112+
uri = "https://cloudbuild.googleapis.com/v1/projects/${var.project_id}/locations/${var.location}/triggers/${google_cloudbuild_trigger.preloader_trigger.trigger_id}:run"
113+
body = base64encode(jsonencode({
114+
source = {
115+
branchName = "main"
116+
}
117+
}))
118+
headers = {
119+
"Content-Type" = "application/json"
120+
}
121+
122+
oauth_token {
123+
service_account_email = local.scheduler_service_account
124+
}
125+
}
126+
}

‎deployment/modules/gcp/cloudbuild/preloader/variables.tf

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
variable "project_id" {
2+
description = "GCP project ID where the log is hosted"
3+
type = string
4+
}
5+
6+
variable "location" {
7+
description = "Location in which to create resources"
8+
type = string
9+
}
10+
11+
variable "env" {
12+
description = "Unique identifier for the env, e.g. dev or ci or prod"
13+
type = string
14+
}
15+
16+
variable "docker_env" {
17+
description = "Unique identifier for the Docker env, e.g. dev or ci or prod"
18+
type = string
19+
}
20+
21+
variable "github_owner" {
22+
description = "GitHub owner used in Cloud Build trigger repository mapping"
23+
type = string
24+
}
25+
26+
variable "submission_url" {
27+
description = "Submission URL of the destination static-ct-api log"
28+
type = string
29+
}
30+
31+
variable "monitoring_url" {
32+
description = "Monitoring URL of the destination static-ct-api log"
33+
type = string
34+
}
35+
36+
variable "source_log_uri" {
37+
description = "URL of the source RFC6962 log"
38+
type = string
39+
}

0 commit comments

Comments
 (0)