source_log_uri as parameter and end index

# Conflicts:
#	deployment/modules/gcp/cloudbuild/preloader/main.tf

# Conflicts:
#	deployment/live/gcp/static-ct-staging/cloudbuild/staging/terragrunt.hcl

Author: phbnf

deployment/modules/gcp/cloudbuild/preloader/main.tf

@@ -65,18 +65,21 @@ resource "google_cloudbuild_trigger" "preloader_trigger" {
     ## TODO(phboneff): move to its own container / cloudrun / batch job.
     ## Preload entries.
     ## Leave enough time for the preloader to run, until the token expires.
+    ## Stop after 40k entries, this is what gets copied within 60 minutes.
     timeout = "4200s" // 60 minutes
     step {
       id       = "ct_preloader"
       name     = "golang"
       script   = <<EOT
-	      START_INDEX=$(curl -H "Authorization: Bearer $(cat /workspace/cb_access)" https://storage.googleapis.com/${var.monitoring_url}/checkpoint | head -2 | tail -1)
-	      echo "Will start preloader at index $START_INDEX"
+	      START_INDEX=$(curl -H "Authorization: Bearer $(cat /workspace/cb_access)" ${var.monitoring_url}/checkpoint | head -2 | tail -1)
+	      END_INDEX=$(($START_INDEX+400000))
+	      echo "Will run preloader between $START_INDEX and $END_INDEX"
         go run github.com/google/certificate-transparency-go/preload/preloader@master \
           --target_log_uri=${var.submission_url}/ \
 	        --target_bearer_token="$(cat /workspace/cb_identity)" \
-          --source_log_uri=https://ct.googleapis.com/logs/us1/argon2025h1 \
+          --source_log_uri=${var.source_log_uri} \
 	        --start_index=$START_INDEX \
+	        --end_index=$END_INDEX \
           --num_workers=20 \
           --parallel_fetch=20 \
           --parallel_submit=20

deployment/modules/gcp/cloudbuild/preloader/variables.tf

@@ -24,11 +24,16 @@ variable "github_owner" {
 }
 
 variable "submission_url" {
-  description = "Submission URL of the destination log"
+  description = "Submission URL of the destination static-ct-api log"
   type        = string
 }
 
 variable "monitoring_url" {
-  description = "Monitoring URL of the destination log"
+  description = "Monitoring URL of the destination static-ct-api log"
+  type        = string
+}
+
+variable "source_log_uri" {
+  description = "URL of the source RFC6962 log"
   type        = string
 }

deployment/modules/gcp/cloudbuild/tesseract/main.tf

@@ -133,7 +133,7 @@ resource "google_cloudbuild_trigger" "build_trigger" {
       id     = "terraform_apply_preloader"
       name   = "alpine/terragrunt"
       script = <<EOT
-        terragrunt --terragrunt-non-interactive --terragrunt-no-color apply -auto-approve -no-color -var="submission_url=$(cat /workspace/conformance_url)/arche2025h1.ct.transparency.dev/" -var="monitoring_url=$(cat /workspace/conformance_bucket_name)" 2>&1
+        terragrunt --terragrunt-non-interactive --terragrunt-no-color apply -auto-approve -no-color -var="submission_url=$(cat /workspace/conformance_url)/arche2025h1.ct.transparency.dev/" -var="monitoring_url=https://storage.googleapis.com/$(cat /workspace/conformance_bucket_name)" 2>&1
       EOT
       dir    = "deployment/live/gcp/static-ct-staging/cloudbuild/preloader"
       env = [
@@ -144,6 +144,7 @@ resource "google_cloudbuild_trigger" "build_trigger" {
         "TF_VAR_location=${var.location}",
         "TF_VAR_env=${var.env}",
         "TF_VAR_github_owner=${var.github_owner}",
+        "TF_VAR_source_log_uri=${var.source_log_uri}",
       ]
       wait_for = ["terraform_apply_conformance_staging"]
     }

deployment/modules/gcp/cloudbuild/tesseract/variables.tf

@@ -22,3 +22,8 @@ variable "github_owner" {
   description = "GitHub owner used in Cloud Build trigger repository mapping"
   type        = string
 }
+
+variable "source_log_uri" {
+  description = "URL of the source RFC6962 log to be preloaded."
+  type        = string
+}