diff --git a/modules/runtime_container_engine_config/variables.tf b/modules/runtime_container_engine_config/variables.tf index 38515f8a..1a264825 100644 --- a/modules/runtime_container_engine_config/variables.tf +++ b/modules/runtime_container_engine_config/variables.tf @@ -345,6 +345,6 @@ variable "vault_secret_id" { } variable "vault_token_renew" { - type = number + type = number description = "Vault token renewal period in seconds. Required when TFE_VAULT_USE_EXTERNAL is true." } diff --git a/modules/tfe_init/functions.tf b/modules/tfe_init/functions.tf index c6d577ae..06c1c244 100644 --- a/modules/tfe_init/functions.tf +++ b/modules/tfe_init/functions.tf @@ -16,5 +16,10 @@ locals { distribution = var.distribution enable_monitoring = var.enable_monitoring != null ? var.enable_monitoring : false }) + quadlet_unit = templatefile("${path.module}/templates/terraform-enterprise.kube.tpl", {}) + + retry = templatefile("${path.module}/templates/retry.func", { + cloud = var.cloud + }) } diff --git a/modules/tfe_init/main.tf b/modules/tfe_init/main.tf index fa1316c3..2bcfca59 100644 --- a/modules/tfe_init/main.tf +++ b/modules/tfe_init/main.tf @@ -45,6 +45,7 @@ locals { get_base64_secrets = local.get_base64_secrets install_packages = local.install_packages install_monitoring_agents = local.install_monitoring_agents + retry = local.retry quadlet_unit = local.quadlet_unit active_active = var.operational_mode == "active-active" diff --git a/modules/tfe_init/templates/install_packages.func b/modules/tfe_init/templates/install_packages.func index cbb54e70..31134037 100644 --- a/modules/tfe_init/templates/install_packages.func +++ b/modules/tfe_init/templates/install_packages.func @@ -16,8 +16,8 @@ function install_packages { systemctl start firewalld %{ else ~} echo "[$(date +"%FT%T")] [Terraform Enterprise] Install unzip with apt-get" | tee -a $log_pathname - apt-get update -y - apt-get install -y unzip + retry 5 apt-get update -y + retry 5 apt-get install -y unzip %{ endif ~} echo "[$(date +"%FT%T")] [Terraform Enterprise] Install AWS CLI" | tee -a $log_pathname @@ -39,4 +39,4 @@ function install_packages { install_packages () { : } -%{ endif ~} \ No newline at end of file +%{ endif ~} diff --git a/modules/tfe_init/templates/retry.func b/modules/tfe_init/templates/retry.func new file mode 100644 index 00000000..62302dba --- /dev/null +++ b/modules/tfe_init/templates/retry.func @@ -0,0 +1,25 @@ +# Retry a command up to a specific numer of times until it exits successfully, +# with exponential back off. +# +# $ retry 5 echo Hello + +function retry { + local retries=$1 + shift + + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** $count)) + count=$(($count + 1)) + if [ $count -lt $retries ]; then + echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 + +} diff --git a/modules/tfe_init/templates/tfe.sh.tpl b/modules/tfe_init/templates/tfe.sh.tpl index d53eb8d3..07973f0a 100644 --- a/modules/tfe_init/templates/tfe.sh.tpl +++ b/modules/tfe_init/templates/tfe.sh.tpl @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -eu pipefail - +${retry} ${get_base64_secrets} ${install_packages} %{ if enable_monitoring ~} diff --git a/modules/tfe_init_replicated/functions.tf b/modules/tfe_init_replicated/functions.tf index c531f291..b4f3f2a4 100644 --- a/modules/tfe_init_replicated/functions.tf +++ b/modules/tfe_init_replicated/functions.tf @@ -16,4 +16,8 @@ locals { distribution = var.distribution enable_monitoring = var.enable_monitoring != null ? var.enable_monitoring : false }) + + retry = templatefile("${path.module}/templates/retry.func", { + cloud = var.cloud + }) } diff --git a/modules/tfe_init_replicated/main.tf b/modules/tfe_init_replicated/main.tf index 0ca9e506..f2239ff1 100644 --- a/modules/tfe_init_replicated/main.tf +++ b/modules/tfe_init_replicated/main.tf @@ -11,6 +11,7 @@ locals { get_base64_secrets = local.get_base64_secrets install_packages = local.install_packages install_monitoring_agents = local.install_monitoring_agents + retry = local.retry # Configuration data active_active = var.tfe_configuration != null ? var.tfe_configuration.enable_active_active.value == "1" ? true : false : null diff --git a/modules/tfe_init_replicated/templates/install_packages.func b/modules/tfe_init_replicated/templates/install_packages.func index 713ea6b9..805e7694 100644 --- a/modules/tfe_init_replicated/templates/install_packages.func +++ b/modules/tfe_init_replicated/templates/install_packages.func @@ -29,8 +29,8 @@ function install_packages { systemctl start firewalld %{ else ~} echo "[$(date +"%FT%T")] [Terraform Enterprise] Install unzip with apt-get" | tee -a $log_pathname - apt-get update -y - apt-get install -y unzip + retry 5 apt-get update -y + retry 5 apt-get install -y unzip %{ endif ~} echo "[$(date +"%FT%T")] [Terraform Enterprise] Install AWS CLI" | tee -a $log_pathname diff --git a/modules/tfe_init_replicated/templates/retry.func b/modules/tfe_init_replicated/templates/retry.func new file mode 100644 index 00000000..62302dba --- /dev/null +++ b/modules/tfe_init_replicated/templates/retry.func @@ -0,0 +1,25 @@ +# Retry a command up to a specific numer of times until it exits successfully, +# with exponential back off. +# +# $ retry 5 echo Hello + +function retry { + local retries=$1 + shift + + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** $count)) + count=$(($count + 1)) + if [ $count -lt $retries ]; then + echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 + +} diff --git a/modules/tfe_init_replicated/templates/tfe_replicated.sh.tpl b/modules/tfe_init_replicated/templates/tfe_replicated.sh.tpl index 8910a45a..b634c49f 100644 --- a/modules/tfe_init_replicated/templates/tfe_replicated.sh.tpl +++ b/modules/tfe_init_replicated/templates/tfe_replicated.sh.tpl @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +${retry} ${get_base64_secrets} ${install_packages} ${install_monitoring_agents}