enabling retry function to resolve failures due to lock on apt

modules/runtime_container_engine_config/variables.tf

@@ -345,6 +345,6 @@ variable "vault_secret_id" {
 }
 
 variable "vault_token_renew" {
-  type = number
+  type        = number
   description = "Vault token renewal period in seconds. Required when TFE_VAULT_USE_EXTERNAL is true."
 }

modules/tfe_init/functions.tf

@@ -16,5 +16,10 @@ locals {
     distribution      = var.distribution
     enable_monitoring = var.enable_monitoring != null ? var.enable_monitoring : false
   })
+
   quadlet_unit = templatefile("${path.module}/templates/terraform-enterprise.kube.tpl", {})
+
+  retry = templatefile("${path.module}/templates/retry.func", {
+    cloud = var.cloud
+  })
 }

modules/tfe_init/main.tf

@@ -45,6 +45,7 @@ locals {
       get_base64_secrets        = local.get_base64_secrets
       install_packages          = local.install_packages
       install_monitoring_agents = local.install_monitoring_agents
+      retry                     = local.retry
       quadlet_unit              = local.quadlet_unit
 
       active_active               = var.operational_mode == "active-active"

modules/tfe_init/templates/install_packages.func

@@ -16,8 +16,8 @@ function install_packages {
 	systemctl start firewalld
 	%{ else ~}
 	echo "[$(date +"%FT%T")] [Terraform Enterprise] Install unzip with apt-get" | tee -a $log_pathname
-	apt-get update -y
-	apt-get install -y unzip
+	retry 5 apt-get update -y
+	retry 5 apt-get install -y unzip
 	%{ endif ~}
 
 	echo "[$(date +"%FT%T")] [Terraform Enterprise] Install AWS CLI" | tee -a $log_pathname
@@ -39,4 +39,4 @@ function install_packages {
 install_packages () {
 	:
 }
-%{ endif ~}
+%{ endif ~}

modules/tfe_init/templates/retry.func

@@ -0,0 +1,25 @@
+# Retry a command up to a specific numer of times until it exits successfully,
+# with exponential back off.
+#
+#  $ retry 5 echo Hello
+
+function retry {
+  local retries=$1
+  shift
+
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** $count))
+    count=$(($count + 1))
+    if [ $count -lt $retries ]; then
+      echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+
+}

modules/tfe_init/templates/tfe.sh.tpl

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 set -eu pipefail
-
+${retry}
 ${get_base64_secrets}
 ${install_packages}
 %{ if enable_monitoring ~}

modules/tfe_init_replicated/functions.tf

@@ -16,4 +16,8 @@ locals {
     distribution      = var.distribution
     enable_monitoring = var.enable_monitoring != null ? var.enable_monitoring : false
   })
+
+  retry = templatefile("${path.module}/templates/retry.func", {
+    cloud = var.cloud
+  })
 }

modules/tfe_init_replicated/main.tf

@@ -11,6 +11,7 @@ locals {
       get_base64_secrets        = local.get_base64_secrets
       install_packages          = local.install_packages
       install_monitoring_agents = local.install_monitoring_agents
+      retry                     = local.retry
 
       # Configuration data
       active_active               = var.tfe_configuration != null ? var.tfe_configuration.enable_active_active.value == "1" ? true : false : null

modules/tfe_init_replicated/templates/install_packages.func

@@ -29,8 +29,8 @@ function install_packages {
 		systemctl start firewalld
 	%{ else ~}
 		echo "[$(date +"%FT%T")] [Terraform Enterprise] Install unzip with apt-get" | tee -a $log_pathname
-		apt-get update -y
-		apt-get install -y unzip
+		retry 5 apt-get update -y
+		retry 5 apt-get install -y unzip
 	%{ endif ~}
 
 	echo "[$(date +"%FT%T")] [Terraform Enterprise] Install AWS CLI" | tee -a $log_pathname

modules/tfe_init_replicated/templates/retry.func

@@ -0,0 +1,25 @@
+# Retry a command up to a specific numer of times until it exits successfully,
+# with exponential back off.
+#
+#  $ retry 5 echo Hello
+
+function retry {
+  local retries=$1
+  shift
+
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** $count))
+    count=$(($count + 1))
+    if [ $count -lt $retries ]; then
+      echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+
+}

modules/tfe_init_replicated/templates/tfe_replicated.sh.tpl

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+${retry}
 ${get_base64_secrets}
 ${install_packages}
 ${install_monitoring_agents}