Merge pull request #736 from Azure/multi-cloud-support

Added multi-cloud support for Lustre

build.sh

@@ -149,6 +149,39 @@ fi
 export TF_VAR_CreatedBy=${logged_user_upn}
 echo "terraform -chdir=$TF_FOLDER $TF_COMMAND $PARAMS"
 
+# Retrieve on which cloud environment we run on
+# env details from here: https://github.com/hashicorp/terraform-provider-azurerm/blob/main/vendor/github.com/Azure/go-autorest/autorest/azure/environments.go
+# TODO: check az environmentName for AzureChina and AzureGerman
+cloud_env="Public"
+account_env=$(az account show | jq '.environmentName' -r)
+case "$account_env" in
+  AzureChinaCloud)
+    export TF_VAR_AzureEnvironment=AZURECHINACLOUD
+    export TF_VAR_KeyVaultSuffix=vault.azure.cn
+    export TF_VAR_BlobStorageSuffix=blob.core.chinacloudapi.cn
+    ;;
+  AzureGermanCloud)
+    export TF_VAR_AzureEnvironment=AZUREGERMANCLOUD
+    export TF_VAR_KeyVaultSuffix=vault.microsoftazure.de
+    export TF_VAR_BlobStorageSuffix=blob.core.cloudapi.de
+    ;;
+  AzureCloud)
+    export TF_VAR_AzureEnvironment=AZUREPUBLICCLOUD
+    export TF_VAR_KeyVaultSuffix=vault.azure.net
+    export TF_VAR_BlobStorageSuffix=blob.core.windows.net
+    ;;
+  AzureUSGovernment)
+    export TF_VAR_AzureEnvironment=AZUREUSGOVERNMENTCLOUD
+    export TF_VAR_KeyVaultSuffix=vault.usgovcloudapi.net
+    export TF_VAR_BlobStorageSuffix=blob.core.usgovcloudapi.net
+    ;;
+  *)
+    echo "ERROR: Unknown Azure environment ${account_env}"
+    exit 1
+    ;;
+esac
+
+
 # -parallelism=30
 TF_LOG="TRACE"
 TF_LOG_PATH="$THIS_DIR/tf/terraform.log"

playbooks/lustre.yml

@@ -4,6 +4,21 @@
   hosts: jumpbox
   become: true
 
+- name: Ensure Lemur version
+  hosts: lustre-oss-*:robinhood
+  become: true
+  gather_facts: no
+  vars_files:
+    - '{{global_config_file}}'
+  tasks:
+  - name: install Lemur if not the required version
+    shell: |
+      min_ver="2.1.2"
+      ver=$(rpm -q --qf "%{VERSION}" lemur-azure-hsm-agent)
+      if [ $? -gt 0 ] || [ "$ver" != "$(printf "$min_ver\n$ver" | sort -t '.' -k 1,1 -k 2,2 -k 3,3 -g | tail -n1)" ]; then
+        yum -y install https://github.com/edwardsp/lemur/releases/download/2.1.2/lemur-azure-data-movers-${min_ver}-lustre_2.12.x86_64.rpm https://github.com/edwardsp/lemur/releases/download/2.1.2/lemur-azure-hsm-agent-${min_ver}-lustre_2.12.x86_64.rpm
+      fi
+
 - name: Lustre MDS setup
   hosts: lustre
   become: true
@@ -132,8 +147,10 @@
       mode: 0600
       content: |
         num_threads=16
+        az_storage_suffix="{{ blob_storage_suffix }}"
         az_storage_account="{{ lustre_hsm_storage_account }}"
         az_kv_name="{{ key_vault }}"
+        az_kv_suffix="{{ key_vault_suffix }}"
         az_kv_secret_name="lustre-{{ lustre_hsm_storage_account }}-{{ lustre_hsm_storage_container }}-sas"
         region="westeurope"
         bandwidth=0
@@ -160,6 +177,7 @@
         ExecStartPre=/bin/mkdir -p /var/run/lhsmd
         ExecStart=/sbin/lhsmd -config /etc/lhsmd/agent
         Restart=always
+        Environment=AZURE_ENVIRONMENT={{ azure_environment }}
         [Install]
         WantedBy=multi-user.target
   - name: systemd daemon reload
@@ -208,23 +226,11 @@
     stat:
       path: /lustre.hydrated
     register: stat_hydrated
-  - name: Read Password from KV
-    command: |
-      az keyvault secret show
-      --vault-name {{ key_vault }}
-      --name lustre-{{ lustre_hsm_storage_account }}-{{ lustre_hsm_storage_container }}-sas
-      --query "value"
-      -o tsv
-    delegate_to: localhost
-    connection: local
-    register: lustre_sas
-    become: false
-    when: not stat_hydrated.stat.exists
   - name: hydrate lustre
     shell: |
-      export STORAGE_SAS="{{ lustre_sas.stdout }}"
+      export AZURE_ENVIRONMENT={{ azure_environment }}
       cd /lustre
-      azure-import -account "{{ lustre_hsm_storage_account }}" -container "{{ lustre_hsm_storage_container }}"
+      azure-import -account "{{ lustre_hsm_storage_account }}" -container "{{ lustre_hsm_storage_container }}" -suffix "{{ blob_storage_suffix }}" -kvname "{{ key_vault }}" -kvsuffix "{{ key_vault_suffix }}" -kvsecret "lustre-{{ lustre_hsm_storage_account }}-{{ lustre_hsm_storage_container }}-sas"
       touch /lustre.hydrated
     when: not stat_hydrated.stat.exists
 
@@ -734,8 +740,9 @@
 
         [Service]
         Type=simple
-        ExecStart=/sbin/changelog-reader -account "{{ lustre_hsm_storage_account }}" -container "{{ lustre_hsm_storage_container }}" -kvname "{{ key_vault }}" -kvsecret "lustre-{{ lustre_hsm_storage_account }}-{{ lustre_hsm_storage_container }}-sas" -mdt LustreFS-MDT0000 -userid cl2
+        ExecStart=/sbin/changelog-reader -account "{{ lustre_hsm_storage_account }}" -suffix "{{ blob_storage_suffix }}" -container "{{ lustre_hsm_storage_container }}" -kvname "{{ key_vault }}" -kvsuffix "{{ key_vault_suffix }}" -kvsecret "lustre-{{ lustre_hsm_storage_account }}-{{ lustre_hsm_storage_container }}-sas" -mdt LustreFS-MDT0000 -userid cl2
         Restart=always
+        Environment=AZURE_ENVIRONMENT={{ azure_environment }}
 
         [Install]
         WantedBy=multi-user.target

tf/outputs.tf

@@ -22,6 +22,9 @@ resource "local_file" "AnsibleInventory" {
 resource "local_file" "global_variables" {
   sensitive_content = templatefile("${local.playbooks_template_dir}/global_variables.tmpl",
     {
+      azure_environment   = local.azure_environment
+      key_vault_suffix    = local.key_vault_suffix
+      blob_storage_suffix = local.blob_storage_suffix
       admin_username      = local.admin_username
       ssh_public_key      = tls_private_key.internal.public_key_openssh
       cc_storage          = azurerm_storage_account.azhop.name

tf/templates/global_variables.tmpl

@@ -19,3 +19,6 @@ lustre_hsm_storage_account    : ${lustre_hsm_storage_account}
 lustre_hsm_storage_container  : ${lustre_hsm_storage_container}
 slurmdb_fqdn                  : ${mysql-fqdn}
 slurmdb_user                  : ${mysql-user}
+azure_environment             : ${azure_environment}
+key_vault_suffix              : ${key_vault_suffix}
+blob_storage_suffix           : ${blob_storage_suffix}

tf/variables.tf

@@ -1,3 +1,15 @@
+variable AzureEnvironment {
+  default = "AZUREPUBLICCLOUD"
+}
+
+variable KeyVaultSuffix {
+  default = "vault.azure.net"
+}
+
+variable BlobStorageSuffix {
+  default = "blob.core.windows.net"
+}
+
 variable CreatedBy {
   default = ""
 }

tf/variables_local.tf

@@ -1,4 +1,9 @@
 locals {
+    # azure environment
+    azure_environment = var.AzureEnvironment
+    key_vault_suffix = var.KeyVaultSuffix
+    blob_storage_suffix = var.BlobStorageSuffix
+
     # azurerm_client_config contains empty values for Managed Identity so use variables instead
     tenant_id = var.tenant_id
     logged_user_objectId = var.logged_user_objectId