Merge pull request #479 from Azure/asg_nsg

Enforce Network Security Group and Application Security Group

.github/workflows/integration.yml

@@ -74,7 +74,7 @@ jobs:
           # Check if we need to build with dual protocol
           #dual_protocol=$(yq eval '.dual_protocol' config.yml | tr '[:upper:]' '[:lower:]')
           build_error=0
-          if [ "$ANF_DUAL" == "false" ]; then
+          if [ "$ANF_DUAL" = "false" ]; then
             set +e
             ./build.sh -a apply -f tf
             build_error=$?

.gitignore

@@ -7,6 +7,7 @@ config.yml
 packer/options.json
 packer/spn.json
 packer/*.log
+packer/md5sum.txt
 
 # Ansible auto-generated inventory
 playbooks/inventory

docs/deploy/how_to.md

@@ -44,7 +44,7 @@ network:
 ### Creating a standalone VNET for AZ-HOP
 There is a way to easily create a standalone VNET for **azhop** without doing a full deployment by following these steps :
 - Create a configuration file with all the required values for creating a VNET
-- run the build command specify the *tf/network* subdirectory `./build -a [plan, apply, destroy] -tf ./tf/network`
+- run the build command specify the *tf/network* subdirectory `./build -a [plan, apply, destroy] -f ./tf/network`
 - Save your config file and create a new one in which you now specify the VNET ID created above
 - Build your **azhop** environment
 

playbooks/create_user.yml

@@ -14,7 +14,6 @@
     state: present
     groups:
       - Domain Users
-      - Domain Admins
     attributes:
       uidNumber: "{{ user.uid }}"
       uid: "{{ user.name }}"

playbooks/ood-overrides.yml

@@ -11,11 +11,17 @@ clusters:
         host: scheduler
         exec: /opt/pbs
       batch_connect:
+        min_port: 61001
+        max_port: 61010
         basic:
+          min_port: 61001
+          max_port: 61010
           script_wrapper: |
             module purge
             %s
         vnc:
+          min_port: 61001
+          max_port: 61010
           script_wrapper: |
             module purge
             export PATH="/opt/TurboVNC/bin:$PATH"

playbooks/ood.yml

@@ -1,3 +1,10 @@
+# This looks crazy but in order for this playbook to run from a pipeline, the jumpbox dummy need to be added
+- name: jumpbox dummy
+  hosts: jumpbox
+  become: true
+  vars_files:
+    - '{{global_config_file}}'
+
 - hosts: ondemand
   #gather_facts: no
   become: true

playbooks/tests.yml

@@ -1,4 +1,11 @@
 ---
+# This looks crazy but in order for this playbook to run from a pipeline, the jumpbox dummy need to be added
+- name: jumpbox dummy
+  hosts: jumpbox
+  become: true
+  vars_files:
+    - '{{global_config_file}}'
+
 - name: Run test suite
   hosts: ondemand
   become: true

tf/active_directory/network_security_group.tf

@@ -0,0 +1 @@
+../network_security_group.tf

tf/ad.tf

@@ -38,3 +38,8 @@ resource "azurerm_windows_virtual_machine" "ad" {
   }
 }
 
+resource "azurerm_network_interface_application_security_group_association" "ad-asg-asso" {
+  for_each = toset(local.asg_associations["ad"])
+  network_interface_id          = azurerm_network_interface.ad-nic.id
+  application_security_group_id = local.create_vnet ? azurerm_application_security_group.asg[each.key].id : data.azurerm_application_security_group.asg[each.key].id
+}

tf/ccportal.tf

@@ -140,3 +140,8 @@ resource "azurerm_role_assignment" "ccportal_rg" {
   principal_id       = azurerm_user_assigned_identity.ccportal.principal_id
 }
 
+resource "azurerm_network_interface_application_security_group_association" "ccportal-asg-asso" {
+  for_each = toset(local.asg_associations["ccportal"])
+  network_interface_id          = azurerm_network_interface.ccportal-nic.id
+  application_security_group_id = local.create_vnet ? azurerm_application_security_group.asg[each.key].id : data.azurerm_application_security_group.asg[each.key].id
+}

tf/grafana.tf

@@ -37,3 +37,9 @@ resource "azurerm_linux_virtual_machine" "grafana" {
     version   = "latest"
   }
 }
+
+resource "azurerm_network_interface_application_security_group_association" "grafana-asg-asso" {
+  for_each = toset(local.asg_associations["grafana"])
+  network_interface_id          = azurerm_network_interface.grafana-nic.id
+  application_security_group_id = local.create_vnet ? azurerm_application_security_group.asg[each.key].id : data.azurerm_application_security_group.asg[each.key].id
+}

tf/jumpbox.tf

@@ -45,3 +45,9 @@ resource "azurerm_linux_virtual_machine" "jumpbox" {
     version   = "latest"
   }
 }
+
+resource "azurerm_network_interface_application_security_group_association" "jumpbox-asg-asso" {
+  for_each = toset(local.asg_associations["jumpbox"])
+  network_interface_id          = azurerm_network_interface.jumpbox-nic.id
+  application_security_group_id = local.create_vnet ? azurerm_application_security_group.asg[each.key].id : data.azurerm_application_security_group.asg[each.key].id
+}

tf/lustre.tf

@@ -53,6 +53,12 @@ resource "azurerm_linux_virtual_machine" "lustre" {
   }
 }
 
+resource "azurerm_network_interface_application_security_group_association" "lustre-asg-asso" {
+  for_each = toset(local.asg_associations["lustre"])
+  network_interface_id          = azurerm_network_interface.lustre-nic.id
+  application_security_group_id = local.create_vnet ? azurerm_application_security_group.asg[each.key].id : data.azurerm_application_security_group.asg[each.key].id
+}
+
 #
 # lustre OSS VMs
 #
@@ -123,6 +129,28 @@ resource "azurerm_key_vault_access_policy" "lustre-oss" {
   secret_permissions = [ "get", "list" ]
 }
 
+# Problem : How to generate associations for all OSS instances as we can't mix count and for_each ???
+# Solution : Use a combined flatten list
+locals {
+  # https://www.daveperrett.com/articles/2021/08/19/nested-for-each-with-terraform/
+  # Nested loop over both lists, and flatten the result.
+  lustre_oss_asgs = distinct(flatten([
+    for oss in range(0, local.lustre_oss_count) : [
+      for asg in local.asg_associations["lustre"] : {
+        oss = oss
+        asg = asg
+      }
+    ]
+  ]))
+}
+
+resource "azurerm_network_interface_application_security_group_association" "lustre-oss-asg-asso" {
+  # We need a map to use for_each, so we convert our list into a map by adding a unique key:
+  for_each = { for entry in local.lustre_oss_asgs: "${entry.oss}.${entry.asg}" => entry }
+  network_interface_id          = azurerm_network_interface.lustre-oss-nic[each.value.oss].id
+  application_security_group_id = local.create_vnet ? azurerm_application_security_group.asg[each.value.asg].id : data.azurerm_application_security_group.asg[each.value.asg].id
+}
+
 #
 # Robinhood VM
 #
@@ -168,3 +196,9 @@ resource "azurerm_linux_virtual_machine" "robinhood" {
       version   = local.lustre_image_reference.version
   }
 }
+
+resource "azurerm_network_interface_application_security_group_association" "robinhood-asg-asso" {
+  for_each = toset(local.asg_associations["robinhood"])
+  network_interface_id          = azurerm_network_interface.robinhood-nic.id
+  application_security_group_id = local.create_vnet ? azurerm_application_security_group.asg[each.key].id : data.azurerm_application_security_group.asg[each.key].id
+}

tf/network.tf

@@ -98,52 +98,3 @@ resource "azurerm_subnet" "compute" {
   service_endpoints    = ["Microsoft.Storage"]
 }
 
-# Network security group for the FrontEnd subnet
-resource "azurerm_network_security_group" "frontend" {
-  count                = local.create_vnet ? 1 : 0
-  name                = "frontendnsg"
-  location            = azurerm_resource_group.rg[0].location
-  resource_group_name = azurerm_resource_group.rg[0].name
-
-  security_rule {
-        name                       = "ssh-in-allow-22"
-        priority                   = "103"
-        direction                  = "Inbound"
-        access                     = "Allow"
-        protocol                   = "tcp"
-        source_port_range          = "*"
-        destination_port_range     = "22"
-        source_address_prefix      = "*"
-        destination_address_prefix = "VirtualNetwork"
-  }
-
-  security_rule {
-        name                       = "https-in-allow-443"
-        priority                   = "104"
-        direction                  = "Inbound"
-        access                     = "Allow"
-        protocol                   = "tcp"
-        source_port_range          = "*"
-        destination_port_range     = "443"
-        source_address_prefix      = "*"
-        destination_address_prefix = "VirtualNetwork"
-  }
-
-  security_rule {
-        name                       = "https-in-allow-80"
-        priority                   = "105"
-        direction                  = "Inbound"
-        access                     = "Allow"
-        protocol                   = "tcp"
-        source_port_range          = "*"
-        destination_port_range     = "80"
-        source_address_prefix      = "*"
-        destination_address_prefix = "VirtualNetwork"
-  }
-}
-
-resource "azurerm_subnet_network_security_group_association" "frontend" {
-  count                     = local.create_vnet ? 1 : 0
-  subnet_id                 = azurerm_subnet.frontend[count.index].id
-  network_security_group_id = azurerm_network_security_group.frontend[count.index].id
-}

tf/network/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "~> 2.41.0"
+      version = "~> 2.61.0"
     }
     random = {
       source  = "hashicorp/random"

tf/network/network_security_group.tf

@@ -0,0 +1 @@
+../network_security_group.tf