Merge pull request #5 from kpeder/feature/alb

Feature/alb

Makefile

@@ -13,7 +13,7 @@ help:
 	@echo '    aws_gitlab_install   Install Terraform, Terragrunt'
 	@echo '    aws_gitlab_lint      Run Go linters'
 	@echo '    aws_gitlab_plan      Show deployment plan'
-	@echo '    aws_gitlab_test      Run deployment tests'
+	@echo '    aws_gitlab_test      Run deployment tests and clean up (CI loop)'
 	@echo ''
 
 .PHONY: pre-commit
@@ -27,11 +27,11 @@ aws_gitlab_clean:
 
 .PHONY: aws_gitlab_configure
 aws_gitlab_configure:
-	@cd aws/gitlab && ./scripts/configure.sh -e demo -o kpeder -p us-east-2 -s us-west-2 -t devops
+	@cd aws/gitlab && ./scripts/configure.sh -d bytecount.net -e demo -o kpeder -p us-east-2 -s us-west-2 -t devops -z Z2OCSN1ZPHG5PO
 
 .PHONY: aws_gitlab_deploy
 aws_gitlab_deploy: aws_gitlab_configure aws_gitlab_init
-	@cd aws/gitlab/test && go test -v
+	@cd aws/gitlab/test && go test -v -timeout 20m
 
 .PHONY: aws_gitlab_init
 aws_gitlab_init: aws_gitlab_configure
@@ -54,4 +54,4 @@ aws_gitlab_plan: aws_gitlab_configure aws_gitlab_init
 
 .PHONY: aws_gitlab_test
 aws_gitlab_test: aws_gitlab_configure aws_gitlab_lint
-	@cd aws/gitlab/test && go test -v -destroy
+	@cd aws/gitlab/test && go test -v -destroy -timeout 20m

README.md

@@ -45,7 +45,51 @@ Targets:
     aws_gitlab_install   Install Terraform, Terragrunt
     aws_gitlab_lint      Run Go linters
     aws_gitlab_plan      Show deployment plan
-    aws_gitlab_test      Run deployment tests
+    aws_gitlab_test      Run deployment tests and clean up (CI loop)
 ```
 
-Note that additional targets can be added in order to configure multiple environments, for example to create development and production environments.
+Note that configuration targets will be specific to the target; particularly domain names, route53 zones and tags. Update these configuration values before deploying to your own environment by editing the aws_gitlab_configure target in the Makefile.
+
+Additional targets can be added in order to configure multiple environments, for example to create development and production environments.
+
+### GitLab Installation
+Installation is decoupled from the instance deployment so that it can be customized, since GitLab installation may contain many environment-specific configuration options, including default root password, external database connection, external object storage location, external authentication, email forwarding configuration, and much more. Follow the instructions below for a vanilla, stand-alone implementation of the product.
+
+1. Connect to the GitLab instance using the configured keypair.
+
+    Update the deployed public key file if necessary by editing the value of pubkey_file value in the following file. It defaults to the ssh-keygen default location.
+
+    ```aws/gitlab/reg-primary/keypairs/gitlab/inputs.yaml```
+
+    Use the associated private key directly or via an SSH agent (recommended), such as ssh-agent (Linux) or Pageant (with PuTTY).
+
+    Use the AWS CLI to connect via the instance connect endpoint.
+
+    ```$ aws ec2-instance-connect ssh --instance-id i-0ea992f180c12345 --os-user ubuntu```
+
+2. Once connected to the instance, follow the [instructions](https://about.gitlab.com/install/#ubuntu) for installing gitlab, generalized below.
+
+    Note that the package repository is already configured on the host. Update the example domain name to match the deployment zone.
+
+    ```$ sudo EXTERNAL_URL="http://gitlab.example.com" apt-get install gitlab-ee```
+
+3. Edit the configuration file to allow health checks from the local VPC range.
+
+    ```$ sudo vi /etc/gitlab/gitlab.rb```
+
+    Uncomment and edit the following line to add the VPC range (the default VPC range for this deployment is 172.16.0.0/16).
+
+    ```gitlab_rails['monitoring_whitelist] = ['127.0.0.1/8', '172.16.0.0/16']```
+
+    Then reconfigure GitLab.
+
+    ```$ sudo gitlab-ctl reconfigure```
+
+### Connection
+1. Copy the installation password from the GitLab instance.
+
+    ```$ sudo cat /etc/gitlab/initial_root_password```
+
+2. Connect to the endpoint using a browser and log in as user 'root' with the retrieved password. Update the example domain name to match the deployment zone.
+
+    ```https://gitlab.example.com/```

aws/gitlab/env.yaml

@@ -1,10 +1,13 @@
 ---
-environment: demo
+dns:
+  domain: "bytecount.net"
+  zone_id: "Z2OCSN1ZPHG5PO"
+environment: "demo"
 labels:
-  deployment: kped
-  environment: demo
-  owner: kpeder
-  team: devops
+  deployment: "kped"
+  environment: "demo"
+  owner: "kpeder"
+  team: "devops"
 
 dependencies:
   custom_vpc_dependency_path: "reg-primary/vpcs/custom"
@@ -51,6 +54,16 @@ dependencies:
     security_group_name: "kped-demo-custom-ice"
     security_group_vpc_id: "vpc-0d8148e657a7787f1"
 
+  gitlab_certificate_dependency_path: "global/certificates/gitlab"
+  gitlab_certificate_mock_outputs:
+    acm_certificate_arn: "arn:aws:acm:us-east-2:123456789012:certificate/a43d3a95-7632-433a-859a-ec2d42145b2b"
+
+  gitlab_instance_dependency_path: "reg-primary/instances/gitlab"
+  gitlab_instance_mock_outputs:
+    id: "i-123456789012"
+    private_ip: "172.16.12.24"
+    public_ip: ""
+
   gitlab_keypair_dependency_path: "reg-primary/keypairs/gitlab"
   gitlab_keypair_mock_outputs:
     key_pair_id: "key-0576e69c4b8faacc2"

aws/gitlab/global/albs/gitlab/inputs.yaml

@@ -0,0 +1,45 @@
+---
+deletion_protection: false
+dns:
+  - name: "gitlab"
+    ttl: 600
+    type: "A"
+egress:
+  all_tcp:
+    cidr_ipv4: "172.16.0.0/16"
+    from_port: 80
+    ip_protocol: "tcp"
+    to_port: 80
+ingress:
+  all_https:
+    cidr_ipv4: "0.0.0.0/0"
+    description: "HTTPS traffic"
+    from_port: 443
+    ip_protocol: "tcp"
+    to_port: 443
+labels: {}
+listeners:
+  https:
+    certificate_arn: ""
+    forward:
+      target_group_key: "gitlab"
+    port: 443
+    protocol: "HTTPS"
+name: "gitlab"
+targets:
+  gitlab:
+    create_attachment: true
+    health_check:
+      enabled: true
+      healthy_threshold: 2
+      interval: 30
+      matcher: "200"
+      path: "/-/health"
+      port: "traffic-port"
+      protocol: "HTTP"
+      timeout: 6
+      unhealthy_threshold: 3
+    name: "gitlab"
+    port: 80
+    protocol: "HTTP"
+    target_type: "instance"

aws/gitlab/global/albs/gitlab/remotestate.tf

@@ -0,0 +1,4 @@
+terraform {
+  # Intentionally unconfigured. Managed by Terragrunt.
+  backend "s3" {}
+}

aws/gitlab/global/albs/gitlab/terragrunt.hcl

@@ -0,0 +1,77 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("aws_gitlab_terragrunt.hcl")
+}
+
+# Resources should not be destroyed without careful consideration of effects
+prevent_destroy = false
+
+locals {
+  env      = yamldecode(file(find_in_parent_folders("env.yaml")))
+  inputs   = yamldecode(file("inputs.yaml"))
+  platform = fileexists(find_in_parent_folders("local.aws.yaml")) ? yamldecode(file(find_in_parent_folders("local.aws.yaml"))) : yamldecode(file(find_in_parent_folders("aws.yaml")))
+  versions = yamldecode(file(find_in_parent_folders("versions.yaml")))
+}
+
+dependency "custom_vpc" {
+  config_path  = find_in_parent_folders(local.env.dependencies.custom_vpc_dependency_path)
+  mock_outputs = local.env.dependencies.custom_vpc_mock_outputs
+
+  mock_outputs_allowed_terraform_commands = ["init", "plan", "validate"]
+}
+
+dependency "gitlab_certificate" {
+  config_path  = find_in_parent_folders(local.env.dependencies.gitlab_certificate_dependency_path)
+  mock_outputs = local.env.dependencies.gitlab_certificate_mock_outputs
+
+  mock_outputs_allowed_terraform_commands = ["init", "plan", "validate"]
+}
+
+dependency "gitlab_instance" {
+  config_path  = find_in_parent_folders(local.env.dependencies.gitlab_instance_dependency_path)
+  mock_outputs = local.env.dependencies.gitlab_instance_mock_outputs
+
+  mock_outputs_allowed_terraform_commands = ["init", "plan", "validate"]
+}
+
+terraform {
+  source = "git::git@github.com:terraform-aws-modules/terraform-aws-alb?ref=${local.versions.aws_module_alb}"
+}
+
+inputs = {
+  enable_deletion_protection = local.inputs.deletion_protection
+
+  name    = local.inputs.name
+  subnets = dependency.custom_vpc.outputs.public_subnets
+  vpc_id  = dependency.custom_vpc.outputs.vpc_id
+
+  listeners                    = merge({
+      for k, v in local.inputs.listeners:
+        k => merge(v,
+          {
+            certificate_arn = dependency.gitlab_certificate.outputs.acm_certificate_arn
+          }) if contains(keys(v), "certificate_arn")
+    },{
+      for k, v in local.inputs.listeners:
+        k => v if !contains(keys(v), "certificate_arn")
+    })
+  route53_records              = {
+      for v in local.inputs.dns: v.name => merge(v,
+        {
+          zone_id = local.env.dns.zone_id
+        })
+    }
+  security_group_egress_rules  = { for k, v in local.inputs.egress: k => v }
+  security_group_ingress_rules = { for k, v in local.inputs.ingress: k => v }
+  target_groups                = {
+    for k, v in local.inputs.targets: k => merge(v,
+    {
+      target_id = dependency.gitlab_instance.outputs.id
+    })
+  }
+
+  tags = merge(local.env.labels, local.inputs.labels)
+}

aws/gitlab/global/certificates/gitlab/inputs.yaml

@@ -0,0 +1,5 @@
+---
+labels: {}
+name: "gitlab"
+validation_method: "DNS"
+wait_for_validation: false

aws/gitlab/global/certificates/gitlab/remotestate.tf

@@ -0,0 +1,4 @@
+terraform {
+  # Intentionally unconfigured. Managed by Terragrunt.
+  backend "s3" {}
+}

aws/gitlab/global/certificates/gitlab/terragrunt.hcl

@@ -0,0 +1,34 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("aws_gitlab_terragrunt.hcl")
+}
+
+# Resources should not be destroyed without careful consideration of effects
+prevent_destroy = false
+
+locals {
+  env      = yamldecode(file(find_in_parent_folders("env.yaml")))
+  inputs   = yamldecode(file("inputs.yaml"))
+  platform = fileexists(find_in_parent_folders("local.aws.yaml")) ? yamldecode(file(find_in_parent_folders("local.aws.yaml"))) : yamldecode(file(find_in_parent_folders("aws.yaml")))
+  versions = yamldecode(file(find_in_parent_folders("versions.yaml")))
+}
+
+terraform {
+  source = "git::git@github.com:terraform-aws-modules/terraform-aws-acm?ref=${local.versions.aws_module_acm}"
+}
+
+inputs = {
+  domain_name         = local.env.dns.domain
+
+  subject_alternative_names = [
+    format("%s.%s", local.inputs.name, local.env.dns.domain)
+  ]
+
+  tags                = merge(local.env.labels, local.inputs.labels)
+  validation_method   = "DNS"
+  wait_for_validation = local.inputs.wait_for_validation
+  zone_id             = local.env.dns.zone_id
+}

aws/gitlab/reg-primary/instances/gitlab/inputs.yaml

@@ -4,4 +4,8 @@ labels: {}
 monitoring: false
 name: "gitlab"
 public_ip: false
-type: "c7i.large"
+type: "m7i.large"
+user_data: |
+  #!/bin/bash
+  apt-get update -y
+  curl https://packages.gitlab.com/install/repositories/gitlab/gitlab-ee/script.deb.sh | bash

aws/gitlab/reg-primary/instances/gitlab/terragrunt.hcl

@@ -50,7 +50,8 @@ inputs = {
   instance_type               = local.inputs.type
   key_name                    = dependency.gitlab_keypair.outputs.key_pair_name
   monitoring                  = local.inputs.monitoring
-  subnet_id                   = dependency.custom_vpc.outputs.public_subnets[0]
+  subnet_id                   = dependency.custom_vpc.outputs.private_subnets[0]
   tags                        = merge(local.env.labels, local.inputs.labels)
+  user_data                   = local.inputs.user_data
   vpc_security_group_ids      = tolist([dependency.gitlab_sg.outputs.security_group_id])
 }

aws/gitlab/reg-primary/sgs/gitlab/inputs.yaml

@@ -1,11 +1,15 @@
 ---
 description: "Security group for network access to the GitLab instance"
-egress_cidr_blocks: []
-egress_rules: []
+egress_cidr_blocks:
+  - "0.0.0.0/0"
+egress_rules:
+  - "http-80-tcp"
+  - "https-443-tcp"
 ingress_cidr_blocks: []
 # order rules lexically, to match output and support testing
 ingress_rules:
-  - "https-8443-tcp"
+  - "http-80-tcp"
+  - "https-443-tcp"
   - "ssh-tcp"
 labels: {}
 name: "gitlab"

aws/gitlab/scripts/configure.sh

@@ -9,19 +9,25 @@ function exit_with_msg {
 
 while [ $# -gt 0 ]; do
   case "${1}" in
+    -d|--dns-domain)
+      DNSDOMAIN="${2}"
+      shift
+      ;;
     -e|--environment)
       ENVIRONMENT="${2}"
       shift
       ;;
     -h|--help)
       echo "Usage:"
       echo "$0 \\"
+      echo "  -d|--dns-domain <domain_name>"
       echo "  -e|--environment <environment_name>"
       echo " [-h|--help]"
       echo "  -o|--owner <owner>"
       echo "  -p|--primaryregion <primary_region>"
       echo "  -s|--secondaryregion <secondary_region>"
       echo "  -t|--team <team>"
+      echo "  -z|--route53-zone-id <route53_zone_id>"
       exit 0
       ;;
     -o|--owner)
@@ -40,6 +46,10 @@ while [ $# -gt 0 ]; do
       TEAM="${2}"
       shift
       ;;
+    -z|--route53-zone-id)
+      DNSZONEID="${2}"
+      shift
+      ;;
     *)
       exit_with_msg "Error: Invalid argument '${1}'."
   esac
@@ -55,13 +65,17 @@ fi
 [[ -z ${PREFIX} ]] && exit_with_msg "Can't locate deployment prefix. Exiting."
 [[ ${#PREFIX} > 5 ]] && exit_with_msg "Prefix '${PREFIX}' is too long. Exiting."
 
+[[ -z ${DNSDOMAIN} ]] && exit_with_msg "-d|--dns-domain is a required parameter. Exiting."
 [[ -z ${ENVIRONMENT} ]] && exit_with_msg "-e|--environment is a required parameter. Exiting."
 [[ -z ${OWNER} ]] && exit_with_msg "-o|--owner is a required parameter. Exiting."
 [[ -z ${PREGION} ]] && exit_with_msg "-p|--primaryregion is a required parameter. Exiting."
 [[ -z ${SREGION} ]] && exit_with_msg "-s|--secondaryregion is a required parameter. Exiting."
 [[ -z ${TEAM} ]] && exit_with_msg "-t|--team is a required parameter. Exiting."
+[[ -z ${DNSZONEID} ]] && exit_with_msg "-z|--route53-zone-id is a required parameter. Exiting."
 
 echo "Deployment Owner: ${OWNER}"
+echo "DNS Domain: ${DNSDOMAIN}"
+echo "DNS Zone ID: ${DNSZONEID}"
 echo "Environment: ${ENVIRONMENT}"
 echo "Name Prefix: ${PREFIX}"
 echo "Primary Region: ${PREGION}"
@@ -72,6 +86,8 @@ cp templates/env.tpl env.yaml
 cp templates/region.tpl reg-primary/region.yaml
 cp templates/region.tpl reg-secondary/region.yaml
 
+sed -i -e "s:DNSDOMAIN:${DNSDOMAIN}:g" env.yaml
+sed -i -e "s:DNSZONEID:${DNSZONEID}:g" env.yaml
 sed -i -e "s:ENVIRONMENT:${ENVIRONMENT}:g" env.yaml
 sed -i -e "s:OWNER:${OWNER}:g" env.yaml
 sed -i -e "s:PREFIX:${PREFIX}:g" env.yaml