Too Many Requests
+ + + EOT + content_type = "text/html" + status = 429 + } + } + + request_setting { + action = null + bypass_busy_wait = false + force_ssl = true + max_stale_age = 86400 + name = "Default cache policy" + xff = "append" + } + request_setting { + action = "pass" + bypass_busy_wait = false + force_ssl = false + max_stale_age = 60 + name = "Force Pass" + request_condition = "Uncacheable URLs" + xff = "append" + } + + response_object { + name = "www redirect" + request_condition = "apex" + response = "Moved Permanently" + status = 301 + } + response_object { + content_type = "text/html" + name = "Generated by IP block list" + request_condition = "Generated by IP block list" + response = "Forbidden" + status = 403 + } + + force_destroy = true +} diff --git a/infra/cdn/providers.tf b/infra/cdn/providers.tf new file mode 100644 index 000000000..2346f041f --- /dev/null +++ b/infra/cdn/providers.tf @@ -0,0 +1,4 @@ +provider "fastly" { + alias = "cdn" + api_key = var.FASTLY_API_KEY +} diff --git a/infra/cdn/variables.tf b/infra/cdn/variables.tf new file mode 100644 index 000000000..da28835cd --- /dev/null +++ b/infra/cdn/variables.tf @@ -0,0 +1,39 @@ +variable "fastly_key" { + type = string + description = "API key for the Fastly VCL edge configuration." +} +variable "fastly_header_token" { + description = "Fastly header token ensure we only allow Fastly to access the service" + type = string + sensitive = true +} +variable "datadog_key" { + type = string + description = "API key for Datadog logging" + sensitive = true +} +variable "fastly_s3_logging" { + type = string + description = "S3 bucket keys for Fastly logging" + sensitive = true +} +variable "name" { + type = string + description = "The name of the Fastly service." +} +variable "domain" { + type = string + description = "The domain name of the service." +} +variable "extra_domains" { + type = list(string) + description = "Extra domains to add to the service." +} +variable "backend_address" { + type = string + description = "The hostname of the backend service." +} +variable "default_ttl" { + type = number + description = "The default TTL for the service." +} \ No newline at end of file diff --git a/infra/cdn/versions.tf b/infra/cdn/versions.tf new file mode 100644 index 000000000..da9c01f79 --- /dev/null +++ b/infra/cdn/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + fastly = { + source = "fastly/fastly" + version = "5.13.0" + } + } +} diff --git a/infra/config.tf b/infra/config.tf new file mode 100644 index 000000000..65b1a5210 --- /dev/null +++ b/infra/config.tf @@ -0,0 +1,9 @@ +# Connect us to TF Cloud for remote deploys +terraform { + cloud { + organization = "psf" + workspaces { + name = "pythondotorg-infra" + } + } +} diff --git a/infra/dns/README.md b/infra/dns/README.md new file mode 100644 index 000000000..7d98749e1 --- /dev/null +++ b/infra/dns/README.md @@ -0,0 +1,48 @@ +# DNS Module + +This Terraform module sets up AWS Route53 resources for managing DNS records, +supporting both apex domains and subdomains with Fastly CDN integration. + +## Features + +- Creates Route53 hosted zones for primary and user content (subdomain) domains +- Supports both apex domains and subdomains +- Automatically configures A/AAAA records for apex domains or CNAME for subdomains +- Integrates with Fastly CDN endpoints +- Provides a consistent set of name servers across hosted zones +- Allows addition of TXT records to the apex domain for verification purposes + +## Usage + +```hcl +module "CoolPythonWebsite" { + source = "./dns" + + primary_domain = "CoolPythonWebsite.com" + user_content_domain = "users.CoolPythonWebsite.com" + apex_txt = ["v=spf1 include:_spf.example.com ~all"] + tags = { + Environment = "Staging" + Project = "CoolPythonWebsite" + } +} +``` + +## Input Variables + +- `primary_domain`: Your main domain (e.g., "CoolPythonWebsite.com") +- `user_content_domain`: Domain for user-generated content (e.g., "users.CoolPythonWebsite.com") +- `apex_txt`: List of TXT records to add to the apex domain +- `tags`: Map of tags to apply to all created resources + +## Outputs + +- `primary_zone_id`: The Zone ID of the primary hosted zone +- `nameservers`: The set of name servers for your domains +- `user_content_zone_id`: The Zone ID of the user content hosted zone + +## Requirements + +Tested on +- Terraform 1.8.5 +- AWS provider ~> 5.0 diff --git a/infra/dns/main.tf b/infra/dns/main.tf new file mode 100644 index 000000000..9cdb28c83 --- /dev/null +++ b/infra/dns/main.tf @@ -0,0 +1,73 @@ +# This Terraform configuration sets up AWS Route53 resources +# ? TODO: It may be nice to allow a var when sourcing this module to +# force destroy/allow overwriting of existing resources + +# Retrieve the current AWS account data (based on secrets provided in .tfvars or TF Cloud) +data "aws_caller_identity" "current" {} + + + +# see if we're dealing with an apex domain or subdomain by splitting the domain name and counting the parts +locals { + apex_domain = length(split(".", var.domain)) > 2 ? false : true +} + +# Create a reusable set of nameservers +# This ensures consistent NS across multiple hosted zones +resource "aws_route53_delegation_set" "ns" {} + +resource "aws_route53_zone" "primary" { + name = var.domain + delegation_set_id = aws_route53_delegation_set.ns.id + tags = var.tags +} + +resource "aws_route53_zone" "user_content" { + name = var.user_content_domain + delegation_set_id = aws_route53_delegation_set.ns.id + tags = var.tags +} + +resource "aws_route53_record" "apex_txt" { + zone_id = aws_route53_zone.primary.zone_id + name = var.primary_domain + type = "TXT" + ttl = 3600 + records = var.apex_txt +} + +# Create the main DNS record, automatically choosing between A (for apex) and CNAME (for subdomain) +# depending on the value of `local.apex_domain` above +# This will point to our Fastly CDN endpoints +resource "aws_route53_record" "primary" { + zone_id = var.zone_id + name = var.domain + type = local.apex_domain ? "A" : "CNAME" + ttl = 86400 + records = var.fastly_endpoints[join("_", concat([var.domain_map[var.domain]], [local.apex_domain ? "A" : "CNAME"]))] +} + +# Same as above, but also create an AAAA record for IPv6 +resource "aws_route53_record" "primary-ipv6" { + count = local.apex_domain ? 1 : 0 + zone_id = var.zone_id + name = var.domain + type = "AAAA" + ttl = 86400 + records = var.fastly_endpoints[join("_", [var.domain_map[var.domain], "AAAA"])] +} + +# Output things we need to ref. elsewhere +output "primary_zone_id" { + value = aws_route53_record.primary.zone_id + description = "The Zone ID of our primary hosted zone." +} + +output "nameservers" { + value = aws_route53_delegation_set.ns.name_servers + description = "The set of name servers for our domains." +} + +output "user_content_zone_id" { + value = aws_route53_zone.user_content.zone_id +} \ No newline at end of file diff --git a/infra/dns/providers.tf b/infra/dns/providers.tf new file mode 100644 index 000000000..92caf800d --- /dev/null +++ b/infra/dns/providers.tf @@ -0,0 +1,6 @@ +provider "aws" { + alias = "dns" + region = "us-east-2" + access_key = var.aws_access_key + secret_key = var.aws_secret_key +} diff --git a/infra/dns/variables.tf b/infra/dns/variables.tf new file mode 100644 index 000000000..35133bef9 --- /dev/null +++ b/infra/dns/variables.tf @@ -0,0 +1,66 @@ +variable "aws_access_key" { + type = string + description = "Access key for the AWS account." + sensitive = true +} + +variable "aws_secret_key" { + type = string + description = "Secret access key for the AWS account." + sensitive = true +} + +variable "route53_zone_id" { + type = string + description = "The Route 53 hosted zone ID" + default = "Z3JUI7A2G39FQL" # python.org +} + +variable "route53_record_name" { + type = string + description = "The name of the CNAME record" + default = "test.python.org" +} + +variable "route53_record_ttl" { + type = number + description = "The TTL for the CNAME record" + default = 60 +} + +variable "tags" { + type = map(any) + description = "Tags to apply to all resources" +} +variable "primary_domain" { + type = string + description = "The primary domain name" +} +variable "user_content_domain" { + type = string + description = "The user content (sub)domain name" +} +variable "apex_txt" { + type = list(any) + description = "The TXT record for the apex domain" +} +variable "name" { + type = string + description = "The name of the Fastly service" +} +variable "zone_id" { + type = string + description = "The Route 53 hosted zone ID" +} +variable "domain" { + type = string + description = "The domain name of the service" +} +variable "fastly_endpoints" { + type = map(any) + description = "The Fastly endpoints" +} +variable "domain_map" { + type = map(any) + description = "The domain map" +} \ No newline at end of file diff --git a/infra/dns/versions.tf b/infra/dns/versions.tf new file mode 100644 index 000000000..1ac87da57 --- /dev/null +++ b/infra/dns/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 000000000..65acb05c3 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,73 @@ +locals { + tags = { + Application = "Python.org" + Environment = "Production" + } +} + +locals { + fastly_endpoints = { + "python.map.fastly.net_A" = ["151.101.128.223", "151.101.192.223", "151.101.0.223", "151.101.64.223"] + "python.map.fastly.net_AAAA" = ["2a04:4e42:200::223", "2a04:4e42:400::223", "2a04:4e42:600::223", "2a04:4e42::223"] + "python.map.fastly.net_CNAME" = ["dualstack.python.map.fastly.net"] + } + domain_map = { + "python.org" = "python.map.fastly.net" + "test.python.org" = "python.map.fastly.net" + } +} + +module "dns" { + # TODO: this doesn't accommodate for DNS management splits between environments + source = "./dns" + tags = local.tags + primary_domain = "python.org" + zone_id = module.dns.primary_zone_id + fastly_endpoints = local.fastly_endpoints + domain_map = local.domain_map + + aws_access_key = var.AWS_ACCESS_KEY_ID + aws_secret_key = var.AWS_SECRET_ACCESS_KEY + + # TODO: the below needs to be parameterized or fixed + apex_txt = [] + domain = "" + name = "" + user_content_domain = "" +} + +module "fastly_production" { + source = "./cdn" + + name = "Python.org" + domain = "python.org" + extra_domains = ["www.python.org"] + backend_address = "pythondotorg.ingress.us-east-2.psfhosted.computer" + default_ttl = 3600 + + datadog_key = var.DATADOG_API_KEY + fastly_key = var.FASTLY_API_KEY + fastly_header_token = var.FASTLY_HEADER_TOKEN + fastly_s3_logging = var.fastly_s3_logging +} + +module "fastly_staging" { + source = "./cdn" + + name = "test.Python.org" + domain = "test.python.org" + extra_domains = [] + backend_address = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" + default_ttl = 3600 + + datadog_key = var.DATADOG_API_KEY + fastly_key = var.FASTLY_API_KEY + fastly_header_token = var.FASTLY_HEADER_TOKEN + fastly_s3_logging = var.fastly_s3_logging +} + + +# module "ngwaf" { +# source = "./ngwaf" +# +# } diff --git a/infra/ngwaf/README.md b/infra/ngwaf/README.md new file mode 100644 index 000000000..9d01f4ced --- /dev/null +++ b/infra/ngwaf/README.md @@ -0,0 +1,4 @@ +# Fastly's NGWAF + +This module sets up the Fastly Next-Gen Web Application Firewall (NGWAF) for our Fastly services +related to python.org / test.python.org. diff --git a/infra/ngwaf/main.tf b/infra/ngwaf/main.tf new file mode 100644 index 000000000..288c48309 --- /dev/null +++ b/infra/ngwaf/main.tf @@ -0,0 +1,149 @@ +# ? TODO: this needs reorgnization :P, was jsut trying to yank out NGWAF stuff from main CDN module +# the goal is to let cdn apply the core config, and then ingest the service ID from that +# and apply ngwaf stuff on top THEN set up signal sciences stuff on top of that + + +# ! TODO: refactor this to be dynamic depending on {test,prod}.python.org +resource "fastly_service_vcl" "ngwaf_service" { + name = var.service_name + activate = true + + domain { + name = var.USER_VCL_SERVICE_DOMAIN_NAME + comment = "NGWAF domain" + } + + backend { + address = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + name = "ngwaf_backend" + port = 443 + use_ssl = true + ssl_cert_hostname = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + ssl_sni_hostname = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + override_host = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + } + + # NGWAF Dynamic Snippets + dynamicsnippet { + name = "ngwaf_config_init" + type = "init" + priority = 0 + } + + dynamicsnippet { + name = "ngwaf_config_miss" + type = "miss" + priority = 9000 + } + + dynamicsnippet { + name = "ngwaf_config_pass" + type = "pass" + priority = 9000 + } + + dynamicsnippet { + name = "ngwaf_config_deliver" + type = "deliver" + priority = 9000 + } + + dictionary { + name = var.Edge_Security_dictionary + } + + product_enablement { + bot_management = true + } + + lifecycle { + ignore_changes = [product_enablement] + } +} + +output "ngwaf_service_id" { + value = fastly_service_vcl.ngwaf_service.id +} + +# Fastly Service Dictionary Items +resource "fastly_service_dictionary_items" "edge_security_dictionary_items" { + for_each = { + for d in fastly_service_vcl.ngwaf_service.dictionary : d.name => d if d.name == var.Edge_Security_dictionary + } + service_id = fastly_service_vcl.ngwaf_service.id + dictionary_id = each.value.dictionary_id + items = { + Enabled : "100" + } +} + +# Fastly Service Dynamic Snippet Contents +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_init" { + for_each = { + for d in fastly_service_vcl.ngwaf_service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_init" + } + service_id = fastly_service_vcl.ngwaf_service.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_init" + manage_snippets = false +} + +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_miss" { + for_each = { + for d in fastly_service_vcl.ngwaf_service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_miss" + } + service_id = fastly_service_vcl.ngwaf_service.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_miss" + manage_snippets = false +} + +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_pass" { + for_each = { + for d in fastly_service_vcl.ngwaf_service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_pass" + } + service_id = fastly_service_vcl.ngwaf_service.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_pass" + manage_snippets = false +} + +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_deliver" { + for_each = { + for d in fastly_service_vcl.ngwaf_service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_deliver" + } + service_id = fastly_service_vcl.ngwaf_service.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_deliver" + manage_snippets = false +} + +# NGWAF Edge Deployment +resource "sigsci_edge_deployment" "ngwaf_edge_site_service" { + site_short_name = var.NGWAF_SITE +} + +resource "sigsci_edge_deployment_service" "ngwaf_edge_service_link" { + site_short_name = var.NGWAF_SITE + fastly_sid = fastly_service_vcl.ngwaf_service.id + activate_version = true + percent_enabled = 100 + depends_on = [ + sigsci_edge_deployment.ngwaf_edge_site_service, + fastly_service_vcl.ngwaf_service, + fastly_service_dictionary_items.edge_security_dictionary_items, + fastly_service_dynamic_snippet_content.ngwaf_config_init, + fastly_service_dynamic_snippet_content.ngwaf_config_miss, + fastly_service_dynamic_snippet_content.ngwaf_config_pass, + fastly_service_dynamic_snippet_content.ngwaf_config_deliver, + ] +} + +resource "sigsci_edge_deployment_service_backend" "ngwaf_edge_service_backend_sync" { + site_short_name = var.NGWAF_SITE + fastly_sid = fastly_service_vcl.ngwaf_service.id + fastly_service_vcl_active_version = fastly_service_vcl.ngwaf_service.active_version + depends_on = [ + sigsci_edge_deployment_service.ngwaf_edge_service_link, + ] +} \ No newline at end of file diff --git a/infra/ngwaf/providers.tf b/infra/ngwaf/providers.tf new file mode 100644 index 000000000..c4a34abff --- /dev/null +++ b/infra/ngwaf/providers.tf @@ -0,0 +1,7 @@ +provider "sigsci" { + alias = "firewall" + corp = var.NGWAF_CORP + email = var.NGWAF_EMAIL + auth_token = var.NGWAF_TOKEN + fastly_api_key = var.FASTLY_API_KEY +} diff --git a/infra/ngwaf/variables.tf b/infra/ngwaf/variables.tf new file mode 100644 index 000000000..0cab70d93 --- /dev/null +++ b/infra/ngwaf/variables.tf @@ -0,0 +1,46 @@ +variable "service_name" { + type = string +} + +variable "USER_VCL_SERVICE_DOMAIN_NAME" { + type = string + description = "The domain name of the service." +} + +variable "USER_VCL_SERVICE_BACKEND_HOSTNAME" { + type = string + description = "The hostname of the backend service." +} + +variable "Edge_Security_dictionary" { + type = string + description = "The dictionary name for the Edge Security product." +} + +variable "NGWAF_CORP" { + type = string + description = "Corp name for NGWAF" + default = "python" +} + +variable "NGWAF_SITE" { + type = string + description = "Site SHORT name for NGWAF" + default = "test" +} + +variable "NGWAF_EMAIL" { + type = string + description = "Email address associated with the token for the NGWAF API." +} + +variable "NGWAF_TOKEN" { + type = string + description = "Secret token for the NGWAF API." + sensitive = true +} + +variable "FASTLY_API_KEY" { + type = string + description = "API key for the Fastly VCL edge configuration." +} diff --git a/infra/ngwaf/versions.tf b/infra/ngwaf/versions.tf new file mode 100644 index 000000000..f8c137ba6 --- /dev/null +++ b/infra/ngwaf/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + fastly = { + source = "fastly/fastly" + version = "5.13.0" + } + sigsci = { + source = "signalsciences/sigsci" + version = "3.3.0" + } + } +} diff --git a/infra/out.tf b/infra/out.tf new file mode 100644 index 000000000..bc969b4ca --- /dev/null +++ b/infra/out.tf @@ -0,0 +1,16 @@ +output "testing-the_ngwaf" { + value = <