From c33f3e550cdbc083a5fd53c7d56e9bb19de9cfc2 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Fri, 23 Aug 2024 12:18:06 -0500 Subject: [PATCH 01/20] feat: testing the waf --- infra/.gitignore | 15 +++++ infra/.terraform.lock.hcl | 69 +++++++++++++++++++++ infra/aws.tf | 7 +++ infra/fastly.tf | 107 +++++++++++++++++++++++++++++++++ infra/main.tf | 34 +++++++++++ infra/ngwaf.tf | 29 +++++++++ infra/out.tf | 16 +++++ infra/terraform.tfvars.example | 5 ++ infra/variables.tf | 79 ++++++++++++++++++++++++ 9 files changed, 361 insertions(+) create mode 100644 infra/.gitignore create mode 100644 infra/.terraform.lock.hcl create mode 100644 infra/aws.tf create mode 100644 infra/fastly.tf create mode 100644 infra/main.tf create mode 100644 infra/ngwaf.tf create mode 100644 infra/out.tf create mode 100644 infra/terraform.tfvars.example create mode 100644 infra/variables.tf diff --git a/infra/.gitignore b/infra/.gitignore new file mode 100644 index 000000000..e61ff928d --- /dev/null +++ b/infra/.gitignore @@ -0,0 +1,15 @@ + +**/.terraform/* +*.tfstate +*.tfstate.* +crash.log +crash.*.log +*.tfvars +*.tfvars.json +override.tf +override.tf.json +*_override.tf +*_override.tf.json +.terraform.tfstate.lock.info +.terraformrc +terraform.rc \ No newline at end of file diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl new file mode 100644 index 000000000..c3317831b --- /dev/null +++ b/infra/.terraform.lock.hcl @@ -0,0 +1,69 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/fastly/fastly" { + version = "5.12.0" + constraints = ">= 3.0.4" + hashes = [ + "h1:y2qTbSz7wIn6wsAHEGmD2qtP6pp420s3kt7+D0n9moc=", + "zh:1b2175821acf437a070b8d33af04a12c4bafb0dd2da767de027d894f6b8a7e0e", + "zh:1e6a95f29f8aaf292c882d69761a266bf948fa9a9433f9fa4d60933ace2ed03a", + "zh:1ef8578a764ea8a85c9d0d72ccf82e211fcd70740ae0b1f03e70952069aafced", + "zh:4c1fba8f37aee1f6708b93891dbc36042a517c3476ca5b208c1d926d3304eadf", + "zh:4d3792115edd259f98732950b8259b7b460eeba8c6141e70424d42699e61c729", + "zh:6f85ecd31b1a6cfb5cd99c1a5b7426519eac065baa6e8a7bf0df7ce17d763e43", + "zh:6ffdf5953efc0f5c5dd21395d545ebdbb9ddcf3821fad89137febba079f49752", + "zh:76eb53907f353a2faa1dc952a2fa085bcd716cf3a7271c87ee46c25588bedf94", + "zh:7dd738afdfa6112d1992130a1285c73b157fe0dba7110c796a247e7328f027b2", + "zh:8c54e5fffd47443d018d792b10ab1425916bfb822416179c6fa3847275023ff1", + "zh:96cbff688e4c47530be9743b5206eb61bdecd6b15459d1a6cf727b1301172331", + "zh:9fbdf427058e90463ce9485ad5ae15605cbe2f306d5eeff9b2c9ea3b9cc5a186", + "zh:aa21ca195fe08e7e0b55515ba78551b7cd6237bd5f111fff3fa0fcfcc814cd6e", + "zh:dc24a157763e401629c25d71a9de3c0674bf9119adf5d4af988b39a7f442e181", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.64.0" + constraints = "~> 5.0" + hashes = [ + "h1:YH4I78rsS9t+YoGMPNzrM53aWi0Rb9Nud16iusrSXMg=", + "zh:1d361f8062c68c9d5ac14b0aa8390709542129b8a9b258e61bbbabc706078b44", + "zh:39dcbf53e3896bdd77071384c8fad4a5862c222c73f3bcf356aca488101f22fd", + "zh:3fad63505f0c5b6f01cc9a6ef02b2226983b79424126a9caf6eb724f654299f4", + "zh:53a8b90d00829cc27e3171a13a8ff1404ee0ea018e73f31d3f916d246cc39613", + "zh:5734c25ef5a04b40f3c1ac5f817f11e42ee3328f74dbc141c0e64afbb0acc834", + "zh:66ea14dbd87f291ce4a877123363933d3ca4022f209f885807a6689c22c24e80", + "zh:68e79654ad0894a3d93134c3377748ace3058d5fad5ec09d1e9a8f8f9b8a47ea", + "zh:7b74259d0ceef0c49cea6bcd171df997b6bad141085bbadded15b440faeb0eee", + "zh:988ebfb5d115dc57070b5abf2e4200ad49cde535f27fd2ba5e34cf9ab336a57f", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a0a2d4efe2835f0101a0a5024e044a3f28c00e10a8d87fce89c707ef6db75cea", + "zh:aecb3e4b9121771dee9cac7975bf5d0657b5f3e8b57788c455beaeb0f3c48d93", + "zh:d2d3393170b8ef761d3146f39f6788c4a3e876e6c5d4cedca4870c2680688ae6", + "zh:daba5a005c1baa4a5eefbfb86d43ccf880eb5b42e8136f0d932f55886d72bda0", + "zh:de16a6ff3baacdaf9609a0a89aa1913fc19cccaf5ee0fc1c49c5a075baa47c02", + ] +} + +provider "registry.terraform.io/signalsciences/sigsci" { + version = "3.3.0" + constraints = ">= 1.2.18" + hashes = [ + "h1:DIoFVzfofY8lQSxFTw9wmQQC28PPMq+5l3xbPNw9gLc=", + "zh:07c25e1cca9c13314429a8430c2e999ad94c7d5e2f2a11501ee2608182387e61", + "zh:07daf79b672f3e0bec7b48e3ac8dcdeec02af06b10d653bd8158a74236b0746b", + "zh:1e24a050c3d3571ec3224c4bb5c82635caf636e707b5993a1cc97c9a1f19fa8f", + "zh:24293ae24b3de13bda8512c47967f01814724805396a1bfbfbfc56f5627615cc", + "zh:2cc6ba7a38d9854146d1d05f4b7a2f8e18a33c1267b768506cbe37168dad01dc", + "zh:42065bfee0cfde04096d6140c65379253359bed49b481a97aff70aa65bf568b3", + "zh:6f7f4d96967dfd92f098b57647d396679b70d92548db6d100c4dc8723569d175", + "zh:a2e4431f045cef16ed152c0d1f8a377b6468351b775ad1ca7ce3fe74fb874be2", + "zh:b0ed1cb03d6f191fe211f10bb59ef8daed6f89e3d99136e7bb5d38f2ac72fa45", + "zh:b61ea18442a65d27b97dd1cd43bdd8d0a56c2b4b8db6355480e89f8507c6782a", + "zh:c31bb2f50ac2a636758f93afec0b9d173be6d7d7476f9e250b4554e70c6d8d82", + "zh:cb7337f7b4678ad7ece28741069c07ce5601d2a103a9667db568cf10ed0ee5a2", + "zh:d521a7dac51733aebb0905e25b8f7c1279d83c06136e87826e010c667528fd3e", + "zh:ef791688acee3b8b1191b3c6dc54dabf69612dbfb666720280b492ce348a3a06", + ] +} diff --git a/infra/aws.tf b/infra/aws.tf new file mode 100644 index 000000000..65be17059 --- /dev/null +++ b/infra/aws.tf @@ -0,0 +1,7 @@ +resource "aws_route53_record" "ngwaf_cname" { + zone_id = var.route53_zone_id + name = var.route53_record_name + type = "CNAME" + ttl = var.route53_record_ttl + records = ["dualstack.python.map.fastly.net"] +} \ No newline at end of file diff --git a/infra/fastly.tf b/infra/fastly.tf new file mode 100644 index 000000000..8124dddf6 --- /dev/null +++ b/infra/fastly.tf @@ -0,0 +1,107 @@ +# Fastly VCL Service +resource "fastly_service_vcl" "frontend-vcl-service" { + name = "NGWAF Testing" + + domain { + name = var.USER_VCL_SERVICE_DOMAIN_NAME + comment = "NGWAF testing" + } + + backend { + address = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + name = "vcl_service_origin_1" + port = 443 + use_ssl = true + ssl_cert_hostname = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + ssl_sni_hostname = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + override_host = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + } + + # NGWAF Dynamic Snippets + dynamicsnippet { + name = "ngwaf_config_init" + type = "init" + priority = 0 + } + + dynamicsnippet { + name = "ngwaf_config_miss" + type = "miss" + priority = 9000 + } + + dynamicsnippet { + name = "ngwaf_config_pass" + type = "pass" + priority = 9000 + } + + dynamicsnippet { + name = "ngwaf_config_deliver" + type = "deliver" + priority = 9000 + } + + dictionary { + name = var.Edge_Security_dictionary + } + + lifecycle { + ignore_changes = [product_enablement] + } + + force_destroy = true +} + +# Fastly Service Dictionary Items +resource "fastly_service_dictionary_items" "edge_security_dictionary_items" { + for_each = { + for d in fastly_service_vcl.frontend-vcl-service.dictionary : d.name => d if d.name == var.Edge_Security_dictionary + } + service_id = fastly_service_vcl.frontend-vcl-service.id + dictionary_id = each.value.dictionary_id + items = { + Enabled: "100" + } +} + +# Fastly Service Dynamic Snippet Contents +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_init" { + for_each = { + for d in fastly_service_vcl.frontend-vcl-service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_init" + } + service_id = fastly_service_vcl.frontend-vcl-service.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_init" + manage_snippets = false +} + +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_miss" { + for_each = { + for d in fastly_service_vcl.frontend-vcl-service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_miss" + } + service_id = fastly_service_vcl.frontend-vcl-service.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_miss" + manage_snippets = false +} + +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_pass" { + for_each = { + for d in fastly_service_vcl.frontend-vcl-service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_pass" + } + service_id = fastly_service_vcl.frontend-vcl-service.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_pass" + manage_snippets = false +} + +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_deliver" { + for_each = { + for d in fastly_service_vcl.frontend-vcl-service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_deliver" + } + service_id = fastly_service_vcl.frontend-vcl-service.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_deliver" + manage_snippets = false +} \ No newline at end of file diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 000000000..6f1e81b20 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,34 @@ +terraform { + required_providers { + fastly = { + source = "fastly/fastly" + version = ">= 3.0.4" + } + sigsci = { + source = "signalsciences/sigsci" + version = ">= 1.2.18" + } + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +# Provider configurations +provider "fastly" { + api_key = var.FASTLY_API_KEY +} + +provider "aws" { + region = "us-east-2" + access_key = var.AWS_ACCESS_KEY_ID + secret_key = var.AWS_SECRET_ACCESS_KEY +} + +provider "sigsci" { + corp = var.NGWAF_CORP + email = var.NGWAF_EMAIL + auth_token = var.NGWAF_TOKEN + fastly_api_key = var.FASTLY_API_KEY +} \ No newline at end of file diff --git a/infra/ngwaf.tf b/infra/ngwaf.tf new file mode 100644 index 000000000..4187fd117 --- /dev/null +++ b/infra/ngwaf.tf @@ -0,0 +1,29 @@ +# NGWAF Edge Deployment +resource "sigsci_edge_deployment" "ngwaf_edge_site_service" { + site_short_name = var.NGWAF_SITE +} + +resource "sigsci_edge_deployment_service" "ngwaf_edge_service_link" { + site_short_name = var.NGWAF_SITE + fastly_sid = fastly_service_vcl.frontend-vcl-service.id + activate_version = true + percent_enabled = 100 + depends_on = [ + sigsci_edge_deployment.ngwaf_edge_site_service, + fastly_service_vcl.frontend-vcl-service, + fastly_service_dictionary_items.edge_security_dictionary_items, + fastly_service_dynamic_snippet_content.ngwaf_config_init, + fastly_service_dynamic_snippet_content.ngwaf_config_miss, + fastly_service_dynamic_snippet_content.ngwaf_config_pass, + fastly_service_dynamic_snippet_content.ngwaf_config_deliver, + ] +} + +resource "sigsci_edge_deployment_service_backend" "ngwaf_edge_service_backend_sync" { + site_short_name = var.NGWAF_SITE + fastly_sid = fastly_service_vcl.frontend-vcl-service.id + fastly_service_vcl_active_version = fastly_service_vcl.frontend-vcl-service.active_version + depends_on = [ + sigsci_edge_deployment_service.ngwaf_edge_service_link, + ] +} \ No newline at end of file diff --git a/infra/out.tf b/infra/out.tf new file mode 100644 index 000000000..0ee354d8c --- /dev/null +++ b/infra/out.tf @@ -0,0 +1,16 @@ +output "testing-the_ngwaf" { + value = < Date: Mon, 26 Aug 2024 14:11:47 -0500 Subject: [PATCH 02/20] feat: import current config --- infra/cdn.tf | 381 +++++++++++++++++++++++++++++++ infra/import/.terraform.lock.hcl | 24 ++ infra/import/README.md | 4 + infra/import/providers.tf | 0 infra/main.tf | 4 +- 5 files changed, 411 insertions(+), 2 deletions(-) create mode 100644 infra/cdn.tf create mode 100644 infra/import/.terraform.lock.hcl create mode 100644 infra/import/README.md create mode 100644 infra/import/providers.tf diff --git a/infra/cdn.tf b/infra/cdn.tf new file mode 100644 index 000000000..ff89e4bca --- /dev/null +++ b/infra/cdn.tf @@ -0,0 +1,381 @@ +# terraform import fastly_service_vcl.cdn 1d1Bii4LcJ9joSaowpIdb3 && terraform show: +resource "fastly_service_vcl" "cdn" { + active_version = 140 + cloned_version = 140 + comment = null + default_host = null + default_ttl = 3600 + force_refresh = false + http3 = false + id = "1d1Bii4LcJ9joSaowpIdb3" + imported = false + name = "www.python.org" + stale_if_error = false + stale_if_error_ttl = 43200 + version_comment = null + + acl { + acl_id = "6Xx3rqKOY0FjQ2CuoS9D8I" + force_destroy = false + name = "Generated_by_IP_block_list" + } + + backend { + # At least one attribute in this block is (or was) sensitive, + # so its contents will not be displayed. + } + backend { + # At least one attribute in this block is (or was) sensitive, + # so its contents will not be displayed. + } + + cache_setting { + action = "pass" + cache_condition = "Force Pass No-Cache No-Store" + name = "Pass No-Cache No-Store" + stale_ttl = 0 + ttl = 0 + } + + condition { + name = "Force Pass No-Cache No-Store" + priority = 10 + statement = "beresp.http.Cache-Control ~ \"(no-cache|no-store)\"" + type = "CACHE" + } + condition { + name = "Generated by IP block list" + priority = 0 + statement = "client.ip ~ Generated_by_IP_block_list" + type = "REQUEST" + } + condition { + name = "HSTS w/ subdomains" + priority = 10 + statement = "req.http.host == \"www.python.org\"" + type = "RESPONSE" + } + condition { + name = "HSTS w/o subdomain" + priority = 10 + statement = "req.http.host == \"python.org\"" + type = "RESPONSE" + } + condition { + name = "Homepage" + priority = 10 + statement = "req.url.path ~ \"^/$\"" + type = "REQUEST" + } + condition { + name = "Is Download" + priority = 10 + statement = "req.url ~ \"^/ftp/\"" + type = "REQUEST" + } + condition { + name = "Is Not Download" + priority = 5 + statement = "req.url !~ \"^/ftp/\"" + type = "REQUEST" + } + condition { + name = "Uncacheable URLs" + priority = 10 + statement = "req.url ~ \"^/(api|admin)/\"" + type = "REQUEST" + } + condition { + name = "apex redirect" + priority = 10 + statement = "req.http.Host == \"python.org\"" + type = "RESPONSE" + } + condition { + name = "apex" + priority = 1 + statement = "req.http.host == \"python.org\"" + type = "REQUEST" + } + + domain { + comment = null + name = "python.org" + } + domain { + comment = null + name = "www.python.org" + } + + gzip { + cache_condition = null + content_types = [ + "application/javascript", + "text/css", + "application/javascript", + "text/javascript", + "application/json", + "application/vnd.ms-fontobject", + "application/x-font-opentype", + "application/x-font-truetype", + "application/x-font-ttf", + "application/xml", + "font/eot", + "font/opentype", + "font/otf", + "image/svg+xml", + "image/vnd.microsoft.icon", + "text/plain", + "text/xml", + ] + extensions = [ + null, + ] + name = "Default rules" + } + + header { + action = "delete" + cache_condition = null + destination = "http.Cookie" + ignore_if_set = false + name = "Remove cookies" + priority = 10 + regex = null + request_condition = "Is Download" + response_condition = null + source = null + substitution = null + type = "request" + } + header { + action = "set" + cache_condition = null + destination = "backend" + ignore_if_set = false + name = "Is Download Director" + priority = 10 + regex = null + request_condition = "Is Download" + response_condition = null + source = "F_lb_nyc1_psf_io" + substitution = null + type = "request" + } + header { + action = "set" + cache_condition = null + destination = "backend" + ignore_if_set = false + name = "Is Not Download Backend" + priority = 10 + regex = null + request_condition = "Is Not Download" + response_condition = null + source = "F_cabotage" + substitution = null + type = "request" + } + header { + action = "set" + cache_condition = null + destination = "http.Fastly-Token" + ignore_if_set = false + name = "Fastly Token" + priority = 10 + regex = null + request_condition = null + response_condition = null + source = "\"bRMyYFEUAtTGmomMj7LKDRFdtGg4cLJXQGPjMYqRiLjLdtewyB\"" + substitution = null + type = "request" + } + header { + action = "set" + cache_condition = null + destination = "http.Location" + ignore_if_set = false + name = "www redirect" + priority = 10 + regex = null + request_condition = null + response_condition = "apex redirect" + source = "\"https://www.python.org\" + req.url" + substitution = null + type = "response" + } + header { + action = "set" + cache_condition = null + destination = "http.Strict-Transport-Security" + ignore_if_set = false + name = "HSTS w/ subdomains" + priority = 10 + regex = null + request_condition = null + response_condition = "HSTS w/ subdomains" + source = "\"max-age=63072000; includeSubDomains; preload\"" + substitution = null + type = "response" + } + header { + action = "set" + cache_condition = null + destination = "http.Strict-Transport-Security" + ignore_if_set = false + name = "HSTS w/o subdomains" + priority = 10 + regex = null + request_condition = null + response_condition = "HSTS w/o subdomain" + source = "\"max-age=315360000; preload\"" + substitution = null + type = "response" + } + header { + action = "set" + cache_condition = null + destination = "url" + ignore_if_set = false + name = "Chop off query string" + priority = 10 + regex = null + request_condition = "Is Download" + response_condition = null + source = "regsub(req.url, \"\\?.*$\", \"\")" + substitution = null + type = "request" + } + header { + action = "set" + cache_condition = null + destination = "url" + ignore_if_set = false + name = "Strip Query Strings" + priority = 10 + regex = null + request_condition = "Homepage" + response_condition = null + source = "req.url.path" + substitution = null + type = "request" + } + + healthcheck { + check_interval = 15000 + expected_response = 200 + headers = [] + host = "python.org" + http_version = "1.1" + initial = 4 + method = "HEAD" + name = "HAProxy Status" + path = "/_haproxy_status" + threshold = 3 + timeout = 5000 + window = 5 + } + + logging_datadog { + # At least one attribute in this block is (or was) sensitive, + # so its contents will not be displayed. + } + + logging_s3 { + # At least one attribute in this block is (or was) sensitive, + # so its contents will not be displayed. + } + + logging_syslog { + # At least one attribute in this block is (or was) sensitive, + # so its contents will not be displayed. + } + + product_enablement { + bot_management = true + brotli_compression = false + domain_inspector = true + image_optimizer = false + name = null + origin_inspector = true + websockets = false + } + + rate_limiter { + action = "log_only" + client_key = "client.ip" + feature_revision = 1 + http_methods = "GET,PUT,TRACE,POST,HEAD,DELETE,PATCH,OPTIONS" + logger_type = "datadog" + name = "python.org backends" + penalty_box_duration = 2 + ratelimiter_id = "6SnVeRHflsd9pTPPFvhYqX" + response_object_name = null + rps_limit = 10 + uri_dictionary_name = null + window_size = 10 + + response { + content = <<-EOT + + + Too Many Requests + + +

Too Many Requests

+ + + EOT + content_type = "text/html" + status = 429 + } + } + + request_setting { + action = null + bypass_busy_wait = false + default_host = null + force_miss = false + force_ssl = true + geo_headers = false + hash_keys = null + max_stale_age = 86400 + name = "Default cache policy" + request_condition = null + timer_support = false + xff = "append" + } + request_setting { + action = "pass" + bypass_busy_wait = false + default_host = null + force_miss = false + force_ssl = false + geo_headers = false + hash_keys = null + max_stale_age = 60 + name = "Force Pass" + request_condition = "Uncacheable URLs" + timer_support = false + xff = "append" + } + + response_object { + cache_condition = null + content = null + content_type = null + name = "www redirect" + request_condition = "apex" + response = "Moved Permanently" + status = 301 + } + response_object { + cache_condition = null + content = null + content_type = "text/html" + name = "Generated by IP block list" + request_condition = "Generated by IP block list" + response = "Forbidden" + status = 403 + } +} diff --git a/infra/import/.terraform.lock.hcl b/infra/import/.terraform.lock.hcl new file mode 100644 index 000000000..552e40475 --- /dev/null +++ b/infra/import/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/fastly/fastly" { + version = "5.13.0" + constraints = "5.13.0" + hashes = [ + "h1:op/7hntTRkfZFIZ5xLNtLb7eBY155ywQIVSy56XCmBE=", + "zh:04f7405ee22a8ace546b90cc3a08d81f1a49dae8b1050500398d4b0244dcbc86", + "zh:0e0c48aca34a1fc7ed7382c8e85b5da770f63f3c9aa79bc2c3c55ed570f9d0ab", + "zh:302d2b9872ab8ffee2082291cc2cfec487633e22c7970b2c9d22268d6b5f7624", + "zh:346ea021dbe2c7128cddc2c9e01a95242b8bceeda20d9d1b00ae09ee90e3962b", + "zh:41fbe18f63154a6a1a46e1b1cc909bfe90f5bba7f5cfab0a80d15be7eceec4c3", + "zh:524c2a54282a92d0d7633bfd511427f6d9aa6b6a52b7d9f71cf5206dedab381e", + "zh:721fe08bfb1b85f8946aeba3bdb7e0de3d74fce94c8657d0086b153f58558d89", + "zh:9c627b3170a5505c73455e6c2a99d2ce4187e225130e12aececdc808357f8b66", + "zh:a61a62cec9612358b08ef1895277a37d4d4ec134972991fa414255ef95683dba", + "zh:bde1a51553c15d333140c2b77481ee668c4af8de93a968d869c02a736db460c4", + "zh:c2683862bd0e9633d3800503a71b3aab51ec8e3aac3f6ef6b71831efe81a2afd", + "zh:dff5ad3766432550974d2f0c24535c572fee5eeb0dce7befeaa97cb6ca3d8443", + "zh:ec3c56fc43344a07b0eef5158df6dd50e68bdcee1b03299bb2acd502d11582d5", + "zh:ec8d899cafd925d3492f00c6523c90599aebc43c1373ad4bd6c55f12d2376230", + ] +} diff --git a/infra/import/README.md b/infra/import/README.md new file mode 100644 index 000000000..7a4301ed9 --- /dev/null +++ b/infra/import/README.md @@ -0,0 +1,4 @@ +# about + +temporary dir holding imports from +`terraform import fastly_service_vcl.cdn 1d1Bii4LcJ9joSaowpIdb3 && terraform show` \ No newline at end of file diff --git a/infra/import/providers.tf b/infra/import/providers.tf new file mode 100644 index 000000000..e69de29bb diff --git a/infra/main.tf b/infra/main.tf index 6f1e81b20..9fd9c697a 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -2,11 +2,11 @@ terraform { required_providers { fastly = { source = "fastly/fastly" - version = ">= 3.0.4" + version = "5.13.0" } sigsci = { source = "signalsciences/sigsci" - version = ">= 1.2.18" + version = "3.3.0" } aws = { source = "hashicorp/aws" From ab14d51f8e92f0ea4b405a40f9640913e19216d9 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Mon, 26 Aug 2024 14:16:00 -0500 Subject: [PATCH 03/20] chore: change names, prep for merger with ngwaf config, add todos --- infra/cdn.tf | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/infra/cdn.tf b/infra/cdn.tf index ff89e4bca..fae315c94 100644 --- a/infra/cdn.tf +++ b/infra/cdn.tf @@ -1,3 +1,4 @@ +# TODO: Combine with fastly.tf so that we can layer the ngwaf stuff for testing # terraform import fastly_service_vcl.cdn 1d1Bii4LcJ9joSaowpIdb3 && terraform show: resource "fastly_service_vcl" "cdn" { active_version = 140 @@ -9,7 +10,7 @@ resource "fastly_service_vcl" "cdn" { http3 = false id = "1d1Bii4LcJ9joSaowpIdb3" imported = false - name = "www.python.org" + name = "test.python.org" stale_if_error = false stale_if_error_ttl = 43200 version_comment = null @@ -20,11 +21,11 @@ resource "fastly_service_vcl" "cdn" { name = "Generated_by_IP_block_list" } - backend { + backend { # TODO: add cabo stuffs # At least one attribute in this block is (or was) sensitive, # so its contents will not be displayed. } - backend { + backend { # TODO: unsure what this onewas, i think maybe lb? # At least one attribute in this block is (or was) sensitive, # so its contents will not be displayed. } @@ -52,13 +53,13 @@ resource "fastly_service_vcl" "cdn" { condition { name = "HSTS w/ subdomains" priority = 10 - statement = "req.http.host == \"www.python.org\"" + statement = "req.http.host == \"test.python.org\"" type = "RESPONSE" } condition { name = "HSTS w/o subdomain" priority = 10 - statement = "req.http.host == \"python.org\"" + statement = "req.http.host == \"test.python.org\"" type = "RESPONSE" } condition { @@ -88,23 +89,23 @@ resource "fastly_service_vcl" "cdn" { condition { name = "apex redirect" priority = 10 - statement = "req.http.Host == \"python.org\"" + statement = "req.http.Host == \"test.python.org\"" type = "RESPONSE" } condition { name = "apex" priority = 1 - statement = "req.http.host == \"python.org\"" + statement = "req.http.host == \"test.python.org\"" type = "REQUEST" } domain { comment = null - name = "python.org" + name = "test.python.org" } domain { comment = null - name = "www.python.org" + name = "test.python.org" } gzip { @@ -200,7 +201,7 @@ resource "fastly_service_vcl" "cdn" { regex = null request_condition = null response_condition = "apex redirect" - source = "\"https://www.python.org\" + req.url" + source = "\"https://test.python.org\" + req.url" substitution = null type = "response" } @@ -265,7 +266,7 @@ resource "fastly_service_vcl" "cdn" { check_interval = 15000 expected_response = 200 headers = [] - host = "python.org" + host = "test.python.org" http_version = "1.1" initial = 4 method = "HEAD" @@ -276,17 +277,17 @@ resource "fastly_service_vcl" "cdn" { window = 5 } - logging_datadog { + logging_datadog { # TODO # At least one attribute in this block is (or was) sensitive, # so its contents will not be displayed. } - logging_s3 { + logging_s3 { # TODO # At least one attribute in this block is (or was) sensitive, # so its contents will not be displayed. } - logging_syslog { + logging_syslog { # TODO # At least one attribute in this block is (or was) sensitive, # so its contents will not be displayed. } @@ -307,7 +308,7 @@ resource "fastly_service_vcl" "cdn" { feature_revision = 1 http_methods = "GET,PUT,TRACE,POST,HEAD,DELETE,PATCH,OPTIONS" logger_type = "datadog" - name = "python.org backends" + name = "test.python.org backends" penalty_box_duration = 2 ratelimiter_id = "6SnVeRHflsd9pTPPFvhYqX" response_object_name = null @@ -337,7 +338,7 @@ resource "fastly_service_vcl" "cdn" { default_host = null force_miss = false force_ssl = true - geo_headers = false + geo_headers = false # ! DEPRECATED hash_keys = null max_stale_age = 86400 name = "Default cache policy" @@ -351,7 +352,7 @@ resource "fastly_service_vcl" "cdn" { default_host = null force_miss = false force_ssl = false - geo_headers = false + geo_headers = false # ! DEPRECATED hash_keys = null max_stale_age = 60 name = "Force Pass" From 2afd560404aaea816053c8c2e39ade8868b85d1b Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Mon, 26 Aug 2024 14:57:53 -0500 Subject: [PATCH 04/20] fix: remove old key --- infra/cdn.tf | 2 +- infra/terraform.tfvars.example | 3 ++- infra/variables.tf | 5 +++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/infra/cdn.tf b/infra/cdn.tf index fae315c94..debd3addc 100644 --- a/infra/cdn.tf +++ b/infra/cdn.tf @@ -187,7 +187,7 @@ resource "fastly_service_vcl" "cdn" { regex = null request_condition = null response_condition = null - source = "\"bRMyYFEUAtTGmomMj7LKDRFdtGg4cLJXQGPjMYqRiLjLdtewyB\"" + source = var.FASTLY_HEADER_TOKEN substitution = null type = "request" } diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index bf23993b5..1588bfb5c 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -2,4 +2,5 @@ AWS_ACCESS_KEY_ID = "NotARealKey" AWS_SECRET_ACCESS_KEY = "NotARealKey" NGWAF_TOKEN = "NotARealKey" -FASTLY_API_KEY = "NotARealKey" \ No newline at end of file +FASTLY_API_KEY = "NotARealKey" +FASTLY_HEADER_TOKEN = "" diff --git a/infra/variables.tf b/infra/variables.tf index c87dc5eca..edc86457f 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -3,6 +3,11 @@ variable "FASTLY_API_KEY" { type = string description = "API key for the Fastly VCL edge configuration." } +variable "FASTLY_HEADER_TOKEN" { + description = "Fastly Token for authentication" + type = string + sensitive = true +} # VCL Service variables variable "USER_VCL_SERVICE_DOMAIN_NAME" { From 15a544e5bbdef2403f05698d937dd92270fb2132 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Mon, 26 Aug 2024 16:01:18 -0500 Subject: [PATCH 05/20] feat: add logging configs (partial) --- infra/.terraform.lock.hcl | 34 ++++++++++++++-------------- infra/cdn.tf | 39 +++++++++++++++++++++++--------- infra/import/.terraform.lock.hcl | 24 -------------------- infra/import/README.md | 4 ---- infra/import/providers.tf | 0 infra/variables.tf | 13 ++++++++--- 6 files changed, 55 insertions(+), 59 deletions(-) delete mode 100644 infra/import/.terraform.lock.hcl delete mode 100644 infra/import/README.md delete mode 100644 infra/import/providers.tf diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl index c3317831b..2d7b597fa 100644 --- a/infra/.terraform.lock.hcl +++ b/infra/.terraform.lock.hcl @@ -2,24 +2,24 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/fastly/fastly" { - version = "5.12.0" - constraints = ">= 3.0.4" + version = "5.13.0" + constraints = "5.13.0" hashes = [ - "h1:y2qTbSz7wIn6wsAHEGmD2qtP6pp420s3kt7+D0n9moc=", - "zh:1b2175821acf437a070b8d33af04a12c4bafb0dd2da767de027d894f6b8a7e0e", - "zh:1e6a95f29f8aaf292c882d69761a266bf948fa9a9433f9fa4d60933ace2ed03a", - "zh:1ef8578a764ea8a85c9d0d72ccf82e211fcd70740ae0b1f03e70952069aafced", - "zh:4c1fba8f37aee1f6708b93891dbc36042a517c3476ca5b208c1d926d3304eadf", - "zh:4d3792115edd259f98732950b8259b7b460eeba8c6141e70424d42699e61c729", - "zh:6f85ecd31b1a6cfb5cd99c1a5b7426519eac065baa6e8a7bf0df7ce17d763e43", - "zh:6ffdf5953efc0f5c5dd21395d545ebdbb9ddcf3821fad89137febba079f49752", - "zh:76eb53907f353a2faa1dc952a2fa085bcd716cf3a7271c87ee46c25588bedf94", - "zh:7dd738afdfa6112d1992130a1285c73b157fe0dba7110c796a247e7328f027b2", - "zh:8c54e5fffd47443d018d792b10ab1425916bfb822416179c6fa3847275023ff1", - "zh:96cbff688e4c47530be9743b5206eb61bdecd6b15459d1a6cf727b1301172331", - "zh:9fbdf427058e90463ce9485ad5ae15605cbe2f306d5eeff9b2c9ea3b9cc5a186", - "zh:aa21ca195fe08e7e0b55515ba78551b7cd6237bd5f111fff3fa0fcfcc814cd6e", - "zh:dc24a157763e401629c25d71a9de3c0674bf9119adf5d4af988b39a7f442e181", + "h1:op/7hntTRkfZFIZ5xLNtLb7eBY155ywQIVSy56XCmBE=", + "zh:04f7405ee22a8ace546b90cc3a08d81f1a49dae8b1050500398d4b0244dcbc86", + "zh:0e0c48aca34a1fc7ed7382c8e85b5da770f63f3c9aa79bc2c3c55ed570f9d0ab", + "zh:302d2b9872ab8ffee2082291cc2cfec487633e22c7970b2c9d22268d6b5f7624", + "zh:346ea021dbe2c7128cddc2c9e01a95242b8bceeda20d9d1b00ae09ee90e3962b", + "zh:41fbe18f63154a6a1a46e1b1cc909bfe90f5bba7f5cfab0a80d15be7eceec4c3", + "zh:524c2a54282a92d0d7633bfd511427f6d9aa6b6a52b7d9f71cf5206dedab381e", + "zh:721fe08bfb1b85f8946aeba3bdb7e0de3d74fce94c8657d0086b153f58558d89", + "zh:9c627b3170a5505c73455e6c2a99d2ce4187e225130e12aececdc808357f8b66", + "zh:a61a62cec9612358b08ef1895277a37d4d4ec134972991fa414255ef95683dba", + "zh:bde1a51553c15d333140c2b77481ee668c4af8de93a968d869c02a736db460c4", + "zh:c2683862bd0e9633d3800503a71b3aab51ec8e3aac3f6ef6b71831efe81a2afd", + "zh:dff5ad3766432550974d2f0c24535c572fee5eeb0dce7befeaa97cb6ca3d8443", + "zh:ec3c56fc43344a07b0eef5158df6dd50e68bdcee1b03299bb2acd502d11582d5", + "zh:ec8d899cafd925d3492f00c6523c90599aebc43c1373ad4bd6c55f12d2376230", ] } diff --git a/infra/cdn.tf b/infra/cdn.tf index debd3addc..814e183ad 100644 --- a/infra/cdn.tf +++ b/infra/cdn.tf @@ -8,7 +8,7 @@ resource "fastly_service_vcl" "cdn" { default_ttl = 3600 force_refresh = false http3 = false - id = "1d1Bii4LcJ9joSaowpIdb3" + id = "z5nOzklFYCXDAUiLeqvS25" imported = false name = "test.python.org" stale_if_error = false @@ -277,19 +277,36 @@ resource "fastly_service_vcl" "cdn" { window = 5 } - logging_datadog { # TODO - # At least one attribute in this block is (or was) sensitive, - # so its contents will not be displayed. + logging_datadog { + name = "ratelimit-debug" + token = var.DATADOG_API_KEY + region = "US" } - logging_s3 { # TODO - # At least one attribute in this block is (or was) sensitive, - # so its contents will not be displayed. + logging_s3 { + name = "psf-fastly-logs" + bucket_name = "psf-fastly-logs-eu-west-1" + domain = "s3-eu-west-1.amazonaws.com" + path = "/www-python-org/%Y/%m/%d/" + period = 3600 + gzip_level = 9 + # %h "%{now}V" %l "%{req.request}V %{req.url}V" %{req.proto}V %>s %{resp.http.Content-Length}V %{resp.http.age}V "%{resp.http.x-cache}V" "%{resp.http.x-cache-hits}V" "%{req.http.content-type}V" "%{req.http.accept-language}V" "%{cstr_escape(req.http.user-agent)}V" + format = "" # TODO + timestamp_format = "%Y-%m-%dT%H:%M:%S.000" + redundancy = "standard" + format_version = 2 + message_type = "classic" + compression_codec = "gzip" + access_key = var.AWS_ACCESS_KEY_ID + secret_key = var.AWS_SECRET_ACCESS_KEY } - logging_syslog { # TODO - # At least one attribute in this block is (or was) sensitive, - # so its contents will not be displayed. + logging_syslog { + name = "syslog" + address = "cdn-logs.nyc1.psf.io" + port = 514 + # %h "%{now}V" %l "%{req.request}V %{req.url}V" %{req.proto}V %>s %{resp.http.Content-Length}V %{resp.http.age}V "%{resp.http.x-cache}V" "%{resp.http.x-cache-hits}V" "%{req.http.content-type}V" "%{req.http.accept-language}V" "%{cstr_escape(req.http.user-agent)}V" + format = "" # TODO } product_enablement { @@ -310,7 +327,7 @@ resource "fastly_service_vcl" "cdn" { logger_type = "datadog" name = "test.python.org backends" penalty_box_duration = 2 - ratelimiter_id = "6SnVeRHflsd9pTPPFvhYqX" + ratelimiter_id = "..." # TODO: create one for ngwaf test service response_object_name = null rps_limit = 10 uri_dictionary_name = null diff --git a/infra/import/.terraform.lock.hcl b/infra/import/.terraform.lock.hcl deleted file mode 100644 index 552e40475..000000000 --- a/infra/import/.terraform.lock.hcl +++ /dev/null @@ -1,24 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/fastly/fastly" { - version = "5.13.0" - constraints = "5.13.0" - hashes = [ - "h1:op/7hntTRkfZFIZ5xLNtLb7eBY155ywQIVSy56XCmBE=", - "zh:04f7405ee22a8ace546b90cc3a08d81f1a49dae8b1050500398d4b0244dcbc86", - "zh:0e0c48aca34a1fc7ed7382c8e85b5da770f63f3c9aa79bc2c3c55ed570f9d0ab", - "zh:302d2b9872ab8ffee2082291cc2cfec487633e22c7970b2c9d22268d6b5f7624", - "zh:346ea021dbe2c7128cddc2c9e01a95242b8bceeda20d9d1b00ae09ee90e3962b", - "zh:41fbe18f63154a6a1a46e1b1cc909bfe90f5bba7f5cfab0a80d15be7eceec4c3", - "zh:524c2a54282a92d0d7633bfd511427f6d9aa6b6a52b7d9f71cf5206dedab381e", - "zh:721fe08bfb1b85f8946aeba3bdb7e0de3d74fce94c8657d0086b153f58558d89", - "zh:9c627b3170a5505c73455e6c2a99d2ce4187e225130e12aececdc808357f8b66", - "zh:a61a62cec9612358b08ef1895277a37d4d4ec134972991fa414255ef95683dba", - "zh:bde1a51553c15d333140c2b77481ee668c4af8de93a968d869c02a736db460c4", - "zh:c2683862bd0e9633d3800503a71b3aab51ec8e3aac3f6ef6b71831efe81a2afd", - "zh:dff5ad3766432550974d2f0c24535c572fee5eeb0dce7befeaa97cb6ca3d8443", - "zh:ec3c56fc43344a07b0eef5158df6dd50e68bdcee1b03299bb2acd502d11582d5", - "zh:ec8d899cafd925d3492f00c6523c90599aebc43c1373ad4bd6c55f12d2376230", - ] -} diff --git a/infra/import/README.md b/infra/import/README.md deleted file mode 100644 index 7a4301ed9..000000000 --- a/infra/import/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# about - -temporary dir holding imports from -`terraform import fastly_service_vcl.cdn 1d1Bii4LcJ9joSaowpIdb3 && terraform show` \ No newline at end of file diff --git a/infra/import/providers.tf b/infra/import/providers.tf deleted file mode 100644 index e69de29bb..000000000 diff --git a/infra/variables.tf b/infra/variables.tf index edc86457f..e68aa94c7 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -13,13 +13,13 @@ variable "FASTLY_HEADER_TOKEN" { variable "USER_VCL_SERVICE_DOMAIN_NAME" { type = string description = "Frontend domain for your service." - default = "ngwaftest.psf.io" + default = "test.python.org" } variable "USER_VCL_SERVICE_BACKEND_HOSTNAME" { type = string description = "Hostname used for backend." - default = "test-ngwaf.psf.io" + default = "test.python.org" } variable "Edge_Security_dictionary" { @@ -74,11 +74,18 @@ variable "route53_zone_id" { variable "route53_record_name" { type = string description = "The name of the CNAME record" - default = "ngwaftest.psf.io" + default = "test.python.org" } variable "route53_record_ttl" { type = number description = "The TTL for the CNAME record" default = 60 +} + +# dd +variable "DATADOG_API_KEY" { + type = string + description = "API key for Datadog" + sensitive = true } \ No newline at end of file From 087da1e256f7eec81cfc2c2d27556d974b874672 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Mon, 26 Aug 2024 16:20:45 -0500 Subject: [PATCH 06/20] feat: add backend configs --- infra/cdn.tf | 38 +++++++++++++++++++++++++++------- infra/terraform.tfvars.example | 3 ++- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/infra/cdn.tf b/infra/cdn.tf index 814e183ad..22ff643c0 100644 --- a/infra/cdn.tf +++ b/infra/cdn.tf @@ -21,13 +21,37 @@ resource "fastly_service_vcl" "cdn" { name = "Generated_by_IP_block_list" } - backend { # TODO: add cabo stuffs - # At least one attribute in this block is (or was) sensitive, - # so its contents will not be displayed. + backend { + name = "cabotage" + address = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" + port = 443 + shield = "iad-va-us" + auto_loadbalance = false + ssl_check_cert = true + ssl_cert_hostname = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" + ssl_sni_hostname = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" + weight = 100 + max_conn = 200 + connect_timeout = 1000 + first_byte_timeout = 30000 + between_bytes_timeout = 10000 } - backend { # TODO: unsure what this onewas, i think maybe lb? - # At least one attribute in this block is (or was) sensitive, - # so its contents will not be displayed. + + backend { + name = "loadbalancer" + address = "lb.nyc1.psf.io" + shield = "iad-va-us" + healthcheck = "HAProxy Status" + auto_loadbalance = false + ssl_check_cert = true + ssl_cert_hostname = "lb.psf.io" + ssl_sni_hostname = "lb.psf.io" + ssl_ca_cert = "" # TODO(@ee) + weight = 100 + max_conn = 200 + connect_timeout = 1000 + first_byte_timeout = 15000 + between_bytes_timeout = 10000 } cache_setting { @@ -327,7 +351,7 @@ resource "fastly_service_vcl" "cdn" { logger_type = "datadog" name = "test.python.org backends" penalty_box_duration = 2 - ratelimiter_id = "..." # TODO: create one for ngwaf test service + ratelimiter_id = "..." # TODO: create specific one for ngwaf test service? prod one is 5S7R6aG8KoT6QqtXFd1Nfk response_object_name = null rps_limit = 10 uri_dictionary_name = null diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 1588bfb5c..e89f21d8c 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -3,4 +3,5 @@ AWS_SECRET_ACCESS_KEY = "NotARealKey" NGWAF_TOKEN = "NotARealKey" FASTLY_API_KEY = "NotARealKey" -FASTLY_HEADER_TOKEN = "" +FASTLY_HEADER_TOKEN = "NotARealKey" +DATADOG_API_KEY = "NotARealKey" From 25f34a26c244363d4c14e866e790db9f8f501922 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Mon, 26 Aug 2024 16:22:58 -0500 Subject: [PATCH 07/20] feat: apply logging formats --- infra/cdn.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/cdn.tf b/infra/cdn.tf index 22ff643c0..ea3caefc5 100644 --- a/infra/cdn.tf +++ b/infra/cdn.tf @@ -315,7 +315,7 @@ resource "fastly_service_vcl" "cdn" { period = 3600 gzip_level = 9 # %h "%{now}V" %l "%{req.request}V %{req.url}V" %{req.proto}V %>s %{resp.http.Content-Length}V %{resp.http.age}V "%{resp.http.x-cache}V" "%{resp.http.x-cache-hits}V" "%{req.http.content-type}V" "%{req.http.accept-language}V" "%{cstr_escape(req.http.user-agent)}V" - format = "" # TODO + format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" timestamp_format = "%Y-%m-%dT%H:%M:%S.000" redundancy = "standard" format_version = 2 @@ -330,7 +330,7 @@ resource "fastly_service_vcl" "cdn" { address = "cdn-logs.nyc1.psf.io" port = 514 # %h "%{now}V" %l "%{req.request}V %{req.url}V" %{req.proto}V %>s %{resp.http.Content-Length}V %{resp.http.age}V "%{resp.http.x-cache}V" "%{resp.http.x-cache-hits}V" "%{req.http.content-type}V" "%{req.http.accept-language}V" "%{cstr_escape(req.http.user-agent)}V" - format = "" # TODO + format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" } product_enablement { From a7cab0a8daa0989aec19d0cf45555c1d50e7d493 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Mon, 26 Aug 2024 16:31:48 -0500 Subject: [PATCH 08/20] fix: adjust for working `tf plan` --- infra/cdn.tf | 23 +++++------------------ infra/terraform.tfvars.example | 2 ++ infra/variables.tf | 11 +++++++++++ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/infra/cdn.tf b/infra/cdn.tf index ea3caefc5..07dd349c6 100644 --- a/infra/cdn.tf +++ b/infra/cdn.tf @@ -1,24 +1,15 @@ # TODO: Combine with fastly.tf so that we can layer the ngwaf stuff for testing # terraform import fastly_service_vcl.cdn 1d1Bii4LcJ9joSaowpIdb3 && terraform show: resource "fastly_service_vcl" "cdn" { - active_version = 140 - cloned_version = 140 - comment = null - default_host = null default_ttl = 3600 - force_refresh = false http3 = false - id = "z5nOzklFYCXDAUiLeqvS25" - imported = false name = "test.python.org" stale_if_error = false stale_if_error_ttl = 43200 - version_comment = null acl { - acl_id = "6Xx3rqKOY0FjQ2CuoS9D8I" - force_destroy = false name = "Generated_by_IP_block_list" + force_destroy = false } backend { @@ -153,9 +144,6 @@ resource "fastly_service_vcl" "cdn" { "text/plain", "text/xml", ] - extensions = [ - null, - ] name = "Default rules" } @@ -321,8 +309,8 @@ resource "fastly_service_vcl" "cdn" { format_version = 2 message_type = "classic" compression_codec = "gzip" - access_key = var.AWS_ACCESS_KEY_ID - secret_key = var.AWS_SECRET_ACCESS_KEY + s3_access_key = var.S3_ACCESS_KEY + s3_secret_key = var.S3_SECRET_KEY } logging_syslog { @@ -351,7 +339,6 @@ resource "fastly_service_vcl" "cdn" { logger_type = "datadog" name = "test.python.org backends" penalty_box_duration = 2 - ratelimiter_id = "..." # TODO: create specific one for ngwaf test service? prod one is 5S7R6aG8KoT6QqtXFd1Nfk response_object_name = null rps_limit = 10 uri_dictionary_name = null @@ -379,7 +366,7 @@ resource "fastly_service_vcl" "cdn" { default_host = null force_miss = false force_ssl = true - geo_headers = false # ! DEPRECATED +# geo_headers = false # ! DEPRECATED hash_keys = null max_stale_age = 86400 name = "Default cache policy" @@ -393,7 +380,7 @@ resource "fastly_service_vcl" "cdn" { default_host = null force_miss = false force_ssl = false - geo_headers = false # ! DEPRECATED +# geo_headers = false # ! DEPRECATED hash_keys = null max_stale_age = 60 name = "Force Pass" diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index e89f21d8c..d801f57fe 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -1,5 +1,7 @@ AWS_ACCESS_KEY_ID = "NotARealKey" AWS_SECRET_ACCESS_KEY = "NotARealKey" +S3_ACCESS_KEY_ID = NotARealKey" +S3_SECRET_KEY = "NotARealKey" NGWAF_TOKEN = "NotARealKey" FASTLY_API_KEY = "NotARealKey" diff --git a/infra/variables.tf b/infra/variables.tf index e68aa94c7..e0da53ebc 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -65,6 +65,17 @@ variable "AWS_SECRET_ACCESS_KEY" { sensitive = true } +variable "S3_ACCESS_KEY" { + type = string + description = "Access key for the S3 bucket." + sensitive = true +} +variable "S3_SECRET_KEY" { + type = string + description = "Secret access key for the S3 bucket." + sensitive = true +} + variable "route53_zone_id" { type = string description = "The Route 53 hosted zone ID" From dbf2764a18c9aae247ad4cabc1c447418c83b625 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Mon, 26 Aug 2024 17:20:44 -0500 Subject: [PATCH 09/20] feat: apply formatting, combine fastly and ngwaf stuff --- infra/cdn.tf | 410 --------------------------------------------- infra/fastly.tf | 363 +++++++++++++++++++++++++++++++++++++-- infra/main.tf | 22 +-- infra/ngwaf.tf | 8 +- infra/out.tf | 6 +- infra/variables.tf | 16 +- infra/versions.tf | 16 ++ 7 files changed, 384 insertions(+), 457 deletions(-) delete mode 100644 infra/cdn.tf create mode 100644 infra/versions.tf diff --git a/infra/cdn.tf b/infra/cdn.tf deleted file mode 100644 index 07dd349c6..000000000 --- a/infra/cdn.tf +++ /dev/null @@ -1,410 +0,0 @@ -# TODO: Combine with fastly.tf so that we can layer the ngwaf stuff for testing -# terraform import fastly_service_vcl.cdn 1d1Bii4LcJ9joSaowpIdb3 && terraform show: -resource "fastly_service_vcl" "cdn" { - default_ttl = 3600 - http3 = false - name = "test.python.org" - stale_if_error = false - stale_if_error_ttl = 43200 - - acl { - name = "Generated_by_IP_block_list" - force_destroy = false - } - - backend { - name = "cabotage" - address = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" - port = 443 - shield = "iad-va-us" - auto_loadbalance = false - ssl_check_cert = true - ssl_cert_hostname = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" - ssl_sni_hostname = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" - weight = 100 - max_conn = 200 - connect_timeout = 1000 - first_byte_timeout = 30000 - between_bytes_timeout = 10000 - } - - backend { - name = "loadbalancer" - address = "lb.nyc1.psf.io" - shield = "iad-va-us" - healthcheck = "HAProxy Status" - auto_loadbalance = false - ssl_check_cert = true - ssl_cert_hostname = "lb.psf.io" - ssl_sni_hostname = "lb.psf.io" - ssl_ca_cert = "" # TODO(@ee) - weight = 100 - max_conn = 200 - connect_timeout = 1000 - first_byte_timeout = 15000 - between_bytes_timeout = 10000 - } - - cache_setting { - action = "pass" - cache_condition = "Force Pass No-Cache No-Store" - name = "Pass No-Cache No-Store" - stale_ttl = 0 - ttl = 0 - } - - condition { - name = "Force Pass No-Cache No-Store" - priority = 10 - statement = "beresp.http.Cache-Control ~ \"(no-cache|no-store)\"" - type = "CACHE" - } - condition { - name = "Generated by IP block list" - priority = 0 - statement = "client.ip ~ Generated_by_IP_block_list" - type = "REQUEST" - } - condition { - name = "HSTS w/ subdomains" - priority = 10 - statement = "req.http.host == \"test.python.org\"" - type = "RESPONSE" - } - condition { - name = "HSTS w/o subdomain" - priority = 10 - statement = "req.http.host == \"test.python.org\"" - type = "RESPONSE" - } - condition { - name = "Homepage" - priority = 10 - statement = "req.url.path ~ \"^/$\"" - type = "REQUEST" - } - condition { - name = "Is Download" - priority = 10 - statement = "req.url ~ \"^/ftp/\"" - type = "REQUEST" - } - condition { - name = "Is Not Download" - priority = 5 - statement = "req.url !~ \"^/ftp/\"" - type = "REQUEST" - } - condition { - name = "Uncacheable URLs" - priority = 10 - statement = "req.url ~ \"^/(api|admin)/\"" - type = "REQUEST" - } - condition { - name = "apex redirect" - priority = 10 - statement = "req.http.Host == \"test.python.org\"" - type = "RESPONSE" - } - condition { - name = "apex" - priority = 1 - statement = "req.http.host == \"test.python.org\"" - type = "REQUEST" - } - - domain { - comment = null - name = "test.python.org" - } - domain { - comment = null - name = "test.python.org" - } - - gzip { - cache_condition = null - content_types = [ - "application/javascript", - "text/css", - "application/javascript", - "text/javascript", - "application/json", - "application/vnd.ms-fontobject", - "application/x-font-opentype", - "application/x-font-truetype", - "application/x-font-ttf", - "application/xml", - "font/eot", - "font/opentype", - "font/otf", - "image/svg+xml", - "image/vnd.microsoft.icon", - "text/plain", - "text/xml", - ] - name = "Default rules" - } - - header { - action = "delete" - cache_condition = null - destination = "http.Cookie" - ignore_if_set = false - name = "Remove cookies" - priority = 10 - regex = null - request_condition = "Is Download" - response_condition = null - source = null - substitution = null - type = "request" - } - header { - action = "set" - cache_condition = null - destination = "backend" - ignore_if_set = false - name = "Is Download Director" - priority = 10 - regex = null - request_condition = "Is Download" - response_condition = null - source = "F_lb_nyc1_psf_io" - substitution = null - type = "request" - } - header { - action = "set" - cache_condition = null - destination = "backend" - ignore_if_set = false - name = "Is Not Download Backend" - priority = 10 - regex = null - request_condition = "Is Not Download" - response_condition = null - source = "F_cabotage" - substitution = null - type = "request" - } - header { - action = "set" - cache_condition = null - destination = "http.Fastly-Token" - ignore_if_set = false - name = "Fastly Token" - priority = 10 - regex = null - request_condition = null - response_condition = null - source = var.FASTLY_HEADER_TOKEN - substitution = null - type = "request" - } - header { - action = "set" - cache_condition = null - destination = "http.Location" - ignore_if_set = false - name = "www redirect" - priority = 10 - regex = null - request_condition = null - response_condition = "apex redirect" - source = "\"https://test.python.org\" + req.url" - substitution = null - type = "response" - } - header { - action = "set" - cache_condition = null - destination = "http.Strict-Transport-Security" - ignore_if_set = false - name = "HSTS w/ subdomains" - priority = 10 - regex = null - request_condition = null - response_condition = "HSTS w/ subdomains" - source = "\"max-age=63072000; includeSubDomains; preload\"" - substitution = null - type = "response" - } - header { - action = "set" - cache_condition = null - destination = "http.Strict-Transport-Security" - ignore_if_set = false - name = "HSTS w/o subdomains" - priority = 10 - regex = null - request_condition = null - response_condition = "HSTS w/o subdomain" - source = "\"max-age=315360000; preload\"" - substitution = null - type = "response" - } - header { - action = "set" - cache_condition = null - destination = "url" - ignore_if_set = false - name = "Chop off query string" - priority = 10 - regex = null - request_condition = "Is Download" - response_condition = null - source = "regsub(req.url, \"\\?.*$\", \"\")" - substitution = null - type = "request" - } - header { - action = "set" - cache_condition = null - destination = "url" - ignore_if_set = false - name = "Strip Query Strings" - priority = 10 - regex = null - request_condition = "Homepage" - response_condition = null - source = "req.url.path" - substitution = null - type = "request" - } - - healthcheck { - check_interval = 15000 - expected_response = 200 - headers = [] - host = "test.python.org" - http_version = "1.1" - initial = 4 - method = "HEAD" - name = "HAProxy Status" - path = "/_haproxy_status" - threshold = 3 - timeout = 5000 - window = 5 - } - - logging_datadog { - name = "ratelimit-debug" - token = var.DATADOG_API_KEY - region = "US" - } - - logging_s3 { - name = "psf-fastly-logs" - bucket_name = "psf-fastly-logs-eu-west-1" - domain = "s3-eu-west-1.amazonaws.com" - path = "/www-python-org/%Y/%m/%d/" - period = 3600 - gzip_level = 9 - # %h "%{now}V" %l "%{req.request}V %{req.url}V" %{req.proto}V %>s %{resp.http.Content-Length}V %{resp.http.age}V "%{resp.http.x-cache}V" "%{resp.http.x-cache-hits}V" "%{req.http.content-type}V" "%{req.http.accept-language}V" "%{cstr_escape(req.http.user-agent)}V" - format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" - timestamp_format = "%Y-%m-%dT%H:%M:%S.000" - redundancy = "standard" - format_version = 2 - message_type = "classic" - compression_codec = "gzip" - s3_access_key = var.S3_ACCESS_KEY - s3_secret_key = var.S3_SECRET_KEY - } - - logging_syslog { - name = "syslog" - address = "cdn-logs.nyc1.psf.io" - port = 514 - # %h "%{now}V" %l "%{req.request}V %{req.url}V" %{req.proto}V %>s %{resp.http.Content-Length}V %{resp.http.age}V "%{resp.http.x-cache}V" "%{resp.http.x-cache-hits}V" "%{req.http.content-type}V" "%{req.http.accept-language}V" "%{cstr_escape(req.http.user-agent)}V" - format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" - } - - product_enablement { - bot_management = true - brotli_compression = false - domain_inspector = true - image_optimizer = false - name = null - origin_inspector = true - websockets = false - } - - rate_limiter { - action = "log_only" - client_key = "client.ip" - feature_revision = 1 - http_methods = "GET,PUT,TRACE,POST,HEAD,DELETE,PATCH,OPTIONS" - logger_type = "datadog" - name = "test.python.org backends" - penalty_box_duration = 2 - response_object_name = null - rps_limit = 10 - uri_dictionary_name = null - window_size = 10 - - response { - content = <<-EOT - - - Too Many Requests - - -

Too Many Requests

- - - EOT - content_type = "text/html" - status = 429 - } - } - - request_setting { - action = null - bypass_busy_wait = false - default_host = null - force_miss = false - force_ssl = true -# geo_headers = false # ! DEPRECATED - hash_keys = null - max_stale_age = 86400 - name = "Default cache policy" - request_condition = null - timer_support = false - xff = "append" - } - request_setting { - action = "pass" - bypass_busy_wait = false - default_host = null - force_miss = false - force_ssl = false -# geo_headers = false # ! DEPRECATED - hash_keys = null - max_stale_age = 60 - name = "Force Pass" - request_condition = "Uncacheable URLs" - timer_support = false - xff = "append" - } - - response_object { - cache_condition = null - content = null - content_type = null - name = "www redirect" - request_condition = "apex" - response = "Moved Permanently" - status = 301 - } - response_object { - cache_condition = null - content = null - content_type = "text/html" - name = "Generated by IP block list" - request_condition = "Generated by IP block list" - response = "Forbidden" - status = 403 - } -} diff --git a/infra/fastly.tf b/infra/fastly.tf index 8124dddf6..25f3b58cf 100644 --- a/infra/fastly.tf +++ b/infra/fastly.tf @@ -1,15 +1,56 @@ -# Fastly VCL Service -resource "fastly_service_vcl" "frontend-vcl-service" { - name = "NGWAF Testing" +resource "fastly_service_vcl" "test_python_org" { + name = "test.python.org" + default_ttl = 3600 + http3 = false + stale_if_error = false + stale_if_error_ttl = 43200 + activate = true +# +# domain { +# name = "test.python.org" +# } domain { name = var.USER_VCL_SERVICE_DOMAIN_NAME - comment = "NGWAF testing" + comment = "NGWAF testing domain" + } + + backend { + name = "cabotage" + address = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" + port = 443 + shield = "iad-va-us" + auto_loadbalance = false + ssl_check_cert = true + ssl_cert_hostname = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" + ssl_sni_hostname = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" + weight = 100 + max_conn = 200 + connect_timeout = 1000 + first_byte_timeout = 30000 + between_bytes_timeout = 10000 + } + + backend { + name = "loadbalancer" + address = "lb.nyc1.psf.io" + shield = "iad-va-us" + healthcheck = "HAProxy Status" + auto_loadbalance = false + ssl_check_cert = true + ssl_cert_hostname = "lb.psf.io" + ssl_sni_hostname = "lb.psf.io" + ssl_ca_cert = "" # TODO(@ee) + weight = 100 + max_conn = 200 + connect_timeout = 1000 + first_byte_timeout = 15000 + between_bytes_timeout = 10000 } backend { address = var.USER_VCL_SERVICE_BACKEND_HOSTNAME - name = "vcl_service_origin_1" + name = "ngwaf_backend" port = 443 use_ssl = true ssl_cert_hostname = var.USER_VCL_SERVICE_BACKEND_HOSTNAME @@ -17,6 +58,294 @@ resource "fastly_service_vcl" "frontend-vcl-service" { override_host = var.USER_VCL_SERVICE_BACKEND_HOSTNAME } + acl { + name = "Generated_by_IP_block_list" + force_destroy = false + } + + cache_setting { + action = "pass" + cache_condition = "Force Pass No-Cache No-Store" + name = "Pass No-Cache No-Store" + stale_ttl = 0 + ttl = 0 + } + + condition { + name = "Force Pass No-Cache No-Store" + priority = 10 + statement = "beresp.http.Cache-Control ~ \"(no-cache|no-store)\"" + type = "CACHE" + } + condition { + name = "Generated by IP block list" + priority = 0 + statement = "client.ip ~ Generated_by_IP_block_list" + type = "REQUEST" + } + condition { + name = "HSTS w/ subdomains" + priority = 10 + statement = "req.http.host == \"test.python.org\"" + type = "RESPONSE" + } + condition { + name = "HSTS w/o subdomain" + priority = 10 + statement = "req.http.host == \"test.python.org\"" + type = "RESPONSE" + } + condition { + name = "Homepage" + priority = 10 + statement = "req.url.path ~ \"^/$\"" + type = "REQUEST" + } + condition { + name = "Is Download" + priority = 10 + statement = "req.url ~ \"^/ftp/\"" + type = "REQUEST" + } + condition { + name = "Is Not Download" + priority = 5 + statement = "req.url !~ \"^/ftp/\"" + type = "REQUEST" + } + condition { + name = "Uncacheable URLs" + priority = 10 + statement = "req.url ~ \"^/(api|admin)/\"" + type = "REQUEST" + } + condition { + name = "apex redirect" + priority = 10 + statement = "req.http.Host == \"test.python.org\"" + type = "RESPONSE" + } + condition { + name = "apex" + priority = 1 + statement = "req.http.host == \"test.python.org\"" + type = "REQUEST" + } + + gzip { + name = "Default rules" + content_types = [ + "application/javascript", + "text/css", + "application/javascript", + "text/javascript", + "application/json", + "application/vnd.ms-fontobject", + "application/x-font-opentype", + "application/x-font-truetype", + "application/x-font-ttf", + "application/xml", + "font/eot", + "font/opentype", + "font/otf", + "image/svg+xml", + "image/vnd.microsoft.icon", + "text/plain", + "text/xml", + ] + } + + header { + action = "delete" + destination = "http.Cookie" + name = "Remove cookies" + priority = 10 + request_condition = "Is Download" + type = "request" + } + header { + action = "set" + destination = "backend" + name = "Is Download Director" + priority = 10 + request_condition = "Is Download" + source = "F_lb_nyc1_psf_io" + type = "request" + } + header { + action = "set" + destination = "backend" + name = "Is Not Download Backend" + priority = 10 + request_condition = "Is Not Download" + source = "F_cabotage" + type = "request" + } + header { + action = "set" + destination = "http.Fastly-Token" + name = "Fastly Token" + priority = 10 + source = var.FASTLY_HEADER_TOKEN + type = "request" + } + header { + action = "set" + destination = "http.Location" + name = "www redirect" + priority = 10 + response_condition = "apex redirect" + source = "\"https://test.python.org\" + req.url" + type = "response" + } + header { + action = "set" + destination = "http.Strict-Transport-Security" + name = "HSTS w/ subdomains" + priority = 10 + response_condition = "HSTS w/ subdomains" + source = "\"max-age=63072000; includeSubDomains; preload\"" + type = "response" + } + header { + action = "set" + destination = "http.Strict-Transport-Security" + name = "HSTS w/o subdomains" + priority = 10 + response_condition = "HSTS w/o subdomain" + source = "\"max-age=315360000; preload\"" + type = "response" + } + header { + action = "set" + destination = "url" + name = "Chop off query string" + priority = 10 + request_condition = "Is Download" + source = "regsub(req.url, \"\\?.*$\", \"\")" + type = "request" + } + header { + action = "set" + destination = "url" + name = "Strip Query Strings" + priority = 10 + request_condition = "Homepage" + source = "req.url.path" + type = "request" + } + + healthcheck { + check_interval = 15000 + expected_response = 200 + host = "test.python.org" + http_version = "1.1" + initial = 4 + method = "HEAD" + name = "HAProxy Status" + path = "/_haproxy_status" + threshold = 3 + timeout = 5000 + window = 5 + } + + logging_datadog { + name = "ratelimit-debug" + token = var.DATADOG_API_KEY + region = "US" + } + + logging_s3 { + name = "psf-fastly-logs" + bucket_name = "psf-fastly-logs-eu-west-1" + domain = "s3-eu-west-1.amazonaws.com" + path = "/www-python-org/%Y/%m/%d/" + period = 3600 + gzip_level = 9 + format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" + timestamp_format = "%Y-%m-%dT%H:%M:%S.000" + redundancy = "standard" + format_version = 2 + message_type = "classic" + s3_access_key = var.S3_ACCESS_KEY + s3_secret_key = var.S3_SECRET_KEY + } + + logging_syslog { + name = "syslog" + address = "cdn-logs.nyc1.psf.io" + port = 514 + format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" + } + + product_enablement { + bot_management = true + brotli_compression = false + domain_inspector = true + image_optimizer = false + origin_inspector = true + websockets = false + } + + rate_limiter { + action = "log_only" + client_key = "client.ip" + feature_revision = 1 + http_methods = "GET,PUT,TRACE,POST,HEAD,DELETE,PATCH,OPTIONS" + logger_type = "datadog" + name = "test.python.org backends" + penalty_box_duration = 2 + rps_limit = 10 + window_size = 10 + + response { + content = <<-EOT + + + Too Many Requests + + +

Too Many Requests

+ + + EOT + content_type = "text/html" + status = 429 + } + } + + request_setting { + action = null + bypass_busy_wait = false + force_ssl = true + max_stale_age = 86400 + name = "Default cache policy" + xff = "append" + } + request_setting { + action = "pass" + bypass_busy_wait = false + force_ssl = false + max_stale_age = 60 + name = "Force Pass" + request_condition = "Uncacheable URLs" + xff = "append" + } + + response_object { + name = "www redirect" + request_condition = "apex" + response = "Moved Permanently" + status = 301 + } + response_object { + content_type = "text/html" + name = "Generated by IP block list" + request_condition = "Generated by IP block list" + response = "Forbidden" + status = 403 + } + # NGWAF Dynamic Snippets dynamicsnippet { name = "ngwaf_config_init" @@ -56,21 +385,21 @@ resource "fastly_service_vcl" "frontend-vcl-service" { # Fastly Service Dictionary Items resource "fastly_service_dictionary_items" "edge_security_dictionary_items" { for_each = { - for d in fastly_service_vcl.frontend-vcl-service.dictionary : d.name => d if d.name == var.Edge_Security_dictionary + for d in fastly_service_vcl.test_python_org.dictionary : d.name => d if d.name == var.Edge_Security_dictionary } - service_id = fastly_service_vcl.frontend-vcl-service.id + service_id = fastly_service_vcl.test_python_org.id dictionary_id = each.value.dictionary_id items = { - Enabled: "100" + Enabled : "100" } } # Fastly Service Dynamic Snippet Contents resource "fastly_service_dynamic_snippet_content" "ngwaf_config_init" { for_each = { - for d in fastly_service_vcl.frontend-vcl-service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_init" + for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_init" } - service_id = fastly_service_vcl.frontend-vcl-service.id + service_id = fastly_service_vcl.test_python_org.id snippet_id = each.value.snippet_id content = "### Fastly managed ngwaf_config_init" manage_snippets = false @@ -78,9 +407,9 @@ resource "fastly_service_dynamic_snippet_content" "ngwaf_config_init" { resource "fastly_service_dynamic_snippet_content" "ngwaf_config_miss" { for_each = { - for d in fastly_service_vcl.frontend-vcl-service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_miss" + for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_miss" } - service_id = fastly_service_vcl.frontend-vcl-service.id + service_id = fastly_service_vcl.test_python_org.id snippet_id = each.value.snippet_id content = "### Fastly managed ngwaf_config_miss" manage_snippets = false @@ -88,9 +417,9 @@ resource "fastly_service_dynamic_snippet_content" "ngwaf_config_miss" { resource "fastly_service_dynamic_snippet_content" "ngwaf_config_pass" { for_each = { - for d in fastly_service_vcl.frontend-vcl-service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_pass" + for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_pass" } - service_id = fastly_service_vcl.frontend-vcl-service.id + service_id = fastly_service_vcl.test_python_org.id snippet_id = each.value.snippet_id content = "### Fastly managed ngwaf_config_pass" manage_snippets = false @@ -98,10 +427,10 @@ resource "fastly_service_dynamic_snippet_content" "ngwaf_config_pass" { resource "fastly_service_dynamic_snippet_content" "ngwaf_config_deliver" { for_each = { - for d in fastly_service_vcl.frontend-vcl-service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_deliver" + for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_deliver" } - service_id = fastly_service_vcl.frontend-vcl-service.id + service_id = fastly_service_vcl.test_python_org.id snippet_id = each.value.snippet_id content = "### Fastly managed ngwaf_config_deliver" manage_snippets = false -} \ No newline at end of file +} diff --git a/infra/main.tf b/infra/main.tf index 9fd9c697a..b2cc0be80 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -1,16 +1,8 @@ terraform { - required_providers { - fastly = { - source = "fastly/fastly" - version = "5.13.0" - } - sigsci = { - source = "signalsciences/sigsci" - version = "3.3.0" - } - aws = { - source = "hashicorp/aws" - version = "~> 5.0" + cloud { + organization = "psf" + workspaces { + name = "test-pythondotorg" } } } @@ -27,8 +19,8 @@ provider "aws" { } provider "sigsci" { - corp = var.NGWAF_CORP - email = var.NGWAF_EMAIL - auth_token = var.NGWAF_TOKEN + corp = var.NGWAF_CORP + email = var.NGWAF_EMAIL + auth_token = var.NGWAF_TOKEN fastly_api_key = var.FASTLY_API_KEY } \ No newline at end of file diff --git a/infra/ngwaf.tf b/infra/ngwaf.tf index 4187fd117..df1b18538 100644 --- a/infra/ngwaf.tf +++ b/infra/ngwaf.tf @@ -5,12 +5,12 @@ resource "sigsci_edge_deployment" "ngwaf_edge_site_service" { resource "sigsci_edge_deployment_service" "ngwaf_edge_service_link" { site_short_name = var.NGWAF_SITE - fastly_sid = fastly_service_vcl.frontend-vcl-service.id + fastly_sid = fastly_service_vcl.test_python_org.id activate_version = true percent_enabled = 100 depends_on = [ sigsci_edge_deployment.ngwaf_edge_site_service, - fastly_service_vcl.frontend-vcl-service, + fastly_service_vcl.test_python_org, fastly_service_dictionary_items.edge_security_dictionary_items, fastly_service_dynamic_snippet_content.ngwaf_config_init, fastly_service_dynamic_snippet_content.ngwaf_config_miss, @@ -21,8 +21,8 @@ resource "sigsci_edge_deployment_service" "ngwaf_edge_service_link" { resource "sigsci_edge_deployment_service_backend" "ngwaf_edge_service_backend_sync" { site_short_name = var.NGWAF_SITE - fastly_sid = fastly_service_vcl.frontend-vcl-service.id - fastly_service_vcl_active_version = fastly_service_vcl.frontend-vcl-service.active_version + fastly_sid = fastly_service_vcl.test_python_org.id + fastly_service_vcl_active_version = fastly_service_vcl.test_python_org.active_version depends_on = [ sigsci_edge_deployment_service.ngwaf_edge_service_link, ] diff --git a/infra/out.tf b/infra/out.tf index 0ee354d8c..721bfb454 100644 --- a/infra/out.tf +++ b/infra/out.tf @@ -1,13 +1,13 @@ output "testing-the_ngwaf" { - value = < Date: Tue, 27 Aug 2024 13:20:15 -0500 Subject: [PATCH 10/20] feat: disable ngwaf somewhat; add cert, header token, --- infra/certs/psf.io.pem | 1 + infra/fastly.tf | 123 ++++++++++----------------------- infra/ngwaf.tf | 113 ++++++++++++++++++++++-------- infra/out.tf | 32 ++++----- infra/terraform.tfvars.example | 2 - infra/variables.tf | 11 --- 6 files changed, 136 insertions(+), 146 deletions(-) create mode 100644 infra/certs/psf.io.pem diff --git a/infra/certs/psf.io.pem b/infra/certs/psf.io.pem new file mode 100644 index 000000000..45571f59b --- /dev/null +++ b/infra/certs/psf.io.pem @@ -0,0 +1 @@ +-----BEGIN CERTIFICATE----- MIIEQzCCAyugAwIBAgIUYH38nEb2KLRgscKhjcNpBLRUz+UwDQYJKoZIhvcNAQEL BQAwgbAxCzAJBgNVBAYTAlVTMQ8wDQYDVQQIDAZPcmVnb24xEjAQBgNVBAcMCUJl YXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xHDAa BgNVBAsME0luZnJhc3RydWN0dXJlIFRlYW0xDzANBgNVBAMMBlBTRl9DQTEoMCYG CSqGSIb3DQEJARYZaW5mcmFzdHJ1Y3R1cmVAcHl0aG9uLm9yZzAeFw0yNDAyMTIx NzU0MDZaFw0yOTAyMTAxNzU0MDZaMIGwMQswCQYDVQQGEwJVUzEPMA0GA1UECAwG T3JlZ29uMRIwEAYDVQQHDAlCZWF2ZXJ0b24xIzAhBgNVBAoMGlB5dGhvbiBTb2Z0 d2FyZSBGb3VuZGF0aW9uMRwwGgYDVQQLDBNJbmZyYXN0cnVjdHVyZSBUZWFtMQ8w DQYDVQQDDAZQU0ZfQ0ExKDAmBgkqhkiG9w0BCQEWGWluZnJhc3RydWN0dXJlQHB5 dGhvbi5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCXAZagv2UK AEnnnnrK/WWcZIKo/l+HTgL01XhReu9CDNs3f3ESlRT3Y4Hbla/pYRu9VM8tMGYS xG5FGJQ2JPVnKCb3mIEC7wy9+VOaQIp3l8+o0lDQhsOZs78ZA8XQpNLD5OURsUHJ re1U6WOTryMJwxpO+DzSBU+oSwfdn2k0BAJqSeIU45hHXeHO24z7GePuk3I1wb+E vfhtdIF/tHvF1I6h7ntmHUeUWYrTKXKB9meMAFwEC1ZNoN1z05X68cSeK8dAsxYh ghmQnUZ4hHH8pLlhYW/QBTol0nutwgHPyC9FIJnZzX50xAMRx3TKP1IbIehWBwF2 CYJq6pRBZ1mfAgMBAAGjUzBRMB0GA1UdDgQWBBQrAQVRNWd6eVr6ZGn8vshzgS09 qDAfBgNVHSMEGDAWgBQrAQVRNWd6eVr6ZGn8vshzgS09qDAPBgNVHRMBAf8EBTAD AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBmtyljZ1q2manMvIMEtXtc9lq3gwxIP4Pq ic5hKuEHDSy5iN0vZRhoqfgPzXMy61zCrvLmvxv8nN2B4Us44KQRzWwDvi8SavfQ LxRZ4KLe5Bg7MNfIKM/ZqYqHIt1FtVFYR7UyEILN/yDCyQC+8n6s8RLmT5OtZHPL 0YAyHgdao4qCICkZShbCukq81ULvkq7i6QvHWZrVGAIc/1nN71QNEUMr9KtlTKO3 TeSd+l13+CDGwMXUpglDiFL329TmG5pKr/zoTCGDmRvEfRPtICwY3FgqGDpmIwhw dXq0JPGHrFODeVrchUMSGqXhAZ+k/9YdJlGLbv3WJmD1GwFTs3Wf -----END CERTIFICATE----- \ No newline at end of file diff --git a/infra/fastly.tf b/infra/fastly.tf index 25f3b58cf..b289b28b0 100644 --- a/infra/fastly.tf +++ b/infra/fastly.tf @@ -17,13 +17,13 @@ resource "fastly_service_vcl" "test_python_org" { backend { name = "cabotage" - address = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" + address = "pythondotorg.ingress.us-east-2.psfhosted.computer" port = 443 shield = "iad-va-us" auto_loadbalance = false ssl_check_cert = true - ssl_cert_hostname = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" - ssl_sni_hostname = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" + ssl_cert_hostname = "pythondotorg.ingress.us-east-2.psfhosted.computer" + ssl_sni_hostname = "pythondotorg.ingress.us-east-2.psfhosted.computer" weight = 100 max_conn = 200 connect_timeout = 1000 @@ -40,7 +40,7 @@ resource "fastly_service_vcl" "test_python_org" { ssl_check_cert = true ssl_cert_hostname = "lb.psf.io" ssl_sni_hostname = "lb.psf.io" - ssl_ca_cert = "" # TODO(@ee) + ssl_ca_cert = file("${path.module}/certs/psf.io.pem") weight = 100 max_conn = 200 connect_timeout = 1000 @@ -169,7 +169,7 @@ resource "fastly_service_vcl" "test_python_org" { name = "Is Download Director" priority = 10 request_condition = "Is Download" - source = "F_lb_nyc1_psf_io" + source = "loadbalancer" type = "request" } header { @@ -178,7 +178,7 @@ resource "fastly_service_vcl" "test_python_org" { name = "Is Not Download Backend" priority = 10 request_condition = "Is Not Download" - source = "F_cabotage" + source = "cabotage" type = "request" } header { @@ -186,7 +186,7 @@ resource "fastly_service_vcl" "test_python_org" { destination = "http.Fastly-Token" name = "Fastly Token" priority = 10 - source = var.FASTLY_HEADER_TOKEN + source = "\"${var.FASTLY_HEADER_TOKEN}\"" type = "request" } header { @@ -267,8 +267,8 @@ resource "fastly_service_vcl" "test_python_org" { redundancy = "standard" format_version = 2 message_type = "classic" - s3_access_key = var.S3_ACCESS_KEY - s3_secret_key = var.S3_SECRET_KEY + s3_access_key = var.AWS_ACCESS_KEY_ID + s3_secret_key = var.AWS_SECRET_ACCESS_KEY } logging_syslog { @@ -347,33 +347,33 @@ resource "fastly_service_vcl" "test_python_org" { } # NGWAF Dynamic Snippets - dynamicsnippet { - name = "ngwaf_config_init" - type = "init" - priority = 0 - } - - dynamicsnippet { - name = "ngwaf_config_miss" - type = "miss" - priority = 9000 - } - - dynamicsnippet { - name = "ngwaf_config_pass" - type = "pass" - priority = 9000 - } - - dynamicsnippet { - name = "ngwaf_config_deliver" - type = "deliver" - priority = 9000 - } +# dynamicsnippet { +# name = "ngwaf_config_init" +# type = "init" +# priority = 0 +# } +# +# dynamicsnippet { +# name = "ngwaf_config_miss" +# type = "miss" +# priority = 9000 +# } +# +# dynamicsnippet { +# name = "ngwaf_config_pass" +# type = "pass" +# priority = 9000 +# } +# +# dynamicsnippet { +# name = "ngwaf_config_deliver" +# type = "deliver" +# priority = 9000 +# } - dictionary { - name = var.Edge_Security_dictionary - } +# dictionary { +# name = var.Edge_Security_dictionary +# } lifecycle { ignore_changes = [product_enablement] @@ -381,56 +381,3 @@ resource "fastly_service_vcl" "test_python_org" { force_destroy = true } - -# Fastly Service Dictionary Items -resource "fastly_service_dictionary_items" "edge_security_dictionary_items" { - for_each = { - for d in fastly_service_vcl.test_python_org.dictionary : d.name => d if d.name == var.Edge_Security_dictionary - } - service_id = fastly_service_vcl.test_python_org.id - dictionary_id = each.value.dictionary_id - items = { - Enabled : "100" - } -} - -# Fastly Service Dynamic Snippet Contents -resource "fastly_service_dynamic_snippet_content" "ngwaf_config_init" { - for_each = { - for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_init" - } - service_id = fastly_service_vcl.test_python_org.id - snippet_id = each.value.snippet_id - content = "### Fastly managed ngwaf_config_init" - manage_snippets = false -} - -resource "fastly_service_dynamic_snippet_content" "ngwaf_config_miss" { - for_each = { - for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_miss" - } - service_id = fastly_service_vcl.test_python_org.id - snippet_id = each.value.snippet_id - content = "### Fastly managed ngwaf_config_miss" - manage_snippets = false -} - -resource "fastly_service_dynamic_snippet_content" "ngwaf_config_pass" { - for_each = { - for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_pass" - } - service_id = fastly_service_vcl.test_python_org.id - snippet_id = each.value.snippet_id - content = "### Fastly managed ngwaf_config_pass" - manage_snippets = false -} - -resource "fastly_service_dynamic_snippet_content" "ngwaf_config_deliver" { - for_each = { - for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_deliver" - } - service_id = fastly_service_vcl.test_python_org.id - snippet_id = each.value.snippet_id - content = "### Fastly managed ngwaf_config_deliver" - manage_snippets = false -} diff --git a/infra/ngwaf.tf b/infra/ngwaf.tf index df1b18538..45f067401 100644 --- a/infra/ngwaf.tf +++ b/infra/ngwaf.tf @@ -1,29 +1,84 @@ -# NGWAF Edge Deployment -resource "sigsci_edge_deployment" "ngwaf_edge_site_service" { - site_short_name = var.NGWAF_SITE -} - -resource "sigsci_edge_deployment_service" "ngwaf_edge_service_link" { - site_short_name = var.NGWAF_SITE - fastly_sid = fastly_service_vcl.test_python_org.id - activate_version = true - percent_enabled = 100 - depends_on = [ - sigsci_edge_deployment.ngwaf_edge_site_service, - fastly_service_vcl.test_python_org, - fastly_service_dictionary_items.edge_security_dictionary_items, - fastly_service_dynamic_snippet_content.ngwaf_config_init, - fastly_service_dynamic_snippet_content.ngwaf_config_miss, - fastly_service_dynamic_snippet_content.ngwaf_config_pass, - fastly_service_dynamic_snippet_content.ngwaf_config_deliver, - ] -} - -resource "sigsci_edge_deployment_service_backend" "ngwaf_edge_service_backend_sync" { - site_short_name = var.NGWAF_SITE - fastly_sid = fastly_service_vcl.test_python_org.id - fastly_service_vcl_active_version = fastly_service_vcl.test_python_org.active_version - depends_on = [ - sigsci_edge_deployment_service.ngwaf_edge_service_link, - ] -} \ No newline at end of file +# # NGWAF Edge Deployment +# +# # Fastly Service Dictionary Items +# resource "fastly_service_dictionary_items" "edge_security_dictionary_items" { +# for_each = { +# for d in fastly_service_vcl.test_python_org.dictionary : d.name => d if d.name == var.Edge_Security_dictionary +# } +# service_id = fastly_service_vcl.test_python_org.id +# dictionary_id = each.value.dictionary_id +# items = { +# Enabled : "100" +# } +# } +# +# # Fastly Service Dynamic Snippet Contents +# resource "fastly_service_dynamic_snippet_content" "ngwaf_config_init" { +# for_each = { +# for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_init" +# } +# service_id = fastly_service_vcl.test_python_org.id +# snippet_id = each.value.snippet_id +# content = "### Fastly managed ngwaf_config_init" +# manage_snippets = false +# } +# +# resource "fastly_service_dynamic_snippet_content" "ngwaf_config_miss" { +# for_each = { +# for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_miss" +# } +# service_id = fastly_service_vcl.test_python_org.id +# snippet_id = each.value.snippet_id +# content = "### Fastly managed ngwaf_config_miss" +# manage_snippets = false +# } +# +# resource "fastly_service_dynamic_snippet_content" "ngwaf_config_pass" { +# for_each = { +# for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_pass" +# } +# service_id = fastly_service_vcl.test_python_org.id +# snippet_id = each.value.snippet_id +# content = "### Fastly managed ngwaf_config_pass" +# manage_snippets = false +# } +# +# resource "fastly_service_dynamic_snippet_content" "ngwaf_config_deliver" { +# for_each = { +# for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_deliver" +# } +# service_id = fastly_service_vcl.test_python_org.id +# snippet_id = each.value.snippet_id +# content = "### Fastly managed ngwaf_config_deliver" +# manage_snippets = false +# } +# +# +# resource "sigsci_edge_deployment" "ngwaf_edge_site_service" { +# site_short_name = var.NGWAF_SITE +# } +# +# resource "sigsci_edge_deployment_service" "ngwaf_edge_service_link" { +# site_short_name = var.NGWAF_SITE +# fastly_sid = fastly_service_vcl.test_python_org.id +# activate_version = true +# percent_enabled = 100 +# depends_on = [ +# sigsci_edge_deployment.ngwaf_edge_site_service, +# fastly_service_vcl.test_python_org, +# fastly_service_dictionary_items.edge_security_dictionary_items, +# fastly_service_dynamic_snippet_content.ngwaf_config_init, +# fastly_service_dynamic_snippet_content.ngwaf_config_miss, +# fastly_service_dynamic_snippet_content.ngwaf_config_pass, +# fastly_service_dynamic_snippet_content.ngwaf_config_deliver, +# ] +# } +# +# resource "sigsci_edge_deployment_service_backend" "ngwaf_edge_service_backend_sync" { +# site_short_name = var.NGWAF_SITE +# fastly_sid = fastly_service_vcl.test_python_org.id +# fastly_service_vcl_active_version = fastly_service_vcl.test_python_org.active_version +# depends_on = [ +# sigsci_edge_deployment_service.ngwaf_edge_service_link, +# ] +# } \ No newline at end of file diff --git a/infra/out.tf b/infra/out.tf index 721bfb454..e56a77748 100644 --- a/infra/out.tf +++ b/infra/out.tf @@ -1,16 +1,16 @@ -output "testing-the_ngwaf" { - value = < Date: Tue, 27 Aug 2024 13:21:33 -0500 Subject: [PATCH 11/20] chore: fix name --- infra/aws.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/aws.tf b/infra/aws.tf index 65be17059..88f36a5b7 100644 --- a/infra/aws.tf +++ b/infra/aws.tf @@ -1,4 +1,4 @@ -resource "aws_route53_record" "ngwaf_cname" { +resource "aws_route53_record" "dns_cname" { zone_id = var.route53_zone_id name = var.route53_record_name type = "CNAME" From 197661f7fb6c3eb02f6c1494de7fab61da573264 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 27 Aug 2024 13:41:23 -0500 Subject: [PATCH 12/20] chore: allow edits for aws dns --- infra/aws.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/aws.tf b/infra/aws.tf index 88f36a5b7..d38d99e65 100644 --- a/infra/aws.tf +++ b/infra/aws.tf @@ -4,4 +4,5 @@ resource "aws_route53_record" "dns_cname" { type = "CNAME" ttl = var.route53_record_ttl records = ["dualstack.python.map.fastly.net"] + allow_overwrite = true } \ No newline at end of file From 49c966d3b856957c03d123725f89d2588c70a119 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 27 Aug 2024 13:41:34 -0500 Subject: [PATCH 13/20] fix: update bad copypasta --- infra/certs/psf.io.pem | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/infra/certs/psf.io.pem b/infra/certs/psf.io.pem index 45571f59b..7952bb36b 100644 --- a/infra/certs/psf.io.pem +++ b/infra/certs/psf.io.pem @@ -1 +1,25 @@ ------BEGIN CERTIFICATE----- MIIEQzCCAyugAwIBAgIUYH38nEb2KLRgscKhjcNpBLRUz+UwDQYJKoZIhvcNAQEL BQAwgbAxCzAJBgNVBAYTAlVTMQ8wDQYDVQQIDAZPcmVnb24xEjAQBgNVBAcMCUJl YXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xHDAa BgNVBAsME0luZnJhc3RydWN0dXJlIFRlYW0xDzANBgNVBAMMBlBTRl9DQTEoMCYG CSqGSIb3DQEJARYZaW5mcmFzdHJ1Y3R1cmVAcHl0aG9uLm9yZzAeFw0yNDAyMTIx NzU0MDZaFw0yOTAyMTAxNzU0MDZaMIGwMQswCQYDVQQGEwJVUzEPMA0GA1UECAwG T3JlZ29uMRIwEAYDVQQHDAlCZWF2ZXJ0b24xIzAhBgNVBAoMGlB5dGhvbiBTb2Z0 d2FyZSBGb3VuZGF0aW9uMRwwGgYDVQQLDBNJbmZyYXN0cnVjdHVyZSBUZWFtMQ8w DQYDVQQDDAZQU0ZfQ0ExKDAmBgkqhkiG9w0BCQEWGWluZnJhc3RydWN0dXJlQHB5 dGhvbi5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCXAZagv2UK AEnnnnrK/WWcZIKo/l+HTgL01XhReu9CDNs3f3ESlRT3Y4Hbla/pYRu9VM8tMGYS xG5FGJQ2JPVnKCb3mIEC7wy9+VOaQIp3l8+o0lDQhsOZs78ZA8XQpNLD5OURsUHJ re1U6WOTryMJwxpO+DzSBU+oSwfdn2k0BAJqSeIU45hHXeHO24z7GePuk3I1wb+E vfhtdIF/tHvF1I6h7ntmHUeUWYrTKXKB9meMAFwEC1ZNoN1z05X68cSeK8dAsxYh ghmQnUZ4hHH8pLlhYW/QBTol0nutwgHPyC9FIJnZzX50xAMRx3TKP1IbIehWBwF2 CYJq6pRBZ1mfAgMBAAGjUzBRMB0GA1UdDgQWBBQrAQVRNWd6eVr6ZGn8vshzgS09 qDAfBgNVHSMEGDAWgBQrAQVRNWd6eVr6ZGn8vshzgS09qDAPBgNVHRMBAf8EBTAD AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBmtyljZ1q2manMvIMEtXtc9lq3gwxIP4Pq ic5hKuEHDSy5iN0vZRhoqfgPzXMy61zCrvLmvxv8nN2B4Us44KQRzWwDvi8SavfQ LxRZ4KLe5Bg7MNfIKM/ZqYqHIt1FtVFYR7UyEILN/yDCyQC+8n6s8RLmT5OtZHPL 0YAyHgdao4qCICkZShbCukq81ULvkq7i6QvHWZrVGAIc/1nN71QNEUMr9KtlTKO3 TeSd+l13+CDGwMXUpglDiFL329TmG5pKr/zoTCGDmRvEfRPtICwY3FgqGDpmIwhw dXq0JPGHrFODeVrchUMSGqXhAZ+k/9YdJlGLbv3WJmD1GwFTs3Wf -----END CERTIFICATE----- \ No newline at end of file +-----BEGIN CERTIFICATE----- +MIIEQzCCAyugAwIBAgIUYH38nEb2KLRgscKhjcNpBLRUz+UwDQYJKoZIhvcNAQEL +BQAwgbAxCzAJBgNVBAYTAlVTMQ8wDQYDVQQIDAZPcmVnb24xEjAQBgNVBAcMCUJl +YXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xHDAa +BgNVBAsME0luZnJhc3RydWN0dXJlIFRlYW0xDzANBgNVBAMMBlBTRl9DQTEoMCYG +CSqGSIb3DQEJARYZaW5mcmFzdHJ1Y3R1cmVAcHl0aG9uLm9yZzAeFw0yNDAyMTIx +NzU0MDZaFw0yOTAyMTAxNzU0MDZaMIGwMQswCQYDVQQGEwJVUzEPMA0GA1UECAwG +T3JlZ29uMRIwEAYDVQQHDAlCZWF2ZXJ0b24xIzAhBgNVBAoMGlB5dGhvbiBTb2Z0 +d2FyZSBGb3VuZGF0aW9uMRwwGgYDVQQLDBNJbmZyYXN0cnVjdHVyZSBUZWFtMQ8w +DQYDVQQDDAZQU0ZfQ0ExKDAmBgkqhkiG9w0BCQEWGWluZnJhc3RydWN0dXJlQHB5 +dGhvbi5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCXAZagv2UK +AEnnnnrK/WWcZIKo/l+HTgL01XhReu9CDNs3f3ESlRT3Y4Hbla/pYRu9VM8tMGYS +xG5FGJQ2JPVnKCb3mIEC7wy9+VOaQIp3l8+o0lDQhsOZs78ZA8XQpNLD5OURsUHJ +re1U6WOTryMJwxpO+DzSBU+oSwfdn2k0BAJqSeIU45hHXeHO24z7GePuk3I1wb+E +vfhtdIF/tHvF1I6h7ntmHUeUWYrTKXKB9meMAFwEC1ZNoN1z05X68cSeK8dAsxYh +ghmQnUZ4hHH8pLlhYW/QBTol0nutwgHPyC9FIJnZzX50xAMRx3TKP1IbIehWBwF2 +CYJq6pRBZ1mfAgMBAAGjUzBRMB0GA1UdDgQWBBQrAQVRNWd6eVr6ZGn8vshzgS09 +qDAfBgNVHSMEGDAWgBQrAQVRNWd6eVr6ZGn8vshzgS09qDAPBgNVHRMBAf8EBTAD +AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBmtyljZ1q2manMvIMEtXtc9lq3gwxIP4Pq +ic5hKuEHDSy5iN0vZRhoqfgPzXMy61zCrvLmvxv8nN2B4Us44KQRzWwDvi8SavfQ +LxRZ4KLe5Bg7MNfIKM/ZqYqHIt1FtVFYR7UyEILN/yDCyQC+8n6s8RLmT5OtZHPL +0YAyHgdao4qCICkZShbCukq81ULvkq7i6QvHWZrVGAIc/1nN71QNEUMr9KtlTKO3 +TeSd+l13+CDGwMXUpglDiFL329TmG5pKr/zoTCGDmRvEfRPtICwY3FgqGDpmIwhw +dXq0JPGHrFODeVrchUMSGqXhAZ+k/9YdJlGLbv3WJmD1GwFTs3Wf +-----END CERTIFICATE----- \ No newline at end of file From 26f535921abede505b9e9b6cc25caa2890838596 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 27 Aug 2024 13:41:45 -0500 Subject: [PATCH 14/20] fix: use correct resource refs --- infra/fastly.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/fastly.tf b/infra/fastly.tf index b289b28b0..217a33684 100644 --- a/infra/fastly.tf +++ b/infra/fastly.tf @@ -169,7 +169,7 @@ resource "fastly_service_vcl" "test_python_org" { name = "Is Download Director" priority = 10 request_condition = "Is Download" - source = "loadbalancer" + source = "F_loadbalancer" type = "request" } header { @@ -178,7 +178,7 @@ resource "fastly_service_vcl" "test_python_org" { name = "Is Not Download Backend" priority = 10 request_condition = "Is Not Download" - source = "cabotage" + source = "F_cabotage" type = "request" } header { From abd398f7391d160f5a4a83c6f9a8773a75281068 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 27 Aug 2024 13:48:47 -0500 Subject: [PATCH 15/20] fix(?): try to make ssl work :(, fake the host --- infra/fastly.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/fastly.tf b/infra/fastly.tf index 217a33684..e719a2eb8 100644 --- a/infra/fastly.tf +++ b/infra/fastly.tf @@ -21,6 +21,7 @@ resource "fastly_service_vcl" "test_python_org" { port = 443 shield = "iad-va-us" auto_loadbalance = false + use_ssl = true ssl_check_cert = true ssl_cert_hostname = "pythondotorg.ingress.us-east-2.psfhosted.computer" ssl_sni_hostname = "pythondotorg.ingress.us-east-2.psfhosted.computer" @@ -29,6 +30,7 @@ resource "fastly_service_vcl" "test_python_org" { connect_timeout = 1000 first_byte_timeout = 30000 between_bytes_timeout = 10000 + override_host = "www.python.org" } backend { @@ -37,6 +39,7 @@ resource "fastly_service_vcl" "test_python_org" { shield = "iad-va-us" healthcheck = "HAProxy Status" auto_loadbalance = false + use_ssl = true ssl_check_cert = true ssl_cert_hostname = "lb.psf.io" ssl_sni_hostname = "lb.psf.io" @@ -46,6 +49,7 @@ resource "fastly_service_vcl" "test_python_org" { connect_timeout = 1000 first_byte_timeout = 15000 between_bytes_timeout = 10000 + override_host = "www.python.org" } backend { From 470dd27868396625008df805dddf94a2101d7cfb Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 27 Aug 2024 14:02:30 -0500 Subject: [PATCH 16/20] chore: reenable ngwaf now that things work --- infra/fastly.tf | 62 +++++++++--------- infra/ngwaf.tf | 168 ++++++++++++++++++++++++------------------------ infra/out.tf | 32 ++++----- 3 files changed, 131 insertions(+), 131 deletions(-) diff --git a/infra/fastly.tf b/infra/fastly.tf index e719a2eb8..2a6eb5b82 100644 --- a/infra/fastly.tf +++ b/infra/fastly.tf @@ -5,10 +5,10 @@ resource "fastly_service_vcl" "test_python_org" { stale_if_error = false stale_if_error_ttl = 43200 activate = true -# -# domain { -# name = "test.python.org" -# } + + domain { + name = "test.python.org" + } domain { name = var.USER_VCL_SERVICE_DOMAIN_NAME @@ -350,34 +350,34 @@ resource "fastly_service_vcl" "test_python_org" { status = 403 } - # NGWAF Dynamic Snippets -# dynamicsnippet { -# name = "ngwaf_config_init" -# type = "init" -# priority = 0 -# } -# -# dynamicsnippet { -# name = "ngwaf_config_miss" -# type = "miss" -# priority = 9000 -# } -# -# dynamicsnippet { -# name = "ngwaf_config_pass" -# type = "pass" -# priority = 9000 -# } -# -# dynamicsnippet { -# name = "ngwaf_config_deliver" -# type = "deliver" -# priority = 9000 -# } +# NGWAF Dynamic Snippets + dynamicsnippet { + name = "ngwaf_config_init" + type = "init" + priority = 0 + } + + dynamicsnippet { + name = "ngwaf_config_miss" + type = "miss" + priority = 9000 + } -# dictionary { -# name = var.Edge_Security_dictionary -# } + dynamicsnippet { + name = "ngwaf_config_pass" + type = "pass" + priority = 9000 + } + + dynamicsnippet { + name = "ngwaf_config_deliver" + type = "deliver" + priority = 9000 + } + + dictionary { + name = var.Edge_Security_dictionary + } lifecycle { ignore_changes = [product_enablement] diff --git a/infra/ngwaf.tf b/infra/ngwaf.tf index 45f067401..612206782 100644 --- a/infra/ngwaf.tf +++ b/infra/ngwaf.tf @@ -1,84 +1,84 @@ -# # NGWAF Edge Deployment -# -# # Fastly Service Dictionary Items -# resource "fastly_service_dictionary_items" "edge_security_dictionary_items" { -# for_each = { -# for d in fastly_service_vcl.test_python_org.dictionary : d.name => d if d.name == var.Edge_Security_dictionary -# } -# service_id = fastly_service_vcl.test_python_org.id -# dictionary_id = each.value.dictionary_id -# items = { -# Enabled : "100" -# } -# } -# -# # Fastly Service Dynamic Snippet Contents -# resource "fastly_service_dynamic_snippet_content" "ngwaf_config_init" { -# for_each = { -# for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_init" -# } -# service_id = fastly_service_vcl.test_python_org.id -# snippet_id = each.value.snippet_id -# content = "### Fastly managed ngwaf_config_init" -# manage_snippets = false -# } -# -# resource "fastly_service_dynamic_snippet_content" "ngwaf_config_miss" { -# for_each = { -# for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_miss" -# } -# service_id = fastly_service_vcl.test_python_org.id -# snippet_id = each.value.snippet_id -# content = "### Fastly managed ngwaf_config_miss" -# manage_snippets = false -# } -# -# resource "fastly_service_dynamic_snippet_content" "ngwaf_config_pass" { -# for_each = { -# for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_pass" -# } -# service_id = fastly_service_vcl.test_python_org.id -# snippet_id = each.value.snippet_id -# content = "### Fastly managed ngwaf_config_pass" -# manage_snippets = false -# } -# -# resource "fastly_service_dynamic_snippet_content" "ngwaf_config_deliver" { -# for_each = { -# for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_deliver" -# } -# service_id = fastly_service_vcl.test_python_org.id -# snippet_id = each.value.snippet_id -# content = "### Fastly managed ngwaf_config_deliver" -# manage_snippets = false -# } -# -# -# resource "sigsci_edge_deployment" "ngwaf_edge_site_service" { -# site_short_name = var.NGWAF_SITE -# } -# -# resource "sigsci_edge_deployment_service" "ngwaf_edge_service_link" { -# site_short_name = var.NGWAF_SITE -# fastly_sid = fastly_service_vcl.test_python_org.id -# activate_version = true -# percent_enabled = 100 -# depends_on = [ -# sigsci_edge_deployment.ngwaf_edge_site_service, -# fastly_service_vcl.test_python_org, -# fastly_service_dictionary_items.edge_security_dictionary_items, -# fastly_service_dynamic_snippet_content.ngwaf_config_init, -# fastly_service_dynamic_snippet_content.ngwaf_config_miss, -# fastly_service_dynamic_snippet_content.ngwaf_config_pass, -# fastly_service_dynamic_snippet_content.ngwaf_config_deliver, -# ] -# } -# -# resource "sigsci_edge_deployment_service_backend" "ngwaf_edge_service_backend_sync" { -# site_short_name = var.NGWAF_SITE -# fastly_sid = fastly_service_vcl.test_python_org.id -# fastly_service_vcl_active_version = fastly_service_vcl.test_python_org.active_version -# depends_on = [ -# sigsci_edge_deployment_service.ngwaf_edge_service_link, -# ] -# } \ No newline at end of file +# NGWAF Edge Deployment + +# Fastly Service Dictionary Items +resource "fastly_service_dictionary_items" "edge_security_dictionary_items" { + for_each = { + for d in fastly_service_vcl.test_python_org.dictionary : d.name => d if d.name == var.Edge_Security_dictionary + } + service_id = fastly_service_vcl.test_python_org.id + dictionary_id = each.value.dictionary_id + items = { + Enabled : "100" + } +} + +# Fastly Service Dynamic Snippet Contents +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_init" { + for_each = { + for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_init" + } + service_id = fastly_service_vcl.test_python_org.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_init" + manage_snippets = false +} + +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_miss" { + for_each = { + for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_miss" + } + service_id = fastly_service_vcl.test_python_org.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_miss" + manage_snippets = false +} + +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_pass" { + for_each = { + for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_pass" + } + service_id = fastly_service_vcl.test_python_org.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_pass" + manage_snippets = false +} + +resource "fastly_service_dynamic_snippet_content" "ngwaf_config_deliver" { + for_each = { + for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_deliver" + } + service_id = fastly_service_vcl.test_python_org.id + snippet_id = each.value.snippet_id + content = "### Fastly managed ngwaf_config_deliver" + manage_snippets = false +} + + +resource "sigsci_edge_deployment" "ngwaf_edge_site_service" { + site_short_name = var.NGWAF_SITE +} + +resource "sigsci_edge_deployment_service" "ngwaf_edge_service_link" { + site_short_name = var.NGWAF_SITE + fastly_sid = fastly_service_vcl.test_python_org.id + activate_version = true + percent_enabled = 100 + depends_on = [ + sigsci_edge_deployment.ngwaf_edge_site_service, + fastly_service_vcl.test_python_org, + fastly_service_dictionary_items.edge_security_dictionary_items, + fastly_service_dynamic_snippet_content.ngwaf_config_init, + fastly_service_dynamic_snippet_content.ngwaf_config_miss, + fastly_service_dynamic_snippet_content.ngwaf_config_pass, + fastly_service_dynamic_snippet_content.ngwaf_config_deliver, + ] +} + +resource "sigsci_edge_deployment_service_backend" "ngwaf_edge_service_backend_sync" { + site_short_name = var.NGWAF_SITE + fastly_sid = fastly_service_vcl.test_python_org.id + fastly_service_vcl_active_version = fastly_service_vcl.test_python_org.active_version + depends_on = [ + sigsci_edge_deployment_service.ngwaf_edge_service_link, + ] +} \ No newline at end of file diff --git a/infra/out.tf b/infra/out.tf index e56a77748..721bfb454 100644 --- a/infra/out.tf +++ b/infra/out.tf @@ -1,16 +1,16 @@ -# output "testing-the_ngwaf" { -# value = < Date: Tue, 27 Aug 2024 14:02:45 -0500 Subject: [PATCH 17/20] fix(route53): adjust zone id --- infra/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/variables.tf b/infra/variables.tf index bcd6983b2..7cfe1b483 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -68,7 +68,7 @@ variable "AWS_SECRET_ACCESS_KEY" { variable "route53_zone_id" { type = string description = "The Route 53 hosted zone ID" - default = "Z2LSM2W8Q3WN11" # psf.io + default = "Z3JUI7A2G39FQL" # python.org } variable "route53_record_name" { From 90da06496a31d08393c7a67bd7678339ec25544b Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 27 Aug 2024 14:10:20 -0500 Subject: [PATCH 18/20] fix: redirects --- infra/fastly.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/fastly.tf b/infra/fastly.tf index 2a6eb5b82..a57b91daf 100644 --- a/infra/fastly.tf +++ b/infra/fastly.tf @@ -126,13 +126,13 @@ resource "fastly_service_vcl" "test_python_org" { condition { name = "apex redirect" priority = 10 - statement = "req.http.Host == \"test.python.org\"" + statement = "req.http.Host == \"python.org\"" type = "RESPONSE" } condition { name = "apex" priority = 1 - statement = "req.http.host == \"test.python.org\"" + statement = "req.http.host == \"python.org\"" type = "REQUEST" } @@ -199,7 +199,7 @@ resource "fastly_service_vcl" "test_python_org" { name = "www redirect" priority = 10 response_condition = "apex redirect" - source = "\"https://test.python.org\" + req.url" + source = "\"https://www.\" + req.http.host + req.url" type = "response" } header { From df5aa002a42b2c065c3cb6e3e90f02c4b38c3126 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Thu, 29 Aug 2024 13:39:59 -0500 Subject: [PATCH 19/20] feat: move into modules --- infra/Makefile | 15 ++++ infra/aws.tf | 8 -- infra/cdn/README.md | 3 + infra/{ => cdn}/certs/psf.io.pem | 0 infra/{fastly.tf => cdn/main.tf} | 117 +++++++++++------------------- infra/cdn/providers.tf | 4 + infra/cdn/variables.tf | 14 ++++ infra/cdn/versions.tf | 8 ++ infra/config.tf | 9 +++ infra/dns/README.md | 48 ++++++++++++ infra/dns/main.tf | 82 +++++++++++++++++++++ infra/dns/providers.tf | 6 ++ infra/dns/variables.tf | 29 ++++++++ infra/dns/versions.tf | 8 ++ infra/main.tf | 82 ++++++++++++++++----- infra/ngwaf/README.md | 4 + infra/{ngwaf.tf => ngwaf/main.tf} | 69 +++++++++++++++++- infra/ngwaf/providers.tf | 7 ++ infra/ngwaf/variables.tf | 42 +++++++++++ infra/{ => ngwaf}/versions.tf | 4 - infra/pythondotorg/README.md | 4 + infra/pythondotorg/main.tf | 2 + infra/pythondotorg/variables.tf | 0 infra/pythondotorg/versions.tf | 4 + infra/variables.tf | 91 ----------------------- 25 files changed, 463 insertions(+), 197 deletions(-) create mode 100644 infra/Makefile delete mode 100644 infra/aws.tf create mode 100644 infra/cdn/README.md rename infra/{ => cdn}/certs/psf.io.pem (100%) rename infra/{fastly.tf => cdn/main.tf} (77%) create mode 100644 infra/cdn/providers.tf create mode 100644 infra/cdn/variables.tf create mode 100644 infra/cdn/versions.tf create mode 100644 infra/config.tf create mode 100644 infra/dns/README.md create mode 100644 infra/dns/main.tf create mode 100644 infra/dns/providers.tf create mode 100644 infra/dns/variables.tf create mode 100644 infra/dns/versions.tf create mode 100644 infra/ngwaf/README.md rename infra/{ngwaf.tf => ngwaf/main.tf} (66%) create mode 100644 infra/ngwaf/providers.tf create mode 100644 infra/ngwaf/variables.tf rename infra/{ => ngwaf}/versions.tf (73%) create mode 100644 infra/pythondotorg/README.md create mode 100644 infra/pythondotorg/main.tf create mode 100644 infra/pythondotorg/variables.tf create mode 100644 infra/pythondotorg/versions.tf delete mode 100644 infra/variables.tf diff --git a/infra/Makefile b/infra/Makefile new file mode 100644 index 000000000..2fa9e4665 --- /dev/null +++ b/infra/Makefile @@ -0,0 +1,15 @@ + +.PHONY: fmt +fmt: + @tf fmt ../**/*.tf + @tf fmt ../**/*.tfvars + @tf fmt ../**/*.tftest.hcl + +.PHONY: check +check: + @tf validate + +.PHONY: yolo +yolo: + @echo "Wise, you are not..." + @tf apply -auto-approve diff --git a/infra/aws.tf b/infra/aws.tf deleted file mode 100644 index d38d99e65..000000000 --- a/infra/aws.tf +++ /dev/null @@ -1,8 +0,0 @@ -resource "aws_route53_record" "dns_cname" { - zone_id = var.route53_zone_id - name = var.route53_record_name - type = "CNAME" - ttl = var.route53_record_ttl - records = ["dualstack.python.map.fastly.net"] - allow_overwrite = true -} \ No newline at end of file diff --git a/infra/cdn/README.md b/infra/cdn/README.md new file mode 100644 index 000000000..f300fdfef --- /dev/null +++ b/infra/cdn/README.md @@ -0,0 +1,3 @@ +# Fastly CDN Config + +This module sets up the Fastly CDN stuff diff --git a/infra/certs/psf.io.pem b/infra/cdn/certs/psf.io.pem similarity index 100% rename from infra/certs/psf.io.pem rename to infra/cdn/certs/psf.io.pem diff --git a/infra/fastly.tf b/infra/cdn/main.tf similarity index 77% rename from infra/fastly.tf rename to infra/cdn/main.tf index a57b91daf..43b2da5a6 100644 --- a/infra/fastly.tf +++ b/infra/cdn/main.tf @@ -1,30 +1,44 @@ -resource "fastly_service_vcl" "test_python_org" { - name = "test.python.org" - default_ttl = 3600 +variable "name" { type = string } +variable "domain" { type = string } +variable "extra_domains" { type = list(string) } +variable "backend_address" { type = string } +variable "default_ttl" { type = number } +variable "stale_if_error" { type = bool } +variable "stale_if_error_ttl" { type = number } +variable "aws_access_key_id" { type = string } +variable "aws_secret_access_key" { type = string } +variable "datadog_api_key" { type = string } +variable "fastly_header_token" { type = string } + +resource "fastly_service_vcl" "python_org" { + name = var.name + default_ttl = var.default_ttl http3 = false stale_if_error = false stale_if_error_ttl = 43200 activate = true domain { - name = "test.python.org" + name = var.domain } - domain { - name = var.USER_VCL_SERVICE_DOMAIN_NAME - comment = "NGWAF testing domain" + dynamic "domain" { + for_each = var.extra_domains + content { + name = domain.value + } } backend { name = "cabotage" - address = "pythondotorg.ingress.us-east-2.psfhosted.computer" + address = var.backend_address port = 443 shield = "iad-va-us" auto_loadbalance = false use_ssl = true ssl_check_cert = true - ssl_cert_hostname = "pythondotorg.ingress.us-east-2.psfhosted.computer" - ssl_sni_hostname = "pythondotorg.ingress.us-east-2.psfhosted.computer" + ssl_cert_hostname = var.backend_address + ssl_sni_hostname = var.backend_address weight = 100 max_conn = 200 connect_timeout = 1000 @@ -43,23 +57,13 @@ resource "fastly_service_vcl" "test_python_org" { ssl_check_cert = true ssl_cert_hostname = "lb.psf.io" ssl_sni_hostname = "lb.psf.io" - ssl_ca_cert = file("${path.module}/certs/psf.io.pem") + ssl_ca_cert = file("${path.module}/cdn/certs/psf.io.pem") weight = 100 max_conn = 200 connect_timeout = 1000 first_byte_timeout = 15000 between_bytes_timeout = 10000 - override_host = "www.python.org" - } - - backend { - address = var.USER_VCL_SERVICE_BACKEND_HOSTNAME - name = "ngwaf_backend" - port = 443 - use_ssl = true - ssl_cert_hostname = var.USER_VCL_SERVICE_BACKEND_HOSTNAME - ssl_sni_hostname = var.USER_VCL_SERVICE_BACKEND_HOSTNAME - override_host = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + override_host = var.domain == "test.python.org" ? "www.python.org" : null } acl { @@ -90,13 +94,13 @@ resource "fastly_service_vcl" "test_python_org" { condition { name = "HSTS w/ subdomains" priority = 10 - statement = "req.http.host == \"test.python.org\"" + statement = "req.http.host == \"${var.domain}\"" type = "RESPONSE" } condition { name = "HSTS w/o subdomain" priority = 10 - statement = "req.http.host == \"test.python.org\"" + statement = "req.http.host == \"${var.domain}\"" type = "RESPONSE" } condition { @@ -242,7 +246,7 @@ resource "fastly_service_vcl" "test_python_org" { healthcheck { check_interval = 15000 expected_response = 200 - host = "test.python.org" + host = var.domain http_version = "1.1" initial = 4 method = "HEAD" @@ -260,19 +264,19 @@ resource "fastly_service_vcl" "test_python_org" { } logging_s3 { - name = "psf-fastly-logs" - bucket_name = "psf-fastly-logs-eu-west-1" - domain = "s3-eu-west-1.amazonaws.com" - path = "/www-python-org/%Y/%m/%d/" - period = 3600 - gzip_level = 9 - format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" - timestamp_format = "%Y-%m-%dT%H:%M:%S.000" - redundancy = "standard" - format_version = 2 - message_type = "classic" - s3_access_key = var.AWS_ACCESS_KEY_ID - s3_secret_key = var.AWS_SECRET_ACCESS_KEY + name = "psf-fastly-logs" + bucket_name = "psf-fastly-logs-eu-west-1" + domain = "s3-eu-west-1.amazonaws.com" + path = "/${replace(var.domain, ".", "-")}/%Y/%m/%d/" + period = 3600 + gzip_level = 9 + format = "%%h \"%%{now}V\" %%l \"%%{req.request}V %%{req.url}V\" %%{req.proto}V %%>s %%{resp.http.Content-Length}V %%{resp.http.age}V \"%%{resp.http.x-cache}V\" \"%%{resp.http.x-cache-hits}V\" \"%%{req.http.content-type}V\" \"%%{req.http.accept-language}V\" \"%%{cstr_escape(req.http.user-agent)}V\"" + timestamp_format = "%Y-%m-%dT%H:%M:%S.000" + redundancy = "standard" + format_version = 2 + message_type = "classic" + s3_access_key = var.s3_logging_keys + s3_secret_key = var.s3_logging_keys } logging_syslog { @@ -297,7 +301,7 @@ resource "fastly_service_vcl" "test_python_org" { feature_revision = 1 http_methods = "GET,PUT,TRACE,POST,HEAD,DELETE,PATCH,OPTIONS" logger_type = "datadog" - name = "test.python.org backends" + name = "${var.domain} backends" penalty_box_duration = 2 rps_limit = 10 window_size = 10 @@ -350,38 +354,5 @@ resource "fastly_service_vcl" "test_python_org" { status = 403 } -# NGWAF Dynamic Snippets - dynamicsnippet { - name = "ngwaf_config_init" - type = "init" - priority = 0 - } - - dynamicsnippet { - name = "ngwaf_config_miss" - type = "miss" - priority = 9000 - } - - dynamicsnippet { - name = "ngwaf_config_pass" - type = "pass" - priority = 9000 - } - - dynamicsnippet { - name = "ngwaf_config_deliver" - type = "deliver" - priority = 9000 - } - - dictionary { - name = var.Edge_Security_dictionary - } - - lifecycle { - ignore_changes = [product_enablement] - } - force_destroy = true -} +} \ No newline at end of file diff --git a/infra/cdn/providers.tf b/infra/cdn/providers.tf new file mode 100644 index 000000000..2346f041f --- /dev/null +++ b/infra/cdn/providers.tf @@ -0,0 +1,4 @@ +provider "fastly" { + alias = "cdn" + api_key = var.FASTLY_API_KEY +} diff --git a/infra/cdn/variables.tf b/infra/cdn/variables.tf new file mode 100644 index 000000000..9eab0eb17 --- /dev/null +++ b/infra/cdn/variables.tf @@ -0,0 +1,14 @@ +variable "FASTLY_API_KEY" { + type = string + description = "API key for the Fastly VCL edge configuration." +} +variable "FASTLY_HEADER_TOKEN" { + description = "Fastly Token for authentication" + type = string + sensitive = true +} +variable "DATADOG_API_KEY" { + type = string + description = "API key for Datadog logging" + sensitive = true +} \ No newline at end of file diff --git a/infra/cdn/versions.tf b/infra/cdn/versions.tf new file mode 100644 index 000000000..da9c01f79 --- /dev/null +++ b/infra/cdn/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + fastly = { + source = "fastly/fastly" + version = "5.13.0" + } + } +} diff --git a/infra/config.tf b/infra/config.tf new file mode 100644 index 000000000..65b1a5210 --- /dev/null +++ b/infra/config.tf @@ -0,0 +1,9 @@ +# Connect us to TF Cloud for remote deploys +terraform { + cloud { + organization = "psf" + workspaces { + name = "pythondotorg-infra" + } + } +} diff --git a/infra/dns/README.md b/infra/dns/README.md new file mode 100644 index 000000000..7d98749e1 --- /dev/null +++ b/infra/dns/README.md @@ -0,0 +1,48 @@ +# DNS Module + +This Terraform module sets up AWS Route53 resources for managing DNS records, +supporting both apex domains and subdomains with Fastly CDN integration. + +## Features + +- Creates Route53 hosted zones for primary and user content (subdomain) domains +- Supports both apex domains and subdomains +- Automatically configures A/AAAA records for apex domains or CNAME for subdomains +- Integrates with Fastly CDN endpoints +- Provides a consistent set of name servers across hosted zones +- Allows addition of TXT records to the apex domain for verification purposes + +## Usage + +```hcl +module "CoolPythonWebsite" { + source = "./dns" + + primary_domain = "CoolPythonWebsite.com" + user_content_domain = "users.CoolPythonWebsite.com" + apex_txt = ["v=spf1 include:_spf.example.com ~all"] + tags = { + Environment = "Staging" + Project = "CoolPythonWebsite" + } +} +``` + +## Input Variables + +- `primary_domain`: Your main domain (e.g., "CoolPythonWebsite.com") +- `user_content_domain`: Domain for user-generated content (e.g., "users.CoolPythonWebsite.com") +- `apex_txt`: List of TXT records to add to the apex domain +- `tags`: Map of tags to apply to all created resources + +## Outputs + +- `primary_zone_id`: The Zone ID of the primary hosted zone +- `nameservers`: The set of name servers for your domains +- `user_content_zone_id`: The Zone ID of the user content hosted zone + +## Requirements + +Tested on +- Terraform 1.8.5 +- AWS provider ~> 5.0 diff --git a/infra/dns/main.tf b/infra/dns/main.tf new file mode 100644 index 000000000..17de554f0 --- /dev/null +++ b/infra/dns/main.tf @@ -0,0 +1,82 @@ +# This Terraform configuration sets up AWS Route53 resources +# ? TODO: It may be nice to allow a var when sourcing this module to +# force destroy/allow overwriting of existing resources + +# Retrieve the current AWS account data (based on secrets provided in .tfvars or TF Cloud) +data "aws_caller_identity" "current" {} + +# Input variables passed in from `$root/infra/main.tf` +variable "tags" { type = map(any) } +variable "primary_domain" { type = string } +variable "user_content_domain" { type = string } +variable "apex_txt" { type = list(any) } +variable "name" { type = string } +variable "zone_id" { type = string } +variable "domain" { type = string } +variable "fastly_endpoints" { type = map(any) } +variable "domain_map" { type = map(any) } + +# see if we're dealing with an apex domain or subdomain by splitting the domain name and counting the parts +locals { + apex_domain = length(split(".", var.domain)) > 2 ? false : true +} + +# Create a reusable set of nameservers +# This ensures consistent NS across multiple hosted zones +resource "aws_route53_delegation_set" "ns" {} + +resource "aws_route53_zone" "primary" { + name = var.domain + delegation_set_id = aws_route53_delegation_set.ns.id + tags = var.tags +} + +resource "aws_route53_zone" "user_content" { + name = var.user_content_domain + delegation_set_id = aws_route53_delegation_set.ns.id + tags = var.tags +} + +resource "aws_route53_record" "apex_txt" { + zone_id = aws_route53_zone.primary.zone_id + name = var.primary_domain + type = "TXT" + ttl = 3600 + records = var.apex_txt +} + +# Create the main DNS record, automatically choosing between A (for apex) and CNAME (for subdomain) +# depending on the value of `local.apex_domain` above +# This will point to our Fastly CDN endpoints +resource "aws_route53_record" "primary" { + zone_id = var.zone_id + name = var.domain + type = local.apex_domain ? "A" : "CNAME" + ttl = 86400 + records = var.fastly_endpoints[join("_", concat([var.domain_map[var.domain]], [local.apex_domain ? "A" : "CNAME"]))] +} + +# Same as above, but also create an AAAA record for IPv6 +resource "aws_route53_record" "primary-ipv6" { + count = local.apex_domain ? 1 : 0 + zone_id = var.zone_id + name = var.domain + type = "AAAA" + ttl = 86400 + records = var.fastly_endpoints[join("_", [var.domain_map[var.domain], "AAAA"])] +} + +# Output things we need to ref. elsewhere +output "primary_zone_id" { + value = aws_route53_record.primary.zone_id + description = "The Zone ID of our primary hosted zone." +} + +output "nameservers" { + value = aws_route53_delegation_set.ns.name_servers + description = "The set of name servers for our domains." +} + +output "user_content_zone_id" { + value = aws_route53_zone.user_content.zone_id +} \ No newline at end of file diff --git a/infra/dns/providers.tf b/infra/dns/providers.tf new file mode 100644 index 000000000..8e5fa868a --- /dev/null +++ b/infra/dns/providers.tf @@ -0,0 +1,6 @@ +provider "aws" { + alias = "dns" + region = "us-east-2" + access_key = var.AWS_ACCESS_KEY_ID + secret_key = var.AWS_SECRET_ACCESS_KEY +} diff --git a/infra/dns/variables.tf b/infra/dns/variables.tf new file mode 100644 index 000000000..980e266ec --- /dev/null +++ b/infra/dns/variables.tf @@ -0,0 +1,29 @@ +variable "AWS_ACCESS_KEY_ID" { + type = string + description = "Access key for the AWS account." + sensitive = true +} + +variable "AWS_SECRET_ACCESS_KEY" { + type = string + description = "Secret access key for the AWS account." + sensitive = true +} + +variable "route53_zone_id" { + type = string + description = "The Route 53 hosted zone ID" + default = "Z3JUI7A2G39FQL" # python.org +} + +variable "route53_record_name" { + type = string + description = "The name of the CNAME record" + default = "test.python.org" +} + +variable "route53_record_ttl" { + type = number + description = "The TTL for the CNAME record" + default = 60 +} diff --git a/infra/dns/versions.tf b/infra/dns/versions.tf new file mode 100644 index 000000000..1ac87da57 --- /dev/null +++ b/infra/dns/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} diff --git a/infra/main.tf b/infra/main.tf index b2cc0be80..a67ac5c8e 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -1,26 +1,70 @@ -terraform { - cloud { - organization = "psf" - workspaces { - name = "test-pythondotorg" - } +variable "fastly_s3_logging" { type = map(any) } + +locals { + tags = { + Application = "Python.org" + Environment = "Production" + } +} + +locals { + fastly_endpoints = { + "python.map.fastly.net_A" = ["151.101.128.223", "151.101.192.223", "151.101.0.223", "151.101.64.223"] + "python.map.fastly.net_AAAA" = ["2a04:4e42:200::223", "2a04:4e42:400::223", "2a04:4e42:600::223", "2a04:4e42::223"] + "python.map.fastly.net_CNAME" = ["dualstack.python.map.fastly.net"] } + domain_map = { + "python.org" = "python.map.fastly.net" + "test.python.org" = "python.map.fastly.net" + } +} + +module "dns" { + source = "./dns" + tags = local.tags + primary_domain = "python.org" } -# Provider configurations -provider "fastly" { - api_key = var.FASTLY_API_KEY +module "pythondotorg_production" { + source = "./cdn" + + name = "Python.org" + domain = "python.org" + extra_domains = ["www.python.org"] + backend_address = "pythondotorg.ingress.us-east-2.psfhosted.computer" + default_ttl = 3600 + stale_if_error = false + stale_if_error_ttl = 43200 + + zone_id = module.dns.primary_zone_id + backend = "pythondotorg.ingress.us-east-2.psfhosted.computer" + s3_logging_keys = var.fastly_s3_logging + + fastly_endpoints = local.fastly_endpoints + domain_map = local.domain_map } -provider "aws" { - region = "us-east-2" - access_key = var.AWS_ACCESS_KEY_ID - secret_key = var.AWS_SECRET_ACCESS_KEY +module "pythondotorg_staging" { + source = "./cdn" + + name = "test.Python.org" + domain = "test.python.org" + extra_domains = [] + backend_address = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" + default_ttl = 3600 + stale_if_error = false + stale_if_error_ttl = 43200 + + zone_id = module.dns.primary_zone_id + backend = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" + s3_logging_keys = var.fastly_s3_logging + + fastly_endpoints = local.fastly_endpoints + domain_map = local.domain_map } -provider "sigsci" { - corp = var.NGWAF_CORP - email = var.NGWAF_EMAIL - auth_token = var.NGWAF_TOKEN - fastly_api_key = var.FASTLY_API_KEY -} \ No newline at end of file + +# module "ngwaf" { +# source = "./ngwaf" +# +# } diff --git a/infra/ngwaf/README.md b/infra/ngwaf/README.md new file mode 100644 index 000000000..9d01f4ced --- /dev/null +++ b/infra/ngwaf/README.md @@ -0,0 +1,4 @@ +# Fastly's NGWAF + +This module sets up the Fastly Next-Gen Web Application Firewall (NGWAF) for our Fastly services +related to python.org / test.python.org. diff --git a/infra/ngwaf.tf b/infra/ngwaf/main.tf similarity index 66% rename from infra/ngwaf.tf rename to infra/ngwaf/main.tf index 612206782..8f60863fd 100644 --- a/infra/ngwaf.tf +++ b/infra/ngwaf/main.tf @@ -1,4 +1,69 @@ -# NGWAF Edge Deployment +# ? TODO: this needs reorgnization :P, was jsut trying to yank out NGWAF stuff from main CDN module +# the goal is to let cdn apply the core config, and then ingest the service ID from that +# and apply ngwaf stuff on top THEN set up signal sciences stuff on top of that + + +# ! TODO: refactor this to be dynamic depending on {test,prod}.python.org +resource "fastly_service_vcl" "ngwaf_service" { + name = var.service_name + activate = true + + domain { + name = var.USER_VCL_SERVICE_DOMAIN_NAME + comment = "NGWAF testing domain" + } + + backend { + address = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + name = "ngwaf_backend" + port = 443 + use_ssl = true + ssl_cert_hostname = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + ssl_sni_hostname = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + override_host = var.USER_VCL_SERVICE_BACKEND_HOSTNAME + } + + # NGWAF Dynamic Snippets + dynamicsnippet { + name = "ngwaf_config_init" + type = "init" + priority = 0 + } + + dynamicsnippet { + name = "ngwaf_config_miss" + type = "miss" + priority = 9000 + } + + dynamicsnippet { + name = "ngwaf_config_pass" + type = "pass" + priority = 9000 + } + + dynamicsnippet { + name = "ngwaf_config_deliver" + type = "deliver" + priority = 9000 + } + + dictionary { + name = var.Edge_Security_dictionary + } + + product_enablement { + bot_management = true + } + + lifecycle { + ignore_changes = [product_enablement] + } +} + +output "ngwaf_service_id" { + value = fastly_service_vcl.ngwaf_service.id +} # Fastly Service Dictionary Items resource "fastly_service_dictionary_items" "edge_security_dictionary_items" { @@ -53,7 +118,7 @@ resource "fastly_service_dynamic_snippet_content" "ngwaf_config_deliver" { manage_snippets = false } - +# NGWAF Edge Deployment resource "sigsci_edge_deployment" "ngwaf_edge_site_service" { site_short_name = var.NGWAF_SITE } diff --git a/infra/ngwaf/providers.tf b/infra/ngwaf/providers.tf new file mode 100644 index 000000000..c4a34abff --- /dev/null +++ b/infra/ngwaf/providers.tf @@ -0,0 +1,7 @@ +provider "sigsci" { + alias = "firewall" + corp = var.NGWAF_CORP + email = var.NGWAF_EMAIL + auth_token = var.NGWAF_TOKEN + fastly_api_key = var.FASTLY_API_KEY +} diff --git a/infra/ngwaf/variables.tf b/infra/ngwaf/variables.tf new file mode 100644 index 000000000..bebe80e81 --- /dev/null +++ b/infra/ngwaf/variables.tf @@ -0,0 +1,42 @@ +variable "service_name" { + type = string +} + +variable "USER_VCL_SERVICE_DOMAIN_NAME" { + type = string + description = "The domain name of the service." +} + +variable "USER_VCL_SERVICE_BACKEND_HOSTNAME" { + type = string + description = "The hostname of the backend service." +} + +variable "Edge_Security_dictionary" { + type = string + description = "The dictionary name for the Edge Security product." +} + +variable "NGWAF_CORP" { + type = string + description = "Corp name for NGWAF" + default = "python" +} + +variable "NGWAF_SITE" { + type = string + description = "Site SHORT name for NGWAF" + default = "test" +} + +variable "NGWAF_EMAIL" { + type = string + description = "Email address associated with the token for the NGWAF API." + default = "jacob.coffee@pyfound.org" +} + +variable "NGWAF_TOKEN" { + type = string + description = "Secret token for the NGWAF API." + sensitive = true +} diff --git a/infra/versions.tf b/infra/ngwaf/versions.tf similarity index 73% rename from infra/versions.tf rename to infra/ngwaf/versions.tf index b920dec3a..f8c137ba6 100644 --- a/infra/versions.tf +++ b/infra/ngwaf/versions.tf @@ -8,9 +8,5 @@ terraform { source = "signalsciences/sigsci" version = "3.3.0" } - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } } } diff --git a/infra/pythondotorg/README.md b/infra/pythondotorg/README.md new file mode 100644 index 000000000..f13505d82 --- /dev/null +++ b/infra/pythondotorg/README.md @@ -0,0 +1,4 @@ +# python.org, the module + +This Terraform module sets up the infrastructure for the Python Software Foundation's website, +[python.org](https://www.python.org/) as well as the test site, [test.python.org](https://test.python.org/). diff --git a/infra/pythondotorg/main.tf b/infra/pythondotorg/main.tf new file mode 100644 index 000000000..e4c6da22e --- /dev/null +++ b/infra/pythondotorg/main.tf @@ -0,0 +1,2 @@ +# todo , i think this module is fully replaced by cdn so maybe that needs to just be renamed +# OR maybe there is app config we can do here alongside cabo? @ee diff --git a/infra/pythondotorg/variables.tf b/infra/pythondotorg/variables.tf new file mode 100644 index 000000000..e69de29bb diff --git a/infra/pythondotorg/versions.tf b/infra/pythondotorg/versions.tf new file mode 100644 index 000000000..dc150e7c8 --- /dev/null +++ b/infra/pythondotorg/versions.tf @@ -0,0 +1,4 @@ +terraform { + required_providers { + } +} diff --git a/infra/variables.tf b/infra/variables.tf deleted file mode 100644 index 7cfe1b483..000000000 --- a/infra/variables.tf +++ /dev/null @@ -1,91 +0,0 @@ -# Fastly variables -variable "FASTLY_API_KEY" { - type = string - description = "API key for the Fastly VCL edge configuration." -} -variable "FASTLY_HEADER_TOKEN" { - description = "Fastly Token for authentication" - type = string - sensitive = true -} - -# VCL Service variables -variable "USER_VCL_SERVICE_DOMAIN_NAME" { - type = string - description = "Frontend domain for your service." - default = "test.python.org" -} - -variable "USER_VCL_SERVICE_BACKEND_HOSTNAME" { - type = string - description = "Hostname used for backend." - default = "test.python.org" -} - -variable "Edge_Security_dictionary" { - type = string - default = "Edge_Security" -} - -# NGWAF variables -variable "NGWAF_CORP" { - type = string - description = "Corp name for NGWAF" - default = "python" -} - -variable "NGWAF_SITE" { - type = string - description = "Site SHORT name for NGWAF" - default = "test" -} - -variable "NGWAF_EMAIL" { - type = string - description = "Email address associated with the token for the NGWAF API." - default = "jacob.coffee@pyfound.org" -} - -variable "NGWAF_TOKEN" { - type = string - description = "Secret token for the NGWAF API." - sensitive = true -} - -# AWS variables -variable "AWS_ACCESS_KEY_ID" { - type = string - description = "Access key for the AWS account." - sensitive = true -} - -variable "AWS_SECRET_ACCESS_KEY" { - type = string - description = "Secret access key for the AWS account." - sensitive = true -} - -variable "route53_zone_id" { - type = string - description = "The Route 53 hosted zone ID" - default = "Z3JUI7A2G39FQL" # python.org -} - -variable "route53_record_name" { - type = string - description = "The name of the CNAME record" - default = "test.python.org" -} - -variable "route53_record_ttl" { - type = number - description = "The TTL for the CNAME record" - default = 60 -} - -# dd -variable "DATADOG_API_KEY" { - type = string - description = "API key for Datadog" - sensitive = true -} \ No newline at end of file From d56525329b08662bf7aedb41deae67f12575be9d Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Sep 2024 15:09:04 -0500 Subject: [PATCH 20/20] feat: cleanup before split --- infra/.terraform.lock.hcl | 22 ----------------- infra/Makefile | 15 ++++++++++++ infra/Makefile | 15 ------------ infra/cdn/main.tf | 22 ++++------------- infra/cdn/variables.tf | 33 ++++++++++++++++++++++--- infra/dns/main.tf | 11 +-------- infra/dns/providers.tf | 4 +-- infra/dns/variables.tf | 41 +++++++++++++++++++++++++++++-- infra/main.tf | 43 ++++++++++++++++++--------------- infra/ngwaf/main.tf | 30 +++++++++++------------ infra/ngwaf/variables.tf | 6 ++++- infra/out.tf | 4 +-- infra/pythondotorg/README.md | 4 --- infra/pythondotorg/main.tf | 2 -- infra/pythondotorg/variables.tf | 0 infra/pythondotorg/versions.tf | 4 --- infra/variables.tf | 28 +++++++++++++++++++++ 17 files changed, 164 insertions(+), 120 deletions(-) create mode 100644 infra/Makefile delete mode 100644 infra/Makefile delete mode 100644 infra/pythondotorg/README.md delete mode 100644 infra/pythondotorg/main.tf delete mode 100644 infra/pythondotorg/variables.tf delete mode 100644 infra/pythondotorg/versions.tf create mode 100644 infra/variables.tf diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl index 2d7b597fa..7d31fccf4 100644 --- a/infra/.terraform.lock.hcl +++ b/infra/.terraform.lock.hcl @@ -45,25 +45,3 @@ provider "registry.terraform.io/hashicorp/aws" { "zh:de16a6ff3baacdaf9609a0a89aa1913fc19cccaf5ee0fc1c49c5a075baa47c02", ] } - -provider "registry.terraform.io/signalsciences/sigsci" { - version = "3.3.0" - constraints = ">= 1.2.18" - hashes = [ - "h1:DIoFVzfofY8lQSxFTw9wmQQC28PPMq+5l3xbPNw9gLc=", - "zh:07c25e1cca9c13314429a8430c2e999ad94c7d5e2f2a11501ee2608182387e61", - "zh:07daf79b672f3e0bec7b48e3ac8dcdeec02af06b10d653bd8158a74236b0746b", - "zh:1e24a050c3d3571ec3224c4bb5c82635caf636e707b5993a1cc97c9a1f19fa8f", - "zh:24293ae24b3de13bda8512c47967f01814724805396a1bfbfbfc56f5627615cc", - "zh:2cc6ba7a38d9854146d1d05f4b7a2f8e18a33c1267b768506cbe37168dad01dc", - "zh:42065bfee0cfde04096d6140c65379253359bed49b481a97aff70aa65bf568b3", - "zh:6f7f4d96967dfd92f098b57647d396679b70d92548db6d100c4dc8723569d175", - "zh:a2e4431f045cef16ed152c0d1f8a377b6468351b775ad1ca7ce3fe74fb874be2", - "zh:b0ed1cb03d6f191fe211f10bb59ef8daed6f89e3d99136e7bb5d38f2ac72fa45", - "zh:b61ea18442a65d27b97dd1cd43bdd8d0a56c2b4b8db6355480e89f8507c6782a", - "zh:c31bb2f50ac2a636758f93afec0b9d173be6d7d7476f9e250b4554e70c6d8d82", - "zh:cb7337f7b4678ad7ece28741069c07ce5601d2a103a9667db568cf10ed0ee5a2", - "zh:d521a7dac51733aebb0905e25b8f7c1279d83c06136e87826e010c667528fd3e", - "zh:ef791688acee3b8b1191b3c6dc54dabf69612dbfb666720280b492ce348a3a06", - ] -} diff --git a/infra/Makefile b/infra/Makefile new file mode 100644 index 000000000..12cd13558 --- /dev/null +++ b/infra/Makefile @@ -0,0 +1,15 @@ + +.PHONY: fmt +fmt: + @tf fmt ../**/*.tf + @tf fmt ../**/*.tfvars + @tf fmt ../**/*.tftest.hcl + +.PHONY: check +check: + @tf validate + +.PHONY: yolo +yolo: + @echo "Wise, you are not..." + @tf apply -auto-approve diff --git a/infra/Makefile b/infra/Makefile deleted file mode 100644 index 2fa9e4665..000000000 --- a/infra/Makefile +++ /dev/null @@ -1,15 +0,0 @@ - -.PHONY: fmt -fmt: - @tf fmt ../**/*.tf - @tf fmt ../**/*.tfvars - @tf fmt ../**/*.tftest.hcl - -.PHONY: check -check: - @tf validate - -.PHONY: yolo -yolo: - @echo "Wise, you are not..." - @tf apply -auto-approve diff --git a/infra/cdn/main.tf b/infra/cdn/main.tf index 43b2da5a6..3a2a4f5b8 100644 --- a/infra/cdn/main.tf +++ b/infra/cdn/main.tf @@ -1,15 +1,3 @@ -variable "name" { type = string } -variable "domain" { type = string } -variable "extra_domains" { type = list(string) } -variable "backend_address" { type = string } -variable "default_ttl" { type = number } -variable "stale_if_error" { type = bool } -variable "stale_if_error_ttl" { type = number } -variable "aws_access_key_id" { type = string } -variable "aws_secret_access_key" { type = string } -variable "datadog_api_key" { type = string } -variable "fastly_header_token" { type = string } - resource "fastly_service_vcl" "python_org" { name = var.name default_ttl = var.default_ttl @@ -194,7 +182,7 @@ resource "fastly_service_vcl" "python_org" { destination = "http.Fastly-Token" name = "Fastly Token" priority = 10 - source = "\"${var.FASTLY_HEADER_TOKEN}\"" + source = "\"${var.fastly_header_token}\"" type = "request" } header { @@ -259,7 +247,7 @@ resource "fastly_service_vcl" "python_org" { logging_datadog { name = "ratelimit-debug" - token = var.DATADOG_API_KEY + token = var.datadog_key region = "US" } @@ -275,8 +263,8 @@ resource "fastly_service_vcl" "python_org" { redundancy = "standard" format_version = 2 message_type = "classic" - s3_access_key = var.s3_logging_keys - s3_secret_key = var.s3_logging_keys + s3_access_key = var.fastly_s3_logging["access_key"] + s3_secret_key = var.fastly_s3_logging["secret_key"] } logging_syslog { @@ -355,4 +343,4 @@ resource "fastly_service_vcl" "python_org" { } force_destroy = true -} \ No newline at end of file +} diff --git a/infra/cdn/variables.tf b/infra/cdn/variables.tf index 9eab0eb17..da28835cd 100644 --- a/infra/cdn/variables.tf +++ b/infra/cdn/variables.tf @@ -1,14 +1,39 @@ -variable "FASTLY_API_KEY" { +variable "fastly_key" { type = string description = "API key for the Fastly VCL edge configuration." } -variable "FASTLY_HEADER_TOKEN" { - description = "Fastly Token for authentication" +variable "fastly_header_token" { + description = "Fastly header token ensure we only allow Fastly to access the service" type = string sensitive = true } -variable "DATADOG_API_KEY" { +variable "datadog_key" { type = string description = "API key for Datadog logging" sensitive = true +} +variable "fastly_s3_logging" { + type = string + description = "S3 bucket keys for Fastly logging" + sensitive = true +} +variable "name" { + type = string + description = "The name of the Fastly service." +} +variable "domain" { + type = string + description = "The domain name of the service." +} +variable "extra_domains" { + type = list(string) + description = "Extra domains to add to the service." +} +variable "backend_address" { + type = string + description = "The hostname of the backend service." +} +variable "default_ttl" { + type = number + description = "The default TTL for the service." } \ No newline at end of file diff --git a/infra/dns/main.tf b/infra/dns/main.tf index 17de554f0..9cdb28c83 100644 --- a/infra/dns/main.tf +++ b/infra/dns/main.tf @@ -5,16 +5,7 @@ # Retrieve the current AWS account data (based on secrets provided in .tfvars or TF Cloud) data "aws_caller_identity" "current" {} -# Input variables passed in from `$root/infra/main.tf` -variable "tags" { type = map(any) } -variable "primary_domain" { type = string } -variable "user_content_domain" { type = string } -variable "apex_txt" { type = list(any) } -variable "name" { type = string } -variable "zone_id" { type = string } -variable "domain" { type = string } -variable "fastly_endpoints" { type = map(any) } -variable "domain_map" { type = map(any) } + # see if we're dealing with an apex domain or subdomain by splitting the domain name and counting the parts locals { diff --git a/infra/dns/providers.tf b/infra/dns/providers.tf index 8e5fa868a..92caf800d 100644 --- a/infra/dns/providers.tf +++ b/infra/dns/providers.tf @@ -1,6 +1,6 @@ provider "aws" { alias = "dns" region = "us-east-2" - access_key = var.AWS_ACCESS_KEY_ID - secret_key = var.AWS_SECRET_ACCESS_KEY + access_key = var.aws_access_key + secret_key = var.aws_secret_key } diff --git a/infra/dns/variables.tf b/infra/dns/variables.tf index 980e266ec..35133bef9 100644 --- a/infra/dns/variables.tf +++ b/infra/dns/variables.tf @@ -1,10 +1,10 @@ -variable "AWS_ACCESS_KEY_ID" { +variable "aws_access_key" { type = string description = "Access key for the AWS account." sensitive = true } -variable "AWS_SECRET_ACCESS_KEY" { +variable "aws_secret_key" { type = string description = "Secret access key for the AWS account." sensitive = true @@ -27,3 +27,40 @@ variable "route53_record_ttl" { description = "The TTL for the CNAME record" default = 60 } + +variable "tags" { + type = map(any) + description = "Tags to apply to all resources" +} +variable "primary_domain" { + type = string + description = "The primary domain name" +} +variable "user_content_domain" { + type = string + description = "The user content (sub)domain name" +} +variable "apex_txt" { + type = list(any) + description = "The TXT record for the apex domain" +} +variable "name" { + type = string + description = "The name of the Fastly service" +} +variable "zone_id" { + type = string + description = "The Route 53 hosted zone ID" +} +variable "domain" { + type = string + description = "The domain name of the service" +} +variable "fastly_endpoints" { + type = map(any) + description = "The Fastly endpoints" +} +variable "domain_map" { + type = map(any) + description = "The domain map" +} \ No newline at end of file diff --git a/infra/main.tf b/infra/main.tf index a67ac5c8e..65acb05c3 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -1,5 +1,3 @@ -variable "fastly_s3_logging" { type = map(any) } - locals { tags = { Application = "Python.org" @@ -20,12 +18,25 @@ locals { } module "dns" { + # TODO: this doesn't accommodate for DNS management splits between environments source = "./dns" tags = local.tags primary_domain = "python.org" + zone_id = module.dns.primary_zone_id + fastly_endpoints = local.fastly_endpoints + domain_map = local.domain_map + + aws_access_key = var.AWS_ACCESS_KEY_ID + aws_secret_key = var.AWS_SECRET_ACCESS_KEY + + # TODO: the below needs to be parameterized or fixed + apex_txt = [] + domain = "" + name = "" + user_content_domain = "" } -module "pythondotorg_production" { +module "fastly_production" { source = "./cdn" name = "Python.org" @@ -33,18 +44,14 @@ module "pythondotorg_production" { extra_domains = ["www.python.org"] backend_address = "pythondotorg.ingress.us-east-2.psfhosted.computer" default_ttl = 3600 - stale_if_error = false - stale_if_error_ttl = 43200 - - zone_id = module.dns.primary_zone_id - backend = "pythondotorg.ingress.us-east-2.psfhosted.computer" - s3_logging_keys = var.fastly_s3_logging - fastly_endpoints = local.fastly_endpoints - domain_map = local.domain_map + datadog_key = var.DATADOG_API_KEY + fastly_key = var.FASTLY_API_KEY + fastly_header_token = var.FASTLY_HEADER_TOKEN + fastly_s3_logging = var.fastly_s3_logging } -module "pythondotorg_staging" { +module "fastly_staging" { source = "./cdn" name = "test.Python.org" @@ -52,15 +59,11 @@ module "pythondotorg_staging" { extra_domains = [] backend_address = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" default_ttl = 3600 - stale_if_error = false - stale_if_error_ttl = 43200 - zone_id = module.dns.primary_zone_id - backend = "test-pythondotorg.ingress.us-east-2.psfhosted.computer" - s3_logging_keys = var.fastly_s3_logging - - fastly_endpoints = local.fastly_endpoints - domain_map = local.domain_map + datadog_key = var.DATADOG_API_KEY + fastly_key = var.FASTLY_API_KEY + fastly_header_token = var.FASTLY_HEADER_TOKEN + fastly_s3_logging = var.fastly_s3_logging } diff --git a/infra/ngwaf/main.tf b/infra/ngwaf/main.tf index 8f60863fd..288c48309 100644 --- a/infra/ngwaf/main.tf +++ b/infra/ngwaf/main.tf @@ -10,7 +10,7 @@ resource "fastly_service_vcl" "ngwaf_service" { domain { name = var.USER_VCL_SERVICE_DOMAIN_NAME - comment = "NGWAF testing domain" + comment = "NGWAF domain" } backend { @@ -68,9 +68,9 @@ output "ngwaf_service_id" { # Fastly Service Dictionary Items resource "fastly_service_dictionary_items" "edge_security_dictionary_items" { for_each = { - for d in fastly_service_vcl.test_python_org.dictionary : d.name => d if d.name == var.Edge_Security_dictionary + for d in fastly_service_vcl.ngwaf_service.dictionary : d.name => d if d.name == var.Edge_Security_dictionary } - service_id = fastly_service_vcl.test_python_org.id + service_id = fastly_service_vcl.ngwaf_service.id dictionary_id = each.value.dictionary_id items = { Enabled : "100" @@ -80,9 +80,9 @@ resource "fastly_service_dictionary_items" "edge_security_dictionary_items" { # Fastly Service Dynamic Snippet Contents resource "fastly_service_dynamic_snippet_content" "ngwaf_config_init" { for_each = { - for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_init" + for d in fastly_service_vcl.ngwaf_service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_init" } - service_id = fastly_service_vcl.test_python_org.id + service_id = fastly_service_vcl.ngwaf_service.id snippet_id = each.value.snippet_id content = "### Fastly managed ngwaf_config_init" manage_snippets = false @@ -90,9 +90,9 @@ resource "fastly_service_dynamic_snippet_content" "ngwaf_config_init" { resource "fastly_service_dynamic_snippet_content" "ngwaf_config_miss" { for_each = { - for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_miss" + for d in fastly_service_vcl.ngwaf_service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_miss" } - service_id = fastly_service_vcl.test_python_org.id + service_id = fastly_service_vcl.ngwaf_service.id snippet_id = each.value.snippet_id content = "### Fastly managed ngwaf_config_miss" manage_snippets = false @@ -100,9 +100,9 @@ resource "fastly_service_dynamic_snippet_content" "ngwaf_config_miss" { resource "fastly_service_dynamic_snippet_content" "ngwaf_config_pass" { for_each = { - for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_pass" + for d in fastly_service_vcl.ngwaf_service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_pass" } - service_id = fastly_service_vcl.test_python_org.id + service_id = fastly_service_vcl.ngwaf_service.id snippet_id = each.value.snippet_id content = "### Fastly managed ngwaf_config_pass" manage_snippets = false @@ -110,9 +110,9 @@ resource "fastly_service_dynamic_snippet_content" "ngwaf_config_pass" { resource "fastly_service_dynamic_snippet_content" "ngwaf_config_deliver" { for_each = { - for d in fastly_service_vcl.test_python_org.dynamicsnippet : d.name => d if d.name == "ngwaf_config_deliver" + for d in fastly_service_vcl.ngwaf_service.dynamicsnippet : d.name => d if d.name == "ngwaf_config_deliver" } - service_id = fastly_service_vcl.test_python_org.id + service_id = fastly_service_vcl.ngwaf_service.id snippet_id = each.value.snippet_id content = "### Fastly managed ngwaf_config_deliver" manage_snippets = false @@ -125,12 +125,12 @@ resource "sigsci_edge_deployment" "ngwaf_edge_site_service" { resource "sigsci_edge_deployment_service" "ngwaf_edge_service_link" { site_short_name = var.NGWAF_SITE - fastly_sid = fastly_service_vcl.test_python_org.id + fastly_sid = fastly_service_vcl.ngwaf_service.id activate_version = true percent_enabled = 100 depends_on = [ sigsci_edge_deployment.ngwaf_edge_site_service, - fastly_service_vcl.test_python_org, + fastly_service_vcl.ngwaf_service, fastly_service_dictionary_items.edge_security_dictionary_items, fastly_service_dynamic_snippet_content.ngwaf_config_init, fastly_service_dynamic_snippet_content.ngwaf_config_miss, @@ -141,8 +141,8 @@ resource "sigsci_edge_deployment_service" "ngwaf_edge_service_link" { resource "sigsci_edge_deployment_service_backend" "ngwaf_edge_service_backend_sync" { site_short_name = var.NGWAF_SITE - fastly_sid = fastly_service_vcl.test_python_org.id - fastly_service_vcl_active_version = fastly_service_vcl.test_python_org.active_version + fastly_sid = fastly_service_vcl.ngwaf_service.id + fastly_service_vcl_active_version = fastly_service_vcl.ngwaf_service.active_version depends_on = [ sigsci_edge_deployment_service.ngwaf_edge_service_link, ] diff --git a/infra/ngwaf/variables.tf b/infra/ngwaf/variables.tf index bebe80e81..0cab70d93 100644 --- a/infra/ngwaf/variables.tf +++ b/infra/ngwaf/variables.tf @@ -32,7 +32,6 @@ variable "NGWAF_SITE" { variable "NGWAF_EMAIL" { type = string description = "Email address associated with the token for the NGWAF API." - default = "jacob.coffee@pyfound.org" } variable "NGWAF_TOKEN" { @@ -40,3 +39,8 @@ variable "NGWAF_TOKEN" { description = "Secret token for the NGWAF API." sensitive = true } + +variable "FASTLY_API_KEY" { + type = string + description = "API key for the Fastly VCL edge configuration." +} diff --git a/infra/out.tf b/infra/out.tf index 721bfb454..bc969b4ca 100644 --- a/infra/out.tf +++ b/infra/out.tf @@ -1,13 +1,13 @@ output "testing-the_ngwaf" { value = <