Skip to content

Commit 4a2678e

Browse files
dsotirho-ucscdsotirho-ucsc
committed
Different rate limits depending on HTTP method (#5555)
1 parent ef1212a commit 4a2678e

File tree

2 files changed

+217
-1
lines changed

2 files changed

+217
-1
lines changed

‎src/azul/__init__.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1774,6 +1774,25 @@ def __attrs_post_init__(self):
17741774
name='rate_limit_alarm',
17751775
value=waf_rate_limit.value * 2)
17761776

1777+
#: The rate limit per IP for requests that trigger a manifest generation
1778+
#:
1779+
waf_rate_limit_manifests = RateLimit(name='rate_limit_manifests',
1780+
value=10,
1781+
period=10 * 60,
1782+
retry_after=30)
1783+
1784+
#: The rate limit for file download requests
1785+
#:
1786+
#: We aim for a global limit of 60 file downloads per 10 minutes. Based on
1787+
#: an observed average of 2.9 distinct IPs concurrently downloading files
1788+
#: in any 10-minute window, the maximum per-IP request rate we can allow is
1789+
#: 20/10min, or 10/5min.
1790+
#:
1791+
waf_rate_limit_files = RateLimit(name='rate_limit_files',
1792+
value=10,
1793+
period=5 * 60,
1794+
retry_after=30)
1795+
17771796
@property
17781797
def waf_bot_control(self) -> bool:
17791798
return self._boolean(self.environ['azul_waf_bot_control'])

‎terraform/api_gateway.tf.json.template.py

Lines changed: 198 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
)
44
import importlib
55
import json
6+
import logging
67

78
from more_itertools import (
89
one,
@@ -20,6 +21,12 @@
2021
from azul.deployment import (
2122
aws,
2223
)
24+
from azul.http import (
25+
http_client,
26+
)
27+
from azul.logging import (
28+
configure_script_logging,
29+
)
2330
from azul.modules import (
2431
load_app_module,
2532
)
@@ -35,6 +42,9 @@
3542
JSONs,
3643
)
3744

45+
log = logging.getLogger(__name__)
46+
configure_script_logging(log)
47+
3848

3949
@dataclass(frozen=True)
4050
class Application:
@@ -110,6 +120,18 @@ def check_waf_rules(rules: JSONs) -> JSONs:
110120
return rules
111121

112122

123+
def public_ip() -> str:
124+
"""
125+
Return the public IPv4 address of the machine running this code.
126+
"""
127+
url = 'https://checkip.amazonaws.com'
128+
http = http_client(log)
129+
response = http.request('GET', url)
130+
assert response.status == 200, R('Unexpected response', response)
131+
ip_address = response.data.decode().strip()
132+
return ip_address
133+
134+
113135
zones_by_domain = {
114136
domain: Zone.for_domain(domain)
115137
for app in apps
@@ -147,6 +169,17 @@ def check_waf_rules(rules: JSONs) -> JSONs:
147169
emit_tf({
148170
'data': [
149171
{
172+
'aws_nat_gateway': {
173+
**{
174+
f'gitlab_{zone}': {
175+
'filter': {
176+
'name': 'tag:Name',
177+
'values': [f'azul-gitlab_{zone}']
178+
},
179+
}
180+
for zone in range(vpc.num_zones)
181+
}
182+
},
150183
'aws_route53_zone': {
151184
zone.slug: {
152185
'name': zone.name,
@@ -223,6 +256,20 @@ def check_waf_rules(rules: JSONs) -> JSONs:
223256
],
224257
'resource': [
225258
{
259+
'aws_wafv2_ip_set': {
260+
'it_v4_ips': {
261+
'name': config.qualified_resource_name('it_v4_ips'),
262+
'scope': 'REGIONAL',
263+
'ip_address_version': 'IPV4',
264+
'addresses': [
265+
public_ip() + '/32',
266+
*[
267+
f'${{data.aws_nat_gateway.gitlab_{zone}.public_ip}}/32'
268+
for zone in range(vpc.num_zones)
269+
]
270+
]
271+
}
272+
},
226273
'aws_wafv2_web_acl': {
227274
'api_gateway': {
228275
'name': config.qualified_resource_name('api_gateway'),
@@ -501,7 +548,157 @@ def check_waf_rules(rules: JSONs) -> JSONs:
501548
config.waf_rate_limit_alarm,
502549
config.waf_rate_limit,
503550
]
504-
]
551+
],
552+
{
553+
'name': 'allow_it_requests',
554+
'statement': {
555+
'and_statement': [
556+
{
557+
'statement': [
558+
{
559+
'ip_set_reference_statement': {
560+
'arn': '${aws_wafv2_ip_set.%s.arn}' % 'it_v4_ips'
561+
}
562+
},
563+
{
564+
'byte_match_statement': {
565+
'field_to_match': {
566+
'method': {}
567+
},
568+
'positional_constraint': 'EXACTLY',
569+
'search_string': 'PUT',
570+
'text_transformation': {
571+
'priority': 0,
572+
'type': 'NONE'
573+
}
574+
}
575+
},
576+
{
577+
'regex_match_statement': {
578+
'regex_string': '^(/fetch)?/manifest/files',
579+
'field_to_match': {
580+
'uri_path': {}
581+
},
582+
'text_transformation': {
583+
'priority': 0,
584+
'type': 'NONE'
585+
}
586+
}
587+
}
588+
]
589+
}
590+
]
591+
},
592+
'action': {
593+
'allow': {}
594+
},
595+
'visibility_config': {
596+
'metric_name': 'allow_it_requests',
597+
'sampled_requests_enabled': True,
598+
'cloudwatch_metrics_enabled': True
599+
}
600+
},
601+
{
602+
'name': config.waf_rate_limit_files.name,
603+
'statement': {
604+
'rate_based_statement': {
605+
'limit': config.waf_rate_limit_files.value,
606+
'evaluation_window_sec': config.waf_rate_limit_files.period,
607+
'aggregate_key_type': 'IP',
608+
'scope_down_statement': {
609+
'regex_match_statement': {
610+
'regex_string': '^(/fetch)?/repository/files',
611+
'field_to_match': {
612+
'uri_path': {}
613+
},
614+
'text_transformation': {
615+
'priority': 0,
616+
'type': 'NONE'
617+
}
618+
}
619+
}
620+
}
621+
},
622+
'action': {
623+
'block': {
624+
'custom_response': {
625+
'response_code': 429,
626+
'response_header': [
627+
{
628+
'name': 'Retry-After',
629+
'value': str(config.waf_rate_limit_files.retry_after)
630+
}
631+
]
632+
}
633+
}
634+
},
635+
'visibility_config': {
636+
'metric_name': config.waf_rate_limit_files.name,
637+
'sampled_requests_enabled': True,
638+
'cloudwatch_metrics_enabled': True
639+
}
640+
},
641+
{
642+
'name': config.waf_rate_limit_manifests.name,
643+
'statement': {
644+
'rate_based_statement': {
645+
'limit': config.waf_rate_limit_manifests.value,
646+
'evaluation_window_sec': config.waf_rate_limit_manifests.period,
647+
'aggregate_key_type': 'IP',
648+
'scope_down_statement': {
649+
'and_statement': [
650+
{
651+
'statement': [
652+
{
653+
'byte_match_statement': {
654+
'field_to_match': {
655+
'method': {}
656+
},
657+
'positional_constraint': 'EXACTLY',
658+
'search_string': 'PUT',
659+
'text_transformation': {
660+
'priority': 0,
661+
'type': 'NONE'
662+
}
663+
}
664+
},
665+
{
666+
'regex_match_statement': {
667+
'regex_string': '^(/fetch)?/manifest/files',
668+
'field_to_match': {
669+
'uri_path': {}
670+
},
671+
'text_transformation': {
672+
'priority': 0,
673+
'type': 'NONE'
674+
}
675+
}
676+
}
677+
]
678+
}
679+
]
680+
}
681+
}
682+
},
683+
'action': {
684+
'block': {
685+
'custom_response': {
686+
'response_code': 429,
687+
'response_header': [
688+
{
689+
'name': 'Retry-After',
690+
'value': str(config.waf_rate_limit_manifests.retry_after)
691+
}
692+
]
693+
}
694+
}
695+
},
696+
'visibility_config': {
697+
'metric_name': config.waf_rate_limit_manifests.name,
698+
'sampled_requests_enabled': True,
699+
'cloudwatch_metrics_enabled': True
700+
}
701+
}
505702
])
506703
]),
507704
'scope': 'REGIONAL',

0 commit comments

Comments
 (0)