Fix: Curl manifest can be impacted by rate based WAF rules (DataBiosphere/azul-private#125, PR #5808)

Author: dsotirho-ucsc

src/azul/service/manifest_service.py

@@ -1397,6 +1397,12 @@ def command_lines(cls,
                 else ()
             )
         ]
+        file_options = [
+            '--fail-early',  # Exit curl with error on the first failure encountered
+            '--continue-at -',  # Resume partially downloaded files
+            '--retry 2',  # Retry a file download up to X times on transient error
+            '--retry-delay 10',  # Sleep for X seconds between retries
+        ]
         return {
             'cmd.exe': ' '.join([
                 'curl.exe',
@@ -1405,6 +1411,7 @@ def command_lines(cls,
                 '|',
                 'curl.exe',
                 *authentication_option,
+                *file_options,
                 '--config',
                 '-'
             ]),
@@ -1415,6 +1422,7 @@ def command_lines(cls,
                 '|',
                 'curl',
                 *authentication_option,
+                *file_options,
                 '--config',
                 '-'
             ])
@@ -1480,10 +1488,6 @@ def _write(file: JSON, is_related_file: bool = False):
                 '--location',  # Follow redirects
                 '--globoff',  # Prevent '#' in file names from being interpreted as output variables
                 '--fail',  # Upon server error don't save the error message to the file
-                '--fail-early',  # Exit curl with error on the first failure encountered
-                '--continue-at -',  # Resume partially downloaded files
-                '--retry 2',  # Retry a file download up to X times on transient error
-                '--retry-delay 10',  # Sleep for X seconds between retries
                 '--write-out "Downloading to: %{filename_effective}\\n\\n"'
             ]
             output.write('\n\n'.join(curl_options))

terraform/api_gateway.tf.json.template.py

@@ -223,7 +223,17 @@ def for_domain(cls, domain):
                             'priority': 1,
                             'name': config.waf_rate_rule_name,
                             'action': {
-                                'block': {}
+                                'block': {
+                                    'custom_response': {
+                                        'response_code': 429,
+                                        'response_header': [
+                                            {
+                                                'name': 'Retry-After',
+                                                'value': '10'
+                                            }
+                                        ]
+                                    }
+                                }
                             },
                             'statement': {
                                 'rate_based_statement': {

test/service/test_manifest.py

@@ -1195,14 +1195,6 @@ def test_curl_manifest(self):
             '',
             '--fail',
             '',
-            '--fail-early',
-            '',
-            '--continue-at -',
-            '',
-            '--retry 2',
-            '',
-            '--retry-delay 10',
-            '',
             '--write-out "Downloading to: %{filename_effective}\\n\\n"',
             '',
         ]
@@ -1496,10 +1488,13 @@ def test(*, format: ManifestFormat, fetch: bool, url: Optional[furl] = None):
                 expected_url = object_url
                 expected_url_for_bash = f"'{expected_url}'"
             if format is ManifestFormat.curl:
-                options = "--location --fail"
+                manifest_options = '--location --fail'
+                file_options = '--fail-early --continue-at - --retry 2 --retry-delay 10'
                 expected = {
-                    'cmd.exe': f'curl.exe {options} "{expected_url}" | curl.exe --config -',
-                    'bash': f'curl {options} {expected_url_for_bash} | curl --config -'
+                    'cmd.exe': f'curl.exe {manifest_options} "{expected_url}"'
+                               f' | curl.exe {file_options} --config -',
+                    'bash': f'curl {manifest_options} {expected_url_for_bash}'
+                            f' | curl {file_options} --config -'
                 }
             else:
                 if format is ManifestFormat.terra_bdbag: