From 1267ab1b845a378e4c91d371a27aed38ad7a574b Mon Sep 17 00:00:00 2001 From: Dominik Hlavac Duran Date: Thu, 17 Oct 2024 10:04:27 +0200 Subject: [PATCH] Base perf test extension for 20-40M tests --- .../performance/load_testing/__init__.py | 0 .../{smoke => load_testing}/conftest.py | 2 +- .../load_testing/templates/__init__.py | 0 .../template_user_key_query_20_40M.hf.yaml | 152 ++++++++++++++++++ .../test_user_key_query_20_40M.py | 126 +++++++++++++++ 5 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 testsuite/tests/performance/load_testing/__init__.py rename testsuite/tests/performance/{smoke => load_testing}/conftest.py (93%) create mode 100644 testsuite/tests/performance/load_testing/templates/__init__.py create mode 100644 testsuite/tests/performance/load_testing/templates/template_user_key_query_20_40M.hf.yaml create mode 100644 testsuite/tests/performance/load_testing/test_user_key_query_20_40M.py diff --git a/testsuite/tests/performance/load_testing/__init__.py b/testsuite/tests/performance/load_testing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testsuite/tests/performance/smoke/conftest.py b/testsuite/tests/performance/load_testing/conftest.py similarity index 93% rename from testsuite/tests/performance/smoke/conftest.py rename to testsuite/tests/performance/load_testing/conftest.py index f3402aab..d0674613 100644 --- a/testsuite/tests/performance/smoke/conftest.py +++ b/testsuite/tests/performance/load_testing/conftest.py @@ -8,4 +8,4 @@ @pytest.fixture(scope="module") def number_of_agents(): """Number of Hyperfoil agents to be spawned""" - return 1 + return 4 diff --git a/testsuite/tests/performance/load_testing/templates/__init__.py b/testsuite/tests/performance/load_testing/templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testsuite/tests/performance/load_testing/templates/template_user_key_query_20_40M.hf.yaml b/testsuite/tests/performance/load_testing/templates/template_user_key_query_20_40M.hf.yaml new file mode 100644 index 00000000..beeba004 --- /dev/null +++ b/testsuite/tests/performance/load_testing/templates/template_user_key_query_20_40M.hf.yaml @@ -0,0 +1,152 @@ +name: test_smoke_user_key_query_20-40M +# http endpoints will be added via test +phases: + - rampUp: + increasingRate: + initialUsersPerSec: 2 + targetUsersPerSec: 20 + duration: 60s + maxSessions: 400 + scenario: &postLargeData + - get: + - randomCsvRow: + file: 'auth_user_key.csv' + skipComments: true + removeQuotes: true + columns: + 0: 0 # hostname + 1: 1 # auth_user_key + 2: 2 # user_key + - randomInt: randomBackend <- 0 .. 9 + - randomInt: randomPath <- 0 .. 9 + - template: + pattern: ${1}=${2} + toVar: param + - template: + pattern: /${randomBackend}/anything/${randomPath}?${param} + toVar: path + - httpRequest: + authority: + fromVar: 0 + POST: + fromVar: path + sync: true + body: + fromFile: message_1kb.txt + - steadyLoad20M : #20M call per day is equivalent of 232 req/s + constantRate: + duration: 30m + usersPerSec: 116 + maxSessions: 400 + startAfter: + phase: rampUp + scenario: *postLargeData + - steadyLoadGet20M: + constantRate: + duration: 30m + usersPerSec: 116 + maxSessions: 400 + startAfter: + phase: rampUp + scenario: + - getData: + - randomCsvRow: + file: 'auth_user_key.csv' + skipComments: true + removeQuotes: true + columns: + 0: 0 # hostname + 1: 1 # auth_user_key + 2: 2 # user_key + - randomInt: randomBackend <- 0 .. 9 + - randomInt: randomPath <- 0 .. 9 + - template: + pattern: ${1}=${2} + toVar: param + - template: + pattern: /${randomBackend}/anything/${randomPath}?${param} + toVar: path + - httpRequest: + authority: + fromVar: 0 + GET: + fromVar: path + sync: true + - steadyLoad30M: #30M call per day is equivalent of 348 req/s + constantRate: + duration: 30m + usersPerSec: 174 + maxSessions: 400 + startAfter: + phase: steadyLoad20M + scenario: *postLargeData + - steadyLoadGet30M: + constantRate: + duration: 30m + usersPerSec: 174 + maxSessions: 400 + startAfter: + phase: steadyLoadGet20M + scenario: + - getData: + - randomCsvRow: + file: 'auth_user_key.csv' + skipComments: true + removeQuotes: true + columns: + 0: 0 # hostname + 1: 1 # auth_user_key + 2: 2 # user_key + - randomInt: randomBackend <- 0 .. 9 + - randomInt: randomPath <- 0 .. 9 + - template: + pattern: ${1}=${2} + toVar: param + - template: + pattern: /${randomBackend}/anything/${randomPath}?${param} + toVar: path + - httpRequest: + authority: + fromVar: 0 + GET: + fromVar: path + sync: true + - steadyLoad40M: #40M call per day is equivalent of 463 req/s + constantRate: + duration: 60m + usersPerSec: 232 + maxSessions: 400 + startAfter: + phase: steadyLoad30M + scenario: *postLargeData + - steadyLoadGet40M: + constantRate: + duration: 60m + usersPerSec: 232 + maxSessions: 400 + startAfter: + phase: steadyLoadGet30M + scenario: + - getData: + - randomCsvRow: + file: 'auth_user_key.csv' + skipComments: true + removeQuotes: true + columns: + 0: 0 # hostname + 1: 1 # auth_user_key + 2: 2 # user_key + - randomInt: randomBackend <- 0 .. 9 + - randomInt: randomPath <- 0 .. 9 + - template: + pattern: ${1}=${2} + toVar: param + - template: + pattern: /${randomBackend}/anything/${randomPath}?${param} + toVar: path + - httpRequest: + authority: + fromVar: 0 + GET: + fromVar: path + sync: true \ No newline at end of file diff --git a/testsuite/tests/performance/load_testing/test_user_key_query_20_40M.py b/testsuite/tests/performance/load_testing/test_user_key_query_20_40M.py new file mode 100644 index 00000000..d97acf64 --- /dev/null +++ b/testsuite/tests/performance/load_testing/test_user_key_query_20_40M.py @@ -0,0 +1,126 @@ +""" + Smoke test that will create 3scale objects for performance testing. + Fill necessary data to benchmark template. + Run the test and assert results. + This test shows usage how to write test where 3scale product is secured with user key. +""" + +import asyncio +import os + +import backoff +import pytest + +from testsuite import rawobj +from testsuite.perf_utils import HyperfoilUtils + +# Maximal runtime of test (need to cover all performance stages) +MAX_RUN_TIME = 130 * 60 + +pytestmark = [pytest.mark.performance] + + +@pytest.fixture(scope="module") +def number_of_products(): + """Number of created services (products)""" + return 10 + + +@pytest.fixture(scope="module") +def number_of_backends(): + """Number of created backends for single service (product)""" + return 10 + + +@pytest.fixture(scope="module") +def number_of_apps(): + """Number of created application for single service (product)""" + return 15 + + +@pytest.fixture(scope="module") +def create_mapping_rules(): + """ + Returns function that will be run for each backend usage + """ + + def _create(i, be_usage): + metric = be_usage.backend.metrics.list()[0] + be_usage.backend.mapping_rules.create(rawobj.Mapping(metric, f"/anything/{i}")) + be_usage.backend.mapping_rules.create(rawobj.Mapping(metric, f"/anything/{i}", "POST")) + + return _create + + +@pytest.fixture(scope="module") +async def services(services, create_mapping_rules, event_loop): + """ + Removes default mapping rule of each product. + For each backend creates 20 mapping rules + """ + for svc in services: + proxy = svc.proxy.list() + proxy.mapping_rules.delete(proxy.mapping_rules.list()[0]["id"]) + for svc in services: + proxy = svc.proxy.list() + futures = [] + for be_usage in svc.backend_usages.list(): + futures += [event_loop.run_in_executor(None, create_mapping_rules, i, be_usage) for i in range(10)] + await asyncio.gather(*futures) + proxy.deploy() + return services + + +@pytest.fixture(scope="module") +def template(root_path): + """Path to template""" + return os.path.join(root_path, "load_testing/templates/template_user_key_query_20_40M.hf.yaml") + + +@pytest.fixture(scope="module") +def hyperfoil_utils(hyperfoil_client, template, request): + """Init of hyperfoil utils""" + utils = HyperfoilUtils(hyperfoil_client, template) + request.addfinalizer(utils.finalizer) + return utils + + +@pytest.fixture(scope="module") +def setup_benchmark(hyperfoil_utils, applications, shared_template, promoted_services): + """Setup of benchmark. It will add necessary host connections, csv data and files.""" + hyperfoil_utils.add_hosts(promoted_services, shared_connections=300) + hyperfoil_utils.add_user_key_auth(applications, "auth_user_key.csv") + hyperfoil_utils.add_file(HyperfoilUtils.message_1kb) + hyperfoil_utils.add_shared_template(shared_template) + return hyperfoil_utils + + +@backoff.on_predicate(backoff.constant, lambda x: not x.is_finished(), interval=5, max_time=MAX_RUN_TIME) +def wait_run(run): + """Waits for the run to end""" + return run.reload() + + +def test_user_key_20_40M(applications, prod_client, setup_benchmark): + """ + Test checks that application is setup correctly. + Runs the created benchmark. + Asserts it was successful. + """ + for app in applications: + client = prod_client(app) + assert client.get("/0/anything/0").status_code == 200 + assert client.post("/0/anything/0").status_code == 200 + + benchmark = setup_benchmark.create_benchmark() + run = benchmark.start() + + run = wait_run(run) + + stats = run.all_stats() + + assert stats + assert stats.get("info", {}).get("errors") == [] + assert stats.get("failures") == [] + assert stats.get("stats", []) != [] + assert len(stats.get("stats", [])) == 3