Merge pull request #698 from viduship/vim-rgw-accounts

CEPH-83581974 : Test LC cloud transition with scale of 1M objects

Author: mergify[bot]

rgw/v2/tests/s3_swift/multisite_configs/test_scale_aws_transition_retain_true.yaml

@@ -0,0 +1,47 @@
+#test_rgw_account_management.py
+#CEPH-83581974
+config:
+  user_count: 1
+  bucket_count: 0
+  objects_count: 10
+  parallel_lc: true
+  test_lc_transition: True
+  enable_resharding: false
+  sharding_type: manual
+  shards: 211
+  pool_name: data.cold
+  storage_class: cold
+  ec_pool_transition: False
+  multiple_transitions: True
+  two_pool_transition: False
+  second_pool_name: data.glacier
+  second_storage_class: glacier
+  remote_zone: secondary
+  objects_size_range:
+    min: 5
+    max: 15
+  test_ops:
+    create_bucket: false
+    create_object: false
+    delete_marker: false
+    test_via_rgw_accounts: true
+    tenant_name: tenant1
+    region: shared
+    reuse_account_bucket: true
+    test_cloud_transition: true
+    test_pool_transition: false
+    test_ibm_cloud_transition: false
+    test_aws_cloud_transition: true
+    test_retain_head: true
+    test_cloud_transition_at_remote: false
+  lifecycle_conf:
+    - ID: LC_Rule_1
+      Filter:
+        Prefix: r
+      Status: Enabled
+      Transitions:
+        - Days: 1
+          StorageClass: CLOUDAWS
+      NoncurrentVersionTransitions:
+        - NoncurrentDays: 1
+          StorageClass: CLOUDAWS

rgw/v2/tests/s3_swift/reusable.py

@@ -1046,36 +1046,78 @@ def put_get_bucket_lifecycle_test(
             raise TestExecError("bucket lifecycle config retrieval failed")
     else:
         raise TestExecError("bucket life cycle retrieved")
-    objs_total = (config.test_ops["version_count"]) * (config.objects_count)
-    if not upload_start_time:
-        upload_start_time = time.time()
-    if not upload_end_time:
-        upload_end_time = time.time()
-    time_diff = math.ceil(upload_end_time - upload_start_time)
-    time_limit = upload_start_time + (
-        config.rgw_lc_debug_interval * config.test_ops.get("actual_lc_days", 20)
-    )
-    for rule in config.lifecycle_conf:
-        if rule.get("Expiration", {}).get("Date", False):
-            # todo: need to get the interval value from yaml file
-            log.info(f"wait for 60 seconds")
-            time.sleep(60)
-        else:
-            while time.time() < time_limit:
-                bucket_stats_op = utils.exec_shell_cmd(
-                    "radosgw-admin bucket stats --bucket=%s" % bucket.name
+    if config.test_ops.get("reuse_account_bucket", False) is True:
+        max_retries = 1500
+        sleep_interval = 30
+        bucket_stats_output = utils.exec_shell_cmd(
+            f"radosgw-admin bucket stats --bucket tenant1/{bucket.name}"
+        )
+        bucket_stats_json = json.loads(bucket_stats_output)
+        objects_before_transition = bucket_stats_json["usage"]["rgw.main"][
+            "num_objects"
+        ]
+        lc_transition_start_time = time.time()
+        for retry in range(max_retries + 2):
+            if retry == 0:
+                time.sleep(
+                    max_retries
+                )  # since value of max_retries is same as rgw_lc_debug_interval
+
+            bucket_stats_output = utils.exec_shell_cmd(
+                f"radosgw-admin bucket stats --bucket tenant1/{bucket.name}"
+            )
+            log.info(f"bucket stats output for {bucket.name}: {bucket_stats_output}")
+            bucket_stats_json = json.loads(bucket_stats_output)
+
+            if (
+                bucket_stats_json["usage"]["rgw.cloudtiered"]["num_objects"]
+                >= objects_before_transition
+                and bucket_stats_json["usage"]["rgw.usage"]["num_objects"] == 0
+            ):
+                log.info(
+                    f" all the objects for bucket successfully cloud transitioned to IBM"
                 )
-                json_doc1 = json.loads(bucket_stats_op)
-                obj_pre_lc = json_doc1["usage"]["rgw.main"]["num_objects"]
-                if obj_pre_lc == objs_total or config.test_lc_transition:
-                    time.sleep(config.rgw_lc_debug_interval)
-                else:
-                    raise TestExecError("Objects expired before the expected days")
-    lc_grace_time = config.test_ops.get("lc_grace_time", 90)
-    log.info(
-        f"sleeping for {time_diff + lc_grace_time} seconds so that all objects gets expired/transitioned"
-    )
-    time.sleep(time_diff + lc_grace_time)
+                break
+            else:
+                log.info(
+                    f"Cloud transition still in progress after {retry} retry, sleep for {sleep_interval} and retry"
+                )
+                time.sleep(sleep_interval)
+        if retry > max_retries:
+            raise AssertionError(
+                f"LC transition to cloud for {objects_before_transition} failed"
+            )
+    else:
+
+        objs_total = (config.test_ops["version_count"]) * (config.objects_count)
+        if not upload_start_time:
+            upload_start_time = time.time()
+        if not upload_end_time:
+            upload_end_time = time.time()
+        time_diff = math.ceil(upload_end_time - upload_start_time)
+        time_limit = upload_start_time + (
+            config.rgw_lc_debug_interval * config.test_ops.get("actual_lc_days", 20)
+        )
+        for rule in config.lifecycle_conf:
+            if rule.get("Expiration", {}).get("Date", False):
+                # todo: need to get the interval value from yaml file
+                log.info("wait for 60 seconds")
+                time.sleep(60)
+            else:
+                while time.time() < time_limit:
+                    bucket_stats_op = utils.exec_shell_cmd(
+                        "radosgw-admin bucket stats --bucket=%s" % bucket.name
+                    )
+                    json_doc1 = json.loads(bucket_stats_op)
+                    obj_pre_lc = json_doc1["usage"]["rgw.main"]["num_objects"]
+                    if obj_pre_lc == objs_total or config.test_lc_transition:
+                        time.sleep(config.rgw_lc_debug_interval)
+                    else:
+                        raise TestExecError("Objects expired before the expected days")
+        log.info(
+            f"sleeping for {time_diff + 90} seconds so that all objects gets expired/transitioned"
+        )
+        time.sleep(time_diff + 90)
 
     if config.test_ops.get("conflict_exp_days"):
         bucket_stats_op = utils.exec_shell_cmd(
@@ -2029,17 +2071,23 @@ def prepare_for_bucket_lc_transition(config):
                     f"radosgw-admin zonegroup placement add  --rgw-zonegroup {zonegroup} --placement-id default-placement --storage-class CLOUDIBM --tier-type=cloud-s3 --tier-config=endpoint={endpoint},access_key={access},secret={secret},target_path={target_path},multipart_sync_threshold=44432,multipart_min_part_size=44432,retain_head_object=false,region=au-syd"
                 )
         else:
-            target_path = "aws-bucket-01"
+            wget_cmd = "curl -o aws_cloud.env http://magna002.ceph.redhat.com/cephci-jenkins/aws_cloud_file"
+            utils.exec_shell_cmd(cmd=f"{wget_cmd}")
+            aws_config = configobj.ConfigObj("aws_cloud.env")
+            target_path = aws_config["TARGET"]
+            access = aws_config["ACCESS"]
+            secret = aws_config["SECRET"]
+            endpoint = aws_config["ENDPOINT"]
             utils.exec_shell_cmd(
                 f"radosgw-admin zonegroup placement add --rgw-zonegroup {zonegroup} --placement-id default-placement --storage-class=CLOUDAWS --tier-type=cloud-s3"
             )
             if config.test_ops.get("test_retain_head", False):
                 utils.exec_shell_cmd(
-                    f"radosgw-admin zonegroup placement add  --rgw-zonegroup {zonegroup}   --placement-id default-placement --storage-class CLOUDAWS --tier-type=cloud-s3 --tier-config=endpoint=http://s3region.amazonaws.com,access_key=awsaccesskey,secret=awssecretkey,target_path={target_path},multipart_sync_threshold=44432,multipart_min_part_size=44432,retain_head_object=true,region=aws-region"
+                    f"radosgw-admin zonegroup placement add  --rgw-zonegroup {zonegroup}   --placement-id default-placement --storage-class CLOUDAWS --tier-type=cloud-s3 --tier-config=endpoint={endpoint},access_key={access},secret={secret},target_path={target_path},multipart_sync_threshold=44432,multipart_min_part_size=44432,retain_head_object=true,region=us-east-1"
                 )
             else:
                 utils.exec_shell_cmd(
-                    f"radosgw-admin zonegroup placement add  --rgw-zonegroup {zonegroup}   --placement-id default-placement --storage-class CLOUDAWS --tier-type=cloud-s3 --tier-config=endpoint=http://s3.aws-region.amazonaws.com,access_key=awsaccesskey,secret=awssecretkey,target_path={target_path},multipart_sync_threshold=44432,multipart_min_part_size=44432,retain_head_object=false,region=us-east-1"
+                    f"radosgw-admin zonegroup placement add  --rgw-zonegroup {zonegroup}   --placement-id default-placement --storage-class CLOUDAWS --tier-type=cloud-s3 --tier-config=endpoint={endpoint},access_key={access},secret={secret},target_path={target_path},multipart_sync_threshold=44432,multipart_min_part_size=44432,retain_head_object=false,region=us-east-1"
                 )
     if is_multisite:
         utils.exec_shell_cmd("radosgw-admin period update --commit")
@@ -3172,3 +3220,45 @@ def bring_up_all_rgws_in_the_site(rgw_service_name, retry=10, delay=10):
                             break
                     if retry_count + 1 == retry:
                         raise AssertionError("Node is not in expected state!!")
+
+
+def configure_rgw_lc_settings():
+    """
+    Retrieves RGW services using 'ceph orch ls | grep rgw' and sets LC debug configs.
+    """
+    log.info("Retrieving RGW service names...")
+
+    # Fetch RGW services
+    rgw_services_output = utils.exec_shell_cmd("ceph orch ls | grep rgw")
+
+    if not rgw_services_output:
+        log.error("No RGW services found or failed to retrieve.")
+        return
+
+    # Extract service names from output
+    rgw_services = []
+    for line in rgw_services_output.split("\n"):
+        line = line.strip()
+        if line:  # Ignore empty lines
+            columns = line.split()
+            if columns:  # Ensure there are columns before accessing
+                rgw_services.append(columns[0])
+
+    if not rgw_services:
+        log.warning("No valid RGW services extracted.")
+        return
+
+    log.info(f"Found RGW services: {rgw_services}")
+
+    # Set LC debug interval for each RGW service
+    for service in rgw_services:
+        lc_config_cmd1 = f"ceph config set client.{service} rgw_lc_debug_interval 600"
+        log.info(f"Setting LC config for {service}: {lc_config_cmd1}")
+        utils.exec_shell_cmd(lc_config_cmd1)
+        lc_config_cmd2 = "ceph config set client.{service} rgw_lc_max_worker 10"
+        log.info(f"Setting LC config for {service}: {lc_config_cmd2}")
+        utils.exec_shell_cmd(lc_config_cmd2)
+        ceph_restart_cmd = f"ceph orch restart {service}"
+        utils.exec_shell_cmd(ceph_restart_cmd)
+
+    log.info("RGW LC debug interval settings updated successfully.")

rgw/v2/tests/s3_swift/reusables/rgw_accounts.py

@@ -6,6 +6,7 @@
 
 import boto
 import boto3
+import v2.lib.resource_op as s3lib
 import v2.utils.utils as utils
 from v2.lib.exceptions import TestExecError
 from v2.lib.rgw_config_opts import CephConfOp, ConfigOpts
@@ -255,6 +256,71 @@ def create_rgw_account_with_iam_user(config, tenant_name, region="shared"):
     return iam_user_details
 
 
+def reuse_account_bucket(config, rgw, user_info, location=None):
+    """
+    Reuse an existing bucket for an RGW account if it meets predefined conditions.
+    If the bucket has more than 1M objects, it is selected for reuse.
+    :param rgw: RGW resource connection
+    :param user_info: Dictionary containing user credentials
+    :return: Bucket resource object if criteria met, else None
+    """
+    rgw_account_id = get_rgw_account()
+    log.info(f"Fetching bucket list for RGW account: {rgw_account_id}")
+
+    # Get the list of buckets
+    cmd = f"radosgw-admin bucket list --account-id {rgw_account_id}"
+    bucket_list_json = utils.exec_shell_cmd(cmd)
+
+    if not bucket_list_json:
+        log.error("Failed to retrieve bucket list or received empty response.")
+        return None
+
+    bucket_list = json.loads(bucket_list_json)
+
+    if not bucket_list:
+        log.warning("No buckets found for the given account.")
+        return None
+
+    log.info(f"Found {len(bucket_list)} buckets. Checking stats...")
+
+    for bucket_name in bucket_list:
+        original_bucket_name = bucket_name  # Keep original for logging
+        if "tenant" in bucket_name:
+            tenant_name, bucket_short_name = bucket_name.split(".", 1)
+            bucket_name = f"{tenant_name}/{bucket_name}"
+
+        log.info(f"Checking stats for bucket: {original_bucket_name}")
+
+        # Fetch bucket statistics
+        stats_cmd = f"radosgw-admin bucket stats --bucket {bucket_name}"
+        bucket_stats_json = utils.exec_shell_cmd(stats_cmd)
+
+        if not bucket_stats_json:
+            log.warning(f"Skipping {original_bucket_name}: Failed to retrieve stats.")
+            continue
+
+        bucket_stats = json.loads(bucket_stats_json)
+        num_objects = (
+            bucket_stats.get("usage", {}).get("rgw.main", {}).get("num_objects", 0)
+        )
+
+        log.info(f"Bucket {original_bucket_name} has {num_objects} objects.")
+
+        if num_objects >= 1100000:
+            log.info(
+                f"Bucket {original_bucket_name} meets criteria. Returning for reuse."
+            )
+
+            # Return bucket in the expected format
+            bucket = s3lib.resource_op(
+                {"obj": rgw, "resource": "Bucket", "args": [original_bucket_name]}
+            )
+            return bucket  # Return the first valid bucket
+
+    log.warning("No buckets met the 1M objects criteria.")
+    return None  # No valid bucket found
+
+
 def perform_user_adoption(config, user_info, bucket):
     """
     Perform user adoption by migrating an RGW user to an RGW account.

rgw/v2/tests/s3_swift/test_rgw_account_management.py

@@ -7,6 +7,7 @@
 The Input yamls are
 # configs/test_account_ownership_change_user_adoption.yaml
 # multisite_configs/test_rgw_accounts_at_scale.yaml
+# multisite_configs/test_scale_aws_transition_retain_true.yaml
 
 Usage  is test_rgw_account_management.py -c  <path_to_config_file>
 
@@ -64,9 +65,17 @@ def test_exec(config, ssh_con):
     for each_user in all_users_info:
         auth = Auth(each_user, ssh_con, ssl=config.ssl)
         rgw_conn = auth.do_auth()
+        rgw_conn2 = auth.do_auth_using_client()
         log.info(f"Creating {config.bucket_count} buckets for {each_user['user_id']}")
         user_buckets = []  # Store buckets for this user
-
+        if config.test_ops.get("reuse_account_bucket", False) is True:
+            life_cycle_rule = {"Rules": config.lifecycle_conf}
+            reusable.prepare_for_bucket_lc_transition(config)
+            bucket = accounts.reuse_account_bucket(config, rgw_conn, each_user)
+            reusable.configure_rgw_lc_settings()
+            reusable.put_get_bucket_lifecycle_test(
+                bucket, rgw_conn, rgw_conn2, life_cycle_rule, config
+            )
         for bc in range(config.bucket_count):
             bucket_name = utils.gen_bucket_name_from_userid(
                 each_user["user_id"], rand_no=bc