Skip to content

Commit f7e5523

Browse files
Hemanth Sai MaheswarlaHemanth Sai Maheswarla
Hemanth Sai Maheswarla
authored and
Hemanth Sai Maheswarla
committed
adding support for lc processing on existing users and buckets passed from the config
Signed-off-by: Hemanth Sai Maheswarla <hemanthsaimaheswarla@Hemanths-MacBook-Pro.local> adding support for setting configs on all rgw daemons and restarting all rgw daemons Signed-off-by: Hemanth Sai Maheswarla <hemanthsaimaheswarla@Hemanths-MacBook-Pro.local> adding rgw_lc_max_wp_worker config parameter Signed-off-by: Hemanth Sai Maheswarla <hemanthsaimaheswarla@Hemanths-MacBook-Pro.local>
1 parent b1e8773 commit f7e5523

6 files changed

+196
-35
lines changed

rgw/v2/lib/resource_op.py

+8-3
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,9 @@ def resource_op(exec_info):
6868
return False
6969

7070

71-
def create_users(no_of_users_to_create, user_names=None, cluster_name="ceph"):
71+
def create_users(
72+
no_of_users_to_create, user_names=None, cluster_name="ceph", config=None
73+
):
7274
"""
7375
This function is to create n users on the cluster
7476
@@ -83,11 +85,11 @@ def create_users(no_of_users_to_create, user_names=None, cluster_name="ceph"):
8385
all_users_details = []
8486
primary = utils.is_cluster_primary()
8587
user_detail_file = os.path.join(lib_dir, "user_details.json")
86-
if primary:
88+
if primary or (config and config.user_names):
8789
for i in range(no_of_users_to_create):
8890
if user_names:
8991
user_details = admin_ops.create_admin_user(
90-
user_id=user_names,
92+
user_id=user_names[i],
9193
displayname=user_names,
9294
cluster_name=cluster_name,
9395
)
@@ -308,6 +310,7 @@ def read(self, ssh_con=None):
308310
"rgw_lifecycle_work_time", "00:00-06:00"
309311
)
310312
self.rgw_lc_max_worker = self.doc["config"].get("rgw_lc_max_worker", 10)
313+
self.rgw_lc_max_wp_worker = self.doc["config"].get("rgw_lc_max_wp_worker", 10)
311314
self.parallel_lc = self.doc["config"].get("parallel_lc", False)
312315
self.multiple_delete_marker_check = self.doc["config"].get(
313316
"multiple_delete_marker_check", False
@@ -440,6 +443,8 @@ def read(self, ssh_con=None):
440443
)
441444
self.user_conflict_write_ops = self.doc["config"].get("user_conflict_write_ops")
442445
self.permutation_count = self.doc["config"].get("permutation_count")
446+
self.user_names = self.doc["config"].get("user_names")
447+
self.bucket_names = self.doc["config"].get("bucket_names")
443448
ceph_version_id, ceph_version_name = utils.get_ceph_version()
444449
# todo: improve Frontend class
445450
if ceph_version_name in ["luminous", "nautilus"]:

rgw/v2/lib/rgw_config_opts.py

+24-5
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ def __init__(self):
3030
rgw_enable_lc_threads = "rgw_enable_lc_threads"
3131
rgw_lifecycle_work_time = "rgw_lifecycle_work_time"
3232
rgw_lc_max_worker = "rgw_lc_max_worker"
33+
rgw_lc_max_wp_worker = "rgw_lc_max_wp_worker"
3334
debug_rgw = "debug_rgw"
3435
log_to_file = "log_to_file"
3536
rgw_crypt_require_ssl = "rgw_crypt_require_ssl"
@@ -126,12 +127,17 @@ def set_to_ceph_conf_file(self, section, option, value=None, ssh_con=None):
126127

127128

128129
class CephConfigSet:
129-
def set_to_ceph_cli(self, key, value, set_to_all=False):
130+
def set_to_ceph_cli(self, key, value, set_to_all=False, remote_ssh_con=None):
130131
log.info("setting key and value using ceph config set cli")
131132
self.prefix = "sudo ceph config set"
132133

133134
cmd_ps = "ceph orch ps --daemon_type rgw -f json"
134-
out_ps = utils.exec_shell_cmd(cmd_ps)
135+
if remote_ssh_con:
136+
out_ps = utils.remote_exec_shell_cmd(
137+
remote_ssh_con, cmd_ps, return_output=True
138+
)
139+
else:
140+
out_ps = utils.exec_shell_cmd(cmd_ps)
135141
out = json.loads(out_ps)
136142
daemon_name_list = []
137143
for node in out:
@@ -146,7 +152,12 @@ def set_to_ceph_cli(self, key, value, set_to_all=False):
146152
log.info(f"got value: {value}")
147153
cmd_list = [self.prefix, self.who, key, str(value)]
148154
cmd = " ".join(cmd_list)
149-
config_set = utils.exec_shell_cmd(cmd)
155+
if remote_ssh_con:
156+
config_set = utils.remote_exec_shell_cmd(
157+
remote_ssh_con, cmd, return_output=False
158+
)
159+
else:
160+
config_set = utils.exec_shell_cmd(cmd)
150161
if config_set is False:
151162
raise InvalidCephConfigOption("Invalid ceph config options")
152163
if not set_to_all:
@@ -158,7 +169,13 @@ def __init__(self, ssh_con=None) -> None:
158169
super().__init__(ssh_con)
159170

160171
def set_to_ceph_conf(
161-
self, section, option, value=None, ssh_con=None, set_to_all=False
172+
self,
173+
section,
174+
option,
175+
value=None,
176+
ssh_con=None,
177+
set_to_all=False,
178+
remote_ssh_con=None,
162179
):
163180
version_id, version_name = utils.get_ceph_version()
164181
log.info(f"ceph version id: {version_id}")
@@ -174,4 +191,6 @@ def set_to_ceph_conf(
174191
log.info("using ceph config cli to set the config values")
175192
log.info(option)
176193
log.info(value)
177-
self.set_to_ceph_cli(option, value, set_to_all=set_to_all)
194+
self.set_to_ceph_cli(
195+
option, value, set_to_all=set_to_all, remote_ssh_con=remote_ssh_con
196+
)

rgw/v2/lib/s3/lifecycle_validation.py

+20-9
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,6 @@ def validate_prefix_rule(bucket, config):
176176
f"Lifecycle cloud transition validation failed with retain_head_object_false for bucket {bucket.name}"
177177
)
178178
else:
179-
180179
log.info("Start the validation of LC pool transition")
181180
curr = 0
182181
ncurr = 0
@@ -244,21 +243,28 @@ def validate_prefix_rule(bucket, config):
244243
)
245244

246245

247-
def validate_prefix_rule_non_versioned(bucket, config):
246+
def validate_prefix_rule_non_versioned(bucket, config, ssh_con=None):
248247
log.info("verification starts")
249248
objects_count = config.objects_count
250249
if config.test_lc_transition:
251-
cmd = utils.exec_shell_cmd(f"radosgw-admin bucket list --bucket {bucket.name}")
250+
cmd = utils.exec_shell_cmd(
251+
f"radosgw-admin bucket list --bucket {bucket.name}{f' --max-entries {config.objects_count}' if config.objects_count > 1000 else ''}"
252+
)
252253
json_doc = json.loads(cmd)
253-
for i in range(0, objects_count):
254+
obj_count = (
255+
config.objects_count if config.objects_count < 1000 else len(json_doc)
256+
)
257+
for i in range(0, obj_count):
254258
storage_class = json_doc[i]["meta"]["storage_class"]
255259
log.info(f"object has transitioned to {storage_class}")
256260
if storage_class != config.storage_class:
257261
raise AssertionError("lc validation for object transition failed")
258262
else:
259-
op = utils.exec_shell_cmd(
260-
"radosgw-admin bucket stats --bucket=%s" % bucket.name
261-
)
263+
bkt_stats_cmd = f"radosgw-admin bucket stats --bucket {bucket.name}"
264+
if ssh_con:
265+
op = utils.remote_exec_shell_cmd(ssh_con, bkt_stats_cmd, return_output=True)
266+
else:
267+
op = utils.exec_shell_cmd(bkt_stats_cmd)
262268
json_doc = json.loads(op)
263269
objects = json_doc["usage"]["rgw.main"]["num_objects"]
264270
if objects != 0:
@@ -276,13 +282,18 @@ def validate_and_rule(bucket, config):
276282
log.info("verification starts")
277283
op = utils.exec_shell_cmd("radosgw-admin bucket stats --bucket=%s" % bucket.name)
278284
json_doc = json.loads(op)
279-
op2 = utils.exec_shell_cmd(f"radosgw-admin bucket list --bucket {bucket.name}")
285+
op2 = utils.exec_shell_cmd(
286+
f"radosgw-admin bucket list --bucket {bucket.name}{f' --max-entries {config.objects_count}' if config.objects_count > 1000 else ''}"
287+
)
280288
json_doc2 = json.loads(op2)
281289
objects = json_doc["usage"]["rgw.main"]["num_objects"]
282290
if config.test_lc_transition and not config.test_ops.get(
283291
"conflict_btw_exp_transition"
284292
):
285-
for i in range(0, config.objects_count):
293+
obj_count = (
294+
config.objects_count if config.objects_count < 1000 else len(json_doc2)
295+
)
296+
for i in range(0, obj_count):
286297
storage_class = json_doc2[i]["meta"]["storage_class"]
287298
if storage_class != config.storage_class:
288299
raise AssertionError("LC transition for AND filters failed")

rgw/v2/tests/s3_swift/reusable.py

+12-7
Original file line numberDiff line numberDiff line change
@@ -1058,7 +1058,7 @@ def put_get_bucket_lifecycle_test(
10581058
for rule in config.lifecycle_conf:
10591059
if rule.get("Expiration", {}).get("Date", False):
10601060
# todo: need to get the interval value from yaml file
1061-
log.info("wait for 60 seconds")
1061+
log.info(f"wait for 60 seconds")
10621062
time.sleep(60)
10631063
else:
10641064
while time.time() < time_limit:
@@ -1071,10 +1071,11 @@ def put_get_bucket_lifecycle_test(
10711071
time.sleep(config.rgw_lc_debug_interval)
10721072
else:
10731073
raise TestExecError("Objects expired before the expected days")
1074+
lc_grace_time = config.test_ops.get("lc_grace_time", 90)
10741075
log.info(
1075-
f"sleeping for {time_diff + 90} seconds so that all objects gets expired/transitioned"
1076+
f"sleeping for {time_diff + lc_grace_time} seconds so that all objects gets expired/transitioned"
10761077
)
1077-
time.sleep(time_diff + 90)
1078+
time.sleep(time_diff + lc_grace_time)
10781079

10791080
if config.test_ops.get("conflict_exp_days"):
10801081
bucket_stats_op = utils.exec_shell_cmd(
@@ -1095,10 +1096,14 @@ def put_get_bucket_lifecycle_test(
10951096
for i, entry in enumerate(json_doc):
10961097
print(i)
10971098
print(entry["status"])
1098-
if entry["status"] == "COMPLETE" or entry["status"] == "PROCESSING":
1099-
log.info("LC is applied on the bucket")
1100-
else:
1101-
raise TestExecError("LC is not applied")
1099+
if bucket.name in entry["bucket"]:
1100+
if entry["status"] == "COMPLETE" or entry["status"] == "PROCESSING":
1101+
log.info("LC is applied on the bucket")
1102+
else:
1103+
raise TestExecError("LC is not applied")
1104+
break
1105+
else:
1106+
raise TestExecError("bucket not listed in lc list")
11021107
if config.test_ops.get("tenant_name"):
11031108
tenant_name = config.test_ops.get("tenant_name")
11041109
op_lc_get = utils.exec_shell_cmd(

rgw/v2/tests/s3_swift/test_bucket_lifecycle_object_expiration_transition.py

+99-5
Original file line numberDiff line numberDiff line change
@@ -82,19 +82,22 @@ def test_exec(config, ssh_con):
8282
ConfigOpts.rgw_lc_debug_interval,
8383
str(config.rgw_lc_debug_interval),
8484
ssh_con,
85+
set_to_all=config.test_ops.get("set_ceph_configs_to_all_daemons", False),
8586
)
8687
if not config.rgw_enable_lc_threads:
8788
ceph_conf.set_to_ceph_conf(
8889
"global",
8990
ConfigOpts.rgw_enable_lc_threads,
9091
str(config.rgw_enable_lc_threads),
9192
ssh_con,
93+
set_to_all=config.test_ops.get("set_ceph_configs_to_all_daemons", False),
9294
)
9395
ceph_conf.set_to_ceph_conf(
9496
"global",
9597
ConfigOpts.rgw_lifecycle_work_time,
9698
str(config.rgw_lifecycle_work_time),
9799
ssh_con,
100+
set_to_all=config.test_ops.get("set_ceph_configs_to_all_daemons", False),
98101
)
99102
_, version_name = utils.get_ceph_version()
100103
if "nautilus" in version_name:
@@ -110,6 +113,72 @@ def test_exec(config, ssh_con):
110113
option=ConfigOpts.rgw_lc_max_worker,
111114
value=str(config.rgw_lc_max_worker),
112115
ssh_con=ssh_con,
116+
set_to_all=config.test_ops.get("set_ceph_configs_to_all_daemons", False),
117+
)
118+
ceph_conf.set_to_ceph_conf(
119+
"global",
120+
ConfigOpts.rgw_lc_max_wp_worker,
121+
str(config.rgw_lc_max_wp_worker),
122+
ssh_con,
123+
set_to_all=config.test_ops.get("set_ceph_configs_to_all_daemons", False),
124+
)
125+
log.info(f"test_lc_on_other_site: {config.test_ops.get('test_lc_on_other_site')}")
126+
if config.test_ops.get("test_lc_on_other_site", False) is True:
127+
log.info(f"other_site_master: {config.test_ops.get('other_site_master')}")
128+
other_site_rgw_ip = utils.get_rgw_ip(
129+
config.test_ops.get("other_site_master", False)
130+
)
131+
log.info(f"other_site_rgw_ip: {other_site_rgw_ip}")
132+
other_site_ssh_con = utils.connect_remote(other_site_rgw_ip)
133+
log.info(f"other_site_ssh_con: {other_site_ssh_con}")
134+
ceph_conf.set_to_ceph_conf(
135+
"global",
136+
ConfigOpts.rgw_lc_debug_interval,
137+
str(config.rgw_lc_debug_interval),
138+
remote_ssh_con=other_site_ssh_con,
139+
set_to_all=config.test_ops.get("set_ceph_configs_to_all_daemons", False),
140+
)
141+
if not config.rgw_enable_lc_threads:
142+
ceph_conf.set_to_ceph_conf(
143+
"global",
144+
ConfigOpts.rgw_enable_lc_threads,
145+
str(config.rgw_enable_lc_threads),
146+
remote_ssh_con=other_site_ssh_con,
147+
set_to_all=config.test_ops.get(
148+
"set_ceph_configs_to_all_daemons", False
149+
),
150+
)
151+
ceph_conf.set_to_ceph_conf(
152+
"global",
153+
ConfigOpts.rgw_lifecycle_work_time,
154+
str(config.rgw_lifecycle_work_time),
155+
remote_ssh_con=other_site_ssh_con,
156+
set_to_all=config.test_ops.get("set_ceph_configs_to_all_daemons", False),
157+
)
158+
_, version_name = utils.get_ceph_version()
159+
if "nautilus" in version_name:
160+
ceph_conf.set_to_ceph_conf(
161+
"global",
162+
ConfigOpts.rgw_lc_max_worker,
163+
str(config.rgw_lc_max_worker),
164+
remote_ssh_con=other_site_ssh_con,
165+
)
166+
else:
167+
ceph_conf.set_to_ceph_conf(
168+
section=None,
169+
option=ConfigOpts.rgw_lc_max_worker,
170+
value=str(config.rgw_lc_max_worker),
171+
remote_ssh_con=other_site_ssh_con,
172+
set_to_all=config.test_ops.get(
173+
"set_ceph_configs_to_all_daemons", False
174+
),
175+
)
176+
ceph_conf.set_to_ceph_conf(
177+
"global",
178+
ConfigOpts.rgw_lc_max_wp_worker,
179+
str(config.rgw_lc_max_wp_worker),
180+
ssh_con,
181+
set_to_all=config.test_ops.get("set_ceph_configs_to_all_daemons", False),
113182
)
114183

115184
if config.test_lc_transition:
@@ -120,6 +189,12 @@ def test_exec(config, ssh_con):
120189

121190
log.info("trying to restart services")
122191
srv_restarted = rgw_service.restart(ssh_con)
192+
193+
if config.test_ops.get("set_ceph_configs_to_all_daemons", False):
194+
utils.restart_rgw(restart_all=True)
195+
if config.test_ops.get("test_lc_on_other_site", False):
196+
utils.restart_rgw(restart_all=True, ssh_con=other_site_ssh_con)
197+
123198
time.sleep(30)
124199
if srv_restarted is False:
125200
raise TestExecError("RGW service restart failed")
@@ -143,7 +218,9 @@ def test_exec(config, ssh_con):
143218
)
144219
else:
145220
log.info(f"user count is {config.user_count}")
146-
user_info = s3lib.create_users(config.user_count)
221+
user_info = s3lib.create_users(
222+
config.user_count, config.user_names, config=config
223+
)
147224
log.info(f"print user info {user_info}")
148225

149226
if config.test_ops.get("send_bucket_notifications", False) is True:
@@ -161,9 +238,12 @@ def test_exec(config, ssh_con):
161238

162239
log.info("no of buckets to create: %s" % config.bucket_count)
163240
for bc in range(config.bucket_count):
164-
bucket_name = utils.gen_bucket_name_from_userid(
165-
each_user["user_id"], rand_no=bc
166-
)
241+
if config.bucket_names:
242+
bucket_name = config.bucket_names[bc]
243+
else:
244+
bucket_name = utils.gen_bucket_name_from_userid(
245+
each_user["user_id"], rand_no=bc
246+
)
167247
obj_list = []
168248
obj_tag = "suffix1=WMV1"
169249
bucket = reusable.create_bucket(bucket_name, rgw_conn, each_user)
@@ -691,10 +771,19 @@ def test_exec(config, ssh_con):
691771
reusable.put_bucket_lifecycle(
692772
bucket, rgw_conn, rgw_conn2, life_cycle_rule
693773
)
774+
if config.test_ops.get("lc_grace_time"):
775+
log.info(
776+
f"sleeping for lc_grace_time {config.test_ops.get('lc_grace_time')} seconds"
777+
)
778+
time.sleep(config.test_ops.get("lc_grace_time"))
694779
time.sleep(60)
695780
for bucket in buckets:
696781
if not config.test_ops.get("enable_versioning", False) is True:
697782
lc_ops.validate_prefix_rule_non_versioned(bucket, config)
783+
if config.test_ops.get("test_lc_on_other_site", False):
784+
lc_ops.validate_prefix_rule_non_versioned(
785+
bucket, config, other_site_ssh_con
786+
)
698787
else:
699788
lc_ops.validate_prefix_rule(bucket, config)
700789
if config.test_ops.get("send_bucket_notifications", False) is True:
@@ -706,9 +795,14 @@ def test_exec(config, ssh_con):
706795
)
707796
rgw_service.restart()
708797
time.sleep(30)
798+
799+
if config.test_ops.get("delete_bucket_object", False):
800+
for bucket in buckets:
801+
reusable.delete_bucket(bucket)
802+
709803
if config.test_ops.get("test_via_rgw_accounts", False) is True:
710804
log.info("do not remove user")
711-
else:
805+
elif config.user_remove:
712806
reusable.remove_user(each_user)
713807
# check for any crashes during the execution
714808
crash_info = reusable.check_for_crash()

0 commit comments

Comments
 (0)