From e22d5e07cb59143482f5c752a97dd87c2e3b7cd8 Mon Sep 17 00:00:00 2001 From: ycggyao Date: Tue, 22 Apr 2025 11:39:11 +0800 Subject: [PATCH] =?UTF-8?q?feat(backend):=20mongodb=20webconsole=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=20#10199=20#=20Reviewed,=20transaction=20id:=2039774?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/db_remote_service/client.py | 10 ++ .../backend/db_services/dbbase/serializers.py | 3 + dbm-ui/backend/db_services/dbbase/views.py | 6 +- .../db_services/mongodb/cluster/handlers.py | 27 ++++- .../db_services/mongodb/resources/views.py | 7 +- dbm-ui/backend/flow/consts.py | 1 + .../flow/utils/mongodb/mongodb_repo.py | 7 +- .../flow/utils/mongodb/mongodb_util.py | 109 ++++++++++++++++++ dbm-ui/backend/iam_app/dataclass/actions.py | 16 +++ 9 files changed, 181 insertions(+), 5 deletions(-) diff --git a/dbm-ui/backend/components/db_remote_service/client.py b/dbm-ui/backend/components/db_remote_service/client.py index 9e5e87f72e..5cae480f22 100644 --- a/dbm-ui/backend/components/db_remote_service/client.py +++ b/dbm-ui/backend/components/db_remote_service/client.py @@ -141,5 +141,15 @@ def __init__(self): description=_("mysql rpc 复杂接口"), ) + self.mongodb_rpc = ProxyAPI( + method="POST", + base=self.BASE_DOMAIN, + url="mongodb/rpc", + module=self.MODULE, + ssl=ssl_flag, + description=_("mongodb 远程执行"), + default_timeout=self.DRS_TIMEOUT, + ) + DRSApi = _DRSApi() diff --git a/dbm-ui/backend/db_services/dbbase/serializers.py b/dbm-ui/backend/db_services/dbbase/serializers.py index 28e1cc3518..75972a6814 100644 --- a/dbm-ui/backend/db_services/dbbase/serializers.py +++ b/dbm-ui/backend/db_services/dbbase/serializers.py @@ -23,6 +23,7 @@ from backend.db_services.ipchooser.query.resource import ResourceQueryHelper from backend.db_services.redis.resources.redis_cluster.query import RedisListRetrieveResource from backend.dbm_init.constants import CC_APP_ABBR_ATTR +from backend.ticket.builders.common.field import DBTimezoneField from backend.ticket.constants import TicketType @@ -188,6 +189,8 @@ class WebConsoleSerializer(serializers.Serializer): # redis 额外参数 db_num = serializers.IntegerField(help_text=_("数据库编号(redis 额外参数)"), required=False) raw = serializers.BooleanField(help_text=_("源编码(redis 额外参数)"), required=False) + # mongodb 额外参数 + session_time = DBTimezoneField(help_text=_("会话创建时间(mongodb 额外参数)"), required=False) class DBConsoleSerializer(serializers.Serializer): diff --git a/dbm-ui/backend/db_services/dbbase/views.py b/dbm-ui/backend/db_services/dbbase/views.py index 6130975001..261c414fe0 100644 --- a/dbm-ui/backend/db_services/dbbase/views.py +++ b/dbm-ui/backend/db_services/dbbase/views.py @@ -55,6 +55,7 @@ WebConsoleSerializer, ) from backend.db_services.ipchooser.query.resource import ResourceQueryHelper +from backend.db_services.mongodb.cluster.handlers import ClusterServiceHandler as MongoClusterServiceHandler from backend.db_services.mysql.remote_service.handlers import RemoteServiceHandler from backend.db_services.redis.toolbox.handlers import ToolboxHandler from backend.iam_app.handlers.drf_perm.base import DBManagePermission @@ -295,7 +296,10 @@ def webconsole(self, request): # redis elif db_type in ClusterType.redis_cluster_types(): data = ToolboxHandler.webconsole_rpc(**data) - + # mongodb + elif db_type == DBType.MongoDB: + data["user_id"] = request.user.id + data = MongoClusterServiceHandler(bk_biz_id=cluster.bk_biz_id).webconsole_rpc(**data) # 对外部查询进行数据脱敏 if getattr(request, "is_external", False) and env.BKDATA_DATA_TOKEN: data = BKBaseApi.data_desensitization(text=json.dumps(data), bk_biz_id=cluster.bk_biz_id) diff --git a/dbm-ui/backend/db_services/mongodb/cluster/handlers.py b/dbm-ui/backend/db_services/mongodb/cluster/handlers.py index 79fd2f8a40..76564c8bc2 100644 --- a/dbm-ui/backend/db_services/mongodb/cluster/handlers.py +++ b/dbm-ui/backend/db_services/mongodb/cluster/handlers.py @@ -9,8 +9,33 @@ specific language governing permissions and limitations under the License. """ +from datetime import datetime + +from django.utils import timezone + +from backend.components import DRSApi +from backend.db_meta.exceptions import ClusterNotExistException, InstanceNotExistException from backend.db_services.dbbase.cluster.handlers import ClusterServiceHandler as BaseClusterServiceHandler +from backend.exceptions import ApiResultError +from backend.flow.utils.mongodb.mongodb_util import MongoUtil class ClusterServiceHandler(BaseClusterServiceHandler): - pass + @staticmethod + def webconsole_rpc(cluster_id: int, cmd: str, **kwargs): + """ + 执行webconsole命令，只支持select语句 + @param cluster_id: 集群ID + @param cmd: 执行命令 + """ + # 获取rpc结果 + try: + session_time = kwargs.get("session_time", datetime.now(timezone.utc).replace(microsecond=0)) + session = f"{kwargs['user_id']}:{session_time}" + rpc_results = DRSApi.mongodb_rpc( + MongoUtil.get_mongodb_webconsole_args(cluster_id=cluster_id, session=session, command=cmd) + ) + except (ApiResultError, InstanceNotExistException, ClusterNotExistException) as err: + return {"query": "", "error_msg": err.message} + + return {"query": rpc_results, "error_msg": ""} diff --git a/dbm-ui/backend/db_services/mongodb/resources/views.py b/dbm-ui/backend/db_services/mongodb/resources/views.py index 31cc38c4d7..e6d1fddf68 100644 --- a/dbm-ui/backend/db_services/mongodb/resources/views.py +++ b/dbm-ui/backend/db_services/mongodb/resources/views.py @@ -101,7 +101,12 @@ class MongoDBViewSet(ResourceViewSet): query_serializer_class = serializers.ListMongoDBResourceSLZ list_instances_slz = serializers.MongoDBListInstancesSerializer - list_perm_actions = [ActionEnum.MONGODB_VIEW, ActionEnum.MONGODB_ENABLE_DISABLE, ActionEnum.MONGODB_DESTROY] + list_perm_actions = [ + ActionEnum.MONGODB_VIEW, + ActionEnum.MONGODB_ENABLE_DISABLE, + ActionEnum.MONGODB_DESTROY, + ActionEnum.MONGODB_WEBCONSOLE, + ] list_instance_perm_actions = [ActionEnum.MONGODB_VIEW] @common_swagger_auto_schema( diff --git a/dbm-ui/backend/flow/consts.py b/dbm-ui/backend/flow/consts.py index 4ef69dbd92..e468d17241 100644 --- a/dbm-ui/backend/flow/consts.py +++ b/dbm-ui/backend/flow/consts.py @@ -1474,6 +1474,7 @@ class MongoDBManagerUser(str, StructuredEnum): AppDbaUser = EnumField("appdba", _("appdba")) MonitorUser = EnumField("monitor", _("monitor")) AppMonitorUser = EnumField("appmonitor", _("appmonitor")) + WebconsoleUser = EnumField("_webconsoleuser", _("_webconsoleuser")) class MongoDBUserPrivileges(str, StructuredEnum): diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py index d701a8323b..456e01296c 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py @@ -40,6 +40,9 @@ def from_instance(cls, s: Union[ProxyInstance, StorageInstance], with_domain: bo node = MongoNode(s.ip_port.split(":")[0], s.port, meta_role, s.machine.bk_cloud_id, s.machine_type, domain) return node + def addr(self) -> str: + return "{}:{}".format(self.ip, self.port) + def equal(self, other: "MongoNode") -> bool: return self.ip == other.ip and self.port == other.port and self.bk_cloud_id == other.bk_cloud_id @@ -432,14 +435,14 @@ def fetch_many_cluster(cls, with_domain: bool, **kwargs): return rows @classmethod - def fetch_one_cluster(cls, withDomain: bool, **kwargs): + def fetch_one_cluster(cls, withDomain: bool, **kwargs) -> MongoDBCluster: rows = cls.fetch_many_cluster(withDomain, **kwargs) if len(rows) > 0: return rows[0] return None @classmethod - def fetch_many_cluster_dict(cls, withDomain: bool = False, **kwargs): + def fetch_many_cluster_dict(cls, withDomain: bool = False, **kwargs) -> dict[int, MongoDBCluster]: clusters = cls.fetch_many_cluster(withDomain, **kwargs) clusters_map = {} for cluster in clusters: diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_util.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_util.py index 7c4b2ae0c2..3006cb44fe 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_util.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_util.py @@ -9,6 +9,7 @@ from backend.flow.consts import ConfigFileEnum, ConfigTypeEnum, MongoDBManagerUser, NameSpaceEnum from backend.flow.utils.mongodb import mongodb_password from backend.flow.utils.mongodb.mongodb_module_operate import MongoDBCCTopoOperator +from backend.flow.utils.mongodb.mongodb_repo import MongoRepository logger = logging.getLogger("flow") @@ -93,3 +94,111 @@ def update_instance_labels(*cluster_id_list: int): MongoDBCCTopoOperator(cluster).transfer_instances_to_cluster_module(proxy_objs, is_increment=True) logger.info("cluster_id:{} update instance labels success".format(cluster_id)) + + @staticmethod + def get_mongodb_webconsole_args(cluster_id: int, session: str, command: str, timeout: int = 15): + """ + 获取mongodb webconsole参数 + @param cluster_id: 集群id + @param session: session，用于标识一个请求，请务必保证带有用户id信息. + @param command: 命令，如：show dbs, 首次连接可以为空 + @param timeout: 超时时间，单位秒 + """ + cluster = MongoRepository().fetch_one_cluster(withDomain=False, id=cluster_id) + if not cluster: + raise Exception("cluster_id:{} not found".format(cluster_id)) + + # 获得连接的节点, 集群为mongos节点，副本集优先使用backup节点，没有backup节点则为m1节点 + connect_nodes = [] + if cluster.is_sharded_cluster(): + connect_nodes = cluster.get_mongos()[:2] # 取两个mongos就行 + else: + shard = cluster.get_shards()[0] + node = shard.get_backup_node() + if node: + connect_nodes.append(node) + else: + connect_nodes.append(shard.get_not_backup_nodes()[0]) + + if len(connect_nodes) == 0: + raise Exception("cluster_id:{} can not get connect node".format(cluster_id)) + + # ip port + node = connect_nodes[0] + adminUserName = MongoDBManagerUser.DbaUser.value + password_out = mongodb_password.MongoDBPassword().get_password_from_db( + node.ip, node.port, node.bk_cloud_id, adminUserName + ) + + if not password_out or "password" not in password_out: + raise Exception( + "can not get webconsole_user password for {}({}:{})".format(cluster_id, node.ip, node.port) + ) + else: + adminPassword = password_out["password"] + + user, pwd, is_created = MongoUtil.cluster_pwd_get_or_create( + cluster_id=cluster_id, bk_cloud_id=node.bk_cloud_id, username=MongoDBManagerUser.WebconsoleUser.value + ) + + logger.info( + "cluster_id:{} webconsole user: {}, password: {}, is_created: {}".format(cluster_id, user, pwd, is_created) + ) + + return { + "cluster_id": cluster.cluster_id, + "cluster_type": cluster.cluster_type, + "cluster_domain": cluster.immute_domain, + "version": cluster.major_version.split("-")[-1], + "addresses": [m.addr() for m in connect_nodes], # 取两个mongos就行 + "set_name": cluster.name, + "command": command, + "timeout": timeout, + "admin_username": adminUserName, + "admin_password": adminPassword, + "username": user, + "password": pwd, + "session": "{}:{}".format(cluster_id, session), + } + + @staticmethod + def cluster_pwd_get_or_create(cluster_id: int, bk_cloud_id: int, username: str) -> (str, str, bool): + """ + 从密码库中获取或生成mongodb用户密码 + 按mongo:+cluster_id为主键获取密码, 密码规则为mongodb_password + 返回 值为 (username, password, is_created) + """ + is_created = False + cluster = "mongodb_cluster:{}".format(cluster_id) + out = mongodb_password.MongoDBPassword().get_password_from_db(cluster, int(0), bk_cloud_id, username) + # 接口返回异常 + if not out or "password" not in out: + raise Exception("can not get dba_user password for {}:{}".format(cluster_id, bk_cloud_id)) + + # 如果密码为空，需要创建密码 + if out["password"] is None or out["password"] == "": + new_pwd = mongodb_password.MongoDBPassword().create_user_password() + if new_pwd["password"] is None: + raise Exception("create password fail, error:{}".format(new_pwd["info"])) + # 密码长度小于8位，表示创建失败. 我们的密码规则不会允许小于8位的密码 + if len(new_pwd["password"]) < 8: + raise Exception("create password fail, password length is {}".format(len(new_pwd["password"]))) + err_msg = mongodb_password.MongoDBPassword().save_password_to_db2( + instances=[ + { + "ip": cluster, + "port": 0, + "bk_cloud_id": bk_cloud_id, + } + ], + username=username, + password=new_pwd["password"], + operator="admin", + ) + + if err_msg != "": + raise Exception("save password to db fail, error:{}".format(err_msg)) + out["password"] = new_pwd["password"] + is_created = True + + return username, out["password"], is_created diff --git a/dbm-ui/backend/iam_app/dataclass/actions.py b/dbm-ui/backend/iam_app/dataclass/actions.py index e1c7091829..c52f864e32 100644 --- a/dbm-ui/backend/iam_app/dataclass/actions.py +++ b/dbm-ui/backend/iam_app/dataclass/actions.py @@ -1534,6 +1534,22 @@ class ActionEnum: common_labels=[CommonActionLabel.BIZ_MAINTAIN], ) + MONGODB_WEBCONSOLE = ActionMeta( + id="mongodb_webconsole", + name=_("MongoDB Webconsole执行"), + name_en="mongodb_webconsole", + type="execute", + related_actions=[DB_MANAGE.id], + related_resource_types=[ResourceEnum.MONGODB], + group=_("MongoDB"), + subgroup=_("集群管理"), + common_labels=[ + CommonActionLabel.BIZ_READ_ONLY, + CommonActionLabel.BIZ_MAINTAIN, + CommonActionLabel.EXTERNAL_DEVELOPER + ], + ) + SQLSERVER_VIEW = ActionMeta( id="sqlserver_view", name=_("SQLServer 集群详情查看"),