diff --git a/dbm-ui/backend/configuration/models/system.py b/dbm-ui/backend/configuration/models/system.py index 40d45f825c..b52c11378b 100644 --- a/dbm-ui/backend/configuration/models/system.py +++ b/dbm-ui/backend/configuration/models/system.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ import logging -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union from django.conf import settings from django.db import connection, models @@ -150,7 +150,7 @@ class Meta: ordering = ("id",) @classmethod - def get_setting_value(cls, bk_biz_id: int, key: str, default: Optional[Any] = None) -> Union[str, Dict]: + def get_setting_value(cls, bk_biz_id: int, key: str, default: Optional[Any] = None) -> Union[str, Dict, List]: return super().get_setting_value(key={"key": key, "bk_biz_id": bk_biz_id}, default=default) @classmethod diff --git a/dbm-ui/backend/db_proxy/urls.py b/dbm-ui/backend/db_proxy/urls.py index 273ee998d5..0427600bfa 100644 --- a/dbm-ui/backend/db_proxy/urls.py +++ b/dbm-ui/backend/db_proxy/urls.py @@ -16,6 +16,7 @@ from backend.db_proxy.views.db_remote_service.views import DRSApiProxyPassViewSet from backend.db_proxy.views.dbconfig.views import DBConfigProxyPassViewSet from backend.db_proxy.views.dns.views import DnsProxyPassViewSet +from backend.db_proxy.views.dumper.views import DumperProxyPassViewSet from backend.db_proxy.views.hadb.views import HADBProxyPassViewSet from backend.db_proxy.views.jobapi.views import JobApiProxyPassViewSet from backend.db_proxy.views.nameservice.views import NameServiceProxyPassViewSet @@ -33,5 +34,6 @@ routers.register(r"", BKRepoProxyPassViewSet, basename="bkrepo") routers.register(r"", DtsApiProxyPassViewSet, basename="redis_dts") routers.register(r"", JobApiProxyPassViewSet, basename="jobapi") +routers.register(r"", DumperProxyPassViewSet, basename="dumper") urlpatterns = routers.urls diff --git a/dbm-ui/backend/db_proxy/views/dumper/__init__.py b/dbm-ui/backend/db_proxy/views/dumper/__init__.py new file mode 100644 index 0000000000..aa5085c628 --- /dev/null +++ b/dbm-ui/backend/db_proxy/views/dumper/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/dbm-ui/backend/db_proxy/views/dumper/serializers.py b/dbm-ui/backend/db_proxy/views/dumper/serializers.py new file mode 100644 index 0000000000..685bb9403f --- /dev/null +++ b/dbm-ui/backend/db_proxy/views/dumper/serializers.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from backend.db_proxy.views.serialiers import BaseProxyPassSerialier + + +class DumperMigrateProxyPassSerializer(BaseProxyPassSerialier): + class DumperSwitchInfoSerializer(serializers.Serializer): + class SwitchInstanceSerializer(serializers.Serializer): + host = serializers.CharField(help_text=_("主机IP")) + port = serializers.IntegerField(help_text=_("主机端口")) + repl_binlog_file = serializers.CharField(help_text=_("待切换后需要同步的binlog文件")) + repl_binlog_pos = serializers.IntegerField(help_text=_("待切换后需要同步的binlog文件的为位点")) + + cluster_domain = serializers.IntegerField(help_text=_("集群域名")) + switch_instances = serializers.ListSerializer(help_text=_("dumper切换信息"), child=SwitchInstanceSerializer()) + + infos = serializers.ListSerializer(child=DumperSwitchInfoSerializer()) + bk_biz_id = serializers.IntegerField(help_text=_("业务ID")) + is_safe = serializers.BooleanField(help_text=_("是否安全切换"), required=False, default=True) diff --git a/dbm-ui/backend/db_proxy/views/dumper/views.py b/dbm-ui/backend/db_proxy/views/dumper/views.py new file mode 100644 index 0000000000..95ec8b187e --- /dev/null +++ b/dbm-ui/backend/db_proxy/views/dumper/views.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import ugettext_lazy as _ +from rest_framework.decorators import action +from rest_framework.response import Response + +from backend.bk_web.swagger import common_swagger_auto_schema +from backend.components.hadb.client import HADBApi +from backend.db_meta.models import Cluster +from backend.db_proxy.constants import SWAGGER_TAG +from backend.ticket.constants import TicketType +from backend.ticket.models import Ticket + +from ..views import BaseProxyPassViewSet +from .serializers import DumperMigrateProxyPassSerializer + + +class DumperProxyPassViewSet(BaseProxyPassViewSet): + """ + Dumper服务接口的透传视图 + """ + + @common_swagger_auto_schema( + operation_summary=_("[dumper]迁移"), + request_body=DumperMigrateProxyPassSerializer(), + tags=[SWAGGER_TAG], + ) + @action( + methods=["POST"], detail=False, serializer_class=DumperMigrateProxyPassSerializer, url_path="dumper/switch" + ) + def dumper_switch(self, request): + data = self.params_validate(self.get_serializer_class()) + # 获取集群ID + cluster_domains = [info["cluster_domain"] for info in data["infos"]] + domain__cluster = { + cluster.immute_domain: cluster for cluster in Cluster.objects.filter(immute_domain__in=cluster_domains) + } + for info in data["infos"]: + info["cluster_id"] = domain__cluster[info["cluster_domain"]] + # 创建dumper迁移单据 + Ticket.create_ticket( + ticket_type=TicketType.TBINLOGDUMPER_SWITCH_NODES, + creator=request.user.username, + bk_biz_id=data.pop("bk_biz_id"), + remark=_("透传接口dumper迁移创建的单据"), + details=data, + ) diff --git a/dbm-ui/backend/db_services/mysql/dumper/mock_data.py b/dbm-ui/backend/db_services/mysql/dumper/mock_data.py new file mode 100644 index 0000000000..63f5a35961 --- /dev/null +++ b/dbm-ui/backend/db_services/mysql/dumper/mock_data.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +DUMPER_INSTANCE_LIST_DATA = [ + { + "id": 1, + "creator": "xxx", + "create_at": "2022-11-11 20:00:00", + "updater": "xxx", + "update_at": "2022-11-11 20:00:00", + "bk_biz_id": 3, + "cluster_id": 64, + "bk_cloud_id": 0, + "ip": "127.0.0.1", + "proc_type": "tbinlogdumper", + "version": "1.0", + "listen_port": 10000, + "need_transfer": True, + "source_cluster": { + "id": 64, + "name": "fortest", + "cluster_type": "tendbha", + "immute_domain": "tendbha57db.xxx.dba.db", + "major_version": "MySQL-5.7", + "bk_cloud_id": 0, + "region": " ", + }, + "dumper_config": { + "id": 1, + "creator": "admin", + "updater": "admin", + "bk_biz_id": 3, + "name": "dumper-1", + "receiver_type": "redis", + "receiver": "redis.com:123", + "subscribe": {"db_name": "db1", "table_names": ["table1"]}, + }, + "dumper_id": 1, + "area_name": 1, + } +] diff --git a/dbm-ui/backend/db_services/mysql/dumper/serializers.py b/dbm-ui/backend/db_services/mysql/dumper/serializers.py index 01fd8ad8e0..0fdc34db56 100644 --- a/dbm-ui/backend/db_services/mysql/dumper/serializers.py +++ b/dbm-ui/backend/db_services/mysql/dumper/serializers.py @@ -15,6 +15,8 @@ from backend.db_meta.models.dumper import DumperSubscribeConfig from backend.db_meta.models.extra_process import ExtraProcessInstance +from . import mock_data + class DumperSubscribeConfigSerializer(serializers.ModelSerializer): class SubscribeInfoSerializer(serializers.Serializer): @@ -32,6 +34,7 @@ class DumperInstanceConfigSerializer(serializers.ModelSerializer): class Meta: model = ExtraProcessInstance fields = "__all__" + swagger_schema_fields = {"example": mock_data.DUMPER_INSTANCE_LIST_DATA} class VerifyDuplicateNamsSerializer(serializers.Serializer): diff --git a/dbm-ui/backend/db_services/mysql/dumper/views/dumper_instance.py b/dbm-ui/backend/db_services/mysql/dumper/views/dumper_instance.py index 6a00784b4e..1d03e82b3e 100644 --- a/dbm-ui/backend/db_services/mysql/dumper/views/dumper_instance.py +++ b/dbm-ui/backend/db_services/mysql/dumper/views/dumper_instance.py @@ -57,8 +57,10 @@ def list(self, request, *args, **kwargs): source_cluster = id__cluster[data["cluster_id"]] master = source_cluster.storageinstance_set.get(instance_inner_role=InstanceInnerRole.MASTER.value) data["need_transfer"] = data["ip"] != master.machine.ip - # 补充集群信息 + # 补充集群信息和集群的master计信系 data["source_cluster"] = source_cluster.simple_desc + data["source_cluster"]["master_ip"] = master.machine.ip + data["source_cluster"]["master_port"] = master.port # 补充订阅配置信息 dumper_config_id = extra_config["dumper_config_id"] data["dumper_config"] = model_to_dict(id__dumper_config[dumper_config_id]) diff --git a/dbm-ui/backend/db_services/mysql/open_area/serializers.py b/dbm-ui/backend/db_services/mysql/open_area/serializers.py index 3394587ab1..6f70ea2c53 100644 --- a/dbm-ui/backend/db_services/mysql/open_area/serializers.py +++ b/dbm-ui/backend/db_services/mysql/open_area/serializers.py @@ -76,3 +76,14 @@ class ConfigDataSerializer(serializers.Serializer): class TendbOpenAreaResultPreviewResponseSerializer(serializers.Serializer): class Meta: swagger_schema_fields = {"example": mock_data.OPENAREA_PREVIEW_DATA} + + +class VarAlterSerializer(serializers.Serializer): + op_type = serializers.CharField(help_text=_("操作类型")) + old_var = serializers.JSONField(help_text=_("旧变量(delete/update)"), required=False) + new_var = serializers.JSONField(help_text=_("新变量(add/update)"), required=False) + + class Meta: + swagger_schema_fields = { + "example": {"op_type": "add(delete, update)", "var": {"name": "APP", "desc": "xxx", "builtin": False}} + } diff --git a/dbm-ui/backend/db_services/mysql/open_area/views.py b/dbm-ui/backend/db_services/mysql/open_area/views.py index 2b40a49a46..a6d6d86a23 100644 --- a/dbm-ui/backend/db_services/mysql/open_area/views.py +++ b/dbm-ui/backend/db_services/mysql/open_area/views.py @@ -21,6 +21,8 @@ ResponseSwaggerAutoSchema, common_swagger_auto_schema, ) +from backend.configuration.constants import BizSettingsEnum +from backend.configuration.models import BizSettings from backend.db_meta.models import Cluster from backend.db_services.mysql.open_area.filters import TendbOpenAreaConfigListFilter from backend.db_services.mysql.open_area.handlers import OpenAreaHandler @@ -29,6 +31,7 @@ TendbOpenAreaConfigSerializer, TendbOpenAreaResultPreviewResponseSerializer, TendbOpenAreaResultPreviewSerializer, + VarAlterSerializer, ) from backend.iam_app.handlers.drf_perm import DBManageIAMPermission @@ -118,3 +121,30 @@ def preview(self, request, *args, **kwargs): validated_data = self.params_validate(self.get_serializer_class()) validated_data["operator"] = request.user.username return Response(OpenAreaHandler.openarea_result_preview(**validated_data)) + + @common_swagger_auto_schema( + operation_summary=_("变量表修改"), + request_body=VarAlterSerializer(), + auto_schema=ResponseSwaggerAutoSchema, + tags=[SWAGGER_TAG], + ) + @action(methods=["POST"], detail=False, serializer_class=VarAlterSerializer) + def alter_var(self, request, *args, **kwargs): + bk_biz_id = kwargs["bk_biz_id"] + biz_vars: list = BizSettings.get_setting_value(bk_biz_id=bk_biz_id, key=BizSettingsEnum.OPEN_AREA_VARS) + data = self.params_validate(self.get_serializer_class()) + + try: + # 对变量表进行简单的新增/删除和更新 + if data["op_type"] in ["add", "update"]: + biz_vars.append(data["new_var"]) + if data["op_type"] in ["delete", "update"]: + biz_vars.remove(data["old_var"]) + except ValueError: + # 如果var不存在，remove会触发ValueError，忽略 + pass + + BizSettings.insert_setting_value( + bk_biz_id=bk_biz_id, key=BizSettingsEnum.OPEN_AREA_VARS, value_type="dict", value=biz_vars + ) + return Response() diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_authorize_rules.py b/dbm-ui/backend/ticket/builders/mysql/mysql_authorize_rules.py index fd32812068..98f6484c30 100644 --- a/dbm-ui/backend/ticket/builders/mysql/mysql_authorize_rules.py +++ b/dbm-ui/backend/ticket/builders/mysql/mysql_authorize_rules.py @@ -14,13 +14,22 @@ from rest_framework import serializers from backend import env +from backend.db_meta.enums import ClusterType from backend.db_services.mysql.permission.authorize.serializers import PreCheckAuthorizeRulesSerializer from backend.db_services.mysql.permission.exceptions import AuthorizeDataHasExpiredException from backend.flow.engine.controller.mysql import MySQLController from backend.ticket import builders from backend.ticket.builders.mysql.base import BaseMySQLTicketFlowBuilder -from backend.ticket.constants import FlowRetryType, FlowType, TicketType -from backend.ticket.models import Flow +from backend.ticket.constants import TicketType + + +class MySQLPluginInfoSerializer(serializers.Serializer): + bk_biz_id = serializers.IntegerField(help_text=_("业务ID")) + user = serializers.CharField(help_text=_("授权账号")) + access_dbs = serializers.ListSerializer(child=serializers.CharField(), help_text=_("准入DB")) + source_ips = serializers.ListField(help_text=_("源IP列表"), child=serializers.CharField()) + target_instances = serializers.ListField(help_text=_("目标集群域名"), child=serializers.CharField()) + cluster_type = serializers.ChoiceField(help_text=_("集群类型"), choices=ClusterType.get_choices()) class MySQLAuthorizeDataSerializer(PreCheckAuthorizeRulesSerializer): @@ -28,8 +37,23 @@ class MySQLAuthorizeDataSerializer(PreCheckAuthorizeRulesSerializer): class MySQLAuthorizeRulesSerializer(serializers.Serializer): - authorize_uid = serializers.CharField(help_text=_("授权数据缓存uid")) - authorize_data = MySQLAuthorizeDataSerializer(help_text=_("授权数据信息")) + authorize_uid = serializers.CharField(help_text=_("授权数据缓存uid"), required=False) + authorize_data = MySQLAuthorizeDataSerializer(help_text=_("授权数据信息"), required=False) + authorize_plugin_infos = serializers.ListSerializer( + help_text=_("第三方接口/插件授权信息"), child=MySQLPluginInfoSerializer(), required=False + ) + + def validate(self, attrs): + if not (attrs.get("authorize_plugin_infos") or attrs.get("authorize_uid")): + raise serializers.ValidationError(_("请保证授权数据存在")) + + if attrs.get("authorize_plugin_infos"): + infos = attrs["authorize_plugin_infos"] + for info in infos: + info["account_rules"] = [{"bk_biz_id": info["bk_biz_id"], "dbname": db} for db in info["access_dbs"]] + info["operator"] = self.context["request"].user.username + + return attrs class MySQLExcelAuthorizeDataSerializer(PreCheckAuthorizeRulesSerializer): @@ -48,11 +72,9 @@ class MySQLAuthorizeRulesFlowParamBuilder(builders.FlowParamBuilder): controller = MySQLController.mysql_authorize_rules def format_ticket_data(self): - authorize_uid = self.ticket_data["authorize_uid"] - data = cache.get(authorize_uid) - + data = cache.get(self.ticket_data.get("authorize_uid")) or self.ticket_data.get("authorize_plugin_infos") if not data: - raise AuthorizeDataHasExpiredException(_("授权数据已过期，请重新提交授权表单或excel文件")) + raise AuthorizeDataHasExpiredException(_("授权数据不存在/已过期，请重新提交授权表单或excel文件")) self.ticket_data.update({"rules_set": data}) diff --git a/dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_clear.py b/dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_apply.py similarity index 92% rename from dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_clear.py rename to dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_apply.py index 8102decd07..7484ba3ff5 100644 --- a/dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_clear.py +++ b/dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_apply.py @@ -23,11 +23,13 @@ class TbinlogdumperApplyDetailSerializer(serializers.Serializer): class DumperInfoSerializer(serializers.Serializer): class AddInfoSerializer(serializers.Serializer): area_name = serializers.IntegerField(help_text=_("dumper安装的大区")) + dumper_id = serializers.IntegerField(help_text=_("dumper实例ID(目前同大区名一样)")) module_id = serializers.IntegerField(help_text=_("dumper的模块")) add_type = serializers.ChoiceField(help_text=_("dumper的安装方式"), choices=TBinlogDumperAddType.get_choices()) cluster_id = serializers.IntegerField(help_text=_("集群ID")) add_infos = serializers.ListSerializer(help_text=_("dumper部署信息"), child=AddInfoSerializer()) + dumper_config_id = serializers.IntegerField(help_text=_("数据订阅配置ID")) infos = serializers.ListSerializer(child=DumperInfoSerializer()) diff --git a/dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_switch.py b/dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_switch.py index 888c9a03e0..392f1865db 100644 --- a/dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_switch.py +++ b/dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_switch.py @@ -30,6 +30,7 @@ class SwitchInstanceSerializer(serializers.Serializer): switch_instances = serializers.ListSerializer(help_text=_("dumper切换信息"), child=SwitchInstanceSerializer()) infos = serializers.ListSerializer(child=DumperSwitchInfoSerializer()) + is_safe = serializers.BooleanField(help_text=_("是否安全切换"), required=False, default=True) class TbinlogdumperSwitchNodesFlowParamBuilder(builders.FlowParamBuilder):