fix(backend): 补充dumper迁移透传 & 直接授权单据 #1827

dbm-ui/backend/configuration/models/system.py

@@ -9,7 +9,7 @@
 specific language governing permissions and limitations under the License.
 """
 import logging
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 from django.conf import settings
 from django.db import connection, models
@@ -150,7 +150,7 @@ class Meta:
         ordering = ("id",)
 
     @classmethod
-    def get_setting_value(cls, bk_biz_id: int, key: str, default: Optional[Any] = None) -> Union[str, Dict]:
+    def get_setting_value(cls, bk_biz_id: int, key: str, default: Optional[Any] = None) -> Union[str, Dict, List]:
         return super().get_setting_value(key={"key": key, "bk_biz_id": bk_biz_id}, default=default)
 
     @classmethod

dbm-ui/backend/db_proxy/urls.py

@@ -16,6 +16,7 @@
 from backend.db_proxy.views.db_remote_service.views import DRSApiProxyPassViewSet
 from backend.db_proxy.views.dbconfig.views import DBConfigProxyPassViewSet
 from backend.db_proxy.views.dns.views import DnsProxyPassViewSet
+from backend.db_proxy.views.dumper.views import DumperProxyPassViewSet
 from backend.db_proxy.views.hadb.views import HADBProxyPassViewSet
 from backend.db_proxy.views.jobapi.views import JobApiProxyPassViewSet
 from backend.db_proxy.views.nameservice.views import NameServiceProxyPassViewSet
@@ -33,5 +34,6 @@
 routers.register(r"", BKRepoProxyPassViewSet, basename="bkrepo")
 routers.register(r"", DtsApiProxyPassViewSet, basename="redis_dts")
 routers.register(r"", JobApiProxyPassViewSet, basename="jobapi")
+routers.register(r"", DumperProxyPassViewSet, basename="dumper")
 
 urlpatterns = routers.urls

dbm-ui/backend/db_proxy/views/dumper/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""

dbm-ui/backend/db_proxy/views/dumper/serializers.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from django.utils.translation import gettext_lazy as _
+from rest_framework import serializers
+
+from backend.db_proxy.views.serialiers import BaseProxyPassSerialier
+
+
+class DumperMigrateProxyPassSerializer(BaseProxyPassSerialier):
+    class DumperSwitchInfoSerializer(serializers.Serializer):
+        class SwitchInstanceSerializer(serializers.Serializer):
+            host = serializers.CharField(help_text=_("主机IP"))
+            port = serializers.IntegerField(help_text=_("主机端口"))
+            repl_binlog_file = serializers.CharField(help_text=_("待切换后需要同步的binlog文件"))
+            repl_binlog_pos = serializers.IntegerField(help_text=_("待切换后需要同步的binlog文件的为位点"))
+
+        cluster_domain = serializers.IntegerField(help_text=_("集群域名"))
+        switch_instances = serializers.ListSerializer(help_text=_("dumper切换信息"), child=SwitchInstanceSerializer())
+
+    infos = serializers.ListSerializer(child=DumperSwitchInfoSerializer())
+    bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
+    is_safe = serializers.BooleanField(help_text=_("是否安全切换"), required=False, default=True)

dbm-ui/backend/db_proxy/views/dumper/views.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from django.utils.translation import ugettext_lazy as _
+from rest_framework.decorators import action
+from rest_framework.response import Response
+
+from backend.bk_web.swagger import common_swagger_auto_schema
+from backend.components.hadb.client import HADBApi
+from backend.db_meta.models import Cluster
+from backend.db_proxy.constants import SWAGGER_TAG
+from backend.ticket.constants import TicketType
+from backend.ticket.models import Ticket
+
+from ..views import BaseProxyPassViewSet
+from .serializers import DumperMigrateProxyPassSerializer
+
+
+class DumperProxyPassViewSet(BaseProxyPassViewSet):
+    """
+    Dumper服务接口的透传视图
+    """
+
+    @common_swagger_auto_schema(
+        operation_summary=_("[dumper]迁移"),
+        request_body=DumperMigrateProxyPassSerializer(),
+        tags=[SWAGGER_TAG],
+    )
+    @action(
+        methods=["POST"], detail=False, serializer_class=DumperMigrateProxyPassSerializer, url_path="dumper/switch"
+    )
+    def dumper_switch(self, request):
+        data = self.params_validate(self.get_serializer_class())
+        # 获取集群ID
+        cluster_domains = [info["cluster_domain"] for info in data["infos"]]
+        domain__cluster = {
+            cluster.immute_domain: cluster for cluster in Cluster.objects.filter(immute_domain__in=cluster_domains)
+        }
+        for info in data["infos"]:
+            info["cluster_id"] = domain__cluster[info["cluster_domain"]]
+        # 创建dumper迁移单据
+        Ticket.create_ticket(
+            ticket_type=TicketType.TBINLOGDUMPER_SWITCH_NODES,
+            creator=request.user.username,
+            bk_biz_id=data.pop("bk_biz_id"),
+            remark=_("透传接口dumper迁移创建的单据"),
+            details=data,
+        )

dbm-ui/backend/db_services/mysql/dumper/mock_data.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+DUMPER_INSTANCE_LIST_DATA = [
+    {
+        "id": 1,
+        "creator": "xxx",
+        "create_at": "2022-11-11 20:00:00",
+        "updater": "xxx",
+        "update_at": "2022-11-11 20:00:00",
+        "bk_biz_id": 3,
+        "cluster_id": 64,
+        "bk_cloud_id": 0,
+        "ip": "127.0.0.1",
+        "proc_type": "tbinlogdumper",
+        "version": "1.0",
+        "listen_port": 10000,
+        "need_transfer": True,
+        "source_cluster": {
+            "id": 64,
+            "name": "fortest",
+            "cluster_type": "tendbha",
+            "immute_domain": "tendbha57db.xxx.dba.db",
+            "major_version": "MySQL-5.7",
+            "bk_cloud_id": 0,
+            "region": " ",
+        },
+        "dumper_config": {
+            "id": 1,
+            "creator": "admin",
+            "updater": "admin",
+            "bk_biz_id": 3,
+            "name": "dumper-1",
+            "receiver_type": "redis",
+            "receiver": "redis.com:123",
+            "subscribe": {"db_name": "db1", "table_names": ["table1"]},
+        },
+        "dumper_id": 1,
+        "area_name": 1,
+    }
+]

dbm-ui/backend/db_services/mysql/dumper/serializers.py

@@ -15,6 +15,8 @@
 from backend.db_meta.models.dumper import DumperSubscribeConfig
 from backend.db_meta.models.extra_process import ExtraProcessInstance
 
+from . import mock_data
+
 
 class DumperSubscribeConfigSerializer(serializers.ModelSerializer):
     class SubscribeInfoSerializer(serializers.Serializer):
@@ -32,6 +34,7 @@ class DumperInstanceConfigSerializer(serializers.ModelSerializer):
     class Meta:
         model = ExtraProcessInstance
         fields = "__all__"
+        swagger_schema_fields = {"example": mock_data.DUMPER_INSTANCE_LIST_DATA}
 
 
 class VerifyDuplicateNamsSerializer(serializers.Serializer):

dbm-ui/backend/db_services/mysql/dumper/views/dumper_instance.py

@@ -57,8 +57,10 @@ def list(self, request, *args, **kwargs):
             source_cluster = id__cluster[data["cluster_id"]]
             master = source_cluster.storageinstance_set.get(instance_inner_role=InstanceInnerRole.MASTER.value)
             data["need_transfer"] = data["ip"] != master.machine.ip
-            # 补充集群信息
+            # 补充集群信息和集群的master计信系
             data["source_cluster"] = source_cluster.simple_desc
+            data["source_cluster"]["master_ip"] = master.machine.ip
+            data["source_cluster"]["master_port"] = master.port
             # 补充订阅配置信息
             dumper_config_id = extra_config["dumper_config_id"]
             data["dumper_config"] = model_to_dict(id__dumper_config[dumper_config_id])

dbm-ui/backend/db_services/mysql/open_area/serializers.py

@@ -76,3 +76,14 @@ class ConfigDataSerializer(serializers.Serializer):
 class TendbOpenAreaResultPreviewResponseSerializer(serializers.Serializer):
     class Meta:
         swagger_schema_fields = {"example": mock_data.OPENAREA_PREVIEW_DATA}
+
+
+class VarAlterSerializer(serializers.Serializer):
+    op_type = serializers.CharField(help_text=_("操作类型"))
+    old_var = serializers.JSONField(help_text=_("旧变量(delete/update)"), required=False)
+    new_var = serializers.JSONField(help_text=_("新变量(add/update)"), required=False)
+
+    class Meta:
+        swagger_schema_fields = {
+            "example": {"op_type": "add(delete, update)", "var": {"name": "APP", "desc": "xxx", "builtin": False}}
+        }

dbm-ui/backend/db_services/mysql/open_area/views.py

@@ -21,6 +21,8 @@
     ResponseSwaggerAutoSchema,
     common_swagger_auto_schema,
 )
+from backend.configuration.constants import BizSettingsEnum
+from backend.configuration.models import BizSettings
 from backend.db_meta.models import Cluster
 from backend.db_services.mysql.open_area.filters import TendbOpenAreaConfigListFilter
 from backend.db_services.mysql.open_area.handlers import OpenAreaHandler
@@ -29,6 +31,7 @@
     TendbOpenAreaConfigSerializer,
     TendbOpenAreaResultPreviewResponseSerializer,
     TendbOpenAreaResultPreviewSerializer,
+    VarAlterSerializer,
 )
 from backend.iam_app.handlers.drf_perm import DBManageIAMPermission
 
@@ -118,3 +121,30 @@ def preview(self, request, *args, **kwargs):
         validated_data = self.params_validate(self.get_serializer_class())
         validated_data["operator"] = request.user.username
         return Response(OpenAreaHandler.openarea_result_preview(**validated_data))
+
+    @common_swagger_auto_schema(
+        operation_summary=_("变量表修改"),
+        request_body=VarAlterSerializer(),
+        auto_schema=ResponseSwaggerAutoSchema,
+        tags=[SWAGGER_TAG],
+    )
+    @action(methods=["POST"], detail=False, serializer_class=VarAlterSerializer)
+    def alter_var(self, request, *args, **kwargs):
+        bk_biz_id = kwargs["bk_biz_id"]
+        biz_vars: list = BizSettings.get_setting_value(bk_biz_id=bk_biz_id, key=BizSettingsEnum.OPEN_AREA_VARS)
+        data = self.params_validate(self.get_serializer_class())
+
+        try:
+            # 对变量表进行简单的新增/删除和更新
+            if data["op_type"] in ["add", "update"]:
+                biz_vars.append(data["new_var"])
+            if data["op_type"] in ["delete", "update"]:
+                biz_vars.remove(data["old_var"])
+        except ValueError:
+            # 如果var不存在，remove会触发ValueError，忽略
+            pass
+
+        BizSettings.insert_setting_value(
+            bk_biz_id=bk_biz_id, key=BizSettingsEnum.OPEN_AREA_VARS, value_type="dict", value=biz_vars
+        )
+        return Response()

dbm-ui/backend/ticket/builders/mysql/mysql_authorize_rules.py

@@ -14,22 +14,46 @@
 from rest_framework import serializers
 
 from backend import env
+from backend.db_meta.enums import ClusterType
 from backend.db_services.mysql.permission.authorize.serializers import PreCheckAuthorizeRulesSerializer
 from backend.db_services.mysql.permission.exceptions import AuthorizeDataHasExpiredException
 from backend.flow.engine.controller.mysql import MySQLController
 from backend.ticket import builders
 from backend.ticket.builders.mysql.base import BaseMySQLTicketFlowBuilder
-from backend.ticket.constants import FlowRetryType, FlowType, TicketType
-from backend.ticket.models import Flow
+from backend.ticket.constants import TicketType
+
+
+class MySQLPluginInfoSerializer(serializers.Serializer):
+    bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
+    user = serializers.CharField(help_text=_("授权账号"))
+    access_dbs = serializers.ListSerializer(child=serializers.CharField(), help_text=_("准入DB"))
+    source_ips = serializers.ListField(help_text=_("源IP列表"), child=serializers.CharField())
+    target_instances = serializers.ListField(help_text=_("目标集群域名"), child=serializers.CharField())
+    cluster_type = serializers.ChoiceField(help_text=_("集群类型"), choices=ClusterType.get_choices())
 
 
 class MySQLAuthorizeDataSerializer(PreCheckAuthorizeRulesSerializer):
     pass
 
 
 class MySQLAuthorizeRulesSerializer(serializers.Serializer):
-    authorize_uid = serializers.CharField(help_text=_("授权数据缓存uid"))
-    authorize_data = MySQLAuthorizeDataSerializer(help_text=_("授权数据信息"))
+    authorize_uid = serializers.CharField(help_text=_("授权数据缓存uid"), required=False)
+    authorize_data = MySQLAuthorizeDataSerializer(help_text=_("授权数据信息"), required=False)
+    authorize_plugin_infos = serializers.ListSerializer(
+        help_text=_("第三方接口/插件授权信息"), child=MySQLPluginInfoSerializer(), required=False
+    )
+
+    def validate(self, attrs):
+        if not (attrs.get("authorize_plugin_infos") or attrs.get("authorize_uid")):
+            raise serializers.ValidationError(_("请保证授权数据存在"))
+
+        if attrs.get("authorize_plugin_infos"):
+            infos = attrs["authorize_plugin_infos"]
+            for info in infos:
+                info["account_rules"] = [{"bk_biz_id": info["bk_biz_id"], "dbname": db} for db in info["access_dbs"]]
+                info["operator"] = self.context["request"].user.username
+
+        return attrs
 
 
 class MySQLExcelAuthorizeDataSerializer(PreCheckAuthorizeRulesSerializer):
@@ -48,11 +72,9 @@ class MySQLAuthorizeRulesFlowParamBuilder(builders.FlowParamBuilder):
     controller = MySQLController.mysql_authorize_rules
 
     def format_ticket_data(self):
-        authorize_uid = self.ticket_data["authorize_uid"]
-        data = cache.get(authorize_uid)
-
+        data = cache.get(self.ticket_data.get("authorize_uid")) or self.ticket_data.get("authorize_plugin_infos")
         if not data:
-            raise AuthorizeDataHasExpiredException(_("授权数据已过期，请重新提交授权表单或excel文件"))
+            raise AuthorizeDataHasExpiredException(_("授权数据不存在/已过期，请重新提交授权表单或excel文件"))
 
         self.ticket_data.update({"rules_set": data})
 

dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_clear.py → dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_apply.py

@@ -23,11 +23,13 @@ class TbinlogdumperApplyDetailSerializer(serializers.Serializer):
     class DumperInfoSerializer(serializers.Serializer):
         class AddInfoSerializer(serializers.Serializer):
             area_name = serializers.IntegerField(help_text=_("dumper安装的大区"))
+            dumper_id = serializers.IntegerField(help_text=_("dumper实例ID(目前同大区名一样)"))
             module_id = serializers.IntegerField(help_text=_("dumper的模块"))
             add_type = serializers.ChoiceField(help_text=_("dumper的安装方式"), choices=TBinlogDumperAddType.get_choices())
 
         cluster_id = serializers.IntegerField(help_text=_("集群ID"))
         add_infos = serializers.ListSerializer(help_text=_("dumper部署信息"), child=AddInfoSerializer())
+        dumper_config_id = serializers.IntegerField(help_text=_("数据订阅配置ID"))
 
     infos = serializers.ListSerializer(child=DumperInfoSerializer())
 

dbm-ui/backend/ticket/builders/tbinlogdumper/dumper_switch.py

@@ -30,6 +30,7 @@ class SwitchInstanceSerializer(serializers.Serializer):
         switch_instances = serializers.ListSerializer(help_text=_("dumper切换信息"), child=SwitchInstanceSerializer())
 
     infos = serializers.ListSerializer(child=DumperSwitchInfoSerializer())
+    is_safe = serializers.BooleanField(help_text=_("是否安全切换"), required=False, default=True)
 
 
 class TbinlogdumperSwitchNodesFlowParamBuilder(builders.FlowParamBuilder):