Merge pull request #447 from TencentBlueKing/develop

v1.5.14

.gitignore

@@ -1 +1,2 @@
-logs
+logs
+.idea

release.md

@@ -1,3 +1,18 @@
+# V1.5.14
+
+### 新增功能
+* 支持初始化接入APIGateway的API和文档
+
+### 功能优化
+* 自动更新资源实例名称对大策略的防御性忽略
+* 新增接入系统管理类API的bk_nocode白名单
+* 调用第三方接口失败时支持返回异常信息
+* 用户组授权调整为立即执行任务
+
+### 缺陷修复
+* 自动更新资源实例名称兼容接入系统回调异常
+* 解决未资源实例视图为空时导致授权异常
+
 # V1.5.13
 
 ### 新增功能

saas/VERSION

@@ -1 +1 @@
-1.5.13
+1.5.14

saas/backend/api/management/migrations/0004_auto_20211130_1137.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-权限中心(BlueKing-IAM) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+# Generated by Django 2.2.24 on 2021-11-30 03:37
+
+from django.db import migrations
+
+from backend.api.constants import ALLOW_ANY
+
+
+def init_allow_list(apps, schema_editor):
+    """初始化授权API白名单"""
+    ManagementAPIAllowListConfig = apps.get_model("management", "ManagementAPIAllowListConfig")
+
+    # 查询已存在白名单，避免重复
+    all_allow_list = ManagementAPIAllowListConfig.objects.all()
+    allow_set = set([(a.system_id, a.api) for a in all_allow_list])
+
+    # 白名单列表
+    system_id_allow_apis = {
+        "bk_nocode": [ALLOW_ANY],
+    }
+
+    # 组装成Model对象
+    mgmt_api_allow_list_config = []
+    for system_id, apis in system_id_allow_apis.items():
+        for api in apis:
+            # 已存在，则直接忽略
+            if (system_id, api) in allow_set:
+                continue
+            mgmt_api_allow_list_config.append(ManagementAPIAllowListConfig(system_id=system_id, api=api))
+    # 批量创建
+    if len(mgmt_api_allow_list_config) != 0:
+        ManagementAPIAllowListConfig.objects.bulk_create(mgmt_api_allow_list_config)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', '0003_systemallowauthsystem'),
+    ]
+
+    operations = [
+        migrations.RunPython(init_allow_list)
+    ]

saas/backend/apps/template/views.py

@@ -559,7 +559,7 @@ def create(self, request, *args, **kwargs):
 
         # 使用长时任务实现用户组授权更新
         task = TaskDetail.create(TaskType.TEMPLATE_UPDATE.value, [template.id])
-        TaskFactory().delay(task.id)
+        TaskFactory()(task.id)
 
         audit_context_setter(template=template)
 

saas/backend/biz/group.py

@@ -519,7 +519,7 @@ def grant(self, role: Role, group: Group, templates: List[GroupTemplateGrantBean
             task = TaskDetail.create(TaskType.GROUP_AUTHORIZATION.value, [subject.dict(), uuid])
 
         # 执行授权流程
-        TaskFactory().delay(task.id)
+        TaskFactory()(task.id)
 
     def get_group_role_dict_by_ids(self, group_ids: List[int]) -> GroupRoleDict:
         """

saas/backend/biz/instance_selection.py

@@ -69,6 +69,9 @@ def _list_system_id(self):
         """
         获取所有视图节点的system_id
         """
+        if not self.selections:
+            return []
+
         return list(set.union(*[selection.get_chain_system_set() for selection in self.selections]))
 
     def fill_chain_node_name(self):
@@ -96,6 +99,9 @@ def list_by_action_resource_type(
         selections = self.svc.list_by_action_resource_type(
             system_id, action_id, resource_type_system_id, resource_type_id
         )
+        if not selections:
+            return []
+
         biz_selections = parse_obj_as(List[InstanceSelectionBean], selections)
         return self._fill_chain_node_name(biz_selections)
 

saas/backend/biz/policy.py

@@ -9,6 +9,7 @@
 specific language governing permissions and limitations under the License.
 """
 import functools
+import logging
 import time
 from copy import deepcopy
 from itertools import chain, groupby
@@ -23,7 +24,7 @@
 from backend.common.error_codes import error_codes
 from backend.common.time import PERMANENT_SECONDS, expired_at_display, generate_default_expired_at
 from backend.service.action import ActionService
-from backend.service.constants import ANY_ID
+from backend.service.constants import ANY_ID, FETCH_MAX_LIMIT
 from backend.service.models import (
     Action,
     BackendThinPolicy,
@@ -49,6 +50,8 @@
 
 from .resource import ResourceBiz, ResourceNodeBean
 
+logger = logging.getLogger(__name__)
+
 
 class PolicyEmptyException(Exception):
     pass
@@ -889,12 +892,17 @@ def check_instance_selection(self, ignore_path=False):
                     continue
                 rrt.check_selection(resource_type.instance_selections, ignore_path)
 
-    def _list_path_node(self) -> List[PathNodeBean]:
+    def _list_path_node(self, is_ignore_big_policy=False) -> List[PathNodeBean]:
         """
         查询策略包含的资源范围 - 所有路径上的节点，包括叶子节点
+        is_ignore_big_policy: 是否忽略大的策略，大策略是指策略里的实例数量大于1000，
+          主要是用于自动更新策略里的资源实例名称时避免大数量的请求接入系统（1000是权限中心的回调接口协议里规定的）
         """
         nodes = []
         for p in self.policies:
+            # 这里是定制逻辑：基于fetch_instance_info限制，避免出现大策略
+            if is_ignore_big_policy and p.count_all_type_instance() > FETCH_MAX_LIMIT:
+                continue
             nodes.extend(p.list_path_node())
         return nodes
 
@@ -937,8 +945,8 @@ def check_instance_count_limit(self):
 
     def get_renamed_resources(self) -> Dict[PathNodeBean, str]:
         """查询已经被重命名的资源实例"""
-        # 获取策略里的资源的所有节点
-        path_nodes = self._list_path_node()
+        # 获取策略里的资源的所有节点，防御性措施：忽略大策略，避免给接入系统请求压力
+        path_nodes = self._list_path_node(is_ignore_big_policy=True)
         # 查询资源实例的实际名称
         resource_name_dict = self.resource_biz.fetch_resource_name(parse_obj_as(List[ResourceNodeBean], path_nodes))
 
@@ -965,8 +973,13 @@ def auto_update_resource_name(self) -> List[PolicyBean]:
         策略里存储的资源名称可能已经变了，需要进行更新
         Note: 该函数仅用于需要对外展示策略数据时调用，不会自动更新DB里数据
         """
-        # 获取策略里被重命名的资源实例
-        renamed_resources = self.get_renamed_resources()
+        # 由于自动更新并非核心功能，若接入系统查询有问题，也需要正常显示
+        try:
+            # 获取策略里被重命名的资源实例
+            renamed_resources = self.get_renamed_resources()
+        except Exception as error:
+            logger.exception(f"auto_update: get_renamed_resources error={error}")
+            return []
 
         # 没有任何被重命名的资源实例，则无需更新策略
         if len(renamed_resources) == 0:

saas/backend/component/engine.py

@@ -48,7 +48,7 @@ def _call_engine_api(http_func, url_path, data, timeout=30):
     if not ok:
         message = "engine api failed, method: %s, info: %s" % (http_func.__name__, kwargs)
         logger.error(message)
-        raise error_codes.ENGINE_REQUEST_ERROR.format("request engine api error")
+        raise error_codes.ENGINE_REQUEST_ERROR.format(f'request engine api error: {data["error"]}')
 
     code = data["code"]
     message = data["message"]

saas/backend/component/esb.py

@@ -45,7 +45,7 @@ def _call_esb_api(http_func, url_path, data, timeout=30):
     if not ok:
         message = "esb api failed, method: %s, info: %s" % (http_func.__name__, kwargs)
         logger.error(message)
-        raise error_codes.REMOTE_REQUEST_ERROR.format("request esb api error")
+        raise error_codes.REMOTE_REQUEST_ERROR.format(f'request esb api error: {data["error"]}')
 
     code = data["code"]
     message = data["message"]

saas/backend/component/http.py

@@ -67,11 +67,11 @@ def _http_request(method, url, headers=None, data=None, timeout=None, verify=Fal
                 url=url, headers=headers, json=data, timeout=timeout, verify=verify, cert=cert, cookies=cookies
             )
         else:
-            return False, None
-    except requests.exceptions.RequestException:
+            return False, {"error": "method not supported"}
+    except requests.exceptions.RequestException as e:
         logger.exception("http request error! method: %s, url: %s, data: %s", method, url, data)
         trace_func(exc=traceback.format_exc())
-        return False, None
+        return False, {"error": str(e)}
     else:
         # record for /metrics
         latency = int((time.time() - st) * 1000)
@@ -90,7 +90,7 @@ def _http_request(method, url, headers=None, data=None, timeout=None, verify=Fal
             logger.error(error_msg, method, url, str(data), resp.status_code, content)
 
             trace_func(status_code=resp.status_code, content=content)
-            return False, None
+            return False, {"error": f"status_code is {resp.status_code}, not 200"}
 
         return True, resp.json()
 

saas/backend/component/iam.py

@@ -52,7 +52,7 @@ def _call_iam_api(http_func, url_path, data, timeout=30):
     if not ok:
         message = "iam api failed, method: %s, info: %s" % (http_func.__name__, kwargs)
         logger.error(message)
-        raise error_codes.REMOTE_REQUEST_ERROR.format("request iam api error")
+        raise error_codes.REMOTE_REQUEST_ERROR.format(f'request iam api error: {data["error"]}')
 
     code = data["code"]
     message = data["message"]

saas/backend/long_task/tasks.py

@@ -232,4 +232,4 @@ def retry_long_task():
         status__in=[TaskStatus.PENDING.value, TaskStatus.RUNNING.value], created_time__lt=day_before
     )
     for t in qs:
-        TaskFactory().delay(t.id)
+        TaskFactory()(t.id)

saas/backend/service/constants.py

@@ -39,6 +39,13 @@ class SubjectRelationType(ChoicesEnum, LowerStrEnum):
     _choices_labels = skip(((GROUP, "用户组"), (DEPARTMENT, "部门")))
 
 
+# ---------------------------------------------------------------------------------------------- #
+# Resource Provider
+# ---------------------------------------------------------------------------------------------- #
+# fetch_instance_info 接口的批量限制
+FETCH_MAX_LIMIT = 1000
+
+
 # ---------------------------------------------------------------------------------------------- #
 # Group Constants
 # ---------------------------------------------------------------------------------------------- #

saas/backend/service/resource.py

@@ -19,6 +19,7 @@
 from backend.util.cache import redis_region, region
 from backend.util.url import url_join
 
+from .constants import FETCH_MAX_LIMIT
 from .models import (
     ResourceAttribute,
     ResourceAttributeValue,
@@ -210,10 +211,9 @@ def fetch_instance_info(
     ) -> List[ResourceInstanceInfo]:
         """批量查询资源实例属性，包括display_name等"""
         # fetch_instance_info 接口的批量限制
-        fetch_limit = 1000
         # 分页查询资源实例属性
         results = []
-        page_ids_list = chunked(ids, fetch_limit)
+        page_ids_list = chunked(ids, FETCH_MAX_LIMIT)
         for page_ids in page_ids_list:
             filter_condition = {"ids": page_ids, "attrs": attributes} if attributes else {"ids": page_ids}
             page_results = self.client.fetch_instance_info(filter_condition)

saas/config/default.py

@@ -353,3 +353,12 @@
 
 # 是否是smart部署方式
 IS_SMART_DEPLOY = os.environ.get("BKAPP_IS_SMART_DEPLOY", "True").lower() == "true"
+
+# apigateway 相关配置
+# NOTE: it sdk will read settings.BK_APP_CODE and settings.BK_APP_SECRET, so you should set it
+BK_APIGW_NAME = "bk-iam"
+BK_API_URL_TMPL = ""
+INSTALLED_APPS += ("apigw_manager.apigw",)
+BK_IAM_BACKEND_SVC = os.environ.get("BK_IAM_BACKEND_SVC", "bkiam-web")
+BK_IAM_ENGINE_SVC = os.environ.get("BK_IAM_ENGINE_SVC", "bkiam-search-engine-web")
+BK_APIGW_RESOURCE_DOCS_BASE_DIR = os.path.join(BASE_DIR, "resources/apigateway/docs/")

saas/config/dev.py

@@ -62,8 +62,8 @@
 # 蓝鲸智云开发者中心的域名，形如：http://paas.bking.com
 BK_PAAS_HOST = ""
 
-APP_ID = os.environ.get("APP_ID", APP_ID)
-APP_TOKEN = os.environ.get("APP_TOKEN", APP_TOKEN)
+APP_ID = BK_APP_CODE = os.environ.get("APP_ID", APP_ID)
+APP_TOKEN = BK_APP_SECRET = os.environ.get("APP_TOKEN", APP_TOKEN)
 BK_PAAS_HOST = os.environ.get("BK_PAAS_HOST", BK_PAAS_HOST)
 BK_PAAS_INNER_HOST = os.environ.get("BK_PAAS_INNER_HOST", BK_PAAS_HOST)
 BK_IAM_HOST = ""

saas/config/prod.py

@@ -51,8 +51,8 @@ def get_app_service_url(app_code: str) -> str:
         return {item["key"]["bk_app_code"]: item["value"]["prod"] for item in decoded_value}[app_code]
 
     # 兼容component的APP_ID,APP_TOKEN
-    APP_CODE = APP_ID = os.environ.get("BKPAAS_APP_ID", APP_CODE)
-    SECRET_KEY = APP_TOKEN = os.environ.get("BKPAAS_APP_SECRET", SECRET_KEY)
+    APP_CODE = APP_ID = BK_APP_CODE = os.environ.get("BKPAAS_APP_ID", APP_CODE)
+    SECRET_KEY = APP_TOKEN = BK_APP_SECRET = os.environ.get("BKPAAS_APP_SECRET", SECRET_KEY)
     BK_PAAS_INNER_HOST = os.environ.get("BK_PAAS2_URL", BK_PAAS_INNER_HOST)
     BK_COMPONENT_API_URL = os.environ.get("BK_COMPONENT_API_URL")
     BK_COMPONENT_INNER_API_URL = BK_COMPONENT_API_URL