From d34619e1de5f4fc16a64874e869e756c25b64061 Mon Sep 17 00:00:00 2001 From: kangwork Date: Thu, 22 Aug 2024 18:47:21 +0900 Subject: [PATCH 1/3] fix: change redefintion to extend to include all recommendations Signed-off-by: kangwork --- .../connector/recommender/cloud_asset.py | 1 - .../connector/recommender/recommendation.py | 5 +- .../manager/recommender/recommendation.py | 534 ------------------ .../{recommendation.yaml => recommender.yaml} | 1 + 4 files changed, 3 insertions(+), 538 deletions(-) delete mode 100644 src/cloudforet/plugin/manager/recommender/recommendation.py rename src/cloudforet/plugin/metadata/recommender/{recommendation.yaml => recommender.yaml} (98%) diff --git a/src/cloudforet/plugin/connector/recommender/cloud_asset.py b/src/cloudforet/plugin/connector/recommender/cloud_asset.py index 8c497e0..9a7a2b7 100644 --- a/src/cloudforet/plugin/connector/recommender/cloud_asset.py +++ b/src/cloudforet/plugin/connector/recommender/cloud_asset.py @@ -1,5 +1,4 @@ import logging - from cloudforet.plugin.connector.base import GoogleCloudConnector __all__ = ["CloudAssetConnector"] diff --git a/src/cloudforet/plugin/connector/recommender/recommendation.py b/src/cloudforet/plugin/connector/recommender/recommendation.py index a34dae9..a51beb4 100644 --- a/src/cloudforet/plugin/connector/recommender/recommendation.py +++ b/src/cloudforet/plugin/connector/recommender/recommendation.py @@ -1,7 +1,6 @@ import logging from cloudforet.plugin.connector.base import GoogleCloudConnector - __all__ = ["RecommendationConnector"] _LOGGER = logging.getLogger(__name__) @@ -26,9 +25,9 @@ def list_recommendations(self, recommendation_parent, **query): while request is not None: response = request.execute() - recommendations = [ + recommendations.extend([ recommendation for recommendation in response.get("recommendations", []) - ] + ]) request = ( self.client.projects() .locations() diff --git a/src/cloudforet/plugin/manager/recommender/recommendation.py b/src/cloudforet/plugin/manager/recommender/recommendation.py deleted file mode 100644 index 4c403e3..0000000 --- a/src/cloudforet/plugin/manager/recommender/recommendation.py +++ /dev/null @@ -1,534 +0,0 @@ -import logging -import requests -import json - -from bs4 import BeautifulSoup -from spaceone.inventory.plugin.collector.lib import * - -from cloudforet.plugin.config.global_conf import ASSET_URL, RECOMMENDATION_MAP -from cloudforet.plugin.connector.recommender import * -from cloudforet.plugin.manager import ResourceManager - -_LOGGER = logging.getLogger(__name__) - -_RECOMMENDATION_TYPE_DOCS_URL = "https://cloud.google.com/recommender/docs/recommenders" - -_UNAVAILABLE_RECOMMENDER_IDS = [ - "google.cloudbilling.commitment.SpendBasedCommitmentRecommender", - "google.accounts.security.SecurityKeyRecommender", - "google.cloudfunctions.PerformanceRecommender", -] - - -class RecommendationManager(ResourceManager): - service = "Recommender" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.cloud_service_group = "Recommender" - self.cloud_service_type = "Recommendation" - self.metadata_path = "plugin/metadata/recommender/recommendation.yaml" - self.recommender_map = RECOMMENDATION_MAP - self.project_id = "" - - def create_cloud_service_type(self): - return make_cloud_service_type( - name=self.cloud_service_type, - group=self.cloud_service_group, - provider=self.provider, - metadata_path=self.metadata_path, - is_primary=True, - is_major=True, - service_code="Recommender", - tags={"spaceone:icon": f"{ASSET_URL}/user_preferences.svg"}, - labels=["Analytics"], - ) - - def create_cloud_service(self, options, secret_data, schema): - error_responses = [] - cloud_services = [] - self.project_id = secret_data["project_id"] - - # Needs periodic updating - # self.recommender_map = self._create_recommendation_id_map_by_crawling() - - cloud_asset_conn = CloudAssetConnector( - options=options, secret_data=secret_data, schema=schema - ) - assets = [asset for asset in cloud_asset_conn.list_assets_in_project()] - self._create_location_field_to_recommendation_map(assets) - - recommendation_parents = self._create_parents_for_request_params() - - recommendation_conn = RecommendationConnector( - options=options, secret_data=secret_data, schema=schema - ) - - preprocessed_recommendations = [] - for recommendation_parent in recommendation_parents: - recommendations = recommendation_conn.list_recommendations( - recommendation_parent - ) - for recommendation in recommendations: - recommendation_name = recommendation["name"] - region, recommender_id = self._get_region_and_recommender_id( - recommendation_name - ) - - display = { - "recommenderId": recommender_id, - "recommenderIdName": self.recommender_map[recommender_id]["name"], - "recommenderIdDescription": self.recommender_map[recommender_id][ - "shortDescription" - ], - "priorityDisplay": self.convert_readable_priority( - recommendation["priority"] - ), - "overview": json.dumps(recommendation["content"]["overview"]), - "operations": json.dumps( - recommendation["content"].get("operationGroups", "") - ), - "operationActions": self._get_actions(recommendation["content"]), - "location": self._get_location(recommendation_parent), - } - - if resource := recommendation["content"].get("overview"): - display["resource"] = resource - - if cost_info := recommendation["primaryImpact"].get("costProjection"): - cost = cost_info.get("cost", {}) - ( - display["cost"], - display["costDescription"], - ) = self._change_cost_to_description(cost) - - recommendation.update({"display": display}) - preprocessed_recommendations.append(recommendation) - - recommenders = self._create_recommenders(preprocessed_recommendations) - collected_recommender_ids = self._list_collected_recommender_ids(recommenders) - for idx, recommender_id in enumerate(collected_recommender_ids): - try: - recommender = self.recommender_map[recommender_id] - total_cost = 0 - resource_count = 0 - total_priority_level = { - "Lowest": 0, - "Second Lowest": 0, - "Highest": 0, - "Second Highest": 0, - } - for recommendation in recommender["recommendations"]: - if ( - recommender["category"] == "COST" - and recommendation.get("cost") - and isinstance(recommendation.get("cost"), float) - ): - total_cost += recommendation.get("cost", 0) - - if recommendation.get("affectedResource"): - resource_count += 1 - - total_priority_level[recommendation.get("priorityLevel")] += 1 - - if total_cost: - recommender["costSavings"] = f"Total ${round(total_cost, 2)}/month" - if resource_count: - recommender["resourceCount"] = resource_count - - ( - recommender["state"], - recommender["primaryPriorityLevel"], - ) = self._get_state_and_priority(total_priority_level) - - self.set_region_code("global") - cloud_services.append( - make_cloud_service( - name=recommender["name"], - cloud_service_type=self.cloud_service_type, - cloud_service_group=self.cloud_service_group, - provider=self.provider, - account=self.project_id, - data=recommender, - region_code="global", - instance_type="", - instance_size=0, - reference={ - "resource_id": recommender["id"], - "external_link": f"https://console.cloud.google.com/cloudpubsub/schema/detail/{recommender['id']}?project={self.project_id}", - }, - ) - ) - except Exception as e: - error_responses.append( - make_error_response( - error=e, - provider=self.provider, - cloud_service_group=self.cloud_service_group, - cloud_service_type=self.cloud_service_type, - ) - ) - - return cloud_services, error_responses - - @staticmethod - def _create_recommendation_id_map_by_crawling(): - res = requests.get(_RECOMMENDATION_TYPE_DOCS_URL) - soup = BeautifulSoup(res.content, "html.parser") - table = soup.find("table") - rows = table.find_all("tr") - - recommendation_id_map = {} - category = "" - for row in rows: - cols = row.find_all("td") - cols = [ele.text.strip() for ele in cols] - if cols: - try: - category, name, recommender_id, short_description, etc = cols - except ValueError: - name, recommender_id, short_description, etc = cols - - recommender_ids = [] - if "Cloud SQL performance recommender" in name: - name = "Cloud SQL performance recommender" - short_description = "Improve Cloud SQL instance performance" - recommender_ids = [ - "google.cloudsql.instance.PerformanceRecommender" - ] - else: - if recommender_id.count("google.") > 1: - re_ids = recommender_id.split("google.")[1:] - for re_id in re_ids: - re_id = "google." + re_id - if re_id not in _UNAVAILABLE_RECOMMENDER_IDS: - recommender_ids.append(re_id) - else: - if recommender_id not in _UNAVAILABLE_RECOMMENDER_IDS: - recommender_ids = [recommender_id] - else: - continue - - for recommender_id in recommender_ids: - recommendation_id_map[recommender_id] = { - "category": category, - "name": name, - "shortDescription": short_description, - } - - return recommendation_id_map - - def _create_parents_for_request_params(self): - recommendation_parents = [] - for recommender_id, recommender_info in self.recommender_map.items(): - for region_or_zone in recommender_info["locations"]: - recommendation_parents.append( - f"projects/{self.project_id}/locations/{region_or_zone}/recommenders/{recommender_id}" - ) - return recommendation_parents - - def _create_location_field_to_recommendation_map(self, assets): - parents_and_locations_map = ( - self._create_parents_and_location_map_by_cloud_asset_api(assets) - ) - - self._add_group_and_service_to_recommender_map() - self._add_locations_to_recommender_map(parents_and_locations_map) - - @staticmethod - def _create_parents_and_location_map_by_cloud_asset_api(assets): - parents_and_locations_map = {} - for asset in assets: - asset_type = asset["assetType"] - locations = asset["resource"].get("location", "global") - - service, cloud_service_type = asset_type.split("/") - cloud_service_group, postfix = service.split(".", 1) - cloud_service_type = cloud_service_type.lower() - - if cloud_service_group not in parents_and_locations_map: - parents_and_locations_map[cloud_service_group] = {} - else: - if ( - cloud_service_type - not in parents_and_locations_map[cloud_service_group] - ): - parents_and_locations_map[cloud_service_group][ - cloud_service_type - ] = [locations] - else: - if ( - locations - not in parents_and_locations_map[cloud_service_group][ - cloud_service_type - ] - ): - parents_and_locations_map[cloud_service_group][ - cloud_service_type - ].append(locations) - - for group, cst_and_locations in parents_and_locations_map.items(): - all_locations = set() - for cst, locations in cst_and_locations.items(): - for location in locations: - all_locations.add(location) - if all_locations: - parents_and_locations_map[group]["all_locations"] = list(all_locations) - - return parents_and_locations_map - - @staticmethod - def _add_group_and_service_to_recommender_map(): - for key, value in RECOMMENDATION_MAP.items(): - prefix, cloud_service_group, cloud_service_type, *others = key.split(".") - if not ( - cloud_service_type.endswith("Commitments") - or cloud_service_type.endswith("Recommender") - ): - if cloud_service_group == "cloudsql": - cloud_service_group = "sqladmin" - RECOMMENDATION_MAP[key]["cloudServiceGroup"] = cloud_service_group - RECOMMENDATION_MAP[key]["cloudServiceType"] = cloud_service_type.lower() - else: - RECOMMENDATION_MAP[key]["cloudServiceGroup"] = cloud_service_group - RECOMMENDATION_MAP[key]["cloudServiceType"] = None - - def _add_locations_to_recommender_map(self, parents_and_locations_map): - delete_services = [] - for service, cst in parents_and_locations_map.items(): - if not cst: - delete_services.append(service) - - for service in delete_services: - del parents_and_locations_map[service] - - for key, value in RECOMMENDATION_MAP.items(): - cloud_service_group = value["cloudServiceGroup"] - cloud_service_type = value["cloudServiceType"] - - for service, cst_and_locations in parents_and_locations_map.items(): - if cloud_service_group == service: - for service_key, locations in cst_and_locations.items(): - if cloud_service_type == service_key: - RECOMMENDATION_MAP[key]["locations"] = locations - - if ( - "locations" not in RECOMMENDATION_MAP[key] - and cloud_service_group == "compute" - ): - RECOMMENDATION_MAP[key]["locations"] = cst_and_locations[ - "instance" - ] - - if cloud_service_type == "commitment": - RECOMMENDATION_MAP[key][ - "locations" - ] = self._change_zone_to_region( - cst_and_locations["instance"] - ) - - if "locations" not in RECOMMENDATION_MAP[key]: - RECOMMENDATION_MAP[key]["locations"] = cst_and_locations[ - "all_locations" - ] - - if "locations" not in RECOMMENDATION_MAP[key]: - RECOMMENDATION_MAP[key]["locations"] = ["global"] - - if "global" not in RECOMMENDATION_MAP[key]["locations"]: - RECOMMENDATION_MAP[key]["locations"].append("global") - - @staticmethod - def _change_zone_to_region(zones): - regions = [] - for zone in zones: - region = zone.rsplit("-", 1)[0] - if region not in regions: - regions.append(region) - return regions - - @staticmethod - def _get_region_and_recommender_id(recommendation_name): - try: - project_id, resource = recommendation_name.split("locations/") - region, _, instance_type, _ = resource.split("/", 3) - return region, instance_type - - except Exception as e: - _LOGGER.error( - f"[_get_region] recommendation passing error (data: {recommendation_name}) => {e}", - exc_info=True, - ) - - @staticmethod - def convert_readable_priority(priority): - if priority == "P1": - return "Highest" - elif priority == "P2": - return "Second Highest" - elif priority == "P3": - return "Second Lowest" - elif priority == "P4": - return "Lowest" - else: - return "Unspecified" - - @staticmethod - def _get_actions(content): - overview = content.get("overview", {}) - operation_groups = content.get("operationGroups", []) - actions = "" - - if recommended_action := overview.get("recommendedAction"): - return recommended_action - - else: - for operation_group in operation_groups: - operations = operation_group.get("operations", []) - for operation in operations: - action = operation.get("action", "test") - first, others = action[0], action[1:] - action = first.upper() + others - - if action == "Test": - continue - elif actions: - actions += f" and {action}" - else: - actions += action - - return actions - - @staticmethod - def _get_location(recommendation_parent): - try: - project_id, parent_info = recommendation_parent.split("locations/") - location, _ = parent_info.split("/", 1) - return location - except Exception as e: - _LOGGER.error( - f"[get_location] recommendation passing error (data: {recommendation_parent}) => {e}", - exc_info=True, - ) - - @staticmethod - def _change_cost_to_description(cost): - currency = cost.get("currencyCode", "USD") - total_cost = 0 - - if nanos := cost.get("nanos", 0): - if nanos < 0: - nanos = -nanos / 1000000000 - else: - nanos = nanos / 1000000000 - total_cost += nanos - - if units := int(cost.get("units", 0)): - if units < 0: - units = -units - total_cost += units - - total_cost = round(total_cost, 2) - description = f"{total_cost}/month" - - if "USD" in currency: - currency = "$" - description = f"{currency}{description}" - - return total_cost, description - - @staticmethod - def _change_resource_name(resource): - try: - resource_name = resource.split("/")[-1] - return resource_name - except ValueError: - return resource - - def _change_target_resources(self, resources): - new_target_resources = [] - for resource in resources: - new_target_resources.append( - {"name": resource, "displayName": self._change_resource_name(resource)} - ) - return new_target_resources - - def _create_recommenders(self, preprocessed_recommendations): - recommenders = [] - for pre_recommendation in preprocessed_recommendations: - redefined_recommendations = [ - { - "description": pre_recommendation["description"], - "state": pre_recommendation["stateInfo"]["state"], - "affectedResource": pre_recommendation["display"].get("resource"), - "viewAffectedResources": self._create_view_affected_resources( - pre_recommendation["display"].get("resource") - ), - "location": pre_recommendation["display"]["location"], - "priorityLevel": pre_recommendation["display"]["priorityDisplay"], - "operations": pre_recommendation["display"]["operationActions"], - "cost": pre_recommendation["display"].get("cost"), - "costSavings": pre_recommendation["display"].get("costDescription"), - } - ] - - recommender = { - "name": pre_recommendation["display"]["recommenderIdName"], - "id": pre_recommendation["display"]["recommenderId"], - "description": pre_recommendation["display"][ - "recommenderIdDescription" - ], - "category": pre_recommendation["primaryImpact"]["category"], - "recommendations": redefined_recommendations, - } - - recommenders.append(recommender) - return recommenders - - def _list_collected_recommender_ids(self, recommenders): - collected_recommender_ids = [] - for recommender in recommenders: - recommender_id = recommender["id"] - if "recommendations" not in self.recommender_map[recommender_id]: - self.recommender_map[recommender_id].update(recommender) - else: - for recommendation in recommender["recommendations"]: - self.recommender_map[recommender_id]["recommendations"].append( - recommendation - ) - - if recommender_id not in collected_recommender_ids: - collected_recommender_ids.append(recommender_id) - return collected_recommender_ids - - @staticmethod - def _create_view_affected_resources(resource): - resource_name = resource.get("resourceName") - - if not resource_name: - resource_name = resource.get("serviceName") - - if not resource_name: - resource_name = resource.get("name") - - if not resource_name: - resource_name = resource.get("name") - - if not resource_name and "asset" in resource: - resource_name = resource["asset"].get("name") - - return resource_name - - @staticmethod - def _get_state_and_priority(total_priority_level): - if total_priority_level["Highest"] > 0: - return "error", "Highest" - - if total_priority_level["Second Highest"] > 0: - return "warning", "Second Highest" - - if total_priority_level["Second Lowest"] > 0: - return "ok", "Second Lowest" - else: - return "ok", "Lowest" diff --git a/src/cloudforet/plugin/metadata/recommender/recommendation.yaml b/src/cloudforet/plugin/metadata/recommender/recommender.yaml similarity index 98% rename from src/cloudforet/plugin/metadata/recommender/recommendation.yaml rename to src/cloudforet/plugin/metadata/recommender/recommender.yaml index 127c856..708e225 100644 --- a/src/cloudforet/plugin/metadata/recommender/recommendation.yaml +++ b/src/cloudforet/plugin/metadata/recommender/recommender.yaml @@ -34,6 +34,7 @@ table: - Service: data.cloudServiceGroup - Service Type: data.cloudServiceType - Cost Savings: data.costSavings + - Google Cloud Project: account tabs.0: name: Details From 5fede80db222b5cce57e7bd81a0f5d6a070d36cf Mon Sep 17 00:00:00 2001 From: kangwork Date: Thu, 22 Aug 2024 18:54:30 +0900 Subject: [PATCH 2/3] feat: add SecurityIAMManagement recommendation Signed-off-by: kangwork --- src/cloudforet/plugin/connector/iam.py | 82 ++++++ .../plugin/connector/recommender/insight.py | 51 ++++ src/cloudforet/plugin/manager/__init__.py | 4 +- .../recommender/security_iam_manager.py | 275 ++++++++++++++++++ .../security_iam_management.yaml | 134 +++++++++ src/setup.py | 4 +- 6 files changed, 547 insertions(+), 3 deletions(-) create mode 100644 src/cloudforet/plugin/connector/iam.py create mode 100644 src/cloudforet/plugin/connector/recommender/insight.py create mode 100644 src/cloudforet/plugin/manager/recommender/security_iam_manager.py create mode 100644 src/cloudforet/plugin/metadata/recommender/recommendation/security_iam_management.yaml diff --git a/src/cloudforet/plugin/connector/iam.py b/src/cloudforet/plugin/connector/iam.py new file mode 100644 index 0000000..582276c --- /dev/null +++ b/src/cloudforet/plugin/connector/iam.py @@ -0,0 +1,82 @@ +import logging +from spaceone.core import cache +from cloudforet.plugin.connector.base import GoogleCloudConnector + + + +__all__ = ["IAMConnector"] + +_LOGGER = logging.getLogger("spaceone") + + +class IAMConnector(GoogleCloudConnector): + google_client_service = "iam" + version = "v1" + + def list_predefined_roles(self): + roles = [] + request = self.client.roles().list(view='FULL') + + while True: + response = request.execute() + + roles.extend(response.get("roles", [])) + + request = ( + self.client.roles() + .list_next(previous_request=request, previous_response=response) + ) + + if request is None: + break + + return roles + + def list_project_roles(self, project_id: str = None): + parent = f"projects/{project_id}" + roles = [] + request = self.client.projects().roles().list(parent=parent, view='FULL') + while True: + response = request.execute() + + roles.extend(response.get("roles", [])) + + request = ( + self.client.projects() + .roles() + .list_next(previous_request=request, previous_response=response) + ) + + if request is None: + break + + return roles + + def list_organization_roles(self, resource): + roles = [] + request = self.client.organizations().roles().list(parent=resource, view='FULL') + + while True: + response = request.execute() + roles.extend(response.get("roles", [])) + + request = ( + self.client.organizations() + .roles() + .list_next(previous_request=request, previous_response=response) + ) + + if request is None: + break + + return roles + + def get_all_roles_to_permissions_dict(self, project_id: str, organization_id: str): + roles_to_permissions = {} + roles = self.list_predefined_roles() + roles.extend(self.list_project_roles(project_id)) + if organization_id: + roles.extend(self.list_organization_roles(organization_id)) + for role in roles: + roles_to_permissions[role.get("name")] = role.get("includedPermissions", []) + return roles_to_permissions \ No newline at end of file diff --git a/src/cloudforet/plugin/connector/recommender/insight.py b/src/cloudforet/plugin/connector/recommender/insight.py new file mode 100644 index 0000000..bcfe912 --- /dev/null +++ b/src/cloudforet/plugin/connector/recommender/insight.py @@ -0,0 +1,51 @@ +import logging + +from cloudforet.plugin.connector.base import GoogleCloudConnector +__all__ = ["InsightConnector"] +_LOGGER = logging.getLogger(__name__) + + +class InsightConnector(GoogleCloudConnector): + google_client_service = "recommender" + version = "v1beta1" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_policy_insight(self, insight_id: str, **query): + insight_parent = f"projects/{self.project_id}/locations/global/insightTypes/google.iam.policy/insights/{insight_id}" + query.update({"parent": insight_parent}) + request = ( + self.client.projects() + .locations() + .insightTypes() + .insights() + .get(**query) + ) + response = request.execute() + return response + + def list_insights(self, insight_parent, **query): + insights = [] + query.update({"parent": insight_parent}) + request = ( + self.client.projects() + .locations() + .insightTypes() + .insights() + .list(**query) + ) + + while request is not None: + response = request.execute() + insights.extend( + insight for insight in response.get("insights", []) + ) + request = ( + self.client.projects() + .locations() + .insightTypes() + .insights() + .list_next(previous_request=request, previous_response=response) + ) + return insights diff --git a/src/cloudforet/plugin/manager/__init__.py b/src/cloudforet/plugin/manager/__init__.py index 9c20d73..2c790ed 100644 --- a/src/cloudforet/plugin/manager/__init__.py +++ b/src/cloudforet/plugin/manager/__init__.py @@ -1,3 +1,3 @@ from cloudforet.plugin.manager.base import ResourceManager - -from cloudforet.plugin.manager.recommender.recommendation import RecommendationManager +from cloudforet.plugin.manager.recommender.security_iam_manager import SecurityIAMRecommendationManager +# from cloudforet.plugin.manager.recommender.recommendation_manager import RecommendationManager diff --git a/src/cloudforet/plugin/manager/recommender/security_iam_manager.py b/src/cloudforet/plugin/manager/recommender/security_iam_manager.py new file mode 100644 index 0000000..c727e32 --- /dev/null +++ b/src/cloudforet/plugin/manager/recommender/security_iam_manager.py @@ -0,0 +1,275 @@ +import logging +from spaceone.inventory.plugin.collector.lib import * +from cloudforet.plugin.config.global_conf import ASSET_URL +from cloudforet.plugin.connector.recommender.insight import InsightConnector +from cloudforet.plugin.connector.iam import IAMConnector +from cloudforet.plugin.connector.recommender.recommendation import RecommendationConnector +from cloudforet.plugin.manager import ResourceManager +_LOGGER = logging.getLogger(__name__) + + +class SecurityIAMRecommendationManager(ResourceManager): + service = "Security Recommendation - IAM Management" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cloud_service_group = "Recommender" + self.cloud_service_type = "SecurityIAMManagement" + self.metadata_path = ( + "plugin/metadata/recommender/recommendation/security_iam_management.yaml" + ) + self.project_id = None + self.organization_id = None + self.all_roles_to_permissions = {} + + def create_cloud_service_type(self): + return make_cloud_service_type( + name=self.cloud_service_type, + group=self.cloud_service_group, + provider=self.provider, + metadata_path=self.metadata_path, + is_primary=True, + is_major=True, + service_code="Recommender", + tags={"spaceone:icon": f"{ASSET_URL}/user_preferences.svg"}, + labels=["Analytics"], + ) + + def create_cloud_service(self, options, secret_data, schema): + cloud_services = [] + error_responses = [] + self.project_id = secret_data.get("project_id") + self.organization_id = secret_data.get("organization_id") + member_to_role_to_data = {} + member_to_overall_values = {} + iam_connector = IAMConnector( + options=options, secret_data=secret_data, schema=schema + ) + self.all_roles_to_permissions = iam_connector.get_all_roles_to_permissions_dict(project_id=self.project_id, organization_id=self.organization_id) + revoked_policy_insights, revoked_service_account_insights = self._list_insights(options, secret_data, schema) + recs = self.list_recommendations(options, secret_data, schema) + + for rec in recs: + data = self._parse_recommendation(rec) + member_id = data.pop("memberId") + member_type = data.pop("memberType") + role_name = data.pop("roleName") + if member_id not in member_to_role_to_data: + member_to_role_to_data[member_id] = {} + member_to_overall_values[member_id] = { + "totalUnusedPermissionsCount": 0, + "rolesCount": 0, + "_priority_count": 0, + "_priority_sum": 0, + "insightSubtypes": ["PERMISSIONS USAGE"], + "memberType": member_type, + } + + if role_name in member_to_role_to_data[member_id]: + for rest_data in member_to_role_to_data[member_id][role_name]: + if rest_data["lastRefreshTime"] < data["lastRefreshTime"]: + member_to_overall_values[member_id][ + "totalUnusedPermissionsCount" + ] -= rest_data["unusedPermissionsCount"] + member_to_overall_values[member_id]["rolesCount"] -= 1 + member_to_overall_values[member_id]["_priority_count"] -= 1 + member_to_overall_values[member_id]["_priority_sum"] -= int( + rest_data["priority"][1] + ) + break + + member_to_role_to_data[member_id][role_name] = data + member_to_overall_values[member_id]["totalUnusedPermissionsCount"] += data[ + "unusedPermissionsCount" + ] + member_to_overall_values[member_id]["rolesCount"] += 1 + member_to_overall_values[member_id]["_priority_count"] += 1 + member_to_overall_values[member_id]["_priority_sum"] += int( + data["priority"][1] + ) + + for insight in revoked_policy_insights: + perm_data = self._parse_permission_usage_insights(insight) + member = perm_data.get("memberId") + if member in member_to_role_to_data: + role = perm_data.get("roleName") + if role not in member_to_role_to_data[member]: + member_to_role_to_data[member][role] = {} + member_to_role_to_data[member][role].update(perm_data["insightSpecificData"]) + + for insight in revoked_service_account_insights: + service_account_data = self._parse_service_account_insights(insight) + member = service_account_data.get("memberId") + member_type = service_account_data.get("memberType") + insight_data = service_account_data.get("insightSpecificData") + if member in member_to_role_to_data: + member_to_overall_values[member]["insightSubtypes"].append("SERVICE ACCOUNT USAGE") + member_to_overall_values[member]["lastRefreshTime"] = insight_data["lastRefreshTime"] + else: + member_to_role_to_data[member] = {} + service_account_data["priority"] = "P4" + member_to_overall_values[member] = { + "_priority_count": 1, + "_priority_sum": 4, + "insightSubtypes": ["SERVICE ACCOUNT USAGE"], + "memberType": member_type + } + member_to_role_to_data[member]["serviceAccount"] = insight_data + + for member in member_to_role_to_data: + avg_priority = member_to_overall_values[member].pop("_priority_sum") / member_to_overall_values[member].pop("_priority_count") + member_to_overall_values[member]["priority"] = self._convert_avg_priority_to_priority(avg_priority) + data = { + "serviceAccountRecommendation": member_to_role_to_data[member].pop("serviceAccount", {}), + "roleRecommendations": [member_to_role_to_data[member][role] for role in member_to_role_to_data[member]], + "memberType": member_to_overall_values[member].pop("memberType"), + "category": "SECURITY", + "product": "IAM", + "productCategory": "Access Management", + "insightSubtypes": member_to_overall_values[member].pop("insightSubtypes"), + "overallValues": member_to_overall_values[member], + } + try: + cloud_services.append( + make_cloud_service( + name=member, + cloud_service_type=self.cloud_service_type, + cloud_service_group=self.cloud_service_group, + provider=self.provider, + account=self.project_id, + data=data, + region_code="global", + instance_type="", + instance_size=0, + reference={ + "resource_id": member, + "external_link": f"https://console.cloud.google.com/active-assist/list/security/recommendations?project={self.project_id}", + }, + ) + ) + except Exception as e: + error_responses.append( + make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + ) + ) + return cloud_services, error_responses + + def list_recommendations(self, options, secret_data, schema) -> list: + rec_parents = self._list_recommendation_parents() + recommendation_conn = RecommendationConnector( + options=options, secret_data=secret_data, schema=schema + ) + recs = [] + for parent in rec_parents: + recs.extend(recommendation_conn.list_recommendations(parent)) + return recs + + def _list_recommendation_parents(self) -> list: + rec_parents = [] + rec_id = "google.iam.policy.Recommender" + if self.organization_id: + rec_parents.append(f"organizations/{self.organization_id}/locations/global/recommenders/{rec_id}") + if self.project_id: + rec_parents.append(f"projects/{self.project_id}/locations/global/recommenders/{rec_id}") + return rec_parents + + def _parse_recommendation(self, rec: dict) -> dict: + overview = rec.get("content", {}).get("overview", {}) + member = overview.get("member", ":") + member_id = member.split(":")[1] + member_type = member.split(":")[0].upper() + if member_type == "SERVICEACCOUNT": + member_type = "SERVICE ACCOUNT" + data = { + "memberType": member_type, + "memberId": member_id, + + "roleName": overview.get("removedRole"), + "unusedPermissionsCount": rec.get("primaryImpact", {}) + .get("securityProjection", {}) + .get("details", {}) + .get("revokedIamPermissionsCount", 0), + "lastRefreshTime": rec.get("lastRefreshTime"), + "priority": rec.get("priority"), + "insightId": rec.get("associatedInsights", [{}])[0].get("insight"), + } + return data + + def _parse_permission_usage_insights(self, insight: dict) -> dict: + content = insight.get("content", {}) + member = content.get("member", ":") + member_type, member_id = member.split(":") + member_type = " ".join([x for x in member_id if x.isupper()]) + member_type = member_type.upper() + observation_period_in_sec = insight.get("observationPeriod") + observation_period_in_days = round(int(observation_period_in_sec[:-1]) / 86400) + role_name = content.get("role") + + all_perms = set(self.all_roles_to_permissions.get(role_name, [])) + _exercised_perms_dict = content.get("exercisedPermissions", []) + + exercised_perms = [perm.get("permission") for perm in _exercised_perms_dict] + unused_permissions = all_perms.difference(set(exercised_perms)) + + inferred_perms = [perm.get("permission") for perm in content.get("inferredPermissions", [])] + data = { + + "memberType": member_type, + "memberId": member_id, + "roleName": role_name, + + "insightSpecificData": { + "roleName": role_name, + "insightId": insight.get("name"), + "unusedPermissions": list(unused_permissions), + "exercisedPermissions": exercised_perms, + "exercisedPermissionsCount": len(content.get("exercisedPermissions", [])), + "inferredPermissions": inferred_perms, + "inferredPermissionsCount": len(inferred_perms), + "currentTotalPermissionsCount": content.get("currentTotalPermissionsCount", 0), + "observationPeriod": observation_period_in_days, + } + } + return data + + def _parse_service_account_insights(self, insight: dict) -> dict: + content = insight.get("content", {}) + observation_period_in_sec = insight.get("observationPeriod") + observation_period_in_days = round(int(observation_period_in_sec[:-1]) / 86400) + data = { + + "memberType": "SERVICE ACCOUNT", + "memberId": content.get("email"), + + "insightSpecificData": { + "insightId": insight.get("name"), + "lastAuthenticatedTime": content.get("lastAuthenticatedTime"), + "lastRefreshTime": insight.get("lastRefreshTime"), + "observationPeriod": observation_period_in_days, + } + } + return data + + def _list_insights(self, options, secret_data, schema) -> (list, list): + insight_connector = InsightConnector( + options=options, secret_data=secret_data, schema=schema + ) + insight_parent = f"projects/{self.project_id}/locations/global/insightTypes/" + revoked_policy_insights = insight_connector.list_insights(insight_parent + "google.iam.policy.Insight") + revoked_service_account_insights = insight_connector.list_insights(insight_parent + "google.iam.serviceAccount.Insight") + return revoked_policy_insights, revoked_service_account_insights + + @staticmethod + def _convert_avg_priority_to_priority(avg_priority: float) -> str: + if avg_priority < 1.5: + return "P1" + elif avg_priority < 2.5: + return "P2" + elif avg_priority < 3.5: + return "P3" + else: + return "P4" diff --git a/src/cloudforet/plugin/metadata/recommender/recommendation/security_iam_management.yaml b/src/cloudforet/plugin/metadata/recommender/recommendation/security_iam_management.yaml new file mode 100644 index 0000000..ec9d9bf --- /dev/null +++ b/src/cloudforet/plugin/metadata/recommender/recommendation/security_iam_management.yaml @@ -0,0 +1,134 @@ +search: + fields: + - Name: name + - State: data.recommendations.state + - Category: data.category + - Location: data.location + - Member Type: data.memberType +# field_type: enum +# enums: +# - SERVICE_ACCOUNT: green.500 +# - USER: yellow.500 +table: +# sort: +# key: data.overallValues.priority +# desc: false + + fields: + - Member Type: data.memberType + type: enum + enums: + - USER: red.500 + - SERVICE ACCOUNT: coral.500 + - Category: data.category + type: enum + enums: + - SECURITY: blue.500 + is_optional: true + - Product: data.product + is_optional: true + - Product Category: data.productCategory + is_optional: true + - Insight Subtypes: data.insightSubtypes + type: list + options: + type: enum + enums: + - PERMISSIONS USAGE: red.500 + - SERVICE ACCOUNT USAGE: coral.500 + - Location: data.location + type: inventory.Region + - Overall Priority Level: data.overallValues.priority + type: enum + enums: + - P1: red.500 + name: Highest + - P2: coral.500 + name: Second Highest + - P3: red.300 + name: Second Lowest + - P4: coral.300 + name: Lowest + - Total Roles Count: data.overallValues.rolesCount + - Total Excess Permissions Count: data.overallValues.totalUnusedPermissionsCount + +tabs.0: + name: Permissions Usage Insights + type: query-search-table + root_path: data.roleRecommendations + unwind: data.roleRecommendations + sort: + key: data.roleRecommendations.unusedPermissionsCount + desc: true + fields: + - Role Name: roleName + - Priority Level: priority + type: enum + enums: + - P1: red.500 + name: Highest + - P2: coral.500 + name: Second Highest + - P3: red.300 + name: Second Lowest + - P4: coral.300 + name: Lowest + - Current Total Permissions Count: currentTotalPermissionsCount + + - name: Unused Permissions Count + key: unusedPermissionsCount +# type: more +# popup_key: data.roleRecommendations.unusedPermissionsCount +# popup_name: "The permissions below were not used during the observation period" +# popup_type: raw + + - name: Unused Permissions List + key: unusedPermissionsCount + type: more + popup_key: data.roleRecommendations.unusedPermissions + popup_name: "The permissions below were not used during the observation period" + popup_type: raw + + - name: Recently Used Permissions Count + key: exercisedPermissionsCount +# type: more +# popup_key: data.roleRecommendations.exercisedPermissionsCount +# popup_name: "The permissions below were used during the observation period" +# popup_type: raw + + - name: Recently Used Permissions List + key: exercisedPermissionsCount + type: more + popup_key: data.roleRecommendations.exercisedPermissions + popup_name: "The permissions below were used during the observation period" + popup_type: raw + + - name: Inferred Permissions Count + key: inferredPermissionsCount + + - name: Inferred Permissions List + key: inferredPermissionsCount + type: more + popup_key: data.roleRecommendations.inferredPermissions + popup_name: "The permissions below are likely to be needed based on the recently used permissions" + popup_type: raw + + - Last Refreshed Time: lastRefreshTime + type: datetime + display_format: 'YYYY-MM-DD HH:mm:ss' + source_type: iso8601 + +tabs.1: + name: Service Account Usage Insight + type: item + fields: + - Observation Period (days): data.serviceAccountRecommendation.observationPeriod + - Last Authenticated Time: data.serviceAccountRecommendation.lastAuthenticatedTime + type: datetime + display_format: 'YYYY-MM-DD HH:mm:ss' + source_type: iso8601 + - Last Refresh Time: data.serviceAccountRecommendation.lastRefreshTime + type: datetime + display_format: 'YYYY-MM-DD HH:mm:ss' + source_type: iso8601 + - Insight ID: data.serviceAccountRecommendation.insightId \ No newline at end of file diff --git a/src/setup.py b/src/setup.py index ab873be..7d6cf6c 100755 --- a/src/setup.py +++ b/src/setup.py @@ -36,6 +36,8 @@ "requests", "beautifulsoup4", ], - package_data={"cloudforet": ["plugin/metadata/*/*.yaml"]}, + package_data={"cloudforet": ["plugin/metadata/*/*.yaml", + "plugin/metadata/*/*/*.yaml" + ]}, zip_safe=False, ) From 65b344356da49428f2da6554dafaac632541e8d3 Mon Sep 17 00:00:00 2001 From: kangwork Date: Wed, 28 Aug 2024 17:15:05 -0400 Subject: [PATCH 3/3] improvement: add pageSize to the list role methods in connector/iam.py Signed-off-by: kangwork --- src/cloudforet/plugin/connector/iam.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cloudforet/plugin/connector/iam.py b/src/cloudforet/plugin/connector/iam.py index 582276c..2d26123 100644 --- a/src/cloudforet/plugin/connector/iam.py +++ b/src/cloudforet/plugin/connector/iam.py @@ -15,7 +15,7 @@ class IAMConnector(GoogleCloudConnector): def list_predefined_roles(self): roles = [] - request = self.client.roles().list(view='FULL') + request = self.client.roles().list(pageSize=1000, view='FULL') while True: response = request.execute() @@ -35,7 +35,7 @@ def list_predefined_roles(self): def list_project_roles(self, project_id: str = None): parent = f"projects/{project_id}" roles = [] - request = self.client.projects().roles().list(parent=parent, view='FULL') + request = self.client.projects().roles().list(parent=parent, pageSize=1000, view='FULL') while True: response = request.execute() @@ -54,7 +54,7 @@ def list_project_roles(self, project_id: str = None): def list_organization_roles(self, resource): roles = [] - request = self.client.organizations().roles().list(parent=resource, view='FULL') + request = self.client.organizations().roles().list(parent=resource, pageSize=1000, view='FULL') while True: response = request.execute()