From 2f741f35a8fe7f6f720af78e22c07bd59ea2da40 Mon Sep 17 00:00:00 2001 From: Sergio Garcia Date: Fri, 28 Feb 2025 10:10:43 +0100 Subject: [PATCH] chore(gcp): enhance GCP APIs logic (#7046) --- docs/tutorials/gcp/authentication.md | 23 +- .../providers/gcp/exceptions/exceptions.py | 44 --- prowler/providers/gcp/gcp_provider.py | 261 +++++++++--------- prowler/providers/gcp/lib/service/service.py | 5 +- .../cloudresourcemanager_service.py | 13 +- 5 files changed, 150 insertions(+), 196 deletions(-) diff --git a/docs/tutorials/gcp/authentication.md b/docs/tutorials/gcp/authentication.md index 1ba42c1020..85707a8f05 100644 --- a/docs/tutorials/gcp/authentication.md +++ b/docs/tutorials/gcp/authentication.md @@ -25,6 +25,9 @@ Prowler will follow the same credentials search as [Google authentication librar Those credentials must be associated to a user or service account with proper permissions to do all checks. To make sure, add the `Viewer` role to the member associated with the credentials. +???+ note + Prowler will use the enabled Google Cloud APIs to get the information needed to perform the checks. + ## Impersonate Service Account If you want to impersonate a GCP service account, you can use the `--impersonate-service-account` argument: @@ -34,23 +37,3 @@ prowler gcp --impersonate-service-account ``` This argument will use the default credentials to impersonate the service account provided. - -## Service APIs - -Prowler will use the Google Cloud APIs to get the information needed to perform the checks. Make sure that the following APIs are enabled in the project: - -- apikeys.googleapis.com -- artifactregistry.googleapis.com -- bigquery.googleapis.com -- sqladmin.googleapis.com -- storage.googleapis.com -- compute.googleapis.com -- dataproc.googleapis.com -- dns.googleapis.com -- containerregistry.googleapis.com -- container.googleapis.com -- iam.googleapis.com -- cloudkms.googleapis.com -- logging.googleapis.com - -You can enable them automatically using our script [enable_apis_in_projects.sh](https://github.com/prowler-cloud/prowler/blob/master/contrib/gcp/enable_apis_in_projects.sh) diff --git a/prowler/providers/gcp/exceptions/exceptions.py b/prowler/providers/gcp/exceptions/exceptions.py index 8b0c2bfdc6..5c5845951c 100644 --- a/prowler/providers/gcp/exceptions/exceptions.py +++ b/prowler/providers/gcp/exceptions/exceptions.py @@ -6,14 +6,6 @@ class GCPBaseException(ProwlerException): """Base class for GCP Errors.""" GCP_ERROR_CODES = { - (3000, "GCPCloudResourceManagerAPINotUsedError"): { - "message": "Cloud Resource Manager API not used", - "remediation": "Enable the Cloud Resource Manager API for the project.", - }, - (3001, "GCPHTTPError"): { - "message": "HTTP error", - "remediation": "Check the HTTP error and ensure the request is properly formatted.", - }, (3002, "GCPNoAccesibleProjectsError"): { "message": "No Project IDs are active or can be accessed via Google Credentials", "remediation": "Ensure the project is active and accessible.", @@ -22,10 +14,6 @@ class GCPBaseException(ProwlerException): "message": "Error setting up session", "remediation": "Check the session setup and ensure it is properly set up.", }, - (3004, "GCPGetProjectError"): { - "message": "Error getting project", - "remediation": "Check the project and ensure it is properly set up.", - }, (3005, "GCPTestConnectionError"): { "message": "Error testing connection to GCP", "remediation": "Check the connection and ensure it is properly set up.", @@ -42,10 +30,6 @@ class GCPBaseException(ProwlerException): "message": "Provider does not match with the expected project_id", "remediation": "Check the provider and ensure it matches the expected project_id.", }, - (3009, "GCPCloudAssetAPINotUsedError"): { - "message": "Cloud Asset API not used", - "remediation": "Enable the Cloud Asset API for the project.", - }, (3010, "GCPLoadServiceAccountKeyFromDictError"): { "message": "Error loading Service Account Private Key credentials from dictionary", "remediation": "Check the dictionary and ensure it contains a Service Account Private Key.", @@ -73,20 +57,6 @@ def __init__(self, code, file=None, original_exception=None, message=None): super().__init__(code, file, original_exception, message) -class GCPCloudResourceManagerAPINotUsedError(GCPBaseException): - def __init__(self, file=None, original_exception=None, message=None): - super().__init__( - 3000, file=file, original_exception=original_exception, message=message - ) - - -class GCPHTTPError(GCPBaseException): - def __init__(self, file=None, original_exception=None, message=None): - super().__init__( - 3001, file=file, original_exception=original_exception, message=message - ) - - class GCPNoAccesibleProjectsError(GCPCredentialsError): def __init__(self, file=None, original_exception=None, message=None): super().__init__( @@ -101,13 +71,6 @@ def __init__(self, file=None, original_exception=None, message=None): ) -class GCPGetProjectError(GCPCredentialsError): - def __init__(self, file=None, original_exception=None, message=None): - super().__init__( - 3004, file=file, original_exception=original_exception, message=message - ) - - class GCPTestConnectionError(GCPBaseException): def __init__(self, file=None, original_exception=None, message=None): super().__init__( @@ -136,13 +99,6 @@ def __init__(self, file=None, original_exception=None, message=None): ) -class GCPCloudAssetAPINotUsedError(GCPBaseException): - def __init__(self, file=None, original_exception=None, message=None): - super().__init__( - 3009, file=file, original_exception=original_exception, message=message - ) - - class GCPLoadServiceAccountKeyFromDictError(GCPCredentialsError): def __init__(self, file=None, original_exception=None, message=None): super().__init__( diff --git a/prowler/providers/gcp/gcp_provider.py b/prowler/providers/gcp/gcp_provider.py index 0154a39507..ee3a1ad4d3 100644 --- a/prowler/providers/gcp/gcp_provider.py +++ b/prowler/providers/gcp/gcp_provider.py @@ -1,3 +1,4 @@ +import json import os import re import sys @@ -20,10 +21,6 @@ from prowler.providers.common.models import Audit_Metadata, Connection from prowler.providers.common.provider import Provider from prowler.providers.gcp.exceptions.exceptions import ( - GCPCloudAssetAPINotUsedError, - GCPCloudResourceManagerAPINotUsedError, - GCPGetProjectError, - GCPHTTPError, GCPInvalidProviderIdError, GCPLoadADCFromDictError, GCPLoadServiceAccountKeyFromDictError, @@ -113,7 +110,6 @@ def __init__( GCPNoAccesibleProjectsError if no project IDs can be accessed via Google Credentials GCPSetUpSessionError if an error occurs during the setup session GCPLoadADCFromDictError if an error occurs during the loading credentials from dict - GCPGetProjectError if an error occurs during the get project Returns: None @@ -182,7 +178,9 @@ def __init__( self._project_ids = [] self._projects = {} self._excluded_project_ids = [] - accessible_projects = self.get_projects(self._session, organization_id) + accessible_projects = self.get_projects( + self._session, organization_id, project_ids, credentials_file + ) if not accessible_projects: logger.critical("No Project IDs can be accessed via Google Credentials.") raise GCPNoAccesibleProjectsError( @@ -500,11 +498,13 @@ def test_connection( if provider_id and project_id != provider_id: # Logic to check if the provider ID matches the project ID GcpProvider.validate_project_id( - provider_id=provider_id, credentials=session + provider_id=provider_id, + credentials=session, ) - service = discovery.build("cloudresourcemanager", "v1", credentials=session) - request = service.projects().list() + # Test the connection using the Service Usage API since it is enabled by default + client = discovery.build("serviceusage", "v1", credentials=session) + request = client.services().list(parent=f"projects/{project_id}") request.execute() return Connection(is_connected=True) @@ -524,18 +524,9 @@ def test_connection( raise setup_session_error return Connection(error=setup_session_error) except HttpError as http_error: - if "Cloud Resource Manager API has not been used" in str(http_error): - logger.critical( - "Cloud Resource Manager API has not been used before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/ then retry." - ) - if raise_on_exception: - raise GCPCloudResourceManagerAPINotUsedError( - file=__file__, original_exception=http_error - ) - else: - logger.critical( - f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}" - ) + logger.critical( + f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}" + ) if raise_on_exception: raise http_error return Connection(error=http_error) @@ -581,7 +572,10 @@ def print_credentials(self): @staticmethod def get_projects( - credentials: Credentials, organization_id: str = None + credentials: Credentials, + organization_id: str = None, + project_ids: list = None, + credentials_file: str = None, ) -> dict[str, GCPProject]: """ Get the projects accessible by the provided credentials. If an organization ID is provided, only the projects under that organization are returned. @@ -589,16 +583,12 @@ def get_projects( Args: credentials: Credentials organization_id: str + project_ids: list + credentials_file: str Returns: dict[str, GCPProject] - Raises: - GCPCloudResourceManagerAPINotUsedError if the Cloud Resource Manager API has not been used before or it is disabled - GCPCloudAssetAPINotUsedError if the Cloud Asset API has not been used before or it is disabled - GCPHTTPError if an error occurs during the HTTP request - GCPGetProjectError if an error occurs during the get project - Usage: >>> GcpProvider.get_projects(credentials=credentials, organization_id=organization_id) """ @@ -606,112 +596,137 @@ def get_projects( projects = {} if organization_id: - # Initialize Cloud Asset Inventory API for recursive project retrieval - asset_service = discovery.build( - "cloudasset", "v1", credentials=credentials - ) - # Set the scope to the specified organization and filter for projects - scope = f"organizations/{organization_id}" - request = asset_service.assets().list( - parent=scope, - assetTypes=["cloudresourcemanager.googleapis.com/Project"], - contentType="RESOURCE", - ) - - while request is not None: - response = request.execute() + try: + # Initialize Cloud Asset Inventory API for recursive project retrieval + asset_service = discovery.build( + "cloudasset", "v1", credentials=credentials + ) + # Set the scope to the specified organization and filter for projects + scope = f"organizations/{organization_id}" + request = asset_service.assets().list( + parent=scope, + assetTypes=["cloudresourcemanager.googleapis.com/Project"], + contentType="RESOURCE", + ) - for asset in response.get("assets", []): - # Extract labels and other project details - labels = { - k: v - for k, v in asset["resource"]["data"] - .get("labels", {}) - .items() - } - project_id = asset["resource"]["data"]["projectId"] - gcp_project = GCPProject( - number=asset["resource"]["data"]["projectNumber"], - id=project_id, - name=asset["resource"]["data"].get("name", project_id), - lifecycle_state=asset["resource"]["data"].get( - "lifecycleState" - ), - labels=labels, - ) - gcp_project.organization = GCPOrganization( - id=organization_id, name=f"organizations/{organization_id}" - ) + while request is not None: + response = request.execute() - projects[project_id] = gcp_project + for asset in response.get("assets", []): + # Extract labels and other project details + labels = { + k: v + for k, v in asset["resource"]["data"] + .get("labels", {}) + .items() + } + project_id = asset["resource"]["data"]["projectId"] + gcp_project = GCPProject( + number=asset["resource"]["data"]["projectNumber"], + id=project_id, + name=asset["resource"]["data"].get("name", project_id), + lifecycle_state=asset["resource"]["data"].get( + "lifecycleState" + ), + labels=labels, + ) + gcp_project.organization = GCPOrganization( + id=organization_id, + name=f"organizations/{organization_id}", + ) - request = asset_service.assets().list_next( - previous_request=request, previous_response=response - ) + projects[project_id] = gcp_project + request = asset_service.assets().list_next( + previous_request=request, previous_response=response + ) + except HttpError as http_error: + if "Cloud Asset API has not been used" in str(http_error): + logger.error( + f"Projects cannot be retrieved from the Organization since Cloud Asset API has not been used before or it is disabled [{http_error.__traceback__.tb_lineno}]. Enable it by visiting https://console.developers.google.com/apis/api/cloudasset.googleapis.com/ then retry." + ) + else: + logger.error( + f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}" + ) else: - # Initialize Cloud Resource Manager API for simple project listing - service = discovery.build( - "cloudresourcemanager", "v1", credentials=credentials - ) - request = service.projects().list() - - while request is not None: - response = request.execute() + try: + # Initialize Cloud Resource Manager API for simple project listing + service = discovery.build( + "cloudresourcemanager", "v1", credentials=credentials + ) + request = service.projects().list() - for project in response.get("projects", []): - # Extract labels and other project details - labels = {k: v for k, v in project.get("labels", {}).items()} - project_id = project["projectId"] - gcp_project = GCPProject( - number=project["projectNumber"], - id=project_id, - name=project.get("name", project_id), - lifecycle_state=project["lifecycleState"], - labels=labels, - ) + while request is not None: + response = request.execute() - # Set organization if present in the project metadata - if ( - "parent" in project - and project["parent"].get("type") == "organization" - ): - parent_org_id = project["parent"]["id"] - gcp_project.organization = GCPOrganization( - id=parent_org_id, name=f"organizations/{parent_org_id}" + for project in response.get("projects", []): + # Extract labels and other project details + labels = { + k: v for k, v in project.get("labels", {}).items() + } + project_id = project["projectId"] + gcp_project = GCPProject( + number=project["projectNumber"], + id=project_id, + name=project.get("name", project_id), + lifecycle_state=project["lifecycleState"], + labels=labels, ) - projects[project_id] = gcp_project + # Set organization if present in the project metadata + if ( + "parent" in project + and project["parent"].get("type") == "organization" + ): + parent_org_id = project["parent"]["id"] + gcp_project.organization = GCPOrganization( + id=parent_org_id, + name=f"organizations/{parent_org_id}", + ) - request = service.projects().list_next( - previous_request=request, previous_response=response - ) + projects[project_id] = gcp_project - except HttpError as http_error: - if "Cloud Resource Manager API has not been used" in str(http_error): - logger.critical( - "Cloud Resource Manager API has not been used before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/ then retry." - ) - raise GCPCloudResourceManagerAPINotUsedError( - file=__file__, original_exception=http_error - ) - elif "Cloud Asset API has not been used" in str(http_error): - logger.critical( - "Cloud Asset API has not been used before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudasset.googleapis.com/ then retry." - ) - raise GCPCloudAssetAPINotUsedError( - file=__file__, original_exception=http_error - ) - else: - logger.error( - f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}" - ) - raise GCPHTTPError(file=__file__, original_exception=http_error) + request = service.projects().list_next( + previous_request=request, previous_response=response + ) + except HttpError as http_error: + if "Cloud Resource Manager API has not been used" in str( + http_error + ): + logger.error( + f"Project information cannot be retrieved since Cloud Resource Manager API has not been used before or it is disabled [{http_error.__traceback__.tb_lineno}]. Enable it by visiting https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/ then retry." + ) + else: + logger.error( + f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}" + ) + if not projects: + # If no projects were able to be accessed via API, add them manually if provided by the user in arguments + if project_ids: + for input_project in project_ids: + projects[input_project] = GCPProject( + id=input_project, + name=input_project, + number=0, + labels={}, + lifecycle_state="ACTIVE", + ) + # If no projects were able to be accessed via API, add them manually from the credentials file + elif credentials_file: + with open(credentials_file, "r", encoding="utf-8") as file: + project_id = json.load(file)["project_id"] + projects[project_id] = GCPProject( + id=project_id, + name=project_id, + number=0, + labels={}, + lifecycle_state="ACTIVE", + ) except Exception as error: logger.critical( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - raise GCPGetProjectError(file=__file__, original_exception=error) finally: return projects @@ -722,10 +737,6 @@ def update_projects_with_organizations(self): Returns: None - Raises: - GCPHTTPError if an error occurs during the HTTP request - GCPGetProjectError if an error occurs during the get project - Usage: >>> GcpProvider.update_projects_with_organizations() """ @@ -835,7 +846,9 @@ def validate_project_id(provider_id: str, credentials: str = None) -> None: """ available_projects = list( - GcpProvider.get_projects(credentials=credentials).keys() + GcpProvider.get_projects( + credentials=credentials, project_ids=[provider_id] + ).keys() ) if len(available_projects) == 0: diff --git a/prowler/providers/gcp/lib/service/service.py b/prowler/providers/gcp/lib/service/service.py index d163e4781a..ce78476f37 100644 --- a/prowler/providers/gcp/lib/service/service.py +++ b/prowler/providers/gcp/lib/service/service.py @@ -2,7 +2,6 @@ import google_auth_httplib2 import httplib2 -from colorama import Fore, Style from google.oauth2.credentials import Credentials from googleapiclient import discovery from googleapiclient.discovery import Resource @@ -66,8 +65,8 @@ def __is_api_active__(self, audited_project_ids): if response.get("state") != "DISABLED": project_ids.append(project_id) else: - print( - f"\n{Fore.YELLOW}{self.service} API {Style.RESET_ALL}has not been used in project {project_id} before or it is disabled.\nEnable it by visiting https://console.developers.google.com/apis/api/{self.service}.googleapis.com/overview?project={project_id} then retry." + logger.error( + f"{self.service} API has not been used in project {project_id} before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/{self.service}.googleapis.com/overview?project={project_id} then retry." ) except Exception as error: logger.error( diff --git a/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py b/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py index 2ca72dee3e..28e2f4d832 100644 --- a/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py +++ b/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py @@ -42,11 +42,14 @@ def _get_iam_policy(self): def _get_organizations(self): try: - response = self.client.organizations().search().execute() - for org in response.get("organizations", []): - self.organizations.append( - Organization(id=org["name"].split("/")[-1], name=org["displayName"]) - ) + if self.project_ids: + response = self.client.organizations().search().execute() + for org in response.get("organizations", []): + self.organizations.append( + Organization( + id=org["name"].split("/")[-1], name=org["displayName"] + ) + ) except Exception as error: logger.error( f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"