aded the azure all billing profiles

cloud_governance/common/clouds/azure/cost_management/cost_management_operations.py

@@ -1,8 +1,9 @@
 import calendar
 import datetime
+import time
 
 import pytz
-from azure.mgmt.costmanagement.models import QueryDataset, QueryAggregation, QueryTimePeriod
+from azure.mgmt.costmanagement.models import QueryDataset, QueryAggregation, QueryTimePeriod, QueryGrouping
 
 from cloud_governance.common.clouds.azure.subscriptions.azure_operations import AzureOperations
 from cloud_governance.common.logger.init_logger import logger
@@ -15,54 +16,64 @@ def __init__(self):
         self.azure_operations = AzureOperations()
 
     @logger_time_stamp
-    def get_usage(self, start_date: datetime = '', end_date: datetime = '', granularity: str = 'Monthly'):
+    def get_usage(self, scope: str, start_date: datetime = '', end_date: datetime = '', granularity: str = 'Monthly', **kwargs):
         """
         This method get the current usage based on month
         @return:
         """
-        if not start_date and not end_date:
-            end_date = datetime.datetime.now(pytz.UTC)
-            start_date = end_date.replace(day=1)
-        response = self.azure_operations.cost_mgmt_client.query.usage(scope=self.azure_operations.scope, parameters={
-            'type': 'Usage', 'timeframe': 'Custom', 'time_period': QueryTimePeriod(from_property=start_date, to=end_date),
-            'dataset': QueryDataset(granularity=granularity, aggregation={
-                "totalCost": QueryAggregation(name="Cost", function="Sum")
+        try:
+            if not start_date and not end_date:
+                end_date = datetime.datetime.now(pytz.UTC)
+                start_date = (end_date - datetime.timedelta(days=30)).replace(day=1)
+            response = self.azure_operations.cost_mgmt_client.query.usage(scope=scope, parameters={
+                'type': 'Usage', 'timeframe': 'Custom', 'time_period': QueryTimePeriod(from_property=start_date, to=end_date),
+                'dataset': QueryDataset(granularity=granularity, aggregation={
+                    "totalCost": QueryAggregation(name="Cost", function="Sum")}, **kwargs
+                )
             })
-        })
-        return response.as_dict()
+            return response.as_dict()
+        except Exception as e:
+            print(e)
+        return []
 
     @logger_time_stamp
-    def get_forecast(self, start_date: datetime = '', end_date: datetime = '', granularity: str = 'Monthly'):
+    def get_forecast(self, scope: str, start_date: datetime = '', end_date: datetime = '', granularity: str = 'Monthly', **kwargs):
         """
         This method gets the forecast of next couple of months
         @param start_date:
         @param end_date:
         @param granularity:
+        @param scope:
         @return:
         """
-        if not start_date and not end_date:
-            current = datetime.datetime.now(pytz.UTC)
-            start_date = (current - datetime.timedelta(30)).replace(day=1)
-            end_date = (current + datetime.timedelta(60))
-            month_end = calendar.monthrange(end_date.year, end_date.month)[1]
-            end_date = end_date.replace(day=month_end)
-        logger.info(f'StartDate: {start_date}, EndDate: {end_date}')
-        response = self.azure_operations.cost_mgmt_client.forecast.usage(scope=self.azure_operations.scope, parameters={
-            'type': 'ActualCost', 'timeframe': 'Custom',
-            'time_period': QueryTimePeriod(from_property=start_date, to=end_date),
-            'dataset': QueryDataset(granularity=granularity, aggregation={
-                "totalCost": QueryAggregation(name="Cost", function="Sum"),
-            }), 'include_actual_cost': True, 'include_fresh_partial_cost': False
-        }).as_dict()
-        result = {'columns': response.get('columns'), 'rows': []}
-        row_data = {}
-        for data in response.get('rows'):
-            data_date = data[1]
-            if data_date in row_data:
-                if row_data[data_date][2] == 'Actual' and data[2] == 'Forecast':
-                    row_data[data_date][2] = data[2]
-                row_data[data_date][0] += data[0]
-            else:
-                row_data[data_date] = data
-        result['rows'] = list(row_data.values())
-        return result
+        try:
+            if not start_date and not end_date:
+                current = datetime.datetime.now(pytz.UTC)
+                start_date = (current - datetime.timedelta(60)).replace(day=1)
+                end_date = (current + datetime.timedelta(365))
+                month_end = calendar.monthrange(end_date.year, end_date.month)[1]
+                end_date = end_date.replace(day=month_end)
+            logger.info(f'StartDate: {start_date}, EndDate: {end_date}')
+            response = self.azure_operations.cost_mgmt_client.forecast.usage(scope=scope, parameters={
+                'type': 'ActualCost', 'timeframe': 'Custom',
+                'time_period': QueryTimePeriod(from_property=start_date, to=end_date),
+                'dataset': QueryDataset(granularity=granularity, aggregation={
+                    "totalCost": QueryAggregation(name="Cost", function="Sum"),
+                }, **kwargs), 'include_actual_cost': True, 'include_fresh_partial_cost': False
+            }).as_dict()
+            result = {'columns': response.get('columns'), 'rows': []}
+            row_data = {}
+            for data in response.get('rows'):
+                data_date = data[1]
+                if data_date in row_data:
+                    if row_data[data_date][2] == 'Actual' and data[2] == 'Forecast':
+                        row_data[data_date][2] = data[2]
+                    row_data[data_date][0] += data[0]
+                else:
+                    row_data[data_date] = data
+            result['rows'] = list(row_data.values())
+            return result
+        except Exception as err:
+            time.sleep(60)
+            return self.get_forecast(scope=scope, **kwargs)
+        return []

cloud_governance/common/clouds/azure/subscriptions/azure_operations.py

@@ -1,4 +1,5 @@
 from azure.identity import DefaultAzureCredential
+from azure.mgmt.billing import BillingManagementClient
 from azure.mgmt.costmanagement import CostManagementClient
 from azure.mgmt.subscription import SubscriptionClient
 
@@ -19,6 +20,8 @@ def __init__(self):
         self.__subscription_client = SubscriptionClient(credential=self.__default_creds)
         self.cost_mgmt_client = CostManagementClient(credential=self.__default_creds)
         self.subscription_id, self.account_name = self.__get_subscription_id()
+        self.billing_client = BillingManagementClient(credential=self.__default_creds, subscription_id=self.subscription_id)
+        self.__account_id = self.__environment_variables_dict.get('AZURE_ACCOUNT_ID')
         self.cloud_name = 'AZURE'
         self.scope = f'subscriptions/{self.subscription_id}'
 
@@ -35,5 +38,13 @@ def __get_subscription_id(self):
             return subscription_id, account_name
         return '', ''
 
+    def get_billing_profiles(self):
+        """
+        This method get the billing profiles from the azure account
+        """
+        response = self.billing_client.billing_profiles.list_by_billing_account(self.__account_id)
+        return response
+
+
 
 

cloud_governance/main/environment_variables.py

@@ -73,12 +73,14 @@ def __init__(self):
         self._environment_variables_dict['cost_explorer_tags'] = EnvironmentVariables.get_env('cost_explorer_tags', '{}')
 
         # AZURE Credentials
+        self._environment_variables_dict['AZURE_ACCOUNT_ID'] = EnvironmentVariables.get_env('AZURE_ACCOUNT_ID', '')
         self._environment_variables_dict['AZURE_CLIENT_ID'] = EnvironmentVariables.get_env('AZURE_CLIENT_ID', '')
         self._environment_variables_dict['AZURE_TENANT_ID'] = EnvironmentVariables.get_env('AZURE_TENANT_ID', '')
         self._environment_variables_dict['AZURE_CLIENT_SECRET'] = EnvironmentVariables.get_env('AZURE_CLIENT_SECRET', '')
         if self._environment_variables_dict['AZURE_CLIENT_ID'] and self._environment_variables_dict['AZURE_TENANT_ID']\
                 and self._environment_variables_dict['AZURE_CLIENT_SECRET']:
             self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'AZURE'
+        self._environment_variables_dict['TOTAL_ACCOUNTS'] = bool(EnvironmentVariables.get_env('TOTAL_ACCOUNTS', ''))
 
         # IBM env vars
         self._environment_variables_dict['IBM_ACCOUNT_ID'] = EnvironmentVariables.get_env('IBM_ACCOUNT_ID', '')

cloud_governance/policy/azure/cost_billing_reports.py

@@ -1,5 +1,8 @@
 import datetime
 
+import pytz
+from azure.mgmt.costmanagement.models import QueryGrouping, QueryFilter, QueryComparisonExpression
+
 from cloud_governance.common.clouds.azure.cost_management.cost_management_operations import CostManagementOperations
 from cloud_governance.common.clouds.azure.subscriptions.azure_operations import AzureOperations
 from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload
@@ -16,13 +19,15 @@ class CostBillingReports:
 
     def __init__(self):
         self.__environment_variables_dict = environment_variables.environment_variables_dict
+        self.__total_account = self.__environment_variables_dict.get('TOTAL_ACCOUNTS', '')
         self.azure_operations = AzureOperations()
         self.cost_mgmt_operations = CostManagementOperations()
         self.elastic_upload = ElasticUpload()
         self.gdrive_operations = GoogleDriveOperations()
         self.__gsheet_id = self.__environment_variables_dict.get('SPREADSHEET_ID')
         self.update_to_gsheet = UploadToGsheet()
-        self.__cost_center, self.__allocated_budget, self.__years = self.update_to_gsheet.get_cost_center_budget_details(account_id=self.azure_operations.subscription_id)
+        self.__cost_center, self.__allocated_budget, self.__years = self.update_to_gsheet.get_cost_center_budget_details(account_id=self.azure_operations.subscription_id, dir_path='/tmp')
+        self.__common_data = self.get_common_data()
 
     def get_common_data(self):
         """
@@ -34,21 +39,38 @@ def get_common_data(self):
         upload_data['Account'] = self.azure_operations.account_name+'-'+self.azure_operations.cloud_name.title()
         upload_data['Budget'] = 0
         upload_data['AllocatedBudget'] = 0
-        upload_data['CostCenter'] = self.__cost_center
+        upload_data['CostCenter'] = int(self.__cost_center)
         upload_data['CloudName'] = self.azure_operations.cloud_name
+        upload_data['Owner'] = 'Others'
         return upload_data
 
     @logger_time_stamp
-    def get_data_from_costs(self, cost_data_rows: list, cost_data_columns: list, cost_type: str, cost_billing_data: dict):
+    def get_data_from_costs(self, cost_data_rows: list, cost_data_columns: list, cost_type: str, cost_billing_data: dict, subscription_id: str = '', account_name: str = '', cost_center: int = 0):
         """
         This method get the data from the cost
         @param cost_billing_data:
         @param cost_type:
         @param cost_data_rows:
         @param cost_data_columns:
+        @param subscription_id:
         @return:
         """
-        common_data = self.get_common_data()
+        common_data = self.__common_data
+        index_id = ''
+        common_data["AccountId"] = subscription_id
+        if account_name:
+            common_data['Account'] = account_name
+        if cost_center > 0:
+            common_data['CostCenter'] = cost_center
+        if subscription_id:
+            cost_center, allocated_budget, years = self.update_to_gsheet.get_cost_center_budget_details(account_id=subscription_id, dir_path='/tmp')
+            if cost_center:
+                common_data['CostCenter'] = int(cost_center)
+                common_data['Owner'] = 'Shai'
+            else:
+                common_data['Owner'] = 'Others'
+        else:
+            allocated_budget, years = self.__allocated_budget, self.__years
         for index, item in enumerate(cost_data_rows):
             cost_data = {}
             start_date = ''
@@ -58,32 +80,44 @@ def get_data_from_costs(self, cost_data_rows: list, cost_data_columns: list, cos
                         cost_data[item[2]] = round(item[key], 2)
                     else:
                         cost_data[cost_type] = round(item[key], 2)
+                elif column.get('name') == 'SubscriptionName':
+                    common_data['Account'] = item[key]
+                elif column.get('name') == 'SubscriptionId':
+                    common_data['AccountId'] = item[key]
+                    cost_center, allocated_budget, years = self.update_to_gsheet.get_cost_center_budget_details(
+                        account_id=item[key], dir_path='/tmp')
+                    if cost_center:
+                        common_data['CostCenter'] = int(cost_center)
+                        common_data['Owner'] = 'Shai'
+                    else:
+                        common_data['Owner'] = 'Others'
                 else:
                     if column.get('type') == 'Datetime':
                         start_date = item[key].split('T')[0]
-            if start_date not in cost_billing_data:
-                if start_date.split('-')[0] in self.__years:
+            index_id = f'{start_date}-{common_data["AccountId"]}'
+            if index_id not in cost_billing_data:
+                if start_date.split('-')[0] in years:
                     if 'Budget' in common_data:
                         common_data.pop('Budget')
                     if 'AllocatedBudget' in common_data:
                         common_data.pop('AllocatedBudget')
-                    cost_data['Budget'] = round(self.__allocated_budget / 12, 3)
-                    cost_data['AllocatedBudget'] = self.__allocated_budget
+                cost_data['Budget'] = round(allocated_budget / 12, 3)
+                cost_data['AllocatedBudget'] = allocated_budget
                 cost_data['start_date'] = start_date
                 cost_data.update(common_data)
-                cost_data['index_id'] = f"""{cost_data['start_date']}-{cost_data['Account'].lower()}-{self.azure_operations.cloud_name.lower()}"""
+                cost_data['index_id'] = f"""{cost_data['start_date']}-{cost_data['AccountId']}"""
                 cost_data['timestamp'] = datetime.datetime.strptime(cost_data['start_date'], '%Y-%m-%d')
                 cost_data['Month'] = datetime.datetime.strftime(cost_data['timestamp'], '%Y %b')
                 cost_data['filter_date'] = f"{cost_data['start_date']}-{cost_data['Month'].split()[-1]}"
-                cost_billing_data[cost_data['start_date']] = cost_data
+                cost_billing_data[index_id] = cost_data
             else:
-                cost_billing_data[start_date].update(cost_data)
+                cost_billing_data[index_id].update(cost_data)
             if start_date:
-                if 'Actual' not in cost_billing_data[start_date]:
-                    cost_billing_data[start_date]['Actual'] = 0
+                if 'Actual' not in cost_billing_data[index_id]:
+                    cost_billing_data[index_id]['Actual'] = 0
                 else:
-                    if 'Forecast' not in cost_billing_data[start_date]:
-                        cost_billing_data[start_date]['Forecast'] = 0
+                    if 'Forecast' not in cost_billing_data[index_id]:
+                        cost_billing_data[index_id]['Forecast'] = 0
         return cost_billing_data
 
     @logger_time_stamp
@@ -92,21 +126,50 @@ def collect_and_upload_cost_data(self):
         This method collect and upload the data
         @return:
         """
-        usage_data = self.cost_mgmt_operations.get_usage()
-        forecast_data = self.cost_mgmt_operations.get_forecast()
         cost_billing_data = {}
-        self.get_data_from_costs(cost_data_rows=usage_data.get('rows'), cost_data_columns=usage_data.get('columns'), cost_type='Actual', cost_billing_data=cost_billing_data)
-        self.get_data_from_costs(cost_data_rows=forecast_data.get('rows'), cost_data_columns=forecast_data.get('columns'), cost_type='Forecast', cost_billing_data=cost_billing_data)
-        cost_billing_data = dict(sorted(cost_billing_data.items()))
-        uploaded_cost_data = list(cost_billing_data.values())
-        self.elastic_upload.es_upload_data(items=uploaded_cost_data, set_index='index_id')
-        upload_data = {
-            'cloud_alias_name': f'{self.azure_operations.account_name}-azure',
-            'cloud_name': 'AZURE',
-            'cloud_data': uploaded_cost_data
-        }
-        self.update_to_gsheet.update_data(cloud_data=upload_data)
-        return uploaded_cost_data
+        if not self.__total_account:
+            usage_data = self.cost_mgmt_operations.get_usage(scope=self.azure_operations.scope)
+            forecast_data = self.cost_mgmt_operations.get_forecast(scope=self.azure_operations.scope)
+            if usage_data:
+                self.get_data_from_costs(cost_data_rows=usage_data.get('rows'), cost_data_columns=usage_data.get('columns'), cost_type='Actual', cost_billing_data=cost_billing_data)
+            if forecast_data:
+                self.get_data_from_costs(cost_data_rows=forecast_data.get('rows'), cost_data_columns=forecast_data.get('columns'), cost_type='Forecast', cost_billing_data=cost_billing_data, subscription_id=self.azure_operations.subscription_id)
+        else:
+            for profile in self.azure_operations.get_billing_profiles():
+                profile = profile.as_dict()
+                scope = profile.get('id')
+                self.__common_data['AccountProfile'] = profile.get('display_name')
+                cost_center = int(profile.get('display_name').split('(CC ')[-1].split(')')[0])
+                self.__common_data['CostCenter'] = cost_center
+                filters = {
+                    'grouping': [QueryGrouping(type='Dimension', name='SubscriptionName'), QueryGrouping(type='Dimension', name='SubscriptionId')]
+                }
+                usage_data = self.cost_mgmt_operations.get_usage(scope=scope, **filters)
+                subscription_ids = []
+                if usage_data.get('rows'):
+                    subscription_ids = [[id[3], id[2]] for id in usage_data.get('rows')]
+                    self.get_data_from_costs(cost_data_rows=usage_data.get('rows'),
+                                             cost_data_columns=usage_data.get('columns'), cost_type='Actual',
+                                             cost_billing_data=cost_billing_data)
+                for subscription in subscription_ids:
+                    query_filter = {'filter': QueryFilter(dimensions=QueryComparisonExpression(name='SubscriptionId', operator='in',
+                                                                                     values=[subscription[0]]))}
+                    forecast_data = self.cost_mgmt_operations.get_forecast(scope=scope, **query_filter)
+                    if forecast_data and forecast_data.get('rows'):
+                        self.get_data_from_costs(cost_data_rows=forecast_data.get('rows'), cost_data_columns=forecast_data.get('columns'), cost_type='Forecast', cost_billing_data=cost_billing_data, subscription_id=subscription[0], account_name=subscription[1], cost_center=cost_center)
+                self.__common_data = self.get_common_data()
+        if cost_billing_data:
+            cost_billing_data = dict(sorted(cost_billing_data.items()))
+            uploaded_cost_data = list(cost_billing_data.values())
+            self.elastic_upload.es_upload_data(items=uploaded_cost_data, set_index='index_id')
+            if not self.__total_account:
+                upload_data = {
+                    'cloud_alias_name': f'{self.azure_operations.account_name}-azure',
+                    'cloud_name': 'AZURE',
+                    'cloud_data': uploaded_cost_data
+                }
+                self.update_to_gsheet.update_data(cloud_data=upload_data)
+        return list(cost_billing_data.values())
 
     def run(self):
         """

jenkins/clouds/azure/daily/cost_reports/Jenkinsfile

@@ -9,6 +9,7 @@ pipeline {
     environment {
         AZURE_CLIENT_SECRET = credentials('cloud-governance-azure-client-secret')
         AZURE_TENANT_ID = credentials('cloud-governance-azure-tenant-id')
+        AZURE_ACCOUNT_ID = credentials('cloud-governance-azure-account-id')
         AZURE_CLIENT_ID = credentials('cloud-governance-azure-client-id')
         ES_HOST = credentials('cloud-governance-es-host')
         ES_PORT = credentials('cloud-governance-es-port')