diff --git a/src/spaceone/inventory/conf/global_conf.py b/src/spaceone/inventory/conf/global_conf.py index a0f6b137..2cf0c767 100644 --- a/src/spaceone/inventory/conf/global_conf.py +++ b/src/spaceone/inventory/conf/global_conf.py @@ -25,7 +25,7 @@ RESOURCE_TERMINATION_TIME = 3 * 30 # 3 Months DEFAULT_DELETE_POLICIES = { 'inventory.CloudService': 48, # 48 Hours - 'inventory.CloudServiceType': 48, # 48 Hours + 'inventory.CloudServiceType': 3 * 30 * 24, # 3 Months 'inventory.Region': 48, # 48 Hours } DEFAULT_DISCONNECTED_STATE_DELETE_POLICY = 3 # 3 Count diff --git a/src/spaceone/inventory/info/cloud_service_query_set_info.py b/src/spaceone/inventory/info/cloud_service_query_set_info.py index 7b6571c9..a9acb3bf 100644 --- a/src/spaceone/inventory/info/cloud_service_query_set_info.py +++ b/src/spaceone/inventory/info/cloud_service_query_set_info.py @@ -21,6 +21,8 @@ def CloudServiceQuerySetInfo(cloud_svc_query_set_vo: CloudServiceQuerySet, minim if not minimal: info.update({ 'query_options': change_analyze_query(cloud_svc_query_set_vo.query_options), + 'keys': cloud_svc_query_set_vo.keys, + 'additional_info_keys': cloud_svc_query_set_vo.additional_info_keys, 'unit': change_struct_type(cloud_svc_query_set_vo.unit), 'tags': change_struct_type(cloud_svc_query_set_vo.tags), 'domain_id': cloud_svc_query_set_vo.domain_id, diff --git a/src/spaceone/inventory/info/cloud_service_stats_info.py b/src/spaceone/inventory/info/cloud_service_stats_info.py index 9166534d..3a3a9ce6 100644 --- a/src/spaceone/inventory/info/cloud_service_stats_info.py +++ b/src/spaceone/inventory/info/cloud_service_stats_info.py @@ -9,19 +9,18 @@ def CloudServiceStatInfo(cloud_svc_stat_vo: CloudServiceStats, minimal=False): info = { - 'key': cloud_svc_stat_vo.key, - 'value': cloud_svc_stat_vo.value, - 'unit': cloud_svc_stat_vo.unit, + 'query_set_id': cloud_svc_stat_vo.query_set_id, + 'values': change_struct_type(cloud_svc_stat_vo.values), + 'unit': change_struct_type(cloud_svc_stat_vo.unit), 'provider': cloud_svc_stat_vo.provider, 'cloud_service_group': cloud_svc_stat_vo.cloud_service_group, 'cloud_service_type': cloud_svc_stat_vo.cloud_service_type, 'project_id': cloud_svc_stat_vo.project_id, - 'created_at': utils.datetime_to_iso8601(cloud_svc_stat_vo.created_at), + 'created_date': cloud_svc_stat_vo.created_date, } if not minimal: info.update({ - 'query_set_id': cloud_svc_stat_vo.query_set_id, 'region_code': cloud_svc_stat_vo.region_code, 'account': cloud_svc_stat_vo.account, 'additional_info': change_struct_type(cloud_svc_stat_vo.additional_info), diff --git a/src/spaceone/inventory/interface/grpc/v1/cloud_service_query_set.py b/src/spaceone/inventory/interface/grpc/v1/cloud_service_query_set.py index 3ea11426..39396bb0 100644 --- a/src/spaceone/inventory/interface/grpc/v1/cloud_service_query_set.py +++ b/src/spaceone/inventory/interface/grpc/v1/cloud_service_query_set.py @@ -33,6 +33,12 @@ def run(self, request, context): cloud_svc_query_set_service.run(params) return self.locator.get_info('EmptyInfo') + def test(self, request, context): + params, metadata = self.parse_request(request, context) + + with self.locator.get_service('CloudServiceQuerySetService', metadata) as cloud_svc_query_set_service: + return self.locator.get_info('AnalyzeInfo', cloud_svc_query_set_service.test(params)) + def enable(self, request, context): params, metadata = self.parse_request(request, context) diff --git a/src/spaceone/inventory/manager/cloud_service_query_set_manager.py b/src/spaceone/inventory/manager/cloud_service_query_set_manager.py index a2784747..972cc1a2 100644 --- a/src/spaceone/inventory/manager/cloud_service_query_set_manager.py +++ b/src/spaceone/inventory/manager/cloud_service_query_set_manager.py @@ -1,7 +1,7 @@ import copy import logging -import time from datetime import datetime +from dateutil.relativedelta import relativedelta from spaceone.core import cache, utils, queue from spaceone.core.manager import BaseManager @@ -59,6 +59,20 @@ def _rollback(cloud_svc_query_set_vo: CloudServiceQuerySet): params['query_hash'] = utils.dict_to_hash(params['query_options']) _LOGGER.debug(f'[create_cloud_service_query_set] create query set: {params["name"]}') + + keys, additional_info_keys = self._get_keys_from_query(params['query_options']) + params['keys'] = keys + params['additional_info_keys'] = additional_info_keys + + provider = params.get('provider') + cloud_service_group = params.get('cloud_service_group') + cloud_service_type = params.get('cloud_service_type') + + if provider and cloud_service_group and cloud_service_type: + params['ref_cloud_service_type'] = self._make_cloud_service_type_key(params['domain_id'], provider, + cloud_service_group, + cloud_service_type) + cloud_svc_query_set_vo: CloudServiceQuerySet = self.cloud_svc_query_set_model.create(params) self.transaction.add_rollback(_rollback, cloud_svc_query_set_vo) @@ -78,6 +92,9 @@ def _rollback(old_data): if 'query_options' in params: params['query_hash'] = utils.dict_to_hash(params['query_options']) + keys, additional_info_keys = self._get_keys_from_query(params['query_options']) + params['keys'] = keys + params['additional_info_keys'] = additional_info_keys _LOGGER.debug(f'[update_cloud_service_query_set_by_vo] update query set: {cloud_svc_query_set_vo.query_set_id}') @@ -137,14 +154,29 @@ def run_cloud_service_query_set(self, cloud_svc_query_set_vo: CloudServiceQueryS try: self._save_query_results(cloud_svc_query_set_vo, results, created_at) - self._delete_old_cloud_service_stats(cloud_svc_query_set_vo, created_at) - self._delete_old_monthly_cloud_service_stats(cloud_svc_query_set_vo, created_at) + self._delete_changed_cloud_service_stats(cloud_svc_query_set_vo, created_at) + self._delete_changed_monthly_cloud_service_stats(cloud_svc_query_set_vo, created_at) except Exception as e: _LOGGER.error(f'[run_cloud_service_query_set] Failed to save query result: {e}', exc_info=True) self._rollback_query_results(cloud_svc_query_set_vo, created_at) raise ERROR_CLOUD_SERVICE_QUERY_SET_RUN_FAILED(query_set_id=cloud_svc_query_set_vo.query_set_id) - self._remove_analyze_cache(cloud_svc_query_set_vo.domain_id) + self._update_status(cloud_svc_query_set_vo, created_at) + self._delete_invalid_cloud_service_stats(cloud_svc_query_set_vo) + self._delete_old_cloud_service_stats(cloud_svc_query_set_vo) + self._remove_analyze_cache(cloud_svc_query_set_vo.domain_id, cloud_svc_query_set_vo.query_set_id) + + def test_cloud_service_query_set(self, cloud_svc_query_set_vo: CloudServiceQuerySet): + if cloud_svc_query_set_vo.state == 'DISABLED': + raise ERROR_CLOUD_SERVICE_QUERY_SET_STATE(state=cloud_svc_query_set_vo.state) + + self.cloud_svc_stats_mgr: CloudServiceStatsManager = self.locator.get_manager('CloudServiceStatsManager') + + _LOGGER.debug(f'[test_cloud_service_query_set] test query set: {cloud_svc_query_set_vo.query_set_id} ' + f'({cloud_svc_query_set_vo.domain_id})') + return { + 'results': self._run_analyze_query(cloud_svc_query_set_vo) + } def _run_analyze_query(self, cloud_svc_query_set_vo: CloudServiceQuerySet): cloud_svc_mgr: CloudServiceManager = self.locator.get_manager('CloudServiceManager') @@ -167,85 +199,129 @@ def _run_analyze_query(self, cloud_svc_query_set_vo: CloudServiceQuerySet): response = cloud_svc_mgr.analyze_cloud_services(analyze_query) return response.get('results', []) - def _delete_old_cloud_service_stats(self, cloud_svc_query_set_vo: CloudServiceQuerySet, created_at): + def _delete_invalid_cloud_service_stats(self, cloud_svc_query_set_vo: CloudServiceQuerySet): domain_id = cloud_svc_query_set_vo.domain_id query_set_id = cloud_svc_query_set_vo.query_set_id - created_date = created_at.strftime('%Y-%m-%d') - timestamp = int(time.mktime(created_at.timetuple())) - query = { + cloud_stats_vos = self.cloud_svc_stats_mgr.filter_cloud_service_stats( + query_set_id=query_set_id, domain_id=domain_id, status='IN_PROGRESS') + + if cloud_stats_vos.count() > 0: + _LOGGER.debug(f'[_delete_invalid_cloud_service_stats] delete stats count: {cloud_stats_vos.count()}') + cloud_stats_vos.delete() + + monthly_stats_vos = self.cloud_svc_stats_mgr.filter_monthly_cloud_service_stats( + query_set_id=query_set_id, domain_id=domain_id, status='IN_PROGRESS') + + if monthly_stats_vos.count() > 0: + _LOGGER.debug(f'[_delete_invalid_cloud_service_stats] delete monthly stats count: {monthly_stats_vos.count()}') + monthly_stats_vos.delete() + + def _delete_old_cloud_service_stats(self, cloud_svc_query_set_vo: CloudServiceQuerySet): + now = datetime.utcnow().date() + query_set_id = cloud_svc_query_set_vo.query_set_id + domain_id = cloud_svc_query_set_vo.domain_id + old_created_month = (now - relativedelta(months=12)).strftime('%Y-%m') + old_created_year = (now - relativedelta(months=36)).strftime('%Y') + + delete_query = { 'filter': [ - {'k': 'domain_id', 'v': domain_id, 'o': 'eq'}, + {'k': 'created_month', 'v': old_created_month, 'o': 'lt'}, {'k': 'query_set_id', 'v': query_set_id, 'o': 'eq'}, - {'k': 'created_date', 'v': created_date, 'o': 'eq'}, - {'k': 'timestamp', 'v': timestamp, 'o': 'not'} + {'k': 'domain_id', 'v': domain_id, 'o': 'eq'} ] } - _LOGGER.debug(f'[_delete_old_cloud_service_stats] Query: {query}') - cloud_stats_vos, total_count = self.cloud_svc_stats_mgr.list_cloud_service_stats(query) - cloud_stats_vos.delete() + stats_vos, total_count = self.cloud_svc_stats_mgr.list_cloud_service_stats(delete_query) - def _delete_old_monthly_cloud_service_stats(self, cloud_svc_query_set_vo: CloudServiceQuerySet, created_at): - domain_id = cloud_svc_query_set_vo.domain_id - query_set_id = cloud_svc_query_set_vo.query_set_id - created_month = created_at.strftime('%Y-%m') - timestamp = int(time.mktime(created_at.timetuple())) + if total_count > 0: + _LOGGER.debug(f'[delete_old_cloud_service_stats] delete stats count: {total_count}') + stats_vos.delete() - query = { + monthly_delete_query = { 'filter': [ - {'k': 'domain_id', 'v': domain_id, 'o': 'eq'}, + {'k': 'created_year', 'v': old_created_year, 'o': 'lt'}, {'k': 'query_set_id', 'v': query_set_id, 'o': 'eq'}, - {'k': 'created_month', 'v': created_month, 'o': 'eq'}, - {'k': 'timestamp', 'v': timestamp, 'o': 'not'} + {'k': 'domain_id', 'v': domain_id, 'o': 'eq'} ] } - _LOGGER.debug(f'[_delete_old_monthly_cloud_service_stats] Query: {query}') - monthly_stats_vos, total_count = self.cloud_svc_stats_mgr.list_monthly_cloud_service_stats(query) + monthly_stats_vos, total_count = self.cloud_svc_stats_mgr.list_monthly_cloud_service_stats(monthly_delete_query) + + if total_count > 0: + _LOGGER.debug(f'[_delete_old_cloud_service_stats] delete monthly stats count: {total_count}') + monthly_stats_vos.delete() + + def _update_status(self, cloud_svc_query_set_vo: CloudServiceQuerySet, created_at): + domain_id = cloud_svc_query_set_vo.domain_id + query_set_id = cloud_svc_query_set_vo.query_set_id + created_date = created_at.strftime('%Y-%m-%d') + created_month = created_at.strftime('%Y-%m') + + cloud_stats_vos = self.cloud_svc_stats_mgr.filter_cloud_service_stats( + query_set_id=query_set_id, domain_id=domain_id, created_date=created_date, status='IN_PROGRESS') + cloud_stats_vos.update({'status': 'DONE'}) + + monthly_stats_vos = self.cloud_svc_stats_mgr.filter_monthly_cloud_service_stats( + query_set_id=query_set_id, domain_id=domain_id, created_month=created_month, status='IN_PROGRESS') + monthly_stats_vos.update({'status': 'DONE'}) + + def _delete_changed_cloud_service_stats(self, cloud_svc_query_set_vo: CloudServiceQuerySet, created_at): + domain_id = cloud_svc_query_set_vo.domain_id + query_set_id = cloud_svc_query_set_vo.query_set_id + created_date = created_at.strftime('%Y-%m-%d') + + cloud_stats_vos = self.cloud_svc_stats_mgr.filter_cloud_service_stats( + query_set_id=query_set_id, domain_id=domain_id, created_date=created_date, status='DONE') + + _LOGGER.debug(f'[_delete_old_cloud_service_stats] delete count: {cloud_stats_vos.count()}') + cloud_stats_vos.delete() + + def _delete_changed_monthly_cloud_service_stats(self, cloud_svc_query_set_vo: CloudServiceQuerySet, created_at): + domain_id = cloud_svc_query_set_vo.domain_id + query_set_id = cloud_svc_query_set_vo.query_set_id + created_month = created_at.strftime('%Y-%m') + + monthly_stats_vos = self.cloud_svc_stats_mgr.filter_monthly_cloud_service_stats( + query_set_id=query_set_id, domain_id=domain_id, created_month=created_month, status='DONE') + + _LOGGER.debug(f'[_delete_old_monthly_cloud_service_stats] delete count: {monthly_stats_vos.count()}') monthly_stats_vos.delete() def _rollback_query_results(self, cloud_svc_query_set_vo: CloudServiceQuerySet, created_at): _LOGGER.debug(f'[_rollback_query_results] Rollback Query Results: {cloud_svc_query_set_vo.query_set_id}') query_set_id = cloud_svc_query_set_vo.query_set_id domain_id = cloud_svc_query_set_vo.domain_id - timestamp = int(time.mktime(created_at.timetuple())) cloud_service_stats_vo = self.cloud_svc_stats_mgr.filter_cloud_service_stats( - query_set_id=query_set_id, domain_id=domain_id, timestamp=timestamp) + query_set_id=query_set_id, domain_id=domain_id, created_date=created_at.strftime('%Y-%m-%d'), + status='IN_PROGRESS') cloud_service_stats_vo.delete() monthly_stats_vo = self.cloud_svc_stats_mgr.filter_monthly_cloud_service_stats( - query_set_id=query_set_id, domain_id=domain_id, timestamp=timestamp) + query_set_id=query_set_id, domain_id=domain_id, created_month=created_at.strftime('%Y-%m'), + status='IN_PROGRESS') monthly_stats_vo.delete() - def _save_query_results(self, cloud_svc_query_set_vo: CloudServiceQuerySet, results, created_at): - query_set_id = cloud_svc_query_set_vo.query_set_id - domain_id = cloud_svc_query_set_vo.domain_id - analyze_query = cloud_svc_query_set_vo.query_options - unit = cloud_svc_query_set_vo.unit - original_group_by = set(analyze_query.get('group_by', [])) - set(_DEFAULT_GROUP_BY) - timestamp = int(time.mktime(created_at.timetuple())) - + def _save_query_results(self, query_set_vo: CloudServiceQuerySet, results, created_at): for result in results: - self._save_query_result(result, query_set_id, original_group_by, unit, domain_id, created_at, timestamp) + self._save_query_result(result, query_set_vo, created_at) - @staticmethod - def _remove_analyze_cache(domain_id): - cache.delete_pattern(f'inventory:cloud-service-stats:{domain_id}:*') - cache.delete_pattern(f'inventory:monthly-cloud-service-stats:{domain_id}:*') - - def _save_query_result(self, result, query_set_id, original_group_by, unit, domain_id, created_at, timestamp): + def _save_query_result(self, result, query_set_vo: CloudServiceQuerySet, created_at): provider = result['provider'] cloud_service_group = result['cloud_service_group'] cloud_service_type = result['cloud_service_type'] region_code = result.get('region_code') + query_set_id = query_set_vo.query_set_id + domain_id = query_set_vo.domain_id ref_cloud_service_type = self._make_cloud_service_type_key(domain_id, provider, cloud_service_group, cloud_service_type) ref_region = self._make_region_key(domain_id, provider, region_code) data = { 'query_set_id': query_set_id, + 'values': {}, + 'unit': {}, 'provider': provider, 'cloud_service_group': cloud_service_group, 'cloud_service_type': cloud_service_type, @@ -257,27 +333,29 @@ def _save_query_result(self, result, query_set_id, original_group_by, unit, doma 'domain_id': domain_id, 'additional_info': {}, 'created_at': created_at, - 'timestamp': timestamp, 'created_year': created_at.strftime('%Y'), 'created_month': created_at.strftime('%Y-%m'), 'created_date': created_at.strftime('%Y-%m-%d') } - group_by_keys = [] - for key in original_group_by: - group_by_key = key.rsplit('.', 1)[-1] - data['additional_info'][group_by_key] = result.get(group_by_key) - group_by_keys.append(group_by_key) + for key in query_set_vo.keys: + data['values'][key] = result.get(key, 0) + + if key in query_set_vo.unit: + data['unit'][key] = query_set_vo.unit[key] + else: + data['unit'][key] = 'Count' + + for key in query_set_vo.additional_info_keys: + data['additional_info'][key] = result.get(key) - field_keys = set(result.keys()) - set(group_by_keys) - set(_DEFAULT_GROUP_BY) - for key in field_keys: - field_data = copy.deepcopy(data) - field_data['key'] = key - field_data['value'] = result.get(key) - field_data['unit'] = unit.get(key, 'Count') + self.cloud_svc_stats_mgr.create_cloud_service_stats(data, False) + self.cloud_svc_stats_mgr.create_monthly_cloud_service_stats(data, False) - self.cloud_svc_stats_mgr.create_cloud_service_stats(field_data, False) - self.cloud_svc_stats_mgr.create_monthly_cloud_service_stats(field_data, False) + @staticmethod + def _remove_analyze_cache(domain_id, query_set_id): + cache.delete_pattern(f'inventory:cloud-service-stats:*:{domain_id}:{query_set_id}:*') + cache.delete_pattern(f'inventory:monthly-cloud-service-stats:*:{domain_id}:{query_set_id}:*') @staticmethod def _make_cloud_service_type_key(domain_id, provider, cloud_service_group, cloud_service_type): @@ -316,3 +394,12 @@ def _make_query_filter(provider=None, cloud_service_group=None, cloud_service_ty }) return _filter + + @staticmethod + def _get_keys_from_query(query): + keys = query.get('fields', {}).keys() + additional_info_keys = [] + for key in query.get('group_by', []): + if key not in _DEFAULT_GROUP_BY: + additional_info_keys.append(key.split('.')[-1:][0]) + return keys, additional_info_keys diff --git a/src/spaceone/inventory/manager/cloud_service_stats_manager.py b/src/spaceone/inventory/manager/cloud_service_stats_manager.py index a6355d2b..b8ad7c67 100644 --- a/src/spaceone/inventory/manager/cloud_service_stats_manager.py +++ b/src/spaceone/inventory/manager/cloud_service_stats_manager.py @@ -58,47 +58,79 @@ def filter_monthly_cloud_service_stats(self, **conditions): return self.monthly_stats_model.filter(**conditions) def list_cloud_service_stats(self, query): + query = self._append_status_filter(query) + return self.cloud_svc_stats_model.query(**query) def list_monthly_cloud_service_stats(self, query): + query = self._append_status_filter(query) + return self.monthly_stats_model.query(**query) - # @cache.cacheable(key='inventory:cloud-service-stats:{domain_id}:{query_hash}', expire=3600 * 24) - def analyze_cloud_service_stats_with_cache(self, query, query_hash, domain_id, target='SECONDARY_PREFERRED'): + def stat_cloud_service_stats(self, query): + query = self._append_status_filter(query) + + return self.cloud_svc_stats_model.stat(**query) + + def stat_monthly_cloud_service_stats(self, query): + query = self._append_status_filter(query) + + return self.monthly_stats_model.stat(**query) + + def analyze_cloud_service_stats(self, query, target='SECONDARY_PREFERRED'): query['target'] = target query['date_field'] = 'created_date' + query['date_field_format'] = '%Y-%m-%d' + _LOGGER.debug(f'[analyze_cloud_service_stats] query: {query}') return self.cloud_svc_stats_model.analyze(**query) - # @cache.cacheable(key='inventory:monthly-cloud-service-stats:{domain_id}:{query_hash}', expire=3600 * 24) - def analyze_monthly_cloud_service_stats_with_cache(self, query, query_hash, domain_id, target='SECONDARY_PREFERRED'): + def analyze_monthly_cloud_service_stats(self, query, target='SECONDARY_PREFERRED'): query['target'] = target query['date_field'] = 'created_month' + query['date_field_format'] = '%Y-%m' + _LOGGER.debug(f'[analyze_monthly_cloud_service_stats] query: {query}') + return self.monthly_stats_model.analyze(**query) + + def analyze_yearly_cloud_service_stats(self, query, target='SECONDARY_PREFERRED'): + query['target'] = target + query['date_field'] = 'created_year' + query['date_field_format'] = '%Y' + _LOGGER.debug(f'[analyze_yearly_cloud_service_stats] query: {query}') return self.monthly_stats_model.analyze(**query) - def analyze_cloud_service_stats(self, query, domain_id): + @cache.cacheable(key='inventory:cloud-service-stats:daily:{domain_id}:{query_set_id}:{query_hash}', expire=3600 * 24) + def analyze_cloud_service_stats_with_cache(self, query, query_hash, domain_id, query_set_id, target='SECONDARY_PREFERRED'): + return self.analyze_cloud_service_stats(query, target) + + @cache.cacheable(key='analyze-costs:monthly:{domain_id}:{query_set_id}:{query_hash}', expire=3600 * 24) + def analyze_monthly_cloud_service_stats_with_cache(self, query, query_hash, domain_id, query_set_id, + target='SECONDARY_PREFERRED'): + return self.analyze_monthly_cloud_service_stats(query, target) + + @cache.cacheable(key='analyze-costs:yearly:{domain_id}:{query_set_id}:{query_hash}', expire=3600 * 24) + def analyze_yearly_cloud_service_stats_with_cache(self, query, query_hash, domain_id, query_set_id, + target='SECONDARY_PREFERRED'): + return self.analyze_yearly_cloud_service_stats(query, target) + + def analyze_cloud_service_stats_by_granularity(self, query, domain_id, query_set_id): + self._check_date_range(query) granularity = query['granularity'] - start, end = self._parse_date_range(query) - del query['start'] - del query['end'] # Save query history to speed up data loading - self._create_analyze_query_history(query, granularity, start, end, domain_id) - - # Add date range filter by granularity - query = self._add_date_range_filter(query, granularity, start, end) query_hash = utils.dict_to_hash(query) + self.create_cloud_service_stats_query_history(query, query_hash, domain_id, query_set_id) - if self._is_monthly_cost(granularity, start, end): - return self.analyze_monthly_cloud_service_stats_with_cache(query, query_hash, domain_id) + if granularity == 'DAILY': + response = self.analyze_cloud_service_stats_with_cache(query, query_hash, domain_id, query_set_id) + elif granularity == 'MONTHLY': + response = self.analyze_monthly_cloud_service_stats_with_cache(query, query_hash, domain_id, query_set_id) else: - return self.analyze_cloud_service_stats_with_cache(query, query_hash, domain_id) + response = self.analyze_yearly_cloud_service_stats_with_cache(query, query_hash, domain_id, query_set_id) - def stat_cloud_service_stats(self, query): - return self.cloud_svc_stats_model.stat(**query) + return response - @cache.cacheable(key='inventory:cloud-service-stats-history:{domain_id}:{query_hash}', expire=600) - def create_cloud_service_stats_query_history(self, domain_id, query, query_hash, granularity=None, - start=None, end=None): + @cache.cacheable(key='inventory:stats-query-history:{domain_id}:{query_set_id}:{query_hash}', expire=600) + def create_cloud_service_stats_query_history(self, query, query_hash, domain_id, query_set_id): def _rollback(history_vo): _LOGGER.info(f'[create_cloud_service_stats_query_history._rollback] Delete query history: {query_hash}') history_vo.delete() @@ -110,119 +142,101 @@ def _rollback(history_vo): history_vo = history_model.create({ 'query_hash': query_hash, 'query_options': copy.deepcopy(query), - 'granularity': granularity, - 'start': start, - 'end': end, + 'query_set_id': query_set_id, 'domain_id': domain_id }) self.transaction.add_rollback(_rollback, history_vo) else: - history_vos[0].update({ - 'start': start, - 'end': end - }) + history_vos[0].update({}) - def _create_analyze_query_history(self, query, granularity, start, end, domain_id): - analyze_query = { - 'group_by': query.get('group_by'), - 'field_group': query.get('field_group'), - 'filter': query.get('filter'), - 'filter_or': query.get('filter_or'), - 'page': query.get('page'), - 'sort': query.get('sort'), - 'fields': query.get('fields'), - 'select': query.get('select'), - } - query_hash = utils.dict_to_hash(analyze_query) - self.create_cloud_service_stats_query_history(domain_id, analyze_query, query_hash, granularity, start, end) - - def _parse_date_range(self, query): + def _check_date_range(self, query): start_str = query.get('start') end_str = query.get('end') granularity = query.get('granularity') - start = self._parse_start_time(start_str) - end = self._parse_end_time(end_str) + start = self._parse_start_time(start_str, granularity) + end = self._parse_end_time(end_str, granularity) + now = datetime.utcnow().date() + + if len(start_str) != len(end_str): + raise ERROR_INVALID_DATE_RANGE(start=start_str, end=end_str, + reason='Start date and end date must be the same format.') if start >= end: raise ERROR_INVALID_DATE_RANGE(start=start_str, end=end_str, reason='End date must be greater than start date.') - if granularity in ['ACCUMULATED', 'MONTHLY']: - if start + relativedelta(months=12) < end: + if granularity == 'DAILY': + if start + relativedelta(months=1) < end: raise ERROR_INVALID_DATE_RANGE(start=start_str, end=end_str, - reason='Request up to a maximum of 12 months.') - elif granularity == 'DAILY': - if start + relativedelta(days=31) < end: + reason='Request up to a maximum of 1 month.') + + if start + relativedelta(months=12) < now.replace(day=1): raise ERROR_INVALID_DATE_RANGE(start=start_str, end=end_str, - reason='Request up to a maximum of 31 days.') + reason='For DAILY, you cannot request data older than 1 year.') - if granularity == 'MONTHLY' and (start.day != 1 or end.day != 1): - raise ERROR_INVALID_DATE_RANGE(start=start_str, end=end_str, - reason='If the granularity is MONTHLY, ' - 'the request must be in YYYY-MM format.') + elif granularity == 'MONTHLY': + if start + relativedelta(months=12) < end: + raise ERROR_INVALID_DATE_RANGE(start=start_str, end=end_str, + reason='Request up to a maximum of 12 months.') - return start, end + if start + relativedelta(months=36) < now.replace(day=1): + raise ERROR_INVALID_DATE_RANGE(start=start_str, end=end_str, + reason='For MONTHLY, you cannot request data older than 3 years.') + elif granularity == 'YEARLY': + if start + relativedelta(years=3) < now.replace(month=1, day=1): + raise ERROR_INVALID_DATE_RANGE(start=start_str, end=end_str, + reason='For YEARLY, you cannot request data older than 3 years.') - def _parse_start_time(self, date_str): - return self._convert_date_from_string(date_str.strip(), 'start') + def _parse_start_time(self, date_str, granularity): + return self._convert_date_from_string(date_str.strip(), 'start', granularity) - def _parse_end_time(self, date_str): - date = self._convert_date_from_string(date_str.strip(), 'end') + def _parse_end_time(self, date_str, granularity): + end = self._convert_date_from_string(date_str.strip(), 'end', granularity) - if len(date_str) == 7: - return date + relativedelta(months=1) + if granularity == 'YEARLY': + return end + relativedelta(years=1) + elif granularity == 'MONTHLY': + return end + relativedelta(months=1) else: - return date + relativedelta(days=1) + return end + relativedelta(days=1) @staticmethod - def _convert_date_from_string(date_str, key): - if len(date_str) == 7: - # Month (YYYY-MM) - date_format = '%Y-%m' + def _convert_date_from_string(date_str, key, granularity): + if granularity == 'YEARLY': + date_format = '%Y' + date_type = 'YYYY' + elif granularity == 'MONTHLY': + if len(date_str) == 4: + date_format = '%Y' + date_type = 'YYYY' + else: + date_format = '%Y-%m' + date_type = 'YYYY-MM' else: - # Date (YYYY-MM-DD) - date_format = '%Y-%m-%d' + if len(date_str) == 4: + date_format = '%Y' + date_type = 'YYYY' + elif len(date_str) == 7: + date_format = '%Y-%m' + date_type = 'YYYY-MM' + else: + date_format = '%Y-%m-%d' + date_type = 'YYYY-MM-DD' try: return datetime.strptime(date_str, date_format).date() except Exception as e: - raise ERROR_INVALID_PARAMETER_TYPE(key=key, type=date_format) - - def _add_date_range_filter(self, query, granularity, start: date, end: date): - query['filter'] = query.get('filter') or [] - - if self._is_monthly_cost(granularity, start, end): - query['filter'].append({ - 'k': 'created_month', - 'v': start.strftime('%Y-%m'), - 'o': 'gte' - }) - - query['filter'].append({ - 'k': 'created_month', - 'v': end.strftime('%Y-%m'), - 'o': 'lt' - }) - else: - query['filter'].append({ - 'k': 'created_date', - 'v': start.strftime('%Y-%m-%d'), - 'o': 'gte' - }) - - query['filter'].append({ - 'k': 'created_date', - 'v': end.strftime('%Y-%m-%d'), - 'o': 'lt' - }) - - return query + raise ERROR_INVALID_PARAMETER_TYPE(key=key, type=date_type) @staticmethod - def _is_monthly_cost(granularity, start, end): - if granularity in ['ACCUMULATED', 'MONTHLY'] and start.day == 1 and end.day == 1: - return True - else: - return False + def _append_status_filter(query): + query_filter = query.get('filter', []) + query_filter.append({ + 'k': 'status', + 'v': 'DONE', + 'o': 'eq' + }) + query['filter'] = query_filter + return query diff --git a/src/spaceone/inventory/manager/cloud_service_type_manager.py b/src/spaceone/inventory/manager/cloud_service_type_manager.py index 16b9402a..eac32ad9 100644 --- a/src/spaceone/inventory/manager/cloud_service_type_manager.py +++ b/src/spaceone/inventory/manager/cloud_service_type_manager.py @@ -34,9 +34,8 @@ def _rollback(cloud_svc_type_vo): return cloud_svc_type_vo def update_cloud_service_type(self, params): - return self.update_cloud_service_type_by_vo(params, - self.get_cloud_service_type(params['cloud_service_type_id'], - params['domain_id'])) + cloud_service_type_vo = self.get_cloud_service_type(params['cloud_service_type_id'], params['domain_id']) + return self.update_cloud_service_type_by_vo(params, cloud_service_type_vo) def update_cloud_service_type_by_vo(self, params, cloud_svc_type_vo): def _rollback(old_data): @@ -50,7 +49,11 @@ def _rollback(old_data): return cloud_svc_type_vo.update(params) def delete_cloud_service_type(self, cloud_service_type_id, domain_id): - self.delete_cloud_service_type_by_vo(self.get_cloud_service_type(cloud_service_type_id, domain_id)) + cloud_svc_type_vo = self.get_cloud_service_type(cloud_service_type_id, domain_id) + query_set_vos = self._filter_cloud_service_query_sets(cloud_svc_type_vo) + query_set_vos.delete() + + self.delete_cloud_service_type_by_vo(cloud_svc_type_vo) def get_cloud_service_type(self, cloud_service_type_id, domain_id, only=None): return self.cloud_svc_type_model.get(cloud_service_type_id=cloud_service_type_id, domain_id=domain_id, diff --git a/src/spaceone/inventory/model/cloud_service_query_set_model.py b/src/spaceone/inventory/model/cloud_service_query_set_model.py index 35c1e400..913e2547 100644 --- a/src/spaceone/inventory/model/cloud_service_query_set_model.py +++ b/src/spaceone/inventory/model/cloud_service_query_set_model.py @@ -1,6 +1,7 @@ from mongoengine import * from spaceone.core.model.mongo_model import MongoModel +from spaceone.inventory.model.cloud_service_type_model import CloudServiceType class CloudServiceQuerySet(MongoModel): @@ -11,9 +12,12 @@ class CloudServiceQuerySet(MongoModel): query_hash = StringField(max_length=255) query_type = StringField(max_length=20, required=True, choices=('MANAGED', 'CUSTOM')) unit = DictField() + keys = ListField(StringField(max_length=255)) + additional_info_keys = ListField(StringField(max_length=255)) provider = StringField(max_length=255, default=None, null=True) cloud_service_group = StringField(max_length=255, default=None, null=True) cloud_service_type = StringField(max_length=255, default=None, null=True) + ref_cloud_service_type = StringField(max_length=255, default=None, null=True) tags = DictField() domain_id = StringField(max_length=40) created_at = DateTimeField(auto_now_add=True) @@ -26,6 +30,8 @@ class CloudServiceQuerySet(MongoModel): 'query_options', 'query_hash', 'unit', + 'keys', + 'additional_info_keys', 'tags', 'updated_at' ], @@ -43,6 +49,12 @@ class CloudServiceQuerySet(MongoModel): 'cloud_service_group', 'cloud_service_type' ], + 'reference_query_keys': { + 'ref_cloud_service_type': { + 'model': CloudServiceType, + 'foreign_key': 'ref_cloud_service_type' + } + }, 'indexes': [ 'name', 'state', diff --git a/src/spaceone/inventory/model/cloud_service_stats_model.py b/src/spaceone/inventory/model/cloud_service_stats_model.py index dcee9fdd..caf38b4b 100644 --- a/src/spaceone/inventory/model/cloud_service_stats_model.py +++ b/src/spaceone/inventory/model/cloud_service_stats_model.py @@ -7,9 +7,9 @@ class CloudServiceStats(MongoModel): query_set_id = StringField(max_length=40, required=True) - key = StringField(max_length=255, required=True) - value = FloatField(default=0) - unit = StringField(max_length=50, default='Count') + status = StringField(max_length=20, default='IN_PROGRESS', choices=('IN_PROGRESS', 'DONE')) + values = DictField() + unit = DictField() provider = StringField(max_length=255) cloud_service_group = StringField(max_length=255) cloud_service_type = StringField(max_length=255) @@ -20,23 +20,21 @@ class CloudServiceStats(MongoModel): additional_info = DictField() project_id = StringField(max_length=255, default=None, null=True) domain_id = StringField(max_length=40) - timestamp = IntField(required=True) - created_at = DateTimeField(required=True) - created_year = StringField(max_length=20) - created_month = StringField(max_length=20) - created_date = StringField(max_length=20) + created_year = StringField(max_length=20, required=True) + created_month = StringField(max_length=20, required=True) + created_date = StringField(max_length=20, required=True) meta = { 'updatable_fields': [], 'minimal_fields': [ - 'key', - 'value', + 'query_set_id', + 'values', 'unit', 'provider', 'cloud_service_group', 'cloud_service_type', 'project_id', - 'created_at' + 'created_date' ], 'change_query_keys': { 'user_projects': 'project_id' @@ -51,21 +49,17 @@ class CloudServiceStats(MongoModel): 'foreign_key': 'ref_region' } }, - 'ordering': [ - '-created_at' - ], 'indexes': [ { - "fields": ['domain_id', '-created_date', 'project_id', 'provider', 'cloud_service_group', - 'cloud_service_type', 'key', 'value'], - "name": "COMPOUND_INDEX_FOR_SEARCH_1" + "fields": ['domain_id', 'query_set_id', 'status', '-created_date', 'project_id'], + "name": "COMPOUND_INDEX_FOR_SEARCH" }, { - "fields": ['domain_id', '-created_date', 'project_id', 'ref_cloud_service_type', 'key', 'value'], - "name": "COMPOUND_INDEX_FOR_SEARCH_2" + "fields": ['domain_id', 'query_set_id', 'status', '-created_month'], + "name": "COMPOUND_INDEX_FOR_SYNC_JOB" }, { - "fields": ['domain_id', 'query_set_id', 'timestamp'], + "fields": ['domain_id', 'query_set_id'], "name": "COMPOUND_INDEX_FOR_DELETE" } ] @@ -74,19 +68,19 @@ class CloudServiceStats(MongoModel): class MonthlyCloudServiceStats(MongoModel): query_set_id = StringField(max_length=40, required=True) - key = StringField(max_length=255, required=True) - value = FloatField(default=0) - unit = StringField(max_length=50, default='Count') + status = StringField(max_length=20, default='IN_PROGRESS', choices=('IN_PROGRESS', 'DONE')) + values = DictField() + unit = DictField() provider = StringField(max_length=255) cloud_service_group = StringField(max_length=255) cloud_service_type = StringField(max_length=255) ref_cloud_service_type = StringField(max_length=255) region_code = StringField(max_length=255, default=None, null=True) + ref_region = StringField(max_length=255, default=None, null=True) account = StringField(max_length=255, default=None, null=True) additional_info = DictField() project_id = StringField(max_length=255, default=None, null=True) domain_id = StringField(max_length=40) - timestamp = IntField(required=True) created_year = StringField(max_length=20) created_month = StringField(max_length=20) @@ -105,21 +99,17 @@ class MonthlyCloudServiceStats(MongoModel): 'foreign_key': 'ref_region' } }, - 'ordering': [ - '-created_at' - ], 'indexes': [ { - "fields": ['domain_id', '-created_month', 'project_id', 'provider', 'cloud_service_group', - 'cloud_service_type', 'key', 'value'], - "name": "COMPOUND_INDEX_FOR_SEARCH_1" + "fields": ['domain_id', 'query_set_id', 'status', '-created_month', 'project_id'], + "name": "COMPOUND_INDEX_FOR_SEARCH" }, { - "fields": ['domain_id', '-created_month', 'project_id', 'ref_cloud_service_type', 'key', 'value'], - "name": "COMPOUND_INDEX_FOR_SEARCH_2" + "fields": ['domain_id', 'query_set_id', 'status', '-created_year'], + "name": "COMPOUND_INDEX_FOR_SYNC_JOB" }, { - "fields": ['domain_id', 'query_set_id', 'timestamp'], + "fields": ['domain_id', 'query_set_id'], "name": "COMPOUND_INDEX_FOR_DELETE" } ] @@ -129,21 +119,17 @@ class MonthlyCloudServiceStats(MongoModel): class CloudServiceStatsQueryHistory(MongoModel): query_hash = StringField(max_length=255) query_options = DictField(default={}) + query_set_id = StringField(max_length=40) domain_id = StringField(max_length=40) - granularity = StringField(max_length=40, default=None, null=True) - start = DateField(defulat=None, null=True) - end = DateField(default=None, null=True) updated_at = DateTimeField(auto_now=True) meta = { 'updatable_fields': [ - 'start', - 'end', 'updated_at' ], 'indexes': [ { - "fields": ['domain_id', 'query_hash'], + "fields": ['domain_id', 'query_set_id', 'query_hash'], "name": "COMPOUND_INDEX_FOR_SEARCH" }, ] diff --git a/src/spaceone/inventory/service/cloud_service_query_set_service.py b/src/spaceone/inventory/service/cloud_service_query_set_service.py index 242c6e26..af6321d5 100644 --- a/src/spaceone/inventory/service/cloud_service_query_set_service.py +++ b/src/spaceone/inventory/service/cloud_service_query_set_service.py @@ -118,6 +118,26 @@ def run(self, params): self.cloud_svc_query_set_mgr.run_cloud_service_query_set(cloud_svc_query_set_vo) + @transaction(append_meta={'authorization.scope': 'DOMAIN'}) + @check_required(['query_set_id', 'domain_id']) + def test(self, params): + """ Test Query Set Manually + Args: + params (dict): { + 'query_set_id': 'str', + 'domain_id': 'str' + } + + Returns: + values (list) : 'list of analyze data' + + """ + + cloud_svc_query_set_vo: CloudServiceQuerySet = \ + self.cloud_svc_query_set_mgr.get_cloud_service_query_set(params['query_set_id'], params['domain_id']) + + return self.cloud_svc_query_set_mgr.test_cloud_service_query_set(cloud_svc_query_set_vo) + @transaction(append_meta={'authorization.scope': 'DOMAIN'}) @check_required(['query_set_id', 'domain_id']) def enable(self, params): diff --git a/src/spaceone/inventory/service/cloud_service_service.py b/src/spaceone/inventory/service/cloud_service_service.py index 5e6666d4..801da7d8 100644 --- a/src/spaceone/inventory/service/cloud_service_service.py +++ b/src/spaceone/inventory/service/cloud_service_service.py @@ -292,8 +292,8 @@ def get(self, params): @transaction(append_meta={'authorization.scope': 'PROJECT'}) @check_required(['domain_id']) @append_query_filter(['cloud_service_id', 'name', 'state', 'account', 'instance_type', 'cloud_service_type', - 'cloud_service_group', 'provider', 'region_code', 'resource_group_id', 'project_id', - 'project_group_id', 'domain_id', 'user_projects', 'ip_address']) + 'cloud_service_group', 'provider', 'region_code', 'project_id', 'project_group_id', + 'domain_id', 'user_projects', 'ip_address']) @append_keyword_filter(_KEYWORD_FILTER) @set_query_page_limit(1000) def list(self, params): @@ -309,7 +309,6 @@ def list(self, params): 'cloud_service_group': 'str', 'provider': 'str', 'region_code': 'str', - 'resource_group_id': 'str', 'project_id': 'str', 'project_group_id': 'str', 'domain_id': 'str', @@ -324,13 +323,12 @@ def list(self, params): """ query = params.get('query', {}) - query = self._append_resource_group_filter(query, params['domain_id']) + # query = self._append_resource_group_filter(query, params['domain_id']) query = self._change_project_group_filter(query, params['domain_id']) query = self._change_filter_tags(query) query = self._change_only_tags(query) query = self._change_sort_tags(query) - print(query) return self.cloud_svc_mgr.list_cloud_services(query) @transaction(append_meta={'authorization.scope': 'PROJECT'}) @@ -348,12 +346,12 @@ def analyze(self, params): } Returns: - values (list) : 'list of statistics data' + results (list) : 'list of analyze data' """ query = params.get('query', {}) - query = self._append_resource_group_filter(query, params['domain_id']) + # query = self._append_resource_group_filter(query, params['domain_id']) query = self._change_project_group_filter(query, params['domain_id']) query = self._change_filter_tags(query) @@ -373,12 +371,12 @@ def stat(self, params): } Returns: - values (list) : 'list of statistics data' + results (list) : 'list of statistics data' """ query = params.get('query', {}) - query = self._append_resource_group_filter(query, params['domain_id']) + # query = self._append_resource_group_filter(query, params['domain_id']) query = self._change_project_group_filter(query, params['domain_id']) query = self._change_filter_tags(query) query = self._change_distinct_tags(query) diff --git a/src/spaceone/inventory/service/cloud_service_stats_service.py b/src/spaceone/inventory/service/cloud_service_stats_service.py index d95f9085..34ef0e1e 100644 --- a/src/spaceone/inventory/service/cloud_service_stats_service.py +++ b/src/spaceone/inventory/service/cloud_service_stats_service.py @@ -18,8 +18,8 @@ def __init__(self, *args, **kwargs): self.cloud_svc_stats_mgr: CloudServiceStatsManager = self.locator.get_manager('CloudServiceStatsManager') @transaction(append_meta={'authorization.scope': 'PROJECT'}) - @check_required(['domain_id']) - @append_query_filter(['query_set_id', 'key', 'provider', 'cloud_service_group', 'cloud_service_type', + @check_required(['query_set_id', 'domain_id']) + @append_query_filter(['query_set_id', 'provider', 'cloud_service_group', 'cloud_service_type', 'region_code', 'account', 'project_id', 'domain_id']) @append_keyword_filter(_KEYWORD_FILTER) @set_query_page_limit(1000) @@ -28,7 +28,6 @@ def list(self, params): Args: params (dict): { 'query_set_id': 'str', - 'key': 'str', 'provider': 'str', 'cloud_service_group': 'str', 'cloud_service_type': 'str', @@ -51,14 +50,16 @@ def list(self, params): return self.cloud_svc_stats_mgr.list_cloud_service_stats(query) @transaction(append_meta={'authorization.scope': 'PROJECT'}) - @check_required(['query', 'query.granularity', 'query.start', 'query.end', 'query.fields', 'domain_id']) - @append_query_filter(['domain_id']) + @check_required(['query', 'query.granularity', 'query.start', 'query.end', 'query.fields', + 'query_set_id', 'domain_id']) + @append_query_filter(['query_set_id', 'domain_id']) @append_keyword_filter(_KEYWORD_FILTER) @set_query_page_limit(1000) def analyze(self, params): """ Analyze Cloud Service Statistics Data Args: params (dict): { + 'query_set_id': 'str', 'domain_id': 'str', 'query': 'dict (spaceone.api.core.v1.TimeSeriesAnalyzeQuery)', 'user_projects': 'list', // from meta @@ -69,20 +70,22 @@ def analyze(self, params): """ domain_id = params['domain_id'] + query_set_id = params['query_set_id'] query = params.get('query', {}) query = self._change_project_group_filter(query, domain_id) - return self.cloud_svc_stats_mgr.analyze_cloud_service_stats(query, domain_id) + return self.cloud_svc_stats_mgr.analyze_cloud_service_stats_by_granularity(query, domain_id, query_set_id) @transaction(append_meta={'authorization.scope': 'PROJECT'}) @check_required(['query', 'domain_id']) - @append_query_filter(['domain_id']) + @append_query_filter(['query_set_id', 'domain_id']) @append_keyword_filter(_KEYWORD_FILTER) def stat(self, params): """ Get Cloud Service Statistics Data Args: params (dict): { + 'query_set_id': 'str', 'domain_id': 'str', 'query': 'dict (spaceone.api.core.v1.StatisticsQuery)', 'user_projects': 'list', // from meta