diff --git a/src/spaceone/inventory/conf/global_conf.py b/src/spaceone/inventory/conf/global_conf.py index 37986fa..ab37d33 100644 --- a/src/spaceone/inventory/conf/global_conf.py +++ b/src/spaceone/inventory/conf/global_conf.py @@ -39,6 +39,7 @@ # Metric Settings METRIC_SCHEDULE_HOUR = 0 # Hour (UTC) +METRIC_QUERY_TTL = 3 # Days # Handler Settings HANDLERS = { diff --git a/src/spaceone/inventory/manager/job_manager.py b/src/spaceone/inventory/manager/job_manager.py index e2c2b42..cf171fb 100644 --- a/src/spaceone/inventory/manager/job_manager.py +++ b/src/spaceone/inventory/manager/job_manager.py @@ -1,13 +1,14 @@ import logging -from typing import Tuple +from typing import Tuple, List from datetime import datetime, timedelta -from spaceone.core import cache +from spaceone.core import cache, config from spaceone.core.manager import BaseManager from spaceone.core.model.mongo_model import QuerySet from spaceone.inventory.error import * from spaceone.inventory.model.collector_model import Collector from spaceone.inventory.model.job_model import Job from spaceone.inventory.manager.metric_manager import MetricManager +from spaceone.inventory.manager.metric_data_manager import MetricDataManager _LOGGER = logging.getLogger(__name__) @@ -101,20 +102,49 @@ def decrease_remained_tasks_by_vo(self, job_vo: Job) -> None: self._delete_metric_cache(job_vo.plugin_id, job_vo.domain_id) self._run_metric_queries(job_vo.plugin_id, job_vo.domain_id) - @staticmethod - def _run_metric_queries(plugin_id: str, domain_id: str) -> None: + def _run_metric_queries(self, plugin_id: str, domain_id: str) -> None: metric_mgr = MetricManager() + recent_metrics = self._get_recent_metrics(domain_id) + managed_metric_vos = metric_mgr.filter_metrics( is_managed=True, domain_id=domain_id ) for managed_metric_vo in managed_metric_vos: - metric_mgr.push_task(managed_metric_vo) + if managed_metric_vo.metric_id in recent_metrics: + metric_mgr.push_task(managed_metric_vo) plugin_metric_vos = metric_mgr.filter_metrics( plugin_id=plugin_id, domain_id=domain_id ) for plugin_metric_vo in plugin_metric_vos: - metric_mgr.push_task(plugin_metric_vo) + if plugin_metric_vo.metric_id in recent_metrics: + metric_mgr.push_task(plugin_metric_vo) + + @staticmethod + def _get_recent_metrics(domain_id: str) -> List[str]: + metric_data_mgr = MetricDataManager() + metric_cache_ttl = config.get_global("METRIC_QUERY_TTL", 3) + ttl_time = datetime.utcnow() - timedelta(days=metric_cache_ttl) + + query = { + "filter": [ + {"k": "domain_id", "v": domain_id, "o": "eq"}, + {"k": "updated_at", "v": ttl_time, "o": "gte"}, + ] + } + + _LOGGER.debug( + f"[_get_metric_query_history] metric_cache_ttl: {metric_cache_ttl} days" + ) + + history_vos, total_count = metric_data_mgr.list_metric_query_history(query) + recent_metrics = [] + for history_vo in history_vos: + recent_metrics.append(history_vo.metric_id) + + _LOGGER.debug(f"[_get_metric_query_history] recent_metrics: {recent_metrics}") + + return recent_metrics @staticmethod def _delete_metric_cache(plugin_id: str, domain_id: str) -> None: diff --git a/src/spaceone/inventory/manager/metric_data_manager.py b/src/spaceone/inventory/manager/metric_data_manager.py index e2eb218..1443372 100644 --- a/src/spaceone/inventory/manager/metric_data_manager.py +++ b/src/spaceone/inventory/manager/metric_data_manager.py @@ -24,6 +24,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.metric_data_model = MetricData self.monthly_metric_data = MonthlyMetricData + self.history_model = MetricQueryHistory def create_metric_data(self, params: dict) -> MetricData: metric_data_vo: MetricData = self.metric_data_model.create(params) @@ -168,6 +169,9 @@ def analyze_metric_data_by_granularity( return response + def list_metric_query_history(self, query: dict) -> Tuple[QuerySet, int]: + return self.history_model.query(**query) + @cache.cacheable( key="inventory:metric-query-history:{domain_id}:{metric_id}", expire=600, @@ -179,10 +183,11 @@ def _rollback(vo: MetricQueryHistory): ) vo.delete() - history_model = MetricQueryHistory() - history_vos = history_model.filter(domain_id=domain_id, metric_id=metric_id) + history_vos = self.history_model.filter( + domain_id=domain_id, metric_id=metric_id + ) if history_vos.count() == 0: - history_vo = history_model.create( + history_vo = self.history_model.create( { "metric_id": metric_id, "domain_id": domain_id,