refactor: change the logic to run metrics after collecting cloud reso…

…urces

Signed-off-by: Jongmin Kim <[email protected]>

src/spaceone/inventory/conf/global_conf.py

@@ -39,6 +39,7 @@
 
 # Metric Settings
 METRIC_SCHEDULE_HOUR = 0  # Hour (UTC)
+METRIC_QUERY_TTL = 3  # Days
 
 # Handler Settings
 HANDLERS = {

src/spaceone/inventory/manager/job_manager.py

@@ -1,13 +1,14 @@
 import logging
-from typing import Tuple
+from typing import Tuple, List
 from datetime import datetime, timedelta
-from spaceone.core import cache
+from spaceone.core import cache, config
 from spaceone.core.manager import BaseManager
 from spaceone.core.model.mongo_model import QuerySet
 from spaceone.inventory.error import *
 from spaceone.inventory.model.collector_model import Collector
 from spaceone.inventory.model.job_model import Job
 from spaceone.inventory.manager.metric_manager import MetricManager
+from spaceone.inventory.manager.metric_data_manager import MetricDataManager
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -101,20 +102,49 @@ def decrease_remained_tasks_by_vo(self, job_vo: Job) -> None:
             self._delete_metric_cache(job_vo.plugin_id, job_vo.domain_id)
             self._run_metric_queries(job_vo.plugin_id, job_vo.domain_id)
 
-    @staticmethod
-    def _run_metric_queries(plugin_id: str, domain_id: str) -> None:
+    def _run_metric_queries(self, plugin_id: str, domain_id: str) -> None:
         metric_mgr = MetricManager()
+        recent_metrics = self._get_recent_metrics(domain_id)
+
         managed_metric_vos = metric_mgr.filter_metrics(
             is_managed=True, domain_id=domain_id
         )
         for managed_metric_vo in managed_metric_vos:
-            metric_mgr.push_task(managed_metric_vo)
+            if managed_metric_vo.metric_id in recent_metrics:
+                metric_mgr.push_task(managed_metric_vo)
 
         plugin_metric_vos = metric_mgr.filter_metrics(
             plugin_id=plugin_id, domain_id=domain_id
         )
         for plugin_metric_vo in plugin_metric_vos:
-            metric_mgr.push_task(plugin_metric_vo)
+            if plugin_metric_vo.metric_id in recent_metrics:
+                metric_mgr.push_task(plugin_metric_vo)
+
+    @staticmethod
+    def _get_recent_metrics(domain_id: str) -> List[str]:
+        metric_data_mgr = MetricDataManager()
+        metric_cache_ttl = config.get_global("METRIC_QUERY_TTL", 3)
+        ttl_time = datetime.utcnow() - timedelta(days=metric_cache_ttl)
+
+        query = {
+            "filter": [
+                {"k": "domain_id", "v": domain_id, "o": "eq"},
+                {"k": "updated_at", "v": ttl_time, "o": "gte"},
+            ]
+        }
+
+        _LOGGER.debug(
+            f"[_get_metric_query_history] metric_cache_ttl: {metric_cache_ttl} days"
+        )
+
+        history_vos, total_count = metric_data_mgr.list_metric_query_history(query)
+        recent_metrics = []
+        for history_vo in history_vos:
+            recent_metrics.append(history_vo.metric_id)
+
+        _LOGGER.debug(f"[_get_metric_query_history] recent_metrics: {recent_metrics}")
+
+        return recent_metrics
 
     @staticmethod
     def _delete_metric_cache(plugin_id: str, domain_id: str) -> None:

src/spaceone/inventory/manager/metric_data_manager.py

@@ -24,6 +24,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.metric_data_model = MetricData
         self.monthly_metric_data = MonthlyMetricData
+        self.history_model = MetricQueryHistory
 
     def create_metric_data(self, params: dict) -> MetricData:
         metric_data_vo: MetricData = self.metric_data_model.create(params)
@@ -168,6 +169,9 @@ def analyze_metric_data_by_granularity(
 
         return response
 
+    def list_metric_query_history(self, query: dict) -> Tuple[QuerySet, int]:
+        return self.history_model.query(**query)
+
     @cache.cacheable(
         key="inventory:metric-query-history:{domain_id}:{metric_id}",
         expire=600,
@@ -179,10 +183,11 @@ def _rollback(vo: MetricQueryHistory):
             )
             vo.delete()
 
-        history_model = MetricQueryHistory()
-        history_vos = history_model.filter(domain_id=domain_id, metric_id=metric_id)
+        history_vos = self.history_model.filter(
+            domain_id=domain_id, metric_id=metric_id
+        )
         if history_vos.count() == 0:
-            history_vo = history_model.create(
+            history_vo = self.history_model.create(
                 {
                     "metric_id": metric_id,
                     "domain_id": domain_id,