feature: monitor CR status & update python version (#34)

* feature: monitor CR status

* fix: update python version

Containerfile

@@ -1,5 +1,5 @@
 FROM registry.access.redhat.com/ubi8/ubi-minimal
-RUN microdnf install -y python3 python3-devel redhat-rpm-config gcc libffi-devel openssl-devel cargo 
+RUN microdnf install -y python38 python38-devel redhat-rpm-config gcc libffi-devel openssl-devel cargo 
 WORKDIR /app
 COPY ./requirements.txt /app/requirements.txt 
 RUN python3 -m pip install setuptools_rust

apiconnect_net.py

@@ -0,0 +1,73 @@
+import alog
+from kubernetes import client, config
+import urllib3
+
+urllib3.disable_warnings()
+logger = alog.use_channel("apiconnect")
+
+# /mgmt/status/apiconnect/TCPSummary
+
+
+class APIConnectNet(object):
+    namespace = 'apic-management'
+    use_kubeconfig = True
+    trawler = None
+
+    def __init__(self, config, trawler):
+        # Takes in config object and trawler instance it's behind
+        # Use kubeconfig or in-cluster config for k8s comms
+        # self.use_kubeconfig = trawler.use_kubeconfig
+        # Namespace to find CRs
+        self.namespace = config.get('namespace', 'default')
+        self.trawler = trawler
+
+    @alog.timed_function(logger.trace)
+    def fish(self):
+        try:
+            if self.use_kubeconfig:
+                logger.info("Using KUBECONFIG")
+                config.load_kube_config()
+            else:
+                logger.info("In cluster, so looking for juhu service")
+                config.load_incluster_config()
+            customObjectsApi = client.CustomObjectsApi()
+            customResources = [
+                {"group": "management.apiconnect.ibm.com", "plural": "managementclusters"},
+                {"group": "gateway.apiconnect.ibm.com", "plural": "gatewayclusters"},
+                {"group": "portal.apiconnect.ibm.com", "plural": "portalclusters"},
+            ]
+            for customResource in customResources:
+                logger.info("Gathering status for {plural}.{group}".format(**customResource))
+                api_response = customObjectsApi.list_cluster_custom_object(
+                    customResource['group'],
+                    'v1beta1',
+                    customResource['plural'])
+                for item in api_response['items']:
+                    for condition in item['status']['conditions']:
+                        self.trawler.set_gauge(
+                            'apiconnect',
+                            "{}_status".format(customResource['plural']),
+                            1,
+                            labels={
+                                "type": condition['type'],
+                                "status": condition['status'],
+                                "name": item['metadata']['name'],
+                                "namespace": item['metadata']['namespace'],
+                            })
+
+            # api_response['status']['conditions']
+            # {'lastTransitionTime': '2021-09-30T13:33:10Z', 'message': '', 'reason': 'na', 'status': 'False', 'type': 'Warning'}
+
+        except client.rest.ApiException as e:
+            logger.error("Error calling kubernetes API")
+            logger.exception(e)
+
+
+if __name__ == "__main__":
+    import trawler
+    boaty = trawler.Trawler()
+    boaty.secret_path = 'test-assets'
+    boaty.use_kubeconfig = True
+    net = APIConnectNet({"namespace": "apic"}, boaty)
+    net.use_kubeconfig = True
+    net.fish()

trawler.py

@@ -9,13 +9,14 @@
 import yaml
 import click
 from certs_net import CertsNet
+from apiconnect_net import APIConnectNet
 from datapower_net import DataPowerNet
 from manager_net import ManagerNet
 from analytics_net import AnalyticsNet
 from watch_pods import Watcher
 from prometheus_client import start_http_server
 import metrics_graphite
-from prometheus_client import Gauge
+from prometheus_client import Gauge, Counter
 
 
 logger = alog.use_channel("trawler")
@@ -42,7 +43,7 @@ def __init__(self, config_file=None):
             self.load_config(config_file)
         if 'logging' in self.config:
             alog.configure(
-              default_level=self.config['logging'].get('level', 'info'),
+              default_level=self.config['logging'].get('level', 'debug'),
               filters=self.config['logging'].get('filters', None),
               formatter=self.config['logging'].get('format', 'json')
             )
@@ -88,12 +89,14 @@ def load_config(self, config_file):
             logger.exception(e)
             exit(2)
 
-    def set_gauge(self, component, target_name, value, pod_name=None, labels={}):
+    def set_gauge(self, component, target_name, value, pod_name=None, labels=None):
+        """ Set or create prometheus gauge """
+        if not labels:
+            labels = {}
         if pod_name:
             labels['pod'] = pod_name
         if 'labels' in self.config['prometheus']:
-            labels =  {**self.config['prometheus']['labels'],**labels}
-        print(labels)
+            labels = {**self.config['prometheus']['labels'],**labels}
         logger.debug("Entering set_gauge - params: ({}, {}, {}, {})".format(component, target_name, value, pod_name))
         logger.debug(labels)
         if type(value) is float or type(value) is int:
@@ -111,8 +114,8 @@ def set_gauge(self, component, target_name, value, pod_name=None, labels={}):
                             prometheus_target,
                             prometheus_target)
 
-                logger.debug("Setting gauge {} to {}".format(
-                    self.gauges[prometheus_target]._name, value))
+                logger.debug("Setting gauge %s to %f",
+                    self.gauges[prometheus_target]._name, value)
                 if labels:
                     self.gauges[prometheus_target].labels(**labels).set(value)
                 else:
@@ -124,13 +127,53 @@ def set_gauge(self, component, target_name, value, pod_name=None, labels={}):
                     metric_name = "{}.{}".format(component, target_name)
                 self.graphite.stage(metric_name, value)
 
+    def inc_counter(self, component, target_name, value, pod_name=None, labels=None):
+        """ Set or increase prometheus counter """
+        if not labels:
+            labels = {}
+        if pod_name:
+            labels['pod'] = pod_name
+        if 'labels' in self.config['prometheus']:
+            labels = {**self.config['prometheus']['labels'],**labels}
+        logger.debug("Entering inc_counter - params: ({}, {}, {}, {})".format(component, target_name, value, pod_name))
+        logger.debug(labels)
+        if type(value) is float or type(value) is int:
+            target_name = target_name.replace('-', '_')
+            if self.config['prometheus']['enabled']:
+                prometheus_target = "{}_{}".format(component, target_name.replace('.', '_'))
+                if prometheus_target not in self.gauges:
+                    logger.info("Creating counter {}".format(prometheus_target))
+                    if labels:
+                        self.gauges[prometheus_target] = Counter(
+                            prometheus_target,
+                            prometheus_target, labels.keys())
+                    else:
+                        self.gauges[prometheus_target] = Counter(
+                            prometheus_target,
+                            prometheus_target)
+
+                logger.debug("Setting gauge %s to %f",
+                    self.gauges[prometheus_target]._name, value)
+                if labels:
+                    self.gauges[prometheus_target].labels(**labels).inc()
+                else:
+                    self.gauges[prometheus_target].inc()
+            if self.config['graphite']['enabled']:
+                if pod_name:
+                    metric_name = "{}.{}.{}".format(component, pod_name, target_name)
+                else: 
+                    metric_name = "{}.{}".format(component, target_name)
+                self.graphite.stage(metric_name, value)
+
     @alog.timed_function(logger.trace)
     def trawl_metrics(self):
         # Initialise
         logger.info("Laying nets...")
         nets = []
         if 'certs' in self.config['nets'] and self.config['nets']['certs'].get('enabled', True):
             nets.append(CertsNet(self.config['nets']['certs'], self))
+        if 'apiconnect' in self.config['nets'] and self.config['nets']['apiconnect'].get('enabled', True):
+            nets.append(APIConnectNet(self.config['nets']['apiconnect'], self))
         if 'datapower' in self.config['nets'] and self.config['nets']['datapower'].get('enabled', True):
             nets.append(DataPowerNet(self.config['nets']['datapower'], self))
         if 'manager' in self.config['nets'] and self.config['nets']['manager'].get('enabled', True):