New services: Cloud Log Manager and Syslog forwarders

Implemented following parts:
- Systemd services file
- Python scripts
- API Actions

imageroot/actions/get-configuration/10get

@@ -5,13 +5,66 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 #
 
+from datetime import datetime, timezone
+
 import os
 import json
 import sys
+import subprocess
+
+# Cloud Log Manager
+
+cloud_log_manager = {}
+
+with subprocess.Popen(['systemctl', '--user', 'is-active', 'cloud-log-manager-forwarder'], stdout=subprocess.PIPE, stderr=sys.stderr) as pipe:
+    response = pipe.stdout.readline().decode('utf-8').replace('\n', '')
+    if 'inactive' in response:
+        cloud_log_manager["status"] = 'inactive'
+    elif 'active' in response:
+        cloud_log_manager["status"] = 'active'
+    elif 'failed' in response:
+        cloud_log_manager["status"] = 'failed'
+
+cloud_log_manager["address"] = os.getenv('CLOUD_LOG_MANAGER_ADDRESS', '')
+cloud_log_manager["tenant"] = os.getenv('CLOUD_LOG_MANAGER_TENANT', '')
+
+try: 
+    with open("./cloud_log_manager_last_timestamp", "r") as clm_file:
+        cloud_log_manager["last_timestamp"] = clm_file.read()
+except:
+    cloud_log_manager["last_timestamp"] = ""
+
+# Syslog
+
+syslog = {}
+
+with subprocess.Popen(['systemctl', '--user', 'is-active', 'syslog-forwarder'], stdout=subprocess.PIPE, stderr=sys.stderr) as pipe:
+    response = pipe.stdout.readline().decode('utf-8').replace('\n', '')
+    if 'inactive' in response:
+        syslog["status"] = 'inactive'
+    elif 'active' in response:
+        syslog["status"] = 'active'
+    elif 'failed' in response:
+        syslog["status"] = 'failed'
+
+syslog["address"] = os.getenv('SYSLOG_ADDRESS', '')
+syslog["port"] = os.getenv('SYSLOG_PORT', '')
+syslog["protocol"] = os.getenv('SYSLOG_PROTOCOL', '')
+syslog["format"] = os.getenv('SYSLOG_FORMAT', '')
+
+try:
+    with open("./syslog_last_timestamp", "r") as syslog_file:
+        syslog["last_timestamp"] = syslog_file.read()
+except:
+    syslog["last_timestamp"] = ""
+
+# General
 
 response = {
     "retention_days": int(os.getenv('LOKI_RETENTION_PERIOD')),
-    "active_from": os.getenv('LOKI_ACTIVE_FROM')
+    "active_from": os.getenv('LOKI_ACTIVE_FROM'),
+    "cloud_log_manager": cloud_log_manager,
+    "syslog": syslog
 }
 
 if os.getenv('LOKI_ACTIVE_TO') is not None:

imageroot/actions/get-configuration/validate-output.json

@@ -6,7 +6,9 @@
     "type": "object",
     "required": [
         "retention_days",
-        "active_from"
+        "active_from",
+        "cloud_log_manager",
+        "syslog"
     ],
     "properties": {
         "retention_days": {
@@ -23,6 +25,65 @@
             "type": "string",
             "format": "date-time",
             "description": "The ISO 8601 date-time when the Loki instance was deactivated."
+        },
+        "cloud_log_manager": {
+            "type": "object",
+            "description": "Cloud Log Manager forwarder configuration.",
+            "required": [
+                "status"
+            ],
+            "properties": {
+                "status": {
+                    "type": "string",
+                    "description": "Forwarder state."
+                },
+                "address": {
+                    "type": "string",
+                    "description": "Forwarder address where datas are sent."
+                },
+                "tenant": {
+                    "type": "string",
+                    "description": "Cloud Log Manager internal data."
+                },
+                "last_timestamp": {
+                    "type": "string",
+                    "description": "Timestamp of the last log sent."
+                }
+            }
+
+        },
+        "syslog": {
+            "type": "object",
+            "description": "Syslog forwarder configuration.",
+            "required": [
+                "status"
+            ],
+            "properties": {
+                "status": {
+                    "type": "string",
+                    "description": "Forwarder state."
+                },
+                "address": {
+                    "type": "string",
+                    "description": "Forwarder address where data are sent."
+                },
+                "port": {
+                    "type": "string",
+                    "description": "External server port."
+                },
+                "protocol": {
+                    "type": "string",
+                    "description": "Protocol used to send datas."
+                },
+                "format": {
+                    "type": "string",
+                    "description": "Log format."
+                },
+                "last_timestamp": {
+                    "type": "string",
+                    "description": "Timestamp of the last log sent."
+                }
+            }
         }
     }
 }

imageroot/actions/set-clm-forwarder/10set

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import subprocess
+import hashlib
+import agent
+import json
+import uuid
+import sys
+
+request = json.load(sys.stdin)
+
+rdb = agent.redis_connect()
+cluster_uuid = rdb.get('cluster/uuid')
+subscription = rdb.hgetall('cluster/subscription')
+
+if not bool(subscription):
+    print("No subscription found !")
+    sys.exit(1)
+
+if (request['active']):
+    agent.set_env('CLOUD_LOG_MANAGER_UUID', uuid.uuid5(uuid.NAMESPACE_DNS, cluster_uuid))
+    agent.set_env('CLOUD_LOG_MANAGER_HOSTNAME', 'cluster-' + hashlib.sha256(cluster_uuid.encode("utf-8")).hexdigest()[:8])
+    agent.set_env('CLOUD_LOG_MANAGER_ADDRESS', f"{request['address']}")
+    agent.set_env('CLOUD_LOG_MANAGER_TENANT', f"{request['tenant']}")
+
+    if (request['start_time'] != ''):
+        agent.set_env('CLOUD_LOG_MANAGER_START_TIME', f"{request['start_time']}")
+
+    # If process is running, restart with new env variables, otherwise enable it
+    with subprocess.Popen(['systemctl', '--user', 'is-active', 'cloud-log-manager-forwarder'], stdout=subprocess.PIPE, stderr=sys.stderr) as pipe:
+        response = pipe.stdout.readline().decode('utf-8').replace('\n', '')
+        if response in {'active', 'failed'}:
+            action = 'restart'
+        else:
+            action = 'enable'
+else:
+    agent.unset_env('CLOUD_LOG_MANAGER_UUID')
+    agent.unset_env('CLOUD_LOG_MANAGER_START_TIME')
+    agent.unset_env('CLOUD_LOG_MANAGER_ADDRESS')
+    agent.unset_env('CLOUD_LOG_MANAGER_TENANT')
+
+    action = 'disable'
+
+subprocess.run(["systemctl", "--user", action, "--now", "cloud-log-manager-forwarder.service"], 
+                stdout=subprocess.PIPE,
+                stderr=sys.stderr,
+                text=True,
+                check=True)

imageroot/actions/set-clm-forwarder/validate-input.json

@@ -0,0 +1,28 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$id": "http://schema.nethserver.org/loki/configure-module.json",
+    "title": "Configure Cloud Log Manager Forwarder",
+    "description": "Configure Cloud Log Manager Forwarder service.",
+    "type": "object",
+    "required": [
+        "active"
+    ],
+    "properties": {
+        "active": {
+            "type": "boolean",
+            "description": "Action condition to be set."
+        },
+        "address": {
+            "type": "string",
+            "description": "Cloud log manager server address."
+        },
+        "tenant": {
+            "type": "string",
+            "description": "Cloud log manager tenant."
+        },
+        "start_time": {
+            "type": "string",
+            "description": "Log forwarding start time."
+        }
+    }
+}

imageroot/actions/set-syslog-forwarder/10set

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import subprocess
+import agent
+import json
+import sys
+
+request = json.load(sys.stdin)
+
+if (request['active']):
+    rdb = agent.redis_connect()
+    cluster_uuid = rdb.get('cluster/uuid')
+
+    agent.set_env('SYSLOG_ADDRESS', f"{request['address']}")
+    agent.set_env('SYSLOG_PORT', f"{request['port']}")
+    agent.set_env('SYSLOG_PROTOCOL', f"{request['protocol']}")
+    agent.set_env('SYSLOG_FORMAT', f"{request['format']}")
+
+    if (request['start_time'] != ''):
+        agent.set_env('SYSLOG_START_TIME', f"{request['start_time']}")
+
+    # If process is running, restart with new env variables, otherwise enable it
+    with subprocess.Popen(['systemctl', '--user', 'is-active', 'syslog-forwarder'], stdout=subprocess.PIPE, stderr=sys.stderr) as pipe:
+        response = pipe.stdout.readline().decode('utf-8').replace('\n', '')
+        if response in {'active', 'failed'}:
+            action = 'restart'
+        else:
+            action = 'enable'
+else:
+    agent.unset_env('SYSLOG_ADDRESS')
+    agent.unset_env('SYSLOG_PORT')
+    agent.unset_env('SYSLOG_PROTOCOL')
+    agent.unset_env('SYSLOG_FORMAT')
+    agent.unset_env('SYSLOG_START_TIME')
+
+    action = 'disable'
+
+subprocess.run(["systemctl", "--user", action, "--now", "syslog-forwarder.service"], 
+                stdout=subprocess.PIPE,
+                stderr=sys.stderr,
+                text=True,
+                check=True)

imageroot/actions/set-syslog-forwarder/validate-input.json

@@ -0,0 +1,36 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$id": "http://schema.nethserver.org/loki/configure-module.json",
+    "title": "Configure Syslog Forwarder",
+    "description": "Configure Syslog Forwarder service.",
+    "type": "object",
+    "required": [
+        "active"
+    ],
+    "properties": {
+        "active": {
+            "type": "boolean",
+            "description": "Action condition to be set."
+        },
+        "address": {
+            "type": "string",
+            "description": "Syslog server address."
+        },
+        "port": {
+            "type": "string",
+            "description": "Syslog server port."
+        },
+        "protocol": {
+            "type": "string",
+            "description": "Protocol to send datas."
+        },
+        "format": {
+            "type": "string",
+            "description": "Log format."
+        },
+        "start_time": {
+            "type": "string",
+            "description": "Log forwarding start time."
+        }
+    }
+}