feat: retention configuration

README.md

@@ -50,6 +50,47 @@ root@leader:~# logcli  query -q --no-labels -t '{nodename="leader"} | json | lin
 
 `logcli` will use the default Loki instance of the cluster, this can be changed using the environment variable [`LOKI_ADDR`](https://grafana.com/docs/loki/latest/getting-started/logcli/#example)
 
+## APIs
+
+The module provides some APIs to interact with the Loki instance:
+
+- `configure-module`
+- `get-configuration`
+
+### `configure-module`
+
+Configure the Loki instance.
+
+#### Parameters
+
+- `retention_days`: The number of days to keep the logs.
+
+#### Example
+
+```bash
+api-cli run module/loki1/configure-module '{"retention_days": 7}'
+```
+
+### `get-configuration`
+
+Get the Loki instance configuration.
+
+#### Example
+
+```bash
+api-cli run module/loki1/get-configuration
+```
+
+```json
+{
+  "retention_days": 7,
+  "active_from": "2021-05-28T15:49:27Z+00:00",
+  "active_to": "2021-05-28T15:49:27Z+00:00"
+}
+```
+
+Note: `active_to` field WILL miss if the instance is still active.
+
 ## Uninstall
 
 To uninstall the instance:

imageroot/actions/configure-module/10set

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import json
+import sys
+
+import agent
+
+request = json.load(sys.stdin)
+
+agent.set_env('LOKI_RETENTION_PERIOD', f"{request['retention_days']}")

imageroot/actions/configure-module/20restart

@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+systemctl --user restart loki.service

imageroot/actions/configure-module/validate-input.json

@@ -0,0 +1,17 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$id": "http://schema.nethserver.org/loki/configure-module.json",
+    "title": "Configure Loki",
+    "description": "Configure Loki instance.",
+    "type": "object",
+    "required": [
+        "retention_days"
+    ],
+    "properties": {
+        "retention_days": {
+            "type": "integer",
+            "description": "Retention period of logs, in days.",
+            "minimum": 1
+        }
+    }
+}

imageroot/actions/create-module/10env

@@ -7,6 +7,7 @@
 
 import agent
 import os
+import datetime
 
 def genuuid():
     uuid =  os.popen("uuidgen")
@@ -24,3 +25,5 @@ agent.set_env('LOKI_API_AUTH_USERNAME', 'loki')
 agent.set_env('LOKI_API_AUTH_PASSWORD', genuuid())
 agent.set_env('LOKI_LOGS_INGRESS_TOKEN', genuuid())
 agent.set_env('LOKI_HTTP_PORT', port)
+agent.set_env('LOKI_RETENTION_PERIOD', '365')
+agent.set_env('LOKI_ACTIVE_FROM', datetime.datetime.now().astimezone().isoformat())

imageroot/actions/get-configuration/10get

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import os
+import json
+import sys
+
+response = {
+    "retention_days": int(os.getenv('LOKI_RETENTION_PERIOD')),
+    "active_from": os.getenv('LOKI_ACTIVE_FROM')
+}
+
+if os.getenv('LOKI_ACTIVE_TO') is not None:
+    response['active_to'] = os.getenv('LOKI_ACTIVE_TO')
+
+json.dump(response, sys.stdout)

imageroot/actions/get-configuration/validate-output.json

@@ -0,0 +1,28 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$id": "http://schema.nethserver.org/loki/get-configuration.json",
+    "title": "Configuration of Loki",
+    "description": "Retrieve the configuration of loki instance.",
+    "type": "object",
+    "required": [
+        "retention_days",
+        "active_from"
+    ],
+    "properties": {
+        "retention_days": {
+            "type": "integer",
+            "description": "Retention period of logs, in days.",
+            "minimum": 1
+        },
+        "active_from": {
+            "type": "string",
+            "format": "date-time",
+            "description": "The ISO 8601 date-time when the Loki instance was activated."
+        },
+        "active_to": {
+            "type": "string",
+            "format": "date-time",
+            "description": "The ISO 8601 date-time when the Loki instance was deactivated."
+        }
+    }
+}

imageroot/actions/restore-module/06copyenv

@@ -34,5 +34,6 @@ for evar in [
         "LOKI_API_AUTH_PASSWORD",
         "LOKI_LOGS_INGRESS_TOKEN",
         "LOKI_HTTP_PORT",
+        "LOKI_RETENTION_PERIOD",
     ]:
     agent.set_env(evar, original_environment[evar])

imageroot/events/default-instance-changed/10set

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import json
+import os
+import sys
+import datetime
+
+import agent
+
+# Check if event comes from the cluster
+if os.getenv("AGENT_EVENT_SOURCE") == 'cluster':
+    # parse data
+    data = json.load(sys.stdin)
+    if 'instance' in data and data['instance'] == 'loki':
+        if 'previous' in data and data['previous'] == os.getenv('MODULE_ID') and os.getenv('LOKI_ACTIVE_TO') is None:
+            agent.set_env('LOKI_ACTIVE_TO', datetime.datetime.now().astimezone().isoformat())

imageroot/loki-config.yaml

@@ -0,0 +1,64 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time will be flushed
+  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
+  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
+  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+  max_transfer_retries: 0     # Chunk transfers disabled
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/boltdb-shipper-active
+    cache_location: /loki/boltdb-shipper-cache
+    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
+    shared_store: filesystem
+  filesystem:
+    directory: /loki/chunks
+
+compactor:
+  working_directory: /loki/boltdb-shipper-compactor
+  shared_store: filesystem
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: ${LOKI_RETENTION_PERIOD:-365}d
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /loki/rules
+  rule_path: /loki/rules-temp
+  alertmanager_url: http://localhost:9093
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true

imageroot/systemd/user/loki-server.service

@@ -17,9 +17,12 @@ ExecStart=/usr/bin/podman run \
     --replace \
     --name=%N \
     --volume=loki-server-data:/loki \
+    --volume=%S/loki-config.yaml:/etc/loki/local-config.yaml:Z \
+    --env=LOKI_RETENTION_PERIOD \
     ${LOKI_IMAGE} \
     -config.file=/etc/loki/local-config.yaml \
-    -log.level warn
+    -log.level=warn \
+    -config.expand-env
 
 ExecStop=/usr/bin/podman stop --ignore --cidfile %t/loki-server.ctr-id -t 10
 ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/loki-server.ctr-id

imageroot/update-module.d/10config

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import datetime
+import os
+
+import agent
+
+# Set environment variables
+if os.getenv('LOKI_ACTIVE_FROM') is None:
+    # This is the most unorthodox way to get the file creation time, which is not even guaranteed to be the same as the time the file was written to disk
+    agent.set_env('LOKI_ACTIVE_FROM', datetime.datetime.fromtimestamp(os.stat('environment').st_mtime).astimezone().isoformat())
+if os.getenv('LOKI_RETENTION_PERIOD') is None:
+    agent.set_env('LOKI_RETENTION_PERIOD', '365')