diff --git a/README.md b/README.md index b05446c..4056238 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,47 @@ root@leader:~# logcli query -q --no-labels -t '{nodename="leader"} | json | lin `logcli` will use the default Loki instance of the cluster, this can be changed using the environment variable [`LOKI_ADDR`](https://grafana.com/docs/loki/latest/getting-started/logcli/#example) +## APIs + +The module provides some APIs to interact with the Loki instance: + +- `configure-module` +- `get-configuration` + +### `configure-module` + +Configure the Loki instance. + +#### Parameters + +- `retention_days`: The number of days to keep the logs. + +#### Example + +```bash +api-cli run module/loki1/configure-module '{"retention_days": 7}' +``` + +### `get-configuration` + +Get the Loki instance configuration. + +#### Example + +```bash +api-cli run module/loki1/get-configuration +``` + +```json +{ + "retention_days": 7, + "active_from": "2021-05-28T15:49:27Z+00:00", + "active_to": "2021-05-28T15:49:27Z+00:00" +} +``` + +Note: `active_to` field WILL miss if the instance is still active. + ## Uninstall To uninstall the instance: diff --git a/imageroot/actions/configure-module/10set b/imageroot/actions/configure-module/10set new file mode 100755 index 0000000..3204d59 --- /dev/null +++ b/imageroot/actions/configure-module/10set @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import json +import sys + +import agent + +request = json.load(sys.stdin) + +agent.set_env('LOKI_RETENTION_PERIOD', f"{request['retention_days']}") diff --git a/imageroot/actions/configure-module/20restart b/imageroot/actions/configure-module/20restart new file mode 100755 index 0000000..20c3035 --- /dev/null +++ b/imageroot/actions/configure-module/20restart @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +systemctl --user restart loki.service diff --git a/imageroot/actions/configure-module/validate-input.json b/imageroot/actions/configure-module/validate-input.json new file mode 100644 index 0000000..966f237 --- /dev/null +++ b/imageroot/actions/configure-module/validate-input.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "http://schema.nethserver.org/loki/configure-module.json", + "title": "Configure Loki", + "description": "Configure Loki instance.", + "type": "object", + "required": [ + "retention_days" + ], + "properties": { + "retention_days": { + "type": "integer", + "description": "Retention period of logs, in days.", + "minimum": 1 + } + } +} diff --git a/imageroot/actions/create-module/10env b/imageroot/actions/create-module/10env index 1aa41e2..afe67a7 100755 --- a/imageroot/actions/create-module/10env +++ b/imageroot/actions/create-module/10env @@ -7,6 +7,7 @@ import agent import os +import datetime def genuuid(): uuid = os.popen("uuidgen") @@ -24,3 +25,5 @@ agent.set_env('LOKI_API_AUTH_USERNAME', 'loki') agent.set_env('LOKI_API_AUTH_PASSWORD', genuuid()) agent.set_env('LOKI_LOGS_INGRESS_TOKEN', genuuid()) agent.set_env('LOKI_HTTP_PORT', port) +agent.set_env('LOKI_RETENTION_PERIOD', '365') +agent.set_env('LOKI_ACTIVE_FROM', datetime.datetime.now().astimezone().isoformat()) diff --git a/imageroot/actions/get-configuration/10get b/imageroot/actions/get-configuration/10get new file mode 100755 index 0000000..834ca8b --- /dev/null +++ b/imageroot/actions/get-configuration/10get @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import os +import json +import sys + +response = { + "retention_days": int(os.getenv('LOKI_RETENTION_PERIOD')), + "active_from": os.getenv('LOKI_ACTIVE_FROM') +} + +if os.getenv('LOKI_ACTIVE_TO') is not None: + response['active_to'] = os.getenv('LOKI_ACTIVE_TO') + +json.dump(response, sys.stdout) diff --git a/imageroot/actions/get-configuration/validate-output.json b/imageroot/actions/get-configuration/validate-output.json new file mode 100644 index 0000000..47e69a8 --- /dev/null +++ b/imageroot/actions/get-configuration/validate-output.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "http://schema.nethserver.org/loki/get-configuration.json", + "title": "Configuration of Loki", + "description": "Retrieve the configuration of loki instance.", + "type": "object", + "required": [ + "retention_days", + "active_from" + ], + "properties": { + "retention_days": { + "type": "integer", + "description": "Retention period of logs, in days.", + "minimum": 1 + }, + "active_from": { + "type": "string", + "format": "date-time", + "description": "The ISO 8601 date-time when the Loki instance was activated." + }, + "active_to": { + "type": "string", + "format": "date-time", + "description": "The ISO 8601 date-time when the Loki instance was deactivated." + } + } +} diff --git a/imageroot/actions/restore-module/06copyenv b/imageroot/actions/restore-module/06copyenv index 14f6b8f..1f19667 100755 --- a/imageroot/actions/restore-module/06copyenv +++ b/imageroot/actions/restore-module/06copyenv @@ -34,5 +34,6 @@ for evar in [ "LOKI_API_AUTH_PASSWORD", "LOKI_LOGS_INGRESS_TOKEN", "LOKI_HTTP_PORT", + "LOKI_RETENTION_PERIOD", ]: agent.set_env(evar, original_environment[evar]) diff --git a/imageroot/events/default-instance-changed/10set b/imageroot/events/default-instance-changed/10set new file mode 100755 index 0000000..ebdb671 --- /dev/null +++ b/imageroot/events/default-instance-changed/10set @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import json +import os +import sys +import datetime + +import agent + +# Check if event comes from the cluster +if os.getenv("AGENT_EVENT_SOURCE") == 'cluster': + # parse data + data = json.load(sys.stdin) + if 'instance' in data and data['instance'] == 'loki': + if 'previous' in data and data['previous'] == os.getenv('MODULE_ID') and os.getenv('LOKI_ACTIVE_TO') is None: + agent.set_env('LOKI_ACTIVE_TO', datetime.datetime.now().astimezone().isoformat()) diff --git a/imageroot/loki-config.yaml b/imageroot/loki-config.yaml new file mode 100644 index 0000000..02ffc35 --- /dev/null +++ b/imageroot/loki-config.yaml @@ -0,0 +1,64 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +ingester: + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed + max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h + chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first + chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) + max_transfer_retries: 0 # Chunk transfers disabled + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: /loki/boltdb-shipper-active + cache_location: /loki/boltdb-shipper-cache + cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space + shared_store: filesystem + filesystem: + directory: /loki/chunks + +compactor: + working_directory: /loki/boltdb-shipper-compactor + shared_store: filesystem + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: false + retention_period: ${LOKI_RETENTION_PERIOD:-365}d + +ruler: + storage: + type: local + local: + directory: /loki/rules + rule_path: /loki/rules-temp + alertmanager_url: http://localhost:9093 + ring: + kvstore: + store: inmemory + enable_api: true diff --git a/imageroot/systemd/user/loki-server.service b/imageroot/systemd/user/loki-server.service index b799a6b..52f4639 100644 --- a/imageroot/systemd/user/loki-server.service +++ b/imageroot/systemd/user/loki-server.service @@ -17,9 +17,12 @@ ExecStart=/usr/bin/podman run \ --replace \ --name=%N \ --volume=loki-server-data:/loki \ + --volume=%S/loki-config.yaml:/etc/loki/local-config.yaml:Z \ + --env=LOKI_RETENTION_PERIOD \ ${LOKI_IMAGE} \ -config.file=/etc/loki/local-config.yaml \ - -log.level warn + -log.level=warn \ + -config.expand-env ExecStop=/usr/bin/podman stop --ignore --cidfile %t/loki-server.ctr-id -t 10 ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/loki-server.ctr-id diff --git a/imageroot/update-module.d/10config b/imageroot/update-module.d/10config new file mode 100755 index 0000000..e67ce59 --- /dev/null +++ b/imageroot/update-module.d/10config @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import datetime +import os + +import agent + +# Set environment variables +if os.getenv('LOKI_ACTIVE_FROM') is None: + # This is the most unorthodox way to get the file creation time, which is not even guaranteed to be the same as the time the file was written to disk + agent.set_env('LOKI_ACTIVE_FROM', datetime.datetime.fromtimestamp(os.stat('environment').st_mtime).astimezone().isoformat()) +if os.getenv('LOKI_RETENTION_PERIOD') is None: + agent.set_env('LOKI_RETENTION_PERIOD', '365')