From 28e9c888459d25987d3e4568bd632fd90a8453dc Mon Sep 17 00:00:00 2001 From: Dmitry Volodin Date: Tue, 1 Oct 2024 11:35:05 +0200 Subject: [PATCH] Replace liftbridge with kafka --- CHANGELOG.md | 4 ++ src/gufo/thor/services/activator.py | 4 +- src/gufo/thor/services/auth.py | 4 +- src/gufo/thor/services/chwriter.py | 4 +- src/gufo/thor/services/classifier.py | 4 +- src/gufo/thor/services/correlator.py | 4 +- src/gufo/thor/services/kafka.py | 38 +++++++++++++++ src/gufo/thor/services/liftbridge.py | 46 ------------------- src/gufo/thor/services/migrate.py | 4 +- src/gufo/thor/services/ping.py | 4 +- src/gufo/thor/services/runner.py | 4 +- src/gufo/thor/services/syslogcollector.py | 4 +- src/gufo/thor/services/topo.py | 4 +- src/gufo/thor/services/trapcollector.py | 4 +- src/gufo/thor/services/worker.py | 4 +- .../thor/templates/liftbridge/liftbridge.yml | 17 ------- src/gufo/thor/templates/noc/settings.yml | 4 ++ tests/test_service.py | 6 +-- 18 files changed, 73 insertions(+), 90 deletions(-) create mode 100644 src/gufo/thor/services/kafka.py delete mode 100644 src/gufo/thor/services/liftbridge.py delete mode 100644 src/gufo/thor/templates/liftbridge/liftbridge.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index a1cfb16..0927c15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ To see unreleased changes, please see the [CHANGELOG on the master branch](https * `restart` command. +### Changed + +* `liftbridge` service replaced with `kafka` + ## 0.6.0 - 2024-08-05 ### Added diff --git a/src/gufo/thor/services/activator.py b/src/gufo/thor/services/activator.py index 1d04760..5e0dac9 100644 --- a/src/gufo/thor/services/activator.py +++ b/src/gufo/thor/services/activator.py @@ -11,7 +11,7 @@ """ # Gufo Thor modules -from .liftbridge import liftbridge +from .kafka import kafka from .migrate import migrate from .noc import NocService from .sae import sae @@ -21,7 +21,7 @@ class ActivatorService(NocService): """activator service.""" name = "activator" - dependencies = (liftbridge, migrate, sae) + dependencies = (kafka, migrate, sae) activator = ActivatorService() diff --git a/src/gufo/thor/services/auth.py b/src/gufo/thor/services/auth.py index 6ce1475..1af4c59 100644 --- a/src/gufo/thor/services/auth.py +++ b/src/gufo/thor/services/auth.py @@ -12,7 +12,7 @@ # Gufo Thor modules from .envoy import envoy -from .liftbridge import liftbridge +from .kafka import kafka from .migrate import migrate from .mongo import mongo from .noc import NocHcService @@ -22,7 +22,7 @@ class AuthService(NocHcService): """auth service.""" name = "auth" - dependencies = (envoy, liftbridge, migrate, mongo) + dependencies = (envoy, kafka, migrate, mongo) allow_scale = True expose_http_prefix = "/api/auth/" compose_command = ( diff --git a/src/gufo/thor/services/chwriter.py b/src/gufo/thor/services/chwriter.py index 405f06c..f2aeb5a 100644 --- a/src/gufo/thor/services/chwriter.py +++ b/src/gufo/thor/services/chwriter.py @@ -12,7 +12,7 @@ # Gufo Thor modules from .clickhouse import clickhouse -from .liftbridge import liftbridge +from .kafka import kafka from .migrate import migrate from .noc import NocService @@ -21,7 +21,7 @@ class ChwriterService(NocService): """chwriter service.""" name = "chwriter" - dependencies = (clickhouse, liftbridge, migrate) + dependencies = (clickhouse, kafka, migrate) chwriter = ChwriterService() diff --git a/src/gufo/thor/services/classifier.py b/src/gufo/thor/services/classifier.py index 38a1d8c..2be1567 100644 --- a/src/gufo/thor/services/classifier.py +++ b/src/gufo/thor/services/classifier.py @@ -11,7 +11,7 @@ """ # Gufo Thor modules -from .liftbridge import liftbridge +from .kafka import kafka from .migrate import migrate from .mongo import mongo from .noc import NocService @@ -22,7 +22,7 @@ class ClassifierService(NocService): """classifier service.""" name = "classifier" - dependencies = (liftbridge, migrate, mongo, postgres) + dependencies = (kafka, migrate, mongo, postgres) classifier = ClassifierService() diff --git a/src/gufo/thor/services/correlator.py b/src/gufo/thor/services/correlator.py index dad54f1..c2d8701 100644 --- a/src/gufo/thor/services/correlator.py +++ b/src/gufo/thor/services/correlator.py @@ -11,7 +11,7 @@ """ # Gufo Thor modules -from .liftbridge import liftbridge +from .kafka import kafka from .migrate import migrate from .mongo import mongo from .noc import NocService @@ -22,7 +22,7 @@ class CorrelatorService(NocService): """correlator service.""" name = "correlator" - dependencies = (liftbridge, migrate, mongo, postgres) + dependencies = (kafka, migrate, mongo, postgres) correlator = CorrelatorService() diff --git a/src/gufo/thor/services/kafka.py b/src/gufo/thor/services/kafka.py new file mode 100644 index 0000000..f6073d0 --- /dev/null +++ b/src/gufo/thor/services/kafka.py @@ -0,0 +1,38 @@ +# --------------------------------------------------------------------- +# Gufo Thor: kafka service +# --------------------------------------------------------------------- +# Copyright (C) 2023, Gufo Labs +# --------------------------------------------------------------------- +""" +kafka service. + +Attributes: + kafka: kafka service singleton. +""" + +# Gufo Thor modules +from .base import BaseService + + +class KafkaService(BaseService): + """kafka service.""" + + name = "kafka" + compose_image = "bitnami/kafka:3.6.2" + compose_volumes = [ + "kafka_data:/data/", + ] + compose_volumes_config = {"kafka_data": {}} + service_discovery = {"kafka": 9093} + compose_environment = { + "KAFKA_CFG_NODE_ID": "0", + "KAFKA_CFG_PROCESS_ROLES": "controller,broker", + "KAFKA_CFG_LISTENERS": "PLAINTEXT://:9092,CONTROLLER://:9093", + "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP": "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT", + "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS": "0@kafka:9093", + "KAFKA_CFG_CONTROLLER_LISTENER_NAMES": "CONTROLLER", + "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE": "false", + } + + +kafka = KafkaService() diff --git a/src/gufo/thor/services/liftbridge.py b/src/gufo/thor/services/liftbridge.py deleted file mode 100644 index a7c7302..0000000 --- a/src/gufo/thor/services/liftbridge.py +++ /dev/null @@ -1,46 +0,0 @@ -# --------------------------------------------------------------------- -# Gufo Thor: liftbridge service -# --------------------------------------------------------------------- -# Copyright (C) 2023, Gufo Labs -# --------------------------------------------------------------------- -""" -liftbridge service. - -Attributes: - liftbridge: liftbridge service singleton. -""" - -# Python modules -from pathlib import Path -from typing import List, Optional - -# Gufo Thor modules -from ..config import Config, ServiceConfig -from .base import BaseService - - -class LiftbridgeService(BaseService): - """liftbridge service.""" - - name = "liftbridge" - compose_image = "liftbridge/liftbridge:v1.9.0" - compose_entrypoint = "liftbridge --config /etc/liftbridge.yml" - compose_volumes = [ - "./etc/liftbridge.yml:/etc/liftbridge.yml:ro", - "liftbridge_data:/data/", - ] - compose_volumes_config = {"liftbridge_data": {}} - service_discovery = {"liftbridge": 9292} - compose_extra = {"user": "root"} - - def prepare_compose_config( - self: "LiftbridgeService", - config: Config, - svc: Optional[ServiceConfig], - services: List["BaseService"], - ) -> None: - """Generate config.""" - self.render_file(Path("etc", "liftbridge.yml"), "liftbridge.yml") - - -liftbridge = LiftbridgeService() diff --git a/src/gufo/thor/services/migrate.py b/src/gufo/thor/services/migrate.py index 5f76e03..9bac278 100644 --- a/src/gufo/thor/services/migrate.py +++ b/src/gufo/thor/services/migrate.py @@ -20,7 +20,7 @@ from .base import ComposeDependsCondition from .clickhouse import clickhouse from .consul import consul -from .liftbridge import liftbridge +from .kafka import kafka from .mongo import mongo from .noc import NocService from .postgres import postgres @@ -38,7 +38,7 @@ class MigrateService(NocService): """ name = "migrate" - dependencies = (clickhouse, consul, liftbridge, mongo, postgres) + dependencies = (clickhouse, consul, kafka, mongo, postgres) compose_depends_condition = ComposeDependsCondition.COMPLETED_SUCCESSFULLY compose_command = "./scripts/deploy/migrate.sh" diff --git a/src/gufo/thor/services/ping.py b/src/gufo/thor/services/ping.py index 31c6155..0b9c390 100644 --- a/src/gufo/thor/services/ping.py +++ b/src/gufo/thor/services/ping.py @@ -12,7 +12,7 @@ # Gufo Thor modules from .datastream import datastream -from .liftbridge import liftbridge +from .kafka import kafka from .migrate import migrate from .noc import NocService @@ -21,7 +21,7 @@ class PingService(NocService): """ping service.""" name = "ping" - dependencies = (datastream, liftbridge, migrate) + dependencies = (datastream, kafka, migrate) compose_extra = { "cap_add": [ "NET_RAW", diff --git a/src/gufo/thor/services/runner.py b/src/gufo/thor/services/runner.py index 92c211a..9974ce6 100644 --- a/src/gufo/thor/services/runner.py +++ b/src/gufo/thor/services/runner.py @@ -11,7 +11,7 @@ """ # Gufo Thor modules -from .liftbridge import liftbridge +from .kafka import kafka from .migrate import migrate from .mongo import mongo from .noc import NocService @@ -21,7 +21,7 @@ class RunnerService(NocService): """runner service.""" name = "runner" - dependencies = (liftbridge, migrate, mongo) + dependencies = (kafka, migrate, mongo) allow_scale = False require_slots = False diff --git a/src/gufo/thor/services/syslogcollector.py b/src/gufo/thor/services/syslogcollector.py index 2046523..ba147d1 100644 --- a/src/gufo/thor/services/syslogcollector.py +++ b/src/gufo/thor/services/syslogcollector.py @@ -12,7 +12,7 @@ # Gufo Thor modules from .datastream import datastream -from .liftbridge import liftbridge +from .kafka import kafka from .migrate import migrate from .noc import NocService @@ -21,7 +21,7 @@ class SyslogcollectorService(NocService): """syslogcollector service.""" name = "syslogcollector" - dependencies = (datastream, liftbridge, migrate) + dependencies = (datastream, kafka, migrate) syslogcollector = SyslogcollectorService() diff --git a/src/gufo/thor/services/topo.py b/src/gufo/thor/services/topo.py index 94a5652..b8b606f 100644 --- a/src/gufo/thor/services/topo.py +++ b/src/gufo/thor/services/topo.py @@ -12,7 +12,7 @@ # Gufo Thor modules from .datastream import datastream -from .liftbridge import liftbridge +from .kafka import kafka from .migrate import migrate from .noc import NocService @@ -21,7 +21,7 @@ class TopoService(NocService): """topo service.""" name = "topo" - dependencies = (datastream, liftbridge, migrate) + dependencies = (datastream, kafka, migrate) topo = TopoService() diff --git a/src/gufo/thor/services/trapcollector.py b/src/gufo/thor/services/trapcollector.py index 27c36c6..fdddbd1 100644 --- a/src/gufo/thor/services/trapcollector.py +++ b/src/gufo/thor/services/trapcollector.py @@ -12,7 +12,7 @@ # Gufo Thor modules from .datastream import datastream -from .liftbridge import liftbridge +from .kafka import kafka from .migrate import migrate from .noc import NocService @@ -21,7 +21,7 @@ class TrapcollectorService(NocService): """trapcollector service.""" name = "trapcollector" - dependencies = (datastream, liftbridge, migrate) + dependencies = (datastream, kafka, migrate) trapcollector = TrapcollectorService() diff --git a/src/gufo/thor/services/worker.py b/src/gufo/thor/services/worker.py index b7dc2a5..ebb49e4 100644 --- a/src/gufo/thor/services/worker.py +++ b/src/gufo/thor/services/worker.py @@ -12,7 +12,7 @@ # Gufo Thor modules from .datastream import datastream -from .liftbridge import liftbridge +from .kafka import kafka from .migrate import migrate from .mongo import mongo from .noc import NocService @@ -23,7 +23,7 @@ class WorkerService(NocService): """worker service.""" name = "worker" - dependencies = (datastream, liftbridge, migrate, mongo, postgres) + dependencies = (datastream, kafka, migrate, mongo, postgres) allow_scale = True require_slots = True diff --git a/src/gufo/thor/templates/liftbridge/liftbridge.yml b/src/gufo/thor/templates/liftbridge/liftbridge.yml deleted file mode 100644 index 5337f09..0000000 --- a/src/gufo/thor/templates/liftbridge/liftbridge.yml +++ /dev/null @@ -1,17 +0,0 @@ -listen: 0.0.0.0:9292 -host: liftbridge -data: - dir: /data -nats: - embedded: true -logging: - level: info - raft: true -streams: - compact.enabled: true - retention.max: - age: 24h -cursors: - stream.partitions: 1 -clustering: - raft.bootstrap.seed: true diff --git a/src/gufo/thor/templates/noc/settings.yml b/src/gufo/thor/templates/noc/settings.yml index 515de72..ea89365 100644 --- a/src/gufo/thor/templates/noc/settings.yml +++ b/src/gufo/thor/templates/noc/settings.yml @@ -7,3 +7,7 @@ pg: web: theme: > {{theme}} +msgstream: + client_class: noc.core.msgstream.redpanda.RedPandaClient +redpanda: + addresses: kafka:9092 diff --git a/tests/test_service.py b/tests/test_service.py index 33273d6..f91407d 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -22,7 +22,7 @@ from gufo.thor.services.consul import consul from gufo.thor.services.datastream import datastream from gufo.thor.services.envoy import envoy -from gufo.thor.services.liftbridge import liftbridge +from gufo.thor.services.kafka import kafka from gufo.thor.services.login import login from gufo.thor.services.migrate import migrate from gufo.thor.services.mongo import mongo @@ -55,7 +55,7 @@ def test_depends_sorted(svc: str) -> None: consul, datastream, envoy, - liftbridge, + kafka, login, migrate, mongo, @@ -133,7 +133,7 @@ def test_migrate_deps(svc: str) -> None: assert ( migrate in deps ), "Depends on `clickhouse`, must depend on `migrate`" - if liftbridge in deps: + if kafka in deps: assert ( migrate in deps ), "Depends on `liftbridge`, must depend on `migrate`"