Replace liftbridge with kafka

CHANGELOG.md

@@ -20,6 +20,10 @@ To see unreleased changes, please see the [CHANGELOG on the master branch](https
 
 * `restart` command.
 
+### Changed
+
+* `liftbridge` service replaced with `kafka`
+
 ## 0.6.0 - 2024-08-05
 
 ### Added

src/gufo/thor/services/activator.py

@@ -11,7 +11,7 @@
 """
 
 # Gufo Thor modules
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .migrate import migrate
 from .noc import NocService
 from .sae import sae
@@ -21,7 +21,7 @@ class ActivatorService(NocService):
     """activator service."""
 
     name = "activator"
-    dependencies = (liftbridge, migrate, sae)
+    dependencies = (kafka, migrate, sae)
 
 
 activator = ActivatorService()

src/gufo/thor/services/auth.py

@@ -12,7 +12,7 @@
 
 # Gufo Thor modules
 from .envoy import envoy
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .migrate import migrate
 from .mongo import mongo
 from .noc import NocHcService
@@ -22,7 +22,7 @@ class AuthService(NocHcService):
     """auth service."""
 
     name = "auth"
-    dependencies = (envoy, liftbridge, migrate, mongo)
+    dependencies = (envoy, kafka, migrate, mongo)
     allow_scale = True
     expose_http_prefix = "/api/auth/"
     compose_command = (

src/gufo/thor/services/chwriter.py

@@ -12,7 +12,7 @@
 
 # Gufo Thor modules
 from .clickhouse import clickhouse
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .migrate import migrate
 from .noc import NocService
 
@@ -21,7 +21,7 @@ class ChwriterService(NocService):
     """chwriter service."""
 
     name = "chwriter"
-    dependencies = (clickhouse, liftbridge, migrate)
+    dependencies = (clickhouse, kafka, migrate)
 
 
 chwriter = ChwriterService()

src/gufo/thor/services/classifier.py

@@ -11,7 +11,7 @@
 """
 
 # Gufo Thor modules
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .migrate import migrate
 from .mongo import mongo
 from .noc import NocService
@@ -22,7 +22,7 @@ class ClassifierService(NocService):
     """classifier service."""
 
     name = "classifier"
-    dependencies = (liftbridge, migrate, mongo, postgres)
+    dependencies = (kafka, migrate, mongo, postgres)
 
 
 classifier = ClassifierService()

src/gufo/thor/services/correlator.py

@@ -11,7 +11,7 @@
 """
 
 # Gufo Thor modules
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .migrate import migrate
 from .mongo import mongo
 from .noc import NocService
@@ -22,7 +22,7 @@ class CorrelatorService(NocService):
     """correlator service."""
 
     name = "correlator"
-    dependencies = (liftbridge, migrate, mongo, postgres)
+    dependencies = (kafka, migrate, mongo, postgres)
 
 
 correlator = CorrelatorService()

src/gufo/thor/services/kafka.py

@@ -0,0 +1,38 @@
+# ---------------------------------------------------------------------
+# Gufo Thor: kafka service
+# ---------------------------------------------------------------------
+# Copyright (C) 2023, Gufo Labs
+# ---------------------------------------------------------------------
+"""
+kafka service.
+
+Attributes:
+    kafka: kafka service singleton.
+"""
+
+# Gufo Thor modules
+from .base import BaseService
+
+
+class KafkaService(BaseService):
+    """kafka service."""
+
+    name = "kafka"
+    compose_image = "bitnami/kafka:3.6.2"
+    compose_volumes = [
+        "kafka_data:/data/",
+    ]
+    compose_volumes_config = {"kafka_data": {}}
+    service_discovery = {"kafka": 9093}
+    compose_environment = {
+        "KAFKA_CFG_NODE_ID": "0",
+        "KAFKA_CFG_PROCESS_ROLES": "controller,broker",
+        "KAFKA_CFG_LISTENERS": "PLAINTEXT://:9092,CONTROLLER://:9093",
+        "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP": "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT",
+        "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS": "0@kafka:9093",
+        "KAFKA_CFG_CONTROLLER_LISTENER_NAMES": "CONTROLLER",
+        "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE": "false",
+    }
+
+
+kafka = KafkaService()

src/gufo/thor/services/migrate.py

@@ -20,7 +20,7 @@
 from .base import ComposeDependsCondition
 from .clickhouse import clickhouse
 from .consul import consul
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .mongo import mongo
 from .noc import NocService
 from .postgres import postgres
@@ -38,7 +38,7 @@ class MigrateService(NocService):
     """
 
     name = "migrate"
-    dependencies = (clickhouse, consul, liftbridge, mongo, postgres)
+    dependencies = (clickhouse, consul, kafka, mongo, postgres)
     compose_depends_condition = ComposeDependsCondition.COMPLETED_SUCCESSFULLY
     compose_command = "./scripts/deploy/migrate.sh"
 

src/gufo/thor/services/ping.py

@@ -12,7 +12,7 @@
 
 # Gufo Thor modules
 from .datastream import datastream
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .migrate import migrate
 from .noc import NocService
 
@@ -21,7 +21,7 @@ class PingService(NocService):
     """ping service."""
 
     name = "ping"
-    dependencies = (datastream, liftbridge, migrate)
+    dependencies = (datastream, kafka, migrate)
     compose_extra = {
         "cap_add": [
             "NET_RAW",

src/gufo/thor/services/runner.py

@@ -11,7 +11,7 @@
 """
 
 # Gufo Thor modules
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .migrate import migrate
 from .mongo import mongo
 from .noc import NocService
@@ -21,7 +21,7 @@ class RunnerService(NocService):
     """runner service."""
 
     name = "runner"
-    dependencies = (liftbridge, migrate, mongo)
+    dependencies = (kafka, migrate, mongo)
     allow_scale = False
     require_slots = False
 

src/gufo/thor/services/syslogcollector.py

@@ -12,7 +12,7 @@
 
 # Gufo Thor modules
 from .datastream import datastream
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .migrate import migrate
 from .noc import NocService
 
@@ -21,7 +21,7 @@ class SyslogcollectorService(NocService):
     """syslogcollector service."""
 
     name = "syslogcollector"
-    dependencies = (datastream, liftbridge, migrate)
+    dependencies = (datastream, kafka, migrate)
 
 
 syslogcollector = SyslogcollectorService()

src/gufo/thor/services/topo.py

@@ -12,7 +12,7 @@
 
 # Gufo Thor modules
 from .datastream import datastream
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .migrate import migrate
 from .noc import NocService
 
@@ -21,7 +21,7 @@ class TopoService(NocService):
     """topo service."""
 
     name = "topo"
-    dependencies = (datastream, liftbridge, migrate)
+    dependencies = (datastream, kafka, migrate)
 
 
 topo = TopoService()

src/gufo/thor/services/trapcollector.py

@@ -12,7 +12,7 @@
 
 # Gufo Thor modules
 from .datastream import datastream
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .migrate import migrate
 from .noc import NocService
 
@@ -21,7 +21,7 @@ class TrapcollectorService(NocService):
     """trapcollector service."""
 
     name = "trapcollector"
-    dependencies = (datastream, liftbridge, migrate)
+    dependencies = (datastream, kafka, migrate)
 
 
 trapcollector = TrapcollectorService()

src/gufo/thor/services/worker.py

@@ -12,7 +12,7 @@
 
 # Gufo Thor modules
 from .datastream import datastream
-from .liftbridge import liftbridge
+from .kafka import kafka
 from .migrate import migrate
 from .mongo import mongo
 from .noc import NocService
@@ -23,7 +23,7 @@ class WorkerService(NocService):
     """worker service."""
 
     name = "worker"
-    dependencies = (datastream, liftbridge, migrate, mongo, postgres)
+    dependencies = (datastream, kafka, migrate, mongo, postgres)
     allow_scale = True
     require_slots = True
 

src/gufo/thor/templates/noc/settings.yml

@@ -7,3 +7,7 @@ pg:
 web:
   theme: >
     {{theme}}
+msgstream:
+  client_class: noc.core.msgstream.redpanda.RedPandaClient
+redpanda:
+  addresses: kafka:9092

tests/test_service.py

@@ -22,7 +22,7 @@
 from gufo.thor.services.consul import consul
 from gufo.thor.services.datastream import datastream
 from gufo.thor.services.envoy import envoy
-from gufo.thor.services.liftbridge import liftbridge
+from gufo.thor.services.kafka import kafka
 from gufo.thor.services.login import login
 from gufo.thor.services.migrate import migrate
 from gufo.thor.services.mongo import mongo
@@ -55,7 +55,7 @@ def test_depends_sorted(svc: str) -> None:
                 consul,
                 datastream,
                 envoy,
-                liftbridge,
+                kafka,
                 login,
                 migrate,
                 mongo,
@@ -133,7 +133,7 @@ def test_migrate_deps(svc: str) -> None:
         assert (
             migrate in deps
         ), "Depends on `clickhouse`, must depend on `migrate`"
-    if liftbridge in deps:
+    if kafka in deps:
         assert (
             migrate in deps
         ), "Depends on `liftbridge`, must depend on `migrate`"