fix: update examples and improve builtin ipam

examples/multi-bootstrap/docker-compose.yaml

@@ -8,11 +8,6 @@ networks:
         - subnet: 10.1.0.0/24
           gateway: 10.1.0.100
 
-volumes:
-  bootstrap-node-1:
-  bootstrap-node-2:
-  bootstrap-node-3:
-
 services:
   bootstrap-node-1:
     image: ${IMAGE:-ghcr.io/webmeshproj/node:latest}
@@ -29,31 +24,18 @@ services:
       - --global.no-ipv6
       - --global.detect-endpoints
       - --global.detect-private-endpoints
-      - --global.log-level=debug
       - --bootstrap.enabled
       - --bootstrap.servers=bootstrap-node-1=10.1.0.1:9443,bootstrap-node-2=10.1.0.2:9443,bootstrap-node-3=10.1.0.3:9443
       - --bootstrap.ipv4-network=10.10.10.0/24
-      - --raft.data-dir=/data
-      - --raft.snapshot-threshold=10
-      - --raft.log-level=debug
+      - --raft.in-memory
       - --services.api.peer-discovery
       - --services.api.mesh
-      - --services.api.webrtc
       - --services.metrics.enabled
-      - --services.turn.enabled
-      - --services.turn.public-ip=127.0.0.1
-      - --services.turn.stun-port-range=60000-60001
       - --services.mesh-dns.enabled
-      - --wireguard.key-file=/data/wireguard.key
     ports:
       - 8443:8443
-      - 3478:3478
       - 5354:5353/udp
       - 51820:51820/udp
-      - 60000-60001:60000-60001/udp
-      - 60000-60001:60000-60001/tcp
-    volumes:
-      - bootstrap-node-1:/data
     cap_add: ["NET_ADMIN", "NET_RAW", "SYS_MODULE"]
 
   bootstrap-node-2:
@@ -68,17 +50,10 @@ services:
       - --global.no-ipv6
       - --global.detect-endpoints
       - --global.detect-private-endpoints
-      - --global.log-level=debug
       - --bootstrap.enabled
       - --bootstrap.servers=bootstrap-node-1=10.1.0.1:9443,bootstrap-node-2=10.1.0.2:9443,bootstrap-node-3=10.1.0.3:9443
       - --bootstrap.ipv4-network=10.10.10.0/24
-      - --raft.data-dir=/data
-      - --raft.snapshot-threshold=10
-      - --raft.log-level=debug
-      - --services.api.leader-proxy
-      - --wireguard.key-file=/data/wireguard.key
-    volumes:
-      - bootstrap-node-2:/data
+      - --raft.in-memory
     cap_add: ["NET_ADMIN", "NET_RAW", "SYS_MODULE"]
 
   bootstrap-node-3:
@@ -93,15 +68,8 @@ services:
       - --global.no-ipv6
       - --global.detect-endpoints
       - --global.detect-private-endpoints
-      - --global.log-level=debug
       - --bootstrap.enabled
       - --bootstrap.servers=bootstrap-node-1=10.1.0.1:9443,bootstrap-node-2=10.1.0.2:9443,bootstrap-node-3=10.1.0.3:9443
       - --bootstrap.ipv4-network=10.10.10.0/24
-      - --raft.data-dir=/data
-      - --raft.snapshot-threshold=10
-      - --raft.log-level=debug
-      - --services.api.leader-proxy
-      - --wireguard.key-file=/data/wireguard.key
-    volumes:
-      - bootstrap-node-3:/data
+      - --raft.in-memory
     cap_add: ["NET_ADMIN", "NET_RAW", "SYS_MODULE"]

examples/site-to-site/docker-compose.yaml

@@ -23,11 +23,6 @@ networks:
       config:
         - subnet: 10.3.0.0/24
 
-volumes:
-  site-1-leader:
-  site-2-leader:
-  site-3-leader:
-
 services:
   # Site 1
 
@@ -52,7 +47,7 @@ services:
       - --bootstrap.servers=site-1-leader=site-1-leader:9443,site-2-leader=site-2-leader:9443,site-3-leader=site-3-leader:9443
       - --bootstrap.ipv4-network=10.10.10.0/24
       - --bootstrap.default-network-policy=accept
-      - --raft.data-dir=/data
+      - --raft.in-memory
       - --mesh.zone-awareness-id=site-1
       - --services.api.mesh
       - --services.api.admin
@@ -66,8 +61,6 @@ services:
       - 8080:8080
       - 8443:8443
       - 5354:5353/udp
-    volumes:
-      - site-1-leader:/data
     cap_add: ["NET_ADMIN", "NET_RAW", "SYS_MODULE"]
 
   site-1-follower-1:
@@ -139,10 +132,8 @@ services:
       - --bootstrap.servers=site-1-leader=site-1-leader:9443,site-2-leader=site-2-leader:9443,site-3-leader=site-3-leader:9443
       - --bootstrap.ipv4-network=10.10.10.0/24
       - --bootstrap.default-network-policy=accept
-      - --raft.data-dir=/data
+      - --raft.in-memory
       - --mesh.zone-awareness-id=site-2
-    volumes:
-      - site-2-leader:/data
     cap_add: ["NET_ADMIN", "NET_RAW", "SYS_MODULE"]
 
   site-2-follower-1:
@@ -214,10 +205,8 @@ services:
       - --bootstrap.servers=site-1-leader=site-1-leader:9443,site-2-leader=site-2-leader:9443,site-3-leader=site-3-leader:9443
       - --bootstrap.ipv4-network=10.10.10.0/24
       - --bootstrap.default-network-policy=accept
-      - --raft.data-dir=/data
+      - --raft.in-memory
       - --mesh.zone-awareness-id=site-3
-    volumes:
-      - site-3-leader:/data
     cap_add: ["NET_ADMIN", "NET_RAW", "SYS_MODULE"]
 
   site-3-follower-1:

pkg/mesh/mesh_open.go

@@ -91,11 +91,6 @@ func (s *meshStore) Open(ctx context.Context, features []v1.Feature) (err error)
 		}
 		return cause
 	}
-	// Register an update hook to watch for node changes.
-	s.kvSubCancel, err = s.raft.Storage().Subscribe(context.Background(), "", s.onDBUpdate)
-	if err != nil {
-		return handleErr(fmt.Errorf("subscribe: %w", err))
-	}
 	// Create the network manager
 	s.nw = net.New(s.Storage(), &net.Options{
 		NodeID:                s.ID(),
@@ -139,5 +134,10 @@ func (s *meshStore) Open(ctx context.Context, features []v1.Feature) (err error)
 			return fmt.Errorf("recover wireguard: %w", err)
 		}
 	}
+	// Register an update hook to watch for network changes.
+	s.kvSubCancel, err = s.raft.Storage().Subscribe(context.Background(), "", s.onDBUpdate)
+	if err != nil {
+		return handleErr(fmt.Errorf("subscribe: %w", err))
+	}
 	return nil
 }

pkg/plugins/clients/streams.go

@@ -29,8 +29,9 @@ import (
 )
 
 func inProcessQueryPipe(ctx context.Context, server v1.PluginServer) v1.Plugin_InjectQuerierClient {
-	schan := make(chan *v1.PluginQuery, 1)
-	rchan := make(chan *v1.PluginQueryResult, 1)
+	// TODO: Make this configurable
+	schan := make(chan *v1.PluginQuery, 100)
+	rchan := make(chan *v1.PluginQueryResult, 100)
 	ctx, cancel := context.WithCancel(ctx)
 	srv := &inProcessStreamServer[v1.PluginQuery, v1.PluginQueryResult]{ctx, schan, rchan}
 	cli := &inProcessStreamClient[v1.PluginQuery, v1.PluginQueryResult]{ctx, cancel, schan, rchan}

pkg/plugins/ipam/plugin.go

@@ -26,6 +26,7 @@ import (
 
 	_ "github.com/mattn/go-sqlite3"
 	v1 "github.com/webmeshproj/api/v1"
+	"golang.org/x/exp/slog"
 	"google.golang.org/protobuf/types/known/emptypb"
 
 	"github.com/webmeshproj/webmesh/pkg/context"
@@ -103,25 +104,24 @@ func (p *Plugin) allocateV4(ctx context.Context, r *v1.AllocateIPRequest) (*v1.A
 	if err != nil {
 		return nil, fmt.Errorf("parse subnet: %w", err)
 	}
-	var allocated []netip.Prefix
 	nodes, err := peers.New(p.data).List(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("list nodes: %w", err)
 	}
+	allocated := make(map[netip.Prefix]struct{}, len(nodes))
+	context.LoggerFrom(ctx).Info("ipam list nodes", slog.Any("nodes", nodes))
 	for _, node := range nodes {
 		n := node
 		if n.PrivateIPv4.IsValid() {
-			allocated = append(allocated, n.PrivateIPv4)
+			allocated[n.PrivateIPv4] = struct{}{}
 		}
 	}
-	allocatedSet, err := toPrefixSet(allocated)
-	if err != nil {
-		return nil, fmt.Errorf("convert allocated IPv4s to set: %w", err)
-	}
-	prefix, err := next32(globalPrefix, allocatedSet)
+	context.LoggerFrom(ctx).Info("ipam allocated ips", slog.Any("allocated", allocated))
+	prefix, err := next32(globalPrefix, allocated)
 	if err != nil {
 		return nil, fmt.Errorf("find next available IPv4: %w", err)
 	}
+	context.LoggerFrom(ctx).Info("ipam allocated", slog.Any("allocated", prefix))
 	return &v1.AllocatedIP{
 		Ip: prefix.String(),
 	}, nil
@@ -157,12 +157,3 @@ func next32(cidr netip.Prefix, set map[netip.Prefix]struct{}) (netip.Prefix, err
 	}
 	return netip.Prefix{}, fmt.Errorf("no more addresses in %s", cidr)
 }
-
-func toPrefixSet(addrs []netip.Prefix) (map[netip.Prefix]struct{}, error) {
-	set := make(map[netip.Prefix]struct{})
-	for _, addr := range addrs {
-		ip := addr
-		set[ip] = struct{}{}
-	}
-	return set, nil
-}

pkg/plugins/manager.go

@@ -299,9 +299,9 @@ func (m *manager) handleQueryClient(plugin string, db storage.Storage, client cl
 				m.log.Error("send query result", "plugin", plugin, "error", err)
 			}
 		case v1.PluginQuery_ITER:
-			var result v1.PluginQueryResult
-			result.Id = query.GetId()
 			err := db.IterPrefix(queries.Context(), query.GetQuery(), func(key, val string) error {
+				var result v1.PluginQueryResult
+				result.Id = query.GetId()
 				result.Key = key
 				result.Value = []string{val}
 				err := queries.Send(&result)
@@ -311,6 +311,8 @@ func (m *manager) handleQueryClient(plugin string, db storage.Storage, client cl
 				m.log.Error("stream query results", "plugin", plugin, "error", err)
 				continue
 			}
+			var result v1.PluginQueryResult
+			result.Id = query.GetId()
 			result.Error = "EOF"
 			err = queries.Send(&result)
 			if err != nil {