Merge pull request #1415 from openziti/fix-xctrl-startup-race

Ensure control chan is available before starting xctrls. Fixes #1414

router/router.go

@@ -21,11 +21,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/openziti/ziti/common/config"
-	"github.com/openziti/ziti/router/link"
 	"github.com/openziti/foundation/v2/debugz"
 	"github.com/openziti/foundation/v2/goroutines"
 	"github.com/openziti/xweb/v2"
+	"github.com/openziti/ziti/common/config"
+	"github.com/openziti/ziti/router/link"
 	metrics2 "github.com/rcrowley/go-metrics"
 	"io/fs"
 	"os"
@@ -39,6 +39,12 @@ import (
 	"github.com/AppsFlyer/go-sundheit/checks"
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
+	"github.com/openziti/foundation/v2/concurrenz"
+	"github.com/openziti/foundation/v2/errorz"
+	"github.com/openziti/foundation/v2/versions"
+	"github.com/openziti/identity"
+	"github.com/openziti/metrics"
+	"github.com/openziti/transport/v2"
 	"github.com/openziti/ziti/common/health"
 	fabricMetrics "github.com/openziti/ziti/common/metrics"
 	"github.com/openziti/ziti/common/pb/ctrl_pb"
@@ -55,12 +61,6 @@ import (
 	"github.com/openziti/ziti/router/xgress_transport_udp"
 	"github.com/openziti/ziti/router/xlink"
 	"github.com/openziti/ziti/router/xlink_transport"
-	"github.com/openziti/foundation/v2/concurrenz"
-	"github.com/openziti/foundation/v2/errorz"
-	"github.com/openziti/foundation/v2/versions"
-	"github.com/openziti/identity"
-	"github.com/openziti/metrics"
-	"github.com/openziti/transport/v2"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/proto"
@@ -531,12 +531,6 @@ func (self *Router) startControlPlane() error {
 
 	self.ctrls.UpdateControllerEndpoints(endpoints)
 
-	for _, x := range self.xrctrls {
-		if err := x.Run(self); err != nil {
-			return err
-		}
-	}
-
 	self.metricsReporter = fabricMetrics.NewControllersReporter(self.ctrls)
 	self.metricsRegistry.StartReporting(self.metricsReporter, self.config.Metrics.ReportInterval, self.config.Metrics.MessageQueueSize)
 
@@ -549,6 +543,13 @@ func (self *Router) startControlPlane() error {
 		}
 	})
 
+	_ = self.ctrls.AnyValidCtrlChannel()
+	for _, x := range self.xrctrls {
+		if err := x.Run(self); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

router/xgress_edge_tunnel/fabric.go

@@ -24,18 +24,18 @@ import (
 	"github.com/openziti/channel/v2"
 	"github.com/openziti/channel/v2/protobufs"
 	"github.com/openziti/edge-api/rest_model"
-	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
-	"github.com/openziti/ziti/router/xgress_common"
-	"github.com/openziti/ziti/tunnel"
-	"github.com/openziti/ziti/common/build"
-	"github.com/openziti/ziti/common/ctrl_msg"
-	"github.com/openziti/ziti/router/xgress"
 	"github.com/openziti/foundation/v2/concurrenz"
 	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/sdk-golang/ziti"
 	"github.com/openziti/sdk-golang/ziti/edge"
 	"github.com/openziti/sdk-golang/ziti/sdkinfo"
 	"github.com/openziti/secretstream/kx"
+	"github.com/openziti/ziti/common/build"
+	"github.com/openziti/ziti/common/ctrl_msg"
+	"github.com/openziti/ziti/common/pb/edge_ctrl_pb"
+	"github.com/openziti/ziti/router/xgress"
+	"github.com/openziti/ziti/router/xgress_common"
+	"github.com/openziti/ziti/tunnel"
 	cmap "github.com/orcaman/concurrent-map/v2"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -168,12 +168,20 @@ func (self *fabricProvider) authenticate() error {
 	}
 
 	ctrlMap := self.factory.ctrls.GetAll()
-	results := newAuthResults(len(ctrlMap))
+	ctrlChMap := map[string]channel.Channel{}
 	for k, v := range ctrlMap {
+		ctrlChMap[k] = v.Channel()
+	}
+	if len(ctrlChMap) == 0 {
+		ctrlCh := self.factory.ctrls.AnyValidCtrlChannel()
+		ctrlChMap[ctrlCh.Id()] = ctrlCh
+	}
+	results := newAuthResults(len(ctrlChMap))
+	for k, v := range ctrlChMap {
 		ctrlId := k
-		ctrl := v
+		ctrlCh := v
 		go func() {
-			respMsg, err := protobufs.MarshalTyped(request).WithTimeout(30 * time.Second).SendForReply(ctrl.Channel())
+			respMsg, err := protobufs.MarshalTyped(request).WithTimeout(30 * time.Second).SendForReply(ctrlCh)
 
 			resp := &edge_ctrl_pb.CreateApiSessionResponse{}
 			if err = xgress_common.GetResultOrFailure(respMsg, err, resp); err != nil {