From 2583621b2cc20fcc1e4773ed9be54f6a92b88b14 Mon Sep 17 00:00:00 2001
From: Ying Huang <sindica2000@yahoo.com>
Date: Wed, 27 Nov 2019 17:03:50 -0800
Subject: [PATCH] Expose workload manager in insecure and secure ports. (#300)

+ Allow workload-controller-manager starts from different port: support multiple instances in the same node.
---
 cmd/kubeadm/app/constants/constants.go  |  4 ++
 cmd/workload-controller-manager/main.go | 57 +++++++++++++++----------
 pkg/master/ports/ports.go               | 10 +++++
 3 files changed, 49 insertions(+), 22 deletions(-)

diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go
index bfc02075f1a..3d20acdd1c4 100644
--- a/cmd/kubeadm/app/constants/constants.go
+++ b/cmd/kubeadm/app/constants/constants.go
@@ -269,6 +269,10 @@ const (
 	KubeAPIServer = "kube-apiserver"
 	// KubeControllerManager defines variable used internally when referring to kube-controller-manager component
 	KubeControllerManager = "kube-controller-manager"
+
+	// WorkloadControllerManager defines variable used internally when referring to workload-controller-manager component
+	WorkloadControllerManager = "workload-controller-manager"
+
 	// KubeScheduler defines variable used internally when referring to kube-scheduler component
 	KubeScheduler = "kube-scheduler"
 	// KubeProxy defines variable used internally when referring to kube-proxy component
diff --git a/cmd/workload-controller-manager/main.go b/cmd/workload-controller-manager/main.go
index 24cd00558f7..36a20dc3347 100644
--- a/cmd/workload-controller-manager/main.go
+++ b/cmd/workload-controller-manager/main.go
@@ -19,20 +19,27 @@ package main
 import (
 	"flag"
 	"fmt"
+	"k8s.io/klog"
+	"net"
 	"os"
 
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/component-base/logs"
+	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
+	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
 	"k8s.io/kubernetes/cmd/workload-controller-manager/app"
 
+	apiserveroptions "k8s.io/apiserver/pkg/server/options"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	controllerManagerConfig "k8s.io/kubernetes/cmd/workload-controller-manager/app/config"
+	"k8s.io/kubernetes/pkg/master/ports"
 )
 
 var kubeconfig string
 var controllerconfigfilepath string
+var workloadControllerPort int
 
 const (
 	// WorkloadControllerManagerUserAgent is the userAgent name when starting workload-controller managers.
@@ -43,49 +50,42 @@ func init() {
 	kubeconfigEnv := os.Getenv("KUBECONFIG")
 	flag.StringVar(&kubeconfig, "kubeconfig", kubeconfigEnv, "absolute path to the kubeconfig file")
 	flag.StringVar(&controllerconfigfilepath, "controllerconfig", "", "absolute path to the controllerconfig file")
+	flag.IntVar(&workloadControllerPort, "port", ports.InsecureWorkloadControllerManagerPort, "port for current workload controller manager rest service")
 	flag.Parse()
 }
 
 func main() {
-
-	var err error
-
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
-	controllerconfig, err := controllerManagerConfig.NewControllerConfig(controllerconfigfilepath)
-
-	if err != nil {
-		panic(err)
-	} else {
-		fmt.Println("using controller configuration from ", controllerconfigfilepath)
-	}
-
+	c, err := getConfig()
 	if err != nil {
 		panic(err)
 	}
 
-	/*
-		store := WatchResources(clientSet)
+	//eventRecorder := createRecorder(client, WorkloadControllerManagerUserAgent)
+	app.StartControllerManager(c.Complete(), wait.NeverStop)
+}
 
-		for {
-			controllermanagersFromStore := store.List()
-			fmt.Printf("controllermanagers in store: %d\n", len(controllermanagersFromStore))
+func getConfig() (*controllerManagerConfig.Config, error) {
+	controllerconfig, err := controllerManagerConfig.NewControllerConfig(controllerconfigfilepath)
 
-			time.Sleep(2 * time.Second)
-		}*/
+	if err != nil {
+		return nil, err
+	} else {
+		fmt.Println("using controller configuration from ", controllerconfigfilepath)
+	}
 
 	controllerManagerKubeConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
 
 	client, err := clientset.NewForConfig(restclient.AddUserAgent(controllerManagerKubeConfig, WorkloadControllerManagerUserAgent))
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
 
-	//eventRecorder := createRecorder(client, WorkloadControllerManagerUserAgent)
 
 	c := &controllerManagerConfig.Config{
 		Client:                  client,
@@ -94,7 +94,20 @@ func main() {
 		//EventRecorder:        eventRecorder,
 	}
 
-	app.StartControllerManager(c.Complete(), wait.NeverStop)
+	klog.Infof("Current workload controller port %d", workloadControllerPort)
+
+	insecureServing := (&apiserveroptions.DeprecatedInsecureServingOptions{
+		BindAddress: net.ParseIP("0.0.0.0"),
+		BindPort:    workloadControllerPort,
+		BindNetwork: "tcp",
+	}).WithLoopback()
+
+	err = insecureServing.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
 }
 
 /*
diff --git a/pkg/master/ports/ports.go b/pkg/master/ports/ports.go
index 23faba1d3ec..07841935d02 100644
--- a/pkg/master/ports/ports.go
+++ b/pkg/master/ports/ports.go
@@ -31,6 +31,16 @@ const (
 	// May be overridden by a flag at startup.
 	// Deprecated: use the secure KubeControllerManagerPort instead.
 	InsecureKubeControllerManagerPort = 10252
+
+	// InsecureWorkloadControllerManagerPort is the default port for the workload controller manager status server.
+	// May be overridden by a flag at startup.
+	// Deprecated TBD: use the secure WorkloadControllerManagerPort instead.
+	InsecureWorkloadControllerManagerPort = 10352
+
+	// WorkloadControllerManagerPort is the default port for the workload controller manager status server.
+	// May be overridden by a flag at startup.
+	WorkloadControllerManagerPort = 10357
+
 	// InsecureCloudControllerManagerPort is the default port for the cloud controller manager server.
 	// This value may be overridden by a flag at startup.
 	// Deprecated: use the secure CloudControllerManagerPort instead.