Merge remote-tracking branch 'remotes/jiangtianli/rs3_hyperv' into rs…

…3_hyperv

cmd/scale.go

@@ -203,6 +203,7 @@ func (sc *scaleCmd) run(cmd *cobra.Command, args []string) error {
 		currentNodeCount = len(indexes)
 
 		if currentNodeCount == sc.newDesiredAgentCount {
+			log.Info("Cluster is currently at the desired agent count.")
 			return nil
 		}
 		highestUsedIndex = indexes[len(indexes)-1]

docs/clusterdefinition.md

@@ -41,7 +41,93 @@ Here are the valid values for the orchestrator types:
 |maxPods|no|The maximum number of pods per node. The minimum valid value, necessary for running kube-system pods, is 5. Default value is 30 when networkPolicy equals azure, 110 otherwise.|
 |gcHighThreshold|no|Sets the --image-gc-high-threshold value on the kublet configuration. Default is 85. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) |
 |gcLowThreshold|no|Sets the --image-gc-low-threshold value on the kublet configuration. Default is 80. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) |
-|disabledAddons.dashboard|no|Disable dashboard addon (boolean - default == false, i.e. not disabled)|
+|useInstanceMetadata|no|Use the Azure cloudprovider instance metadata service for appropriate resource discovery operations. Default is `true`.|
+|addons|no|Configure various Kubernetes addons configuration (currently supported: tiller, kubernetes-dashboard). See `addons` configuration below.|
+
+`addons` describes various addons configuration. It is a child property of `kubernetesConfig`. Below is a list of currently available addons:
+
+|Name of addon|Enabled by default?|How many containers|Description|
+|tiller|true|Delivers the Helm server-side component: tiller. See https://github.com/kubernetes/helm for more info.|
+|kubernetes-dashboard|true|1|Delivers the kubernetes dashboard component. See https://github.com/kubernetes/dashboard for more info.|
+
+To give a bit more info on the `addons` property: We've tried to expose the basic bits of data that allow useful configuration of these cluster features. Here are some example usage patterns that will unpack what `addons` provide:
+
+To enable an addon (using "tiller" as an example):
+
+```
+"kubernetesConfig": {
+    "addons": [
+        {
+            "name": "tiller",
+            "enabled" : true
+        }
+    ]
+}
+```
+
+As you can see above, `addons` is an array child property of `kubernetesConfig`. Each addon that you want to add custom configuration to would be represented as an object item in the array. For example, to disable both tiller and dashboard:
+
+```
+"kubernetesConfig": {
+    "addons": [
+        {
+            "name": "tiller",
+            "enabled" : false
+        },
+        {
+            "name": "dashboard",
+            "enabled" : false
+        }
+    ]
+}
+```
+
+More usefully, let's add some custom configuration to both of the above addons:
+
+```
+"kubernetesConfig": {
+    "addons": [
+        {
+            "name": "tiller",
+            "containers": [
+                {
+                  "name": "tiller",
+                  "image": "myDockerHubUser/tiller:v3.0.0-alpha
+                  "cpuRequests": "1",
+                  "memoryRequests": "1024Mi",
+                  "cpuLimits": "1",
+                  "memoryLimits": "1024Mi"
+                }
+              ]
+        },
+        {
+            "name": "kubernetes-dashboard",
+            "containers": [
+                {
+                  "name": "kubernetes-dashboard",
+                  "cpuRequests": "50m",
+                  "memoryRequests": "512Mi",
+                  "cpuLimits": "50m",
+                  "memoryLimits": "512Mi"
+                }
+              ]
+        }
+    ]
+}
+```
+
+Above you see custom configuration for both tiller and kubernetes-dashboard. Both include specific resource limit values across the following dimensions:
+
+- cpuRequests
+- memoryRequests
+- cpuLimits
+- memoryLimits
+
+See https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ for more on Kubernetes resource limits.
+
+Additionally above, we specified a custom docker image for tiller, let's say we want to build a cluster and test an alpha version of tiller in it.
+
+Finally, the `addons.enabled` boolean property was omitted above; that's by design. If you specify a `containers` configuration, acs-engine assumes you're enabling the addon. The very first example above demonstrates a simple "enable this addon with default configuration" declaration.
 
 ### masterProfile
 `masterProfile` describes the settings for master configuration.

docs/kubernetes/monitoring.md

@@ -73,15 +73,14 @@ Another option to see stats is via Grafana and Influx DB. Grafana is a powerful
 
 To set up Grafana, we will need to deploy Grafana and InfluxDB. We will also need to configure Heapster to use InfluxDB as its storage backend. 
 
-1. `git clone https://github.com/kubernetes/Heapster.git $HOME/heapster`
-1. `cd $HOME/heapster`
-1. `git checkout release-1.4`
-1. `git cherry-pick c674a16f74782b326f02345486b5f9520891f395` (This works around the [open issue](https://github.com/kubernetes/Heapster/issues/1783) with Grafana deployments currently)
-1. `kubectl create -f deploy/kube-config/influxdb/influxdb.yaml`
-1. `kubectl create -f deploy/kube-config/influxdb/grafana.yaml`
+1. `wget https://raw.githubusercontent.com/kubernetes/heapster/release-1.5/deploy/kube-config/influxdb/influxdb.yaml`
+1. `wget https://raw.githubusercontent.com/kubernetes/heapster/release-1.5/deploy/kube-config/influxdb/grafana.yaml`
+1. `kubectl create -f influxdb.yaml`
+1. `kubectl create -f grafana.yaml`
 1. `kubectl get pods --namespace=kube-system` Ensure that Heapster, Grafana and InfluxDB are in the `Running` state
 1. `kubectl edit deployment/heapster --namespace=kube-system`
-1. We need to configure Heapster to use InfluxDB as the the data store. To do that under the spec > containers > command property change the command field from:
+
+    We need to configure Heapster to use InfluxDB as the the data store. To do that under the spec > containers > command property change the command field from:
    ``` yaml
    - command:
      - /heapster

docs/kubernetes/windows.md

@@ -46,7 +46,7 @@ Once your Kubernetes cluster has been created you will have a resource group con
 
 1. 1 master accessible by SSH on port 22 or kubectl on port 443
 
-2. a set of windows and linux nodes.  The windows nodes can be accessed through an RDP SSH tunnel via the master node.  To do this, follow these [instructions](../ssh.md#create-port-80-tunnel-to-the-master), replacing port 80 with 3389.  Since your windows machine is already using port 3389, it is recommended to use 3390 to Windows Node 0, 10.240.245.5, 3391 to Windows Node 1, 10.240.245.6, and so on as shown in the following image:
+2. a set of windows and linux nodes.  The windows nodes can be accessed through an RDP SSH tunnel via the master node.  To do this, follow these [instructions](../ssh.md#ssh-to-the-machine), replacing port 80 with 3389.  Since your windows machine is already using port 3389, it is recommended to use 3390 to Windows Node 0, 10.240.0.4, 3391 to Windows Node 1, 10.240.0.5, and so on as shown in the following image:
 
 ![Image of Windows RDP tunnels](../images/rdptunnels.png)
 

docs/ssh.md

@@ -38,21 +38,21 @@ Pageant enables agent forwarding.  This means that you can SSH from any of the m
 
 The DC/OS admin router requires an SSH tunnel from port 80 to port 80.
 
- 1. Verify you have nothing listening on port 80:
+### Verify you have nothing listening on port 80:
   1. on linux or Mac type `sudo netstat -anpe | grep ":80"` and stop any service or process listening on port 80
   2. on windows, at the command prompt type `netstat -ano | findstr /c:":80"` and stop any service or process listening on port 80.
 
- 2. Get the "sshMaster0" command:
+### Get the "sshMaster0" command:
   1. If using Powershell or CLI, the output parameters are the last values printed
   2. If using Portal, to get the output you need to:
-    1. navigate to "resource group"
-    2. click on the resource group you just created
-    3. then click on "Succeeded" under *last deployment*
-    4. then click on the "Microsoft.Template"
-    5. now you can copy the output of "sshMaster0"
+      1. navigate to "resource group"
+      2. click on the resource group you just created
+      3. then click on "Succeeded" under *last deployment*
+      4. then click on the "Microsoft.Template"
+      5. now you can copy the output of "sshMaster0"
     ![Image of docker scaling](images/findingoutputs.png)
 
-2. SSH to the machine
+### SSH to the machine
   1. on linux or Mac, modify the ssh command from "sshMaster0", and add `-L 80:localhost:80`, then use that command to connect
   2. on Windows, open Putty and in addition to the instructions from [agent forwarding](#key-management-and-agent-forwarding-with-windows-pageant), browse to Connection->SSH->Tunnel and add "80" to Source Port and "localhost:80" to Destination.
   ![Image of putty port 80](images/putty-port80tunnel.png)

examples/e2e-tests/kubernetes/release/default/definition.json

@@ -4,6 +4,7 @@
     "orchestratorProfile": {
       "orchestratorType": "Kubernetes",
       "kubernetesConfig": {
+        "enableRbac": true,
         "networkPolicy": "calico",
         "clusterSubnet": "10.230.0.0/16"
       }

examples/kubernetes-config/kubernetes-no-dashboard.json

@@ -4,9 +4,12 @@
     "orchestratorProfile": {
       "orchestratorType": "Kubernetes",
       "kubernetesConfig": {
-        "disabledAddons": {
-          "dashboard": true
-        }
+        "addons": [
+          {
+            "name": "dashboard",
+            "enabled" : false
+          }
+        ]
       }
     },
     "masterProfile": {

parts/kubernetesagentcustomdata.yml

@@ -36,13 +36,6 @@ write_files:
   content: |
     {{WrapAsVariable "caCertificate"}}
 
-- path: "/etc/kubernetes/certs/apiserver.crt"
-  permissions: "0644"
-  encoding: "base64"
-  owner: "root"
-  content: |
-    {{WrapAsVariable "apiserverCertificate"}}
-
 - path: "/etc/kubernetes/certs/client.crt"
   permissions: "0644"
   encoding: "base64"
@@ -108,7 +101,7 @@ write_files:
     DOCKER_OPTS=
     CUSTOM_CMD=/bin/true
     KUBELET_REGISTER_SCHEDULABLE=true
-    KUBELET_NODE_LABELS={{ GetKubernetesLabels . }}
+    KUBELET_NODE_LABELS={{GetAgentKubernetesLabels . "',variables('labelResourceGroup'),'"}}
     KUBELET_POD_INFRA_CONTAINER_IMAGE={{WrapAsVariable "kubernetesPodInfraContainerSpec"}}
     KUBELET_NODE_STATUS_UPDATE_FREQUENCY={{WrapAsVariable "kubernetesNodeStatusUpdateFrequency"}}
     KUBE_CTRL_MGR_NODE_MONITOR_GRACE_PERIOD={{WrapAsVariable "kubernetesCtrlMgrNodeMonitorGracePeriod"}}

parts/kubernetesmasteraddons-kubernetes-dashboard-deployment.yaml

@@ -75,6 +75,13 @@ spec:
         ports:
         - containerPort: 9090
           protocol: TCP
+        resources:
+          requests:
+            cpu: <kubernetesDashboardCPURequests>
+            memory: <kubernetesDashboardMemoryRequests>
+          limits:
+            cpu: <kubernetesDashboardCPULimit>
+            memory: <kubernetesDashboardMemoryLimit>
       serviceAccountName: kubernetes-dashboard
       nodeSelector:
         beta.kubernetes.io/os: linux 

parts/kubernetesmasteraddons-kubernetes-dashboard-deployment1.5.yaml

@@ -74,6 +74,13 @@ spec:
         ports:
         - containerPort: 9090
           protocol: TCP
+        resources:
+          requests:
+            cpu: <kubernetesDashboardCPURequests>
+            memory: <kubernetesDashboardMemoryRequests>
+          limits:
+            cpu: <kubernetesDashboardCPULimit>
+            memory: <kubernetesDashboardMemoryLimit>
       serviceAccountName: kubernetes-dashboard
       nodeSelector:
         beta.kubernetes.io/os: linux

parts/kubernetesmastercustomdata.yml

@@ -43,13 +43,6 @@ write_files:
   content: |
     {{WrapAsVariable "caCertificate"}}
 
-- path: "/etc/kubernetes/certs/apiserver.crt"
-  permissions: "0644"
-  encoding: "base64"
-  owner: "root"
-  content: |
-    {{WrapAsVariable "apiServerCertificate"}}
-
 - path: "/etc/kubernetes/certs/client.crt"
   permissions: "0644"
   encoding: "base64"
@@ -138,7 +131,7 @@ write_files:
   content: !!binary |
     MASTER_ADDON_KUBE_PROXY_DAEMONSET_B64_GZIP_STR
 
-{{if not .OrchestratorProfile.KubernetesConfig.DisabledAddons.Dashboard}}
+{{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}}
 {{end}}
 - path: /etc/kubernetes/addons/azure-storage-classes.yaml
   permissions: "0644"
@@ -147,12 +140,14 @@ write_files:
   content: !!binary |
     MASTER_ADDON_AZURE_STORAGE_CLASSES_B64_GZIP_STR
 
+{{if .OrchestratorProfile.KubernetesConfig.IsTillerEnabled}}
 - path: /etc/kubernetes/addons/kube-tiller-deployment.yaml
   permissions: "0644"
   encoding: gzip
   owner: "root"
   content: !!binary |
     MASTER_ADDON_TILLER_DEPLOYMENT_B64_GZIP_STR
+{{end}}
 
 {{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "calico"}}
 - path: /etc/kubernetes/addons/calico-daemonset.yaml
@@ -196,7 +191,7 @@ write_files:
     KUBELET_NETWORK_PLUGIN=
     KUBELET_MAX_PODS=110
     DOCKER_OPTS=
-    KUBELET_NODE_LABELS=kubernetes.io/role=master
+    KUBELET_NODE_LABELS={{GetMasterKubernetesLabels "',variables('labelResourceGroup'),'"}}
     KUBELET_POD_INFRA_CONTAINER_IMAGE={{WrapAsVariable "kubernetesPodInfraContainerSpec"}}
     KUBELET_NODE_STATUS_UPDATE_FREQUENCY={{WrapAsVariable "kubernetesNodeStatusUpdateFrequency"}}
     KUBE_CTRL_MGR_NODE_MONITOR_GRACE_PERIOD={{WrapAsVariable "kubernetesCtrlMgrNodeMonitorGracePeriod"}}
@@ -267,13 +262,19 @@ write_files:
     sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml"
     sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml"
     sed -i "s|<kubernetesKubeDNSSpec>|{{WrapAsVariable "kubernetesKubeDNSSpec"}}|g; s|<kubernetesDNSMasqSpec>|{{WrapAsVariable "kubernetesDNSMasqSpec"}}|g; s|<kubernetesExecHealthzSpec>|{{WrapAsVariable "kubernetesExecHealthzSpec"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml"
-{{if not .OrchestratorProfile.KubernetesConfig.DisabledAddons.Dashboard}}
+{{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}}
+    sed -i "s|<kubernetesDashboardCPURequests>|{{WrapAsVariable "kubernetesDashboardCPURequests"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
+    sed -i "s|<kubernetesDashboardMemoryRequests>|{{WrapAsVariable "kubernetesDashboardMemoryRequests"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
+    sed -i "s|<kubernetesDashboardCPULimit>|{{WrapAsVariable "kubernetesDashboardCPULimit"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
+    sed -i "s|<kubernetesDashboardMemoryLimit>|{{WrapAsVariable "kubernetesDashboardMemoryLimit"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
 {{end}}
+{{if .OrchestratorProfile.KubernetesConfig.IsTillerEnabled}}
     sed -i "s|<kubernetesTillerSpec>|{{WrapAsVariable "kubernetesTillerSpec"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
     sed -i "s|<kubernetesTillerCPURequests>|{{WrapAsVariable "kubernetesTillerCPURequests"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
     sed -i "s|<kubernetesTillerMemoryRequests>|{{WrapAsVariable "kubernetesTillerMemoryRequests"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
     sed -i "s|<kubernetesTillerCPULimit>|{{WrapAsVariable "kubernetesTillerCPULimit"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
     sed -i "s|<kubernetesTillerMemoryLimit>|{{WrapAsVariable "kubernetesTillerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
+{{end}}
     sed -i "s|<kubeDNSServiceIP>|{{WrapAsVariable "kubeDNSServiceIP"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml"
 
 {{if .OrchestratorProfile.KubernetesConfig.EnableRbac }}

parts/kubernetesmastercustomscript.sh

@@ -75,6 +75,12 @@ chmod 0600 "${KUBELET_PRIVATE_KEY_PATH}"
 chown root:root "${KUBELET_PRIVATE_KEY_PATH}"
 echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}"
 
+APISERVER_PUBLIC_KEY_PATH="/etc/kubernetes/certs/apiserver.crt"
+touch "${APISERVER_PUBLIC_KEY_PATH}"
+chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}"
+chown root:root "${APISERVER_PUBLIC_KEY_PATH}"
+echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
+
 AZURE_JSON_PATH="/etc/kubernetes/azure.json"
 touch "${AZURE_JSON_PATH}"
 chmod 0600 "${AZURE_JSON_PATH}"