Skip to content

Commit

Permalink
Merge branch 'openyurt' into openyurt-deploy
Browse files Browse the repository at this point in the history
  • Loading branch information
@jchua99
jchua99 committed Dec 21, 2023
2 parents 9fa3225 + 1b5a74c commit 3c4a66f
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 45 deletions.
2 changes: 1 addition & 1 deletion scripts/openyurt-deployer/node/go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
module github.com/vhive-serverless/vhive/scripts/openyurt_deployer/node

go 1.20
go 1.20
27 changes: 11 additions & 16 deletions scripts/openyurt-deployer/node/knative.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,6 @@ func (node *Node) InstallKnativeServing() {

node.CreateTmpDir()

// untaint no schedule on master node
utils.WaitPrintf("Untaint single master node to build metalLB on single node cluster")
_, err = node.ExecShellCmd("kubectl taint node --all node-role.kubernetes.io/control-plane:NoSchedule-")
utils.CheckErrorWithMsg(err, "Failed to remove NoSchedule on master node!")

// Install and configure MetalLB
utils.WaitPrintf("Installing and configuring MetalLB")
_, err = node.ExecShellCmd(`kubectl get configmap kube-proxy -n kube-system -o yaml | sed -e "s/strictARP: false/strictARP: true/" | kubectl apply -f - -n kube-system`)
Expand All @@ -36,11 +31,11 @@ func (node *Node) InstallKnativeServing() {
// Download istio
utils.WaitPrintf("Downloading istio")
istioFilePath, err := node.DownloadToTmpDir(node.GetIstioDownloadUrl())
utils.CheckErrorWithMsg(err, "Failed to download istio!")
utils.CheckErrorWithTagAndMsg(err, "Failed to download istio!")
// Extract istio
utils.WaitPrintf("Extracting istio")
err = node.ExtractToDir(istioFilePath, "/usr/local", true)
utils.CheckErrorWithMsg(err, "Failed to extract istio!")
utils.CheckErrorWithTagAndMsg(err, "Failed to extract istio!")
// Update PATH
err = node.AppendDirToPath("/usr/local/istio-%s/bin", node.Configs.Knative.IstioVersion)
utils.CheckErrorWithMsg(err, "Failed to update PATH!")
Expand All @@ -49,14 +44,14 @@ func (node *Node) InstallKnativeServing() {
operatorConfigPath, err := node.DownloadToTmpDir(node.Configs.Knative.IstioOperatorConfigUrl)
utils.CheckErrorWithMsg(err, "Failed to download istio operator config!")
_, err = node.ExecShellCmd("sudo /usr/local/istio-%s/bin/istioctl install -y -f %s", node.Configs.Knative.IstioVersion, operatorConfigPath)
utils.CheckErrorWithMsg(err, "Failed to deploy istio operator!")
utils.CheckErrorWithTagAndMsg(err, "Failed to deploy istio operator!")

// Install Knative Serving component
utils.WaitPrintf("Installing Knative Serving component")
_, err = node.ExecShellCmd("kubectl apply -f https://github.com/knative/serving/releases/download/knative-v%s/serving-crds.yaml", node.Configs.Knative.KnativeVersion)
utils.CheckErrorWithMsg(err, "Failed to install Knative Serving component!")
_, err = node.ExecShellCmd("kubectl apply -f https://github.com/knative/serving/releases/download/knative-v%s/serving-core.yaml", node.Configs.Knative.KnativeVersion)
utils.CheckErrorWithMsg(err, "Failed to install Knative Serving component!")
utils.CheckErrorWithTagAndMsg(err, "Failed to install Knative Serving component!")

// Install local cluster registry
utils.WaitPrintf("Installing local cluster registry")
Expand All @@ -67,17 +62,17 @@ func (node *Node) InstallKnativeServing() {
_, err = node.ExecShellCmd("REPO_VOL_SIZE=%s envsubst < %s | kubectl create --filename -", node.Configs.Knative.LocalRegistryRepoVolumeSize, configFilePath)
utils.CheckErrorWithMsg(err, "Failed to install local cluster registry!")
_, err = node.ExecShellCmd("kubectl create -f %s && kubectl apply -f %s", node.Configs.Knative.LocalRegistryDockerRegistryConfigUrl, node.Configs.Knative.LocalRegistryHostUpdateConfigUrl)
utils.CheckErrorWithMsg(err, "Failed to install local cluster registry!")
utils.CheckErrorWithTagAndMsg(err, "Failed to install local cluster registry!")

// Configure Magic DNS
utils.WaitPrintf("Configuring Magic DNS")
_, err = node.ExecShellCmd("kubectl apply -f %s", node.Configs.Knative.MagicDNSConfigUrl)
utils.CheckErrorWithMsg(err, "Failed to configure Magic DNS!")
utils.CheckErrorWithTagAndMsg(err, "Failed to configure Magic DNS!")

// Install networking layer
utils.WaitPrintf("Installing networking layer")
_, err = node.ExecShellCmd("kubectl apply -f https://github.com/knative/net-istio/releases/download/knative-v%s/net-istio.yaml", node.Configs.Knative.KnativeVersion)
utils.CheckErrorWithMsg(err, "Failed to install networking layer!")
utils.CheckErrorWithTagAndMsg(err, "Failed to install networking layer!")

// Logs for verification
_, err = node.ExecShellCmd("kubectl get pods -n knative-serving")
Expand All @@ -94,7 +89,7 @@ func (node *Node) InstallKnativeServing() {
--type merge \
-p '{"data":{"kubernetes.podspec-nodeselector":"enabled"}}'
`)
utils.CheckErrorWithMsg(err, "Failed to enable node selector in knative serving")
utils.CheckErrorWithTagAndMsg(err, "Failed to enable node selector in knative serving")
// node.enableNodeSelect()
}

Expand All @@ -105,7 +100,7 @@ func (node *Node) InstallKnativeEventing() {
_, err := node.ExecShellCmd("kubectl apply -f https://github.com/knative/eventing/releases/download/knative-v%s/eventing-crds.yaml", node.Configs.Knative.KnativeVersion)
utils.CheckErrorWithMsg(err, "Failed to install Knative Eventing component!")
_, err = node.ExecShellCmd("kubectl apply -f https://github.com/knative/eventing/releases/download/knative-v%s/eventing-core.yaml", node.Configs.Knative.KnativeVersion)
utils.CheckErrorWithMsg(err, "Failed to install Knative Eventing component!")
utils.CheckErrorWithTagAndMsg(err, "Failed to install Knative Eventing component!")

// Logs for verification
_, err = node.ExecShellCmd("kubectl get pods -n knative-eventing")
Expand All @@ -114,12 +109,12 @@ func (node *Node) InstallKnativeEventing() {
// Install a default Channel (messaging) layer
utils.WaitPrintf("Installing a default Channel (messaging) layer")
_, err = node.ExecShellCmd("kubectl apply -f https://github.com/knative/eventing/releases/download/knative-v%s/in-memory-channel.yaml", node.Configs.Knative.KnativeVersion)
utils.CheckErrorWithMsg(err, "Failed to install a default Channel (messaging) layer!")
utils.CheckErrorWithTagAndMsg(err, "Failed to install a default Channel (messaging) layer!")

// Install a Broker layer
utils.WaitPrintf("Installing a Broker layer")
_, err = node.ExecShellCmd("kubectl apply -f https://github.com/knative/eventing/releases/download/knative-v%s/mt-channel-broker.yaml", node.Configs.Knative.KnativeVersion)
utils.CheckErrorWithMsg(err, "Failed to install a Broker layer!")
utils.CheckErrorWithTagAndMsg(err, "Failed to install a Broker layer!")

// Logs for verification
_, err = node.ExecShellCmd("kubectl --namespace istio-system get service istio-ingressgateway")
Expand Down
12 changes: 6 additions & 6 deletions scripts/openyurt-deployer/node/kube.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func (node *Node) KubeMasterInit() (string, string, string, string) {
shellCmd = fmt.Sprintf(shellCmd+"--image-repository %s ", node.Configs.Kube.AlternativeImageRepo)
}
_, err = node.ExecShellCmd(shellCmd)
utils.CheckErrorWithMsg(err, "Failed to pre-pull required images!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to pre-pull required images!\n")

// Deploy Kubernetes
utils.WaitPrintf("Deploying Kubernetes(version %s)", node.Configs.Kube.K8sVersion)
Expand All @@ -35,20 +35,20 @@ func (node *Node) KubeMasterInit() (string, string, string, string) {
}
shellCmd = fmt.Sprintf(shellCmd+"| tee %s/masterNodeInfo", node.Configs.System.TmpDir)
_, err = node.ExecShellCmd(shellCmd)
utils.CheckErrorWithMsg(err, "Failed to deploy Kubernetes(version %s)!\n", node.Configs.Kube.K8sVersion)
utils.CheckErrorWithTagAndMsg(err, "Failed to deploy Kubernetes(version %s)!\n", node.Configs.Kube.K8sVersion)

// Make kubectl work for non-root user
utils.WaitPrintf("Making kubectl work for non-root user")
_, err = node.ExecShellCmd("mkdir -p %s/.kube && sudo cp -i /etc/kubernetes/admin.conf %s/.kube/config && sudo chown $(id -u):$(id -g) %s/.kube/config",
node.Configs.System.UserHomeDir,
node.Configs.System.UserHomeDir,
node.Configs.System.UserHomeDir)
utils.CheckErrorWithMsg(err, "Failed to make kubectl work for non-root user!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to make kubectl work for non-root user!\n")

// Install Calico network add-on
utils.WaitPrintf("Installing pod network")
_, err = node.ExecShellCmd("kubectl apply -f %s", node.Configs.Kube.PodNetworkAddonConfigURL)
utils.CheckErrorWithMsg(err, "Failed to install pod network!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to install pod network!\n")

// Extract master node information from logs
utils.WaitPrintf("Extracting master node information from logs")
Expand Down Expand Up @@ -92,7 +92,7 @@ func (node *Node) KubeClean() {
utils.WaitPrintf("Reseting kube cluster")
_, err = node.ExecShellCmd("sudo kubeadm reset -f && rm -rf /etc/cni/net.d")
}
utils.CheckErrorWithMsg(err, "Failed to clean kube cluster!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to clean kube cluster!\n")

}

Expand All @@ -105,7 +105,7 @@ func (node *Node) KubeWorkerJoin(apiServerAddr string, apiServerPort string, api
// Join Kubernetes cluster
utils.WaitPrintf("Joining Kubernetes cluster")
_, err = node.ExecShellCmd("sudo kubeadm join %s:%s --token %s --discovery-token-ca-cert-hash %s", apiServerAddr, apiServerPort, apiServerToken, apiServerTokenHash)
utils.CheckErrorWithMsg(err, "Failed to join Kubernetes cluster!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to join Kubernetes cluster!\n")
}

func (node *Node) check_kube_environment() {
Expand Down
44 changes: 22 additions & 22 deletions scripts/openyurt-deployer/node/system.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,14 +76,14 @@ func (node *Node) CreateTmpDir() {
tmpDir := "~/yurt_tmp"
_, err = node.ExecShellCmd("mkdir -p %s", tmpDir)
node.Configs.System.TmpDir = tmpDir
utils.CheckErrorWithMsg(err, "Failed to create temporary directory!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to create temporary directory!\n")
}

// Clean up temporary directory
func (node *Node) CleanUpTmpDir() {
utils.InfoPrintf("Cleaning up temporary directory")
_, err := node.ExecShellCmd("rm -rf %s/*", node.Configs.System.TmpDir)
utils.CheckErrorWithMsg(err, "Failed to create temporary directory!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to create temporary directory!\n")
}

// Extract arhive file to specific directory(currently support .tar.gz file only)
Expand Down Expand Up @@ -240,32 +240,32 @@ func (node *Node) SystemInit() {
// Turn off unattended-upgrades on ubuntu
utils.InfoPrintf("Turning off automatic upgrade")
_, err = node.TurnOffAutomaticUpgrade()
utils.CheckErrorWithMsg(err, "Failed to turn off automatic upgrade!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to turn off automatic upgrade!\n")

// Disable swap
utils.InfoPrintf("Disabling swap")
_, err = node.ExecShellCmd("sudo swapoff -a && sudo cp /etc/fstab /etc/fstab.old") // Turn off Swap && Backup fstab file
utils.CheckErrorWithMsg(err, "Failed to disable swap!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to disable swap!\n")

utils.InfoPrintf("Modifying fstab")
// Modify fstab to disable swap permanently
_, err = node.ExecShellCmd("sudo sed -i 's/#\\s*\\(.*swap.*\\)/\\1/g' /etc/fstab && sudo sed -i 's/.*swap.*/# &/g' /etc/fstab")
utils.CheckErrorWithMsg(err, "Failed to dodify fstab!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to dodify fstab!\n")

// Install dependencies
utils.InfoPrintf("Installing dependencies")
err = node.InstallPackages(node.Configs.System.Dependencies)
utils.CheckErrorWithMsg(err, "Failed to install dependencies!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to install dependencies!\n")

// Install Golang
if !node.Configs.System.GoInstalled {
// Download & Extract Golang
utils.InfoPrintf("Downloading Golang(ver %s)", node.Configs.System.GoVersion)
filePathName, err := node.DownloadToTmpDir(node.Configs.System.GoDownloadUrlTemplate, node.Configs.System.GoVersion, node.Configs.System.CurrentArch)
utils.CheckErrorWithMsg(err, "Failed to download Golang(ver %s)!\n", node.Configs.System.GoVersion)
utils.CheckErrorWithTagAndMsg(err, "Failed to download Golang(ver %s)!\n", node.Configs.System.GoVersion)
utils.InfoPrintf("Extracting Golang")
_, err = node.ExecShellCmd("sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf %s", filePathName)
utils.CheckErrorWithMsg(err, "Failed to extract Golang!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to extract Golang!\n")

// For bash
_, err = node.ExecShellCmd("echo 'export PATH=$PATH:/usr/local/go/bin' >> %s/.bashrc", node.Configs.System.UserHomeDir)
Expand All @@ -287,18 +287,18 @@ func (node *Node) SystemInit() {
node.Configs.System.ContainerdVersion,
node.Configs.System.ContainerdVersion,
node.Configs.System.CurrentArch)
utils.CheckErrorWithMsg(err, "Failed to Download containerd(ver %s)\n", node.Configs.System.ContainerdVersion)
utils.CheckErrorWithTagAndMsg(err, "Failed to Download containerd(ver %s)\n", node.Configs.System.ContainerdVersion)
// Extract containerd
utils.InfoPrintf("Extracting containerd")
_, err = node.ExecShellCmd("sudo tar Cxzvf /usr/local %s", filePathName)
utils.CheckErrorWithMsg(err, "Failed to extract containerd!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to extract containerd!\n")
// Start containerd via systemd
utils.InfoPrintf("Downloading systemd profile for containerd")
filePathName, err = node.DownloadToTmpDir(node.Configs.System.ContainerdSystemdProfileDownloadUrl)
utils.CheckErrorWithMsg(err, "Failed to download systemd profile for containerd!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to download systemd profile for containerd!\n")
utils.InfoPrintf("Starting containerd via systemd")
_, err = node.ExecShellCmd("sudo cp %s /lib/systemd/system/ && sudo systemctl daemon-reload && sudo systemctl enable --now containerd", filePathName)
utils.CheckErrorWithMsg(err, "Failed to start containerd via systemd!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to start containerd via systemd!\n")
}

// Install runc
Expand All @@ -309,11 +309,11 @@ func (node *Node) SystemInit() {
node.Configs.System.RuncDownloadUrlTemplate,
node.Configs.System.RuncVersion,
node.Configs.System.CurrentArch)
utils.CheckErrorWithMsg(err, "Failed to download runc(ver %s)!\n", node.Configs.System.RuncVersion)
utils.CheckErrorWithTagAndMsg(err, "Failed to download runc(ver %s)!\n", node.Configs.System.RuncVersion)
// Install runc
utils.InfoPrintf("Installing runc")
_, err = node.ExecShellCmd("sudo install -m 755 %s /usr/local/sbin/runc", filePathName)
utils.CheckErrorWithMsg(err, "Failed to install runc!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to install runc!\n")
}

// Install CNI plugins
Expand All @@ -324,10 +324,10 @@ func (node *Node) SystemInit() {
node.Configs.System.CniPluginsVersion,
node.Configs.System.CurrentArch,
node.Configs.System.CniPluginsVersion)
utils.CheckErrorWithMsg(err, "Failed to download CNI plugins(ver %s)!\n", node.Configs.System.CniPluginsVersion)
utils.CheckErrorWithTagAndMsg(err, "Failed to download CNI plugins(ver %s)!\n", node.Configs.System.CniPluginsVersion)
utils.InfoPrintf("Extracting CNI plugins")
_, err = node.ExecShellCmd("sudo mkdir -p /opt/cni/bin && sudo tar Cxzvf /opt/cni/bin %s", filePathName)
utils.CheckErrorWithMsg(err, "Failed to extract CNI plugins!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to extract CNI plugins!\n")
}

// Configure the systemd cgroup driver
Expand All @@ -336,32 +336,32 @@ func (node *Node) SystemInit() {
"containerd config default > %s && sudo mkdir -p /etc/containerd && sudo cp %s /etc/containerd/config.toml && sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml && sudo systemctl restart containerd",
node.Configs.System.TmpDir+"/config.toml",
node.Configs.System.TmpDir+"/config.toml")
utils.CheckErrorWithMsg(err, "Failed to configure the systemd cgroup driver!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to configure the systemd cgroup driver!\n")

// Enable IP forwading & br_netfilter
utils.InfoPrintf("Enabling IP forwading & br_netfilter")
_, err = node.ExecShellCmd("sudo modprobe br_netfilter && sudo modprobe overlay && sudo sysctl -w net.ipv4.ip_forward=1 && sudo sysctl -w net.ipv4.conf.all.forwarding=1 && sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 && sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1")
utils.CheckErrorWithMsg(err, "Failed to enable IP forwading & br_netfilter!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to enable IP forwading & br_netfilter!\n")
// Ensure Boot-Resistant
utils.InfoPrintf("Ensuring Boot-Resistant")
_, err = node.ExecShellCmd("echo 'br_netfilter' | sudo tee /etc/modules-load.d/netfilter.conf && echo 'overlay' | sudo tee -a /etc/modules-load.d/netfilter.conf && sudo sed -i 's/# *net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g' /etc/sysctl.conf && sudo sed -i 's/net.ipv4.ip_forward=0/net.ipv4.ip_forward=1/g' /etc/sysctl.conf && echo 'net.bridge.bridge-nf-call-iptables=1\nnet.bridge.bridge-nf-call-ip6tables=1\nnet.ipv4.conf.all.forwarding=1' | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf")
utils.CheckErrorWithMsg(err, "Failed to ensure Boot-Resistant!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to ensure Boot-Resistant!\n")

// Install kubeadm, kubelet, kubectl
switch node.Configs.System.CurrentOS {
case "ubuntu":
// Download Google Cloud public signing key and Add the Kubernetes apt repository
utils.InfoPrintf("Adding the Kubernetes apt repository")
_, err = node.ExecShellCmd("sudo mkdir -p /etc/apt/keyrings && sudo curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://dl.k8s.io/apt/doc/apt-key.gpg && echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main' | sudo tee /etc/apt/sources.list.d/kubernetes.list")
utils.CheckErrorWithMsg(err, "Failed to add the Kubernetes apt repository!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to add the Kubernetes apt repository!\n")
// Install kubeadm, kubelet, kubectl via apt
utils.InfoPrintf("Installing kubeadm, kubelet, kubectl")
err = node.InstallPackages("kubeadm=%s kubelet=%s kubectl=%s", node.Configs.System.KubeVersion, node.Configs.System.KubeVersion, node.Configs.System.KubeVersion)
utils.CheckErrorWithMsg(err, "Failed to install kubeadm, kubelet, kubectl!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to install kubeadm, kubelet, kubectl!\n")
// Lock kubeadm, kubelet, kubectl version
utils.InfoPrintf("Locking kubeadm, kubelet, kubectl version")
_, err = node.ExecShellCmd("sudo apt-mark hold kubelet kubeadm kubectl")
utils.CheckErrorWithMsg(err, "Failed to lock kubeadm, kubelet, kubectl version!\n")
utils.CheckErrorWithTagAndMsg(err, "Failed to lock kubeadm, kubelet, kubectl version!\n")
default:
utils.FatalPrintf("Unsupported Linux distribution: %s\n", node.Configs.System.CurrentOS)
}
Expand Down

0 comments on commit 3c4a66f

Please sign in to comment.