chore(*): update config.yaml filed and add remove function (#6)

* chore(*): add inventory filed in config.yaml * chore(*): config compose.yaml nginx ports * cleanup(build): remove build and push kubeplay image * chore(*): update config.yaml filed and add remove function
k8sli · Aug 22, 2021 · 4af115c · 4af115c
1 parent 95fd0de
commit 4af115c
Show file tree

Hide file tree

Showing 6 changed files with 145 additions and 96 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -1,5 +1,5 @@
 ---
-name: Build kubeplay package
+name: Build kubeplay packages
 
 on:
   push:
@@ -20,7 +20,7 @@ env:
   KUBESPRAY_IMAGES_IMAGE: "ghcr.io/${{ github.repository_owner }}/kubespray-images"
 
 jobs:
-  build-image:
+  build-package:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
@@ -84,67 +84,24 @@ jobs:
           sed -n 's|image: nginx:|NGINX_VERSION=|p' compose.yaml | tr -d ' ' >> $GITHUB_ENV
           sed -n 's|image: registry:|RERGISRRY_VERSION=|p' compose.yaml | tr -d ' ' >> $GITHUB_ENV
 
-      - name: Build and push kubeplay images
+      - name: Build kubeplay image to local
         uses: docker/build-push-action@v2
         with:
           context: .
-          push: ${{ github.event_name != 'pull_request' }}
           file: Dockerfile
           platforms: linux/amd64,linux/arm64
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
           build-args: |
             NGINX_VERSION=${{ env.NGINX_VERSION }}
             RERGISRRY_VERSION=${{ env.RERGISRRY_VERSION }}
             KUBESPRAY_IMAGE=${{ env.KUBESPRAY_IMAGE }}
             KUBESPRAY_VERSION=${{ env.KUBESPRAY_VERSION }}
-          tags: |
-            ${{ env.KUBEPLAY_IMAGE_REPO }}:${{ env.IMAGE_TAG }}
-
-  build-package:
-    if: startsWith(github.ref, 'refs/tags/')
-    needs: [build-image]
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          # fetch all git repo tag for define image tag
-          fetch-depth: 0
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Log in to GitHub Docker Registry
-        uses: docker/login-action@v1
-        with:
-          registry: ${{ env.IMAGE_REGISTRY }}
-          username: ${{ env.REGISTRY_USER }}
-          password: ${{ env.REGISTRY_TOKEN }}
-
-      - name: Prepare for build images
-        shell: bash
-        run: |
-          git describe --tags --always | sed 's/^/IMAGE_TAG=/' >> $GITHUB_ENV
-          source $GITHUB_ENV
-          echo "FROM ${KUBEPLAY_IMAGE_REPO}:${IMAGE_TAG}" > Dockerfile
-
-      - name: Build package
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: Dockerfile
-          platforms: linux/amd64,linux/arm64
-          cache-from: type=local,src=/tmp/.buildx-cache
           outputs: type=local,dest=./
 
       - name: Prepare for upload package
         shell: bash
+        if: startsWith(github.ref, 'refs/tags/')
         run: |
-          rm -rf linux_{amd64,arm64}/build
+          rm -rf linux_{amd64,arm64}/{Dockerfile,LICENSE}
           mv linux_amd64 kubeplay
           tar -I pigz -cf kubeplay-${IMAGE_TAG}-linux-amd64.tar.gz kubeplay --remove-files
           mv linux_arm64 kubeplay
@@ -153,6 +110,7 @@ jobs:
 
       - name: Release and upload packages
         uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

diff --git a/config.yaml b/config.yaml
@@ -1,33 +1,81 @@
 compose:
   # Nginx http server bind port for download files and packages
-  http_port: 8080
+  nginx_http_port: 8080
+  # For container runtime pull image, use 443 as default
+  registry_https_port: 443
+  # For user push image to this registry, use 5000 as default
+  registry_push_port: 5000
   # Registry host ip for add to /etc/hosts by domain
   registry_ip: 172.20.0.25
   # Registry domain for CRI runtime download image
-  registry_domain: "kube.registry.local"
+  registry_domain: kube.registry.local
   # Auto generate self-signed certificate for registry domain
   generate_crt: true
-  self_signed_cert: true
   # User and password for push image to registry
   registry_auth_user: "Username"
   registry_auth_password: "Password"
+  # Registry image repo, default library
+  image_repo: library
 
 kubespray:
   # Kubernetes version by default, only support v1.20.6
   kube_version: v1.21.3
   # For deploy HA cluster you must configure a external apiserver access ip
-  external_apiserver_access_ip: "127.0.0.1"
+  external_apiserver_access_ip: 127.0.0.1
   # Set network plugin to calico with vxlan mode by default
-  kube_network_plugin: "calico"
+  kube_network_plugin: calico
   # Set true to download and cache container
   download_container: false
-  # Use nginx and registry as all resources
+  # Use nginx and registry provide all offline resources
   offline_resources_enabled: true
-  ## Container runtime
-  ## docker for docker, crio for cri-o and containerd for containerd.
+  #Container runtime, only support containerd if offline deploy
   container_manager: containerd
-  # Now only support host because use containerd as CRI runtime
+  # Now only support host if use containerd as CRI runtime
   etcd_deployment_type: host
   # Settings for etcd event server
   etcd_events_cluster_setup: true
   etcd_events_cluster_enabled: true
+  # Kubespray container image for deploy user cluster
+  kubespray_image: ""
+
+inventory:
+  all:
+    vars:
+      ansible_port: 22
+      ansible_user: root
+      ansible_ssh_pass: Password
+      # ansible_ssh_private_key_file: /kubespray/config/id_rsa
+    hosts:
+      node1:
+        ansible_host: 172.20.0.21
+      node2:
+        ansible_host: 172.20.0.22
+      node3:
+        ansible_host: 172.20.0.23
+      node4:
+        ansible_host: 172.20.0.24
+    children:
+      kube_control_plane:
+        hosts:
+          node1:
+          node2:
+          node3:
+      kube_node:
+        hosts:
+          node1:
+          node2:
+          node3:
+          node4:
+      etcd:
+        hosts:
+          node1:
+          node2:
+          node3:
+      k8s_cluster:
+        children:
+          kube_control_plane:
+          kube_node:
+      gpu:
+        hosts: {}
+      calico_rr:
+        hosts: {}
diff --git a/install.sh b/install.sh
@@ -16,14 +16,14 @@ ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')
 KUBE_ROOT="$(cd "$(dirname "$0")" && pwd)"
 CERTS_DIR="${KUBE_ROOT}/config/certs"
 CONFIG_FILE="${KUBE_ROOT}/config.yaml"
+CA_CONFIGFILE="${KUBE_ROOT}/config/rootCA.cnf"
 COMPOSE_YAML_FILE="${KUBE_ROOT}/compose.yaml"
 IMAGES_DIR="${KUBE_ROOT}/resources/images"
 COMPOSE_CONFIG_DIR="${KUBE_ROOT}/config/compose"
-INSTALL_STEPS_FILE="${KUBE_ROOT}/.install_steps"
 OUTPUT_ENV_FILE="${KUBE_ROOT}/.install-env.sh"
 RESOURCES_NGINX_DIR="${KUBE_ROOT}/resources/nginx"
 KUBESPRAY_CONFIG_DIR="${KUBE_ROOT}/config/kubespray"
-CA_CONFIGFILE="${KUBE_ROOT}/config/rootCA.cnf"
+INSTALL_STEPS_FILE="${KUBESPRAY_CONFIG_DIR}/.install_steps"
 
 # Import all functions from scripts/*.sh
 for file in ${KUBE_ROOT}/scripts/*.sh; do source ${file}; done
@@ -64,23 +64,21 @@ EOF
 }
 
 deploy_cluster(){
+  common::rudder_config
+  common::push_kubespray_image
   common::run_kubespray "bash /kubespray/run.sh deploy-cluster"
 }
 
-remove_cluster(){
-  common::run_kubespray "bash /kubespray/run.sh remove-cluster"
-}
-
 add_nodes(){
-  common::run_kubespray "bash /kubespray/run.sh add-node $1"
+  common::run_kubespray "bash /kubespray/run.sh add-node $2"
 }
 
 remove_nodes(){
-  common::run_kubespray "bash /kubespray/run.sh remove-node $1"
+  common::run_kubespray "bash /kubespray/run.sh remove-node $2"
 }
 
 kubespray_debug(){
-  common::run_kubespray "/usr/bin/bash"
+  common::run_kubespray "bash"
 }
 
 install_all(){
@@ -100,16 +98,25 @@ main(){
       deploy_cluster
       ;;
     remove)
-      remove_cluster
-      remove_compose
+      common::rudder_config
+      remove::remove_cluster
+      remove::remove_compose
       ;;
     remove-cluster)
-      remove_cluster
+      common::rudder_config
+      remove::remove_cluster
+      ;;
+    remove-compose)
+      common::rudder_config
+      remove::remove_compose
       ;;
     add-nodes)
       ;;
     remove-node)
       ;;
+    health-check)
+      common::health_check
+      ;;
     debug)
       kubespray_debug
       ;;

diff --git a/scripts/common.sh b/scripts/common.sh
@@ -55,29 +55,38 @@ common::install_tools(){
 }
 
 common::rudder_config(){
-  HTTP_PORT=$(yq eval '.compose.http_port' ${CONFIG_FILE})
+  # Gather variables form config.yaml
+  NGINX_HTTP_PORT=$(yq eval '.compose.nginx_http_port' ${CONFIG_FILE})
+  REGISTRY_HTTPS_PORT=$(yq eval '.compose.registry_https_port' ${CONFIG_FILE})
+  REGISTRY_PUSH_PORT=$(yq eval '.compose.registry_push_port' ${CONFIG_FILE})
   REGISTRY_IP=$(yq eval '.compose.registry_ip' ${CONFIG_FILE})
   REGISTRY_DOMAIN=$(yq eval '.compose.registry_domain' ${CONFIG_FILE})
   REGISTRY_AUTH_USER=$(yq eval '.compose.registry_auth_user' ${CONFIG_FILE})
   REGISTRY_AUTH_PASSWORD=$(yq eval '.compose.registry_auth_password' ${CONFIG_FILE})
   GENERATE_CRT=$(yq eval '.compose.generate_crt' ${CONFIG_FILE})
-  HTTP_URL="http://${REGISTRY_IP}:${HTTP_PORT}"
-  yq eval '.compose' ${CONFIG_FILE} > ${KUBESPRAY_CONFIG_DIR}/env.yml
+  IMAGE_REPO=$(yq eval '.compose.image_repo' ${CONFIG_FILE})
+  PUSH_REGISTRY="${REGISTRY_DOMAIN}:${REGISTRY_PUSH_PORT}"
+
+  # Update compose.yaml nginx ports filed
+  nginx_http_port="${NGINX_HTTP_PORT}:8080" yq eval --inplace '.services.nginx.ports[0] = strenv(nginx_http_port)' ${COMPOSE_YAML_FILE}
+  registry_https_port="${REGISTRY_HTTPS_PORT}:443" yq eval --inplace '.services.nginx.ports[1] = strenv(registry_https_port)' ${COMPOSE_YAML_FILE}
+  registry_push_port="${REGISTRY_PUSH_PORT}:5000" yq eval --inplace '.services.nginx.ports[2] = strenv(registry_push_port)' ${COMPOSE_YAML_FILE}
+
+  # Generate kubespray's env.yaml and inventory file
+  : ${NGINX_HTTP_URL:="http://${REGISTRY_IP}:${NGINX_HTTP_PORT}"}
+  : ${REGISTRY_HTTPS_URL:="https://${REGISTRY_DOMAIN}:${REGISTRY_HTTPS_PORT}"}
+  echo "offline_resources_url: ${NGINX_HTTP_URL}" > ${KUBESPRAY_CONFIG_DIR}/env.yml
+  yq eval '.compose' ${CONFIG_FILE} >> ${KUBESPRAY_CONFIG_DIR}/env.yml
   yq eval '.kubespray' ${CONFIG_FILE} >> ${KUBESPRAY_CONFIG_DIR}/env.yml
-  echo -e "\noffline_resources_url: ${HTTP_URL}" >> ${KUBESPRAY_CONFIG_DIR}/env.yml
-}
-
-common::generate_inventory(){
-:
-# TDDO
+  yq eval '.inventory' ${CONFIG_FILE} > ${KUBESPRAY_CONFIG_DIR}/inventory
 }
 
 # Generate registry domain cert
 common::generate_domain_certs(){
   if [[ ${GENERATE_CRT} == "true" ]]; then
     rm -rf ${CERTS_DIR} ${RESOURCES_NGINX_DIR}/certs
     mkdir -p ${CERTS_DIR} ${RESOURCES_NGINX_DIR}/certs
-    cp ${CA_CONFIGFILE} ${CERTS_DIR}
+    cp -f ${CA_CONFIGFILE} ${CERTS_DIR}
     infolog "Generating TLS cert for domain: ${REGISTRY_DOMAIN}"
     # Creating rootCA directory structure
     sed -i "s|CERTS_DIR|${CERTS_DIR}|" ${CERTS_DIR}/rootCA.cnf
@@ -132,7 +141,7 @@ common::generate_auth_htpasswd(){
   htpasswd -cB -b ${COMPOSE_CONFIG_DIR}/auth.htpasswd ${REGISTRY_AUTH_USER} ${REGISTRY_AUTH_PASSWORD}
 }
 
-# Insect registry domain hosts to /etc/hosts file
+# Add registry domain with ip to /etc/hosts file
 common::update_hosts(){
   sed -i "/${REGISTRY_DOMAIN}/d" /etc/hosts
   echo "${REGISTRY_IP} ${REGISTRY_DOMAIN}" >> /etc/hosts
@@ -146,6 +155,9 @@ common::local_images(){
       infolog "Load ${image} image successfully"
     fi
   done
+  : ${KUBESPRAY_IMAGE:=$(nerdctl images | awk '{print $1":"$2}' | grep '^kubespray:*' | sort -r --version-sort | head -n1)}
+  kubespray_image="${REGISTRY_DOMAIN}/${KUBESPRAY_IMAGE}" yq eval --inplace '.kubespray.kubespray_image = strenv(kubespray_image)' ${CONFIG_FILE}
+  kubespray_image="${REGISTRY_DOMAIN}/${KUBESPRAY_IMAGE}" yq eval --inplace '.kubespray.kubespray_image = strenv(kubespray_image)' ${KUBESPRAY_CONFIG_DIR}/env.yml
 }
 
 common::compose_up(){
@@ -185,22 +197,23 @@ common::http_check(){
 }
 
 common::health_check(){
-  common::http_check ${HTTP_URL}/certs/rootCA.crt
-  common::http_check https://${REGISTRY_DOMAIN}/v2/_catalog
+  common::http_check ${NGINX_HTTP_URL}/certs/rootCA.crt && common::http_check ${REGISTRY_HTTPS_URL}/v2/_catalog
 }
 
+# Run kubespray container
 common::run_kubespray(){
-  local KUBESPRAY_IMAGE=$(nerdctl images | awk '{print $1":"$2}' | grep '^kubespray:*' | sort -r --version-sort | head -n1)
+  : ${KUBESPRAY_IMAGE:=$(nerdctl images | awk '{print $1":"$2}' | grep '^kubespray:*' | sort -r --version-sort | head -n1)}
   nerdctl rm -f kubespray-runner >/dev/null 2>&1 || true
   nerdctl run --rm -it --net=host --name kubespray-runner \
   -v ${KUBESPRAY_CONFIG_DIR}:/kubespray/config \
+  -e KUBESPRAY_IMAGE=${KUBESPRAY_IMAGE} \
   ${KUBESPRAY_IMAGE} $1
 }
 
 # Push kubespray image to registry
 common::push_kubespray_image(){
-  nerdctl login "${REGISTRY_DOMAIN}:5000" -u "${REGISTRY_AUTH_USER}" -p "${REGISTRY_AUTH_PASSWORD}"
-  local KUBESPRAY_IMAGE=$(nerdctl images | awk '{print $1":"$2}' | grep '^kubespray:*' | sort -r --version-sort | head -n1)
-  nerdctl tag ${KUBESPRAY_IMAGE} ${REGISTRY_DOMAIN}/library/${KUBESPRAY_IMAGE}
-  nerdctl push ${REGISTRY_DOMAIN}/library/${KUBESPRAY_IMAGE}
+  : ${KUBESPRAY_IMAGE:=$(nerdctl images | awk '{print $1":"$2}' | grep '^kubespray:*' | sort -r --version-sort | head -n1)}
+  nerdctl login -u "${REGISTRY_AUTH_USER}" -p "${REGISTRY_AUTH_PASSWORD}" ${PUSH_REGISTRY}
+  nerdctl tag ${KUBESPRAY_IMAGE} ${PUSH_REGISTRY}/${IMAGE_REPO}/${KUBESPRAY_IMAGE}
+  nerdctl push ${PUSH_REGISTRY}/${IMAGE_REPO}/${KUBESPRAY_IMAGE}
 }