.github/CODEOWNERS

@@ -0,0 +1,8 @@
+## Default
+* @Rory-Z
+
+## Docs
+/docs/ @emqx/cloud-native
+
+## Release note
+/RELEASE.md @emqx/cloud-native

.github/ISSUE_TEMPLATE/bug_report.md

@@ -21,5 +21,5 @@ A clear and concise description of what you expected to happen.
 **Environment details:**:
 - Kubernetes version:
 - Cloud-provider/provisioner:
-- emqx-operator version: 
+- emqx-operator version:
 - Install method: e.g. helm/static manifests

.github/ISSUE_TEMPLATE/config.yml

@@ -2,4 +2,4 @@ blank_issues_enabled: false
 contact_links:
   - name: GitHub Community Support
     url: https://github.com/emqx/emqx-operator/discussions
-    about: Please ask and answer questions here.
+    about: Please ask and answer questions here.

.github/actions/deploy-emqx/action.yaml

@@ -24,11 +24,17 @@ runs:
       shell: bash
       run: |
         while
-          condition="$(kubectl get ${{ inputs.kind }} ${{ inputs.name }} -o json | jq '.status.conditions[0]')";
-          [[ "$condition" == "null" ]] \
-          || [[ "$(echo $condition | jq --raw-output '.type')" != "Running" ]] \
-          || [[ "$(echo $condition | jq --raw-output '.status')" != "True" ]]
+          type="$(kubectl get ${{ inputs.kind }} ${{ inputs.name }} -o json |jq '.status.conditions[0] | select(.status == "True")' | jq --raw-output '.type')"
+          [[ "$type" != "Ready" ]] && [[ "$type" != "Running" ]]
         do
           echo "waiting"
           sleep 1
-        done
+        done
+    - name: Check statefulSet doesn't update
+      shell: bash
+      run: |
+        generation=$(kubectl get sts -l "apps.emqx.io/instance=${{ inputs.name }}" -o 'jsonpath={.items[0].status.observedGeneration}')
+        if [ $generation != 1 ]; then
+          kubectl get sts -l "apps.emqx.io/instance=${{ inputs.name }}" -o json
+          exit 1;
+        fi

.github/actions/deploy-operator/action.yaml

@@ -16,28 +16,50 @@ runs:
   - name: Install cert-manager
     shell: bash
     run: |
+      cert_manager_version=""
+      kube_min_version=$(kubectl version --output=yaml | yq '.serverVersion.minor')
+      if [ ${kube_min_version} -le 21 ]; then
+        cert_manager_version="v1.11.5"
+      fi
+      if [ ${kube_min_version} -le 20 ]; then
+        cert_manager_version="v1.10.2"
+      fi
+      if [ ${kube_min_version} -le 19 ]; then
+        cert_manager_version="v1.8.2"
+      fi
+      if [ ${kube_min_version} -le 18 ]; then
+        cert_manager_version="v1.7.3"
+      fi
+      if [ ${kube_min_version} -le 17 ]; then
+        cert_manager_version="v1.6.3"
+      fi
+      if [ ${kube_min_version} -le 16 ]; then
+        cert_manager_version="v1.5.5"
+      fi
+      if [ ${kube_min_version} -le 15 ]; then
+        echo "Kubernetes version is too low, please upgrade to 1.15+"
+        exit 1
+      fi
+
       helm repo add jetstack https://charts.jetstack.io
       helm repo update
       helm install \
         cert-manager jetstack/cert-manager \
         --namespace cert-manager \
         --create-namespace \
-        --version v1.8.2 \
-        --set installCRDs=true
+        --set crds.enabled=true \
+        --version "$cert_manager_version"
   - name: Deploy operator by helm
     shell: bash
     run: |
+      repository=$(echo ${{ inputs.repository }} | tr '[:upper:]' '[:lower:]')
+      tag=$(echo ${{ inputs.tag }} | tr '/' '-')
       helm install emqx-operator deploy/charts/emqx-operator\
-        --set installCRDs=true \
-        --set image.repository=${{ inputs.repository }} \
-        --set image.tag=${{ inputs.tag }} \
+        --set image.repository=$repository \
+        --set image.tag=$tag \
+        --set development=true \
         --namespace emqx-operator-system \
         --create-namespace
   - name: Check operator
     shell: bash
-    run: |
-      set -euo pipefail
-      while [ "$(kubectl get pods -l "control-plane=controller-manager" -n emqx-operator-system -o json | jq '.items[0].status.containerStatuses[] | select(.ready==true) | .containerID')" = "" ]; do
-        echo "waiting operator controller pod running"
-        sleep 1
-      done
+    run: kubectl wait --for=condition=Ready pods -l "control-plane=controller-manager" -n emqx-operator-system

.github/scripts/directory_check.py

@@ -16,24 +16,30 @@ def check_md_content(md_file):
         success = False
         return
 
-    md_content = open(md_file, 'r').read()
+    md_content = re.sub(r'<!--([\s\S]*?)-->', '', open(md_file, 'r').read())
+
     if 'ee' in directory_file:
         md_content = re.sub(r'{% emqxce %}([\s\S]*?){% endemqxce %}', '', md_content)
     else:
         md_content = re.sub(r'{% emqxee %}([\s\S]*?){% endemqxee %}', '', md_content)
 
+    for i in re.findall(r'([\u4e00-\u9fa5]+)([a-zA-Z]+)|([a-zA-Z]+)([\u4e00-\u9fa5]+)', md_content):
+        print(f'In {md_file} ', end='')
+        print(f'No space between Chinese and English: {i[0]}{i[1]}{i[2]}{i[3]}')
+        success = False
+
     image_list = re.findall('(.*?)!\[(.*?)\]\((.*?)\)', md_content)
     url_list = re.findall('(.*?)\[(.*?)\]\((.*?)\)', md_content)
     for url in url_list:
         if url[0].endswith('!'):
             continue
-        if url[2].startswith(('http://', 'https://', '<', '#')):
+        if url[2].startswith(('http://', 'https://', '<', '#', 'mailto:', 'tel:')):
             continue
         url_path = url[2].split('.md')[0]
         ref_md_path = os.path.join(f'{"/".join(md_file.split("/")[:-1])}/', f'{url_path}.md')
 
         if not os.path.exists(ref_md_path):
-            print(f'In {md_file}：', end='')
+            print(f'In {md_file}: ', end='')
             print(f'{url[2]} not found or not in {directory_file}')
             success = False
 
@@ -45,7 +51,7 @@ def check_md_content(md_file):
         image_path = os.path.join(f'{"/".join(md_file.split("/")[:-1])}/', image[2])
 
         if not os.path.exists(image_path):
-            print(f'In {md_file}：', end='')
+            print(f'In {md_file}: ', end='')
             print(image[2], 'does not exist')
             success = False
 
@@ -56,20 +62,18 @@ def get_md_files(dir_config, path):
     for i in dir_config:
         md_name = i.get('path')
         md_children = i.get('children')
-        if md_name and md_children:
-            print(f'{i.get("title")} has path and children')
-            success = False
 
-        if md_children:
-            md_list += get_md_files(md_children, path)
-        else:
+        if md_name:
             if md_name.startswith(('http://', 'https://')):
                 continue
             elif md_name == './':
-                md_list.append(f'{docs_path}/{path}/README.md')
+                md_list.append(f'{docs_path}/{path}/index.md')
             else:
                 md_list.append(f'{docs_path}/{path}/{md_name}.md')
 
+        if md_children:
+            md_list += get_md_files(md_children, path)
+
     return list(set(md_list))
 
 

.github/scripts/remove_unused.py

@@ -8,13 +8,13 @@
 def get_markdown_file(dir_config, base_path):
     current_files = []
     for row in dir_config:
-        if row.get('children'):
-            current_files += get_markdown_file(row['children'], base_path)
-        else:
+        if row.get('path'):
             current_files.append(
-                f'{base_path}/README.md' if row['path'] == './'
+                f'{base_path}/index.md' if row['path'] == './'
                 else f'{base_path}/{row["path"]}.md'
             )
+        if row.get('children'):
+            current_files += get_markdown_file(row['children'], base_path)
     return current_files
 
 

.github/workflows/build-reloader-image.yaml

@@ -1,4 +1,4 @@
-name: Build reloader image
+name: Build Reloader Image
 
 on:
   push:
@@ -14,7 +14,7 @@ jobs:
   build-reloader-image:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Get reloader image version
       id: version
       run: |
@@ -28,34 +28,34 @@ jobs:
         else
           echo "::set-output name=registry::ghcr.io"
         fi
-    - uses: docker/setup-qemu-action@v1
-    - uses: docker/setup-buildx-action@v1
-    - uses: docker/login-action@v1
+    - uses: docker/setup-qemu-action@v3
+    - uses: docker/setup-buildx-action@v3
+    - uses: docker/login-action@v3
       if: github.ref_type == 'tag'
       with:
         registry: docker.io
         username: ${{ secrets.DOCKER_HUB_USER }}
         password: ${{ secrets.DOCKER_HUB_TOKEN }}
-    - uses: docker/login-action@v1
+    - uses: docker/login-action@v3
       if: github.ref_type == 'branch'
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ github.token }}
-    - uses: docker/metadata-action@v3
+    - uses: docker/metadata-action@v5
       id: meta
       with:
         images: ${{ steps.registry.outputs.registry }}/${{ github.repository }}-reloader
         flavor: |
           latest=true
         tags: |
           type=semver,pattern={{version}},value=${{ steps.version.outputs.version }}
-    - uses: docker/build-push-action@v2
+    - uses: docker/build-push-action@v5
       with:
         push: true
         pull: true
         no-cache: true
         platforms: linux/amd64,linux/arm64,linux/arm/v7
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
-        context: sidecar/reloader
+        context: sidecar/reloader

.github/workflows/check-file.yaml → .github/workflows/check-docs.yaml

@@ -1,30 +1,8 @@
-name: Check file
+name: Check Docs
 
 on: [pull_request]
 
 jobs:
-  check-manifests-and-generate:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '1.18.3'
-      - name: Check manifests
-        run: |
-          make manifests
-          if ! git diff --quiet; then
-            echo "Need run make manifests"
-            exit 1
-          fi
-      - name: Check generate
-        run: |
-          make generate
-          if ! git diff --quiet; then
-            echo "Need run make generate"
-            exit 1
-          fi
-
   check-docs-markdown:
     runs-on: ubuntu-latest
     steps:

.github/workflows/check-helm.yaml

@@ -1,4 +1,4 @@
-name: Check helm version
+name: Check Helm Version
 
 on:
   push:
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - run: minikube start
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Update helm
         run: |
           helm repo add emqx https://repos.emqx.io/charts

.github/workflows/cts.yaml

@@ -4,79 +4,81 @@ on:
   workflow_dispatch:
   schedule:
     - cron: '0 */6 * * *'
-  push:
-    tags:
-      - "*"
 
 jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
+        kubernetes-env:
+          - minikube
+          - k3s
         kubernetes-version:
-          - "v1.20.15"
-          - "v1.21.14"
-          - "v1.22.12"
-          - "v1.23.9"
-          - "v1.24.3"
+          - "v1.23.0"
+          - "v1.24.0"
           - "v1.25.0"
+          - "v1.26.0"
+          - "v1.27.0"
+          - "v1.28.0"
+          - "v1.29.0"
+          - "v1.30.0"
+          - "v1.31.0"
+          - "v1.32.0"
     steps:
-      - run: minikube start --kubernetes-version="${{ matrix.kubernetes-version }}"
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - if: matrix.kubernetes-env == 'k3s'
+        run: |
+          INSTALL_K3S_VERSION="${{ matrix.kubernetes-version }}+k3s1" curl -sfL https://get.k3s.io | sh -
+          sudo chmod 644 /etc/rancher/k3s/k3s.yaml
+          echo "KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV
+      - if: matrix.kubernetes-env == 'minikube'
+        run: minikube start --kubernetes-version="${{ matrix.kubernetes-version }}"
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.18.3'
+          go-version-file: 'go.mod'
       - name: install kubebuilder
         run: |
           OS=$(uname -s | tr '[:upper:]' '[:lower:]')
           ARCH=$(uname -m | sed 's/x86_64/amd64/')
           curl -fsL "https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-1.16.4-${OS}-${ARCH}.tar.gz" -o kubebuilder-tools
           tar -zxvf kubebuilder-tools
           sudo mv kubebuilder/ /usr/local/kubebuilder
-      - run: make test
-      - name: Install goveralls
-        run: go install github.com/mattn/goveralls@latest
-      - name: Send coverage
+      - name: Install Telepresence
         env:
-          COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: goveralls -coverprofile=cover.out -service=github -parallel -flagname=go-test-${{ matrix.kubernetes-version }}
-
-  finish:
-    needs: test
-    runs-on: ubuntu-latest
-    steps:
-    - name: Coveralls Finished
-      uses: coverallsapp/github-action@master
-      with:
-        github-token: ${{ secrets.github_token }}
-        parallel-finished: true
+          TELEPRESENCE_VERSION: 2.13.3
+        run: |
+          sudo curl -fL https://app.getambassador.io/download/tel2/linux/amd64/${TELEPRESENCE_VERSION}/telepresence -o /usr/local/bin/telepresence
+          sudo chmod a+x /usr/local/bin/telepresence
+      - run: telepresence helm install
+      - run: telepresence connect
+      - run: make test
 
   build-image:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - uses: docker/setup-qemu-action@v1
+    - uses: actions/checkout@v4
+    - uses: docker/setup-buildx-action@v3
+    - uses: docker/setup-qemu-action@v3
       with:
         image: tonistiigi/binfmt:latest
         platforms: all
-    - uses: docker/login-action@v1
+    - uses: docker/login-action@v3
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ github.token }}
-    - uses: docker/metadata-action@v3
+    - uses: docker/metadata-action@v5
       id: meta
       with:
-        images: ghcr.io/${{ github.repository }}
+        images: ghcr.io/${{ github.repository_owner }}/emqx-operator
         flavor: |
           latest=true
         tags: |
           type=ref,event=branch
           type=ref,event=tag
           type=semver,pattern={{version}}
-    - uses: docker/build-push-action@v2
+    - uses: docker/build-push-action@v5
       with:
         push: true
         pull: true
@@ -89,29 +91,49 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-image
     strategy:
+      ## Set max parallel to 8 to avoid hitting the GitHub Actions concurrency limit
+      max-parallel: 8
       fail-fast: false
       matrix:
+        kubernetes-env:
+          - minikube
+          - k3s
         kubernetes-version:
-          - "v1.20.15"
-          - "v1.21.14"
-          - "v1.22.12"
-          - "v1.23.9"
-          - "v1.24.3"
+          - "v1.23.0"
+          - "v1.24.0"
           - "v1.25.0"
+          - "v1.26.0"
+          - "v1.27.0"
+          - "v1.28.0"
+          - "v1.29.0"
+          - "v1.30.0"
+          - "v1.31.0"
+          - "v1.32.0"
         emqx:
-          - [EmqxBroker, emqx, "config/samples/emqx/v1beta3/emqx.yaml"]
-          - [EmqxEnterprise, emqx-ee, "config/samples/emqx/v1beta3/emqx-ee.yaml"]
+          # - [EmqxBroker, emqx, "config/samples/emqx/v1beta4/emqxbroker-slim.yaml"]
+          # - [EmqxBroker, emqx, "config/samples/emqx/v1beta4/emqxbroker-full.yaml"]
+          - [EmqxEnterprise, emqx-ee, "config/samples/emqx/v1beta4/emqxenterprise-slim.yaml"]
+          - [EmqxEnterprise, emqx-ee, "config/samples/emqx/v1beta4/emqxenterprise-full.yaml"]
+          - [EMQX, emqx, "config/samples/emqx/v2beta1/emqx-slim.yaml"]
+          - [EMQX, emqx, "config/samples/emqx/v2beta1/emqx-full.yaml"]
+
     steps:
-      - run: minikube start --kubernetes-version="${{ matrix.kubernetes-version }}"
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - if: matrix.kubernetes-env == 'k3s'
+        run: |
+          curl -sfL https://get.k3s.io | sh -
+          sudo chmod 644 /etc/rancher/k3s/k3s.yaml
+          echo "KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV
+      - if: matrix.kubernetes-env == 'minikube'
+        run: minikube start --kubernetes-version="${{ matrix.kubernetes-version }}"
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.18.3'
+          go-version-file: 'go.mod'
       - name: Deploy operator
         timeout-minutes: 5
         uses: ./.github/actions/deploy-operator
         with:
-          repository: ghcr.io/${{ github.repository }}
+          repository: ghcr.io/${{ github.repository_owner }}/emqx-operator
           tag: ${{ github.ref_name }}
       - name: Deploy emqx
         timeout-minutes: 5
@@ -120,7 +142,17 @@ jobs:
           kind: ${{ matrix.emqx[0] }}
           name: ${{ matrix.emqx[1] }}
           file: ${{ matrix.emqx[2] }}
+      - name: Make sure all of pods can be deleted
+        run: |
+          kubectl delete ${{ matrix.emqx[0] }} ${{ matrix.emqx[1] }}
+          kubectl wait --for=delete pods -l "apps.emqx.io/instance=${{ matrix.emqx[1] }}" --timeout=300s
       - if: failure()
         run: kubectl logs -l "control-plane=controller-manager" -n emqx-operator-system -c manager --tail=1000
       - if: failure()
         run: kubectl get ${{ matrix.emqx[0] }} ${{ matrix.emqx[1] }} -o json
+      - if: failure()
+        run: kubectl get events --sort-by='.lastTimestamp'
+      - if: failure()
+        run: kubectl get pods -l "apps.emqx.io/managed-by=emqx-operator" -o json
+      - if: failure()
+        run: kubectl logs -l "apps.emqx.io/managed-by=emqx-operator" -c emqx

.github/workflows/deploy-docs.yaml

@@ -1,57 +1,92 @@
 name: Deploy Docs
 
-concurrency:
-  group: ${{ github.ref }}
-  cancel-in-progress: true
-
 on:
-  push:
-    tags:
-      - "*"
+  workflow_dispatch:
+    inputs:
+      tag:
+        required: true
+        type: string
+  release:
+    types:
+      - published
 
 jobs:
+  prepare:
+    outputs:
+      tags: ${{ steps.tags.outputs.tags}}
+
+    runs-on: ubuntu-20.04
+    if: ${{ !github.event.release.prerelease }}
+
+    steps:
+    - uses: actions/checkout@main
+      with:
+        fetch-depth: 0
+    - name: get_tags
+      id: tags
+      run: |
+        ref=$(git describe --tags --abbrev=0)
+        if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+          ref="${{ inputs.tag }}"
+        fi
+        if [ "${{ github.event_name }}" = "release" ]; then
+          ref="${{ github.event.release.tag_name }}"
+        fi
+        latest_tag="$(curl --silent --show-error \
+            -H "Accept: application/vnd.github.v3+json" \
+            -H "Authorization: token ${{ github.token }}" \
+            https://api.github.com/repos/emqx/emqx-operator/releases/latest \
+            | jq '.tag_name' | tr -d '"')"
+        if [ "$latest_tag" = "$ref" ];then
+          tag=("$ref" "latest")
+        else
+          tag=("$ref")
+        fi
+        tags=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${tag[@]}")
+        echo "tags=$tags" >> $GITHUB_OUTPUT
+
   deploy:
+    needs: prepare
     strategy:
       fail-fast: false
       matrix:
-        ref:
-          - "latest"
-          - ""
+        tags: ${{ fromJSON(needs.prepare.outputs.tags) }}
 
     runs-on: ubuntu-20.04
     if: github.repository_owner == 'emqx'
     steps:
     - name: clone docs
-      uses: actions/checkout@v2
+      uses: actions/checkout@main
       with:
         fetch-depth: 0
         path: docs-files
 
     - name: clone frontend
-      uses: actions/checkout@v2
+      uses: actions/checkout@main
       with:
         repository: 'emqx/docs-emqx-com-frontend'
+        ref: next
         token: ${{ secrets.CI_GIT_TOKEN }}
         path: frontend
 
     - name: use node.js
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v4
+      with:
+        node-version-file: 'frontend/.nvmrc'
+
+    - name: use pnpm
+      uses: pnpm/action-setup@v2
       with:
-        node-version: 14.15
+        version: 8
 
     - name: use python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: '3.8'
-        architecture: 'x64'
 
     - name: set env
       run: |
-        if [ "${{ matrix.ref }}" == "latest" ]; then
-          echo "VERSION=latest" >> $GITHUB_ENV
-        else
-          echo "VERSION=${{ github.ref_name }}" >> $GITHUB_ENV
-        fi
+        echo "VERSION=${{ matrix.tags }}" >> $GITHUB_ENV
         echo "DOCS_TYPE=emqx-operator" >> $GITHUB_ENV
 
     - name: remove unused files
@@ -61,71 +96,61 @@ jobs:
 
     - name: move files
       run: |
-        rm frontend/docs/en/README.md || true
-        rm frontend/docs/zh/README.md || true
+        rm frontend/docs/en/index.md || true
+        rm frontend/docs/zh/index.md || true
         rm frontend/docs/*.md || true
-        rm frontend/README.md
+        rm frontend/index.md || true
         mkdir -p frontend/docs/en/${DOCS_TYPE}/${VERSION}/
         mkdir -p frontend/docs/zh/${DOCS_TYPE}/${VERSION}/
-        mkdir -p frontend/docs/.vuepress/public/api/
+        mkdir -p frontend/docs/public/api/
         cp -r docs-files/docs/en_US/* frontend/docs/en/${DOCS_TYPE}/${VERSION}/
         cp -r docs-files/docs/zh_CN/* frontend/docs/zh/${DOCS_TYPE}/${VERSION}/
-        cp docs-files/docs/directory.json frontend/docs/.vuepress/config/directory.json
-        cd docs-files && git tag -l | egrep "^[1-9]\.[0-9]\.[0-9]?" | jq -R -s -c 'split("\n")[:-1] | . += ["latest"] | reverse' > ../frontend/docs/.vuepress/public/api/${DOCS_TYPE}_versions.json
+        cp docs-files/docs/directory.json frontend/docs/.vitepress/config/directory.json
+        cd docs-files && git tag -l | egrep "^[1-9]+\.[0-9]+\.[0-9]+$" | jq -R -s -c 'split("\n")[:-1] | . += ["latest"] | reverse' > ../frontend/docs/public/api/${DOCS_TYPE}_versions.json
 
     - name: build docs
       run: |
         cd frontend
-        yarn && yarn build
+        pnpm install
+        pnpm build
+
+    - name: set aws credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+
+    - name: upload dist to s3
+      run: |
+        aws s3 rm --recursive s3://docs-emqx-com/zh/${DOCS_TYPE}/${VERSION} || true
+        aws s3 rm --recursive s3://docs-emqx-com/en/${DOCS_TYPE}/${VERSION} || true
+        aws s3 cp --recursive frontend/docs/.vitepress/dist/ s3://docs-emqx-com/
+        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_DOCS_CLOUDFRONT_ID }} --paths "/zh/${DOCS_TYPE}/${VERSION}/*" "/en/${DOCS_TYPE}/${VERSION}/*" "/api/${DOCS_TYPE}_versions.json" "/sitemap_${DOCS_TYPE}_${VERSION}.xml"
 
-    - name: upload dist
+    - name: upload dist to cos
       run: |
         pip3 install coscmd
-        coscmd config -a ${{ secrets.COS_ID }} -s ${{ secrets.COS_KEY }} -b ${{ secrets.COS_NAME }} -r ${{ secrets.COS_REGION }}
+        coscmd config -a ${{ secrets.TENCENT_COS_ID }} -s ${{ secrets.TENCENT_COS_KEY }} -b docs-1302406139 -r ap-shanghai
         coscmd delete -r -f en/${DOCS_TYPE}/${VERSION} || true
         coscmd delete -r -f zh/${DOCS_TYPE}/${VERSION} || true
-        coscmd config -a ${{ secrets.COS_ID }} -s ${{ secrets.COS_KEY }} -b ${{ secrets.COS_NAME }} -e cos.accelerate.myqcloud.com
-        cd frontend/docs/.vuepress/dist/
-        zip -rq docs-dist-${DOCS_TYPE}-${VERSION}.zip ./
-        coscmd upload docs-dist-${DOCS_TYPE}-${VERSION}.zip docs-zip/
-        sleep 90
+        coscmd config -a ${{ secrets.TENCENT_COS_ID }} -s ${{ secrets.TENCENT_COS_KEY }} -b docs-1302406139 -e cos.accelerate.myqcloud.com
+        coscmd upload -r frontend/docs/.vitepress/dist/ /
 
     - name: flush cdn
       run: |
         pip3 install tccli
-        tccli configure set secretId ${{ secrets.COS_ID }}
-        tccli configure set secretKey ${{ secrets.COS_KEY }}
-        tccli configure set region ${{ secrets.COS_REGION }}
+        tccli configure set secretId ${{ secrets.TENCENT_COS_ID }}
+        tccli configure set secretKey ${{ secrets.TENCENT_COS_KEY }}
+        tccli configure set region ap-shanghai
         tccli cdn PurgePathCache --cli-unfold-argument --Paths https://docs.emqx.com/zh/${DOCS_TYPE}/${VERSION}/ https://docs.emqx.com/en/${DOCS_TYPE}/${VERSION}/ --FlushType delete
         tccli cdn PurgeUrlsCache --cli-unfold-argument --Urls https://docs.emqx.com/api/${DOCS_TYPE}_versions.json https://docs.emqx.com/sitemap_${DOCS_TYPE}_${VERSION}.xml
 
-    - name: clone docsearch
-      uses: actions/checkout@v2
-      with:
-        repository: 'Swilder-M/docsearch-scraper-simple'
-        path: docsearch
-
-    - name: install pipenv
-      run: |
-          python -m pip install --upgrade pipenv wheel
-
-    - id: cache-pipenv
-      uses: actions/cache@v1
-      with:
-        path: ~/.local/share/virtualenvs
-        key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
-
-    - name: install dependencies
-      if: steps.cache-pipenv.outputs.cache-hit != 'true'
-      run: |
-        cd docsearch
-        pipenv install --keep-outdated
-
-    - name: update current version
-      if: env.VERSION == 'latest'
+    - name: update search index
+      uses: Swilder-M/docsearch-scraper-simple@next
       env:
-        APPLICATION_ID: ${{ secrets.DOCS_APPLICATION_ID }}
-        API_KEY: ${{ secrets.DOCS_API_KEY }}
-      run: |
-        cd docsearch
-        pipenv run python -m src.index config ${DOCS_TYPE} ${VERSION}
+        APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID_NEXT }}
+        API_KEY: ${{ secrets.ALGOLIA_API_KEY_NEXT }}
+      with:
+        docs_type: ${{ env.DOCS_TYPE }}
+        docs_version: ${{ env.VERSION }}

.github/workflows/deploy.yaml

@@ -0,0 +1,112 @@
+name: Deployment Test Suite
+
+concurrency:
+  group: deploy-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+
+jobs:
+  deployment:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        install:
+          - static
+          - helm
+        emqx:
+          # - [EmqxBroker, emqx, "config/samples/emqx/v1beta3/emqxbroker-slim.yaml"]
+          # - [EmqxBroker, emqx, "config/samples/emqx/v1beta3/emqxbroker-full.yaml"]
+          - [EmqxEnterprise, emqx-ee, "config/samples/emqx/v1beta3/emqxenterprise-slim.yaml"]
+          - [EmqxEnterprise, emqx-ee, "config/samples/emqx/v1beta3/emqxenterprise-full.yaml"]
+
+          # - [EmqxBroker, emqx, "config/samples/emqx/v1beta4/emqxbroker-slim.yaml"]
+          # - [EmqxBroker, emqx, "config/samples/emqx/v1beta4/emqxbroker-full.yaml"]
+          - [EmqxEnterprise, emqx-ee, "config/samples/emqx/v1beta4/emqxenterprise-slim.yaml"]
+          - [EmqxEnterprise, emqx-ee, "config/samples/emqx/v1beta4/emqxenterprise-full.yaml"]
+
+          - [EMQX, emqx, "config/samples/emqx/v2alpha1/emqx-slim.yaml"]
+          - [EMQX, emqx, "config/samples/emqx/v2alpha1/emqx-full.yaml"]
+
+          - [EMQX, emqx, "config/samples/emqx/v2beta1/emqx-slim.yaml"]
+          - [EMQX, emqx, "config/samples/emqx/v2beta1/emqx-full.yaml"]
+        single_namespace:
+          - false
+          - true
+        enable_webhook:
+          - true
+        exclude:
+          - install: static
+            single_namespace: true
+        include:
+          - enable_webhook: false
+            install: helm
+            single_namespace: true
+            emqx: [EMQX, emqx, "config/samples/emqx/v2beta1/emqx-slim.yaml"]
+          - enable_webhook: false
+            install: helm
+            single_namespace: true
+            emqx: [EMQX, emqx, "config/samples/emqx/v2beta1/emqx-full.yaml"]
+
+    steps:
+      - run: minikube start
+      - name: install cert-manager
+        run: |
+          helm repo add jetstack https://charts.jetstack.io
+          helm repo update
+          helm install \
+            cert-manager jetstack/cert-manager \
+            --namespace cert-manager \
+            --create-namespace \
+            --set crds.enabled=true
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+      - name: Build image
+        env:
+          IMG: "emqx/emqx-operator-controller:${{ github.sha }}"
+        run: eval $(minikube docker-env) && docker build -t $IMG .
+      - name: Deploy controller
+        if: matrix.install == 'static'
+        env:
+          IMG: "emqx/emqx-operator-controller:${{ github.sha }}"
+        run: make deploy
+      - name: Deploy controller
+        if: matrix.install == 'helm'
+        run: |
+          helm install emqx-operator deploy/charts/emqx-operator \
+            --set image.tag=${{ github.sha }} \
+            --set development=true \
+            --set singleNamespace=${{ matrix.single_namespace }} \
+            --namespace ${{ matrix.single_namespace && 'default' || 'emqx-operator-system' }} \
+            --create-namespace
+      - name: Check operator
+        timeout-minutes: 5
+        run: |
+          kubectl wait --for=condition=Ready pods \
+            -l "control-plane=controller-manager" \
+            -n ${{ matrix.single_namespace && 'default' || 'emqx-operator-system' }}
+      - name: Deployment emqx
+        timeout-minutes: 5
+        uses: ./.github/actions/deploy-emqx
+        with:
+          kind: ${{ matrix.emqx[0] }}
+          name: ${{ matrix.emqx[1] }}
+          file: ${{ matrix.emqx[2] }}
+      - name: Make sure all of pods can be deleted
+        run: |
+          kubectl delete ${{ matrix.emqx[0] }} ${{ matrix.emqx[1] }}
+          kubectl wait --for=delete pods -l "apps.emqx.io/instance=${{ matrix.emqx[1] }}"
+      - if: failure()
+        run: kubectl logs -l "control-plane=controller-manager" -n emqx-operator-system -c manager --tail=1000
+      - if: failure()
+        run: kubectl get ${{ matrix.emqx[0] }} ${{ matrix.emqx[1] }} -o json
+      - if: failure()
+        run: kubectl get events --sort-by='.lastTimestamp'
+      - if: failure()
+        run: kubectl get pods -l "apps.emqx.io/managed-by=emqx-operator" -o json
+      - if: failure()
+        run: kubectl logs -l "apps.emqx.io/managed-by=emqx-operator" -c emqx

.github/workflows/gitlint.yaml

@@ -1,4 +1,4 @@
-name: Run gitlint
+name: Run Gitlint
 
 on: [pull_request]
 
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install gitlint
         run: |
           sudo apt-get update

.github/workflows/issue.yaml

@@ -1,4 +1,4 @@
-name: Manage stale issues
+name: Manage Stale Issues
 
 on:
   schedule:
@@ -26,4 +26,4 @@ jobs:
             for your contributions.
           # we don't want stalebot to analyze pull requests
           only-pr-labels: "ZZZDisabledZZZ"
-          operations-per-run: 80
+          operations-per-run: 80

.github/workflows/markdownlint.json

@@ -12,9 +12,6 @@
 		"level": 1,
 		"front_matter_title": ""
 	},
-    "MD033": {
-        "allowed_elements": ["table", "tr", "td", "th"]
-    },
 	"MD042": true,
 	"MD046": {
 		"style": "fenced"

.github/workflows/push-helm.yaml

@@ -0,0 +1,36 @@
+name: Push Helm Chart
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        required: true
+        type: string
+  release:
+    types:
+      - published
+
+jobs:
+  helm:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Get tag
+      id: get_tag
+      run: |
+        if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+          tag="${{ inputs.tag }}"
+        fi
+        if [ "${{ github.event_name }}" = "release" ]; then
+          tag="${{ github.event.release.tag_name }}"
+        fi
+        echo "tag=$tag" >> $GITHUB_OUTPUT
+    - name: Update helm repo
+      uses: emqx/push-helm-action@v1.1
+      with:
+        charts_dir: "${{ github.workspace }}/deploy/charts/emqx-operator"
+        version: ${{ steps.get_tag.outputs.tag }}
+        aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws_region: "us-west-2"
+        aws_bucket_name: "repos-emqx-io"

.github/workflows/release.yml

@@ -1,76 +1,104 @@
-name: Release emqx operator
+name: Release EMQX Operator
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
 
 on:
   push:
     tags:
       - "*"
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Ref to release'
+        required: true
+        default: HEAD
+
+env:
+  GIT_TAG: ${{ github.event_name == 'push' && github.ref || inputs.tag }}
 
 jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Check image tag for yaml
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ env.GIT_TAG }}
+    - name: Check helm
+      if: github.event_name == 'push'
       run: |
-        tag=$(cat deploy/manifests/emqx-operator-controller.yaml | egrep "image:[[:space:]]emqx/emqx-operator-controller" | sed -r 's|image:[[:space:]]emqx/emqx-operator-controller:(.*)|\1|g' | tr -d '[:space:]')
-        if [ "$tag" != "${GITHUB_REF##*/}" ]; then
-          echo "Need update image tag for deploy/manifests/emqx-operator-controller.yaml"
+        version=$(grep -E "^version" deploy/charts/emqx-operator/Chart.yaml | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9])?")
+        if [ "$version" != "${GITHUB_REF##*/}" ]; then
+          echo "Need update version for Chart.yaml"
           exit 1
         fi
-    - name: Check appVersion for helm
-      run: |
-        appVersion=$(egrep "^appVersion" deploy/charts/emqx-operator/Chart.yaml | sed -r 's|^appVersion:[[:space:]]([0-9]+.[0-9]+.[0-9]+)$|\1|g')
+        appVersion=$(grep -E "^appVersion" deploy/charts/emqx-operator/Chart.yaml | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9])?")
         if [ "$appVersion" != "${GITHUB_REF##*/}" ]; then
           echo "Need update appVersion for Chart.yaml"
           exit 1
         fi
-    - uses: docker/setup-qemu-action@v1
-    - uses: docker/setup-buildx-action@v1
-    - uses: docker/metadata-action@v3
+    - name: Generate CRDs file
+      run: |
+        make kustomize
+        ./bin/kustomize build ./config/crd > crds.yaml
+    - uses: docker/setup-qemu-action@v3
+    - uses: docker/setup-buildx-action@v3
+    - uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ github.token }}
+    - uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
+    - uses: docker/metadata-action@v5
       id: meta
       with:
-        images: ${{ github.repository_owner }}/emqx-operator-controller
+        images: |
+          ${{ github.repository_owner }}/emqx-operator-controller
+          ghcr.io/${{ github.repository_owner }}/emqx-operator
         tags: |
           type=ref,event=branch
           type=ref,event=pr
           type=ref,event=tag
-          type=semver,pattern={{version}}
-          type=semver,pattern={{major}}.{{minor}}
-    - uses: docker/login-action@v1
-      if: startsWith(github.ref, 'refs/tags/')
-      with:
-        username: ${{ secrets.DOCKER_HUB_USER }}
-        password: ${{ secrets.DOCKER_HUB_TOKEN }}
-    - uses: docker/build-push-action@v2
+          type=semver,pattern={{version}},value=${{ env.GIT_TAG }}
+          type=semver,pattern={{major}}.{{minor}},value=${{ env.GIT_TAG }}
+    - uses: docker/build-push-action@v5
       with:
         platforms: linux/amd64,linux/arm64,linux/arm/v7
-        push: ${{ startsWith(github.ref, 'refs/tags/') }}
+        push: true
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         context: .
+    - uses: peter-evans/dockerhub-description@v3
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
+        repository: "${{ github.repository_owner }}/emqx-operator-controller"
+        readme-filepath: ./README.md
+        short-description: "EMQX Operator creates and manages EMQX clusters running in Kubernetes."
     - id: prerelease
       run: |
-        if echo "${{ github.ref_name }}" |egrep -q "^[0-9].[0-9].[0-9]$"; then
-          echo "::set-output name=prerelease::false"
+        if echo "${{ github.ref_name }}" |egrep -q "^[0-9]+.[0-9]+.[0-9]+$"; then
+          echo "prerelease=false" >> $GITHUB_OUTPUT
         else
-          echo "::set-output name=prerelease::true"
+          echo "prerelease=true" >> $GITHUB_OUTPUT
         fi
-    - uses: softprops/action-gh-release@v1
-      if: startsWith(github.ref, 'refs/tags/')
+    - uses: softprops/action-gh-release@v2
       with:
+        tag_name: ${{ env.GIT_TAG }}
         prerelease: ${{ steps.prerelease.outputs.prerelease }}
-        token: ${{ github.token }}
+        generate_release_notes: true
+        ## When you use the repository's GITHUB_TOKEN to perform tasks,
+        ## events triggered by the GITHUB_TOKEN, with the exception of workflow_dispatch and repository_dispatch,
+        ## will not create a new workflow run.
+        ## This prevents you from accidentally creating recursive workflow runs.
+        ## More info: https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
+        # token: ${{ github.token }}
+        token: ${{ secrets.CI_GIT_TOKEN }}
         name: EMQX Operator ${{ github.ref_name }} Released
         body_path: RELEASE.md
-        generate_release_notes: true
         files: |
-          deploy/manifests/emqx-operator-controller.yaml
-    - name: Update helm repo
-      uses: emqx/push-helm-action@v1
-      with:
-        charts_dir: "${{ github.workspace }}/deploy/charts/emqx-operator"
-        version: ${{ github.ref_name }}
-        aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        aws_region: "us-west-2"
-        aws_bucket_name: "repos-emqx-io"
+          crds.yaml

.github/workflows/test.yaml

@@ -1,137 +1,145 @@
-name: Run emqx operator test case
+name: Run EMQX Operator Test Case
 
 concurrency:
   group: test-${{ github.event_name }}-${{ github.ref }}
   cancel-in-progress: true
 
 on:
   pull_request:
+  push:
+    ## For update codecov.io
+    branches:
+      - main**
 
 jobs:
   lint:
     runs-on: ubuntu-latest
+    outputs:
+      e2e: ${{ steps.e2e.outputs.e2e }}
+
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.18.3'
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+          go-version-file: 'go.mod'
+      - uses: golangci/golangci-lint-action@v4
         with:
-          version: latest
+          ##https://github.com/golangci/golangci-lint-action/issues/807
+          skip-pkg-cache: true
           args: --timeout=5m
+      - run: go install github.com/google/go-licenses@latest
+      - run: $(go env GOPATH)/bin/go-licenses check . --disallowed_types forbidden,restricted,unknown
+      - name: Check manifests
+        run: |
+          make manifests
+          if ! git diff --quiet; then
+            echo "Need run make manifests"
+            exit 1
+          fi
+      - name: Check generate
+        run: |
+          make generate
+          if ! git diff --quiet; then
+            echo "Need run make generate"
+            exit 1
+          fi
+      - name: Check helm crds
+        run: |
+          make helm-crds
+          if ! git diff --quiet; then
+            echo "Need run make helm-crds"
+            exit 1
+          fi
+      - name: Check line-break at EOF
+        run: ./scripts/check-nl-at-eof.sh
+      - name: Check space at EOL
+        run: ./scripts/check-space-at-eol.sh
+      - name: Run shellcheck
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends shellcheck
+          ./scripts/shellcheck.sh
+      - name: Output e2e test cases
+        id: e2e
+        run: |
+          e2e=$(find e2e -type f ! -name "suite_test.go" | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "e2e test cases: $e2e"
+          echo "e2e=$e2e" >> $GITHUB_OUTPUT
 
-  test:
+  e2e-test:
     runs-on: ubuntu-latest
+    needs: lint
+
     strategy:
       fail-fast: false
       matrix:
-        dir:
-        - apis
-        - controllers
-        - pkg
-        - e2e/v1beta3
+        file: ${{fromJSON(needs.lint.outputs.e2e)}}
 
     steps:
       - run: minikube start
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.18.3'
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: install kubebuilder
+          go-version-file: 'go.mod'
+      - run: go install github.com/onsi/ginkgo/v2/ginkgo@latest
+      - name: Install Telepresence
+        env:
+          TELEPRESENCE_VERSION: 2.13.3
         run: |
-          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
-          ARCH=$(uname -m | sed 's/x86_64/amd64/')
-          curl -fsL "https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-1.16.4-${OS}-${ARCH}.tar.gz" -o kubebuilder-tools
-          tar -zxvf kubebuilder-tools
-          sudo mv kubebuilder/ /usr/local/kubebuilder
-      - name: Run unit tests
-        run: go test -coverpkg=./... -coverprofile=cover.out --timeout=20m ./${{ matrix.dir }}/...
-      - name: Install goveralls
-        run: go install github.com/mattn/goveralls@latest
-      - name: Send coverage
+          sudo curl -fL https://app.getambassador.io/download/tel2/linux/amd64/${TELEPRESENCE_VERSION}/telepresence -o /usr/local/bin/telepresence
+          sudo chmod a+x /usr/local/bin/telepresence
+      - run: telepresence helm install
+      - run: telepresence connect
+      - name: Run e2e test cases
+        id: e2e_test
         env:
-          COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: goveralls -coverprofile=cover.out -service=github -parallel -flagname=go-test-${{ matrix.dir }}
-
-  finish:
-    needs: test
+          FILE: ${{ matrix.file }}
+        run: |
+          echo "Run e2e test cases: $FILE"
+          filename="$(basename $FILE)"
+          filename="${filename%.*}"
+          $(go env GOPATH)/bin/ginkgo -v --cover -covermode=atomic -coverpkg=./... -coverprofile=${filename}.out --focus-file=${filename} $(dirname $FILE)
+          echo "filename=${filename}" >> "$GITHUB_OUTPUT"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverprofile-${{ steps.e2e_test.outputs.filename }}
+          path: |
+            *.out
+
+  unit-test:
     runs-on: ubuntu-latest
     steps:
-    - name: Coveralls Finished
-      uses: coverallsapp/github-action@master
-      with:
-        github-token: ${{ secrets.github_token }}
-        parallel-finished: true
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+      - run: make test
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverprofile-unit-test
+          path: |
+            *.out
 
-  deployment:
+  codecov:
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        install:
-          - static
-          - helm
-        emqx:
-          - [EmqxBroker, emqx, "config/samples/emqx/v1beta3/emqx.yaml"]
-          - [EmqxEnterprise, emqx-ee, "config/samples/emqx/v1beta3/emqx-ee.yaml"]
+    needs:
+      - e2e-test
+      - unit-test
     steps:
-      - run: minikube start
-      - name: install cert-manager
-        run: |
-          helm repo add jetstack https://charts.jetstack.io
-          helm repo update
-          helm install \
-            cert-manager jetstack/cert-manager \
-            --namespace cert-manager \
-            --create-namespace \
-            --version v1.8.2 \
-            --set installCRDs=true
-      - uses: actions/setup-go@v2
+      - uses: actions/download-artifact@v4
         with:
-          go-version: '1.18.3'
-      - uses: actions/checkout@v2
-      - name: Build image
-        env:
-          IMG: "emqx/emqx-operator-controller:${{ github.sha }}"
-        run: eval $(minikube docker-env) && docker build -t $IMG .
-      - name: Deploy controller
-        if: matrix.install == 'static'
-        env:
-          IMG: "emqx/emqx-operator-controller:${{ github.sha }}"
-        run: make deploy
-      - name: Deploy controller
-        if: matrix.install == 'helm'
-        run: |
-          helm install emqx-operator deploy/charts/emqx-operator \
-            --set installCRDs=true \
-            --set image.tag=${{ github.sha }} \
-            --namespace emqx-operator-system \
-            --create-namespace
-      - name: Check operator
-        timeout-minutes: 5
+          pattern: coverprofile-*
+      - name: Get cover files
+        id: files
         run: |
-          set -euo pipefail
-          while [ "$(kubectl get pods -l "control-plane=controller-manager" -n emqx-operator-system -o json | jq '.items[0].status.containerStatuses[] | select(.ready==true) | .containerID')" = "" ]; do
-            echo "waiting operator controller pod running"
-            sleep 1
-          done
-      - name: Deployment emqx
-        timeout-minutes: 5
-        uses: ./.github/actions/deploy-emqx
+          files="$(find . -type f -name '*.out' -exec readlink -f '{}' ';' | tr '\n' ',' | sed 's/,$//g')"
+          echo "files=$files" >> $GITHUB_OUTPUT
+      - uses: codecov/codecov-action@v4
+        with:
+          files: ${{ steps.files.outputs.files }} # optional
+          fail_ci_if_error: true # optional (default = false)
+          verbose: true # optional (default = false)
+          token: ${{ secrets.CODECOV_TOKEN }} # required
+      - uses: geekyeggo/delete-artifact@v5
         with:
-          kind: ${{ matrix.emqx[0] }}
-          name: ${{ matrix.emqx[1] }}
-          file: ${{ matrix.emqx[2] }}
-      - if: failure()
-        run: kubectl logs -l "control-plane=controller-manager" -n emqx-operator-system -c manager --tail=1000
-      - if: failure()
-        run: kubectl get ${{ matrix.emqx[0] }} ${{ matrix.emqx[1] }} -o json
+          name: coverprofile-*

.github/workflows/upgrade.yaml

@@ -1,5 +1,4 @@
-
-name: Upgrade emqx operator
+name: Upgrade Test Suite
 
 concurrency:
   group: upgrade-${{ github.event_name }}-${{ github.ref }}
@@ -14,14 +13,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        install:
-          - static
-          - helm
-        apiVersion:
-          - v1beta3
         emqx:
-          - [EmqxBroker, "emqx/emqx:4.4.8"]
-          - [EmqxEnterprise, "emqx/emqx-ee:4.4.8"]
+          - [EmqxBroker, emqx, "config/samples/emqx/v1beta3/emqxbroker-slim.yaml"]
+          # - [EmqxBroker, emqx, "config/samples/emqx/v1beta3/emqxbroker-full.yaml"]
+          - [EmqxEnterprise, emqx-ee, "config/samples/emqx/v1beta3/emqxenterprise-slim.yaml"]
+          # - [EmqxEnterprise, emqx-ee, "config/samples/emqx/v1beta3/emqxenterprise-full.yaml"]
+          - [EMQX, emqx, "config/samples/emqx/v2alpha1/emqx-slim.yaml"]
+          - [EMQX, emqx, "config/samples/emqx/v2alpha1/emqx-full.yaml"]
+
     steps:
       - run: minikube start
       - name: install cert-manager
@@ -32,112 +31,68 @@ jobs:
             cert-manager jetstack/cert-manager \
             --namespace cert-manager \
             --create-namespace \
-            --version v1.8.2 \
-            --set installCRDs=true
-      - uses: actions/setup-go@v2
+            --set crds.enabled=true
+      - uses: actions/checkout@v4
         with:
-          go-version: '1.18.3'
-      - uses: actions/checkout@v2
-      - name: Install storage operator
-        if: matrix.install == 'static'
+          fetch-depth: 0
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+      - name: Get pre stable version
+        ## Just get the last stable version, ignore the pre-release version
         run: |
-          browser_download_url="$(curl --silent --show-error \
-            -H "Accept: application/vnd.github.v3+json" \
-            -H "Authorization: token ${{ github.token }}" \
-            https://api.github.com/repos/emqx/emqx-operator/releases/latest \
-            | jq '.assets[0].browser_download_url')"
-          kubectl apply -f "$browser_download_url" --server-side --force-conflicts
+          echo "PRE_VERSION=$(git tag -l | grep -oE "^[0-9]+.[0-9]+.[0-9]+$" | sed -n '$p'  )" >> $GITHUB_ENV
       - name: Install storage operator
-        if: matrix.install == 'helm'
         run: |
           helm repo add emqx https://repos.emqx.io/charts
           helm repo update
           helm install emqx-operator emqx/emqx-operator \
-            --set installCRDs=true \
             --namespace emqx-operator-system \
-            --create-namespace
+            --create-namespace \
+            --version ${{ env.PRE_VERSION }}
       - name: Check operator
         timeout-minutes: 5
-        run: |
-          set -euo pipefail
-          while [ "$(kubectl get pods -l "control-plane=controller-manager" -n emqx-operator-system -o json | jq '.items[0].status.containerStatuses[] | select(.ready==true) | .containerID')" = "" ]; do
-            echo "waiting operator controller pod running"
-            sleep 1
-          done
-      - name: Deploy emqx
-        run: |
-          cat << "EOF" | kubectl apply -f -
-          apiVersion: apps.emqx.io/${{ matrix.apiVersion }}
-          kind: ${{ matrix.emqx[0] }}
-          metadata:
-            name: emqx
-            labels:
-              cluster: emqx
-          spec:
-            replicas: 3
-            persistent:
-              storageClassName: standard
-              resources:
-                requests:
-                  storage: 20Mi
-              accessModes:
-              - ReadWriteOnce
-            emqxTemplate:
-              image: ${{ matrix.emqx[1] }}
-          EOF
-      - name: Check emqx status
+        run: kubectl wait --for=condition=Ready pods -l "control-plane=controller-manager" -n emqx-operator-system
+      - name: Deployment emqx
         timeout-minutes: 5
-        run: |
-          while
-            condition="$(kubectl get ${{ matrix.emqx[0] }} emqx -o json | jq '.status.conditions[0]')";
-            [[ "$condition" == "null" ]] \
-            || [[ "$(echo $condition | jq --raw-output '.type')" != "Running" ]] \
-            || [[ "$(echo $condition | jq --raw-output '.status')" != "True" ]]
-          do
-            echo "waiting"
-            sleep 1
-          done
+        uses: ./.github/actions/deploy-emqx
+        with:
+          kind: ${{ matrix.emqx[0] }}
+          name: ${{ matrix.emqx[1] }}
+          file: https://raw.githubusercontent.com/emqx/emqx-operator/${{ env.PRE_VERSION }}/${{ matrix.emqx[2] }}
       - name: Build image
         env:
           IMG: "emqx/emqx-operator-controller:${{ github.sha }}"
         run: eval $(minikube docker-env) && docker build -t $IMG .
       - name: Deploy controller
-        if: matrix.install == 'static'
-        env:
-          IMG: "emqx/emqx-operator-controller:${{ github.sha }}"
-        run: make deploy
-      - name: Deploy controller
-        if: matrix.install == 'helm'
         run: |
           helm upgrade --install emqx-operator deploy/charts/emqx-operator \
-            --set installCRDs=true \
-            --set image.tag=${{ github.sha }} \
             --namespace emqx-operator-system \
-            --create-namespace
+            --create-namespace \
+            --set image.tag=${{ github.sha }} \
+            --set development=true \
+            --debug
       - name: Check operator
         env:
           IMG: "emqx/emqx-operator-controller:${{ github.sha }}"
         timeout-minutes: 5
-        run: |
-          set -euo pipefail
-          while [ "$(kubectl get pods -l "control-plane=controller-manager" -n emqx-operator-system -o json | jq '.items[0].status.containerStatuses[] | select(.ready==true) | .image' | tr -d '"')" != "$IMG" ]; do
-            echo "waiting operator controller pod running"
-            sleep 1
-          done
+        run: kubectl wait --for=condition=Ready pods -l "control-plane=controller-manager" -n emqx-operator-system
       - name: Check emqx status
         timeout-minutes: 5
         run: |
           sleep 30
           while
-            condition="$(kubectl get ${{ matrix.emqx[0] }} emqx -o json | jq '.status.conditions[0]')";
-            [[ "$condition" == "null" ]] \
-            || [[ "$(echo $condition | jq --raw-output '.type')" != "Running" ]] \
-            || [[ "$(echo $condition | jq --raw-output '.status')" != "True" ]]
+            type="$(kubectl get ${{ matrix.emqx[0] }} ${{ matrix.emqx[1] }} -o json |jq '.status.conditions[0] | select(.status == "True")' | jq --raw-output '.type')"
+            [[ "$type" != "Ready" ]] && [[ "$type" != "Running" ]]
           do
             echo "waiting"
             sleep 1
           done
       - if: failure()
         run: kubectl logs -l "control-plane=controller-manager" -n emqx-operator-system -c manager --tail=1000
       - if: failure()
-        run: kubectl get ${{ matrix.emqx[0] }} emqx -o json
+        run: kubectl get pods -l "control-plane=controller-manager" -n emqx-operator-system -o json
+      - if: failure()
+        run: kubectl get ${{ matrix.emqx[0] }} ${{ matrix.emqx[1] }}  -o json
+      - if: failure()
+        run: kubectl get event

.gitignore

@@ -26,3 +26,5 @@ cspell.config.yaml
 .idea/
 
 *.tgz
+
+ginkgo.report

CONTRIBUTING.md

@@ -79,4 +79,4 @@ Just as in the **subject**, use the imperative, present tense: "change" not "cha
 
 The footer should contain any information about **Breaking Changes** and is also the place to reference GitHub issues that this commit **Closes**.
 
-**Breaking Changes** should start with the word `BREAKING CHANGE:` with a space or two newlines. The rest of the commit message is then used for this.
+**Breaking Changes** should start with the word `BREAKING CHANGE:` with a space or two newlines. The rest of the commit message is then used for this.

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.18.3 as builder
+FROM golang:1.22 AS builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -13,7 +13,7 @@ RUN go work init && go work use . && go mod download
 COPY main.go main.go
 COPY apis/ apis/
 COPY controllers/ controllers/
-COPY pkg/ pkg/
+COPY internal/ internal/
 
 # Build
 RUN CGO_ENABLED=0 GOOS=linux go build -a -o manager main.go

Makefile

@@ -1,8 +1,7 @@
 # Image URL to use all building/pushing image targets
 IMG ?= emqx/emqx-operator-controller:$(shell $(CURDIR)/scripts/get-version.sh)
-# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
-#CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
-CRD_OPTIONS ?= "crd:generateEmbeddedObjectMeta=true"
+# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
+ENVTEST_K8S_VERSION = 1.25.0
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -37,8 +36,9 @@ help: ## Display this help.
 
 ##@ Development
 
-manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+manifests: crd-ref-docs controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+	$(CONTROLLER_GEN) crd:maxDescLen=0,generateEmbeddedObjectMeta=true rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(call gen-crd-ref-docs)
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
@@ -50,22 +50,25 @@ vet: ## Run go vet against code.
 	go vet ./...
 
 test: manifests generate fmt vet envtest ## Run tests.
-	go test -coverpkg=./... -coverprofile=cover.out --timeout=20m ./...
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -race --cover -covermode=atomic -coverpkg=./... -coverprofile=cover.out $(shell go list ./...|grep -v e2e)
 
 ##@ Build
 
 build: generate fmt vet ## Build manager binary.
 	go build -o bin/manager main.go
 
 run: manifests generate fmt vet ## Run a controller from your host.
-	go run ./main.go
+	go run ./main.go --zap-devel=true
 
 docker-build: test ## Build docker image with the manager.
 	docker build --no-cache -t ${IMG} .
 
 docker-push: ## Push docker image with the manager.
 	docker push ${IMG}
 
+helm-crds: manifests kustomize ## build CRDs to helm template
+	$(PROJECT_DIR)/scripts/gen-helm-crds.sh
+
 ##@ Deployment
 
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
@@ -77,29 +80,84 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
+	kubectl wait --for=condition=Ready pods -l "control-plane=controller-manager" -n emqx-operator-system
 
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/default | kubectl delete -f -
 
-CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
-controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.2)
+.PHONY: dev
+dev: manifests kustomize local-webhook ## Instanll all the dependencies and run the controller locally
+	$(KUSTOMIZE) build config/dev | kubectl apply -f -
+
+.PHONY: undev
+undev: manifests kustomize local-webhook ## Uninstanll all the dependencies and run the controller locally
+	$(KUSTOMIZE) build config/dev | kubectl delete -f -
 
-KUSTOMIZE = $(shell pwd)/bin/kustomize
-kustomize: ## Download kustomize locally if necessary.
-	$(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.2)
 
-ENVTEST = $(shell pwd)/bin/setup-envtest
-envtest: ## Download envtest-setup locally if necessary.
-	$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
-	$(ENVTEST) use 1.21
+##@ Build Dependencies
 
-# go-install-tool will 'go install' any package $2 and install it to $1.
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-install-tool
-@[ -f $(1) ] || { \
-set -e ;\
-echo "Install $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
-}
+
+## Location to install dependencies to
+LOCALBIN ?= $(PROJECT_DIR)/bin
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+## Tool Binaries
+KUSTOMIZE ?= $(LOCALBIN)/kustomize
+CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
+ENVTEST ?= $(LOCALBIN)/setup-envtest
+CRD_REF_DOCS = $(LOCALBIN)/crd-ref-docs
+TELEPRESENCE_GEN ?= $(LOCALBIN)/telepresence
+
+## Tool Versions
+KUSTOMIZE_VERSION ?= v4
+CONTROLLER_TOOLS_VERSION ?= latest
+ENVTEST_VERSION ?= latest
+CRD_REF_DOCS_VERSION ?= latest
+
+## Certs for webhook testing locally
+CERT_PATH=/tmp/k8s-webhook-server/serving-certs
+
+.PHONY: kustomize
+kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
+$(KUSTOMIZE): $(LOCALBIN)
+	test -s $(LOCALBIN)/kustomize || GOBIN=$(LOCALBIN) go install sigs.k8s.io/kustomize/kustomize/$(KUSTOMIZE_VERSION)@latest
+
+.PHONY: controller-gen
+controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
+$(CONTROLLER_GEN): $(LOCALBIN)
+	test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
+
+.PHONY: envtest
+envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
+$(ENVTEST): $(LOCALBIN)
+	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION)
+
+.PHONY: crd-ref-docs
+crd-ref-docs: $(CRD_REF_DOCS) ## Download crd-ref-docs locally if necessary.
+$(CRD_REF_DOCS): $(LOCALBIN)
+	test -s $(LOCALBIN)/crd-ref-docs || GOBIN=$(LOCALBIN) go install github.com/elastic/crd-ref-docs@$(CRD_REF_DOCS_VERSION)
+
+define gen-crd-ref-docs
+@for API_DIR in $$(find '$(PROJECT_DIR)/apis/apps' -maxdepth 1 -mindepth 1 -type d); do \
+	$(CRD_REF_DOCS) \
+		--source-path=$${API_DIR} \
+		--config=$(PROJECT_DIR)/crd-reference-config.yaml \
+		--output-path=$(PROJECT_DIR)/docs/en_US/reference/$$(basename $${API_DIR})-reference.md \
+		--renderer=markdown \
+		--log-level=error; \
+done
+@mkdir -p $(PROJECT_DIR)/docs/zh_CN/reference && cp -r $(PROJECT_DIR)/docs/en_US/reference/* $(PROJECT_DIR)/docs/zh_CN/reference
 endef
+
+.PHONY: local-webhook
+local-webhook: $(CERT_PATH)
+	test -s $(CERT_PATH)/tls.crt && test -s $(CERT_PATH)/tls.key || mkdir -p $(CERT_PATH) && cp config/dev/cert/* $(CERT_PATH)
+
+.PHONY: telepresence
+telepresence: $(TELEPRESENCE_GEN)
+$(TELEPRESENCE_GEN): $(LOCALBIN)
+	test -s $(LOCALBIN)/telepresence || curl -fL https://app.getambassador.io/download/tel2/$(shell go env GOOS)/$(shell go env GOARCH)/latest/telepresence -o $(TELEPRESENCE_GEN)
+	chmod +x $(TELEPRESENCE_GEN)
+	$(TELEPRESENCE_GEN) helm install

PROJECT

@@ -1,3 +1,7 @@
+# Code generated by tool. DO NOT EDIT.
+# This file is used to track the info used to scaffold your project
+# and allow the plugins properly work.
+# More info: https://book.kubebuilder.io/reference/project-config.html
 domain: emqx.io
 layout:
 - go.kubebuilder.io/v3
@@ -44,4 +48,97 @@ resources:
     defaulting: true
     validation: true
     webhookVersion: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: emqx.io
+  group: apps
+  kind: EMQX
+  path: github.com/emqx/emqx-operator/apis/apps/v2alpha1
+  version: v2alpha1
+  webhooks:
+    defaulting: true
+    validation: true
+    webhookVersion: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: emqx.io
+  group: apps
+  kind: EmqxBroker
+  path: github.com/emqx/emqx-operator/apis/apps/v1beta4
+  version: v1beta4
+  webhooks:
+    conversion: true
+    defaulting: true
+    validation: true
+    webhookVersion: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: emqx.io
+  group: apps
+  kind: EmqxEnterprise
+  path: github.com/emqx/emqx-operator/apis/apps/v1beta4
+  version: v1beta4
+  webhooks:
+    conversion: true
+    defaulting: true
+    validation: true
+    webhookVersion: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: emqx.io
+  group: apps
+  kind: EmqxPlugin
+  path: github.com/emqx/emqx-operator/apis/apps/v1beta4
+  version: v1beta4
+  webhooks:
+    conversion: true
+    defaulting: true
+    validation: true
+    webhookVersion: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: emqx.io
+  group: apps
+  kind: Rebalance
+  path: github.com/emqx/emqx-operator/apis/apps/v1beta4
+  version: v1beta4
+  webhooks:
+    validation: true
+    webhookVersion: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: emqx.io
+  group: apps
+  kind: EMQX
+  path: github.com/emqx/emqx-operator/apis/apps/v2beta1
+  version: v2beta1
+  webhooks:
+    conversion: true
+    webhookVersion: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: emqx.io
+  group: apps
+  kind: Rebalance
+  path: github.com/emqx/emqx-operator/apis/apps/v2beta1
+  version: v2beta1
+  webhooks:
+    conversion: true
+    defaulting: true
+    validation: true
+    webhookVersion: v1
 version: "3"

README.md

@@ -1,56 +1,90 @@
-# EMQX Operator
+<h1 align="center" style="border-bottom: none">
+    EMQX Operator
+</h1>
+
+<p align="center">Visit <a href="//www.emqx.com/en/emqx-kubernetes-operator" target="_blank">emqx.io</a> for the full documentation, examples and guides.</p>
+
+<div align="center">
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx-operator?color=brightgreen)](https://github.com/emqx/emqx-operator/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx-operator-controller)](https://hub.docker.com/r/emqx/emqx-operator-controller)
-[![Coverage Status](https://coveralls.io/repos/github/emqx/emqx-operator/badge.svg?branch=main)](https://coveralls.io/github/emqx/emqx-operator?branch=main)
+[![codecov](https://codecov.io/gh/emqx/emqx-operator/branch/main/graph/badge.svg?token=RNMH7K52JZ)](https://codecov.io/gh/emqx/emqx-operator)
 
-A Kubernetes Operator for [EMQX](https://www.emqx.io)
+</div>
 
 ## Overview
 
-The EMQX Operator provides [Kubernetes](https://kubernetes.io/) native deployment and management of [EMQX](https://www.emqx.io/) including EMQX Broker and EMQX Enterprise. The purpose of this project is to simplify and automate the configuration of the EMQX cluster.
+The EMQX Operator provides [Kubernetes](https://kubernetes.io/) native deployment and management of [EMQX](https://www.emqx.io/), including EMQX Broker and EMQX Enterprise. The purpose of this project is to simplify and automate the configuration of the EMQX cluster.
 
 The EMQX Operator includes, but is not limited to, the following features:
 
-* **Kubernetes Custom Resource**: Deploy and manage EMQX Cluster with pre-defined custom resources.
+* **Simplified Deployment EMQX**: Declare EMQX clusters with EMQX custom resources and deploy them quickly. For more details, please check [Getting Started](./docs/en_US/getting-started/getting-started.md).
+
+* **Manage EMQX Cluster**: Automate operations and maintenance for EMQX, including cluster upgrades, runtime data persistence, updating Kubernetes resources based on the status of EMQX, etc. For more details, please check [Manage EMQX](./docs/en_US/tasks/overview.md).
+
+![](.docs/../docs/en_US/introduction/assets/architecture.png)
+
+## EMQX and EMQX Operator compatibility
+
+### EMQX Enterprise
 
-* **Simplified Deployment Configuration**: Configure the fundamentals of EMQX Cluster, including persistence, configuration, license and etc, in a Kubernetes-native way.
+|  EMQX Enterprise Version   |              EMQX Operator Version              |                          APIVersion                          |      Kind      |
+| :------------------------: | :---------------------------------------------: | :----------------------------------------------------------: | :------------: |
+| 5.6.0 (included) ~ latest | 2.2.24 ~ latest | [apps.emqx.io/v2beta1](./docs/en_US/reference/v2beta1-reference.md) | EMQX |
+| 5.1.1 (included) ~ latest      |    2.2.0 ~ latest                      | [apps.emqx.io/v2beta1](./docs/en_US/reference/v2beta1-reference.md) |      EMQX      |
+| 5.0.0 (included) ~ 5.0.23        |    2.0.0, 2.0.1, 2.0.2, 2.0.3, 2.1.0, 2.1.1     | [apps.emqx.io/v2alpha1](./docs/en_US/reference/v2alpha1-reference.md) |      EMQX      |
+| 4.4.14 (included) or higher 4.4.x   |                  2.1.0, 2.1.1                   | [apps.emqx.io/v1beta4](./docs/en_US/reference/v1beta4-reference.md) | EmqxEnterprise |
+| 4.4.8 (included) ～ 4.4.14 | 1.2.6, 1.2.7, 1.2.8, 2.0.0, 2.0.1, 2.0.2, 2.0.3 | [apps.emqx.io/v1beta3](./docs/en_US/reference/v1beta3-reference.md) | EmqxEnterprise |
+| 4.4.6 (included) ～ 4.4.8  |                      1.2.5                      | [apps.emqx.io/v1beta3](./docs/en_US/reference/v1beta3-reference.md) | EmqxEnterprise |
+|  4.3.x (included) ～ 4.4   |               1.2.1, 1.2.2, 1.2.3               | [apps.emqx.io/v1beta3](./docs/en_US/reference/v1beta3-reference.md) | EmqxEnterprise |
 
-For an introduction to the EMQX Operator, see the [introduction](docs/en_US/README.md).
+### EMQX Open Source
 
-## Prerequisites
+|      EMQX Open Source Version      |     EMQX Operator Version                            |     APIVersion    |    Kind    |
+|:----------------------:|:-----------------:|-------------------|:-----------------:|
+| 5.6.0 (included) ~ latest | 2.2.24 ~ latest | [apps.emqx.io/v2beta1](./docs/en_US/reference/v2beta1-reference.md) | EMQX |
+| 5.1.1 (included) ~ latest  |                 2.2.0 ~ latest                  | [apps.emqx.io/v2beta1](./docs/en_US/reference/v2beta1-reference.md) |    EMQX    |
+| 5.0.14 (included)  ~ 5.0.23 | 2.1.0, 2.1.1                                                | [apps.emqx.io/v2alpha1](./docs/en_US/reference/v2alpha1-reference.md)         | EMQX     |
+| 5.0.8 (included) ～  5.0.14 | 2.0.2                                            |  [apps.emqx.io/v2alpha1](./docs/en_US/reference/v2alpha1-reference.md)         |  EMQX     |
+| 5.0.6 (included) ～ 5.0.8 | 2.0.0, 2.0.1, 2.0.3                                |  [apps.emqx.io/v2alpha1](./docs/en_US/reference/v2alpha1-reference.md)         |  EMQX     |
+| 4.4.14 or higher 4.4.x | 2.1.0, 2.1.1                                                 |  [apps.emqx.io/v1beta4](./docs/en_US/reference/v1beta4-reference.md)          | EmqxBroker |
+| 4.4.6 (included) ～ 4.4.8 | 1.2.5                                                 | [apps.emqx.io/v1beta3](./docs/en_US/reference/v1beta3-reference.md)          | EmqxBroker |
+| 4.4.8 (included) ～ 4.4.14 | 1.2.6, 1.2.7, 1.2.8, 2.0.0, 2.0.1, 2.0.2, 2.0.3   |  [apps.emqx.io/v1beta3](./docs/en_US/reference/v1beta3-reference.md)          | EmqxBroker |
+| 4.3.x (included) ～ 4.4 | 1.2.1, 1.2.2, 1.2.3                                 |  [apps.emqx.io/v1beta3](./docs/en_US/reference/v1beta3-reference.md)          |  EmqxBroker  |
 
-The EMQX Operator requires a Kubernetes cluster of version `>=1.20.0`.
+## How to selector Kubernetes version
+
+The EMQX Operator requires a Kubernetes cluster of version `>=1.24`.
+
+| Kubernetes Versions     | EMQX Operator Compatibility                                  | Notes                                                        |
+| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1.24 (included) ~ latest | All functions supported                                      |  |
+| 1.22 (included) ～ 1.23 | Supported, except [MixedProtocolLBService](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/) | EMQX cluster can only use one protocol in `LoadBalancer` type of Service, for example TCP or UDP. |
+| Lower than 1.22       | Not supported                                                | `unknown field "x-kubernetes-validations" in io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.JSONSchemaProps]` |
 
 ## CustomResourceDefinitions
 
 A core feature of the EMQX Operator is to monitor the Kubernetes API server for changes to specific objects and ensure that the running EMQX deployments match these objects.
 The Operator acts on the following [custom resource definitions (CRDs)](https://kubernetes.io/docs/tasks/access-kubernetes-api/extend-api-custom-resource-definitions/).
 
-For the example of EMQX Broker see the [`emqx.yaml`](config/samples/emqx/v1beta3/emqx.yaml) and for the example of EMQX Enterprise see the [emqx-ee.yaml](config/samples/emqx/v1beta3/emqx-ee.yaml).
+For more details on EMQX, please check the [reference document](https://github.com/emqx/emqx-operator/blob/main/docs/en_US/reference/v2alpha1-reference.md).
 
 The EMQX Operator automatically detects changes on any of the above custom resource objects and ensures that running deployments are kept in sync with the changes.
 
-## EMQX Operator compatibility 
-
-|                          | EMQX 4.2 latest | EMQX 4.3 latest | EMQX 4.4 latest |
-|--------------------------|-----------------|-----------------|-----------------|
-| EMQX Operator 1.1 latest | ✓               | ✓               | ✓               |
-| EMQX Operator 1.2 latest |                 |                 | ✓               |
-
 ## Getting Start
 
-For more information on getting started, see the [getting started](docs/en_US/getting-started/getting-started.md)
+For more information on getting started, see the [getting started](docs/en_US/getting-started/getting-started.md).
 
 ## Public Cloud Platform Deployment Guide
 
-|  Public Cloud Platform   | Deployment Guide                                         | 
+|  Public Cloud Platform   | Deployment Guide                                         |
 |--------------------------|----------------------------------------------------------|
-|    AWS                   | [EKS](docs/en_US/deployment/aws-eks-deployment.md)       | 
-|    Azure                 | [Azure](docs/en_US/deployment/azure-deployment.md)       | 
-|    Aliyun                | [ACK](docs/zh_CN/deployment/aliyun-ack-deployment.md)    | 
-|    Huawei                | [CCE](docs/zh_CN/deployment/cce-deployment.md)           | 
-|    Tencent               | [TKE](docs/zh_CN/deployment/tencent-tke-deployment.md)   | 
+|    AWS                   | [Deploy EMQX on Amazon Elastic Kubernetes Service](docs/en_US/deployment/on-aws-eks.md)       |
+|    Azure                 | [Deploy EMQX on Azure Kubernetes Service](docs/en_US/deployment/on-azure-aks.md)       |
+|    Google Cloud          | [Deploy EMQX on Google Cloud GKE](docs/en_US/deployment/on-gcp-gke.md)       |
+|    Alibaba Cloud         | [Deploy EMQX on Alibaba Cloud ACK](docs/zh_CN/deployment/on-alibaba-cloud.md)    |
+|    Huawei Cloud          | [Deploy EMQX on Huawei Cloud CCE](docs/zh_CN/deployment/on-huawei-cloud.md)           |
+|    Tencent Cloud         | [Deploy EMQX on Tencent Cloud TKE](docs/zh_CN/deployment/on-tencent-cloud.md)   |
 
 
 ## Development
@@ -60,13 +94,24 @@ For more information on getting started, see the [getting started](docs/en_US/ge
 - Golang environment
 - docker (used for creating container images, etc.)
 - Kubernetes cluster
+- teleperence
+
+### Install Teleperence for once
+```shell
+make telepresence
+```
+
+### Connect to cluster
+```shell
+./bin/telepresence connect
+```
 
 ## Contributing
 Many files (API, config, controller, hack,...) in this repository are auto-generated.
 Before proposing a pull request:
 
 1. Commit your changes.
-2. Run `make` and `make manifests`
+2. `make` and `make manifests`
 3. Commit the generated changes.
 
 ## Troubleshooting

RELEASE.md

@@ -1,21 +1,35 @@
-## Release Note 🍻
+# Release Note 🍻
 
-EMQX Operator 1.2.6 is released.
+EMQX Operator 2.2.29-beta.1 has been released.
 
-### Supported EMQX version
+## Supported version
++ apps.emqx.io/v2beta1
 
-- EMQX 4.4.8 and later
+  + EMQX at 5.1.1 and later
+  + EMQX Enterprise at 5.1.1 and later
 
-- EMQX Enterprise 4.4.8 and later
++ apps.emqx.io/v1beta4
 
-### Features 🌈
+  + EMQX at 4.4.14 and later
+  + EMQX Enterprise at 4.4.14 and later
 
-- Add `.spec.emqxTemplate.license.secretName` for EMQX Enterprise Custom Resource, the user can create the EMQX license as a Kubernetes secret resource and use in MQX Enterprise Custom Resource
+## Fix 🐞
 
-- After the user updates the license, the EMQX Operator completes the runtime update via the EMQX API
++ Fix the issue that the replicas of the statefulSet is not current when the `emqx` CR is updated
 
-### Fixes 🛠
+## How to install/upgrade EMQX Operator 💡
 
-- Now it's not possible to update `.spec.persistent` in the EMQX Custom Resource runtime
+> Need make sure the [cert-manager](https://cert-manager.io/) is ready
 
-- Now it does not create `loaded_plugins` configMap for EMQX Custom Resource, this is to fix the `erofs` error in EMQX
+```
+helm repo add emqx https://repos.emqx.io/charts
+helm repo update
+helm upgrade --install emqx-operator emqx/emqx-operator \
+  --namespace emqx-operator-system \
+  --create-namespace \
+  --version 2.2.29-beta.1
+kubectl wait --for=condition=Ready pods -l "control-plane=controller-manager" -n emqx-operator-system
+```
+
+## Warning 🚨
+`apps.emqx.io/v1beta3` and `apps.emqx.io/v2alpha1` will be dropped soon

apis/apps/v1beta3/emqx_interface.go

@@ -19,11 +19,17 @@ package v1beta3
 import (
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-//+kubebuilder:object:generate=false
+// +kubebuilder:object:generate=false
 type EmqxSpec interface {
+	Default()
+	ValidateCreate() error
+	ValidateUpdate(runtime.Object) error
+	ValidateDelete() error
+
 	GetReplicas() *int32
 	SetReplicas(replicas *int32)
 
@@ -101,6 +107,9 @@ type EmqxSpec interface {
 
 	GetStatus() Status
 	SetStatus(status Status)
+
+	GetRegistry() string
+	SetRegistry(registry string)
 }
 
 // +kubebuilder:object:generate=false

apis/apps/v1beta3/emqxbroker_conversion.go

@@ -0,0 +1,208 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta3
+
+import (
+	"reflect"
+	"regexp"
+	"strings"
+
+	"github.com/emqx/emqx-operator/apis/apps/v1beta4"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/conversion"
+)
+
+// ConvertTo converts this version to the Hub version (v1).
+func (src *EmqxBroker) ConvertTo(dstRaw conversion.Hub) error {
+	dst := dstRaw.(*v1beta4.EmqxBroker)
+	dst.ObjectMeta = src.ObjectMeta
+	// Replicas
+	dst.Spec.Replicas = src.Spec.Replicas
+	// ClusterDomain
+	dst.Spec.ClusterDomain = "cluster.local"
+	// ServiceTemplate
+	if !reflect.ValueOf(src.Spec.EmqxTemplate.ServiceTemplate).IsZero() {
+		dst.Spec.ServiceTemplate = v1beta4.ServiceTemplate(src.Spec.EmqxTemplate.ServiceTemplate)
+	}
+	// Persistent
+	if !reflect.ValueOf(src.Spec.Persistent).IsZero() {
+		names := Names{Object: src}
+		dst.Spec.Persistent = &corev1.PersistentVolumeClaimTemplate{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: names.Data(),
+			},
+			Spec: src.Spec.Persistent,
+		}
+	}
+
+	// Template
+	dst.Spec.Template.ObjectMeta.Labels = src.Labels
+	dst.Spec.Template.ObjectMeta.Annotations = src.Annotations
+
+	dst.Spec.Template.Spec.EmqxContainer.Image.Version = "latest"
+	compile := regexp.MustCompile(`(latest|[0-9]+(\.[0-9]+)?(\.[0-9]+)?(-(alpha|beta|rc)\.[0-9]+)?)`)
+	image := strings.Split(src.Spec.EmqxTemplate.Image, ":")
+	dst.Spec.Template.Spec.EmqxContainer.Image.Repository = image[0]
+	if compile.MatchString(image[1]) {
+		dst.Spec.Template.Spec.EmqxContainer.Image.Version = compile.FindString(image[1])
+		index := compile.FindStringIndex(image[1])
+		if index != nil {
+			dst.Spec.Template.Spec.EmqxContainer.Image.Prefix = image[1][:index[0]]
+			dst.Spec.Template.Spec.EmqxContainer.Image.Suffix = image[1][index[1]:]
+		}
+	}
+	if len(src.Spec.EmqxTemplate.ImagePullPolicy) != 0 {
+		dst.Spec.Template.Spec.EmqxContainer.Image.PullPolicy = src.Spec.EmqxTemplate.ImagePullPolicy
+	}
+	if src.Spec.EmqxTemplate.EmqxConfig != nil {
+		dst.Spec.Template.Spec.EmqxContainer.EmqxConfig = src.Spec.EmqxTemplate.EmqxConfig
+	}
+	if len(src.Spec.EmqxTemplate.ACL) != 0 {
+		dst.Spec.Template.Spec.EmqxContainer.EmqxACL = src.Spec.EmqxTemplate.ACL
+	}
+	if len(src.Spec.EmqxTemplate.Args) != 0 {
+		dst.Spec.Template.Spec.EmqxContainer.Args = src.Spec.EmqxTemplate.Args
+	}
+	if !reflect.ValueOf(src.Spec.EmqxTemplate.Resources).IsZero() {
+		dst.Spec.Template.Spec.EmqxContainer.Resources = src.Spec.EmqxTemplate.Resources
+	}
+	if src.Spec.EmqxTemplate.ReadinessProbe != nil {
+		dst.Spec.Template.Spec.EmqxContainer.ReadinessProbe = src.Spec.EmqxTemplate.ReadinessProbe
+	}
+	if src.Spec.EmqxTemplate.LivenessProbe != nil {
+		dst.Spec.Template.Spec.EmqxContainer.LivenessProbe = src.Spec.EmqxTemplate.LivenessProbe
+	}
+	if src.Spec.EmqxTemplate.StartupProbe != nil {
+		dst.Spec.Template.Spec.EmqxContainer.StartupProbe = src.Spec.EmqxTemplate.StartupProbe
+	}
+	if len(src.Spec.EmqxTemplate.ExtraVolumeMounts) != 0 {
+		dst.Spec.Template.Spec.EmqxContainer.VolumeMounts = src.Spec.EmqxTemplate.ExtraVolumeMounts
+	}
+	if len(src.Spec.EmqxTemplate.ExtraVolumes) != 0 {
+		dst.Spec.Template.Spec.Volumes = src.Spec.EmqxTemplate.ExtraVolumes
+	}
+	if src.Spec.EmqxTemplate.SecurityContext != nil {
+		dst.Spec.Template.Spec.PodSecurityContext = src.Spec.EmqxTemplate.SecurityContext
+	}
+	if len(src.Spec.InitContainers) != 0 {
+		dst.Spec.Template.Spec.InitContainers = src.Spec.InitContainers
+	}
+	if len(src.Spec.ExtraContainers) != 0 {
+		dst.Spec.Template.Spec.ExtraContainers = src.Spec.ExtraContainers
+	}
+	if len(src.Spec.ImagePullSecrets) != 0 {
+		dst.Spec.Template.Spec.ImagePullSecrets = src.Spec.ImagePullSecrets
+	}
+	if len(src.Spec.Env) != 0 {
+		dst.Spec.Template.Spec.EmqxContainer.Env = append(dst.Spec.Template.Spec.EmqxContainer.Env, src.Spec.Env...)
+	}
+	if len(src.Spec.ToleRations) != 0 {
+		dst.Spec.Template.Spec.Tolerations = src.Spec.ToleRations
+	}
+	if len(src.Spec.NodeName) != 0 {
+		dst.Spec.Template.Spec.NodeName = src.Spec.NodeName
+	}
+	if src.Spec.NodeSelector != nil {
+		dst.Spec.Template.Spec.NodeSelector = src.Spec.NodeSelector
+	}
+	if src.Spec.Affinity != nil {
+		dst.Spec.Template.Spec.Affinity = src.Spec.Affinity
+	}
+
+	// +kubebuilder:docs-gen:collapse=rote conversion
+	return nil
+}
+
+// ConvertFrom converts from the Hub version (v1) to this version.
+func (dst *EmqxBroker) ConvertFrom(srcRaw conversion.Hub) error {
+	src := srcRaw.(*v1beta4.EmqxBroker)
+	dst.ObjectMeta = src.ObjectMeta
+	//Replicas
+	dst.Spec.Replicas = src.Spec.Replicas
+	// Persistent
+	if !reflect.ValueOf(src.Spec.Persistent).IsZero() {
+		dst.Spec.Persistent = src.Spec.Persistent.Spec
+	}
+	// ServiceTemplate
+	if !reflect.ValueOf(src.Spec.ServiceTemplate).IsZero() {
+		dst.Spec.EmqxTemplate.ServiceTemplate = ServiceTemplate(src.Spec.ServiceTemplate)
+	}
+	// Template
+	dst.Spec.EmqxTemplate.Image = v1beta4.GetEmqxImage(src)
+
+	if len(src.Spec.Template.Spec.EmqxContainer.Image.PullPolicy) != 0 {
+		dst.Spec.EmqxTemplate.ImagePullPolicy = src.Spec.Template.Spec.EmqxContainer.Image.PullPolicy
+	}
+	if src.Spec.Template.Spec.EmqxContainer.EmqxConfig != nil {
+		dst.Spec.EmqxTemplate.EmqxConfig = src.Spec.Template.Spec.EmqxContainer.EmqxConfig
+	}
+	if len(src.Spec.Template.Spec.EmqxContainer.EmqxACL) != 0 {
+		dst.Spec.EmqxTemplate.ACL = src.Spec.Template.Spec.EmqxContainer.EmqxACL
+	}
+	if len(src.Spec.Template.Spec.EmqxContainer.Args) != 0 {
+		dst.Spec.EmqxTemplate.Args = src.Spec.Template.Spec.EmqxContainer.Args
+	}
+	if !reflect.ValueOf(src.Spec.Template.Spec.EmqxContainer.Resources).IsZero() {
+		dst.Spec.EmqxTemplate.Resources = src.Spec.Template.Spec.EmqxContainer.Resources
+	}
+	if src.Spec.Template.Spec.EmqxContainer.ReadinessProbe != nil {
+		dst.Spec.EmqxTemplate.ReadinessProbe = src.Spec.Template.Spec.EmqxContainer.ReadinessProbe
+	}
+	if src.Spec.Template.Spec.EmqxContainer.LivenessProbe != nil {
+		dst.Spec.EmqxTemplate.LivenessProbe = src.Spec.Template.Spec.EmqxContainer.LivenessProbe
+	}
+	if src.Spec.Template.Spec.EmqxContainer.StartupProbe != nil {
+		dst.Spec.EmqxTemplate.StartupProbe = src.Spec.Template.Spec.EmqxContainer.StartupProbe
+	}
+	if len(src.Spec.Template.Spec.EmqxContainer.VolumeMounts) != 0 {
+		dst.Spec.EmqxTemplate.ExtraVolumeMounts = src.Spec.Template.Spec.EmqxContainer.VolumeMounts
+	}
+	if len(src.Spec.Template.Spec.Volumes) != 0 {
+		dst.Spec.EmqxTemplate.ExtraVolumes = src.Spec.Template.Spec.Volumes
+	}
+	if src.Spec.Template.Spec.PodSecurityContext != nil {
+		dst.Spec.EmqxTemplate.SecurityContext = src.Spec.Template.Spec.PodSecurityContext
+	}
+	if len(src.Spec.Template.Spec.InitContainers) != 0 {
+		dst.Spec.InitContainers = src.Spec.Template.Spec.InitContainers
+	}
+	if len(src.Spec.Template.Spec.ExtraContainers) != 0 {
+		dst.Spec.ExtraContainers = src.Spec.Template.Spec.ExtraContainers
+	}
+	if len(src.Spec.Template.Spec.ImagePullSecrets) != 0 {
+		dst.Spec.ImagePullSecrets = src.Spec.Template.Spec.ImagePullSecrets
+	}
+	if len(src.Spec.Template.Spec.EmqxContainer.Env) != 0 {
+		dst.Spec.Env = src.Spec.Template.Spec.EmqxContainer.Env
+	}
+	if len(src.Spec.Template.Spec.Tolerations) != 0 {
+		dst.Spec.ToleRations = src.Spec.Template.Spec.Tolerations
+	}
+	if len(src.Spec.Template.Spec.NodeName) != 0 {
+		dst.Spec.NodeName = src.Spec.Template.Spec.NodeName
+	}
+	if src.Spec.Template.Spec.NodeSelector != nil {
+		dst.Spec.NodeSelector = src.Spec.Template.Spec.NodeSelector
+	}
+	if src.Spec.Template.Spec.Affinity != nil {
+		dst.Spec.Affinity = src.Spec.Template.Spec.Affinity
+	}
+
+	// +kubebuilder:docs-gen:collapse=rote conversion
+	return nil
+}

apis/apps/v1beta3/emqxbroker_types.go

@@ -22,6 +22,10 @@ import (
 )
 
 type EmqxBrokerTemplate struct {
+	// Registry will used for EMQX owner image,
+	// like ${registry}/emqx/emqx and ${registry}/emqx/emqx-operator-reloader,
+	// but it will not be used by other images, like sidecar container or else.
+	Registry string `json:"registry,omitempty"`
 	//+kubebuilder:validation:Required
 	Image string `json:"image,omitempty"`
 	// Image pull policy.
@@ -129,9 +133,8 @@ type EmqxBrokerSpec struct {
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
-//+kubebuilder:resource:shortName=emqx
 //+kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas
-//+kubebuilder:storageversion
+//+kubebuilder:deprecatedversion
 
 // EmqxBroker is the Schema for the emqxbrokers API
 type EmqxBroker struct {
@@ -312,3 +315,6 @@ func (emqx *EmqxBroker) SetPassword(password string) {
 
 func (emqx *EmqxBroker) GetStatus() Status       { return emqx.Status }
 func (emqx *EmqxBroker) SetStatus(status Status) { emqx.Status = status }
+
+func (emqx *EmqxBroker) GetRegistry() string         { return emqx.Spec.EmqxTemplate.Registry }
+func (emqx *EmqxBroker) SetRegistry(registry string) { emqx.Spec.EmqxTemplate.Registry = registry }

apis/apps/v1beta3/emqxenterprise_conversion.go

@@ -0,0 +1,213 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta3
+
+import (
+	"reflect"
+	"regexp"
+	"strings"
+
+	"github.com/emqx/emqx-operator/apis/apps/v1beta4"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/conversion"
+)
+
+// ConvertTo converts this version to the Hub version (v1).
+func (src *EmqxEnterprise) ConvertTo(dstRaw conversion.Hub) error {
+	dst := dstRaw.(*v1beta4.EmqxEnterprise)
+	dst.ObjectMeta = src.ObjectMeta
+	// Replicas
+	dst.Spec.Replicas = src.Spec.Replicas
+	// ClusterDomain
+	dst.Spec.ClusterDomain = "cluster.local"
+	// ServiceTemplate
+	if !reflect.ValueOf(src.Spec.EmqxTemplate.ServiceTemplate).IsZero() {
+		dst.Spec.ServiceTemplate = v1beta4.ServiceTemplate(src.Spec.EmqxTemplate.ServiceTemplate)
+	}
+	// Persistent
+	if !reflect.ValueOf(src.Spec.Persistent).IsZero() {
+		names := Names{Object: src}
+		dst.Spec.Persistent = &corev1.PersistentVolumeClaimTemplate{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: names.Data(),
+			},
+			Spec: src.Spec.Persistent,
+		}
+	}
+	// Template
+	dst.Spec.Template.ObjectMeta.Labels = src.Labels
+	dst.Spec.Template.ObjectMeta.Annotations = src.Annotations
+
+	compile := regexp.MustCompile(`(latest|[0-9]+(\.[0-9]+)?(\.[0-9]+)?(-(alpha|beta|rc)\.[0-9]+)?)`)
+	image := strings.Split(src.Spec.EmqxTemplate.Image, ":")
+	dst.Spec.Template.Spec.EmqxContainer.Image.Repository = image[0]
+	if compile.MatchString(image[1]) {
+		dst.Spec.Template.Spec.EmqxContainer.Image.Version = compile.FindString(image[1])
+		index := compile.FindStringIndex(image[1])
+		if index != nil {
+			dst.Spec.Template.Spec.EmqxContainer.Image.Prefix = image[1][:index[0]]
+			dst.Spec.Template.Spec.EmqxContainer.Image.Suffix = image[1][index[1]:]
+		}
+	}
+	if len(src.Spec.EmqxTemplate.ImagePullPolicy) != 0 {
+		dst.Spec.Template.Spec.EmqxContainer.Image.PullPolicy = src.Spec.EmqxTemplate.ImagePullPolicy
+	}
+	if !reflect.ValueOf(src.Spec.EmqxTemplate.License).IsZero() {
+		dst.Spec.License = v1beta4.EmqxLicense(src.Spec.EmqxTemplate.License)
+	}
+	if src.Spec.EmqxTemplate.EmqxConfig != nil {
+		dst.Spec.Template.Spec.EmqxContainer.EmqxConfig = src.Spec.EmqxTemplate.EmqxConfig
+	}
+	if len(src.Spec.EmqxTemplate.ACL) != 0 {
+		dst.Spec.Template.Spec.EmqxContainer.EmqxACL = src.Spec.EmqxTemplate.ACL
+	}
+	if len(src.Spec.EmqxTemplate.Args) != 0 {
+		dst.Spec.Template.Spec.EmqxContainer.Args = src.Spec.EmqxTemplate.Args
+	}
+	if !reflect.ValueOf(src.Spec.EmqxTemplate.Resources).IsZero() {
+		dst.Spec.Template.Spec.EmqxContainer.Resources = src.Spec.EmqxTemplate.Resources
+	}
+	if src.Spec.EmqxTemplate.ReadinessProbe != nil {
+		dst.Spec.Template.Spec.EmqxContainer.ReadinessProbe = src.Spec.EmqxTemplate.ReadinessProbe
+	}
+	if src.Spec.EmqxTemplate.LivenessProbe != nil {
+		dst.Spec.Template.Spec.EmqxContainer.LivenessProbe = src.Spec.EmqxTemplate.LivenessProbe
+	}
+	if src.Spec.EmqxTemplate.StartupProbe != nil {
+		dst.Spec.Template.Spec.EmqxContainer.StartupProbe = src.Spec.EmqxTemplate.StartupProbe
+	}
+	if len(src.Spec.EmqxTemplate.ExtraVolumeMounts) != 0 {
+		dst.Spec.Template.Spec.EmqxContainer.VolumeMounts = src.Spec.EmqxTemplate.ExtraVolumeMounts
+	}
+	if len(src.Spec.EmqxTemplate.ExtraVolumes) != 0 {
+		dst.Spec.Template.Spec.Volumes = src.Spec.EmqxTemplate.ExtraVolumes
+	}
+	if src.Spec.EmqxTemplate.SecurityContext != nil {
+		dst.Spec.Template.Spec.PodSecurityContext = src.Spec.EmqxTemplate.SecurityContext
+	}
+	if len(src.Spec.InitContainers) != 0 {
+		dst.Spec.Template.Spec.InitContainers = src.Spec.InitContainers
+	}
+	if len(src.Spec.ExtraContainers) != 0 {
+		dst.Spec.Template.Spec.ExtraContainers = src.Spec.ExtraContainers
+	}
+	if len(src.Spec.ImagePullSecrets) != 0 {
+		dst.Spec.Template.Spec.ImagePullSecrets = src.Spec.ImagePullSecrets
+	}
+	if len(src.Spec.Env) != 0 {
+		dst.Spec.Template.Spec.EmqxContainer.Env = append(dst.Spec.Template.Spec.EmqxContainer.Env, src.Spec.Env...)
+	}
+	if len(src.Spec.ToleRations) != 0 {
+		dst.Spec.Template.Spec.Tolerations = src.Spec.ToleRations
+	}
+	if len(src.Spec.NodeName) != 0 {
+		dst.Spec.Template.Spec.NodeName = src.Spec.NodeName
+	}
+	if src.Spec.NodeSelector != nil {
+		dst.Spec.Template.Spec.NodeSelector = src.Spec.NodeSelector
+	}
+	if src.Spec.Affinity != nil {
+		dst.Spec.Template.Spec.Affinity = src.Spec.Affinity
+	}
+
+	// +kubebuilder:docs-gen:collapse=rote conversion
+	return nil
+}
+
+// ConvertFrom converts from the Hub version (v1) to this version.
+func (dst *EmqxEnterprise) ConvertFrom(srcRaw conversion.Hub) error {
+	src := srcRaw.(*v1beta4.EmqxEnterprise)
+	dst.ObjectMeta = src.ObjectMeta
+	//Replicas
+	dst.Spec.Replicas = src.Spec.Replicas
+	// Persistent
+	if !reflect.ValueOf(src.Spec.Persistent).IsZero() {
+		dst.Spec.Persistent = src.Spec.Persistent.Spec
+	}
+	// ServiceTemplate
+	if !reflect.ValueOf(src.Spec.ServiceTemplate).IsZero() {
+		dst.Spec.EmqxTemplate.ServiceTemplate = ServiceTemplate(src.Spec.ServiceTemplate)
+	}
+	// Template
+	dst.Spec.EmqxTemplate.Image = v1beta4.GetEmqxImage(src)
+
+	if len(src.Spec.Template.Spec.EmqxContainer.Image.PullPolicy) != 0 {
+		dst.Spec.EmqxTemplate.ImagePullPolicy = src.Spec.Template.Spec.EmqxContainer.Image.PullPolicy
+	}
+
+	if !reflect.ValueOf(dst.Spec.EmqxTemplate.License).IsZero() {
+		dst.Spec.EmqxTemplate.License = License(src.Spec.License)
+	}
+	if src.Spec.Template.Spec.EmqxContainer.EmqxConfig != nil {
+		dst.Spec.EmqxTemplate.EmqxConfig = src.Spec.Template.Spec.EmqxContainer.EmqxConfig
+	}
+	if len(src.Spec.Template.Spec.EmqxContainer.EmqxACL) != 0 {
+		dst.Spec.EmqxTemplate.ACL = src.Spec.Template.Spec.EmqxContainer.EmqxACL
+	}
+	if len(src.Spec.Template.Spec.EmqxContainer.Args) != 0 {
+		dst.Spec.EmqxTemplate.Args = src.Spec.Template.Spec.EmqxContainer.Args
+	}
+	if !reflect.ValueOf(src.Spec.Template.Spec.EmqxContainer.Resources).IsZero() {
+		dst.Spec.EmqxTemplate.Resources = src.Spec.Template.Spec.EmqxContainer.Resources
+	}
+	if src.Spec.Template.Spec.EmqxContainer.ReadinessProbe != nil {
+		dst.Spec.EmqxTemplate.ReadinessProbe = src.Spec.Template.Spec.EmqxContainer.ReadinessProbe
+	}
+	if src.Spec.Template.Spec.EmqxContainer.LivenessProbe != nil {
+		dst.Spec.EmqxTemplate.LivenessProbe = src.Spec.Template.Spec.EmqxContainer.LivenessProbe
+	}
+	if src.Spec.Template.Spec.EmqxContainer.StartupProbe != nil {
+		dst.Spec.EmqxTemplate.StartupProbe = src.Spec.Template.Spec.EmqxContainer.StartupProbe
+	}
+	if len(src.Spec.Template.Spec.EmqxContainer.VolumeMounts) != 0 {
+		dst.Spec.EmqxTemplate.ExtraVolumeMounts = src.Spec.Template.Spec.EmqxContainer.VolumeMounts
+	}
+	if len(src.Spec.Template.Spec.Volumes) != 0 {
+		dst.Spec.EmqxTemplate.ExtraVolumes = src.Spec.Template.Spec.Volumes
+	}
+	if src.Spec.Template.Spec.PodSecurityContext != nil {
+		dst.Spec.EmqxTemplate.SecurityContext = src.Spec.Template.Spec.PodSecurityContext
+	}
+	if len(src.Spec.Template.Spec.InitContainers) != 0 {
+		dst.Spec.InitContainers = src.Spec.Template.Spec.InitContainers
+	}
+	if len(src.Spec.Template.Spec.ExtraContainers) != 0 {
+		dst.Spec.ExtraContainers = src.Spec.Template.Spec.ExtraContainers
+	}
+	if len(src.Spec.Template.Spec.ImagePullSecrets) != 0 {
+		dst.Spec.ImagePullSecrets = src.Spec.Template.Spec.ImagePullSecrets
+	}
+	if len(src.Spec.Template.Spec.EmqxContainer.Env) != 0 {
+		dst.Spec.Env = src.Spec.Template.Spec.EmqxContainer.Env
+	}
+	if len(src.Spec.Template.Spec.Tolerations) != 0 {
+		dst.Spec.ToleRations = src.Spec.Template.Spec.Tolerations
+	}
+	if len(src.Spec.Template.Spec.NodeName) != 0 {
+		dst.Spec.NodeName = src.Spec.Template.Spec.NodeName
+	}
+	if src.Spec.Template.Spec.NodeSelector != nil {
+		dst.Spec.NodeSelector = src.Spec.Template.Spec.NodeSelector
+	}
+	if src.Spec.Template.Spec.Affinity != nil {
+		dst.Spec.Affinity = src.Spec.Template.Spec.Affinity
+	}
+
+	// +kubebuilder:docs-gen:collapse=rote conversion
+	return nil
+}

apis/apps/v1beta3/emqxenterprise_types.go

@@ -39,8 +39,13 @@ type License struct {
 }
 
 type EmqxEnterpriseTemplate struct {
+	// Registry will used for EMQX owner image,
+	// like ${registry}/emqx/emqx-ee and ${registry}/emqx/emqx-operator-reloader,
+	// but it will not be used by other images, like sidecar container or else.
+	Registry string `json:"registry,omitempty"`
 	//+kubebuilder:validation:Required
 	Image string `json:"image,omitempty"`
+
 	// Image pull policy.
 	// One of Always, Never, IfNotPresent.
 	// Defaults to Always if :latest tag is specified, or IfNotPresent otherwise.
@@ -148,9 +153,8 @@ type EmqxEnterpriseSpec struct {
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
-//+kubebuilder:resource:shortName=emqx-ee
 //+kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas
-//+kubebuilder:storageversion
+//+kubebuilder:deprecatedversion
 
 // EmqxEnterprise is the Schema for the emqxEnterprises API
 type EmqxEnterprise struct {
@@ -338,3 +342,6 @@ func (emqx *EmqxEnterprise) SetPassword(password string) {
 
 func (emqx *EmqxEnterprise) GetStatus() Status       { return emqx.Status }
 func (emqx *EmqxEnterprise) SetStatus(status Status) { emqx.Status = status }
+
+func (emqx *EmqxEnterprise) GetRegistry() string         { return emqx.Spec.EmqxTemplate.Registry }
+func (emqx *EmqxEnterprise) SetRegistry(registry string) { emqx.Spec.EmqxTemplate.Registry = registry }

apis/apps/v1beta3/emqxplugin_conversion.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta3
+
+import (
+	"github.com/emqx/emqx-operator/apis/apps/v1beta4"
+	"sigs.k8s.io/controller-runtime/pkg/conversion"
+)
+
+// ConvertTo converts this version to the Hub version (v1).
+func (src *EmqxPlugin) ConvertTo(dstRaw conversion.Hub) error {
+	dst := dstRaw.(*v1beta4.EmqxPlugin)
+	dst.ObjectMeta = src.ObjectMeta
+	dst.Spec = v1beta4.EmqxPluginSpec{
+		PluginName: src.Spec.PluginName,
+		Selector:   src.Spec.Selector,
+		Config:     src.Spec.Config,
+	}
+
+	// +kubebuilder:docs-gen:collapse=rote conversion
+	return nil
+}
+
+// ConvertFrom converts from the Hub version (v1) to this version.
+func (dst *EmqxPlugin) ConvertFrom(srcRaw conversion.Hub) error {
+	src := srcRaw.(*v1beta4.EmqxPlugin)
+	src.ObjectMeta = dst.ObjectMeta
+	src.Spec = v1beta4.EmqxPluginSpec{
+		PluginName: dst.Spec.PluginName,
+		Selector:   dst.Spec.Selector,
+		Config:     dst.Spec.Config,
+	}
+
+	// +kubebuilder:docs-gen:collapse=rote conversion
+	return nil
+}

apis/apps/v1beta3/emqxplugin_conversion_test.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta3
+
+import (
+	"testing"
+
+	"github.com/emqx/emqx-operator/apis/apps/v1beta4"
+	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var v1beta3Plugin = &EmqxPlugin{
+	ObjectMeta: metav1.ObjectMeta{
+		Name:        "test",
+		Namespace:   "test",
+		Labels:      map[string]string{"test": "test"},
+		Annotations: map[string]string{"test": "test"},
+	},
+	Spec: EmqxPluginSpec{
+		PluginName: "test",
+		Selector:   map[string]string{"test": "test"},
+		Config:     map[string]string{"test": "test"},
+	},
+}
+
+var v1beta4Plugin = &v1beta4.EmqxPlugin{
+	ObjectMeta: metav1.ObjectMeta{
+		Name:        "test",
+		Namespace:   "test",
+		Labels:      map[string]string{"test": "test"},
+		Annotations: map[string]string{"test": "test"},
+	},
+	Spec: v1beta4.EmqxPluginSpec{
+		PluginName: "test",
+		Selector:   map[string]string{"test": "test"},
+		Config:     map[string]string{"test": "test"},
+	},
+}
+
+func TestPluginConvertTo(t *testing.T) {
+	plugin := &v1beta4.EmqxPlugin{}
+	err := v1beta3Plugin.ConvertTo(plugin)
+	assert.Nil(t, err)
+	assert.ObjectsAreEqualValues(v1beta4Plugin, plugin)
+}
+
+func TestPluginConvertFrom(t *testing.T) {
+	plugin := &EmqxPlugin{}
+	err := plugin.ConvertFrom(v1beta4Plugin)
+	assert.Nil(t, err)
+	assert.ObjectsAreEqualValues(v1beta3Plugin, plugin)
+}

apis/apps/v1beta3/emqxplugin_types.go

@@ -45,6 +45,7 @@ type EmqxPluginStatus struct {
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
+//+kubebuilder:deprecatedversion
 
 // EmqxPlugin is the Schema for the emqxplugins API
 type EmqxPlugin struct {

apis/apps/v1beta3/groupversion_info.go

@@ -15,8 +15,8 @@ limitations under the License.
 */
 
 // Package v1beta3 contains API Schema definitions for the apps v1beta3 API group
-//+kubebuilder:object:generate=true
-//+groupName=apps.emqx.io
+// +kubebuilder:object:generate=true
+// +groupName=apps.emqx.io
 package v1beta3
 
 import (

apis/apps/v1beta3/modules_type.go

@@ -7,13 +7,13 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
-//+kubebuilder:object:generate=true
+// +kubebuilder:object:generate=true
 type EmqxBrokerModule struct {
 	Name   string `json:"name,omitempty"`
 	Enable bool   `json:"enable,omitempty"`
 }
 
-//+kubebuilder:object:generate=false
+// +kubebuilder:object:generate=false
 type EmqxBrokerModuleList struct {
 	Items []EmqxBrokerModule
 }
@@ -57,15 +57,15 @@ func (list *EmqxBrokerModuleList) String() string {
 	return str
 }
 
-//+kubebuilder:object:generate=true
+// +kubebuilder:object:generate=true
 type EmqxEnterpriseModule struct {
 	Name   string `json:"name,omitempty"`
 	Enable bool   `json:"enable,omitempty"`
 	// +kubebuilder:pruning:PreserveUnknownFields
 	Configs runtime.RawExtension `json:"configs,omitempty"`
 }
 
-//+kubebuilder:object:generate=false
+// +kubebuilder:object:generate=false
 type EmqxEnterpriseModuleList struct {
 	Items []EmqxEnterpriseModule
 }

apis/apps/v1beta3/namer.go

@@ -6,7 +6,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-//+kubebuilder:object:generate=false
+// +kubebuilder:object:generate=false
 type Names struct {
 	client.Object
 }
@@ -27,10 +27,6 @@ func (n Names) PluginsConfig() string {
 	return fmt.Sprintf("%s-%s", n.Object.GetName(), "plugins-config")
 }
 
-func (n Names) LoadedPlugins() string {
-	return fmt.Sprintf("%s-%s", n.Object.GetName(), "loaded-plugins")
-}
-
 func (n Names) LoadedModules() string {
 	return fmt.Sprintf("%s-%s", n.Object.GetName(), "loaded-modules")
 }