diff --git a/.ci/chart_test.sh b/.ci/chart_test.sh index d4ac20d9..43e07bd3 100755 --- a/.ci/chart_test.sh +++ b/.ci/chart_test.sh @@ -35,6 +35,8 @@ source ${PULSAR_HOME}/.ci/helm.sh # create cluster ci::create_cluster +ci::helm_repo_add + extra_opts="" if [[ "x${SYMMETRIC}" == "xtrue" ]]; then extra_opts="-s" diff --git a/.ci/helm.sh b/.ci/helm.sh index c7e1ec9e..e23a3d63 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -83,6 +83,13 @@ function ci::install_cert_manager() { echo "Successfully installed the cert manager." } +function ci::helm_repo_add() { + echo "Adding the helm repo ..." + ${HELM} repo add prometheus-community https://prometheus-community.github.io/helm-charts + ${HELM} repo update + echo "Successfully added the helm repo." +} + function ci::print_pod_logs() { echo "Logs for all pulsar containers:" for k8sobject in $(${KUBECTL} get pods,jobs -n ${NAMESPACE} -l app=pulsar -o=name); do diff --git a/.github/workflows/pulsar-helm-chart-ci.yaml b/.github/workflows/pulsar-helm-chart-ci.yaml index 2a9c81ed..bb0932c2 100644 --- a/.github/workflows/pulsar-helm-chart-ci.yaml +++ b/.github/workflows/pulsar-helm-chart-ci.yaml @@ -127,6 +127,29 @@ jobs: --validate-maintainers=false \ --target-branch ${{ github.event.repository.default_branch }} + - name: Run kubeconform check for helm template with every major k8s version 1.21.0-1.29.0 + if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + run: | + PULSAR_CHART_HOME=$(pwd) + source ${PULSAR_CHART_HOME}/hack/common.sh + source ${PULSAR_CHART_HOME}/.ci/helm.sh + hack::ensure_kubectl + hack::ensure_helm + hack::ensure_kubeconform + ci::helm_repo_add + helm dependency build charts/pulsar + validate_helm_template_with_k8s_version() { + local kube_version=$1 + echo "Validating helm template with kubeconform for k8s version $kube_version" + helm template charts/pulsar --set kube-prometheus-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version | \ + kubeconform -schema-location default -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -strict -kubernetes-version $kube_version -summary + } + set -o pipefail + for k8s_version_part in {21..29}; do + k8s_version="1.${k8s_version_part}.0" + echo "Validating helm template with kubeconform for k8s version $k8s_version" + validate_helm_template_with_k8s_version $k8s_version + done - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks uses: ./.github/actions/ssh-access diff --git a/charts/pulsar/values.yaml b/charts/pulsar/values.yaml index 778c17ac..bbbf58bd 100644 --- a/charts/pulsar/values.yaml +++ b/charts/pulsar/values.yaml @@ -601,11 +601,33 @@ bookkeeper: -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem - # configure the memory settings based on jvm memory settings - dbStorage_writeCacheMaxSizeMb: "32" - dbStorage_readAheadCacheMaxSizeMb: "32" - dbStorage_rocksDB_writeBufferSizeMB: "8" - dbStorage_rocksDB_blockCacheSize: "8388608" + # + # Bookkeeper configuration reference: https://bookkeeper.apache.org/docs/reference/config + # + # https://bookkeeper.apache.org/docs/reference/config#db-ledger-storage-settings + # You could use the below example settings for a minimal configuration + # dbStorage_writeCacheMaxSizeMb: "32" + # dbStorage_readAheadCacheMaxSizeMb: "32" + # dbStorage_rocksDB_writeBufferSizeMB: "8" + # dbStorage_rocksDB_blockCacheSize: "8388608" + # + # configure the data compaction (bookie entry log compaction and gc) settings + # https://bookkeeper.apache.org/docs/reference/config#garbage-collection-settings + # https://bookkeeper.apache.org/docs/reference/config#entry-log-compaction-settings + minorCompactionThreshold: "0.2" # default 0.2 (use default) + minorCompactionInterval: "360" # default 3600 seconds (6 minutes vs default 1 hour) + majorCompactionThreshold: "0.8" # default 0.5 + majorCompactionInterval: "10800" # default 86400 seconds (3 hours vs default 1 day) + gcWaitTime: "300000" # default 900000 milli-seconds (5 minutes vs default 15 minutes) + isForceGCAllowWhenNoSpace: "true" # default false + # disk utilization configuration + # https://bookkeeper.apache.org/docs/reference/config#disk-utilization + # Make sure that diskUsageLwmThreshold <= diskUsageWarnThreshold <= diskUsageThreshold + diskUsageLwmThreshold: "0.85" # default 0.90 + diskUsageWarnThreshold: "0.9" # default 0.95 + diskUsageThreshold: "0.95" # default 0.95 (use default) + diskCheckInterval: "1800" # default 10000 + ## Add a custom command to the start up process of the bookie pods (e.g. update-ca-certificates, jvm commands, etc) additionalCommand: ## Bookkeeper Service diff --git a/examples/values-minikube.yaml b/examples/values-minikube.yaml index f7842c7e..6c52107f 100644 --- a/examples/values-minikube.yaml +++ b/examples/values-minikube.yaml @@ -35,6 +35,13 @@ zookeeper: bookkeeper: replicaCount: 1 + configData: + # minimal memory use for bookkeeper + # https://bookkeeper.apache.org/docs/reference/config#db-ledger-storage-settings + dbStorage_writeCacheMaxSizeMb: "32" + dbStorage_readAheadCacheMaxSizeMb: "32" + dbStorage_rocksDB_writeBufferSizeMB: "8" + dbStorage_rocksDB_blockCacheSize: "8388608" broker: replicaCount: 1 diff --git a/hack/common.sh b/hack/common.sh index a776c23f..15003ca6 100755 --- a/hack/common.sh +++ b/hack/common.sh @@ -33,6 +33,8 @@ HELM_BIN=$OUTPUT_BIN/helm KIND_BIN=$OUTPUT_BIN/kind CR_BIN=$OUTPUT_BIN/cr : "${CR_VERSION:=1.6.0}" +KUBECONFORM_BIN=$OUTPUT_BIN/kubeconform +: "${KUBECONFORM_VERSION:=0.6.4}" export PATH="$OUTPUT_BIN:$PATH" test -d "$OUTPUT_BIN" || mkdir -p "$OUTPUT_BIN" @@ -134,3 +136,9 @@ function hack::ensure_cr() { chmod +x $CR_BIN $CR_BIN version } + +function hack::ensure_kubeconform() { + echo "Installing kubeconform v$KUBECONFORM_VERSION..." + curl -s --retry 10 -L https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-${OS}-${ARCH}.tar.gz | tar -xzO kubeconform > $KUBECONFORM_BIN + chmod +x $KUBECONFORM_BIN +} \ No newline at end of file