From 76248242d06171ba156a7b4dd175003204c4a9e0 Mon Sep 17 00:00:00 2001 From: kogeler <25884155+kogeler@users.noreply.github.com> Date: Fri, 11 Aug 2023 11:50:05 +0300 Subject: [PATCH 1/5] backup downloading was fixed for node helm chart (#283) * backup downloading was fixed for node helm chart Signed-off-by: kogeler --- charts/node/Chart.yaml | 2 +- charts/node/README.md | 2 +- charts/node/templates/statefulset.yaml | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/node/Chart.yaml b/charts/node/Chart.yaml index 78da359e..ca699db9 100644 --- a/charts/node/Chart.yaml +++ b/charts/node/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: node description: A Helm chart to deploy Substrate/Polkadot nodes type: application -version: 5.1.5 +version: 5.1.6 maintainers: - name: Parity url: https://github.com/paritytech/helm-charts diff --git a/charts/node/README.md b/charts/node/README.md index 8a8e61fe..e76a91d5 100644 --- a/charts/node/README.md +++ b/charts/node/README.md @@ -18,7 +18,7 @@ This is intended behaviour. Make sure to run `git add -A` once again to stage ch # Substrate/Polkadot node helm chart -![Version: 5.1.5](https://img.shields.io/badge/Version-5.1.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 5.1.6](https://img.shields.io/badge/Version-5.1.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ## Maintainers diff --git a/charts/node/templates/statefulset.yaml b/charts/node/templates/statefulset.yaml index 64db8f7d..29ce603a 100644 --- a/charts/node/templates/statefulset.yaml +++ b/charts/node/templates/statefulset.yaml @@ -102,25 +102,25 @@ spec: if [ "${METHOD}" == "http-single-tar-lz4" ]; then apk add lz4 --no-cache - rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --stdout --error-on-no-transfer --retries 6 --retries-sleep 10 ${SNAPSHOT_URL} | lz4 -c -d - | tar -x -C /chain-data/chains/${CHAIN_PATH}/ + rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --stdout --retries 1 --error-on-no-transfer --no-gzip-encoding ${SNAPSHOT_URL} | lz4 -c -d - | tar -x -C /chain-data/chains/${CHAIN_PATH}/ chown -R {{ .Values.podSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} /chain-data/chains/${CHAIN_PATH}/ elif [ "${METHOD}" == "http-single-tar" ]; then - rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --stdout --error-on-no-transfer --retries 6 --retries-sleep 10 ${SNAPSHOT_URL} | tar -x -C /chain-data/chains/${CHAIN_PATH}/ + rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --stdout --retries 1 --error-on-no-transfer --no-gzip-encoding ${SNAPSHOT_URL} | tar -x -C /chain-data/chains/${CHAIN_PATH}/ elif [ "${METHOD}" == "gcs" ]; then LATEST=$(rclone cat {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --quiet :gcs:${SNAPSHOT_URL}/latest_version.meta.txt) if [ -z "$LATEST" ]; then echo "Failed to retrieve latest_version.meta.txt file. Will download everything from ${SNAPSHOT_URL} instead" fi - rclone sync {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --fast-list --transfers $PARALLEL_TRANFERS --progress --retries 6 --retries-sleep 10 --error-on-no-transfer :gcs:${SNAPSHOT_URL}/${LATEST} /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/ + rclone sync {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --fast-list --transfers $PARALLEL_TRANFERS --progress --retries 6 --retries-sleep 10 --error-on-no-transfer --inplace --no-gzip-encoding :gcs:${SNAPSHOT_URL}/${LATEST} /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/ elif [ "${METHOD}" == "s3" ]; then LATEST=$(rclone cat {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --quiet :s3:${SNAPSHOT_URL}/latest_version.meta.txt ) if [ -z "$LATEST" ]; then echo "Failed to retrieve latest_version.meta.txt file. Will download everything from ${SNAPSHOT_URL} instead" fi - rclone sync {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --fast-list --transfers $PARALLEL_TRANFERS --progress --retries 6 --retries-sleep 10 --error-on-no-transfer :s3:${SNAPSHOT_URL}/${LATEST} /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/ + rclone sync {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --fast-list --transfers $PARALLEL_TRANFERS --progress --retries 6 --retries-sleep 10 --error-on-no-transfer --inplace --no-gzip-encoding :s3:${SNAPSHOT_URL}/${LATEST} /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/ elif [ "${METHOD}" == "http-filelist" ]; then LATEST=$(rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --stdout ${SNAPSHOT_URL}/latest_version.meta.txt ) @@ -129,8 +129,8 @@ spec: else SNAPSHOT_URL="${SNAPSHOT_URL}/${LATEST}" fi - rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --error-on-no-transfer ${SNAPSHOT_URL}/{{ .Values.node.chainData.chainSnapshot.filelistName }} /tmp/filelist.txt - rclone copy {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --progress --error-on-no-transfer --transfers $PARALLEL_TRANFERS --http-url ${SNAPSHOT_URL} --no-traverse --http-no-head --disable-http2 --retries 6 --retries-sleep 10 --files-from /tmp/filelist.txt :http: /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/ + rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --retries 6 --retries-sleep 10 --error-on-no-transfer --inplace --no-gzip-encoding ${SNAPSHOT_URL}/{{ .Values.node.chainData.chainSnapshot.filelistName }} /tmp/filelist.txt + rclone copy {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --transfers $PARALLEL_TRANFERS --progress --retries 6 --retries-sleep 10 --error-on-no-transfer --inplace --no-gzip-encoding --http-url ${SNAPSHOT_URL} --no-traverse --http-no-head --disable-http2 --size-only --files-from /tmp/filelist.txt :http: /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/ fi fi env: From 7dfb38633fa02b5612fe842e2ecf75e098e269bf Mon Sep 17 00:00:00 2001 From: Artyom Bakhtin Date: Wed, 16 Aug 2023 09:37:46 +0100 Subject: [PATCH 2/5] Switch to using GH App for pushing to gh-pages branch (#285) --- .github/workflows/release.yaml | 10 +++++++++- charts/node/Chart.yaml | 2 +- charts/node/README.md | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fd6cd63e..17b177b2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -13,6 +13,7 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 + persist-credentials: false - name: Configure Git run: | @@ -28,9 +29,16 @@ jobs: run: | helm repo add bitnami https://charts.bitnami.com/bitnami + - name: Retrieve GH app token + id: generate-github-app-token + uses: getsentry/action-github-app-token@v2 + with: + app_id: ${{ secrets.CHART_UPDATER_APP_ID }} + private_key: ${{ secrets.CHART_UPDATER_APP_KEY }} + - name: Run chart-releaser uses: helm/chart-releaser-action@v1.5.0 env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + CR_TOKEN: ${{ steps.generate-github-app-token.outputs.token }} with: charts_dir: charts diff --git a/charts/node/Chart.yaml b/charts/node/Chart.yaml index ca699db9..06bd31c0 100644 --- a/charts/node/Chart.yaml +++ b/charts/node/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: node description: A Helm chart to deploy Substrate/Polkadot nodes type: application -version: 5.1.6 +version: 5.1.7 maintainers: - name: Parity url: https://github.com/paritytech/helm-charts diff --git a/charts/node/README.md b/charts/node/README.md index e76a91d5..9854e72f 100644 --- a/charts/node/README.md +++ b/charts/node/README.md @@ -18,7 +18,7 @@ This is intended behaviour. Make sure to run `git add -A` once again to stage ch # Substrate/Polkadot node helm chart -![Version: 5.1.6](https://img.shields.io/badge/Version-5.1.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 5.1.7](https://img.shields.io/badge/Version-5.1.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ## Maintainers From 772f273f940aadb475ec0914ead24506463a939d Mon Sep 17 00:00:00 2001 From: Yuri Volkov <0@mcornholio.ru> Date: Wed, 23 Aug 2023 12:57:33 +0200 Subject: [PATCH 3/5] Added cloudsql to faucet (#286) * Added postgres to faucet * add env variable with postgresql creds --------- Co-authored-by: BulatSaif --- charts/substrate-faucet/Chart.yaml | 6 +++++- charts/substrate-faucet/README.md | 3 ++- .../substrate-faucet/templates/configmap.yaml | 5 +++++ .../templates/deployment.yaml | 19 +++++-------------- charts/substrate-faucet/templates/secret.yaml | 1 + charts/substrate-faucet/values.yaml | 16 ++++++++++++++++ 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/charts/substrate-faucet/Chart.yaml b/charts/substrate-faucet/Chart.yaml index 04440491..a5c4bf2f 100644 --- a/charts/substrate-faucet/Chart.yaml +++ b/charts/substrate-faucet/Chart.yaml @@ -2,7 +2,11 @@ apiVersion: v2 name: substrate-faucet description: A Helm chart to deploy substrate-faucet type: application -version: 2.2.2 +version: 3.0.0 maintainers: - name: Parity url: https://github.com/paritytech/helm-charts +dependencies: + - name: postgresql + version: "12.8.3" + repository: "https://charts.bitnami.com/bitnami" diff --git a/charts/substrate-faucet/README.md b/charts/substrate-faucet/README.md index 1c0af20d..e87939b5 100644 --- a/charts/substrate-faucet/README.md +++ b/charts/substrate-faucet/README.md @@ -7,11 +7,12 @@ The helm chart installs the [Substrate Matrix faucet](https://github.com/parityt To deploy a Westend faucet: ```console helm repo add parity https://paritytech.github.io/helm-charts/ +helm dependency update helm install substrate-faucet parity/substrate-faucet \ --set faucet.secret.SMF_CONFIG_FAUCET_ACCOUNT_MNEMONIC="//Alice" \ --set faucet.secret.SMF_CONFIG_MATRIX_ACCESS_TOKEN="******" \ --set faucet.config.SMF_CONFIG_MATRIX_SERVER="https://matrix.org" \ - --set faucet.config.SMF_CONFIG_MATRIX_BOT_USER_ID="@test_bot_faucet:matrix.org" + --set faucet.config.SMF_CONFIG_MATRIX_BOT_USER_ID="@test_bot_faucet:matrix.org" \ --set faucet.config.SMF_CONFIG_NETWORK="westend" ``` diff --git a/charts/substrate-faucet/templates/configmap.yaml b/charts/substrate-faucet/templates/configmap.yaml index fac2ebfd..aa8a8956 100644 --- a/charts/substrate-faucet/templates/configmap.yaml +++ b/charts/substrate-faucet/templates/configmap.yaml @@ -7,4 +7,9 @@ data: {{- range $key, $val := .Values.faucet.config }} {{ $key }}: {{ $val | quote }} {{- end }} + POSTGRESQL_HOST: "{{ .Release.Name }}-postgresql" + POSTGRESQL_PORT: "5432" + POSTGRESQL_USERNAME: postgres + POSTGRESQL_DATABASE: postgres + POSTGRESQL_SSLMODE: disable {{- end }} diff --git a/charts/substrate-faucet/templates/deployment.yaml b/charts/substrate-faucet/templates/deployment.yaml index cec9c33b..cc1aca8d 100644 --- a/charts/substrate-faucet/templates/deployment.yaml +++ b/charts/substrate-faucet/templates/deployment.yaml @@ -33,24 +33,15 @@ spec: env: - name: SMF_CONFIG_PORT value: "5555" - {{- range $key, $val := .Values.faucet.secret }} - - name: {{ $key }} - valueFrom: - secretKeyRef: - key: {{ $key }} - name: {{ $.Values.faucet.existingSecret | default (printf "%s-secret" $.Release.Name) }} - {{- end }} - {{- range $key, $val := .Values.faucet.config }} - - name: {{ $key }} - valueFrom: - configMapKeyRef: - key: {{ $key }} - name: {{ $.Values.faucet.existingConfigMap | default (printf "%s-config" $.Release.Name) }} - {{- end }} - name: SMF_CONFIG_DEPLOYED_REF value: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - name: SMF_CONFIG_EXTERNAL_ACCESS value: {{ .Values.faucet.externalAccess | quote }} + envFrom: + - configMapRef: + name: {{ $.Values.faucet.existingConfigMap | default (printf "%s-config" $.Release.Name) }} + - secretRef: + name: {{ $.Values.faucet.existingSecret | default (printf "%s-secret" $.Release.Name) }} ports: - name: http containerPort: 5555 diff --git a/charts/substrate-faucet/templates/secret.yaml b/charts/substrate-faucet/templates/secret.yaml index 573758c2..4297794b 100644 --- a/charts/substrate-faucet/templates/secret.yaml +++ b/charts/substrate-faucet/templates/secret.yaml @@ -8,4 +8,5 @@ data: {{- range $key, $val := .Values.faucet.secret }} {{ $key }}: {{ $val | b64enc }} {{- end }} + POSTGRESQL_PASSWORD: {{ .Values.postgresql.global.postgresql.auth.postgresPassword | b64enc | quote -}} {{- end }} diff --git a/charts/substrate-faucet/values.yaml b/charts/substrate-faucet/values.yaml index 9082d8d2..40d57f7b 100644 --- a/charts/substrate-faucet/values.yaml +++ b/charts/substrate-faucet/values.yaml @@ -125,3 +125,19 @@ ingress: ## @param ingress.enabled Specifies whether as Ingress should be created ## enabled: false + +postgresql: + global: + postgresql: + auth: + postgresPassword: "Secret!" + primary: + persistence: + size: 4Gi + resources: + limits: + cpu: 500m + memory: 1024Mi + requests: + cpu: 250m + memory: 512Mi From f9f80956ab7bf8ed4629bb518231fa22382fcd36 Mon Sep 17 00:00:00 2001 From: Yuri Volkov <0@mcornholio.ru> Date: Wed, 23 Aug 2023 16:22:55 +0200 Subject: [PATCH 4/5] Fix variable names (#287) * Fix variable names Even though I was prompted to doublecheck variable names, I've mixed up variables accepted by application and variables set by application... * fix db name --------- Co-authored-by: BulatSaif --- charts/substrate-faucet/Chart.yaml | 2 +- charts/substrate-faucet/templates/configmap.yaml | 9 ++++----- charts/substrate-faucet/templates/secret.yaml | 2 +- charts/substrate-faucet/values.yaml | 1 + 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/substrate-faucet/Chart.yaml b/charts/substrate-faucet/Chart.yaml index a5c4bf2f..b159d41d 100644 --- a/charts/substrate-faucet/Chart.yaml +++ b/charts/substrate-faucet/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: substrate-faucet description: A Helm chart to deploy substrate-faucet type: application -version: 3.0.0 +version: 3.0.1 maintainers: - name: Parity url: https://github.com/paritytech/helm-charts diff --git a/charts/substrate-faucet/templates/configmap.yaml b/charts/substrate-faucet/templates/configmap.yaml index aa8a8956..571e0f6e 100644 --- a/charts/substrate-faucet/templates/configmap.yaml +++ b/charts/substrate-faucet/templates/configmap.yaml @@ -7,9 +7,8 @@ data: {{- range $key, $val := .Values.faucet.config }} {{ $key }}: {{ $val | quote }} {{- end }} - POSTGRESQL_HOST: "{{ .Release.Name }}-postgresql" - POSTGRESQL_PORT: "5432" - POSTGRESQL_USERNAME: postgres - POSTGRESQL_DATABASE: postgres - POSTGRESQL_SSLMODE: disable + SMF_CONFIG_DB_HOST: "{{ .Release.Name }}-postgresql" + SMF_CONFIG_DB_PORT: "5432" + SMF_CONFIG_DB_USERNAME: postgres + SMF_CONFIG_DB_DATABASE_NAME: "{{ .Values.postgresql.global.postgresql.auth.database }}" {{- end }} diff --git a/charts/substrate-faucet/templates/secret.yaml b/charts/substrate-faucet/templates/secret.yaml index 4297794b..04a4014e 100644 --- a/charts/substrate-faucet/templates/secret.yaml +++ b/charts/substrate-faucet/templates/secret.yaml @@ -8,5 +8,5 @@ data: {{- range $key, $val := .Values.faucet.secret }} {{ $key }}: {{ $val | b64enc }} {{- end }} - POSTGRESQL_PASSWORD: {{ .Values.postgresql.global.postgresql.auth.postgresPassword | b64enc | quote -}} + SMF_CONFIG_DB_PASSWORD: {{ .Values.postgresql.global.postgresql.auth.postgresPassword | b64enc | quote -}} {{- end }} diff --git a/charts/substrate-faucet/values.yaml b/charts/substrate-faucet/values.yaml index 40d57f7b..a65102ca 100644 --- a/charts/substrate-faucet/values.yaml +++ b/charts/substrate-faucet/values.yaml @@ -130,6 +130,7 @@ postgresql: global: postgresql: auth: + database: faucet postgresPassword: "Secret!" primary: persistence: From ce9d2376bada6a71e5e144fb5aae8ec3369dfaf7 Mon Sep 17 00:00:00 2001 From: Artyom Bakhtin Date: Mon, 28 Aug 2023 10:33:48 +0100 Subject: [PATCH 5/5] K8s manifests validation pipeline (#288) --- .github/workflows/pr.yaml | 21 +++++++++++++++++++++ .pre-commit-hooks.yaml | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b071e1d1..cd242269 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -90,3 +90,24 @@ jobs: --target-branch ${{ github.event.repository.default_branch }} \ --charts ${{ steps.list-changed.outputs.changed }} if: steps.list-changed.outputs.changed + + validate-manifests: + runs-on: ubuntu-latest + container: docker.io/paritytech/kube-manifests-validation:k8s-1.25.9-gator-3.12.0-datree-1.9.19-9196b4c + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - run: | + git config --system --add safe.directory $GITHUB_WORKSPACE + git fetch origin "+${GITHUB_BASE_REF}:${GITHUB_BASE_REF}" + + - name: Validate manifests + run: | + /app/validate-k8s-manifests.sh \ + --datree-policy-config /app/datree-policies.yaml \ + --git-ref-changed-paths $GITHUB_BASE_REF \ + --skip-gatekeeper \ + charts diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 9b77c52c..f74dec69 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,7 +1,7 @@ - id: helm-docs args: [] description: Uses 'helm-docs' to create documentation from the Helm chart's 'values.yaml' file, and inserts the result into a corresponding 'README.md' file. - entry: git-hook/helm-docs + entry: git-hooks/helm-docs files: (README\.md\.gotmpl|(Chart|requirements|values)\.yaml)$ language: script name: Helm Docs