Merge branch 'main' into pib-sidecar-prom-metrics

paritytech · Aug 29, 2023 · 80ff152 · 80ff152
2 parents b2a4041 + ce9d237
commit 80ff152
Show file tree

Hide file tree

Showing 12 changed files with 73 additions and 26 deletions.
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -90,3 +90,24 @@ jobs:
         --target-branch ${{ github.event.repository.default_branch }} \
         --charts ${{ steps.list-changed.outputs.changed }}
       if: steps.list-changed.outputs.changed
+
+  validate-manifests:
+    runs-on: ubuntu-latest
+    container: docker.io/paritytech/kube-manifests-validation:k8s-1.25.9-gator-3.12.0-datree-1.9.19-9196b4c
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - run: |
+        git config --system --add safe.directory $GITHUB_WORKSPACE
+        git fetch origin "+${GITHUB_BASE_REF}:${GITHUB_BASE_REF}"
+
+    - name: Validate manifests
+      run: |
+        /app/validate-k8s-manifests.sh \
+          --datree-policy-config /app/datree-policies.yaml \
+          --git-ref-changed-paths $GITHUB_BASE_REF \
+          --skip-gatekeeper \
+          charts
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -13,6 +13,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - name: Configure Git
         run: |
@@ -28,9 +29,16 @@ jobs:
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
 
+      - name: Retrieve GH app token
+        id: generate-github-app-token
+        uses: getsentry/action-github-app-token@v2
+        with:
+          app_id: ${{ secrets.CHART_UPDATER_APP_ID }}
+          private_key: ${{ secrets.CHART_UPDATER_APP_KEY }}
+
       - name: Run chart-releaser
         uses: helm/[email protected]
         env:
-          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          CR_TOKEN: ${{ steps.generate-github-app-token.outputs.token }}
         with:
           charts_dir: charts
diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml
@@ -1,7 +1,7 @@
 - id: helm-docs
   args: []
   description: Uses 'helm-docs' to create documentation from the Helm chart's 'values.yaml' file, and inserts the result into a corresponding 'README.md' file.
-  entry: git-hook/helm-docs
+  entry: git-hooks/helm-docs
   files: (README\.md\.gotmpl|(Chart|requirements|values)\.yaml)$
   language: script
   name: Helm Docs

diff --git a/charts/node/Chart.yaml b/charts/node/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: node
 description: A Helm chart to deploy Substrate/Polkadot nodes
 type: application
-version: 5.2.0
+version: 5.2.1
 maintainers:
   - name: Parity
     url: https://github.com/paritytech/helm-charts

diff --git a/charts/node/README.md b/charts/node/README.md
@@ -18,7 +18,7 @@ This is intended behaviour. Make sure to run `git add -A` once again to stage ch
 
 # Substrate/Polkadot node helm chart
 
-![Version: 5.2.0](https://img.shields.io/badge/Version-5.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 5.2.1](https://img.shields.io/badge/Version-5.2.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 ## Maintainers
 

diff --git a/charts/node/templates/statefulset.yaml b/charts/node/templates/statefulset.yaml
@@ -102,25 +102,25 @@ spec:
 
                 if [ "${METHOD}" == "http-single-tar-lz4" ]; then
                   apk add lz4 --no-cache
-                  rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --stdout --error-on-no-transfer --retries 6 --retries-sleep 10 ${SNAPSHOT_URL} | lz4 -c -d - | tar -x -C /chain-data/chains/${CHAIN_PATH}/
+                  rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --stdout --retries 1 --error-on-no-transfer --no-gzip-encoding ${SNAPSHOT_URL} | lz4 -c -d - | tar -x -C /chain-data/chains/${CHAIN_PATH}/
                   chown -R {{ .Values.podSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} /chain-data/chains/${CHAIN_PATH}/
 
                 elif [ "${METHOD}" == "http-single-tar" ]; then
-                  rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --stdout --error-on-no-transfer --retries 6 --retries-sleep 10 ${SNAPSHOT_URL} | tar -x -C /chain-data/chains/${CHAIN_PATH}/
+                  rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --stdout --retries 1 --error-on-no-transfer --no-gzip-encoding ${SNAPSHOT_URL} | tar -x -C /chain-data/chains/${CHAIN_PATH}/
 
                 elif [ "${METHOD}" == "gcs" ]; then
                   LATEST=$(rclone cat {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --quiet :gcs:${SNAPSHOT_URL}/latest_version.meta.txt)
                   if [ -z "$LATEST" ]; then
                     echo "Failed to retrieve latest_version.meta.txt file. Will download everything from ${SNAPSHOT_URL} instead"
                   fi
-                  rclone sync {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --fast-list --transfers $PARALLEL_TRANFERS --progress --retries 6 --retries-sleep 10 --error-on-no-transfer :gcs:${SNAPSHOT_URL}/${LATEST} /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/
+                  rclone sync {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --fast-list --transfers $PARALLEL_TRANFERS --progress --retries 6 --retries-sleep 10 --error-on-no-transfer --inplace --no-gzip-encoding :gcs:${SNAPSHOT_URL}/${LATEST} /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/
 
                 elif [ "${METHOD}" == "s3" ]; then
                   LATEST=$(rclone cat {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --quiet :s3:${SNAPSHOT_URL}/latest_version.meta.txt )
                   if [ -z "$LATEST" ]; then
                     echo "Failed to retrieve latest_version.meta.txt file. Will download everything from ${SNAPSHOT_URL} instead"
                   fi
-                  rclone sync {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --fast-list --transfers $PARALLEL_TRANFERS --progress --retries 6 --retries-sleep 10 --error-on-no-transfer :s3:${SNAPSHOT_URL}/${LATEST} /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/
+                  rclone sync {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --fast-list --transfers $PARALLEL_TRANFERS --progress --retries 6 --retries-sleep 10 --error-on-no-transfer --inplace --no-gzip-encoding :s3:${SNAPSHOT_URL}/${LATEST} /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/
 
                 elif [ "${METHOD}" == "http-filelist" ]; then
                   LATEST=$(rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --stdout ${SNAPSHOT_URL}/latest_version.meta.txt )
@@ -129,8 +129,8 @@ spec:
                   else
                     SNAPSHOT_URL="${SNAPSHOT_URL}/${LATEST}"
                   fi
-                  rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --error-on-no-transfer ${SNAPSHOT_URL}/{{ .Values.node.chainData.chainSnapshot.filelistName }} /tmp/filelist.txt
-                  rclone copy {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --progress --error-on-no-transfer --transfers $PARALLEL_TRANFERS --http-url ${SNAPSHOT_URL} --no-traverse --http-no-head --disable-http2 --retries 6 --retries-sleep 10 --files-from /tmp/filelist.txt :http: /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/
+                  rclone copyurl {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --retries 6 --retries-sleep 10 --error-on-no-transfer --inplace --no-gzip-encoding ${SNAPSHOT_URL}/{{ .Values.node.chainData.chainSnapshot.filelistName }} /tmp/filelist.txt
+                  rclone copy {{ .Values.initContainers.downloadChainSnapshot.cmdArgs }} --transfers $PARALLEL_TRANFERS --progress --retries 6 --retries-sleep 10 --error-on-no-transfer --inplace --no-gzip-encoding --http-url ${SNAPSHOT_URL} --no-traverse --http-no-head --disable-http2 --size-only --files-from /tmp/filelist.txt :http: /chain-data/chains/${CHAIN_PATH}/{{ $databasePath }}/
                 fi
               fi
           env:

diff --git a/charts/substrate-faucet/Chart.yaml b/charts/substrate-faucet/Chart.yaml
@@ -2,7 +2,11 @@ apiVersion: v2
 name: substrate-faucet
 description: A Helm chart to deploy substrate-faucet
 type: application
-version: 2.2.2
+version: 3.0.1
 maintainers:
   - name: Parity
     url: https://github.com/paritytech/helm-charts
+dependencies:
+  - name: postgresql
+    version: "12.8.3"
+    repository: "https://charts.bitnami.com/bitnami"
diff --git a/charts/substrate-faucet/README.md b/charts/substrate-faucet/README.md
@@ -7,11 +7,12 @@ The helm chart installs the [Substrate Matrix faucet](https://github.com/parityt
 To deploy a Westend faucet:
 ```console
 helm repo add parity https://paritytech.github.io/helm-charts/
+helm dependency update
 helm install substrate-faucet parity/substrate-faucet \
     --set faucet.secret.SMF_CONFIG_FAUCET_ACCOUNT_MNEMONIC="//Alice" \
     --set faucet.secret.SMF_CONFIG_MATRIX_ACCESS_TOKEN="******" \
     --set faucet.config.SMF_CONFIG_MATRIX_SERVER="https://matrix.org" \
-    --set faucet.config.SMF_CONFIG_MATRIX_BOT_USER_ID="@test_bot_faucet:matrix.org" 
+    --set faucet.config.SMF_CONFIG_MATRIX_BOT_USER_ID="@test_bot_faucet:matrix.org" \
     --set faucet.config.SMF_CONFIG_NETWORK="westend" 
 ```
 

diff --git a/charts/substrate-faucet/templates/configmap.yaml b/charts/substrate-faucet/templates/configmap.yaml
@@ -7,4 +7,8 @@ data:
   {{- range $key, $val := .Values.faucet.config }}
   {{ $key }}: {{ $val | quote }}
   {{- end }}
+  SMF_CONFIG_DB_HOST: "{{ .Release.Name }}-postgresql"
+  SMF_CONFIG_DB_PORT: "5432"
+  SMF_CONFIG_DB_USERNAME: postgres
+  SMF_CONFIG_DB_DATABASE_NAME: "{{ .Values.postgresql.global.postgresql.auth.database }}"
 {{- end }}
diff --git a/charts/substrate-faucet/templates/deployment.yaml b/charts/substrate-faucet/templates/deployment.yaml
@@ -33,24 +33,15 @@ spec:
           env:
             - name: SMF_CONFIG_PORT
               value: "5555"
-          {{- range $key, $val := .Values.faucet.secret }}
-            - name: {{ $key }}
-              valueFrom:
-                secretKeyRef:
-                  key: {{ $key }}
-                  name: {{ $.Values.faucet.existingSecret | default (printf "%s-secret" $.Release.Name) }}
-          {{- end }}
-          {{- range $key, $val := .Values.faucet.config }}
-            - name: {{ $key }}
-              valueFrom:
-                configMapKeyRef:
-                  key: {{ $key }}
-                  name: {{ $.Values.faucet.existingConfigMap | default (printf "%s-config" $.Release.Name) }}
-          {{- end }}
             - name: SMF_CONFIG_DEPLOYED_REF
               value: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
             - name: SMF_CONFIG_EXTERNAL_ACCESS
               value: {{ .Values.faucet.externalAccess | quote }}
+          envFrom:
+            - configMapRef:
+                name: {{ $.Values.faucet.existingConfigMap | default (printf "%s-config" $.Release.Name) }}
+            - secretRef:
+                name: {{ $.Values.faucet.existingSecret | default (printf "%s-secret" $.Release.Name) }}
           ports:
             - name: http
               containerPort: 5555

diff --git a/charts/substrate-faucet/templates/secret.yaml b/charts/substrate-faucet/templates/secret.yaml
@@ -8,4 +8,5 @@ data:
   {{- range $key, $val := .Values.faucet.secret }}
   {{ $key }}: {{ $val | b64enc }}
   {{- end }}
+  SMF_CONFIG_DB_PASSWORD: {{ .Values.postgresql.global.postgresql.auth.postgresPassword  | b64enc | quote -}}
 {{- end }}
diff --git a/charts/substrate-faucet/values.yaml b/charts/substrate-faucet/values.yaml
@@ -125,3 +125,20 @@ ingress:
   ## @param ingress.enabled Specifies whether as Ingress should be created
   ##
   enabled: false
+
+postgresql:
+  global:
+    postgresql:
+      auth:
+        database: faucet
+        postgresPassword: "Secret!"
+  primary:
+    persistence:
+      size: 4Gi
+    resources:
+      limits:
+        cpu: 500m
+        memory: 1024Mi
+      requests:
+        cpu: 250m
+        memory: 512Mi