From a203fe713489d85d9a4e374daf8b5f350724c2d1 Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Wed, 18 Oct 2023 11:54:02 +0100 Subject: [PATCH 01/10] Don't wait for migrations for capture pods These pods don't care about migrations and waiting for them slows down scaling and deployments --- charts/posthog/Chart.yaml | 2 +- charts/posthog/templates/events-deployment.yaml | 1 - charts/posthog/templates/recordings-deployment.yaml | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/charts/posthog/Chart.yaml b/charts/posthog/Chart.yaml index de153e16..373b7eff 100644 --- a/charts/posthog/Chart.yaml +++ b/charts/posthog/Chart.yaml @@ -11,7 +11,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 30.31.0 +version: 30.32.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. diff --git a/charts/posthog/templates/events-deployment.yaml b/charts/posthog/templates/events-deployment.yaml index afe2e04e..a3d9df4f 100644 --- a/charts/posthog/templates/events-deployment.yaml +++ b/charts/posthog/templates/events-deployment.yaml @@ -174,5 +174,4 @@ spec: {{- end }} initContainers: {{- include "_snippet-initContainers-wait-for-service-dependencies" . | indent 8 }} - {{- include "_snippet-initContainers-wait-for-migrations" . | indent 8 }} {{- end }} diff --git a/charts/posthog/templates/recordings-deployment.yaml b/charts/posthog/templates/recordings-deployment.yaml index 42b18822..1a736c30 100644 --- a/charts/posthog/templates/recordings-deployment.yaml +++ b/charts/posthog/templates/recordings-deployment.yaml @@ -174,5 +174,4 @@ spec: {{- end }} initContainers: {{- include "_snippet-initContainers-wait-for-service-dependencies" . | indent 8 }} - {{- include "_snippet-initContainers-wait-for-migrations" . | indent 8 }} {{- end }} From 6c1bbaff4f13920bf69ff565045555d10b91847e Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Wed, 18 Oct 2023 13:23:49 +0100 Subject: [PATCH 02/10] Fix test --- charts/posthog/tests/_initContainers-wait-for-migrations.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/posthog/tests/_initContainers-wait-for-migrations.yaml b/charts/posthog/tests/_initContainers-wait-for-migrations.yaml index 3d193d83..a9c082d8 100644 --- a/charts/posthog/tests/_initContainers-wait-for-migrations.yaml +++ b/charts/posthog/tests/_initContainers-wait-for-migrations.yaml @@ -1,6 +1,5 @@ suite: _initContainers-wait-for-migrations templates: - - templates/events-deployment.yaml - templates/plugins-deployment.yaml - templates/web-deployment.yaml - templates/worker-deployment.yaml @@ -10,7 +9,6 @@ templates: tests: - it: spec.template.spec.initContainers[1].env override via 'env' should work templates: # TODO: remove once secrets.yaml will be fixed/removed - - templates/events-deployment.yaml - templates/plugins-deployment.yaml - templates/web-deployment.yaml - templates/worker-deployment.yaml From 6b6e954b0e504f44a2c725caf81a0074fd1de19b Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 19 Oct 2023 09:38:00 +0100 Subject: [PATCH 03/10] ? --- charts/posthog/templates/events-deployment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/posthog/templates/events-deployment.yaml b/charts/posthog/templates/events-deployment.yaml index a3d9df4f..afe2e04e 100644 --- a/charts/posthog/templates/events-deployment.yaml +++ b/charts/posthog/templates/events-deployment.yaml @@ -174,4 +174,5 @@ spec: {{- end }} initContainers: {{- include "_snippet-initContainers-wait-for-service-dependencies" . | indent 8 }} + {{- include "_snippet-initContainers-wait-for-migrations" . | indent 8 }} {{- end }} From 75cfed4bb89bd88cdf9142c6e6cdb20d595c5d5d Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 19 Oct 2023 09:50:31 +0100 Subject: [PATCH 04/10] more logs --- .github/workflows/test-kubetest.yaml | 3 ++- charts/posthog/templates/events-deployment.yaml | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-kubetest.yaml b/.github/workflows/test-kubetest.yaml index 6d46355e..f6468123 100644 --- a/.github/workflows/test-kubetest.yaml +++ b/.github/workflows/test-kubetest.yaml @@ -64,7 +64,8 @@ jobs: export PYTHONUNBUFFERED=1 python -m pytest ${{ matrix.test-file }} \ -s \ - --maxfail=1 + --maxfail=1 \ + --kube-log-level=info - name: Emit namespace report uses: jupyterhub/action-k8s-namespace-report@v1 diff --git a/charts/posthog/templates/events-deployment.yaml b/charts/posthog/templates/events-deployment.yaml index afe2e04e..a3d9df4f 100644 --- a/charts/posthog/templates/events-deployment.yaml +++ b/charts/posthog/templates/events-deployment.yaml @@ -174,5 +174,4 @@ spec: {{- end }} initContainers: {{- include "_snippet-initContainers-wait-for-service-dependencies" . | indent 8 }} - {{- include "_snippet-initContainers-wait-for-migrations" . | indent 8 }} {{- end }} From 91af21efeea213147fdfc26b2df1cf48eb2d7985 Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 19 Oct 2023 10:01:06 +0100 Subject: [PATCH 05/10] log --- ci/kubetest/helpers/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/kubetest/helpers/utils.py b/ci/kubetest/helpers/utils.py index cfe72dcb..75c71f29 100644 --- a/ci/kubetest/helpers/utils.py +++ b/ci/kubetest/helpers/utils.py @@ -110,6 +110,7 @@ def wait_for_pods_to_be_ready(kube, labels=None, expected_count=None, namespace= # NOTE 2: the test is currently a bit flaky likely due to # liveness/readiness checks. I'm bumping the expected restart # count from 0 to 2 for now. + print([c.get_logs() for c in pod.get_containers()]) assert get_pod_restart_count(pod) < 2, f"Detected too many restarts in pod {pod.obj.metadata.name}" # Note we assume that if "job-name" is a label then it is a job pod. From a8c1688017f87ae7d20660be93b443927c1f3f40 Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 19 Oct 2023 10:01:24 +0100 Subject: [PATCH 06/10] logs --- ci/kubetest/helpers/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/kubetest/helpers/utils.py b/ci/kubetest/helpers/utils.py index 75c71f29..8787bd28 100644 --- a/ci/kubetest/helpers/utils.py +++ b/ci/kubetest/helpers/utils.py @@ -110,7 +110,9 @@ def wait_for_pods_to_be_ready(kube, labels=None, expected_count=None, namespace= # NOTE 2: the test is currently a bit flaky likely due to # liveness/readiness checks. I'm bumping the expected restart # count from 0 to 2 for now. + print("container logs start") print([c.get_logs() for c in pod.get_containers()]) + print("container logs finish") assert get_pod_restart_count(pod) < 2, f"Detected too many restarts in pod {pod.obj.metadata.name}" # Note we assume that if "job-name" is a label then it is a job pod. From 89684dbe101faf865e5560d98a7aece6670449d6 Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 19 Oct 2023 10:08:05 +0100 Subject: [PATCH 07/10] ignore fail logs --- ci/kubetest/helpers/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/kubetest/helpers/utils.py b/ci/kubetest/helpers/utils.py index 8787bd28..1b63ee87 100644 --- a/ci/kubetest/helpers/utils.py +++ b/ci/kubetest/helpers/utils.py @@ -111,7 +111,10 @@ def wait_for_pods_to_be_ready(kube, labels=None, expected_count=None, namespace= # liveness/readiness checks. I'm bumping the expected restart # count from 0 to 2 for now. print("container logs start") - print([c.get_logs() for c in pod.get_containers()]) + try: + print([c.get_logs() for c in pod.get_containers()]) + except Exception: + pass print("container logs finish") assert get_pod_restart_count(pod) < 2, f"Detected too many restarts in pod {pod.obj.metadata.name}" From c81db1431732fe82b7ac5abb1597bd977d8414c5 Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 19 Oct 2023 12:18:13 +0100 Subject: [PATCH 08/10] debug --- charts/posthog/templates/events-deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/posthog/templates/events-deployment.yaml b/charts/posthog/templates/events-deployment.yaml index a3d9df4f..8d553bb5 100644 --- a/charts/posthog/templates/events-deployment.yaml +++ b/charts/posthog/templates/events-deployment.yaml @@ -105,6 +105,8 @@ spec: value: '1' - name: IS_BEHIND_PROXY value: '1' + - name: DEBUG + value: 'true' {{- if eq .Values.web.secureCookies false }} - name: SECURE_COOKIES value: '0' From 9eab7f56f68c7f7ea81f6fba4e04ef6333db889a Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 19 Oct 2023 12:28:08 +0100 Subject: [PATCH 09/10] skip migration checks on events/recordings --- ci/values/k3s.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ci/values/k3s.yaml b/ci/values/k3s.yaml index b5274522..711f73bc 100644 --- a/ci/values/k3s.yaml +++ b/ci/values/k3s.yaml @@ -10,6 +10,16 @@ ingress: web: secureCookies: false +events: + env: + - name: POSTHOG_SKIP_MIGRATION_CHECKS + value: "true" + +recordings: + env: + - name: POSTHOG_SKIP_MIGRATION_CHECKS + value: "true" + # Use small PVC in CI clickhouse: persistence: From 56ecf41c6585f6369ae9973455904e2b68ee1de7 Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 19 Oct 2023 12:33:19 +0100 Subject: [PATCH 10/10] skip migration checks everywhere --- ci/values/k3s.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/values/k3s.yaml b/ci/values/k3s.yaml index 711f73bc..e59c250f 100644 --- a/ci/values/k3s.yaml +++ b/ci/values/k3s.yaml @@ -10,6 +10,10 @@ ingress: web: secureCookies: false +env: + - name: POSTHOG_SKIP_MIGRATION_CHECKS + value: "true" + events: env: - name: POSTHOG_SKIP_MIGRATION_CHECKS