chore: make local setup idempotent

Signed-off-by: Spazzy <[email protected]>

.circleci/config.yml

@@ -40,7 +40,7 @@ jobs:
               kubectl config use-context kind-test
       - run:
           name: Run tests
-          command: cd devops && ./integration-test.sh
+          command: cd devops && make integration-tests
 
 workflows:
   version: 2

devops/Makefile

@@ -0,0 +1,54 @@
+DB_VERSION=v21.1.9
+PORT=5432
+DATABASE=chatroach
+
+dev: create-kind-cluster setup-dev-secret setup-kind-registry bootstrap-fly
+
+start-testrunner:
+	@kubectl apply -f testing/facebot.yaml
+	@kubectl wait --for=condition=available deployment/facebot --timeout 5m
+	@kubectl delete job testrunner || true
+	@envsubst < testing/testrunner.yaml | kubectl apply -f -
+
+.PHONY: check-test-status
+check-test-status:
+	$(eval SUCCESS=$(shell kubectl get job testrunner -o jsonpath='{.status.succeeded}'))
+	@if [ "$(SUCCESS)" != "1" ]; then\
+			exit 1; \
+		fi
+	@echo "Test Succesful"
+
+.PHONY: integration-tests
+integration-tests:	create-integration-secret bootstrap-fly start-testrunner
+	@kubectl wait --for=condition=complete job/testrunner --timeout 10m
+	@kubectl logs -l app=testrunner --tail -1
+	$(MAKE) check-test-status
+
+create-kind-cluster:
+	@kind create cluster --config dev/kind-cluster.yaml
+
+create-integration-secret:
+	@kubectl create secret generic bot-envs --from-env-file=./testing/.test-env || true
+
+create-dev-secret:
+	@kubectl create secret generic bot-envs --from-env-file=./dev/.env || true
+
+bootstrap-fly:
+	@bash ./scripts/bootstrap-fly.sh
+
+setup-kind-registry:
+	@bash ./dev/kind-with-registry.sh
+
+test-db:
+	@docker stop vlab-cockroach && docker rm vlab-cockroach
+	@docker run --name vlab-cockroach -d \
+		-p $(PORT):26257 \
+		cockroachdb/cockroach:$(DB_VERSION) start-single-node --insecure
+	@cat ./sql/* | docker run -i \
+		--net=host \
+		--rm cockroachdb/cockroach:$(DB_VERSION) \
+		sql --insecure --host localhost --port $(PORT) --database chatroach
+	@echo "set sql_safe_updates = false;" | docker run -i \
+		--net=host \
+		--rm cockroachdb/cockroach:$(DB_VERSION) \
+		sql --insecure --host localhost --port $(PORT) --database chatroach

devops/README.md

@@ -1,7 +1,42 @@
 # FLY Deployment
 
+## Setting up dev environment
 
-## Updateing  Helm Charts
+You will need to install [kind][1] in order to run the full development setup
+
+
+Once installed you can simply run:
+
+```bash
+make dev
+```
+> This takes quite a while, you can watch the pods starting by running `kubectl get pods` in another terminal
+
+
+Once this is done you can run the integration tests by running:
+
+```bash
+make dev-integration-tests
+```
+
+This will create a test runner which you can follow by running
+
+```bash
+kubectl logs -f -l app=testrunner --tail -1
+```
+
+## Running Integration Tests
+**NOTE:** this is a resource intensive process that takes a fair amount of time
+please be patient
+
+In order to run integration tests you will need to have access to a kubernetes
+cluster. All the dependencies will be created in a single namespace
+
+```bash
+make integration-tests
+```
+
+## Updating  Helm Charts
 
 We use the OCI registry to configure helm charts therefore you should do the
 following to update a helm chart
@@ -50,3 +85,4 @@ cd vlab/
 helm dependency update
 ```
 
+[1]: https://kind.sigs.k8s.io/docs/user/quick-start/

devops/dev/kind-cluster.yaml

@@ -0,0 +1,15 @@
+---
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
+    endpoint = ["http://kind-registry:5000"]
+# We should have a kind cluster that has some workers
+# to speed up deployment
+nodes:
+- role: control-plane
+- role: worker
+- role: worker
+- role: worker
+- role: worker

devops/dev/kind-with-registry.sh

@@ -11,16 +11,6 @@ if [ "${running}" != 'true' ]; then
     registry:2
 fi
 
-# create a cluster with the local registry enabled in containerd
-cat <<EOF | kind create cluster --config=-
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-containerdConfigPatches:
-- |-
-  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
-    endpoint = ["http://${reg_name}:${reg_port}"]
-EOF
-
 # connect the registry to the cluster network
 # (the network may already be connected)
 docker network connect "kind" "${reg_name}" || true

devops/scripts/bootstrap-fly.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+set -e
+
+######################
+# add third party charts
+######################
+helm repo add bitnami https://charts.bitnami.com/bitnami
+helm repo add cockroachdb https://charts.cockroachdb.com/
+helm repo update
+
+######################
+# install db
+######################
+helm upgrade --install db cockroachdb/cockroachdb \
+  --values values/integrations/cdb.yaml \
+  --timeout 10m \
+  --wait
+
+######################
+# create database
+######################
+cat ./sql/* > tmp.sql
+
+# Note we skip this if it fails as 
+# currently migrations are not idempotent
+cat tmp.sql | kubectl run -i \
+  --rm cockroach-client \
+  --image=cockroachdb/cockroach:v2.1.4 \
+  --restart=Never \
+  --command -- ./cockroach sql --insecure --host db-cockroachdb-public || true
+rm -f tmp.sql
+
+######################
+# install kafka
+######################
+helm upgrade --install kafka bitnami/kafka \
+  --values values/integrations/kafka.yaml \
+  --timeout 10m0s \
+  --wait
+
+######################
+# install fly
+######################
+helm upgrade --install \
+  fly vlab \
+  -f values/integrations/fly.yaml \
+  --timeout 10m0s \
+  --wait
+

devops/scripts/clear-data.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+kubectl scale statefulset fly-cockroachdb --replicas=0
+kubectl delete pvc datadir-fly-cockroachdb-0
+kubectl scale statefulset fly-cockroachdb --replicas=1

devops/scripts/run-individual-migrations.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# This is just a file to run each sql migration individual in order to pick up 
+# which migration failed
+set -e
+
+FILES="sql/*.sql"
+for f in $FILES
+do
+  echo "Processing $f file..."
+  # take action on each file. $f store current file name
+  cat "$f" | kubectl run -i   \
+    --rm cockroach-client   \
+    --image=cockroachdb/cockroach:v21.1.9 \
+    --restart=Never \
+    --command -- ./cockroach sql --insecure --host fly-cockroachdb-public
+done

devops/sql/migrate-04.sql

@@ -3,11 +3,11 @@
 -- Add created columns
 ALTER TABLE chatroach.states ADD COLUMN fb_error_code varchar AS (state_json->'error'->>'code') STORED;
 ALTER TABLE chatroach.states ADD COLUMN timeout_date TIMESTAMPTZ AS (CASE
-      WHEN state_json->'wait'->>'type' = 'timeout' THEN (CEILING((state_json->>'waitStart')::INT/1000)::INT::TIMESTAMPTZ + (state_json->'wait'->>'value')::INTERVAL)
+      WHEN state_json->'wait'->>'type' = 'timeout' 
+      THEN (timezone('UCT', (CEILING((state_json->>'waitStart')::INT/1000)::INT::TIMESTAMP + (state_json->'wait'->>'value')::INTERVAL)))
       ELSE NULL
 END) STORED;
 
-
 -- Index those columns for queries!
 -- TODO: Should include updated, for time window filtering in grafana!!
 CREATE INDEX ON chatroach.states (current_state, fb_error_code) STORING (state_json);

devops/sql/migrate-05.sql

@@ -3,7 +3,7 @@
 -- fix weird farmhash int and make reasonable primary key!
 ALTER TABLE chatroach.messages ADD COLUMN hsh INT AS (fnv64a(content)) STORED NOT NULL;
 BEGIN;
-ALTER TABLE chatroach.messages DROP CONSTRAINT "primary";
+ALTER TABLE chatroach.messages DROP CONSTRAINT IF EXISTS "primary";
 ALTER TABLE chatroach.messages ADD CONSTRAINT "primary" PRIMARY KEY (hsh, userid);
 COMMIT;
 

devops/sql/migrate-17.sql

@@ -7,11 +7,15 @@ ALTER TABLE chatroach.states DROP COLUMN timeout_date;
 SET sql_safe_updates=TRUE;
 
 -- CREATE NEW INDEX/COLUMN FOR TIMEOUTS
-ALTER TABLE chatroach.states ADD COLUMN timeout_date TIMESTAMPTZ AS (CASE
-      WHEN state_json->'wait'->>'type' = 'timeout' AND state_json->'wait'->'value'->>'type' = 'absolute' THEN (state_json->'wait'->'value'->>'timeout')::TIMESTAMPTZ
-      WHEN state_json->'wait'->>'type' = 'timeout' AND state_json->'wait'->'value'->>'type' = 'relative' THEN (CEILING((state_json->>'waitStart')::INT/1000)::INT::TIMESTAMPTZ + (state_json->'wait'->'value'->>'timeout')::INTERVAL)
-      WHEN state_json->'wait'->>'type' = 'timeout' THEN (CEILING((state_json->>'waitStart')::INT/1000)::INT::TIMESTAMPTZ + (state_json->'wait'->>'value')::INTERVAL)
-      ELSE NULL
+ALTER TABLE chatroach.states ADD COLUMN timeout_date TIMESTAMPTZ AS (
+CASE
+  WHEN state_json->'wait'->>'type' = 'timeout' AND state_json->'wait'->'value'->>'type' = 'absolute' 
+    THEN (timezone('UCT',parse_timestamp(state_json->'wait'->'value'->>'timeout')))
+  WHEN state_json->'wait'->>'type' = 'timeout' AND state_json->'wait'->'value'->>'type' = 'relative' 
+    THEN  (timezone('UCT',(CEILING((state_json->>'waitStart')::INT/1000)::INT::TIMESTAMP + (state_json->'wait'->'value'->>'timeout')::INTERVAL)))
+  WHEN state_json->'wait'->>'type' = 'timeout' 
+    THEN (timezone('UCT', (CEILING((state_json->>'waitStart')::INT/1000)::INT::TIMESTAMP + (state_json->'wait'->>'value')::INTERVAL)))
+  ELSE NULL
 END) STORED;
 
 CREATE INDEX ON chatroach.states (current_state, timeout_date) STORING (state_json);