-
Notifications
You must be signed in to change notification settings - Fork 1
/
tiltfile
194 lines (171 loc) · 9.57 KB
/
tiltfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
# Welcome to Tilt!
# More info: https://docs.tilt.dev
print("""
-----------------------------------------------------------------
✨ Let's Build a Data Platform! ✨
-----------------------------------------------------------------
""".strip())
S3_ACCESS_KEY = "minio-sa"
S3_SECRET_KEY = "minio123"
GITHUB_ACCESS_TOKEN = os.getenv("GITHUB_ACCESS_TOKEN")
# default_registry(
# 'localhost:5005',
# host_from_cluster='ctlptl-registry:5000'
# )
# Minikube
# # look for a local registry
# local_registry = local('minikube -p minikube service -n kube-system registry --url', quiet=True).strip()
# if local_registry:
# default_registry(local_registry)
# Enable Minikube addons
# if k8s_context() == 'minikube':
# local("minikube addons enable metrics-server")
# local("minikube addons enable ingress")
# local("minikube addons enable registry")
# Get the Minikube registry IP and port
# minikube_registry_ip = local("minikube ip", quiet=True).strip()
# minikube_registry_port = local("kubectl get svc registry -n kube-system -o=jsonpath='{.spec.ports[0].nodePort}'", quiet=True).strip()
# Build Podman Open Container Initiative (OCI) image
# custom_build('datagen', 'podman build -f Dockerfiles/dockerfile.datagen -t datagen:latest -t $EXPECTED_REF .', '.', skips_local_docker=True)
# custom_build('dbt-argo', 'podman build -f Dockerfiles/dockerfile.dbt -t dbt-argo:latest -t $EXPECTED_REF .', '.', skips_local_docker=True)
# custom_build('dbz_connect', 'podman build -f Dockerfiles/dockerfile.debezium -t debezium:latest -t $EXPECTED_REF .', '.', skips_local_docker=True)
# custom_build('go_loader', 'podman build -f Dockerfiles/dockerfile.go_loader -t go_loader:latest -t $EXPECTED_REF .', '.',skips_local_docker=True)
# custom_build('psql_db', 'podman build -f Dockerfiles/dockerfile.psql_db -t psql_db:latest -t $EXPECTED_REF .', '.', skips_local_docker=True)
# custom_build('py_deequ', 'podman build -f Dockerfiles/dockerfile.deequ -t deequ:latest -t $EXPECTED_REF .', '.',skips_local_docker=True)
# custom_build('py_gx', 'podman build -f Dockerfiles/dockerfile.gx -t gx:latest -t $EXPECTED_REF .', '.',skips_local_docker=True)
# Build Docker Container Images
docker_build('evidence_dev', '.', dockerfile='Dockerfiles/Dockerfile.evidence_dev')
docker_build('datagen', '.', dockerfile='Dockerfiles/Dockerfile.datagen')
docker_build('dbz_connect', '.', dockerfile='Dockerfiles/Dockerfile.debezium')
docker_build('go_loader', '.', dockerfile='Dockerfiles/Dockerfile.go_loader')
docker_build('psql_db', '.', dockerfile='Dockerfiles/Dockerfile.psql_db')
# docker_build('dbt_argo', '.', dockerfile='Dockerfiles/Dockerfile.dbt', target='final', match_in_env_vars=True,extra_tag=["dbt-argo:local", "ctlptl-registry:5000/dbt-argo:latest"]) # NOTE: container is now built with github actions
# docker_build('py_deequ', '.', dockerfile='Dockerfiles/Dockerfile.deequ', target='final', match_in_env_vars=True,extra_tag=["deequ:local"]) # NOTE: container is now built with github actions
# docker_build('py_gx', '.', dockerfile='Dockerfiles/Dockerfile.gx', target='final' ,match_in_env_vars=True,extra_tag=["gx:local"]) # NOTE: container is now built with github actions
# Customize Kubernetes resources
k8s_yaml(kustomize("./ops/dev-stack"))
k8s_resource('postgres-db', port_forwards=[5432], labels="psql_db")
k8s_resource('go-loader', port_forwards=[8000], labels="go_loader")
# k8s_resource('py-gx', labels="argo_workflow", auto_init=False, trigger_mode=TRIGGER_MODE_MANUAL)# dummy resource to trigger gx container build
# k8s_resource('py-deequ', labels="argo_workflow", auto_init=False, trigger_mode=TRIGGER_MODE_MANUAL )# dummy resource to trigger gx container build
# k8s_resource('dbt-argo', labels="argo_workflow", auto_init=False, trigger_mode=TRIGGER_MODE_MANUAL) # dummy resource to trigger gx container build
k8s_resource('evidence-dev', labels="analytics", port_forwards=['4000'])
# Install Prometheus Operator
# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
load('ext://helm_remote', 'helm_remote')
helm_remote('kube-prometheus-stack', repo_url='https://prometheus-community.github.io/helm-charts')
k8s_resource("kube-prometheus-stack-prometheus-node-exporter", labels="prometheus")
k8s_resource("kube-prometheus-stack-grafana", port_forwards=[3000], labels="prometheus")
k8s_resource("kube-prometheus-stack-operator", port_forwards=[9090], labels="prometheus")
k8s_resource("kube-prometheus-stack-kube-state-metrics", labels="prometheus")
k8s_resource("kube-prometheus-stack-admission-create", labels="prometheus")
k8s_resource("kube-prometheus-stack-admission-patch", labels="prometheus")
# Install Minio Object storage
# https://github.com/minio/minio/tree/master/helm/minio
load('ext://namespace', 'namespace_create')
namespace_create('minio')
load('ext://helm_remote', 'helm_remote')
values = [
"./ops/dev-stack/minio/values.yaml",
]
helm_remote('minio', namespace='minio', repo_url='https://charts.min.io/', values=values, version='5.0.8')
k8s_resource("minio", port_forwards=['9001','9000'], labels="object")
k8s_resource("minio-post-job", labels="object")
# Install Redpanda Kafka
# https://github.com/redpanda-data-blog/2022-redpanda-duckdb/blob/main/docker-compose.yml
# https://github.com/redpanda-data/helm-charts
load('ext://namespace', 'namespace_create')
namespace_create('kafka')
load('ext://helm_remote', 'helm_remote')
values = [
"./ops/dev-stack/redpanda/values.yaml"
]
helm_remote('redpanda', namespace='kafka', repo_url='https://charts.redpanda.com/', values=values, version='4.0.13')
k8s_resource("redpanda-configuration", labels="kafka",resource_deps=["redpanda"])
## Redpanda Console
load('ext://helm_remote', 'helm_remote')
values = [
"./ops/dev-stack/redpanda/console/values.yaml",
]
helm_remote('console', namespace='kafka', repo_url='https://charts.redpanda.com/', values=values)
k8s_resource("redpanda-configuration", labels="kafka",resource_deps=["redpanda"])
k8s_resource("console",port_forwards=['8080'], labels="kafka",resource_deps=["redpanda" ])
k8s_resource("redpanda-post-upgrade", labels="kafka", resource_deps=["redpanda"],)
k8s_resource("redpanda", labels="kafka", resource_deps=["minio"],)
k8s_resource("configure-kafka-connectors", labels="kafka")
k8s_resource("dbz-connect", port_forwards=["8083"], labels="kafka", resource_deps=["redpanda", "minio"])
k8s_yaml(blob("""
apiVersion: v1
kind: Secret
data:
.dockerconfigjson: eyJhdXRocyI6eyJjdGxwdGwtcmVnaXN0cnk6NTAwMCI6eyJ1c2VybmFtZSI6IiIsInBhc3N3b3JkIjoiIiwiZW1haWwiOiJyb290QGxvY2FsaG9zdCIsImF1dGgiOiIifX19Cg==
metadata:
name: ctlptl-regcred
namespace: argo
type: kubernetes.io/dockerconfigjson
---
apiVersion: v1
kind: Secret
metadata:
name: argo-workflow.service-account-token
namespace: argo
annotations:
kubernetes.io/service-account.name: argo-workflow
type: kubernetes.io/service-account-token
---
apiVersion: v1
kind: Secret
metadata:
name: argo-secrets
namespace: argo
type: Opaque
stringData:
S3_ARTIFACT_ACCESS_KEY: {s3_access}
S3_ARTIFACT_SECRET_KEY: {s3_secret}
GITHUB_ACCESS_TOKEN: {github_access_token}
""".format(
s3_access=S3_ACCESS_KEY,
s3_secret=S3_SECRET_KEY,
github_access_token=GITHUB_ACCESS_TOKEN,
)))
# Install Argo Workflows
load('ext://namespace', 'namespace_create')
namespace_create('argo')
load('ext://helm_remote', 'helm_remote')
values = [
"./ops/dev-stack/argo/values.yaml",
# "server=[serviceAccount.create=false,serviceAccount.name=argo-workflows,service.type=NodePort,extraArgs=[--secure=false]]", "--debug=true"
]
helm_remote('argo-workflows', namespace='argo', repo_url='https://argoproj.github.io/argo-helm', values=values, version='0.22.9')
# k8s_resource("argo-workflows-workflow-controller", port_forwards=['2746'])
k8s_resource("argo-workflows-server", port_forwards=['2746'], labels="argo")
k8s_resource("argo-workflows-workflow-controller", labels="argo")
k8s_resource("create-go-loader-user", labels="psql_db")
# Install Argo Events
load('ext://helm_remote', 'helm_remote')
values = [
"./ops/dev-stack/argo-events/values.yaml",
# "server=[serviceAccount.create=false,serviceAccount.name=argo-workflows,service.type=NodePort,extraArgs=[--secure=false]]", "--debug=true"
]
helm_remote('argo-events', namespace='argo', repo_url='https://argoproj.github.io/argo-helm', values=values)
k8s_resource("argo-events-controller-manager", labels="argo")
# Install Argo Cluster Workflows
k8s_yaml(helm('./ops/dev-stack/argo/helm/workflows'))
script = """\
CREATE_TOKEN="Bearer $(kubectl create token argo-workflows-server -n argo)"
echo $CREATE_TOKEN > ./.argo-token.txt
"""
local_resource("create-argo-server-token", script, resource_deps=["argo-workflows-server"], labels="argo")
run_argo_workflow = """\
eval ARGO_TOKEN=$(cat ./.argo-token.txt)
eval ARGO_IP=$(kubectl get pods --namespace=argo -l="app.kubernetes.io/name=argo-workflows-server" -o=jsonpath="{range .items[*]}{.status.podIP}")
eval ARGO_TASK_NAME=$(argo --namespace=argo submit --token "$ARGO_TOKEN" --output=name --from workflowtemplate/data-workflow --parameter ip="$ARGO_IP")
echo ARGO_IP=$ARGO_IP
echo ARGO_TASK_NAME=$ARGO_TASK_NAME
argo --namespace=argo logs $ARGO_TASK_NAME --follow
argo --namespace=argo get $ARGO_TASK_NAME --output=json | jq -r '.status.phase' | grep -q 'Succeeded'
"""
local_resource("run-argo-workflow", run_argo_workflow, resource_deps=["argo-workflows-server", "create-argo-server-token"], labels="argo", trigger_mode=TRIGGER_MODE_MANUAL, auto_init=False)
# More info: https://github.com/tilt-dev/tilt-extensions
secret_settings(disable_scrub=True)
load('ext://git_resource', 'git_checkout')