Merge pull request #13 from geobeyond/#5_kube_configs

#5 kube configs

.dockerignore

@@ -0,0 +1,7 @@
+.cookiecutter.json
+.pytest_cache/
+dist/
+.k3d/
+.github
+examples/
+tests/

.github/workflows/release.yml

@@ -69,11 +69,3 @@ jobs:
           user: __token__
           password: ${{ secrets.TEST_PYPI_TOKEN }}
           repository_url: https://test.pypi.org/legacy/
-
-      - name: Publish the release notes
-        uses: release-drafter/[email protected]
-        with:
-          publish: ${{ steps.check-version.outputs.tag != '' }}
-          tag: ${{ steps.check-version.outputs.tag }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -5,7 +5,6 @@ repos:
       - id: trailing-whitespace
       - id: check-added-large-files
       - id: check-toml
-      - id: check-yaml
       - id: end-of-file-fixer
   - repo: local
     hooks:

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8
+FROM python:3.10
 
 WORKDIR /app
 
@@ -7,8 +7,9 @@ RUN poetry config virtualenvs.create false --local
 COPY ./fastflows /app/fastflows
 COPY ./flows /app/flows
 RUN ls /app
+COPY ./README.md /app
 COPY ./pyproject.toml /app
 
 RUN poetry install --no-dev
 
-CMD ["fastflows", "server"]
+CMD ["uvicorn", "fastflows.main:app", "--host", "0.0.0.0", "--port", "5010"]

docker/README.md

@@ -0,0 +1,94 @@
+**Note:** Run all commands from base repo dir
+
+### Build Fastflows Docker Image
+
+```console
+
+    docker build . -f ./docker/Dockerfile -t geobeyond/fastflows:latest
+
+```
+
+### Run Fastflows on Kube cluster
+
+Steps to run:
+
+1. **(For local env only)** Install minikube (will be used to create & manage local cluster): https://minikube.sigs.k8s.io/docs/start/
+
+2. Install kubectl if you don't have it - https://kubernetes.io/docs/tasks/tools/
+
+3. **(For local env only)** Create kubernetes cluster:
+
+```console
+
+    minikube start
+
+    # Important!!!
+    # On MacOS to make Ingress work you should run minikube with hyperkit:
+
+    minikube start --driver=hyperkit
+
+
+```
+
+4. **(For local env only)**
+
+Enable Ingress & host resolving (full explanation here)
+
+```console
+
+    minikube addons enable ingress
+
+```
+
+Get minikube cluster ip
+
+```console
+
+    minikube ip
+
+```
+
+Add domains to /etc/hosts linked to minikube cluster ip.
+
+Add this to /etc/hosts file:
+
+```console
+
+    your-mini-kube-ip fastflows.geo
+    your-mini-kube-ip prefect.geo
+
+    # for example (where 192.168.64.2 - minikube ip):
+    # 192.168.64.2 fastflows.geo
+    # 192.168.64.2 prefect.geo
+
+```
+
+5. Apply Kube manifests to the cluster
+
+```console
+
+    kubectl apply -f docker/kube-infra
+
+```
+
+5. Check pods statuses & wait until all pods will not be in status "Running"
+
+```console
+
+    kubectl get pods
+
+```
+
+Now you can access service by links:
+
+http://fastflows.geo/ & http://prefect.geo/
+
+### Tips
+
+1.  **(For local env only)** If for some reason you need to delete cluster, use:
+
+```console
+
+    minikube delete
+
+```

docker/docker-compose.yml

@@ -22,15 +22,15 @@ services:
   #  - prefect
   #  - db
   prefect:
-    image: prefecthq/prefect:2.0b7-python3.9
+    image: prefecthq/prefect:2.0b8-python3.9
     entrypoint: prefect orion start --host 0.0.0.0
     environment:
       - PREFECT_ORION_DATABASE_CONNECTION_URL=postgresql+asyncpg://postgres:postgres@db:5432/orion
     ports:
       - 4200:4200
 
   prefect_work_queue:
-    image: prefecthq/prefect:2.0b7-python3.9
+    image: prefecthq/prefect:2.0b8-python3.9
     entrypoint: prefect work-queue create base-queue
     environment:
       - PREFECT_API_URL=http://prefect:4200/api
@@ -40,7 +40,7 @@ services:
       - db
 
   prefect_agent:
-    image: prefecthq/prefect:2.0b7-python3.9
+    image: prefecthq/prefect:2.0b8-python3.9
     entrypoint: prefect agent start base-queue
     environment:
       - PREFECT_API_URL=http://prefect:4200/api
@@ -61,24 +61,24 @@ services:
       - "5432:5432"
     volumes:
       - db:/var/lib/postgresql/data
-  keycloak:
-    image: jboss/keycloak:latest
-    ports:
-      - 8282:8080
-    environment:
-      - KEYCLOAK_USER=admin
-      - KEYCLOAK_PASSWORD=admin
-  opa:
-    image: openpolicyagent/opa:latest
-    ports:
-      - 8383:8181
-    command:
-      - "run"
-      - "--server"
-      - "--log-level=debug"
-      - "/policy/auth.rego"
-    volumes:
-      - ./policy:/policy
+  #keycloak:
+  #  image: jboss/keycloak:latest
+  #  ports:
+  #    - 8282:8080
+  # environment:
+  #    - KEYCLOAK_USER=admin
+  #    - KEYCLOAK_PASSWORD=admin
+  #opa:
+  #  image: openpolicyagent/opa:latest
+  #  ports:
+  #    - 8383:8181
+  #  command:
+  #    - "run"
+  #    - "--server"
+  #    - "--log-level=debug"
+  #    - "/policy/auth.rego"
+  #  volumes:
+  #    - ./policy:/policy
 volumes:
   db:
     driver: local

docker/kube-infra/fastflows.yaml

@@ -0,0 +1,93 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fastflows
+spec:
+  selector:
+    matchLabels:
+      app: fastflows
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: fastflows
+    spec:
+      initContainers:
+        - name: check-orion-is-ready
+          image: curlimages/curl
+          command:
+            [
+              "sh",
+              "-c",
+              'while [ `curl -fsS http://localhost:4200/" --silent --output /dev/null"` -ne 200 ]; do sleep 2; done',
+            ]
+        - name: orion-queue
+          image: prefecthq/prefect:2.0b8-python3.9
+          command:
+            [
+              "sh",
+              "-c",
+              'prefect work-queue create default-queue; r=$?; [ "$r" -gt 0 ] && exit 0; exit 0',
+            ]
+          imagePullPolicy: "IfNotPresent"
+          ports:
+            - containerPort: 4200
+          env:
+            - name: PREFECT_API_URL
+              value: http://orion:4200/api
+      containers:
+        - name: main
+          image: geobeyond/fastflows:latest
+          imagePullPolicy: "IfNotPresent"
+          ports:
+            - containerPort: 5010
+          env:
+            - name: PREFECT_URI
+              value: http://orion:4200
+            - name: FASTFLOWS_PORT
+              value: "5010"
+          command: ["fastflows", "server"]
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: fastflows
+  labels:
+    app: fastflows
+spec:
+  ports:
+    - name: http
+      protocol: TCP
+      port: 5010
+      targetPort: 5010
+  selector:
+    app: fastflows
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: default
+  name: flow-runner
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "pods/log", "pods/status"]
+    verbs: ["get", "watch", "list"]
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: flow-runner-role-binding
+  namespace: default
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: default
+roleRef:
+  kind: Role
+  name: flow-runner
+  apiGroup: rbac.authorization.k8s.io

docker/kube-infra/ingress.yaml

@@ -0,0 +1,28 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: fastflows-cluster
+  annotations:
+    kubernetes.io/ingress.class: nginx
+spec:
+  rules:
+    - host: prefect.geo
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: orion
+                port:
+                  number: 4200
+    - host: fastflows.geo
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: fastflows
+                port:
+                  number: 5010