From 7f1ea68246d1923ac307d974d5cb00a6a2b55f22 Mon Sep 17 00:00:00 2001 From: xnuinside Date: Tue, 12 Jul 2022 17:21:17 +0300 Subject: [PATCH 1/5] update version of Prefect --- .github/workflows/release.yml | 8 ------- docker/docker-compose.yml | 42 +++++++++++++++++------------------ pyproject.toml | 2 +- 3 files changed, 22 insertions(+), 30 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aae2d97..e179917 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,11 +69,3 @@ jobs: user: __token__ password: ${{ secrets.TEST_PYPI_TOKEN }} repository_url: https://test.pypi.org/legacy/ - - - name: Publish the release notes - uses: release-drafter/release-drafter@v5.15.0 - with: - publish: ${{ steps.check-version.outputs.tag != '' }} - tag: ${{ steps.check-version.outputs.tag }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 8d04407..249b81b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -22,7 +22,7 @@ services: # - prefect # - db prefect: - image: prefecthq/prefect:2.0b7-python3.9 + image: prefecthq/prefect:2.0b8-python3.9 entrypoint: prefect orion start --host 0.0.0.0 environment: - PREFECT_ORION_DATABASE_CONNECTION_URL=postgresql+asyncpg://postgres:postgres@db:5432/orion @@ -30,7 +30,7 @@ services: - 4200:4200 prefect_work_queue: - image: prefecthq/prefect:2.0b7-python3.9 + image: prefecthq/prefect:2.0b8-python3.9 entrypoint: prefect work-queue create base-queue environment: - PREFECT_API_URL=http://prefect:4200/api @@ -40,7 +40,7 @@ services: - db prefect_agent: - image: prefecthq/prefect:2.0b7-python3.9 + image: prefecthq/prefect:2.0b8-python3.9 entrypoint: prefect agent start base-queue environment: - PREFECT_API_URL=http://prefect:4200/api @@ -61,24 +61,24 @@ services: - "5432:5432" volumes: - db:/var/lib/postgresql/data - keycloak: - image: jboss/keycloak:latest - ports: - - 8282:8080 - environment: - - KEYCLOAK_USER=admin - - KEYCLOAK_PASSWORD=admin - opa: - image: openpolicyagent/opa:latest - ports: - - 8383:8181 - command: - - "run" - - "--server" - - "--log-level=debug" - - "/policy/auth.rego" - volumes: - - ./policy:/policy + #keycloak: + # image: jboss/keycloak:latest + # ports: + # - 8282:8080 + # environment: + # - KEYCLOAK_USER=admin + # - KEYCLOAK_PASSWORD=admin + #opa: + # image: openpolicyagent/opa:latest + # ports: + # - 8383:8181 + # command: + # - "run" + # - "--server" + # - "--log-level=debug" + # - "/policy/auth.rego" + # volumes: + # - ./policy:/policy volumes: db: driver: local diff --git a/pyproject.toml b/pyproject.toml index 77adc95..004c024 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fastflows" -version = "0.1.0.dev.1657623062" +version = "0.1.0.dev.1657629308" description = "FastFlows is a FastAPI server & command line tool to comunicate with Prefect 2.0 as a Workflow manager to deploy, run, track flows and more." authors = ["Francesco Bartoli ", "Iuliia Volkova "] license = "MIT" From 34084a0d5f474738acee850d93e4ddf9a7f433a6 Mon Sep 17 00:00:00 2001 From: xnuinside Date: Mon, 18 Jul 2022 23:37:20 +0300 Subject: [PATCH 2/5] kube configs --- .dockerignore | 7 +++ .pre-commit-config.yaml | 1 - Makefile | 39 ++++++++++++ docker/Dockerfile | 7 ++- docker/README.md | 79 ++++++++++++++++++++++++ docker/kube-infra/fastflows.yaml | 68 +++++++++++++++++++++ docker/kube-infra/ingress.yaml | 31 ++++++++++ docker/kube-infra/postgres.yaml | 87 +++++++++++++++++++++++++++ docker/kube-infra/prefect-queue.yaml | 34 +++++++++++ docker/kube-infra/prefect.yaml | 90 ++++++++++++++++++++++++++++ fastflows/config/app.py | 4 +- fastflows/main.py | 1 + fastflows/providers/utils.py | 8 ++- 13 files changed, 450 insertions(+), 6 deletions(-) create mode 100644 .dockerignore create mode 100644 Makefile create mode 100644 docker/README.md create mode 100644 docker/kube-infra/fastflows.yaml create mode 100644 docker/kube-infra/ingress.yaml create mode 100644 docker/kube-infra/postgres.yaml create mode 100644 docker/kube-infra/prefect-queue.yaml create mode 100644 docker/kube-infra/prefect.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fd092b1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.cookiecutter.json +.pytest_cache/ +dist/ +.k3d/ +.github +examples/ +tests/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f40fcd3..45e4704 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,6 @@ repos: - id: trailing-whitespace - id: check-added-large-files - id: check-toml - - id: check-yaml - id: end-of-file-fixer - repo: local hooks: diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..211c420 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ + +export CLUSTER_NAME=fastflows-cluster +export KUBECONFIG=.k3d/kubeconfig-${CLUSTER_NAME}.yaml + +## create cluster and install minio and prefect +kubes: cluster kubes-prefect + +## create k3s cluster +cluster: +# enable ephmeral containers for profiling + k3d cluster create ${CLUSTER_NAME} --registry-create ${CLUSTER_NAME}-registry:0.0.0.0:5550 \ + -p 4200:80@loadbalancer -p 9000:9000@loadbalancer -p 9001:9001@loadbalancer \ + --k3s-arg '--kube-apiserver-arg=feature-gates=EphemeralContainers=true@server:*' \ + --k3s-arg '--kube-scheduler-arg=feature-gates=EphemeralContainers=true@server:*' \ + --k3s-arg '--kubelet-arg=feature-gates=EphemeralContainers=true@agent:*' \ + --wait + @echo "Probing until traefik CRDs are created (~60 secs)..." && export KUBECONFIG=$$(k3d kubeconfig write ${CLUSTER_NAME}) && \ + while : ; do kubectl get crd ingressroutes.traefik.containo.us > /dev/null && break; sleep 10; done + @echo -e "\nTo use your cluster set:\n" + @echo "export KUBECONFIG=$$(k3d kubeconfig write ${CLUSTER_NAME})" + +## install prefect api and agent into kubes cluster +kubes-prefect: export PREFECT_API_URL=http://localhost:4200/api +kubes-prefect: + kubectl apply -f docker/kube-infra/postgres.yaml + kubectl wait pod --for=condition=ready --timeout=120s -lapp=postgres + kubectl apply -f docker/kube-infra/prefect.yaml + kubectl apply -f docker/kube-infra/fastflows.yaml + kubectl apply -f docker/kube-infra/ingress.yaml + kubectl apply -f docker/kube-infra/prefect-queue.yaml + kubectl wait pod --for=condition=ready --timeout=120s -lapp=orion + +## show kube logs for orion +kube-logs-orion: + kubectl logs -lapp=orion --all-containers + +## show kube logs for orion +kube-logs-fastflows: + kubectl logs -lapp=fastflows --all-containers diff --git a/docker/Dockerfile b/docker/Dockerfile index 6b63721..256f381 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8 +FROM python:3.10 WORKDIR /app @@ -7,8 +7,11 @@ RUN poetry config virtualenvs.create false --local COPY ./fastflows /app/fastflows COPY ./flows /app/flows RUN ls /app +COPY ./README.md /app COPY ./pyproject.toml /app RUN poetry install --no-dev -CMD ["fastflows", "server"] +EXPOSE 80 + +CMD ["uvicorn", "fastflows.main:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..50d9333 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,79 @@ +Run all commands from base source dir + +### Build Fastflows Docker Image + +```console + + docker build . -f ./docker/Dockerfile -t geobeyond/fastflows:latest + +``` + +### Run Fastflows on Kube cluster + +Steps to run: + +1. Create dir '/etc/data/postgresql' it will be used to store persistent PostgreSQL data + +2. **(For local env only)** Install k3d.io (will be used as K8s local solution): https://k3d.io/v5.4.4/#installation + +3. Install kubectl if you don't have it - https://kubernetes.io/docs/tasks/tools/ + +4. Create k3d cluster with an image registry, minio (for remote storage), the prefect agent and api: + +```console + + make kubes + +``` + +P.S: If you will see error like 'Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "ingressroutes.traefik.containo.us" not found' it is okay - just wait. + +4. (**For local run only**) Import fastflows image + +To get possible run Fastflows in k3d locally you need to import image in cluster (by default k3d cannot find images from local env) + +```console + + k3d images import fastflows -c fastflows-cluster + +``` + +### Tips + +1. **(For local env only)** If for some reason you need to delete cluster, use: + +```console + + k3d cluster delete fastflows + +``` + +2. If you need to connect to PostgreSQL DB from localhose, first find host with + +```console + + kubectl get svc + +``` + +when use with information to connect wtih PgAdmin or psql + +3. If you cannot access from you local terminal Kube cluster & if you run command, for example: `kubectl get svc` see the error like: + 'The connection to the server localhost:8080 was refused - did you specify the right host or port?' + +Then in your terminal that you use define path to cluster config like this: + +```console + + export CLUSTER_NAME=fastflows-cluster + export KUBECONFIG=.k3d/kubeconfig-${CLUSTER_NAME}.yaml + +``` + +4. To check pods status: + +```console + + kubectl get pods + +``` diff --git a/docker/kube-infra/fastflows.yaml b/docker/kube-infra/fastflows.yaml new file mode 100644 index 0000000..5e08755 --- /dev/null +++ b/docker/kube-infra/fastflows.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fastflows +spec: + selector: + matchLabels: + app: fastflows + replicas: 2 + template: + metadata: + labels: + app: fastflows + spec: + containers: + - name: main + image: geobeyond/fastflows:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5010 + env: + - name: PREFECT_URI + value: http://orion:4200 + - name: FASTFLOWS_PORT + value: "5010" + command: ["fastflows", "server"] +--- +apiVersion: v1 +kind: Service +metadata: + name: fastflows + labels: + app: fastflows +spec: + ports: + - port: 5010 + protocol: TCP + selector: + app: fastflows + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: default + name: flow-runner +rules: + - apiGroups: [""] + resources: ["pods", "pods/log", "pods/status"] + verbs: ["get", "watch", "list"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: flow-runner-role-binding + namespace: default +subjects: + - kind: ServiceAccount + name: default + namespace: default +roleRef: + kind: Role + name: flow-runner + apiGroup: rbac.authorization.k8s.io diff --git a/docker/kube-infra/ingress.yaml b/docker/kube-infra/ingress.yaml new file mode 100644 index 0000000..7795259 --- /dev/null +++ b/docker/kube-infra/ingress.yaml @@ -0,0 +1,31 @@ +apiVersion: traefik.containo.us/v1alpha1 +kind: IngressRoute +metadata: + name: orion-api +spec: + entryPoints: + - web + routes: + - kind: Rule + match: PathPrefix(`/`) + services: + - kind: Service + name: orion + namespace: default + port: 4200 +--- +apiVersion: traefik.containo.us/v1alpha1 +kind: IngressRoute +metadata: + name: fastflows-api +spec: + entryPoints: + - web + routes: + - kind: Rule + match: PathPrefix(`/`) + services: + - kind: Service + name: fastflows + namespace: default + port: 5010 diff --git a/docker/kube-infra/postgres.yaml b/docker/kube-infra/postgres.yaml new file mode 100644 index 0000000..1163fc9 --- /dev/null +++ b/docker/kube-infra/postgres.yaml @@ -0,0 +1,87 @@ +# Create ConfigMap postgres-secret for the postgres app +# Define default database name, user, and password +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-secret + labels: + app: postgres +data: + POSTGRES_DB: orion + POSTGRES_USER: prefect_db + POSTGRES_PASSWORD: 71ccca6076254c528a3b7b34d184459b +--- +apiVersion: v1 +kind: PersistentVolume # Create PV +metadata: + name: postgres-volume # Sets PV name + labels: + type: local # Sets PV's type + app: postgres +spec: + storageClassName: manual + capacity: + storage: 10Gi # Sets PV's size + accessModes: + - ReadWriteMany + hostPath: + path: "/data/postgresql" # Sets PV's host path +--- +apiVersion: v1 +kind: PersistentVolumeClaim # Create PVC +metadata: + name: postgres-volume-claim # Sets PVC's name + labels: + app: postgres # Defines app to create PVC for +spec: + storageClassName: manual + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi # Sets PVC's size +--- +apiVersion: apps/v1 +kind: Deployment # Create a deployment +metadata: + name: postgres # Set the name of the deployment +spec: + replicas: 3 # Set 3 deployment replicas + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:13.7 # Docker image + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5432 # Exposing the container port 5432 for PostgreSQL client connections. + envFrom: + - configMapRef: + name: postgres-secret # Using the ConfigMap postgres-secret + + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: postgresdata + volumes: + - name: postgresdata + persistentVolumeClaim: + claimName: postgres-volume-claim +--- +apiVersion: v1 +kind: Service # Create service +metadata: + name: postgres # Sets the service name + labels: + app: postgres # Defines app to create service for +spec: + type: NodePort # Sets the service type + ports: + - port: 5432 # Sets the port to run the postgres application + selector: + app: postgres diff --git a/docker/kube-infra/prefect-queue.yaml b/docker/kube-infra/prefect-queue.yaml new file mode 100644 index 0000000..3b88319 --- /dev/null +++ b/docker/kube-infra/prefect-queue.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: orion-queue +spec: + selector: + matchLabels: + app: orion-queue + replicas: 1 + template: + metadata: + labels: + app: orion-queue + spec: + initContainers: + - name: check-orion-is-ready + image: curlimages/curl + command: + [ + "sh", + "-c", + 'while [ `curl -fsS http://localhost:4200/" --silent --output /dev/null"` -ne 200 ]; do sleep 2; done', + ] + containers: + - name: orion-queue + image: prefecthq/prefect:2.0b8-python3.9 + command: ["prefect", "work-queue", "create", "default-queue"] + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 4200 + env: + - name: PREFECT_API_URL + value: http://orion:4200/api diff --git a/docker/kube-infra/prefect.yaml b/docker/kube-infra/prefect.yaml new file mode 100644 index 0000000..9fa6f6e --- /dev/null +++ b/docker/kube-infra/prefect.yaml @@ -0,0 +1,90 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: orion +spec: + selector: + matchLabels: + app: orion + replicas: 2 + template: + metadata: + labels: + app: orion + spec: + initContainers: + - name: check-db-ready + image: postgres:13.0 + command: [ + "sh", + "-c", + "until pg_isready -h postgres -p 5432; + do echo waiting for database; sleep 2; done;", + ] + containers: + - name: api + image: prefecthq/prefect:2.0b8-python3.9 + command: + [ + "prefect", + "orion", + "start", + "--host", + "0.0.0.0", + "--log-level", + "DEBUG", + ] + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 4200 + env: + - name: PREFECT_ORION_DATABASE_CONNECTION_URL + value: postgresql+asyncpg://prefect_db:71ccca6076254c528a3b7b34d184459b@postgres:5432/orion + - name: agent + image: prefecthq/prefect:2.0b8-python3.9 + command: ["prefect", "agent", "start", "kubernetes"] + imagePullPolicy: "IfNotPresent" + env: + - name: PREFECT_API_URL + value: http://orion:4200/api +--- +apiVersion: v1 +kind: Service +metadata: + name: orion + labels: + app: orion +spec: + ports: + - port: 4200 + protocol: TCP + selector: + app: orion +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: default + name: flow-runner +rules: + - apiGroups: [""] + resources: ["pods", "pods/log", "pods/status"] + verbs: ["get", "watch", "list"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: flow-runner-role-binding + namespace: default +subjects: + - kind: ServiceAccount + name: default + namespace: default +roleRef: + kind: Role + name: flow-runner + apiGroup: rbac.authorization.k8s.io diff --git a/fastflows/config/app.py b/fastflows/config/app.py index 5cf1298..18cb9ab 100644 --- a/fastflows/config/app.py +++ b/fastflows/config/app.py @@ -37,9 +37,9 @@ class Config(BaseSettings): LOG_FORMAT: str = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} [id:{extra[request_id]}] - {message}" # uvicorn setting - HOST: str = "0.0.0.0" + FASTFLOWS_HOST: str = "0.0.0.0" FASTFLOWS_PORT: int = 5010 - RELOAD: bool = True + RELOAD: bool = False ACCESS_LOG: bool = False ROOT_PATH: str = "" diff --git a/fastflows/main.py b/fastflows/main.py index 62d3aad..76c087a 100644 --- a/fastflows/main.py +++ b/fastflows/main.py @@ -91,6 +91,7 @@ async def startup_event(): def app_run(): uvicorn.run( "fastflows.main:app", + host=cfg.FASTFLOWS_HOST, port=cfg.FASTFLOWS_PORT, reload=cfg.RELOAD, access_log=cfg.ACCESS_LOG, diff --git a/fastflows/providers/utils.py b/fastflows/providers/utils.py index 33dc7fd..f525392 100644 --- a/fastflows/providers/utils.py +++ b/fastflows/providers/utils.py @@ -1,6 +1,7 @@ from typing import Optional, _GenericAlias, Callable from pydantic import BaseModel import fastflows +import httpx from fastflows import errors from fastflows.schemas.misc import DefaultAPIResponseModel @@ -33,7 +34,12 @@ def decorator(func): _response_model = get_response_model(func, response_model) def wrapper(*args, **kwargs): - response = func(*args, **kwargs) + try: + response = func(*args, **kwargs) + except httpx.ConnectError as e: + raise fastflows.errors.FastFlowException( + f"Problems with resolving Prefect host: {str(e)}." + ) status_code_check = str(response.status_code).startswith if status_code_check("2"): response = response.json() From c143e676156f454489e93a9046985567440c7611 Mon Sep 17 00:00:00 2001 From: xnuinside Date: Mon, 18 Jul 2022 23:37:33 +0300 Subject: [PATCH 3/5] return pre-commit --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 45e4704..f40fcd3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,7 @@ repos: - id: trailing-whitespace - id: check-added-large-files - id: check-toml + - id: check-yaml - id: end-of-file-fixer - repo: local hooks: From 12fe9b49d5bc7242fc0c96e7eb6552e7b0fa1554 Mon Sep 17 00:00:00 2001 From: xnuinside Date: Wed, 20 Jul 2022 09:03:29 +0300 Subject: [PATCH 4/5] config changes for kube --- .pre-commit-config.yaml | 1 - Makefile | 6 ++---- docker/README.md | 2 +- docker/kube-infra/fastflows.yaml | 5 ++++- docker/kube-infra/ingress.yaml | 15 ++++++++------- docker/kube-infra/postgres.yaml | 1 + docker/kube-infra/prefect.yaml | 4 +++- fastflows/cli/flows.py | 2 +- fastflows/core/flow.py | 5 +++-- 9 files changed, 23 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f40fcd3..45e4704 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,6 @@ repos: - id: trailing-whitespace - id: check-added-large-files - id: check-toml - - id: check-yaml - id: end-of-file-fixer - repo: local hooks: diff --git a/Makefile b/Makefile index 211c420..06aae2b 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,8 @@ kubes: cluster kubes-prefect ## create k3s cluster cluster: -# enable ephmeral containers for profiling k3d cluster create ${CLUSTER_NAME} --registry-create ${CLUSTER_NAME}-registry:0.0.0.0:5550 \ - -p 4200:80@loadbalancer -p 9000:9000@loadbalancer -p 9001:9001@loadbalancer \ + -v "data:/data" -p 4200:80@loadbalancer -p 9090:9090@loadbalancer -p 9000:9000@loadbalancer -p 9001:9001@loadbalancer \ --k3s-arg '--kube-apiserver-arg=feature-gates=EphemeralContainers=true@server:*' \ --k3s-arg '--kube-scheduler-arg=feature-gates=EphemeralContainers=true@server:*' \ --k3s-arg '--kubelet-arg=feature-gates=EphemeralContainers=true@agent:*' \ @@ -20,13 +19,12 @@ cluster: @echo "export KUBECONFIG=$$(k3d kubeconfig write ${CLUSTER_NAME})" ## install prefect api and agent into kubes cluster -kubes-prefect: export PREFECT_API_URL=http://localhost:4200/api kubes-prefect: kubectl apply -f docker/kube-infra/postgres.yaml kubectl wait pod --for=condition=ready --timeout=120s -lapp=postgres kubectl apply -f docker/kube-infra/prefect.yaml kubectl apply -f docker/kube-infra/fastflows.yaml - kubectl apply -f docker/kube-infra/ingress.yaml + kubectl apply -f docker/kube-infra/ingress.yaml kubectl apply -f docker/kube-infra/prefect-queue.yaml kubectl wait pod --for=condition=ready --timeout=120s -lapp=orion diff --git a/docker/README.md b/docker/README.md index 50d9333..930bc12 100644 --- a/docker/README.md +++ b/docker/README.md @@ -44,7 +44,7 @@ To get possible run Fastflows in k3d locally you need to import image in cluster ```console - k3d cluster delete fastflows + k3d cluster delete fastflows-cluster ``` diff --git a/docker/kube-infra/fastflows.yaml b/docker/kube-infra/fastflows.yaml index 5e08755..c14e99a 100644 --- a/docker/kube-infra/fastflows.yaml +++ b/docker/kube-infra/fastflows.yaml @@ -34,10 +34,13 @@ metadata: app: fastflows spec: ports: - - port: 5010 + - name: http protocol: TCP + port: 80 + targetPort: 5010 selector: app: fastflows + type: LoadBalancer --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/docker/kube-infra/ingress.yaml b/docker/kube-infra/ingress.yaml index 7795259..a483a1c 100644 --- a/docker/kube-infra/ingress.yaml +++ b/docker/kube-infra/ingress.yaml @@ -1,7 +1,8 @@ +--- apiVersion: traefik.containo.us/v1alpha1 kind: IngressRoute metadata: - name: orion-api + name: fastflows-api spec: entryPoints: - web @@ -10,22 +11,22 @@ spec: match: PathPrefix(`/`) services: - kind: Service - name: orion + name: fastflows namespace: default - port: 4200 + port: 5010 --- apiVersion: traefik.containo.us/v1alpha1 kind: IngressRoute metadata: - name: fastflows-api + name: orion-api spec: entryPoints: - - web + - other routes: - kind: Rule match: PathPrefix(`/`) services: - kind: Service - name: fastflows + name: orion namespace: default - port: 5010 + port: 4200 diff --git a/docker/kube-infra/postgres.yaml b/docker/kube-infra/postgres.yaml index 1163fc9..daa377c 100644 --- a/docker/kube-infra/postgres.yaml +++ b/docker/kube-infra/postgres.yaml @@ -10,6 +10,7 @@ data: POSTGRES_DB: orion POSTGRES_USER: prefect_db POSTGRES_PASSWORD: 71ccca6076254c528a3b7b34d184459b + PGSSLMODE: require --- apiVersion: v1 kind: PersistentVolume # Create PV diff --git a/docker/kube-infra/prefect.yaml b/docker/kube-infra/prefect.yaml index 9fa6f6e..0cdb46e 100644 --- a/docker/kube-infra/prefect.yaml +++ b/docker/kube-infra/prefect.yaml @@ -57,8 +57,10 @@ metadata: app: orion spec: ports: - - port: 4200 + - port: 80 + targetPort: 4200 protocol: TCP + type: LoadBalancer selector: app: orion --- diff --git a/fastflows/cli/flows.py b/fastflows/cli/flows.py index f79d9ef..59fe84f 100644 --- a/fastflows/cli/flows.py +++ b/fastflows/cli/flows.py @@ -54,7 +54,7 @@ def run( def list(flow_path: Optional[str] = configuration.FLOWS_HOME): """List all flows from FLOWS_HOME""" typer.echo("\nAll flows from FLOWS_HOME: \n") - typer.echo(f"\nAvailable flows: {list_flows(flow_path)}\n") + typer.echo(f"\nAvailable flows: {list_flows(flows_home_path=flow_path)}\n") @flows_app.command() diff --git a/fastflows/core/flow.py b/fastflows/core/flow.py index 86ee676..ad8d6db 100644 --- a/fastflows/core/flow.py +++ b/fastflows/core/flow.py @@ -4,6 +4,7 @@ from fastflows.errors import FlowNotFound from fastflows.core.catalog.catalog import catalog, Catalog, catalog_by_id from fastflows.providers import provider +from fastflows.config.app import configuration as cfg def get_flow_id(flow_name: str, by_id: bool): @@ -63,9 +64,9 @@ def get_flow_runs_list(flow_name: str, by_id: bool) -> List[FlowRunResponse]: return provider.list_flow_runs(flow_id) -def list_flows(flow_path: Optional[str]) -> List[Flow]: +def list_flows(flows_home_path: Optional[str] = cfg.FLOWS_HOME) -> List[Flow]: # they cannot be registered in Prefect, just list from FLOWS_HOME - Catalog(flow_path).register_and_deploy() + Catalog(flows_home_path=flows_home_path).register_and_deploy() return list(catalog.keys()) From e85c7733913daa62c985b52c95b7fd47aa233e2d Mon Sep 17 00:00:00 2001 From: xnuinside Date: Thu, 21 Jul 2022 22:55:12 +0300 Subject: [PATCH 5/5] finally worked kube infra with ingress --- Makefile | 37 --------------- docker/Dockerfile | 4 +- docker/README.md | 67 +++++++++++++++++----------- docker/kube-infra/fastflows.yaml | 26 ++++++++++- docker/kube-infra/ingress.yaml | 56 +++++++++++------------ docker/kube-infra/postgres.yaml | 62 +++++++++++++------------ docker/kube-infra/prefect-queue.yaml | 34 -------------- docker/kube-infra/prefect.yaml | 8 ++-- docker/kube-infra/secrets.yaml | 13 ++++++ 9 files changed, 140 insertions(+), 167 deletions(-) delete mode 100644 Makefile delete mode 100644 docker/kube-infra/prefect-queue.yaml create mode 100644 docker/kube-infra/secrets.yaml diff --git a/Makefile b/Makefile deleted file mode 100644 index 06aae2b..0000000 --- a/Makefile +++ /dev/null @@ -1,37 +0,0 @@ - -export CLUSTER_NAME=fastflows-cluster -export KUBECONFIG=.k3d/kubeconfig-${CLUSTER_NAME}.yaml - -## create cluster and install minio and prefect -kubes: cluster kubes-prefect - -## create k3s cluster -cluster: - k3d cluster create ${CLUSTER_NAME} --registry-create ${CLUSTER_NAME}-registry:0.0.0.0:5550 \ - -v "data:/data" -p 4200:80@loadbalancer -p 9090:9090@loadbalancer -p 9000:9000@loadbalancer -p 9001:9001@loadbalancer \ - --k3s-arg '--kube-apiserver-arg=feature-gates=EphemeralContainers=true@server:*' \ - --k3s-arg '--kube-scheduler-arg=feature-gates=EphemeralContainers=true@server:*' \ - --k3s-arg '--kubelet-arg=feature-gates=EphemeralContainers=true@agent:*' \ - --wait - @echo "Probing until traefik CRDs are created (~60 secs)..." && export KUBECONFIG=$$(k3d kubeconfig write ${CLUSTER_NAME}) && \ - while : ; do kubectl get crd ingressroutes.traefik.containo.us > /dev/null && break; sleep 10; done - @echo -e "\nTo use your cluster set:\n" - @echo "export KUBECONFIG=$$(k3d kubeconfig write ${CLUSTER_NAME})" - -## install prefect api and agent into kubes cluster -kubes-prefect: - kubectl apply -f docker/kube-infra/postgres.yaml - kubectl wait pod --for=condition=ready --timeout=120s -lapp=postgres - kubectl apply -f docker/kube-infra/prefect.yaml - kubectl apply -f docker/kube-infra/fastflows.yaml - kubectl apply -f docker/kube-infra/ingress.yaml - kubectl apply -f docker/kube-infra/prefect-queue.yaml - kubectl wait pod --for=condition=ready --timeout=120s -lapp=orion - -## show kube logs for orion -kube-logs-orion: - kubectl logs -lapp=orion --all-containers - -## show kube logs for orion -kube-logs-fastflows: - kubectl logs -lapp=fastflows --all-containers diff --git a/docker/Dockerfile b/docker/Dockerfile index 256f381..5aedd0f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,6 +12,4 @@ COPY ./pyproject.toml /app RUN poetry install --no-dev -EXPOSE 80 - -CMD ["uvicorn", "fastflows.main:app", "--host", "0.0.0.0", "--port", "80"] +CMD ["uvicorn", "fastflows.main:app", "--host", "0.0.0.0", "--port", "5010"] diff --git a/docker/README.md b/docker/README.md index 930bc12..48e753c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,4 +1,4 @@ -Run all commands from base source dir +**Note:** Run all commands from base repo dir ### Build Fastflows Docker Image @@ -12,68 +12,83 @@ Run all commands from base source dir Steps to run: -1. Create dir '/etc/data/postgresql' it will be used to store persistent PostgreSQL data +1. **(For local env only)** Install minikube (will be used to create & manage local cluster): https://minikube.sigs.k8s.io/docs/start/ -2. **(For local env only)** Install k3d.io (will be used as K8s local solution): https://k3d.io/v5.4.4/#installation +2. Install kubectl if you don't have it - https://kubernetes.io/docs/tasks/tools/ -3. Install kubectl if you don't have it - https://kubernetes.io/docs/tasks/tools/ - -4. Create k3d cluster with an image registry, minio (for remote storage), the prefect agent and api: +3. **(For local env only)** Create kubernetes cluster: ```console - make kubes + minikube start -``` + # Important!!! + # On MacOS to make Ingress work you should run minikube with hyperkit: + + minikube start --driver=hyperkit -P.S: If you will see error like 'Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "ingressroutes.traefik.containo.us" not found' it is okay - just wait. -4. (**For local run only**) Import fastflows image +``` + +4. **(For local env only)** -To get possible run Fastflows in k3d locally you need to import image in cluster (by default k3d cannot find images from local env) +Enable Ingress & host resolving (full explanation here) ```console - k3d images import fastflows -c fastflows-cluster + minikube addons enable ingress ``` -### Tips - -1. **(For local env only)** If for some reason you need to delete cluster, use: +Get minikube cluster ip ```console - k3d cluster delete fastflows-cluster + minikube ip ``` -2. If you need to connect to PostgreSQL DB from localhose, first find host with +Add domains to /etc/hosts linked to minikube cluster ip. + +Add this to /etc/hosts file: ```console - kubectl get svc + your-mini-kube-ip fastflows.geo + your-mini-kube-ip prefect.geo + + # for example (where 192.168.64.2 - minikube ip): + # 192.168.64.2 fastflows.geo + # 192.168.64.2 prefect.geo ``` -when use with information to connect wtih PgAdmin or psql +5. Apply Kube manifests to the cluster -3. If you cannot access from you local terminal Kube cluster & if you run command, for example: `kubectl get svc` see the error like: - 'The connection to the server localhost:8080 was refused - did you specify the right host or port?' +```console -Then in your terminal that you use define path to cluster config like this: + kubectl apply -f docker/kube-infra + +``` + +5. Check pods statuses & wait until all pods will not be in status "Running" ```console - export CLUSTER_NAME=fastflows-cluster - export KUBECONFIG=.k3d/kubeconfig-${CLUSTER_NAME}.yaml + kubectl get pods ``` -4. To check pods status: +Now you can access service by links: + +http://fastflows.geo/ & http://prefect.geo/ + +### Tips + +1. **(For local env only)** If for some reason you need to delete cluster, use: ```console - kubectl get pods + minikube delete ``` diff --git a/docker/kube-infra/fastflows.yaml b/docker/kube-infra/fastflows.yaml index c14e99a..d56ffeb 100644 --- a/docker/kube-infra/fastflows.yaml +++ b/docker/kube-infra/fastflows.yaml @@ -13,6 +13,29 @@ spec: labels: app: fastflows spec: + initContainers: + - name: check-orion-is-ready + image: curlimages/curl + command: + [ + "sh", + "-c", + 'while [ `curl -fsS http://localhost:4200/" --silent --output /dev/null"` -ne 200 ]; do sleep 2; done', + ] + - name: orion-queue + image: prefecthq/prefect:2.0b8-python3.9 + command: + [ + "sh", + "-c", + 'prefect work-queue create default-queue; r=$?; [ "$r" -gt 0 ] && exit 0; exit 0', + ] + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 4200 + env: + - name: PREFECT_API_URL + value: http://orion:4200/api containers: - name: main image: geobeyond/fastflows:latest @@ -36,11 +59,10 @@ spec: ports: - name: http protocol: TCP - port: 80 + port: 5010 targetPort: 5010 selector: app: fastflows - type: LoadBalancer --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/docker/kube-infra/ingress.yaml b/docker/kube-infra/ingress.yaml index a483a1c..daf6bb2 100644 --- a/docker/kube-infra/ingress.yaml +++ b/docker/kube-infra/ingress.yaml @@ -1,32 +1,28 @@ ---- -apiVersion: traefik.containo.us/v1alpha1 -kind: IngressRoute +apiVersion: networking.k8s.io/v1 +kind: Ingress metadata: - name: fastflows-api + name: fastflows-cluster + annotations: + kubernetes.io/ingress.class: nginx spec: - entryPoints: - - web - routes: - - kind: Rule - match: PathPrefix(`/`) - services: - - kind: Service - name: fastflows - namespace: default - port: 5010 ---- -apiVersion: traefik.containo.us/v1alpha1 -kind: IngressRoute -metadata: - name: orion-api -spec: - entryPoints: - - other - routes: - - kind: Rule - match: PathPrefix(`/`) - services: - - kind: Service - name: orion - namespace: default - port: 4200 + rules: + - host: prefect.geo + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: orion + port: + number: 4200 + - host: fastflows.geo + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: fastflows + port: + number: 5010 diff --git a/docker/kube-infra/postgres.yaml b/docker/kube-infra/postgres.yaml index daa377c..01e86f7 100644 --- a/docker/kube-infra/postgres.yaml +++ b/docker/kube-infra/postgres.yaml @@ -1,53 +1,39 @@ -# Create ConfigMap postgres-secret for the postgres app -# Define default database name, user, and password apiVersion: v1 -kind: ConfigMap +kind: PersistentVolume metadata: - name: postgres-secret + name: postgres-volume labels: - app: postgres -data: - POSTGRES_DB: orion - POSTGRES_USER: prefect_db - POSTGRES_PASSWORD: 71ccca6076254c528a3b7b34d184459b - PGSSLMODE: require ---- -apiVersion: v1 -kind: PersistentVolume # Create PV -metadata: - name: postgres-volume # Sets PV name - labels: - type: local # Sets PV's type + type: local app: postgres spec: storageClassName: manual capacity: - storage: 10Gi # Sets PV's size + storage: 10Gi accessModes: - ReadWriteMany hostPath: - path: "/data/postgresql" # Sets PV's host path + path: /data/postgresql --- apiVersion: v1 kind: PersistentVolumeClaim # Create PVC metadata: - name: postgres-volume-claim # Sets PVC's name + name: postgres-volume-claim labels: - app: postgres # Defines app to create PVC for + app: postgres spec: storageClassName: manual accessModes: - ReadWriteMany resources: requests: - storage: 10Gi # Sets PVC's size + storage: 10Gi --- apiVersion: apps/v1 -kind: Deployment # Create a deployment +kind: Deployment metadata: - name: postgres # Set the name of the deployment + name: postgres spec: - replicas: 3 # Set 3 deployment replicas + replicas: 3 selector: matchLabels: app: postgres @@ -58,16 +44,28 @@ spec: spec: containers: - name: postgres - image: postgres:13.7 # Docker image + image: postgres:13.0 imagePullPolicy: "IfNotPresent" ports: - - containerPort: 5432 # Exposing the container port 5432 for PostgreSQL client connections. - envFrom: - - configMapRef: - name: postgres-secret # Using the ConfigMap postgres-secret - + - containerPort: 5432 + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_USER + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_DB + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD volumeMounts: - - mountPath: /var/lib/postgresql/data + - mountPath: "/var/lib/postgresql/data/:Z" name: postgresdata volumes: - name: postgresdata diff --git a/docker/kube-infra/prefect-queue.yaml b/docker/kube-infra/prefect-queue.yaml deleted file mode 100644 index 3b88319..0000000 --- a/docker/kube-infra/prefect-queue.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: orion-queue -spec: - selector: - matchLabels: - app: orion-queue - replicas: 1 - template: - metadata: - labels: - app: orion-queue - spec: - initContainers: - - name: check-orion-is-ready - image: curlimages/curl - command: - [ - "sh", - "-c", - 'while [ `curl -fsS http://localhost:4200/" --silent --output /dev/null"` -ne 200 ]; do sleep 2; done', - ] - containers: - - name: orion-queue - image: prefecthq/prefect:2.0b8-python3.9 - command: ["prefect", "work-queue", "create", "default-queue"] - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 4200 - env: - - name: PREFECT_API_URL - value: http://orion:4200/api diff --git a/docker/kube-infra/prefect.yaml b/docker/kube-infra/prefect.yaml index 0cdb46e..148abdb 100644 --- a/docker/kube-infra/prefect.yaml +++ b/docker/kube-infra/prefect.yaml @@ -40,7 +40,10 @@ spec: - containerPort: 4200 env: - name: PREFECT_ORION_DATABASE_CONNECTION_URL - value: postgresql+asyncpg://prefect_db:71ccca6076254c528a3b7b34d184459b@postgres:5432/orion + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_URI - name: agent image: prefecthq/prefect:2.0b8-python3.9 command: ["prefect", "agent", "start", "kubernetes"] @@ -57,10 +60,9 @@ metadata: app: orion spec: ports: - - port: 80 + - port: 4200 targetPort: 4200 protocol: TCP - type: LoadBalancer selector: app: orion --- diff --git a/docker/kube-infra/secrets.yaml b/docker/kube-infra/secrets.yaml new file mode 100644 index 0000000..2f2d572 --- /dev/null +++ b/docker/kube-infra/secrets.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-credentials + namespace: default +type: Opaque +data: + POSTGRES_USER: cHJlZmVjdF9kYg== + POSTGRES_PASSWORD: b3Jpb25EYnBhc3N3b3JkMjAyMA== + POSTGRES_DB: b3Jpb24= + # uri to connect to DB should be in forma like: + # postgresql+asyncpg://user:password@postgres:5432/db_name + POSTGRES_URI: cG9zdGdyZXNxbCthc3luY3BnOi8vcHJlZmVjdF9kYjpvcmlvbkRicGFzc3dvcmQyMDIwQHBvc3RncmVzOjU0MzIvb3Jpb24=