From dd346a7ca562a4741b4e1e6a8b6021ea6a509343 Mon Sep 17 00:00:00 2001 From: nspmx Date: Thu, 25 Jan 2024 11:25:14 -0300 Subject: [PATCH 1/5] add playground workflow --- .../google-registry-gke-playground.yaml | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 .github/workflows/google-registry-gke-playground.yaml diff --git a/.github/workflows/google-registry-gke-playground.yaml b/.github/workflows/google-registry-gke-playground.yaml new file mode 100644 index 0000000..e50299e --- /dev/null +++ b/.github/workflows/google-registry-gke-playground.yaml @@ -0,0 +1,201 @@ +# This workflow build and push a Docker container to Google Artifact Registry and deploy it on Google Kubernetes Engine when a commit is pushed to the "develop" branch +# You can start your commit with `#update` and the workflow will just trigger an update of the Helm installation, without building a new image +# +# To configure this workflow: +# +# 1. Ensure the required Google Cloud APIs are enabled in the project: +# +# Cloud Build cloudbuild.googleapis.com +# Kubernetes Engine API container.googleapis.com +# Artifact Registry artifactregistry.googleapis.com +# +# 2. Create a service account (if you don't have one) with the following fields: +# +# Service Account Name -github-actions +# Service Account ID -github-actions +# +# 3. Ensure the service account have the required IAM permissions granted: +# +# Kubernetes Engine Developer +# roles/container.developer (kubernetes engine developer) +# +# Artifact Registry +# roles/artifactregistry.repoAdmin (artifact registry repository administrator) +# roles/artifactregistry.admin (artifact registry administrator) +# +# Service Account +# roles/iam.serviceAccountUser (act as the Cloud Run runtime service account) +# +# Basic Roles +# roles/viewer (viewer) +# +# NOTE: You should always follow the principle of least privilege when assigning IAM roles +# +# 4. Ensure you have the following GitHub Secrets and Variables: +# +# GitHub Secrets +# GCP_SA_KEY (Google Cloud Project Service Account Key) ref visit https://github.com/Datawheel/company/wiki/Setting-Up-a-Service-Account-for-Workflows#use-the-service-account-on-github-secrets +# +# GitHub Variables +# GCP_PROJECT_ID (Google Cloud Project ID) +# GCP_ARTIFACT_REGISTRY_NAME (Google Cloud Articaft Registry Repository Name) +# GCP_ARTIFACT_REGISTRY_LOCATION (Google Cloud Artifact Registry Reposotiry Location) +# +# 5. Ensure you have the following GitHub Variables for each environment that you will set up: +# +# GitHub Variables +# GCP_IMAGE_NAME (Docker Image Name) +# GKE_APP_NAME (Google Kubernetes Engine Deployment Name) +# GKE_APP_NAMESPACE (Google Kubernetes Engine Deployment Namespace) +# GKE_CLUSTER (Google Kubernetes Engine Cluster Name) +# GKE_ZONE (Google Kubernetes Engine Cluster Zone) +# +# Further reading: +# Kubernetes Developer - https://cloud.google.com/iam/docs/understanding-roles#container.developer +# Artifact Registry IAM permissions - https://cloud.google.com/artifact-registry/docs/access-control#roles +# Container Registry vs Artifact Registry - https://cloud.google.com/blog/products/application-development/understanding-artifact-registry-vs-container-registry +# Principle of least privilege - https://cloud.google.com/blog/products/identity-security/dont-get-pwned-practicing-the-principle-of-least-privilege +# Deploy CloudRun Github Actions - https://github.com/google-github-actions/deploy-cloudrun +name: "[GCP][DEV] Build API to Registry and Deploy via Helm" + +on: + push: + branches: [ "tesseract-python" ] + paths: + - .github/workflows/google-registry-gke-playground.yaml + - helm/development.yaml + - schema/** + +env: + GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} + GCP_ARTIFACT_REGISTRY_NAME: ${{ vars.GCP_ARTIFACT_REGISTRY_NAME }} + GCP_ARTIFACT_REGISTRY_LOCATION: ${{ vars.GCP_ARTIFACT_REGISTRY_LOCATION }} + GCP_IMAGE_NAME: ${{ vars.GCP_IMAGE_NAME }} + GKE_APP_NAME: ${{ vars.GKE_APP_NAME }} + GKE_APP_NAMESPACE: ${{ vars.GKE_APP_NAMESPACE }} + GKE_CLUSTER: ${{ vars.GKE_CLUSTER }} + GKE_ZONE: ${{ vars.GKE_ZONE }} + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + +jobs: + build: + environment: playground + runs-on: ubuntu-latest + # runs-on: self-hosted + if: ${{ !contains(github.event.head_commit.message, '#update') }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + # Authentication via credentials json + - name: Google Auth + id: auth + uses: google-github-actions/auth@v2 + with: + project_id: ${{ env.GCP_PROJECT_ID }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + + # Install Cloud SDK + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1 + with: + install_components: beta + + # Build image on Google Cloud Artifact Registry + - name: Build Docker Image + run: |- + gcloud builds submit \ + --quiet \ + --timeout=40m \ + --config=cloudbuild.yml \ + --substitutions=_GCP_PROJECT_ID=${{ env.GCP_PROJECT_ID }},_GCP_ARTIFACT_REGISTRY_NAME=${{ env.GCP_ARTIFACT_REGISTRY_NAME }},_GCP_ARTIFACT_REGISTRY_LOCATION=${{ env.GCP_ARTIFACT_REGISTRY_LOCATION }},_GCP_IMAGE_NAME=${{ env.GCP_IMAGE_NAME }},_GCP_IMAGE_TAG=${{ github.sha }},_GCP_IMAGE_ENVIRONMENT=${{ env.GKE_APP_NAMESPACE }} + + deploy: + needs: build + environment: playground + runs-on: ubuntu-latest + # runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v3 + + # Authentication via credentials json + - name: Google Auth + id: auth + uses: google-github-actions/auth@v2 + with: + project_id: ${{ env.GCP_PROJECT_ID }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + + # Get google kubernetes engine credentials + - name: Get GKE Credentials + uses: google-github-actions/get-gke-credentials@v2 + with: + cluster_name: ${{ env.GKE_CLUSTER }} + location: ${{ env.GKE_ZONE }} + + # Transform GitHub secrets to base64 encoded + - name: Set encoded secret values + run: | + echo "ENCODED_TESSERACT_BACKEND=$(echo -n "${{ secrets.TESSERACT_BACKEND }}" | base64 | tr -d '\n')" >> $GITHUB_ENV + + # Install Helm chart + - name: Helm install + uses: WyriHaximus/github-action-helm3@v2 + with: + exec: | + helm upgrade --install --create-namespace \ + --namespace ${{ env.GKE_APP_NAMESPACE }} \ + --set app.environment=${{ env.GKE_APP_NAMESPACE }} \ + --set app.release=${{ env.GKE_APP_NAMESPACE }} \ + --set image.repository=${{ env.GCP_ARTIFACT_REGISTRY_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GCP_ARTIFACT_REGISTRY_NAME }}/${{ env.GCP_IMAGE_NAME }} \ + --set image.tag=${{ github.sha }} \ + --set nameOverride=${{ env.GKE_APP_NAME }} \ + --set fullnameOverride=${{ env.GKE_APP_NAME }} \ + --set secrets.TESSERACT_BACKEND=$ENCODED_TESSERACT_BACKEND \ + ${{ env.GKE_APP_NAME }} --values=./helm/development.yaml ./helm + + update: + runs-on: ubuntu-latest + environment: playground + # runs-on: self-hosted + if: ${{ contains(github.event.head_commit.message, '#update') }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + # Authentication via credentials json + - name: Google Auth + id: auth + uses: google-github-actions/auth@v2 + with: + project_id: ${{ env.GCP_PROJECT_ID }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + + # Get google kubernetes engine credentials + - name: Get GKE Credentials + uses: google-github-actions/get-gke-credentials@v2 + with: + cluster_name: ${{ env.GKE_CLUSTER }} + location: ${{ env.GKE_ZONE }} + + # Transform GitHub secrets to base64 encoded + - name: Set encoded secret values + run: | + echo "ENCODED_TESSERACT_BACKEND=$(echo -n "${{ secrets.TESSERACT_BACKEND }}" | base64 | tr -d '\n')" >> $GITHUB_ENV + + # Install Helm chart + - name: Helm install + uses: WyriHaximus/github-action-helm3@v2 + with: + exec: | + helm upgrade --install --create-namespace \ + --namespace ${{ env.GKE_APP_NAMESPACE }} \ + --set app.environment=${{ env.GKE_APP_NAMESPACE }} \ + --set app.release=${{ env.GKE_APP_NAMESPACE }} \ + --set image.repository=${{ env.GCP_ARTIFACT_REGISTRY_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GCP_ARTIFACT_REGISTRY_NAME }}/${{ env.GCP_IMAGE_NAME }} \ + --set image.tag=${{ github.sha }} \ + --set nameOverride=${{ env.GKE_APP_NAME }} \ + --set fullnameOverride=${{ env.GKE_APP_NAME }} \ + --set secrets.TESSERACT_BACKEND=$ENCODED_TESSERACT_BACKEND \ + ${{ env.GKE_APP_NAME }} --values=./helm/development.yaml ./helm \ No newline at end of file From ea6f5d13e9627396fad716aa26af5c7d3207c19b Mon Sep 17 00:00:00 2001 From: nspmx Date: Thu, 25 Jan 2024 12:29:16 -0300 Subject: [PATCH 2/5] add playground zrch --- .../google-registry-gke-playground.yaml | 6 +- helm/playground.yaml | 86 +++++++++++++++++++ 2 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 helm/playground.yaml diff --git a/.github/workflows/google-registry-gke-playground.yaml b/.github/workflows/google-registry-gke-playground.yaml index e50299e..450605d 100644 --- a/.github/workflows/google-registry-gke-playground.yaml +++ b/.github/workflows/google-registry-gke-playground.yaml @@ -63,7 +63,7 @@ on: branches: [ "tesseract-python" ] paths: - .github/workflows/google-registry-gke-playground.yaml - - helm/development.yaml + - helm/playground.yaml - schema/** env: @@ -153,7 +153,7 @@ jobs: --set nameOverride=${{ env.GKE_APP_NAME }} \ --set fullnameOverride=${{ env.GKE_APP_NAME }} \ --set secrets.TESSERACT_BACKEND=$ENCODED_TESSERACT_BACKEND \ - ${{ env.GKE_APP_NAME }} --values=./helm/development.yaml ./helm + ${{ env.GKE_APP_NAME }} --values=./helm/playground.yaml ./helm update: runs-on: ubuntu-latest @@ -198,4 +198,4 @@ jobs: --set nameOverride=${{ env.GKE_APP_NAME }} \ --set fullnameOverride=${{ env.GKE_APP_NAME }} \ --set secrets.TESSERACT_BACKEND=$ENCODED_TESSERACT_BACKEND \ - ${{ env.GKE_APP_NAME }} --values=./helm/development.yaml ./helm \ No newline at end of file + ${{ env.GKE_APP_NAME }} --values=./helm/playground.yaml ./helm \ No newline at end of file diff --git a/helm/playground.yaml b/helm/playground.yaml new file mode 100644 index 0000000..a236313 --- /dev/null +++ b/helm/playground.yaml @@ -0,0 +1,86 @@ +image: + pullPolicy: Always + +imagePullSecrets: + - name: github + +replicaCount: 1 + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 4 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 120 + +resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: 500m + memory: 4Gi + +livenessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 7777 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 30 + successThreshold: 1 + +readinessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 7777 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 30 + successThreshold: 2 + +service: + type: ClusterIP + port: 7777 + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +configMap: + TESSERACT_DEBUG: "true" + TESSERACT_SCHEMA: "schema" + +# secrets: +# TESSERACT_BACKEND: + +ingress: + enabled: true + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + acme.cert-manager.io/http01-edit-in-place: "true" + ingress.kubernetes.io/ssl-redirect: "true" + nginx.org/proxy-connect-timeout: "120s" + nginx.org/proxy-read-timeout: "120s" + nginx.org/proxy-buffers: "8 16k" + nginx.org/proxy-buffer-size: "16k" + nginx.org/proxy-busy-buffers-size: "64k" + nginx.org/location-snippets: | + add_header Access-Control-Allow-Origin *; + hosts: + - host: playground.api.datawheel.us + paths: + - / + tls: + - secretName: tesseract-api-ingress-tls + hosts: + - playground.api.datawheel.us From abc74e7f17ee28bd4dd37e2bd8faeb8c24a513cc Mon Sep 17 00:00:00 2001 From: Samuel Osorio Carriman <35802301+D2SD@users.noreply.github.com> Date: Thu, 15 Feb 2024 16:42:35 -0300 Subject: [PATCH 3/5] Update google-registry-gke-playground.yaml --- .github/workflows/google-registry-gke-playground.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/google-registry-gke-playground.yaml b/.github/workflows/google-registry-gke-playground.yaml index 450605d..a40b476 100644 --- a/.github/workflows/google-registry-gke-playground.yaml +++ b/.github/workflows/google-registry-gke-playground.yaml @@ -64,6 +64,7 @@ on: paths: - .github/workflows/google-registry-gke-playground.yaml - helm/playground.yaml + - requirements.txt - schema/** env: @@ -198,4 +199,4 @@ jobs: --set nameOverride=${{ env.GKE_APP_NAME }} \ --set fullnameOverride=${{ env.GKE_APP_NAME }} \ --set secrets.TESSERACT_BACKEND=$ENCODED_TESSERACT_BACKEND \ - ${{ env.GKE_APP_NAME }} --values=./helm/playground.yaml ./helm \ No newline at end of file + ${{ env.GKE_APP_NAME }} --values=./helm/playground.yaml ./helm From ac12b238486e3888faa54c8760b8f5cdc894d80d Mon Sep 17 00:00:00 2001 From: D2SD Date: Mon, 25 Mar 2024 16:44:15 -0300 Subject: [PATCH 4/5] fix pandas version for peii --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 03398a2..a7e17e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ lz4==4.3.3 numpy==1.24.4 ; python_version >= "3.8" and python_version < "3.9" numpy==1.26.4 ; python_version >= "3.9" and python_version < "4.0" orjson==3.9.13 -pandas==2.0.3 +pandas==1.5.3 pydantic-core==2.16.2 pydantic==2.6.1 pypika==0.48.9 From 64c56d1efa90736e93f0b230550a0c46268f60fe Mon Sep 17 00:00:00 2001 From: hermosillajelmy Date: Thu, 4 Jul 2024 17:46:10 -0400 Subject: [PATCH 5/5] change branch --- .github/workflows/google-registry-gke-playground.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/google-registry-gke-playground.yaml b/.github/workflows/google-registry-gke-playground.yaml index a40b476..6d73048 100644 --- a/.github/workflows/google-registry-gke-playground.yaml +++ b/.github/workflows/google-registry-gke-playground.yaml @@ -60,7 +60,7 @@ name: "[GCP][DEV] Build API to Registry and Deploy via Helm" on: push: - branches: [ "tesseract-python" ] + branches: [ "main" ] paths: - .github/workflows/google-registry-gke-playground.yaml - helm/playground.yaml