Mage setup

.github/workflows/mage-ai-develop.yml

@@ -0,0 +1,41 @@
+name: Push mageai to the latest deploy helm chart to Amazon EKS
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - global-api/importer-mage/**
+      - charts/mage-ai/**
+      - .github/workflows/mage-ai-develop.yml
+    branches: ["develop"]
+  pull_request:
+    paths:
+      - global-api/importer-mage/**
+
+jobs:
+
+  pushToGHCR:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./global-api/importer-mage
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pushing mageai to GHCR
+        env:
+          VERSION: ${{ github.sha }}
+          IMAGE: ghcr.io/open-earth-foundation/citycatalyst-mage-ai
+        run: |
+          docker build -t $IMAGE:$VERSION .
+          docker tag $IMAGE:$VERSION $IMAGE:latest
+          docker push $IMAGE:$VERSION
+          docker push $IMAGE:latest

.gitignore

@@ -150,5 +150,6 @@ global-api/importer/osm/cache/**
 global-api/importer/osm/data/**
 global-api/data/**
 
+
 #Python virtual env
-global-api/env
+global-api/env

charts/mage-ai/values.yml

@@ -0,0 +1,164 @@
+# Default values for mageai.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+standaloneScheduler: false
+
+# Effective if standaloneScheduler is true
+scheduler:
+  replicaCount: 1
+  name: mageai-schedulers
+
+
+# Effective if standaloneScheduler is true
+webServer:
+  replicaCount: 1
+  name: mageai-webserver
+
+# Enable redis if you want more replica
+redis:
+  enabled: false
+  architecture: standalone
+  auth:
+    enabled: false
+  # Your custom redis url (make sure redis.enabled is set to false)
+  customRedisURL: ""
+
+image:
+  repository: ghcr.io/open-earth-foundation/citycatalyst-mage-ai
+  pullPolicy: Always
+  tag: latest
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: "mageai"
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+service:
+  type: ClusterIP
+  port: 6789
+  # Annotations to add to the service
+  annotations: {}
+
+# Configure extra options for containers' liveness probes
+# If not configured, the probe is enabled using the following values
+livenessProbe:
+  enabled: true
+  path: /api/status
+  port: http
+  initialDelaySeconds: 5
+  # failureThreshold:
+  # periodSeconds:
+  # successThreshold:
+  # terminationGracePeriodSeconds:
+  timeoutSeconds: 10
+
+# Configure extra options for containers' readiness probes
+# If not configured, the probe is enabled using the following values
+readinessProbe:
+  enabled: true
+  path: /api/status
+  port: http
+  initialDelaySeconds: 5
+  # failureThreshold:
+  # periodSeconds:
+  # successThreshold:
+  # terminationGracePeriodSeconds:
+  timeoutSeconds: 1
+
+# Custom liveness probe
+customLivenessProbe: {}
+
+# Custom readiness probe
+customReadinessProbe: {}
+
+# We recommend creating the ingress separately instead of creating it using this chart.
+# There is a corresponding Mage-Ingress chart to create an ingress for Mage if needed.
+# This section is kept here for backwards compatibility.
+ingress:
+  enabled: false
+  className: ""
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+  hosts:
+    - host: chart-example.local
+      paths:
+        - path: /
+          pathType: ImplementationSpecific
+  tls: []
+  #  - secretName: chart-example-tls
+  #    hosts:
+  #      - chart-example.local
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+# extraVolumeMounts:
+#   - name: mage-fs
+#     mountPath: /home/src
+
+extraVolumes:
+  - name: mage-fs
+    persistentVolumeClaim:
+      claimName: mage-fs-pvc
+
+# config: Default configuration for mageai as environment variables. These get injected directly in the container.
+config: {}
+
+# existingSecret: Specifies an existing secret to be used as environment variables. These get injected directly in the container.
+existingSecret: ""
+
+# secrets: Default secrets for mageai as environment variables. These get injected directly in the container.
+# Consider using a secret manager first, before sourcing secrets as environment variables.
+secrets: {}
+
+# extraEnvs: Extra environment variables
+extraEnvs:
+  - name: KUBE_NAMESPACE
+    valueFrom:
+      fieldRef:
+        fieldPath: metadata.namespace
+
+env:
+  - name: REQUIRE_USER_AUTHENTICATION
+    value: "1"
+  - name: DEFAULT_OWNER_EMAIL
+    value: ""
+  - name: DEFAULT_OWNER_PASSWORD
+    value: ""
+  - name: DEFAULT_OWNER_USERNAME
+    value: ""

global-api/import_everything.sh

@@ -134,7 +134,8 @@ popd
 
 
 # Import EDGAR extract
+# no longer require sample data to be loaded. 
 
-pushd importer/edgar/pilot_cities
-$python_cmd load_edgar_pilot_city.py --database_uri $DB_URI
-popd
+# pushd importer/edgar/pilot_cities
+# $python_cmd load_edgar_pilot_city.py --database_uri $DB_URI
+# popd