Merge pull request #222 from sunya-ch/ci

add local db, remove kubelet, update push-pr

.github/workflows/build-push.yml

@@ -1,84 +1,111 @@
-name: BuildPushDeployImage
+name: Build-Push Kepler Model Server Image
 
 on:
-  push:
-    branches: [ main ]
+  workflow_call:
+    secrets:
+      docker_username:
+        description: 'Docker username'
+        required: false
+      docker_password:
+        description: 'Docker password'
+        required: false
+    inputs:
+      base_change:
+        description: 'Change flag on base image'
+        required: true
+        type: string
+      image_repo:
+        description: 'The image repo to use'
+        required: true
+        type: string
+      image_tag:
+        description: 'The image tag to use'
+        required: true
+        type: string
+      push:
+        description: 'Push image'
+        required: false
+        type: string
+        default: false
+
+
+env:
+  base_change: ${{ inputs.base_change }}
+  base_image: ${{ inputs.image_repo }}/kepler_model_server_base:${{ inputs.image_tag }}
+  image: ${{ inputs.image_repo }}/kepler_model_server:${{ inputs.image_tag }}
 
 jobs:
 
-  changes:
+  check-secret:
     runs-on: ubuntu-latest
+
     outputs:
-      src: ${{ steps.changes.outputs.src }}
+      available: ${{ steps.check-secret.outputs.available }}
+
     steps:
-    - uses: actions/checkout@v4
-    - uses: dorny/paths-filter@v2
-      id: changes
-      with:
-        filters: |
-          src:
-            - 'dockerfiles/requirements.txt'
-            - 'dockerfiles/Dockerfile.base'
-            - '.github/workflows/build-push.yml'
+      - name: Check Secret
+        id: check-secret
+        env:
+          SECRET: ${{ secrets.docker_password }}
+        run: |
+          if [ "$SECRET" == "" ]; then
+              echo "available=false" >> "$GITHUB_OUTPUT"
+          else
+              echo "available=true" >> "$GITHUB_OUTPUT"
+          fi
+
+
+  check-base-exist:
+    runs-on: ubuntu-latest
 
-  buildbase:
-    needs: changes
-    if: ${{ needs.changes.outputs.src == 'true' }}
+    outputs:
+      exists: ${{ steps.check-base-exist.outputs.exists }}
+
+    steps:
+    - name: Check if Docker base image exists
+      id: check-base-exist
+      run: |
+        if docker pull ${{ env.base_image }}; then
+              echo "exists=true" >> "$GITHUB_OUTPUT"
+        else
+              echo "exists=false" >> "$GITHUB_OUTPUT"
+        fi
+
+  build:
     runs-on: ubuntu-latest
+    needs: [check-secret, check-base-exist]
     steps:
     - name: checkout
       uses: actions/checkout@v4
     - name: set up QEMU
       uses: docker/setup-qemu-action@v3
     - name: set up Docker Buildx
       uses: docker/setup-buildx-action@v3
-    - name: Login to Quay
-      if: ${{ (github.repository_owner == 'sustainable-computing-io') && (github.ref == 'refs/heads/main') }}
+    - name: Login to Docker
+      if: ${{ needs.check-secret.outputs.available == 'true' }} 
       uses: docker/login-action@v3
       with:
-          registry: quay.io/sustainable_computing_io
-          username: ${{ secrets.BOT_NAME }}
-          password: ${{ secrets.BOT_TOKEN }}      
-    - name: Build and push base image
+          registry: ${{ inputs.image_repo }}
+          username: ${{ secrets.docker_username }}
+          password: ${{ secrets.docker_password }}    
+    - name: Build-push base image
+      if: ${{ (needs.check-secret.outputs.available == 'true') && ((needs.check-base-exist.outputs.exists == 'false') || (env.base_change == 'true')) }} 
       uses: docker/build-push-action@v5
       with:
         context: dockerfiles
         platforms: linux/amd64
         push: true
-        tags: quay.io/sustainable_computing_io/kepler_model_server_base:v0.7
+        tags: ${{ env.base_image }}
         file: dockerfiles/Dockerfile.base
-    - name: Build and push Docker image
-      uses: docker/build-push-action@v5
-      with:
-        context: .
-        platforms: linux/amd64
-        push: true
-        tags: quay.io/sustainable_computing_io/kepler_model_server:latest,quay.io/sustainable_computing_io/kepler_model_server:v0.7
-        file: dockerfiles/Dockerfile
-
-  build:
-    needs: [changes]
-    if: ${{ needs.changes.outputs.src == 'false' }}
-    runs-on: ubuntu-latest
-    steps:
-    - name: checkout
-      uses: actions/checkout@v4
-    - name: set up QEMU
-      uses: docker/setup-qemu-action@v3
-    - name: set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
-    - name: Login to Quay
-      if: ${{ (github.repository_owner == 'sustainable-computing-io') && (github.ref == 'refs/heads/main') }}
-      uses: docker/login-action@v3
-      with:
-          registry: quay.io/sustainable_computing_io
-          username: ${{ secrets.BOT_NAME }}
-          password: ${{ secrets.BOT_TOKEN }}      
-    - name: Build and push Docker image
+    - name: Replace value in file
+      if: ${{ (needs.check-secret.outputs.available == 'true') && ((needs.check-base-exist.outputs.exists == 'false') || (env.base_change == 'true')) }} 
+      run: |
+        sed -i "s|quay.io/sustainable_computing_io/kepler_model_server_base:v0.7|${{ env.base_image }}|" dockerfiles/Dockerfile
+    - name: Build-push image
       uses: docker/build-push-action@v5
       with:
         context: .
         platforms: linux/amd64
-        push: true
-        tags: quay.io/sustainable_computing_io/kepler_model_server:latest,quay.io/sustainable_computing_io/kepler_model_server:v0.7
+        push: ${{ inputs.push }}
+        tags: ${{ env.image }}
         file: dockerfiles/Dockerfile

.github/workflows/train-model-self-hosted.yml → .github/workflows/collect-data-self-hosted.yml

@@ -1,4 +1,4 @@
-name: Reusable workflow example
+name: Self-hosted Collect Data Workflow
 
 on:
   workflow_call:
@@ -31,10 +31,13 @@ on:
         description: 'The instance type to use for the EC2 instance'
         required: true
         type: string
+      model_server_image:
+        description: 'Kepler Model Server image'
+        required: true
+        type: string
 
 env:
   KUBECONFIG: /root/.kube/config
-  VERSION: 0.7
 
 jobs:
   setup-runner:
@@ -69,8 +72,8 @@ jobs:
           echo "instance_ip ${{ steps.create-runner.outputs.instance_ip }}"
           echo "runner_name ${{ steps.create-runner.outputs.runner_name }}"
           whoami
-  train-power-model:
-    name: Train Power Model
+  collect-data:
+    name: Collect Data
     needs: setup-runner
     runs-on: [self-hosted, linux, x64]
 
@@ -121,6 +124,7 @@ jobs:
           kubectl apply --filename https://storage.googleapis.com/tekton-releases/pipeline/latest/release.yaml
           ./hack/k8s_helper.sh rollout_ns_status tekton-pipelines
           ./hack/k8s_helper.sh rollout_ns_status tekton-pipelines-resolvers
+          
       - name: Prepare PVC
         working-directory: model_training/tekton
         run: |
@@ -145,13 +149,13 @@ jobs:
           kubectl apply -f tasks
           kubectl apply -f tasks/s3-pusher
           kubectl apply -f pipelines
-      - name: Run Tekton Pipeline
+      - name: Run Tekton Pipeline with S3Push
         run: |
           cat <<EOF | kubectl apply -f -
           apiVersion: tekton.dev/v1
           kind: PipelineRun
           metadata:
-            name: self-hosted-aws
+            name: self-hosted-aws-collect
           spec:
             timeouts:
               pipeline: "6h"
@@ -161,25 +165,21 @@ jobs:
               persistentVolumeClaim:
                 claimName: task-pvc
             params:
-            - name: PIPELINE_NAME
-              value: std_v${VERSION}
-            - name: OUTPUT_TYPE
-              value: AbsPower
+            - name: MODEL_SERVER_IMAGE
+              value: ${{ inputs.model_server_image }}}
             - name: COS_PROVIDER
               value: aws
             - name: COS_SECRET_NAME
               value: aws-cos-secret
             - name: MACHINE_ID
               value: ${{ inputs.instance_type }}-${{ inputs.ami_id }}
             pipelineRef:
-              name: single-train-pipeline
+              name: collect-data-pipeline
           EOF
-          ./hack/k8s_helper.sh wait_for_pipelinerun self-hosted-aws
-          df -h
-  
+
   destroy-runner:
     if: always()
-    needs: [setup-runner, train-power-model]
+    needs: [setup-runner, collect-data]
     name: Destroy Self Hosted Runner
     runs-on: ubuntu-latest
     steps:

.github/workflows/collect-train.yml

@@ -0,0 +1,40 @@
+# manually run on collect needed
+on:
+  workflow_dispatch:
+
+jobs:
+  collect-data:
+    uses: ./.github/workflows/collect-data-self-hosted.yml
+    strategy:
+      matrix:
+        instance_type: [i3.metal]
+      max-parallel: 1
+    with:
+      instance_type: ${{ matrix.instance_type }}
+      ami_id: 'ami-0e4d0bb9670ea8db0'
+      github_repo: ${{ github.repository }}
+      model_server_image: ${{ vars.IMAGE_REPO }}/kepler-model-server:v0.7
+    secrets:
+      self_hosted_github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }}
+      aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      security_group_id: ${{ secrets.AWS_SECURITY_GROUP_ID }}
+      aws_region: ${{ secrets.AWS_REGION }}
+
+  train-model:
+    strategy:
+      matrix:
+        instance_type: [i3.metal]
+    uses: ./.github/workflows/train-model.yml
+    with:
+      pipeline_name: std_v0.7
+      instance_type: ${{ matrix.instance_type }}
+      ami_id: 'ami-0e4d0bb9670ea8db0'
+      github_repo: ${{ github.repository }}
+      model_server_image: ${{ vars.IMAGE_REPO }}/kepler-model-server:v0.7
+      trainers: LogisticRegressionTrainer,ExponentialRegressionTrainer,SGDRegressorTrainer,GradientBoostingRegressorTrainer,XgboostFitTrainer
+    secrets:
+      self_hosted_github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }}
+      aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      aws_region: ${{ secrets.AWS_REGION }}

.github/workflows/integration-test.yaml → .github/workflows/integration-test.yml

@@ -1,8 +1,29 @@
+name: Integration Test
 on:
-  push:
-  pull_request:
+  workflow_call:
+    secrets:
+      docker_username:
+        description: 'Docker username'
+        required: false
+      docker_password:
+        description: 'Docker password'
+        required: false
+    inputs:
+      base_change:
+        description: 'Change flag on base image'
+        required: true
+        type: string
+      image_repo:
+        description: 'The image repo to use'
+        required: true
+        type: string
+      image_tag:
+        description: 'The image tag to use'
+        required: true
+        type: string
 
 env:
+  BASE_IMAGE: ${{ inputs.image_repo }}/kepler_model_server_base:${{ inputs.image_tag }}
   IMAGE: localhost:5001/kepler_model_server:devel
 
 jobs:
@@ -21,6 +42,21 @@ jobs:
           kind load docker-image quay.io/sustainable_computing_io/kepler:latest
       - name: checkout
         uses: actions/checkout@v4
+      - name: Login to Docker
+        if: ${{ inputs.base_change == 'true' }} 
+        uses: docker/login-action@v3
+        with:
+            registry: ${{ inputs.image_repo }}
+            username: ${{ secrets.docker_username }}
+            password: ${{ secrets.docker_password }} 
+      - name: Replace value in Dockerfile
+        if: ${{ inputs.base_change == 'true' }} 
+        run: |
+          sed -i "s|quay.io/sustainable_computing_io/kepler_model_server_base:v0.7|${{ env.BASE_IMAGE }}|" dockerfiles/Dockerfile
+      - name: Replace value in Dockerfile.test
+        if: ${{ inputs.base_change == 'true' }} 
+        run: |
+          sed -i "s|quay.io/sustainable_computing_io/kepler_model_server_base:v0.7|${{ env.BASE_IMAGE }}|" dockerfiles/Dockerfile.test
       - name: set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: set up Docker Buildx