Merge pull request #272 from sthaha/upgrade-py-3.10

chore(python): upgrade to 3.10

Makefile

@@ -31,14 +31,14 @@ exec-test:
 
 test-pipeline:
 	mkdir -p ${MODEL_PATH}
-	$(CTR_CMD) run --platform linux/amd64 -v ${MODEL_PATH}:/mnt/models -i $(TEST_IMAGE) /bin/bash -c "python3.8 -u ./tests/pipeline_test.py"
+	$(CTR_CMD) run --platform linux/amd64 -v ${MODEL_PATH}:/mnt/models -i $(TEST_IMAGE) /bin/bash -c "python3.10 -u ./tests/pipeline_test.py"
 
 # test collector --> estimator
 run-estimator:
-	$(CTR_CMD) run -d --platform linux/amd64 -e "MODEL_TOPURL=http://localhost:8110" -v ${MODEL_PATH}:/mnt/models -p 8100:8100 --name estimator $(TEST_IMAGE) /bin/bash -c "python3.8 tests/http_server.py & sleep 5 && python3.8 src/estimate/estimator.py"
+	$(CTR_CMD) run -d --platform linux/amd64 -e "MODEL_TOPURL=http://localhost:8110" -v ${MODEL_PATH}:/mnt/models -p 8100:8100 --name estimator $(TEST_IMAGE) /bin/bash -c "python3.10 tests/http_server.py & sleep 5 && python3.10 src/estimate/estimator.py"
 
 run-collector-client:
-	$(CTR_CMD) exec estimator /bin/bash -c "while [ ! -S "/tmp/estimator.sock" ]; do sleep 1; done; python3.8 -u ./tests/estimator_power_request_test.py"
+	$(CTR_CMD) exec estimator /bin/bash -c "while [ ! -S "/tmp/estimator.sock" ]; do sleep 1; done; python3.10 -u ./tests/estimator_power_request_test.py"
 
 clean-estimator:
 	$(CTR_CMD) stop estimator
@@ -48,11 +48,11 @@ test-estimator: run-estimator run-collector-client clean-estimator
 
 # test estimator --> model-server
 run-model-server:
-	$(CTR_CMD) run -d --platform linux/amd64 -e "MODEL_TOPURL=http://localhost:8110" -v ${MODEL_PATH}:/mnt/models -p 8100:8100 --name model-server $(TEST_IMAGE) /bin/bash -c "python3.8 tests/http_server.py & sleep 10 &&  python3.8 src/server/model_server.py"
+	$(CTR_CMD) run -d --platform linux/amd64 -e "MODEL_TOPURL=http://localhost:8110" -v ${MODEL_PATH}:/mnt/models -p 8100:8100 --name model-server $(TEST_IMAGE) /bin/bash -c "python3.10 tests/http_server.py & sleep 10 &&  python3.10 src/server/model_server.py"
 	while ! docker logs model-server | grep -q Serving; do   echo "waiting for model-server to serve";  sleep 5; done
 
 run-estimator-client:
-	$(CTR_CMD) exec model-server /bin/bash -c "python3.8 -u ./tests/estimator_model_request_test.py"
+	$(CTR_CMD) exec model-server /bin/bash -c "python3.10 -u ./tests/estimator_model_request_test.py"
 
 clean-model-server:
 	@$(CTR_CMD) stop model-server
@@ -62,11 +62,11 @@ test-model-server: run-model-server run-estimator-client clean-model-server
 
 # test offline trainer
 run-offline-trainer:
-	$(CTR_CMD) run -d --platform linux/amd64  -p 8102:8102 --name offline-trainer $(TEST_IMAGE) python3.8 src/train/offline_trainer.py
+	$(CTR_CMD) run -d --platform linux/amd64  -p 8102:8102 --name offline-trainer $(TEST_IMAGE) python3.10 src/train/offline_trainer.py
 	sleep 5
 
 run-offline-trainer-client:
-	$(CTR_CMD) exec offline-trainer /bin/bash -c "python3.8 -u ./tests/offline_trainer_test.py"
+	$(CTR_CMD) exec offline-trainer /bin/bash -c "python3.10 -u ./tests/offline_trainer_test.py"
 
 clean-offline-trainer:
 	@$(CTR_CMD) stop offline-trainer
@@ -112,4 +112,4 @@ patch-power-request-client:
 	kubectl patch ds kepler-exporter -n kepler --patch-file ./manifests/test/power-request-client.yaml
 
 patch-model-request-client:
-	kubectl patch ds kepler-exporter -n kepler --patch-file ./manifests/test/model-request-client.yaml
+	kubectl patch ds kepler-exporter -n kepler --patch-file ./manifests/test/model-request-client.yaml

dockerfiles/Dockerfile

@@ -18,4 +18,4 @@ EXPOSE 8101
 # port for Offline Trainer
 EXPOSE 8102
 
-ENTRYPOINT ["python3.8", "cmd/main.py"]
+ENTRYPOINT ["python3.10", "cmd/main.py"]

dockerfiles/Dockerfile.base

@@ -1,3 +1,3 @@
-FROM python:3.8-slim
+FROM python:3.10-slim
 COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt

dockerfiles/Dockerfile.test

@@ -24,4 +24,4 @@ EXPOSE 8101
 # port for Offline Trainer
 EXPOSE 8102
 
-CMD [ "python3.8", "-u", "src/server/model_server.py" ]
+CMD [ "python3.10", "-u", "src/server/model_server.py" ]

dockerfiles/Dockerfile.test-nobase

@@ -1,5 +1,5 @@
 # Include base requirements
-FROM python:3.8-slim
+FROM python:3.10-slim
 COPY dockerfiles/requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
@@ -25,4 +25,4 @@ EXPOSE 8101
 # port for Offline Trainer
 EXPOSE 8102
 
-CMD [ "python3.8", "-u", "src/server/model_server.py" ]
+CMD [ "python3.10", "-u", "src/server/model_server.py" ]

manifests/base/patch/patch-estimator-sidecar.yaml

@@ -30,7 +30,7 @@ spec:
         name: kepler-exporter
       - image: kepler_model_server
         imagePullPolicy: IfNotPresent
-        command: [ "python3.8" ]
+        command: [ "python3.10" ]
         args: ["-u", "src/estimate/estimator.py" ]
         name: estimator
         volumeMounts:
@@ -45,4 +45,4 @@ spec:
       - emptyDir: {}
         name: tmp
       - emptyDir: {}
-        name: mnt
+        name: mnt

manifests/offline-trainer/offline-trainer.yaml

@@ -38,7 +38,7 @@ spec:
           - name: mnt
             mountPath: /mnt
             readOnly: false
-        command: [ "python3.8" ]
+        command: [ "python3.10" ]
         args: ["-u", "src/train/offline_trainer.py" ]
 
 ---
@@ -58,4 +58,4 @@ spec:
   ports:
   - name: http
     port: 8102
-    targetPort: http
+    targetPort: http

manifests/server/online-train/patch-trainer.yaml

@@ -30,5 +30,5 @@ spec:
           - name: mnt
             mountPath: /mnt
             readOnly: false
-        command: [ "python3.8" ]
+        command: [ "python3.10" ]
         args: ["-u", "src/train/online_trainer.py" ]

manifests/server/server.yaml

@@ -44,7 +44,7 @@ spec:
           - name: mnt
             mountPath: /mnt
             readOnly: false
-        command: [ "python3.8" ]
+        command: [ "python3.10" ]
         args: ["-u", "src/server/model_server.py" ]
 ---
 kind: Service
@@ -63,4 +63,4 @@ spec:
   ports:
   - name: http
     port: 8100
-    targetPort: http
+    targetPort: http

manifests/test/file-server.yaml

@@ -11,7 +11,7 @@ spec:
     image: localhost:5001/kepler_model_server:devel-test
     imagePullPolicy: IfNotPresent
     command: [ "/bin/sh" ]
-    args: ["-c", "python3.8 -u tests/http_server.py"]
+    args: ["-c", "python3.10 -u tests/http_server.py"]
     ports:
     - containerPort: 8110
       name: http
@@ -23,7 +23,7 @@ spec:
     image: localhost:5001/kepler_model_server:devel-test
     imagePullPolicy: IfNotPresent
     command: [ "/bin/sh" ]
-    args: ["-c", "python3.8 -u tests/minimal_trainer.py"]
+    args: ["-c", "python3.10 -u tests/minimal_trainer.py"]
     volumeMounts:
     - name: mnt
       mountPath: /mnt
@@ -46,4 +46,4 @@ spec:
   ports:
   - name: http
     port: 8110
-    targetPort: http
+    targetPort: http

manifests/test/model-request-client.yaml

@@ -11,7 +11,7 @@ spec:
         image: localhost:5001/kepler_model_server:devel-test
         imagePullPolicy: IfNotPresent
         command: [ "/bin/sh" ]
-        args: ["-c", "python3.8 -u tests/weight_model_request_test.py && echo Done && sleep infinity"]
+        args: ["-c", "python3.10 -u tests/weight_model_request_test.py && echo Done && sleep infinity"]
         volumeMounts:
         - name: cfm
           mountPath: /etc/kepler/kepler.config

manifests/test/power-request-client.yaml

@@ -11,7 +11,7 @@ spec:
         image: localhost:5001/kepler_model_server:devel-test
         imagePullPolicy: IfNotPresent
         command: [ "/bin/sh" ]
-        args: ["-c", "until [ -e /tmp/estimator.sock ]; do sleep 1; done && python3.8 -u tests/estimator_power_request_test.py && echo Done && sleep infinity"]
+        args: ["-c", "until [ -e /tmp/estimator.sock ]; do sleep 1; done && python3.10 -u tests/estimator_power_request_test.py && echo Done && sleep infinity"]
         volumeMounts:
         - name: cfm
           mountPath: /etc/kepler/kepler.config

model_training/s3/Dockerfile

@@ -6,4 +6,4 @@ COPY . /usr/local
 
 RUN pip install boto3 ibm-cos-sdk
 
-ENTRYPOINT ["python3.8"]
+ENTRYPOINT ["python3.10"]

model_training/tekton/pipelines/collect.yaml

@@ -96,7 +96,7 @@ spec:
             - --benchmark=idle
             - -o=idle
             - --id=$(params.MACHINE_ID)
-            command: ["python3.8"]
+            command: ["python3.10"]
             env:
             - name: PROM_SERVER
               value: http://prometheus-k8s.monitoring.svc:9090
@@ -151,7 +151,7 @@ spec:
             - --id=$(params.MACHINE_ID)
             - --benchmark=stressng
             - -o=kepler_query
-            command: ["python3.8"]
+            command: ["python3.10"]
             env:
             - name: PROM_SERVER
               value: http://prometheus-k8s.monitoring.svc:9090
@@ -190,4 +190,4 @@ spec:
       - name: MACHINE_ID
         value: $(params.MACHINE_ID)
       taskRef:
-        name: aws-s3-push
+        name: aws-s3-push

model_training/tekton/pipelines/complete-train.yaml

@@ -112,7 +112,7 @@ spec:
             - --benchmark=idle
             - -o=idle
             - --id=$(params.MACHINE_ID)
-            command: ["python3.8"]
+            command: ["python3.10"]
             env:
             - name: PROM_SERVER
               value: http://prometheus-k8s.monitoring.svc:9090
@@ -167,7 +167,7 @@ spec:
             - --benchmark=stressng
             - -o=kepler_query
             - --id=$(params.MACHINE_ID)
-            command: ["python3.8"]
+            command: ["python3.10"]
             env:
             - name: PROM_SERVER
               value: http://prometheus-k8s.monitoring.svc:9090
@@ -231,4 +231,4 @@ spec:
       - name: MACHINE_ID
         value: $(params.MACHINE_ID)
       taskRef:
-        name: aws-s3-push
+        name: aws-s3-push

model_training/tekton/pipelines/single-train.yaml

@@ -120,7 +120,7 @@ spec:
             - --benchmark=idle
             - -o=idle
             - --id=$(params.MACHINE_ID)
-            command: ["python3.8"]
+            command: ["python3.10"]
             env:
             - name: PROM_SERVER
               value: http://prometheus-k8s.monitoring.svc:9090
@@ -175,7 +175,7 @@ spec:
             - --benchmark=stressng
             - -o=kepler_query
             - --id=$(params.MACHINE_ID)
-            command: ["python3.8"]
+            command: ["python3.10"]
             env:
             - name: PROM_SERVER
               value: http://prometheus-k8s.monitoring.svc:9090
@@ -323,4 +323,4 @@ spec:
       - name: MACHINE_ID
         value: $(params.MACHINE_ID)
       taskRef:
-        name: aws-s3-push
+        name: aws-s3-push

model_training/tekton/tasks/extract-task.yaml

@@ -37,7 +37,7 @@ spec:
   steps:
     - name: extract
       image: $(params.MODEL_SERVER_IMAGE)
-      command: ["python3.8"]
+      command: ["python3.10"]
       args:
       - cmd/main.py
       - extract
@@ -48,4 +48,4 @@ spec:
       - --feature-group=$(params.FEATURE_GROUP)
       - --energy-source=$(params.ENERGY_SOURCE)
       - --output-type=$(params.OUTPUT_TYPE)
-      - --thirdparty-metrics="$(params.THIRDPARTY_METRICS)"
+      - --thirdparty-metrics="$(params.THIRDPARTY_METRICS)"

model_training/tekton/tasks/isolate-task.yaml

@@ -47,7 +47,7 @@ spec:
   steps:
     - name: isolate
       image: $(params.MODEL_SERVER_IMAGE)
-      command: ["python3.8"]
+      command: ["python3.10"]
       args:
       - cmd/main.py
       - isolate
@@ -64,4 +64,4 @@ spec:
       - --abs-pipeline-name=$(params.ABS_PIPELINE_NAME)
       - --profile=idle
       - --target-hints="$(params.TARGET_HINTS)"
-      - --bg-hints="$(params.BG_HINTS)"
+      - --bg-hints="$(params.BG_HINTS)"

model_training/tekton/tasks/original-pipeline-task.yaml

@@ -49,7 +49,7 @@ spec:
   steps:
     - name: pipeline-train
       image: $(params.MODEL_SERVER_IMAGE)
-      command: ["python3.8"]
+      command: ["python3.10"]
       env:
       - name: MODEL_PATH
         value: $(workspaces.mnt.path)/models
@@ -68,4 +68,4 @@ spec:
       - --dyn-trainers=$(params.DYN_TRAINERS)
       - --energy-source=$(params.ENERGY_SOURCE)
       - --thirdparty-metrics="$(params.THIRDPARTY_METRICS)"
-      - --id=$(params.MACHINE_ID)
+      - --id=$(params.MACHINE_ID)

model_training/tekton/tasks/s3/aws-s3-load.yaml

@@ -45,7 +45,7 @@ spec:
           secretKeyRef:
             name: $(params.COS_SECRET_NAME)
             key: bucketName
-      command: ["python3.8"]
+      command: ["python3.10"]
       args:
       - s3-loader.py
       - aws
@@ -55,4 +55,4 @@ spec:
       - --bucket-name=$(BUCKET_NAME)
       - --mnt-path=$(workspaces.mnt.path)
       - --pipeline-name=$(params.PIPELINE_NAME)
-      - --machine-id=$(params.MACHINE_ID)
+      - --machine-id=$(params.MACHINE_ID)

model_training/tekton/tasks/s3/aws-s3-push.yaml

@@ -42,7 +42,7 @@ spec:
           secretKeyRef:
             name: $(params.COS_SECRET_NAME)
             key: bucketName
-      command: ["python3.8"]
+      command: ["python3.10"]
       args:
       - s3-pusher.py
       - aws
@@ -53,4 +53,4 @@ spec:
       - --mnt-path=$(workspaces.mnt.path)
       - --query-data=kepler_query
       - --idle-data=idle
-      - --machine-id=$(params.MACHINE_ID)
+      - --machine-id=$(params.MACHINE_ID)

model_training/tekton/tasks/s3/ibmcloud-s3-load.yaml

@@ -45,7 +45,7 @@ spec:
           secretKeyRef:
             name: $(params.COS_SECRET_NAME)
             key: bucketName
-      command: ["python3.8"]
+      command: ["python3.10"]
       args:
       - s3-loader.py
       - ibmcloud
@@ -55,4 +55,4 @@ spec:
       - --bucket-name=$(BUCKET_NAME)
       - --mnt-path=$(workspaces.mnt.path)
       - --pipeline-name=$(params.PIPELINE_NAME)
-      - --machine-id=$(params.MACHINE_ID)
+      - --machine-id=$(params.MACHINE_ID)

model_training/tekton/tasks/s3/ibmcloud-s3-push.yaml

@@ -42,7 +42,7 @@ spec:
           secretKeyRef:
             name: $(params.COS_SECRET_NAME)
             key: bucketName
-      command: ["python3.8"]
+      command: ["python3.10"]
       args:
       - s3-pusher.py
       - ibmcloud
@@ -53,4 +53,4 @@ spec:
       - --mnt-path=$(workspaces.mnt.path)
       - --query-data=kepler_query
       - --idle-data=idle
-      - --machine-id=$(params.MACHINE_ID)
+      - --machine-id=$(params.MACHINE_ID)

model_training/tekton/tasks/train-task.yaml

@@ -43,7 +43,7 @@ spec:
   steps:
     - name: train-from-data
       image: $(params.MODEL_SERVER_IMAGE)
-      command: ["python3.8"]
+      command: ["python3.10"]
       env:
       - name: MODEL_PATH
         value: $(workspaces.mnt.path)/models
@@ -58,4 +58,4 @@ spec:
       - --output-type=$(params.OUTPUT_TYPE)
       - --trainers=$(params.TRAINERS)
       - --thirdparty-metrics="$(params.THIRDPARTY_METRICS)"
-      - --id=$(params.MACHINE_ID)
+      - --id=$(params.MACHINE_ID)