Merge pull request #38 from NASA-IMPACT/ab/add-smce-config

Ab/add smce config
NASA-IMPACT · Mar 5, 2024 · 6c5d602 · 6c5d602
2 parents 257817b + 795efef
commit 6c5d602
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 32 deletions.
diff --git a/.github/workflows/config.py b/.github/workflows/config.py
@@ -1,17 +1,5 @@
 import os
 
-bucket_choice = os.environ.get("S3_BUCKET")
-bucket_options = {
-    "default": "s3://gcorradini-forge-runner-test",
-    "test": "s3://gcorradini-forge-runner-test",
-}
-s3_uri = bucket_options.get(bucket_choice)
-if not s3_uri:
-    raise ValueError(
-        f"'S3_BUCKET_OPTIONS_MAP' did not have a key for '{bucket_choice}'. Options are {bucket_options}"
-    )
-
-
 def calc_task_manager_resources(task_manager_process_memory):
     """
     illustration of Flink memory model:
@@ -104,8 +92,6 @@ def calc_task_manager_resources(task_manager_process_memory):
 )
 print(f"[ CALCULATED TASK MANAGER RESOURCES ]: {task_manager_resources}")
 
-
-BUCKET_PREFIX = s3_uri
 c.Bake.prune = bool(int(os.environ.get("PRUNE_OPTION")))
 c.Bake.container_image = "apache/beam_python3.10_sdk:2.52.0"
 c.Bake.bakery_class = "pangeo_forge_runner.bakery.flink.FlinkOperatorBakery"
@@ -127,11 +113,10 @@ def calc_task_manager_resources(task_manager_process_memory):
     "taskmanager.memory.managed.fraction": f"{task_manager_resources['task_memory_managed_fraction']}"
 }
 
+BUCKET_PREFIX = os.environ.get("OUTPUT_BUCKET")
 c.TargetStorage.fsspec_class = "s3fs.S3FileSystem"
 c.TargetStorage.root_path = f"{BUCKET_PREFIX}/{{job_name}}/output"
 c.TargetStorage.fsspec_args = {
-    "key": os.environ.get("S3_DEFAULT_AWS_ACCESS_KEY_ID"),
-    "secret": os.environ.get("S3_DEFAULT_AWS_SECRET_ACCESS_KEY"),
     "anon": False,
     "client_kwargs": {"region_name": "us-west-2"},
 }

diff --git a/.github/workflows/job-runner.yaml b/.github/workflows/job-runner.yaml
@@ -14,22 +14,18 @@ on:
         description: 'The subdir of the feedstock directory in the repo'
         required: true
         default: 'feedstock'
-      bucket:
-        description: 'This job runner leverages s3fs.S3FileSystem for your recipe cache and output. Choices currently are: "default"'
-        required: true
-        default: 'default'
       prune:
         description: 'Only run the first two time steps'
         required: true
         default: '0'
       parallelism:
-        description: 'Number of task managers to spin up'
+        description: 'Number of workers to run in parallel'
         required: true
         default: '1'
-      protocol:
-        description: 'What protocol to use when accessing files (s3 or https).'
+      auth_mode:
+        description: 'What auth mode (edl or iamrole) to use when accessing files.'
         required: false
-        default: 's3'
+        default: 'iamrole'
       resource_profile:
         description: 'jobs have different memory requirements so choose (small[7168M], medium[10240M], large[15360M], xlarge[20480M])'
         required: false
@@ -50,6 +46,7 @@ jobs:
     if: contains('["ranchodeluxe","abarciauskas-bgse", "norlandrhagen", "sharkinsspatial", "moradology", "thodson-usgs"]', github.actor)
     name: kickoff job ${{ needs.name-job.outputs.repo_name }}@${{ github.event.inputs.ref }}
     needs: name-job
+    environment: veda-smce
     outputs:
       job_name: ${{ steps.report_ids.outputs.job_name }}
       job_id: ${{ steps.report_ids.outputs.job_id }}
@@ -70,7 +67,6 @@ jobs:
         echo "Manually triggered workflow:  \
           ${{ github.event.inputs.repo }} \
           ${{ github.event.inputs.ref }} \
-          ${{ github.event.inputs.bucket }} \
           ${{ github.event.inputs.parallelism }} \
           ${{ github.event.inputs.prune }}"
 
@@ -94,7 +90,7 @@ jobs:
 
     - name: update kubeconfig with cluster
       run: |
-        aws eks update-kubeconfig --name pangeo-forge-v3 --region ${{ secrets.GH_ACTIONS_AWS_REGION }}
+        aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region ${{ secrets.GH_ACTIONS_AWS_REGION }}
 
     - name: execute recipe on k8s cluster
       id: executejob
@@ -122,18 +118,15 @@ jobs:
         FLINK_DASH=$(cat execute.log | grep -oP "You can run '\K[^']+(?=')")
         echo "FLINK_DASH=$FLINK_DASH" >> $GITHUB_ENV
       env:
-        EARTHDATA_TOKEN: ${{ secrets.EARTHDATA_TOKEN }}
         EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }}
         EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }}
         REPO: ${{ github.event.inputs.repo }}
         REF: ${{ github.event.inputs.ref }}
         FEEDSTOCK_SUBDIR: ${{ github.event.inputs.feedstock_subdir }}
         PRUNE_OPTION: ${{ github.event.inputs.prune }}
         PARALLELISM_OPTION: ${{ github.event.inputs.parallelism }}
-        S3_BUCKET: ${{ github.event.inputs.bucket }}
-        S3_DEFAULT_AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEFAULT_AWS_ACCESS_KEY_ID }}
-        S3_DEFAULT_AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEFAULT_AWS_SECRET_ACCESS_KEY }}
-        PROTOCOL: ${{ github.event.inputs.protocol }}
+        OUTPUT_BUCKET: $OUTPUT_BUCKET
+        AUTH_MODE: ${{ github.event.inputs.auth_mode }}
         RESOURCE_PROFILE: ${{ github.event.inputs.resource_profile }}
 
     - name: cleanup if "pangeo-forge-runner bake" failed
@@ -213,7 +206,7 @@ jobs:
 
       - name: update kubeconfig with cluster
         run: |
-          aws eks update-kubeconfig --name pangeo-forge-v3 --region ${{ secrets.GH_ACTIONS_AWS_REGION }}
+          aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region ${{ secrets.GH_ACTIONS_AWS_REGION }}
 
 #      - name: Setup upterm session
 #        uses: lhotari/action-upterm@v1