Adds ci-queue-pct lambda code to aws/lambdas and include it to the re…

…lease (#6259) Move from meta closed repository to test-infra repository the code for ci-queue-pct lambda. This lambda is the one that computes the queue percentile times used in https://hud.pytorch.org/sli The reason for this migration is related to start the effort to organize our Lambdas, using this as an example of how we should be moving forward. Test workflow: https://github.com/pytorch/test-infra/actions/runs/13162528997 Created release: https://github.com/pytorch/test-infra/releases/tag/v20250205-165758
pytorch · Feb 5, 2025 · 3cf0808 · 3cf0808
1 parent 88e4f1e
commit 3cf0808
Show file tree

Hide file tree

Showing 6 changed files with 710 additions and 1 deletion.
diff --git a/.github/workflows/lambda-do-release-runners.yml b/.github/workflows/lambda-do-release-runners.yml
@@ -6,7 +6,7 @@ on:
         description: tag to create release
         required: true
 
-name: Upload Release for runners lambdas
+name: Upload Release for lambdas
 
 jobs:
   release-lambdas:
@@ -23,6 +23,7 @@ jobs:
         with:
           ref: ${{ inputs.tag }}
 
+      # Runner Binaries Syncer
       - name: Install dependencies - Runner Binaries Syncer
         run: yarn install
         working-directory: terraform-aws-github-runner/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer
@@ -38,6 +39,7 @@ jobs:
         with:
           args: zip runner-binaries-syncer.zip index.js
 
+      # ALI scaleUp scaleDown
       - name: Install dependencies - Runners
         run: yarn install
         working-directory: terraform-aws-github-runner/modules/runners/lambdas/runners
@@ -53,6 +55,7 @@ jobs:
         with:
           args: zip runners.zip index.js
 
+      # Webhook
       - name: Install dependencies - Webhook
         run: yarn install
         working-directory: terraform-aws-github-runner/modules/webhook/lambdas/webhook
@@ -71,5 +74,64 @@ jobs:
       - uses: ncipollo/release-action@v1
         with:
           artifacts: "runner-binaries-syncer.zip,runners.zip,webhook.zip"
+          allowUpdates: true
+          draft: true
           name: ${{ inputs.tag }}
           tag: ${{ inputs.tag }}
+          updateOnlyUnreleased: true
+
+  release-ci-queue-pct:
+    name: Upload Release for ci-queue-pct lambda
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      REF: ${{ inputs.tag }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.tag }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Build deployment.zip
+        working-directory: aws/lambda/ci-queue-pct
+        run: make deployment.zip
+
+      - name: Copy deployment.zip to root
+        run: cp aws/lambda/ci-queue-pct/deployment.zip ci-queue-pct.zip
+
+      - uses: ncipollo/release-action@v1
+        with:
+          artifacts: "ci-queue-pct.zip"
+          allowUpdates: true
+          draft: true
+          name: ${{ inputs.tag }}
+          tag: ${{ inputs.tag }}
+          updateOnlyUnreleased: true
+
+  finish-release:
+    needs:
+      - release-lambdas
+      - release-ci-queue-pct
+    name: Mark the release as final and publish it
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      REF: ${{ inputs.tag }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.tag }}
+      - uses: ncipollo/release-action@v1
+        with:
+          allowUpdates: true
+          draft: false
+          name: ${{ inputs.tag }}
+          tag: ${{ inputs.tag }}
+          updateOnlyUnreleased: true
diff --git a/aws/lambda/ci-queue-pct/.gitignore b/aws/lambda/ci-queue-pct/.gitignore
@@ -0,0 +1,3 @@
+*.zip
+deployment/
+venv/
diff --git a/aws/lambda/ci-queue-pct/Makefile b/aws/lambda/ci-queue-pct/Makefile
@@ -0,0 +1,27 @@
+all: run-local
+
+clean:
+	rm -rf deployment
+	rm -rf venv
+	rm -rf deployment.zip
+
+venv/bin/python:
+	virtualenv venv
+	venv/bin/pip install -r requirements.txt
+
+.PHONY: run-local
+run-local: venv/bin/python
+	venv/bin/python ci_queue_pct.py --max-hours 70 --worker-pool-size 8  # --rebuild-table
+
+.PHONY: run-local-loop
+run-local-loop: venv/bin/python
+	while true ; do venv/bin/python ci_queue_pct.py --max-hours 70 --worker-pool-size 8 ; done
+
+deployment.zip:
+	mkdir -p deployment
+	cp ci_queue_pct.py ./deployment/.
+	pip3.10 install -r requirements.txt -t ./deployment/. --platform manylinux2014_x86_64 --only-binary=:all: --implementation cp --python-version 3.10 --upgrade
+	cd ./deployment && zip -q -r ../deployment.zip .
+
+.PHONY: create-deployment-package
+create-deployment-package: deployment.zip