Benchmark #52
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # name of the workflow, what it is doing (optional) | |
| name: Benchmark | |
| # events that trigger the workflow (required) | |
| on: | |
| push: | |
| # pushes to the following branches | |
| branches: | |
| - main | |
| pull_request: | |
| # pull request where master is target | |
| branches: | |
| - main | |
| workflow_dispatch: # Add this line to allow manual triggering | |
| jobs: | |
| mot-metrics-benchmark: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ['3.11'] | |
| tracker: ["ocsort", "bytetrack", "botsort", "hybridsort", "deepocsort", "imprassoc", "strongsort"] | |
| timeout-minutes: 50 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'pip' | |
| - name: Install requirements | |
| run: | | |
| sudo apt-get install -y jq curl | |
| if [[ "$OSTYPE" == "darwin"* ]]; then | |
| # macOS | |
| sed -i '' 's/source="torch_cuda121"/source="torchcpu"/g' pyproject.toml | |
| elif [[ "$OSTYPE" == "linux-gnu"* ]]; then | |
| # Linux | |
| sed -i 's/source="torch_cuda121"/source="torchcpu"/g' pyproject.toml | |
| fi | |
| python -m pip install --upgrade pip setuptools wheel poetry | |
| poetry config virtualenvs.create false | |
| poetry lock --no-update | |
| poetry install --with yolo | |
| - name: Download eval tools repo | |
| run: | | |
| git clone https://github.com/JonathonLuiten/TrackEval.git tracking/val_utils | |
| # Restore the cached dataset (if available) | |
| - name: Restore MOT17 dataset cache | |
| uses: actions/cache@v3 | |
| with: | |
| # Specify the path where the dataset is stored | |
| path: tracking/val_utils/MOT17-50.zip | |
| # Create a cache key, you can use a fixed key if the dataset is static | |
| key: mot17-50-dataset-cache-v3 | |
| - name: Download eval data | |
| run: | | |
| wget https://github.com/mikel-brostrom/boxmot/releases/download/v10.0.83/runs.zip -O runs.zip | |
| wget https://github.com/mikel-brostrom/boxmot/releases/download/v10.0.83/MOT17-50.zip -O tracking/val_utils/MOT17-50.zip | |
| unzip runs.zip -d ./ | |
| mkdir -p tracking/val_utils/data | |
| unzip tracking/val_utils/MOT17-50.zip -d ./tracking/val_utils/data/ | |
| # Cache data for future runs (only if the cache was not already restored) | |
| - name: Cache MOT17.zip if not already cached | |
| if: steps.cache-restore.outputs.cache-hit != 'true' # Only run if the cache was not hit | |
| uses: actions/cache@v3 | |
| with: | |
| path: tracking/val_utils/MOT17-50.zip | |
| key: mot17-50-dataset-cache-v3 | |
| - name: Evaluation and Summarize Results | |
| run: | | |
| pwd | |
| ls tracking/val_utils | |
| ls tracking/val_utils/data | |
| if python3 tracking/val.py --imgsz 320 --classes 0 --benchmark MOT17-50 --yolo-model yolov8x.pt --reid-model osnet_x1_0_dukemtmcreid.pt --tracking-method ${{ matrix.tracker }} --verbose --source ./tracking/val_utils/data/MOT17-50/train; then | |
| STATUS="✅" | |
| else | |
| STATUS="❌" | |
| fi | |
| if [ -f ${{ matrix.tracker }}_output.json ]; then | |
| HOTA=$(jq -r '.HOTA' ${{ matrix.tracker }}_output.json) | |
| MOTA=$(jq -r '.MOTA' ${{ matrix.tracker }}_output.json) | |
| IDF1=$(jq -r '.IDF1' ${{ matrix.tracker }}_output.json) | |
| else | |
| HOTA="" | |
| MOTA="" | |
| IDF1="" | |
| fi | |
| mkdir results | |
| TRACKER_NAME=$(echo ${{ matrix.tracker }} | awk '{print toupper(substr($0,1,1)) tolower(substr($0,2))}') | |
| echo "$TRACKER_NAME,$STATUS,$HOTA,$MOTA,$IDF1" > results/${{ matrix.tracker }}.txt | |
| - name: Show Results | |
| run: cat results/${{ matrix.tracker }}.txt | |
| - name: Upload Results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: results | |
| path: results/${{ matrix.tracker }}.txt | |
| combine-results: | |
| runs-on: ubuntu-latest | |
| needs: mot-metrics-benchmark | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download all results | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: results | |
| path: results | |
| - name: Check downloaded files | |
| run: | | |
| echo "Downloaded files in the results directory:" | |
| ls -la results/ | |
| - name: Combine results | |
| run: | | |
| for file in results/*; do | |
| if [ -f "$file" ]; then | |
| cat "$file" >> combined_results.csv # Use cat to include all lines | |
| fi | |
| done | |
| # Sort the results by the third column (HOTA) in descending order | |
| sort -t, -k3 -nr combined_results.csv > sorted_results.csv | |
| # Create a pretty table from the sorted_results.csv file | |
| column -s, -t sorted_results.csv > pretty_results.txt | |
| - name: Show Combined Results | |
| run: cat pretty_results.txt | |
| - name: Set up Git | |
| run: | | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "mikel-brostrom" | |
| - name: Update README with tracker results | |
| run: | | |
| # Paths | |
| RESULTS_FILE="pretty_results.txt" | |
| README_FILE="README.md" | |
| # Backup original README in case needed | |
| cp "$README_FILE" "${README_FILE}.bak" | |
| # Create the markdown table header | |
| new_table="| Tracker | Status | HOTA↑ | MOTA↑ | IDF1↑ |\n" | |
| new_table+="| ------- | ------- | ----- | ----- | ----- |\n" | |
| # Append the contents of pretty_results.txt to the table | |
| while read -r line; do | |
| new_table+="| $(echo "$line" | awk '{print $1}') | $(echo "$line" | awk '{print $2}') | $(echo "$line" | awk '{print $3}') | $(echo "$line" | awk '{print $4}') | $(echo "$line" | awk '{print $5}') |\n" | |
| done < "$RESULTS_FILE" | |
| # Define unique markers to locate the table | |
| start_marker="<!-- START TRACKER TABLE -->" | |
| end_marker="<!-- END TRACKER TABLE -->" | |
| # Use awk to replace lines between markers, preserving non-table content | |
| awk -v start_marker="$start_marker" -v end_marker="$end_marker" -v new_table="$new_table" ' | |
| $0 == start_marker { print $0; print new_table; in_table=1; next } | |
| $0 == end_marker { in_table=0; print $0; next } | |
| !in_table | |
| ' "$README_FILE" > temp_readme.md | |
| # Replace original README with updated version | |
| mv temp_readme.md "$README_FILE" | |
| # Check for changes | |
| - name: Check for changes | |
| id: check_changes | |
| run: | | |
| if git diff --quiet; then | |
| echo "No changes to commit." | |
| echo "::set-output name=changed::false" | |
| else | |
| echo "::set-output name=changed::true" | |
| fi | |
| # Commit and push changes if there are any | |
| - name: Commit changes | |
| if: steps.check_changes.outputs.changed == 'true' | |
| run: | | |
| BRANCH_NAME="update-tracker-results-$(date +%Y%m%d%H%M%S)" | |
| git checkout -b "$BRANCH_NAME" | |
| git add README.md | |
| git commit -m "Overwrote tracker results in README.md" | |
| git push origin "$BRANCH_NAME" | |
| # Create a pull request (PR) | |
| - name: Create Pull Request | |
| if: steps.check_changes.outputs.changed == 'true' | |
| uses: peter-evans/create-pull-request@v5 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: ${{ github.ref }} | |
| commit-message: Overwrote tracker results in README.md | |
| title: Overwrite tracker results in README.md | |
| body: "This PR updates the tracker results table in the README.md." |