From c716cf1b4d6360ddd96f65b562b61c67ed4b3972 Mon Sep 17 00:00:00 2001 From: Dmitry Rogozhkin Date: Fri, 20 Dec 2024 15:08:39 -0800 Subject: [PATCH 1/7] ci/transformers: run tests in tests/utils Signed-off-by: Dmitry Rogozhkin --- .github/workflows/_linux_transformers.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/_linux_transformers.yml b/.github/workflows/_linux_transformers.yml index 65dde1b6d..8d4936705 100644 --- a/.github/workflows/_linux_transformers.yml +++ b/.github/workflows/_linux_transformers.yml @@ -146,6 +146,14 @@ jobs: not TestTrainerDistributedXPU and \ not TestFSDPTrainer" python3 -m pytest -rsf --make-reports=tests_trainer tests/trainer -k "$pattern" + - name: Run tests/utils + run: | + source activate huggingface_transformers_test + cd transformers + # Excluding tests due to: + # * Network proxy connection issue, reason unknown + pattern="not test_load_img_url_timeout" + python3 -m pytest -rsf --make-reports=tests_utils tests/utils -k "$pattern" - name: Print results table if: ${{ ! cancelled() }} run: | From 1073af04813a36e03133bae2db7aef32f43dff58 Mon Sep 17 00:00:00 2001 From: Dmitry Rogozhkin Date: Fri, 20 Dec 2024 15:49:19 -0800 Subject: [PATCH 2/7] ci/transformers: run all test groups even if one failed Signed-off-by: Dmitry Rogozhkin --- .github/workflows/_linux_transformers.yml | 24 +++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_linux_transformers.yml b/.github/workflows/_linux_transformers.yml index 8d4936705..5b340d4ae 100644 --- a/.github/workflows/_linux_transformers.yml +++ b/.github/workflows/_linux_transformers.yml @@ -122,18 +122,25 @@ jobs: pip show torchvision | grep Version | grep xpu python -c 'import torch; exit(not torch.xpu.is_available())' - name: Run -k backbone tests + env: + TEST_CASE: 'tests_backbone' run: | source activate huggingface_transformers_test cd transformers - python3 -m pytest -rsf --make-reports=tests_backbone -k backbone tests + python3 -m pytest -rsf --make-reports=$TEST_CASE -k backbone tests || \ + (echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV) - name: Run tests/pipelines + env: + TEST_CASE: 'tests_pipelines' run: | source activate huggingface_transformers_test cd transformers # Some tests are known to fail w/o clear pattern # TODO: drop ||true after triage and fixes - python3 -m pytest -rsf --make-reports=tests_pipelines tests/pipelines || true + python3 -m pytest -rsf --make-reports=$TEST_CASE tests/pipelines || true - name: Run tests/trainer + env: + TEST_CASE: 'tests_trainer' run: | source activate huggingface_transformers_test cd transformers @@ -145,15 +152,24 @@ jobs: not TestTrainerDistributed and \ not TestTrainerDistributedXPU and \ not TestFSDPTrainer" - python3 -m pytest -rsf --make-reports=tests_trainer tests/trainer -k "$pattern" + python3 -m pytest -rsf --make-reports=$TEST_CASE tests/trainer -k "$pattern" || \ + (echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV) - name: Run tests/utils + env: + TEST_CASE: 'tests_utils' run: | source activate huggingface_transformers_test cd transformers # Excluding tests due to: # * Network proxy connection issue, reason unknown pattern="not test_load_img_url_timeout" - python3 -m pytest -rsf --make-reports=tests_utils tests/utils -k "$pattern" + python3 -m pytest -rsf --make-reports=$TEST_CASE tests/utils -k "$pattern" || \ + (echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV) + - name: Check for errors in tests + run: | + FAILED_CASES=$(echo $FAILED_CASES | sed 's/^,//') + echo "Failed cases: [$(echo $FAILED_CASES | sed 's/,/, /g')]" + test -z "$FAILED_CASES" - name: Print results table if: ${{ ! cancelled() }} run: | From 989cb4833af59e1cad8e889e173513450ff22213 Mon Sep 17 00:00:00 2001 From: Dmitry Rogozhkin Date: Fri, 20 Dec 2024 16:36:41 -0800 Subject: [PATCH 3/7] ci/transformers: run tests in tests/benchmark Signed-off-by: Dmitry Rogozhkin --- .github/workflows/_linux_transformers.yml | 24 ++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/_linux_transformers.yml b/.github/workflows/_linux_transformers.yml index 5b340d4ae..7b07b2a52 100644 --- a/.github/workflows/_linux_transformers.yml +++ b/.github/workflows/_linux_transformers.yml @@ -129,6 +129,13 @@ jobs: cd transformers python3 -m pytest -rsf --make-reports=$TEST_CASE -k backbone tests || \ (echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV) + - name: Run tests/benchmark + env: + TEST_CASE: 'tests_benchmark' + run: | + source activate huggingface_transformers_test + cd transformers + python3 -m pytest -rsf --make-reports=$TEST_CASE tests/benchmark || true - name: Run tests/pipelines env: TEST_CASE: 'tests_pipelines' @@ -204,24 +211,27 @@ jobs: cd transformers { echo "### Failure lines" - echo "| File | Error | Comment |" - echo "| --- | --- | --- |" + echo "| Test group |File | Error | Comment |" + echo "| --- | --- | --- | --- |" rm -rf _failures.txt for failure in $(find reports -name failures_line.txt); do - tail -n +2 $failure >> _failures.txt + # Each failure_line.txt is located in: reports/$test_group/failure_line.txt + test_group=$(echo $failure | cut -f2 -d/) + tail -n +2 $failure | sed "s/^/$test_group /" >> _failures.txt done # failures_line.txt file does not have test case information, # so we can just sort the output and report uniq values sort _failures.txt | uniq > _failures_uniq.txt while read line; do - file=$(echo $line | cut -f1 -d" " | sed "s/\(.*\):$/\1/") - error=$(echo $line | cut -f2 -d" " | sed "s/\(.*\):$/\1/") + test_group=$(echo $line | cut -f1 -d" ") + file=$(echo $line | cut -f2 -d" " | sed "s/\(.*\):$/\1/") + error=$(echo $line | cut -f3 -d" " | sed "s/\(.*\):$/\1/") # Failure comments often contain special characters which complicate # parsing failure lines. But fortunately we know for sure where comments # start. So we just output all contents starting from this position and # wrap everything in 
 to avoid collisions with Markdown formatting.
-              comment="
$(echo $line | cut -f3- -d' ' | sed 's/\(.*\):$/\1/')
"
-              echo "| $file | $error | $comment |"
+              comment="
$(echo $line | cut -f4- -d' ' | sed 's/\(.*\):$/\1/')
"
+              echo "| $test_group | $file | $error | $comment |"
             done <_failures_uniq.txt
           } >> $GITHUB_STEP_SUMMARY
       - name: Print annotations

From 80d51928fa3e293d7dc5e3b40565768f7bcfddcb Mon Sep 17 00:00:00 2001
From: Dmitry Rogozhkin 
Date: Fri, 20 Dec 2024 16:58:59 -0800
Subject: [PATCH 4/7] ci/transformers: run tests in tests/generation

Signed-off-by: Dmitry Rogozhkin 
---
 .github/workflows/_linux_transformers.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/_linux_transformers.yml b/.github/workflows/_linux_transformers.yml
index 7b07b2a52..1b23f2f0d 100644
--- a/.github/workflows/_linux_transformers.yml
+++ b/.github/workflows/_linux_transformers.yml
@@ -136,6 +136,16 @@ jobs:
           source activate huggingface_transformers_test
           cd transformers
           python3 -m pytest -rsf --make-reports=$TEST_CASE tests/benchmark || true
+      - name: Run tests/generation
+        env:
+          TEST_CASE: 'tests_generation'
+        run: |
+          source activate huggingface_transformers_test
+          cd transformers
+          # Excluding tests due to:
+          # * torch.distributed.* not yet supported by XPU
+          pattern="not TestFSDPGeneration"
+          python3 -m pytest -rsf --make-reports=$TEST_CASE tests/generation -k "$pattern" || true
       - name: Run tests/pipelines
         env:
           TEST_CASE: 'tests_pipelines'

From 29a1969f959803383acd3c3dfc36ef25d8795e7a Mon Sep 17 00:00:00 2001
From: Dmitry Rogozhkin 
Date: Fri, 20 Dec 2024 17:14:14 -0800
Subject: [PATCH 5/7] ci/transformers: print number of deselected tests

Signed-off-by: Dmitry Rogozhkin 
---
 .github/workflows/_linux_transformers.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/_linux_transformers.yml b/.github/workflows/_linux_transformers.yml
index 1b23f2f0d..50a862835 100644
--- a/.github/workflows/_linux_transformers.yml
+++ b/.github/workflows/_linux_transformers.yml
@@ -201,18 +201,19 @@ jobs:
           cd transformers
           {
             echo "### Results"
-            echo "| Test group | Errors | Failed | Passed | Skipped |"
-            echo "| --- | --- | --- | --- | --- |"
+            echo "| Test group | Errors | Failed | Deselected | Passed | Skipped |"
+            echo "| --- | --- | --- | --- | --- | --- |"
             for stat in $(find reports -name stats.txt); do
               # Each stat.txt is located in: reports/$test_group/stats.txt
               test_group=$(echo $stat | cut -f 2 -d/)
               # Get failed, passed, skipped, etc. counters
               failed=$(parse_stat $stat failed)
               passed=$(parse_stat $stat passed)
+              deselected=$(parse_stat $stat deselected)
               skipped=$(parse_stat $stat skipped)
               warnings=$(parse_stat $stat warnings)
               errors=$(parse_stat $stat errors)
-              echo "| $test_group | $errors | $failed | $passed | $skipped |"
+              echo "| $test_group | $errors | $failed | $deselected | $passed | $skipped |"
             done
           } >> $GITHUB_STEP_SUMMARY
       - name: Print failure lines

From cb2af7e49126edf60ce50cb8ca5da8291499e83b Mon Sep 17 00:00:00 2001
From: Dmitry Rogozhkin 
Date: Fri, 20 Dec 2024 20:14:31 -0800
Subject: [PATCH 6/7] ci/transformers: run tests in tests/*.py

Signed-off-by: Dmitry Rogozhkin 
---
 .github/workflows/_linux_transformers.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/_linux_transformers.yml b/.github/workflows/_linux_transformers.yml
index 50a862835..ccc9c3afa 100644
--- a/.github/workflows/_linux_transformers.yml
+++ b/.github/workflows/_linux_transformers.yml
@@ -129,6 +129,13 @@ jobs:
           cd transformers
           python3 -m pytest -rsf --make-reports=$TEST_CASE -k backbone tests || \
             (echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV)
+      - name: Run tests/*.py
+        env:
+          TEST_CASE: 'tests_py'
+        run: |
+          source activate huggingface_transformers_test
+          cd transformers
+          python3 -m pytest -rsf --make-reports=$TEST_CASE tests/*.py || true
       - name: Run tests/benchmark
         env:
           TEST_CASE: 'tests_benchmark'

From 0a0ed3d4bc40452273bce09274bc81228bf01f15 Mon Sep 17 00:00:00 2001
From: Dmitry Rogozhkin 
Date: Fri, 20 Dec 2024 21:00:01 -0800
Subject: [PATCH 7/7] ci/transformers: run tests in tests/models

Signed-off-by: Dmitry Rogozhkin 
---
 .github/workflows/_linux_transformers.yml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/.github/workflows/_linux_transformers.yml b/.github/workflows/_linux_transformers.yml
index ccc9c3afa..f79227658 100644
--- a/.github/workflows/_linux_transformers.yml
+++ b/.github/workflows/_linux_transformers.yml
@@ -153,6 +153,21 @@ jobs:
           # * torch.distributed.* not yet supported by XPU
           pattern="not TestFSDPGeneration"
           python3 -m pytest -rsf --make-reports=$TEST_CASE tests/generation -k "$pattern" || true
+      - name: Run tests/models
+        env:
+          TEST_CASE: 'tests_models'
+        run: |
+          source activate huggingface_transformers_test
+          cd transformers
+          # Excluding tests due to:
+          # * https://github.com/huggingface/transformers/issues/35252 (CUDA specific tests)
+          # * https://github.com/pytorch/pytorch/issues/140965 (aten::_linalg_eigvals)
+          pattern=" \
+            not test_model_parallelization and \
+            not test_model_parallel_equal_results and \
+            not test_resize_embeddings_untied and \
+            not test_resize_tokens_embeddings"
+          python3 -m pytest -rsf --make-reports=$TEST_CASE tests/models -k "$pattern" || true
       - name: Run tests/pipelines
         env:
           TEST_CASE: 'tests_pipelines'