Skip to content

Commit

Permalink
Merge branch 'main' into jianyi/safe_softmax
Browse files Browse the repository at this point in the history
  • Loading branch information
@xytintel
xytintel authored Dec 24, 2024
2 parents 4ec9145 + bc99386 commit 9cff8f2
Show file tree
Hide file tree
Showing 3 changed files with 193 additions and 47 deletions.
3 changes: 1 addition & 2 deletions .github/scripts/apply_torch_pr.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@
# Fallback to CPU for XPU FP64
"https://github.com/pytorch/pytorch/pull/126516",
# Modify the tolerance level in TIMM benchmark
# "https://github.com/pytorch/pytorch/pull/129735",
"https://github.com/mengfei25/pytorch/pull/21",
"https://github.com/pytorch/pytorch/pull/143739",
]
)
parser.add_argument('--extra-pr-list', '-e', nargs='+',default=[])
Expand Down
235 changes: 190 additions & 45 deletions .github/workflows/_linux_transformers.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,6 @@ jobs:
rm -rf reports
cp ${{ github.workspace }}/torch-xpu-ops/.github/scripts/spec.py ./
- name: Report installed versions
id: installed
run: |
source activate huggingface_transformers_test
echo "pip installed packages:"
Expand All @@ -113,48 +112,6 @@ jobs:
lspci -d ::0380 | tee ${{ github.workspace }}/transformers/tests_log/lspci_0380.txt
echo "GPU render nodes:"
cat /sys/class/drm/render*/device/device | tee ${{ github.workspace }}/transformers/tests_log/device_IDs.txt
# printing annotations for the key packages
echo "### Annotations" >> $GITHUB_STEP_SUMMARY
echo "| | |" >> $GITHUB_STEP_SUMMARY
echo "| --- | --- |" >> $GITHUB_STEP_SUMMARY
echo "| jobs.$GITHUB_JOB.versions.os | $(source /etc/os-release && echo $VERSION_ID) |" >> $GITHUB_STEP_SUMMARY
echo "| jobs.$GITHUB_JOB.versions.linux-kernel | $(uname -r) |" >> $GITHUB_STEP_SUMMARY
echo "| jobs.$GITHUB_JOB.versions.python | $(python --version | cut -f2 -d' ') |" >> $GITHUB_STEP_SUMMARY
packages=" \
level-zero \
libigc1 \
libigc2 \
libze1 \
libze-intel-gpu1 \
intel-i915-dkms \
intel-level-zero-gpu \
intel-opencl-icd"
for package in $packages; do
package_version=$(dpkg -l | grep $package | grep ii | head -1 | sed "s/ */ /g" | cut -f3 -d" ")
echo "| jobs.$GITHUB_JOB.versions.$package | $package_version |" >> $GITHUB_STEP_SUMMARY
done
packages="accelerate \
numpy \
torch \
torchaudio \
torchvision \
transformers"
for package in $packages; do
package_version=$(python -c "import $package; print($package.__version__)")
echo "| jobs.$GITHUB_JOB.versions.$package | $package_version |" >> $GITHUB_STEP_SUMMARY
done
# printing annotations for GPU cards
var="[$(cat /sys/class/drm/render*/device/vendor)]"
echo "| jobs.$GITHUB_JOB.drm.render_nodes_vendor_ids | $(echo $var | sed "s/ /,/g") |" >> $GITHUB_STEP_SUMMARY
var="[$(cat /sys/class/drm/render*/device/device)]"
echo "| jobs.$GITHUB_JOB.drm.render_nodes_device_ids | $(echo $var | sed "s/ /,/g") |" >> $GITHUB_STEP_SUMMARY
var=$(python -c "import torch; print(torch.version.xpu)")
echo "| jobs.$GITHUB_JOB.torch.version.xpu | $var |" >> $GITHUB_STEP_SUMMARY
var=$(python -c "import torch; print(torch.xpu.device_count())")
echo "| jobs.$GITHUB_JOB.torch.xpu.device_count | $var |" >> $GITHUB_STEP_SUMMARY
# printing annotations with key environment variables
echo "| jobs.$GITHUB_JOB.env.ZE_AFFINITY_MASK | $ZE_AFFINITY_MASK |" >> $GITHUB_STEP_SUMMARY
echo "| jobs.$GITHUB_JOB.env.NEOReadDebugKeys | $NEOReadDebugKeys |" >> $GITHUB_STEP_SUMMARY
- name: Sanitry check installed packages
run: |
source activate huggingface_transformers_test
Expand All @@ -164,11 +121,199 @@ jobs:
pip show torchaudio | grep Version | grep xpu
pip show torchvision | grep Version | grep xpu
python -c 'import torch; exit(not torch.xpu.is_available())'
- name: Run XPU backbone
- name: Run -k backbone tests
env:
TEST_CASE: 'tests_backbone'
run: |
source activate huggingface_transformers_test
cd transformers
python3 -m pytest -rsf --make-reports=tests_benchmark -k backbone tests
python3 -m pytest -rsf --make-reports=$TEST_CASE -k backbone tests || \
(echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV)
- name: Run tests/*.py
env:
TEST_CASE: 'tests_py'
run: |
source activate huggingface_transformers_test
cd transformers
python3 -m pytest -rsf --make-reports=$TEST_CASE tests/*.py || true
- name: Run tests/benchmark
env:
TEST_CASE: 'tests_benchmark'
run: |
source activate huggingface_transformers_test
cd transformers
python3 -m pytest -rsf --make-reports=$TEST_CASE tests/benchmark || true
- name: Run tests/generation
env:
TEST_CASE: 'tests_generation'
run: |
source activate huggingface_transformers_test
cd transformers
# Excluding tests due to:
# * torch.distributed.* not yet supported by XPU
pattern="not TestFSDPGeneration"
python3 -m pytest -rsf --make-reports=$TEST_CASE tests/generation -k "$pattern" || true
- name: Run tests/models
env:
TEST_CASE: 'tests_models'
run: |
source activate huggingface_transformers_test
cd transformers
# Excluding tests due to:
# * https://github.com/huggingface/transformers/issues/35252 (CUDA specific tests)
# * https://github.com/pytorch/pytorch/issues/140965 (aten::_linalg_eigvals)
pattern=" \
not test_model_parallelization and \
not test_model_parallel_equal_results and \
not test_resize_embeddings_untied and \
not test_resize_tokens_embeddings"
python3 -m pytest -rsf --make-reports=$TEST_CASE tests/models -k "$pattern" || true
- name: Run tests/pipelines
env:
TEST_CASE: 'tests_pipelines'
run: |
source activate huggingface_transformers_test
cd transformers
# Some tests are known to fail w/o clear pattern
# TODO: drop ||true after triage and fixes
python3 -m pytest -rsf --make-reports=$TEST_CASE tests/pipelines || true
- name: Run tests/trainer
env:
TEST_CASE: 'tests_trainer'
run: |
source activate huggingface_transformers_test
cd transformers
# Excluding tests due to:
# * Some ray tests hang, reason unknown
# * torch.distributed.* not yet supported by XPU
pattern=" \
not ray and \
not TestTrainerDistributed and \
not TestTrainerDistributedXPU and \
not TestFSDPTrainer"
python3 -m pytest -rsf --make-reports=$TEST_CASE tests/trainer -k "$pattern" || \
(echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV)
- name: Run tests/utils
env:
TEST_CASE: 'tests_utils'
run: |
source activate huggingface_transformers_test
cd transformers
# Excluding tests due to:
# * Network proxy connection issue, reason unknown
pattern="not test_load_img_url_timeout"
python3 -m pytest -rsf --make-reports=$TEST_CASE tests/utils -k "$pattern" || \
(echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV)
- name: Check for errors in tests
run: |
FAILED_CASES=$(echo $FAILED_CASES | sed 's/^,//')
echo "Failed cases: [$(echo $FAILED_CASES | sed 's/,/, /g')]"
test -z "$FAILED_CASES"
- name: Print results table
if: ${{ ! cancelled() }}
run: |
# Helper function to return number preceeding given pattern, i.e:
# === 25 failed, 11 warnings, 0 errors ===
# Call as follows:
# parse_stat $line "failed"
function parse_stat() {
stat=$(cat $1 | grep $2 | sed "s/.* \([0-9]*\) $2.*/\1/")
if [ -n "$stat" ]; then echo $stat; else echo "0"; fi
}
cd transformers
{
echo "### Results"
echo "| Test group | Errors | Failed | Deselected | Passed | Skipped |"
echo "| --- | --- | --- | --- | --- | --- |"
for stat in $(find reports -name stats.txt); do
# Each stat.txt is located in: reports/$test_group/stats.txt
test_group=$(echo $stat | cut -f 2 -d/)
# Get failed, passed, skipped, etc. counters
failed=$(parse_stat $stat failed)
passed=$(parse_stat $stat passed)
deselected=$(parse_stat $stat deselected)
skipped=$(parse_stat $stat skipped)
warnings=$(parse_stat $stat warnings)
errors=$(parse_stat $stat errors)
echo "| $test_group | $errors | $failed | $deselected | $passed | $skipped |"
done
} >> $GITHUB_STEP_SUMMARY
- name: Print failure lines
if: ${{ ! cancelled() }}
run: |
cd transformers
{
echo "### Failure lines"
echo "| Test group |File | Error | Comment |"
echo "| --- | --- | --- | --- |"
rm -rf _failures.txt
for failure in $(find reports -name failures_line.txt); do
# Each failure_line.txt is located in: reports/$test_group/failure_line.txt
test_group=$(echo $failure | cut -f2 -d/)
tail -n +2 $failure | sed "s/^/$test_group /" >> _failures.txt
done
# failures_line.txt file does not have test case information,
# so we can just sort the output and report uniq values
sort _failures.txt | uniq > _failures_uniq.txt
while read line; do
test_group=$(echo $line | cut -f1 -d" ")
file=$(echo $line | cut -f2 -d" " | sed "s/\(.*\):$/\1/")
error=$(echo $line | cut -f3 -d" " | sed "s/\(.*\):$/\1/")
# Failure comments often contain special characters which complicate
# parsing failure lines. But fortunately we know for sure where comments
# start. So we just output all contents starting from this position and
# wrap everything in <pre></pre> to avoid collisions with Markdown formatting.
comment="<pre>$(echo $line | cut -f4- -d' ' | sed 's/\(.*\):$/\1/')</pre>"
echo "| $test_group | $file | $error | $comment |"
done <_failures_uniq.txt
} >> $GITHUB_STEP_SUMMARY
- name: Print annotations
if: ${{ ! cancelled() }}
run: |
source activate huggingface_transformers_test
{
echo "### Annotations"
echo "| | |"
echo "| --- | --- |"
echo "| jobs.$GITHUB_JOB.versions.os | $(source /etc/os-release && echo $VERSION_ID) |"
echo "| jobs.$GITHUB_JOB.versions.linux-kernel | $(uname -r) |"
echo "| jobs.$GITHUB_JOB.versions.python | $(python --version | cut -f2 -d' ') |"
packages=" \
level-zero \
libigc1 \
libigc2 \
libze1 \
libze-intel-gpu1 \
intel-i915-dkms \
intel-level-zero-gpu \
intel-opencl-icd"
for package in $packages; do
package_version=$(dpkg -l | grep $package | grep ii | head -1 | sed "s/ */ /g" | cut -f3 -d" ")
echo "| jobs.$GITHUB_JOB.versions.$package | $package_version |"
done
packages="accelerate \
numpy \
torch \
torchaudio \
torchvision \
transformers"
for package in $packages; do
package_version=$(python -c "import $package; print($package.__version__)" || true)
echo "| jobs.$GITHUB_JOB.versions.$package | $package_version |"
done
# printing annotations for GPU cards
var="[$(cat /sys/class/drm/render*/device/vendor || true)]"
echo "| jobs.$GITHUB_JOB.drm.render_nodes_vendor_ids | $(echo $var | sed 's/ /,/g') |"
var="[$(cat /sys/class/drm/render*/device/device || true)]"
echo "| jobs.$GITHUB_JOB.drm.render_nodes_device_ids | $(echo $var | sed 's/ /,/g') |"
var=$(python -c "import torch; print(torch.version.xpu)" || true)
echo "| jobs.$GITHUB_JOB.torch.version.xpu | $var |"
var=$(python -c "import torch; print(torch.xpu.device_count())" || true)
echo "| jobs.$GITHUB_JOB.torch.xpu.device_count | $var |"
# printing annotations with key environment variables
echo "| jobs.$GITHUB_JOB.env.ZE_AFFINITY_MASK | $ZE_AFFINITY_MASK |"
echo "| jobs.$GITHUB_JOB.env.NEOReadDebugKeys | $NEOReadDebugKeys |"
} >> $GITHUB_STEP_SUMMARY
- name: Upload Test log
if: ${{ ! cancelled() }}
uses: actions/upload-artifact@v4
Expand Down
2 changes: 2 additions & 0 deletions src/ATen/native/transformers/SDPUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

namespace sdp {

using c10::array_of;

bool check_all_tensors_on_device(sdp_params const& params, bool debug) {
// Check that all tensors are on the GPU device
// This should be handled by the stub dispatch, but whe call
Expand Down

0 comments on commit 9cff8f2

Please sign in to comment.