[CI] Compile with pytorch 2.4.0.dev20240514

.github/workflows/publish.yml

@@ -44,7 +44,7 @@ jobs:
           # manylinux docker image, but I haven't figured out how to install CUDA on manylinux.
           os: [ubuntu-20.04]
           python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
-          torch-version: ['2.0.1', '2.1.2', '2.2.2', '2.3.1', '2.4.0.dev20240512']
+          torch-version: ['2.0.1', '2.1.2', '2.2.2', '2.3.1', '2.4.0.dev20240514']
           cuda-version: ['11.8.0', '12.2.2']
           # We need separate wheels that either uses C++11 ABI (-D_GLIBCXX_USE_CXX11_ABI) or not.
           # Pytorch wheels currently don't use it, but nvcr images have Pytorch compiled with C++11 ABI.

flash_attn/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.6.0"
+__version__ = "2.6.0.post1"
 
 from flash_attn.flash_attn_interface import (
     flash_attn_func,

training/Dockerfile

@@ -85,7 +85,7 @@ RUN pip install transformers==4.25.1 datasets==2.8.0 pytorch-lightning==1.8.6 tr
 RUN pip install git+https://github.com/mlcommons/[email protected]
 
 # Install FlashAttention
-RUN pip install flash-attn==2.6.0
+RUN pip install flash-attn==2.6.0.post1
 
 # Install CUDA extensions for fused dense
-RUN pip install git+https://github.com/HazyResearch/[email protected]#subdirectory=csrc/fused_dense_lib
+RUN pip install git+https://github.com/HazyResearch/[email protected].post1#subdirectory=csrc/fused_dense_lib