Pkg 6001 win

abs.yaml

@@ -2,7 +2,4 @@
 # variant, so it's specified for both.
 extra_labels_for_os:
   osx-arm64: [ventura]
-aggregate_check: false
-
-channels:
-  - https://staging.continuum.io/prefect/fs/sympy-feedstock/pr10/3afd78c
+aggregate_check: false

recipe/bld.bat

@@ -1,12 +1,12 @@
 @echo On
+setlocal enabledelayedexpansion
 
 :: The PyTorch test suite includes some symlinks, which aren't resolved on Windows, leading to packaging errors.
 :: ATTN! These change and have to be updated manually, often with each release. 
 :: (no current symlinks being packaged. Leaving this information here as it took some months to find the issue. Look out
 :: for a failure with error message: "conda_package_handling.exceptions.ArchiveCreationError: <somefile> Cannot stat
 :: while writing file")
 
-set TH_BINARY_BUILD=1
 set PYTORCH_BUILD_VERSION=%PKG_VERSION%
 set PYTORCH_BUILD_NUMBER=%PKG_BUILDNUM%
 
@@ -21,6 +21,12 @@ if "%pytorch_variant%" == "gpu" (
     set USE_CUDA=0
 )
 
+:: KINETO seems to require CUPTI and will look quite hard for it.
+:: CUPTI seems to cause trouble when users install a version of
+:: cudatoolkit different than the one specified at compile time.
+:: https://github.com/conda-forge/pytorch-cpu-feedstock/issues/135
+set "USE_KINETO=OFF"
+
 :: =============================== CUDA FLAGS> ======================================
 if "%build_with_cuda%" == "" goto cuda_flags_end
 
@@ -41,6 +47,7 @@ set USE_MKLDNN=1
 set USE_TENSORPIPE=0
 set DISTUTILS_USE_SDK=1
 set BUILD_TEST=0
+set INSTALL_TEST=0
 :: Don't increase MAX_JOBS to NUMBER_OF_PROCESSORS, as it will run out of heap
 set CPU_COUNT=1
 set MAX_JOBS=%CPU_COUNT%
@@ -64,9 +71,12 @@ set CUDNN_INCLUDE_DIR=%LIBRARY_PREFIX%\include
 :: =============================== CUDA< ======================================
 
 set CMAKE_GENERATOR=Ninja
+set "CMAKE_GENERATOR_TOOLSET="
 set "CMAKE_GENERATOR_PLATFORM="
 set "CMAKE_PREFIX_PATH=%LIBRARY_PREFIX%"
-set CMAKE_BUILD_TYPE=Release
+set "CMAKE_INCLUDE_PATH=%LIBRARY_INC%"
+set "CMAKE_LIBRARY_PATH=%LIBRARY_LIB%"
+set "CMAKE_BUILD_TYPE=Release"
 :: This is so that CMake finds the environment's Python, not another one
 set Python_EXECUTABLE=%PYTHON%
 set Python3_EXECUTABLE=%PYTHON%
@@ -81,10 +91,45 @@ set BLAS=MKL
 set INTEL_MKL_DIR=%LIBRARY_PREFIX%
 
 set "libuv_ROOT=%LIBRARY_PREFIX%"
-set "USE_SYSTEM_SLEEF=OFF"
-:: Note that BUILD_CUSTOM_PROTOBUF=OFF (which would use our protobuf) doesn't work properly as of last testing, and results in
-:: duplicate symbols at link time.
-:: set "BUILD_CUSTOM_PROTOBUF=OFF"
+set "USE_SYSTEM_SLEEF=ON"
+
+:: Use our protobuf
+set "BUILD_CUSTOM_PROTOBUF=OFF"
+set "USE_LITE_PROTO=ON"
+
+:: Here we split the build into two parts.
+:: 
+:: Both the packages libtorch and pytorch use this same build script.
+:: - The output of the libtorch package should just contain the binaries that are 
+::   not related to Python.
+:: - The output of the pytorch package contains everything except for the 
+::   non-python specific binaries.
+::
+:: This ensures that a user can quickly switch between python versions without the
+:: need to redownload all the large CUDA binaries.
+
+if "%PKG_NAME%" == "libtorch" (
+  :: For the main script we just build a wheel for libtorch so that the C++/CUDA
+  :: parts are built. Then they are reused in each python version.
+  set BUILD_LIBTORCH_WHL=1
+  set BUILD_PYTHON_ONLY=0
+
+  %PYTHON% setup.py bdist_wheel
+  %PYTHON% -m pip install --find-links=dist torch_no_python --no-build-isolation --no-deps
+
+  :: building libtorch generates an empty __init__.py file. We don't want to package that, it should come from pytorch.
+  rm %SP_DIR%\torch\__init__.py
+) else (
+  set BUILD_LIBTORCH_WHL=0
+  :: In theory we want BUILD_PYTHON_ONLY=1 but that ends up causing lots of linking problems.
+  :: set BUILD_PYTHON_ONLY=1
+
+  :: NOTE: Passing --cmake is necessary here since the torch frontend has it's
+  :: own cmake files that it needs to generate
+  cmake -DPython_EXECUTABLE="%PYTHON%" --build build --target clean
+  %PYTHON% setup.py bdist_wheel --cmake
+  %PYTHON% -m pip install --find-links=dist torch --no-build-isolation --no-deps
+)
 
-%PYTHON% -m pip install . --no-deps --no-build-isolation -vv
 if errorlevel 1 exit /b 1
+

recipe/build_pytorch.bat

@@ -1 +1,5 @@
+@echo On
+setlocal enabledelayedexpansion
+
 call %RECIPE_DIR%\bld.bat
+if errorlevel 1 exit /b 1

recipe/meta.yaml

@@ -75,7 +75,7 @@ build:
     - python *                               # [megabuild]
     - numpy *                                # [megabuild]
   skip: True  # [py<39]
-  skip: True  # [win]
+  skip: True  # [not win]
 
 requirements:
   # Keep this list synchronized (except for python*, numpy*) in outputs
@@ -114,12 +114,13 @@ requirements:
     # This has a strong run_export so we don't need to put it in `host` or `run`
     # We use llvm-openmp for openblas variants on osx.
     - llvm-openmp 14.0.6              # [osx and not (blas_impl == "mkl")]
+    - libuv     # [win]
     - cmake
     - ninja-base
     # Keep libprotobuf here so that a compatibile version
     # of protobuf is installed between build and host
-    - libprotobuf    # [not win]
-    - protobuf       # [not win]
+    - libprotobuf
+    - protobuf
     - make      # [linux]
     # Uncomment to use ccache, see README and build_pytorch.sh
     # - ccache
@@ -147,15 +148,15 @@ requirements:
     # other requirements
     - python 3.12  # [megabuild]
     - python       # [not megabuild]
-    - numpy 2.*
+    - numpy 2
     - pip
-    - setuptools
+    - setuptools <=72.1.0
     - wheel
     - pyyaml
     - requests
     - future
     - six
-    - mkl-devel {{ mkl }}.*           # [blas_impl == "mkl"]
+    - mkl-devel {{ mkl }}           # [blas_impl == "mkl"]
     - openblas-devel {{ openblas }}   # [blas_impl == "openblas"]
     # - libcblas * *_mkl      # [blas_impl == "mkl"]
     # - libcblas              # [blas_impl != "mkl"]
@@ -167,8 +168,8 @@ requirements:
     - intel-openmp   {{ mkl }}        # [blas_impl == "mkl"]
     - llvm-openmp 14.0.6              # [osx and not (blas_impl == "mkl")]
     - libabseil
-    - libprotobuf {{ libprotobuf }}   # [not win]
-    - sleef 3.5.1                     # [not win]
+    - libprotobuf {{ libprotobuf }}
+    - sleef 3.5.1
     - typing
     - libuv
     - pkg-config  # [unix]
@@ -180,6 +181,7 @@ requirements:
   # satisfy overlinking checks
   run:
     - {{ pin_compatible('intel-openmp') }}   # [blas_impl == "mkl"]
+    - libuv  # [win]
 
 # these tests are for the libtorch output below, but due to
 # a particularity of conda-build, that output is defined in
@@ -199,6 +201,13 @@ outputs:
   - name: libtorch
     build:
       missing_dso_whitelist:
+        # The are dynamically loaded from %SP_DIR%\torch\lib\
+        - "**/asmjit.dll"             # [win]
+        - "**/c10.dll"                # [win]
+        - "**/fbgemm.dll"             # [win]
+        - "**/shm.dll"                # [win]
+        - "**/torch_cpu.dll"          # [win]
+        - "**/torch_python.dll"       # [win]
         - $RPATH/ld64.so.1  # [s390x]
   - name: pytorch
     build:
@@ -210,13 +219,19 @@ outputs:
       ignore_run_exports:             # [osx]
         - libuv                       # [osx]
       missing_dso_whitelist:
+        # The are dynamically loaded from %SP_DIR%\torch\lib\
+        - "**/asmjit.dll"             # [win]
+        - "**/c10.dll"                # [win]
+        - "**/fbgemm.dll"             # [win]
+        - "**/shm.dll"                # [win]
+        - "**/torch_cpu.dll"          # [win]
+        - "**/torch_python.dll"       # [win]
         - $RPATH/ld64.so.1  # [s390x]
       detect_binary_files_with_prefix: false
       run_exports:
         - {{ pin_subpackage('pytorch', max_pin='x.x') }}
         - {{ pin_subpackage('libtorch', max_pin='x.x') }}
       skip: True  # [py<39]
-      skip: True  # [win]
 
     script: build_pytorch.sh   # [unix]
     script: build_pytorch.bat  # [win]
@@ -256,8 +271,8 @@ outputs:
         - ninja-base
         # Keep libprotobuf here so that a compatibile version
         # of protobuf is installed between build and host
-        - libprotobuf  # [not win]
-        - protobuf     # [not win]
+        - libprotobuf
+        - protobuf
         - make      # [linux]
         # Uncomment to use ccache, see README and build_pytorch.sh
         # - ccache
@@ -283,15 +298,15 @@ outputs:
         {% endif %}
         # other requirements
         - python
-        - numpy 2.*
+        - numpy 2
         - pip
-        - setuptools
+        - setuptools <=72.1.0
         - wheel
         - pyyaml
         - requests
         - future
         - six
-        - mkl-devel {{ mkl }}.*           # [blas_impl == "mkl"]
+        - mkl-devel {{ mkl }}           # [blas_impl == "mkl"]
         - openblas-devel {{ openblas }}   # [blas_impl == "openblas"]
         # - libcblas * *_mkl      # [blas_impl == "mkl"]
         # - libcblas              # [blas_impl != "mkl"]
@@ -303,8 +318,8 @@ outputs:
         - intel-openmp   {{ mkl }}        # [blas_impl == "mkl"]
         - llvm-openmp 14.0.6              # [osx and not (blas_impl == "mkl")]
         - libabseil
-        - libprotobuf {{ libprotobuf }}   # [not win]
-        - sleef 3.5.1                     # [not win]
+        - libprotobuf {{ libprotobuf }}
+        - sleef 3.5.1
         - typing
         - libuv
         - pkg-config  # [unix]
@@ -371,6 +386,8 @@ outputs:
         - pytest-rerunfailures
         - pytest-flakefinder
         - pytest-xdist
+        # Needed for test_autograd.py
+        - pybind11
       imports:
         - torch
       source_files:
@@ -379,6 +396,8 @@ outputs:
         # as of pytorch=2.0.0, there is a bug when trying to run tests without the tools
         - tools
         - smoke_test
+        # See files needed: https://github.com/pytorch/pytorch/blob/main/test/test_ops.py#L271-L274
+        - aten/src/ATen/native
       commands:
         # the smoke test script takes a bunch of env variables, defined below
         - set MATRIX_GPU_ARCH_VERSION="{{ '.'.join(cudatoolkit.split('.')[:2]) }}"              # [(gpu_variant == "cuda-11") and (win)]
@@ -411,7 +430,10 @@ outputs:
         # Note that the `|| true` expression will make the build continue even if the whole script falls over completely
         # (for example, in the case of missing imports). There doesn't seem to be a way of making a script exception return
         # non-zero but failing tests return zero.
-        - python ./test/run_test.py --core --continue-through-error || true
+        - python ./test/run_test.py --core --continue-through-error || true  # [not win]
+        # Hack to fix test_autograd.py so it can find pybind11
+        - mv %PREFIX%/include/pybind11_global/pybind11 %PREFIX%/include  # [win]
+        - cmd.exe /c "python ./test/run_test.py --core --continue-through-error" || ver > nul 2> nul  # [win]
         # The inductor tests test the torch.compile backend. Using the options below avoids running distributed tests,
         # which would be run if we used the --inductor option. (Distributed tests would only be correctly run on a multi-gpu test platform,
         # which we don't have.)