Gitlab CI: Windows Configs (spack#43967)

Add support for Gitlab CI on Windows This PR adds the config changes required to configure and execute Gitlab pipelines running Windows builds on Windows runners using the existing Gitlab CI infrastructure (and newly added Windows infrastructure). * Adds support for generating child pipelines dispatched to Windows runners * Refactors the relevant pre-scripts, scripts, and post scripts to be compatible with Windows * Adds Windows config section describing Windows jobs * Adds VTK as Windows build stack (to be expanded later) * Modifies proj to build on Windows * Refactors Windows rpath symlinking to avoid system libs and externals --------- Co-authored-by: Ryan Krattiger <[email protected]> Co-authored-by: Mike VanDenburgh <[email protected]> Co-authored-by: Todd Gamblin <[email protected]> Co-authored-by: Scott Wittenburg <[email protected]>
johnwparent · May 16, 2024 · 81fe460 · 81fe460
1 parent b894f99
commit 81fe460
Show file tree

Hide file tree

Showing 16 changed files with 220 additions and 31 deletions.
diff --git a/bin/spack.ps1 b/bin/spack.ps1
@@ -144,3 +144,5 @@ switch($SpackSubCommand)
     "unload" {Invoke-SpackLoad}
     default  {python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs}
 }
+
+exit $LASTEXITCODE
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
@@ -2531,8 +2531,14 @@ def establish_link(self):
 
         # for each binary install dir in self.pkg (i.e. pkg.prefix.bin, pkg.prefix.lib)
         # install a symlink to each dependent library
-        for library, lib_dir in itertools.product(self.rpaths, self.library_dependents):
-            self._link(library, lib_dir)
+
+        # do not rpath for system libraries included in the dag
+        # we should not be modifying libraries managed by the Windows system
+        # as this will negatively impact linker behavior and can result in permission
+        # errors if those system libs are not modifiable by Spack
+        if "windows-system" not in getattr(self.pkg, "tags", []):
+            for library, lib_dir in itertools.product(self.rpaths, self.library_dependents):
+                self._link(library, lib_dir)
 
 
 @system_path_filter

diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
@@ -684,6 +684,22 @@ def generate_gitlab_ci_yaml(
             "instead.",
         )
 
+    def ensure_expected_target_path(path):
+        """Returns passed paths with all Windows path separators exchanged
+        for posix separators only if copy_only_pipeline is enabled
+
+        This is required as copy_only_pipelines are a unique scenario where
+        the generate job and child pipelines are run on different platforms.
+        To make this compatible w/ Windows, we cannot write Windows style path separators
+        that will be consumed on by the Posix copy job runner.
+
+        TODO (johnwparent): Refactor config + cli read/write to deal only in posix
+        style paths
+        """
+        if copy_only_pipeline and path:
+            path = path.replace("\\", "/")
+        return path
+
     pipeline_mirrors = spack.mirror.MirrorCollection(binary=True)
     deprecated_mirror_config = False
     buildcache_destination = None
@@ -807,7 +823,7 @@ def generate_gitlab_ci_yaml(
             if scope not in include_scopes and scope not in env_includes:
                 include_scopes.insert(0, scope)
         env_includes.extend(include_scopes)
-        env_yaml_root["spack"]["include"] = env_includes
+        env_yaml_root["spack"]["include"] = [ensure_expected_target_path(i) for i in env_includes]
 
         if "gitlab-ci" in env_yaml_root["spack"] and "ci" not in env_yaml_root["spack"]:
             env_yaml_root["spack"]["ci"] = env_yaml_root["spack"].pop("gitlab-ci")
@@ -1228,6 +1244,9 @@ def main_script_replacements(cmd):
             "SPACK_REBUILD_EVERYTHING": str(rebuild_everything),
             "SPACK_REQUIRE_SIGNING": os.environ.get("SPACK_REQUIRE_SIGNING", "False"),
         }
+        output_vars = output_object["variables"]
+        for item, val in output_vars.items():
+            output_vars[item] = ensure_expected_target_path(val)
 
         # TODO: Remove this block in Spack 0.23
         if deprecated_mirror_config and remote_mirror_override:
@@ -1284,7 +1303,6 @@ def main_script_replacements(cmd):
     sorted_output = {}
     for output_key, output_value in sorted(output_object.items()):
         sorted_output[output_key] = output_value
-
     if known_broken_specs_encountered:
         tty.error("This pipeline generated hashes known to be broken on develop:")
         display_broken_spec_messages(broken_specs_url, known_broken_specs_encountered)

diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py
@@ -161,7 +161,11 @@ def windows_establish_runtime_linkage(self):
 
         Performs symlinking to incorporate rpath dependencies to Windows runtime search paths
         """
-        if sys.platform == "win32":
+        # If spec is an external, we should not be modifying its bin directory, as we would
+        # be doing in this method
+        # Spack should in general not modify things it has not installed
+        # we can reasonably expect externals to have their link interface properly established
+        if sys.platform == "win32" and not self.spec.external:
             self.win_rpath.add_library_dependent(*self.win_add_library_dependent())
             self.win_rpath.add_rpath(*self.win_add_rpath())
             self.win_rpath.establish_link()
@@ -2446,9 +2450,18 @@ def rpath(self):
 
         # on Windows, libraries of runtime interest are typically
         # stored in the bin directory
+        # Do not include Windows system libraries in the rpath interface
+        # these libraries are handled automatically by VS/VCVARS and adding
+        # Spack derived system libs into the link path or address space of a program
+        # can result in conflicting versions, which makes Spack packages less useable
         if sys.platform == "win32":
             rpaths = [self.prefix.bin]
-            rpaths.extend(d.prefix.bin for d in deps if os.path.isdir(d.prefix.bin))
+            rpaths.extend(
+                d.prefix.bin
+                for d in deps
+                if os.path.isdir(d.prefix.bin)
+                and "windows-system" not in getattr(d.package, "tags", [])
+            )
         else:
             rpaths = [self.prefix.lib, self.prefix.lib64]
             rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib))

diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -64,6 +64,11 @@ default:
     SPACK_TARGET_PLATFORM: "linux"
     SPACK_TARGET_ARCH: "ppc64le"
 
+.win64-msvc2019:
+  variables:
+    SPACK_TARGET_PLATFORM: "win64"
+    SPACK_TARGET_ARCH: "x86_64"
+
 ########################################
 # Job templates
 ########################################
@@ -72,6 +77,8 @@ default:
     PIPELINE_MIRROR_TEMPLATE: "single-src-protected-mirrors.yaml.in"
     # TODO: We can remove this when we drop the "deprecated" stack
     PUSH_BUILDCACHE_DEPRECATED: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+    SPACK_CI_CONFIG_ROOT: "${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/configs"
+    SPACK_CI_SCRIPTS_ROOT: "${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/scripts"
 
   rules:
     - if: $SPACK_CI_DISABLE_STACKS =~ /.+/ && $SPACK_CI_STACK_NAME =~ $SPACK_CI_DISABLE_STACKS
@@ -114,16 +121,8 @@ default:
 .generate-common:
   stage: generate
   script:
-    - uname -a || true
-    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
-    - nproc || true
-    - cat /proc/loadavg || true
-    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
-    - . "./share/spack/setup-env.sh"
     - spack --version
-    - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
-    - spack env activate --without-view .
-    - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
+    - spack env activate --without-view share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
     - spack
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
@@ -134,29 +133,25 @@ default:
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
-      ${CI_STACK_CONFIG_SCOPES}
       audit configs
-    - spack python -c "import os,sys; print(os.path.expandvars(sys.stdin.read()))"
-      < "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}" > "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
+    # Command below needs to be `spack python` due to naming differences accross platforms
+    - spack python ${SPACK_CI_SCRIPTS_ROOT}/common/expand_vars.py
+      "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}"
+      "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
     - spack config add -f "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
-    - mkdir -p "${CI_PROJECT_DIR}/jobs_scratch_dir"
+    - mkdir "${CI_PROJECT_DIR}/jobs_scratch_dir"
     - spack
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
-      ${CI_STACK_CONFIG_SCOPES}
       config blame > "${CI_PROJECT_DIR}/jobs_scratch_dir/spack.yaml.blame"
     - spack -v  --color=always
       --config-scope "${SPACK_CI_CONFIG_ROOT}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
       --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
-      ${CI_STACK_CONFIG_SCOPES}
       ci generate --check-index-only
       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
-  after_script:
-    - cat /proc/loadavg || true
-    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
   artifacts:
     paths:
       - "${CI_PROJECT_DIR}/jobs_scratch_dir"
@@ -179,6 +174,16 @@ default:
 # Generate without tags for cases using external runners
 .generate-base:
   extends: [ ".base-job", ".generate-common" ]
+  before_script:
+    - uname -a || true
+    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
+    - nproc || true
+    - cat /proc/loadavg || true
+    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
+    - . "./share/spack/setup-env.sh"
+  after_script:
+    - cat /proc/loadavg || true
+    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
 
 .generate-x86_64:
   extends: [ ".generate-base" ]
@@ -196,6 +201,25 @@ default:
   extends: [ ".generate-base" ]
   tags: ["spack", "public", "medium", "neoverse_v2"]
 
+.generate-win64:
+  extends: [ ".base-job", ".generate-common" ]
+  before_script:
+    - $ErrorActionOld=$ErrorActionPreference
+    - $ErrorActionPreference="SilentlyContinue"
+    - python -c"import psutil;print(psutil.getloadavg())"
+    - (Get-WmiObject Win32_PhysicalMemory | measure-object Capacity -sum).sum/1kb
+    - $ErrorActionPreference=$ErrorActionOld
+    - . .\share\spack\setup-env.ps1
+  after_script:
+    - $ErrorActionOld=$ErrorActionPreference
+    - $ErrorActionPreference="SilentlyContinue"
+    - python -c"import psutil;print(psutil.getloadavg())"
+    - (Get-WmiObject Win32_PhysicalMemory | measure-object Capacity -sum).sum/1kb
+    - $ErrorActionPreference=$ErrorActionOld
+
+  tags: ["spack", "public", "medium", "x86_64-win"]
+  image: "ghcr.io/johnwparent/windows-server21h2:sha-c749cf3"
+
 .generate-deprecated:
   extends: [ ".base-job" ]
   stage: generate
@@ -859,6 +883,15 @@ aws-pcluster-build-neoverse_v1:
     - echo $PATH
     - module avail
     - module list
+    - uname -a || true
+    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
+    - nproc || true
+    - cat /proc/loadavg || true
+    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
+    - . "./share/spack/setup-env.sh"
+  after_script:
+    - cat /proc/loadavg || true
+    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
 
 .generate-cray-rhel:
   tags: [ "cray-rhel-zen4", "public" ]
@@ -912,3 +945,25 @@ e4s-cray-sles-build:
   needs:
     - artifacts: True
       job: e4s-cray-sles-generate
+
+#######################################
+# Windows Visualization Tools
+#######################################
+.windows-vis:
+  extends: [".win64-msvc2019"]
+  variables:
+    SPACK_CI_STACK_NAME: windows-vis
+
+windows-vis-generate:
+  extends: [ ".generate-win64", ".windows-vis" ]
+
+windows-vis-build:
+  extends: [ ".build", ".windows-vis"]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: windows-vis-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: windows-vis-generate
diff --git a/share/spack/gitlab/cloud_pipelines/configs/win64/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/win64/ci.yaml
@@ -0,0 +1,18 @@
+ci:
+  pipeline-gen:
+  - build-job:
+      after_script::
+      - Write-Output "Done"
+
+      before_script::
+      - fsutil 8dot3name set C:\ 0
+      - . .\share\spack\setup-env.ps1
+      - If (Test-Path -path C:\\key\intermediate_ci_signing_key.gpg) { spack.ps1 gpg trust C:\\key\intermediate_ci_signing_key.gpg }
+      - If (Test-Path -path C:\\key\spack_public_key.gpg) { spack.ps1 gpg trust C:\\key\spack_public_key.gpg }
+
+      script::
+      - spack.ps1 env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
+      - spack.ps1 config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{hash}'"
+      - mkdir ${SPACK_ARTIFACTS_ROOT}/user_data
+      - spack.ps1 --backtrace ci rebuild | Tee-Object -FilePath "${env:SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt" 2>&1 | Tee-Object -FilePath "${env:SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt"
+      image: "ghcr.io/johnwparent/windows-server21h2:sha-c749cf3"
diff --git a/share/spack/gitlab/cloud_pipelines/configs/win64/config.yaml b/share/spack/gitlab/cloud_pipelines/configs/win64/config.yaml
@@ -0,0 +1,10 @@
+config:
+  build_stage::
+    - 'C:/spack stage'
+  install_tree:
+    root: "C:/spack install"
+    # Path lengths on windows doesn't support much padding
+    padded_length: 0
+    # Reduce the projections to only including the hash to avoid path length issues
+    projections:
+      all: '{hash}'
diff --git a/share/spack/gitlab/cloud_pipelines/configs/win64/packages.yaml b/share/spack/gitlab/cloud_pipelines/configs/win64/packages.yaml
@@ -0,0 +1,25 @@
+packages:
+  all:
+    target: [x86_64]
+  tbb:
+    require: "intel-tbb"
+  cmake:
+    externals:
+    - spec: [email protected]
+      prefix: "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake"
+    buildable: False
+  ninja:
+    externals:
+    - spec: [email protected]
+      prefix: "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\Ninja"
+    buildable: False
+  wgl:
+    externals:
+    - spec: [email protected] plat=x64
+      prefix: "C:\\Program Files (x86)\\Windows Kits\\10"
+    buildable: False
+  win-sdk:
+    externals:
+    - spec: [email protected] plat=x64
+      prefix: "C:\\Program Files (x86)\\Windows Kits\\10"
+    buildable: False
diff --git a/share/spack/gitlab/cloud_pipelines/configs/win64/x86_64/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/win64/x86_64/ci.yaml
@@ -0,0 +1,4 @@
+ci:
+  pipeline-gen:
+  - build-job:
+      tags: [x86_64-win]
diff --git a/share/spack/gitlab/cloud_pipelines/configs/win64/x86_64/packages.yaml b/share/spack/gitlab/cloud_pipelines/configs/win64/x86_64/packages.yaml
@@ -0,0 +1,3 @@
+packages:
+  all:
+    target: [x86_64]
diff --git a/share/spack/gitlab/cloud_pipelines/scripts/common/expand_vars.py b/share/spack/gitlab/cloud_pipelines/scripts/common/expand_vars.py
@@ -0,0 +1,10 @@
+import argparse
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument("input", type=argparse.FileType("r"))
+parser.add_argument("out", type=argparse.FileType("w"))
+
+args = parser.parse_args()
+
+args.out.write(os.path.expandvars(args.input.read()))
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/windows-vis/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/windows-vis/spack.yaml
@@ -0,0 +1,12 @@
+# Windows Visualization Stack
+# maintainers:
+# - John Parent (@johnwparent)
+# - Ryan Krattiger (@kwryankrattiger)
+
+spack:
+  view: false
+  specs:
+  - vtk
+
+  cdash:
+    build-group: Windows Visualization (Kitware)
diff --git a/var/spack/repos/builtin/packages/proj/package.py b/var/spack/repos/builtin/packages/proj/package.py
@@ -87,15 +87,14 @@ class Proj(CMakePackage, AutotoolsPackage):
         when="@6.2:9.1",
     )
 
-    patch("proj.cmakelists.5.0.patch", when="@5.0")
-    patch("proj.cmakelists.5.1.patch", when="@5.1:5.2")
-
     # https://proj.org/install.html#build-requirements
     with when("build_system=cmake"):
         # CMake 3.19 refactored the FindTiff module interface, update older proj's
         # to be compatible with this "new" interface
         # patch replaces the TIFF_LIBRARY variable (no longer used) with TIFF_LIBRARIES
-        patch("proj-8.1-cmake-3.29-new-tiff-interface.patch", when="+tiff @:9.1.0 ^[email protected]:")
+        patch("proj-8.1-cmake-3.29-new-tiff-interface.patch", when="+tiff @7:9.1.0 ^[email protected]:")
+        patch("proj.cmakelists.5.0.patch", when="@5.0")
+        patch("proj.cmakelists.5.1.patch", when="@5.1:5.2")
         conflicts("[email protected]:", when="@:7")
         depends_on("[email protected]:", when="@6:", type="build")
         depends_on("[email protected]:", when="@5", type="build")