Merge branch 'main' into update-perplexity-ci

.github/workflows/build_packages.yml

@@ -12,6 +12,9 @@ on:
     # Runs at 11:00 AM UTC, which is 3:00 AM PST (UTC-8)
     - cron: '0 11 * * *'
 
+permissions:
+  contents: read
+
 jobs:
   # Note: metadata generation could happen in a separate trigger/schedule
   # workflow. For cross platform builds, it's useful to just generate the
@@ -40,7 +43,7 @@ jobs:
           sharktank_package_version=$(python3 build_tools/python_deploy/compute_local_version.py sharktank)
           shortfin_package_version=$(python3 build_tools/python_deploy/compute_local_version.py shortfin)
       - name: Upload version_local.json
-        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: version_local
           path: |
@@ -50,6 +53,8 @@ jobs:
   build_packages:
     name: "${{ matrix.package }} :: ${{ matrix.platform }} :: ${{ matrix.python-version }}"
     runs-on: ${{ matrix.runs-on }}
+    permissions:
+      contents: write
     needs: [setup_metadata]
     strategy:
       fail-fast: false
@@ -116,7 +121,7 @@ jobs:
         ./c/shortfin/build_tools/build_linux_package.sh
 
     - name: Upload python wheels
-      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
       with:
         if-no-files-found: error
         name: snapshot-${{ matrix.package }}-${{ matrix.platform }}-${{ matrix.python-version }}
@@ -126,7 +131,6 @@ jobs:
       uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5 # v1.14.0
       with:
         artifacts: bindist/*.whl
-        token: "${{ secrets.RELEASE_PUBLISH_ACCESS_TOKEN }}"
         tag: "dev-wheels"
         name: "dev-wheels"
         body: "Automatic snapshot release of shark-ai python wheels."

.github/workflows/ci-llama-large-tests.yaml

@@ -70,8 +70,8 @@ jobs:
 
           # Test with pinned nightly releases, not what iree-turbine uses.
           pip install -f https://iree.dev/pip-release-links.html --upgrade \
-            iree-base-compiler==3.0.0rc20241115 \
-            iree-base-runtime==3.0.0rc20241115
+            iree-base-compiler==3.0.0rc20241118 \
+            iree-base-runtime==3.0.0rc20241118
 
       - name: Run llama tests
         run: pytest sharktank/tests/models/llama/benchmark_amdgpu_test.py -v -s --run-all-llama --iree-hip-target=gfx942 --html=out/index.html

.github/workflows/ci-llama-quick-tests.yaml

@@ -71,14 +71,14 @@ jobs:
 
           # Test with pinned nightly releases, not what iree-turbine uses.
           pip install -f https://iree.dev/pip-release-links.html --upgrade \
-            iree-base-compiler==3.0.0rc20241115 \
-            iree-base-runtime==3.0.0rc20241115
+            iree-base-compiler==3.0.0rc20241118 \
+            iree-base-runtime==3.0.0rc20241118
 
       - name: Run llama 8b tests
         run: pytest sharktank/tests/models/llama/benchmark_amdgpu_test.py -v -s --iree-hip-target=gfx942 --run-8b-llama
 
       - name: Upload llama executable files
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: llama-files
           path: ${{ github.workspace }}/${{ steps.date.outputs.date }}

.github/workflows/ci-sdxl.yaml

@@ -64,7 +64,7 @@ jobs:
         repository: iree-org/iree
         path: ${{ env.IREE_REPO_DIR }}
         submodules: false
-        ref: iree-3.0.0rc20241115
+        ref: iree-3.0.0rc20241118
 
     - name: Initalize IREE submodules
       working-directory: ${{ env.IREE_REPO_DIR }}

.github/workflows/ci-sglang-benchmark.yml

@@ -69,8 +69,8 @@ jobs:
           # We could also pin to a known working or stable version.
           # This should eventually stabilize. Do the best we can for now.
           pip install -f https://iree.dev/pip-release-links.html --upgrade \
-            iree-base-compiler==3.0.0rc20241115 \
-            iree-base-runtime==3.0.0rc20241115 \
+            iree-base-compiler==3.0.0rc20241118 \
+            iree-base-runtime==3.0.0rc20241118 \
             "numpy<2.0"
 
       - name: Install SGLang

.github/workflows/ci_linux_x64-libshortfin.yml

@@ -59,7 +59,7 @@ jobs:
         repository: iree-org/iree
         path: ${{ env.IREE_REPO_DIR }}
         submodules: false
-        ref: iree-3.0.0rc20241115
+        ref: iree-3.0.0rc20241118
 
     - name: Initalize IREE submodules
       working-directory: ${{ env.IREE_REPO_DIR }}

.github/workflows/ci_linux_x64_asan-libshortfin.yml

@@ -109,7 +109,7 @@ jobs:
         repository: iree-org/iree
         path: ${{ env.IREE_SOURCE_DIR }}
         submodules: false
-        ref: iree-3.0.0rc20241115
+        ref: iree-3.0.0rc20241118
 
     - name: Initalize IREE submodules
       working-directory: ${{ env.IREE_SOURCE_DIR }}

.github/workflows/ci_linux_x64_nogil-libshortfin.yml

@@ -57,7 +57,7 @@ jobs:
         repository: iree-org/iree
         path: ${{ env.IREE_REPO_DIR }}
         submodules: false
-        ref: iree-3.0.0rc20241115
+        ref: iree-3.0.0rc20241118
 
     - name: Initalize IREE submodules
       working-directory: ${{ env.IREE_REPO_DIR }}

.github/workflows/ci_windows_x64-libshortfin.yml

@@ -54,7 +54,7 @@ jobs:
         repository: iree-org/iree
         path: ${{ env.IREE_REPO_DIR }}
         submodules: false
-        ref: iree-3.0.0rc20241115
+        ref: iree-3.0.0rc20241118
 
     - name: Initalize IREE submodules
       working-directory: ${{ env.IREE_REPO_DIR }}

docs/user_guide.md

@@ -17,7 +17,7 @@ Our current user guide requires that you have:
 
 This section will help you install Python and set up a Python environment with venv.
 
-Officially we support Python versions: 3.11, 3.12, 3.13, 3.13t
+Officially we support Python versions: 3.11, 3.12, 3.13
 
 The rest of this guide assumes you are using Python 3.11.
 
@@ -39,6 +39,10 @@ Setup your Python environment with the following commands:
 # Set up a virtual environment to isolate packages from other envs.
 python3.11 -m venv 3.11.venv
 source 3.11.venv/bin/activate
+
+# Optional: faster installation of torch with just CPU support.
+# See other options at https://pytorch.org/get-started/locally/
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
 ```
 
 ## Install SHARK and its dependencies

sharktank/version.json

@@ -1,3 +1,3 @@
 {
-  "package-version": "2.9.2.dev"
+  "package-version": "3.0.0.dev"
 }

shortfin/CMakeLists.txt

@@ -40,7 +40,7 @@ if(NOT WIN32)
 endif()
 
 # Pins
-set(SHORTFIN_IREE_GIT_TAG "iree-3.0.0rc20241115")
+set(SHORTFIN_IREE_GIT_TAG "iree-3.0.0rc20241118")
 
 # build options
 option(SHORTFIN_BUILD_PYTHON_BINDINGS "Builds Python Bindings" OFF)

shortfin/python/lib_ext.cc

@@ -428,16 +428,15 @@ ConfigOptions CreateConfigOptions(std::optional<std::string> &env_prefix,
 }  // namespace
 
 NB_MODULE(lib, m) {
-// Tragically, debug builds of Python do the right thing and don't immortalize
-// many identifiers and such. This makes the last chance leak checking that
-// nanobind does somewhat unreliable since the reports it prints may be
-// to identifiers that are no longer live (at a time in process shutdown
-// where it is expected that everything left just gets dropped on the floor).
-// This causes segfaults or ASAN violations in the leak checker on exit in
-// certain scenarios where we have spurious "leaks" of global objects.
-#if defined(Py_DEBUG)
+  // Tragically, debug builds of Python do the right thing and don't immortalize
+  // many identifiers and such. This makes the last chance leak checking that
+  // nanobind does somewhat unreliable since the reports it prints may be
+  // to identifiers that are no longer live (at a time in process shutdown
+  // where it is expected that everything left just gets dropped on the floor).
+  // This causes segfaults or ASAN violations in the leak checker on exit in
+  // certain scenarios where we have spurious "leaks" of global objects.
+
   py::set_leak_warnings(false);
-#endif
 
   logging::InitializeFromEnv();
 

shortfin/python/shortfin_apps/sd/_deps.py

@@ -9,7 +9,7 @@
 try:
     import transformers
 except ModuleNotFoundError as e:
-    raise ShortfinDepNotFoundError(__name__, "diffusers") from e
+    raise ShortfinDepNotFoundError(__name__, "transformers") from e
 
 try:
     import tokenizers

shortfin/python/shortfin_apps/sd/components/builders.py

@@ -5,9 +5,10 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from iree.build import *
-from iree.build.executor import FileNamespace
+from iree.build.executor import FileNamespace, BuildAction, BuildContext, BuildFile
 import itertools
 import os
+import urllib
 import shortfin.array as sfnp
 import copy
 
@@ -24,7 +25,7 @@
     sfnp.bfloat16: "bf16",
 }
 
-ARTIFACT_VERSION = "11132024"
+ARTIFACT_VERSION = "11182024"
 SDXL_BUCKET = (
     f"https://sharkpublic.blob.core.windows.net/sharkpublic/sdxl/{ARTIFACT_VERSION}/"
 )
@@ -162,34 +163,93 @@ def needs_update(ctx):
     return False
 
 
-def needs_file(filename, ctx, namespace=FileNamespace.GEN):
+def needs_file(filename, ctx, url=None, namespace=FileNamespace.GEN):
     out_file = ctx.allocate_file(filename, namespace=namespace).get_fs_path()
+    needed = True
     if os.path.exists(out_file):
-        needed = False
-    else:
-        # name_path = "bin" if namespace == FileNamespace.BIN else ""
-        # if name_path:
-        #     filename = os.path.join(name_path, filename)
-        filekey = os.path.join(ctx.path, filename)
-        ctx.executor.all[filekey] = None
-        needed = True
-    return needed
+        if url:
+            needed = not is_valid_size(out_file, url)
+        if not needed:
+            return False
+    filekey = os.path.join(ctx.path, filename)
+    ctx.executor.all[filekey] = None
+    return True
 
 
 def needs_compile(filename, target, ctx):
-    device = "amdgpu" if "gfx" in target else "llvmcpu"
-    vmfb_name = f"{filename}_{device}-{target}.vmfb"
+    vmfb_name = f"{filename}_{target}.vmfb"
     namespace = FileNamespace.BIN
-    return needs_file(vmfb_name, ctx, namespace)
+    return needs_file(vmfb_name, ctx, namespace=namespace)
 
 
 def get_cached_vmfb(filename, target, ctx):
-    device = "amdgpu" if "gfx" in target else "llvmcpu"
-    vmfb_name = f"{filename}_{device}-{target}.vmfb"
-    namespace = FileNamespace.BIN
+    vmfb_name = f"{filename}_{target}.vmfb"
     return ctx.file(vmfb_name)
 
 
+def is_valid_size(file_path, url):
+    if not url:
+        return True
+    with urllib.request.urlopen(url) as response:
+        content_length = response.getheader("Content-Length")
+    local_size = get_file_size(str(file_path))
+    if content_length:
+        content_length = int(content_length)
+        if content_length != local_size:
+            return False
+    return True
+
+
+def get_file_size(file_path):
+    """Gets the size of a local file in bytes as an integer."""
+
+    file_stats = os.stat(file_path)
+    return file_stats.st_size
+
+
+def fetch_http_check_size(*, name: str, url: str) -> BuildFile:
+    context = BuildContext.current()
+    output_file = context.allocate_file(name)
+    action = FetchHttpWithCheckAction(
+        url=url, output_file=output_file, desc=f"Fetch {url}", executor=context.executor
+    )
+    output_file.deps.add(action)
+    return output_file
+
+
+class FetchHttpWithCheckAction(BuildAction):
+    def __init__(self, url: str, output_file: BuildFile, **kwargs):
+        super().__init__(**kwargs)
+        self.url = url
+        self.output_file = output_file
+
+    def _invoke(self, retries=4):
+        path = self.output_file.get_fs_path()
+        self.executor.write_status(f"Fetching URL: {self.url} -> {path}")
+        try:
+            urllib.request.urlretrieve(self.url, str(path))
+        except urllib.error.HTTPError as e:
+            if retries > 0:
+                retries -= 1
+                self._invoke(retries=retries)
+            else:
+                raise IOError(f"Failed to fetch URL '{self.url}': {e}") from None
+        local_size = get_file_size(str(path))
+        try:
+            with urllib.request.urlopen(self.url) as response:
+                content_length = response.getheader("Content-Length")
+            if content_length:
+                content_length = int(content_length)
+                if content_length != local_size:
+                    raise IOError(
+                        f"Size of downloaded artifact does not match content-length header! {content_length} != {local_size}"
+                    )
+        except IOError:
+            if retries > 0:
+                retries -= 1
+                self._invoke(retries=retries)
+
+
 @entrypoint(description="Retreives a set of SDXL submodels.")
 def sdxl(
     model_json=cl_arg(
@@ -224,7 +284,7 @@ def sdxl(
     mlir_filenames = get_mlir_filenames(model_params, model)
     mlir_urls = get_url_map(mlir_filenames, mlir_bucket)
     for f, url in mlir_urls.items():
-        if update or needs_file(f, ctx):
+        if update or needs_file(f, ctx, url):
             fetch_http(name=f, url=url)
 
     vmfb_filenames = get_vmfb_filenames(model_params, model=model, target=target)
@@ -244,15 +304,14 @@ def sdxl(
                 vmfb_filenames[idx] = get_cached_vmfb(file_stem, target, ctx)
     else:
         for f, url in vmfb_urls.items():
-            if update or needs_file(f, ctx):
+            if update or needs_file(f, ctx, url):
                 fetch_http(name=f, url=url)
 
     params_filenames = get_params_filenames(model_params, model=model, splat=splat)
     params_urls = get_url_map(params_filenames, SDXL_WEIGHTS_BUCKET)
     for f, url in params_urls.items():
-        out_file = os.path.join(ctx.executor.output_dir, f)
-        if needs_file(f, ctx):
-            fetch_http(name=f, url=url)
+        if needs_file(f, ctx, url):
+            fetch_http_check_size(name=f, url=url)
     filenames = [*vmfb_filenames, *params_filenames, *mlir_filenames]
     return filenames