feat(workflow): Added new dev workflow for submitter

Signed-off-by: Evan Spearman <[email protected]>

.gitignore

@@ -13,3 +13,4 @@ houdini_version.txt
 /dev_install/
 /wheels/
 **/otls/backup
+plugin_env

DEVELOPMENT.md

@@ -2,15 +2,39 @@
 
 ## Submitter Development Workflow
 
-WARNING: This workflow installs additional Python packages into your Houdini's python distribution.
-
-1. Create a development location within which to do your git checkouts. For example `~/deadline-clients`. Clone packages from this directory with commands like `git clone [email protected]:casillas2/deadline-cloud-for-houdini.git`. You'll also want the `deadline-cloud` and `openjd-adaptor-runtime` repos.
-2. Switch to your Houdini directory, like `cd "C:\Program Files\hfs19.5"`.
-3. Run `.\bin\hython -m pip install -e C:\Users\<username>\deadline-clients\deadline-cloud` to install the Amazon Deadline Cloud Client Library in edit mode.
-4. Run `.\bin\hython -m pip install -e C:\Users\<username>\deadline-clients\openjd-adaptor-runtime-for-python` to install the Open Job Description Adaptor Runtime Library in edit mode.
-5. Run `.\bin\hython -m pip install -e C:\Users\<username>\deadline-clients\deadline-cloud-for-houdini` to install the Houdini submitter in edit mode.
-6. Run Houdini. Go to File > Import > Houdini Digital Asset Select the `src\deadline\houdini_submitter\otls\deadline_cloud.hda` directory. Press "Install and Create", which should create a "deadline_cloud1" node in the output context.
-7. To edit the deadline_cloud hda, go to Assets > Asset Manager. Under Operator Type Libraries > Current HIP File, you will find "Driver/deadline_cloud". Right click, select Type Properties. From the Parameter tab you can modify the parameter interface, as you hit Apply you will see that the "DialogScript" file in the hda source files has been updated.
+> Note: This workflow assumes you are using the same version of Python as the version of Houdini you are developing for uses.
+
+This workflow creates a "houdini package", a JSON file which tells Houdini where to find the plugin files. This workflow is preferred because it does not install any files directly into your Houdini installation, and it uses the same functionality to load the plugin as is used by the submitter installer. Because we use the paths of our clone of the repository and our hatch environment, we only need to run this script once after creating a new development environment, and then changes to the code will be present the next time you launch Houdini.
+
+1. Clone this repository somewhere on the machine you have Houdini installed on:
+
+   ```sh
+   git clone [email protected]:casillas2/deadline-cloud-for-houdini.git
+   cd deadline-cloud-for-houdini
+   ```
+
+2. Create a hatch environment:
+
+   > Note: If you do not have hatch installed yet, you can install it with `pip install hatch`
+
+   ```sh
+   hatch env create
+   ```
+
+3. Create a Houdini package using the provided script, specifying the full houdini version:
+
+   ```sh
+   python scripts/install_dev_submitter.py --houdini-version X.Y.Z
+   ```
+
+4. (Optional) If you need to make changes to the Houdini submitter and deadline-cloud at the same time, you can do the following to do an in-place install of deadline-cloud from a clone of the deadline-cloud repository. Note that this will print an error message if the current version of deadline-cloud is greater than specified in deadline-cloud-for-houdini's dependencies, but in most cases this can be ignored:
+
+   ```sh
+   cd ..
+   git clone [email protected]:casillas2/deadline-cloud.git
+   cd deadline-cloud-for-houdini
+   hatch run pip install -e ../deadline-cloud
+   ```
 
 ## Application Interface Adaptor Development Workflow
 

hatch.toml

@@ -19,7 +19,7 @@ lint = [
   "style",
   "typing",
 ]
-install = "bash {root}/scripts/install_dev_submitter.sh {args:}"
+install = "python {root}/scripts/install_dev_submitter.py {args:}"
 
 [[envs.test.matrix]]
 python = ["3.9", "3.10", "3.11"]

scripts/deps_bundle.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import platform
 import re
 import shutil
 import subprocess
@@ -12,7 +13,7 @@
 from typing import Any
 
 SUPPORTED_PYTHON_VERSIONS = ["3.9", "3.10", "3.11"]
-SUPPORTED_PLATFORMS = ["win_amd64", "manylinux2014_x86_64", "macosx_10_9_x86_64"]
+SUPPORTED_PLATFORMS = ["Windows", "Linux", "Darwin"]
 NATIVE_DEPENDENCIES = ["xxhash"]
 
 
@@ -81,9 +82,9 @@ def _download_native_dependencies(working_directory: Path, base_env: Path) -> li
     ]
     native_dependency_paths = []
     for version in SUPPORTED_PYTHON_VERSIONS:
-        for platform in SUPPORTED_PLATFORMS:
+        for plat in map(_get_pip_platform, SUPPORTED_PLATFORMS):
             native_dependency_path = (
-                working_directory / "native" / f"{version.replace('.', '_')}_{platform}"
+                working_directory / "native" / f"{version.replace('.', '_')}_{plat}"
             )
             native_dependency_paths.append(native_dependency_path)
             native_dependency_path.mkdir(parents=True)
@@ -93,7 +94,7 @@ def _download_native_dependencies(working_directory: Path, base_env: Path) -> li
                 "--target",
                 str(native_dependency_path),
                 "--platform",
-                platform,
+                plat,
                 "--python-version",
                 version,
                 "--only-binary=:all:",
@@ -138,6 +139,17 @@ def _copy_zip_to_destination(zip_path: Path) -> None:
     shutil.copy(str(zip_path), str(zip_desntination))
 
 
+def _get_pip_platform(system_platform: str) -> str:
+    if system_platform == "Windows":
+        return "win_amd64"
+    elif system_platform == "Darwin":
+        return "macosx_10_9_x86_64"
+    elif system_platform == "Linux":
+        return "manylinux2014_x86_64"
+    else:
+        raise Exception(f"Unsupported platform: {system_platform}")
+
+
 def build_deps_bundle() -> None:
     with TemporaryDirectory() as working_directory:
         working_directory = Path(working_directory)
@@ -152,5 +164,27 @@ def build_deps_bundle() -> None:
         _copy_zip_to_destination(zip_path)
 
 
+def build_deps_env(destination: Path, python_version: str) -> None:
+    project_dict = _get_project_dict()
+
+    destination.mkdir(parents=True, exist_ok=True)
+    if not destination.is_dir():
+        raise Exception(f"{str(destination)} is not a directory")
+
+    args = [
+        "pip",
+        "install",
+        "--target",
+        str(destination),
+        "--platform",
+        _get_pip_platform(platform.system()),
+        "--python-version",
+        python_version,
+        "--only-binary=:all:",
+        *_get_dependencies(project_dict),
+    ]
+    subprocess.run(args, check=True)
+
+
 if __name__ == "__main__":
     build_deps_bundle()

scripts/install_dev_submitter.py

@@ -0,0 +1,125 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+import argparse
+import json
+import os
+import platform
+import re
+
+from pathlib import Path
+from typing import Optional
+
+from deps_bundle import build_deps_env
+
+
+SUBMITTER_PACKAGE_TEMPLATE = {
+    "env": [],
+    "hpath": "$DEADLINE_CLOUD_FOR_HOUDINI",
+}
+
+
+def _get_git_root() -> Path:
+    return Path(__file__).parents[1].resolve()
+
+
+class HoudiniVersion:
+    major: int
+    minor: int
+    patch: Optional[int]
+
+    VERSION_REGEX = re.compile(r"^([0-9]+)\.([0-9]+)(?:\.([0-9]+))?")
+
+    PYTHON_VERSIONS = {
+        "19.5": "3.9",
+    }
+
+    def __init__(self, arg_version: Optional[str]):
+        version = self._get_houdini_version(arg_version)
+        match = self.VERSION_REGEX.match(version)
+        if match is None:
+            raise ValueError(f"Invalid version: {version}")
+        self.major = int(match.group(1))
+        self.minor = int(match.group(2))
+        self.patch = int(match.group(3)) if match.group(3) is not None else None
+
+    def major_minor(self) -> str:
+        return f"{self.major}.{self.minor}"
+
+    def python_major_minor(self) -> str:
+        major_minor = self.major_minor()
+        if major_minor in self.PYTHON_VERSIONS:
+            return self.PYTHON_VERSIONS[major_minor]
+        raise ValueError(f"Unknown Houdini major minor version {major_minor}")
+
+    @classmethod
+    def _validate_version(cls, version: str) -> str:
+        match = cls.VERSION_REGEX.match(version)
+        if match is None:
+            raise ValueError(f"Invalid version: {version}")
+        return version
+
+    @classmethod
+    def _get_houdini_version(cls, arg: Optional[str]) -> str:
+        if arg is not None:
+            return cls._validate_version(arg)
+        houdini_version_file = _get_git_root() / "houdini_version.txt"
+        if houdini_version_file.exists():
+            with open(houdini_version_file, "r", encoding="utf-8") as f:
+                return cls._validate_version(f.read().strip())
+        return cls._validate_version(
+            input("Please enter the Houdini version (Major.Minor[.Patch]): ")
+        )
+
+
+def _get_houdini_user_prefs_path(major_minor: str) -> Path:
+    if platform.system() == "Windows":
+        return Path.home() / "Documents" / f"houdini{major_minor}"
+    elif platform.system() == "Darwin":
+        return Path.home() / "Library" / "Preferences" / "houdini" / major_minor
+    elif platform.system() == "Linux":
+        return Path.home() / f"houdini{major_minor}"
+    else:
+        raise RuntimeError(f"Unsupported platform: {platform.system()}")
+
+
+def _get_submitter_src_path() -> Path:
+    return _get_git_root() / "src" / "deadline" / "houdini_submitter"
+
+
+def install_submitter_package(houdini_version_arg: Optional[str]) -> None:
+    houdini_version = HoudiniVersion(houdini_version_arg)
+    major_minor = houdini_version.major_minor()
+
+    plugin_env_path = _get_git_root() / "plugin_env"
+    build_deps_env(plugin_env_path, houdini_version.python_major_minor())
+
+    submitter_package = SUBMITTER_PACKAGE_TEMPLATE.copy()
+    submitter_package["env"].append({"DEADLINE_CLOUD_FOR_HOUDINI": str(_get_submitter_src_path())})
+    python_path = os.pathsep.join(
+        [
+            str(_get_submitter_src_path() / "python"),
+            str(plugin_env_path),
+        ]
+    )
+    submitter_package["env"].append({"PYTHONPATH": python_path})
+
+    user_prefs_path = _get_houdini_user_prefs_path(major_minor)
+    packages_path = user_prefs_path / "packages"
+    packages_path.mkdir(parents=True, exist_ok=True)
+    submitter_package_path = packages_path / "deadline_submitter_for_houdini.json"
+
+    with open(submitter_package_path, "w", encoding="utf-8") as f:
+        json.dump(submitter_package, f, indent=4)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--houdini-version",
+        help="Houdini version to install the submitter for",
+        type=str,
+        default=None,
+    )
+    args = parser.parse_args()
+
+    install_submitter_package(args.houdini_version)