feat(workflow): Added new dev workflow for submitter

Signed-off-by: Evan Spearman <[email protected]>

.gitignore

@@ -14,3 +14,4 @@ houdini_version.txt
 /dev_install/
 /wheels/
 **/otls/backup
+plugin_env

DEVELOPMENT.md

@@ -2,15 +2,29 @@
 
 ## Submitter Development Workflow
 
-WARNING: This workflow installs additional Python packages into your Houdini's python distribution.
-
-1. Create a development location within which to do your git checkouts. For example `~/deadline-clients`. Clone packages from this directory with commands like `git clone [email protected]:casillas2/deadline-cloud-for-houdini.git`. You'll also want the `deadline-cloud` and `openjd-adaptor-runtime` repos.
-2. Switch to your Houdini directory, like `cd "C:\Program Files\hfs19.5"`.
-3. Run `.\bin\hython -m pip install -e C:\Users\<username>\deadline-clients\deadline-cloud` to install the Amazon Deadline Cloud Client Library in edit mode.
-4. Run `.\bin\hython -m pip install -e C:\Users\<username>\deadline-clients\openjd-adaptor-runtime-for-python` to install the Open Job Description Adaptor Runtime Library in edit mode.
-5. Run `.\bin\hython -m pip install -e C:\Users\<username>\deadline-clients\deadline-cloud-for-houdini` to install the Houdini submitter in edit mode.
-6. Run Houdini. Go to File > Import > Houdini Digital Asset Select the `src\deadline\houdini_submitter\otls\deadline_cloud.hda` directory. Press "Install and Create", which should create a "deadline_cloud1" node in the output context.
-7. To edit the deadline_cloud hda, go to Assets > Asset Manager. Under Operator Type Libraries > Current HIP File, you will find "Driver/deadline_cloud". Right click, select Type Properties. From the Parameter tab you can modify the parameter interface, as you hit Apply you will see that the "DialogScript" file in the hda source files has been updated.
+This workflow creates a "houdini package", a JSON file which tells Houdini where to find the plugin files. This workflow is preferred because it does not install any files directly into your Houdini installation, and it uses the same functionality to load the plugin as is used by the submitter installer. Because we use the paths of our clone of the repository, we only need to run this script once after creating a new development environment, or if the dependencies change, and then changes to the code will be present the next time you launch Houdini.
+
+1. Clone this repository somewhere on the machine you have Houdini installed on:
+
+   ```sh
+   git clone [email protected]:casillas2/deadline-cloud-for-houdini.git
+   cd deadline-cloud-for-houdini
+   ```
+
+2. Create a Houdini package using the provided script, specifying the full houdini version:
+
+   ```sh
+   hatch run install --houdini-version X.Y
+   ```
+
+4. (Optional) If you need to make changes to the Houdini submitter and deadline-cloud at the same time, you can do the following to do an in-place install of deadline-cloud from a clone of the deadline-cloud repository. Note that this will print an error message if the current version of deadline-cloud is greater than specified in deadline-cloud-for-houdini's dependencies, but in most cases this can be ignored:
+
+   ```sh
+   cd ..
+   git clone [email protected]:casillas2/deadline-cloud.git
+   cd deadline-cloud-for-houdini
+   hatch run install --houdini-version X.Y --local-dep ../deadline-cloud
+   ```
 
 ## Application Interface Adaptor Development Workflow
 

hatch.toml

@@ -19,7 +19,7 @@ lint = [
   "style",
   "typing",
 ]
-install = "bash {root}/scripts/install_dev_submitter.sh {args:}"
+install = "python {root}/scripts/install_dev_submitter.py {args:}"
 
 [[envs.test.matrix]]
 python = ["3.9", "3.10", "3.11"]

requirements-testing.txt

@@ -6,4 +6,5 @@ twine ~= 4.0
 mypy ~= 1.6
 black ~= 23.11
 ruff ~= 0.1.4
-types-pyyaml ~= 6.0
+types-pyyaml ~= 6.0
+pipgrip ~= 0.10

scripts/_project.py

@@ -0,0 +1,82 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+from __future__ import annotations
+
+import subprocess
+import sys
+
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Optional
+
+
+ADAPTOR_ONLY_DEPENDENCIES = {"openjd-adaptor-runtime"}
+
+
+def get_project_dict(project_path: Optional[Path] = None) -> dict[str, Any]:
+    if sys.version_info < (3, 11):
+        with TemporaryDirectory() as toml_env:
+            toml_install_pip_args = ["pip", "install", "--target", toml_env, "toml"]
+            subprocess.run(toml_install_pip_args, check=True)
+            sys.path.insert(0, toml_env)
+            import toml
+        mode = "r"
+    else:
+        import tomllib as toml
+
+        mode = "rb"
+
+    with open(str((project_path or get_git_root()) / "pyproject.toml"), mode) as pyproject_toml:
+        return toml.load(pyproject_toml)
+
+
+class Dependency:
+    name: str
+    operator: Optional[str]
+    version: Optional[str]
+
+    def __init__(self, dep: str):
+        components = dep.split(" ")
+        self.name = components[0]
+        if len(components) > 2:
+            self.operator = components[1]
+            self.version = components[2]
+        else:
+            self.operator = None
+            self.version = None
+
+    def for_pip(self) -> str:
+        if self.operator is not None and self.version is not None:
+            return f"{self.name}{self.operator}{self.version}"
+        return self.name
+
+    def __repr__(self) -> str:
+        return self.for_pip()
+
+
+def get_dependencies(pyproject_dict: dict[str, Any], exclude_adaptor_only=True) -> list[Dependency]:
+    if "project" not in pyproject_dict:
+        raise Exception("pyproject.toml is missing project section")
+    if "dependencies" not in pyproject_dict["project"]:
+        raise Exception("pyproject.toml is missing dependencies section")
+
+    return [
+        Dependency(dep_str)
+        for dep_str in pyproject_dict["project"]["dependencies"]
+        if exclude_adaptor_only or dep_str not in ADAPTOR_ONLY_DEPENDENCIES
+    ]
+
+
+def get_git_root() -> Path:
+    return Path(__file__).parents[1].resolve()
+
+
+def get_pip_platform(system_platform: str) -> str:
+    if system_platform == "Windows":
+        return "win_amd64"
+    elif system_platform == "Darwin":
+        return "macosx_10_9_x86_64"
+    elif system_platform == "Linux":
+        return "manylinux2014_x86_64"
+    else:
+        raise Exception(f"Unsupported platform: {system_platform}")

scripts/deps_bundle.py

@@ -5,45 +5,18 @@
 import re
 import shutil
 import subprocess
-import sys
 
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any
 
+from _project import get_project_dict, get_dependencies, get_pip_platform
+
 SUPPORTED_PYTHON_VERSIONS = ["3.9", "3.10", "3.11"]
-SUPPORTED_PLATFORMS = ["win_amd64", "manylinux2014_x86_64", "macosx_10_9_x86_64"]
+SUPPORTED_PLATFORMS = ["Windows", "Linux", "Darwin"]
 NATIVE_DEPENDENCIES = ["xxhash"]
 
 
-def _get_project_dict() -> dict[str, Any]:
-    if sys.version_info < (3, 11):
-        with TemporaryDirectory() as toml_env:
-            toml_install_pip_args = ["pip", "install", "--target", toml_env, "toml"]
-            subprocess.run(toml_install_pip_args, check=True)
-            sys.path.insert(0, toml_env)
-            import toml
-        mode = "r"
-    else:
-        import tomllib as toml
-
-        mode = "rb"
-
-    with open(str(Path(__file__).parents[1] / "pyproject.toml"), mode) as pyproject_toml:
-        return toml.load(pyproject_toml)
-
-
-def _get_dependencies(pyproject_dict: dict[str, Any]) -> list[str]:
-    if "project" not in pyproject_dict:
-        raise Exception("pyproject.toml is missing project section")
-    if "dependencies" not in pyproject_dict["project"]:
-        raise Exception("pyproject.toml is missing dependencies section")
-
-    dependencies = pyproject_dict["project"]["dependencies"]
-    deps_noopenjd = filter(lambda dep: not dep.startswith("openjd"), dependencies)
-    return list(map(lambda dep: dep.replace(" ", ""), deps_noopenjd))
-
-
 def _get_package_version_regex(package: str) -> re.Pattern:
     return re.compile(rf"^{re.escape(package)} *(.*)$")
 
@@ -81,9 +54,9 @@ def _download_native_dependencies(working_directory: Path, base_env: Path) -> li
     ]
     native_dependency_paths = []
     for version in SUPPORTED_PYTHON_VERSIONS:
-        for platform in SUPPORTED_PLATFORMS:
+        for plat in map(get_pip_platform, SUPPORTED_PLATFORMS):
             native_dependency_path = (
-                working_directory / "native" / f"{version.replace('.', '_')}_{platform}"
+                working_directory / "native" / f"{version.replace('.', '_')}_{plat}"
             )
             native_dependency_paths.append(native_dependency_path)
             native_dependency_path.mkdir(parents=True)
@@ -93,7 +66,7 @@ def _download_native_dependencies(working_directory: Path, base_env: Path) -> li
                 "--target",
                 str(native_dependency_path),
                 "--platform",
-                platform,
+                plat,
                 "--python-version",
                 version,
                 "--only-binary=:all:",
@@ -141,9 +114,10 @@ def _copy_zip_to_destination(zip_path: Path) -> None:
 def build_deps_bundle() -> None:
     with TemporaryDirectory() as working_directory:
         working_directory = Path(working_directory)
-        project_dict = _get_project_dict()
-        dependencies = _get_dependencies(project_dict)
-        base_env = _build_base_environment(working_directory, dependencies)
+        project_dict = get_project_dict()
+        dependencies = get_dependencies(project_dict)
+        deps_noopenjd = filter(lambda dep: not dep.startswith("openjd"), dependencies)
+        base_env = _build_base_environment(working_directory, deps_noopenjd)
         native_dependency_paths = _download_native_dependencies(working_directory, base_env)
         _copy_native_to_base_env(base_env, native_dependency_paths)
         zip_path = _get_zip_path(working_directory, project_dict)