From 95a030389288f08a30139f2bf4d9c8d08644fd1f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 13:52:44 -0500 Subject: [PATCH 01/19] pants-plugins/pack_metadata: add pack_content_resource target type Now pack_metadata targets will generate pack_content_resource instead of just resource. pack_content_resource is still a resource, but this setup allows us to find the generated resource targets more simply. This also harmonizes the implementation of pack_metadata to follow the fields definition of resources (esp moving dependencies into moved_fields instead of core_fields). --- pants-plugins/pack_metadata/register.py | 2 ++ pants-plugins/pack_metadata/target_types.py | 30 +++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 36c11079d9..789ba35ffb 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -13,6 +13,7 @@ # limitations under the License. from pack_metadata import tailor, target_types_rules from pack_metadata.target_types import ( + PackContentResourceTarget, PackMetadata, PackMetadataInGitSubmodule, PacksGlob, @@ -28,6 +29,7 @@ def rules(): def target_types(): return [ + PackContentResourceTarget, PackMetadata, PackMetadataInGitSubmodule, PacksGlob, diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 4c7c2c854f..817fe54aa3 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -15,8 +15,12 @@ from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies from pants.core.target_types import ( + ResourceDependenciesField, ResourcesGeneratingSourcesField, ResourcesGeneratorTarget, + ResourcesOverridesField, + ResourceSourceField, + ResourceTarget, GenericTarget, ) @@ -25,6 +29,10 @@ class UnmatchedGlobsError(Exception): """Error thrown when a required set of globs didn't match.""" +class PackContentResourceSourceField(ResourceSourceField): + pass + + class PackMetadataSourcesField(ResourcesGeneratingSourcesField): required = False default = ( @@ -58,9 +66,25 @@ def validate_resolved_files(self, files: Sequence[str]) -> None: super().validate_resolved_files(files) +class PackContentResourceTarget(ResourceTarget): + alias = "pack_content_resource" + core_fields = ( + *COMMON_TARGET_FIELDS, + ResourceDependenciesField, + PackContentResourceSourceField, + ) + help = "A single pack content resource file (mostly for metadata files)." + + class PackMetadata(ResourcesGeneratorTarget): alias = "pack_metadata" - core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSourcesField) + core_fields = ( + *COMMON_TARGET_FIELDS, + PackMetadataSourcesField, + ResourcesOverridesField, + ) + moved_fields = (ResourceDependenciesField,) + generated_target_cls = PackContentResourceTarget help = ( "Loose pack metadata files.\n\n" "Pack metadata includes top-level files (pack.yaml, .yaml.example, " @@ -73,9 +97,11 @@ class PackMetadataInGitSubmodule(PackMetadata): alias = "pack_metadata_in_git_submodule" core_fields = ( *COMMON_TARGET_FIELDS, - Dependencies, PackMetadataInGitSubmoduleSources, + ResourcesOverridesField, ) + moved_fields = (ResourceDependenciesField,) + generated_target_cls = PackContentResourceTarget help = PackMetadata.help + ( "\npack_metadata_in_git_submodule variant errors if the sources field " "has unmatched globs. It prints instructions on how to checkout git " From 2222faff8aa11cba0bdc09340e7557e3978f45fe Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 13:58:13 -0500 Subject: [PATCH 02/19] pants-plugins/pack_metadata: classify metadata type of pack_content_resource targets This will allow rules to look up just action and sensor metadata (for example). --- .../pack_metadata/python_rules/BUILD | 1 + .../pack_metadata/python_rules/__init__.py | 0 .../python_rules/python_pack_content.py | 59 +++++++++++++++ pants-plugins/pack_metadata/register.py | 3 + pants-plugins/pack_metadata/target_types.py | 75 ++++++++++++++++++- 5 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 pants-plugins/pack_metadata/python_rules/BUILD create mode 100644 pants-plugins/pack_metadata/python_rules/__init__.py create mode 100644 pants-plugins/pack_metadata/python_rules/python_pack_content.py diff --git a/pants-plugins/pack_metadata/python_rules/BUILD b/pants-plugins/pack_metadata/python_rules/BUILD new file mode 100644 index 0000000000..db46e8d6c9 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/pants-plugins/pack_metadata/python_rules/__init__.py b/pants-plugins/pack_metadata/python_rules/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content.py b/pants-plugins/pack_metadata/python_rules/python_pack_content.py new file mode 100644 index 0000000000..45716efff4 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content.py @@ -0,0 +1,59 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass + +from pants.engine.rules import collect_rules, rule +from pants.engine.target import ( + AllTargets, + Targets, +) +from pants.util.logging import LogLevel + +from pack_metadata.target_types import ( + PackContentResourceSourceField, + PackContentResourceTypeField, + PackContentResourceTypes, + PackMetadata, +) + + +@dataclass(frozen=True) +class PackContentResourceTargetsOfTypeRequest: + types: tuple[PackContentResourceTypes, ...] + + +class PackContentResourceTargetsOfType(Targets): + pass + + +@rule( + desc=f"Find all `{PackMetadata.alias}` targets in project filtered by content type", + level=LogLevel.DEBUG, +) +async def find_pack_metadata_targets_of_types( + request: PackContentResourceTargetsOfTypeRequest, targets: AllTargets +) -> PackContentResourceTargetsOfType: + return PackContentResourceTargetsOfType( + tgt + for tgt in targets + if tgt.has_field(PackContentResourceSourceField) + and ( + not request.types + or tgt[PackContentResourceTypeField].value in request.types + ) + ) + + +def rules(): + return (*collect_rules(),) diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 789ba35ffb..2e04bc6675 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from pack_metadata import tailor, target_types_rules +from pack_metadata.python_rules import python_pack_content from pack_metadata.target_types import ( PackContentResourceTarget, PackMetadata, @@ -24,6 +26,7 @@ def rules(): return [ *tailor.rules(), *target_types_rules.rules(), + *python_pack_content.rules(), ] diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 817fe54aa3..74fd27d5da 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -11,9 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Sequence +from enum import Enum +from pathlib import PurePath +from typing import Optional, Sequence, Tuple -from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies +from pants.engine.internals.native_engine import Address +from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, StringField from pants.core.target_types import ( ResourceDependenciesField, ResourcesGeneratingSourcesField, @@ -29,6 +32,73 @@ class UnmatchedGlobsError(Exception): """Error thrown when a required set of globs didn't match.""" +class PackContentResourceTypes(Enum): + # in root of pack + pack_metadata = "pack_metadata" + pack_config_schema = "pack_config_schema" + pack_config_example = "pack_config_example" + pack_icon = "pack_icon" + # in subdirectory (see _content_type_by_path_parts below + action_metadata = "action_metadata" + action_chain_workflow = "action_chain_workflow" + orquesta_workflow = "orquesta_workflow" + alias_metadata = "alias_metadata" + policy_metadata = "policy_metadata" + rule_metadata = "rule_metadata" + sensor_metadata = "sensor_metadata" + trigger_metadata = "trigger_metadata" + # other + unknown = "unknown" + + +_content_type_by_path_parts: dict[Tuple[str, ...], PackContentResourceTypes] = { + ("actions",): PackContentResourceTypes.action_metadata, + ("actions", "chains"): PackContentResourceTypes.action_chain_workflow, + ("actions", "workflows"): PackContentResourceTypes.orquesta_workflow, + ("aliases",): PackContentResourceTypes.alias_metadata, + ("policies",): PackContentResourceTypes.policy_metadata, + ("rules",): PackContentResourceTypes.rule_metadata, + ("sensors",): PackContentResourceTypes.sensor_metadata, + ("triggers",): PackContentResourceTypes.trigger_metadata, +} + + +class PackContentResourceTypeField(StringField): + alias = "type" + help = ( + "The content type of the resource." + "\nDo not use this field in BUILD files. It is calculated automatically" + "based on the conventional location of files in the st2 pack." + ) + valid_choices = PackContentResourceTypes + value: PackContentResourceTypes + + @classmethod + def compute_value( + cls, raw_value: Optional[str], address: Address + ) -> PackContentResourceTypes: + value = super().compute_value(raw_value, address) + if value is not None: + return PackContentResourceTypes(value) + path = PurePath(address.relative_file_path) + _yaml_suffixes = ("yaml", "yml") + if len(path.parent.parts) == 0: + # in the pack root + if path.name == "pack.yaml": + return PackContentResourceTypes.pack_metadata + if path.stem == "pack.schema" and path.suffix in _yaml_suffixes: + return PackContentResourceTypes.pack_config_schema + if path.suffix == "example" and path.suffixes[0] in _yaml_suffixes: + return PackContentResourceTypes.pack_config_example + if path.name == "icon.png": + return PackContentResourceTypes.pack_config_example + return PackContentResourceTypes.unknown + resource_type = _content_type_by_path_parts.get(path.parent.parts, None) + if resource_type is not None: + return resource_type + return PackContentResourceTypes.unknown + + class PackContentResourceSourceField(ResourceSourceField): pass @@ -72,6 +142,7 @@ class PackContentResourceTarget(ResourceTarget): *COMMON_TARGET_FIELDS, ResourceDependenciesField, PackContentResourceSourceField, + PackContentResourceTypeField, ) help = "A single pack content resource file (mostly for metadata files)." From 050b17e568f24c84299552c7d7568bee858f2544 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 14:28:01 -0500 Subject: [PATCH 03/19] pants-plugins/pack_metadata: register actions/sensors in pants python module mapping Only handles the actual action/sensor python files. It does not yet handle: - /lib - /actions/lib --- .../python_rules/python_module_mapper.py | 70 +++++++++ .../python_rules/python_pack_content.py | 144 +++++++++++++++++- pants-plugins/pack_metadata/register.py | 6 +- 3 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 pants-plugins/pack_metadata/python_rules/python_module_mapper.py diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py new file mode 100644 index 0000000000..37ecefbcf1 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py @@ -0,0 +1,70 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import defaultdict +from typing import DefaultDict + +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, + FirstPartyPythonMappingImplMarker, + ModuleProvider, + ModuleProviderType, + ResolveName, +) +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackPythonLibs, + PackPythonLibsRequest, +) +from pack_metadata.target_types import PackMetadata + + +# This is only used to register our implementation with the plugin hook via unions. +class St2PythonPackContentMappingMarker(FirstPartyPythonMappingImplMarker): + pass + + +@rule( + desc=f"Creating map of `{PackMetadata.alias}` targets to Python modules in pack content", + level=LogLevel.DEBUG, +) +async def map_pack_content_to_python_modules( + _: St2PythonPackContentMappingMarker, +) -> FirstPartyPythonMappingImpl: + resolves_to_modules_to_providers: DefaultDict[ + ResolveName, DefaultDict[str, list[ModuleProvider]] + ] = defaultdict(lambda: defaultdict(list)) + + pack_content_python_entry_points = await Get( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest(), + ) + + for pack_content in pack_content_python_entry_points: + resolves_to_modules_to_providers[pack_content.resolve][ + pack_content.module + ].append(ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL)) + + return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers) + + +def rules(): + return ( + *collect_rules(), + UnionRule(FirstPartyPythonMappingImplMarker, St2PythonPackContentMappingMarker), + ) diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content.py b/pants-plugins/pack_metadata/python_rules/python_pack_content.py index 45716efff4..d73ba9fe8c 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content.py @@ -11,11 +11,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import yaml +from collections import defaultdict from dataclasses import dataclass +from pathlib import PurePath +from typing import DefaultDict -from pants.engine.rules import collect_rules, rule +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import PythonResolveField +from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior +from pants.base.specs import FileLiteralSpec, RawSpecs +from pants.engine.collection import Collection +from pants.engine.fs import DigestContents +from pants.engine.internals.native_engine import Address, Digest +from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import ( AllTargets, + HydrateSourcesRequest, + HydratedSources, + Target, Targets, ) from pants.util.logging import LogLevel @@ -55,5 +69,133 @@ async def find_pack_metadata_targets_of_types( ) +@dataclass(frozen=True) +class PackContentPythonEntryPoint: + metadata_address: Address + content_type: PackContentResourceTypes + entry_point: str + python_address: Address + resolve: str + module: str + + +class PackContentPythonEntryPoints(Collection[PackContentPythonEntryPoint]): + pass + + +class PackContentPythonEntryPointsRequest: + pass + + +def get_possible_modules(path: PurePath) -> list[str]: + module = path.stem if path.suffix == ".py" else path.name + modules = [module] + + try: + start = path.parent.parts.index("actions") + 1 + except ValueError: + start = path.parent.parts.index("sensors") + 1 + + # st2 adds the parent dir of the python file to sys.path at runtime. + # by convention, however, just actions/ is on sys.path during tests. + # so, also construct the module name from actions/ to support tests. + if start < len(path.parent.parts): + modules.append(".".join((*path.parent.parts[start:], module))) + return modules + + +@rule(desc="Find all Pack Content entry_points that are python", level=LogLevel.DEBUG) +async def find_pack_content_python_entry_points( + python_setup: PythonSetup, _: PackContentPythonEntryPointsRequest +) -> PackContentPythonEntryPoints: + action_or_sensor = ( + PackContentResourceTypes.action_metadata, + PackContentResourceTypes.sensor_metadata, + ) + + action_and_sensor_metadata_targets = await Get( + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest(action_or_sensor), + ) + action_and_sensor_metadata_sources = await MultiGet( + Get(HydratedSources, HydrateSourcesRequest(tgt[PackContentResourceSourceField])) + for tgt in action_and_sensor_metadata_targets + ) + action_and_sensor_metadata_contents = await MultiGet( + Get(DigestContents, Digest, source.snapshot.digest) + for source in action_and_sensor_metadata_sources + ) + + # python file path -> list of info about metadata files that refer to it + pack_content_entry_points_by_spec: DefaultDict[ + str, list[tuple[Address, PackContentResourceTypes, str]] + ] = defaultdict(list) + + tgt: Target + contents: DigestContents + for tgt, contents in zip( + action_and_sensor_metadata_targets, action_and_sensor_metadata_contents + ): + content_type = tgt[PackContentResourceTypeField].value + if content_type not in action_or_sensor: + continue + assert len(contents) == 1 + try: + metadata = yaml.safe_load(contents[0].content) or {} + except yaml.YAMLError: + continue + if content_type == PackContentResourceTypes.action_metadata: + runner_type = metadata.get("runner_type", "") or "" + if runner_type != "python-script": + # only python-script has special PYTHONPATH rules + continue + # get the entry_point to find subdirectory that contains the module + entry_point = metadata.get("entry_point", "") or "" + if entry_point: + # address.filename is basically f"{spec_path}/{relative_file_path}" + path = PurePath(tgt.address.filename).parent / entry_point + pack_content_entry_points_by_spec[str(path)].append( + (tgt.address, content_type, entry_point) + ) + + python_targets = await Get( + Targets, + RawSpecs( + file_literals=tuple( + FileLiteralSpec(spec_path) + for spec_path in pack_content_entry_points_by_spec + ), + unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, + description_of_origin="pack_metadata python module mapper", + ), + ) + + pack_content_entry_points: list[PackContentPythonEntryPoint] = [] + for tgt in python_targets: + if not tgt.has_field(PythonResolveField): + # this is unexpected + continue + for ( + metadata_address, + content_type, + entry_point, + ) in pack_content_entry_points_by_spec[tgt.address.filename]: + resolve = tgt[PythonResolveField].normalized_value(python_setup) + + for module in get_possible_modules(PurePath(tgt.address.filename)): + pack_content_entry_points.append( + PackContentPythonEntryPoint( + metadata_address=metadata_address, + content_type=content_type, + entry_point=entry_point, + python_address=tgt.address, + resolve=resolve, + module=module, + ) + ) + + return PackContentPythonEntryPoints(pack_content_entry_points) + + def rules(): return (*collect_rules(),) diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 2e04bc6675..de349ee4c8 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -13,7 +13,10 @@ # limitations under the License. from pack_metadata import tailor, target_types_rules -from pack_metadata.python_rules import python_pack_content +from pack_metadata.python_rules import ( + python_module_mapper, + python_pack_content, +) from pack_metadata.target_types import ( PackContentResourceTarget, PackMetadata, @@ -27,6 +30,7 @@ def rules(): *tailor.rules(), *target_types_rules.rules(), *python_pack_content.rules(), + *python_module_mapper.rules(), ] From 9b5a90b206460b6b0169cffb32fb1691b171c4c0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 17:45:10 -0500 Subject: [PATCH 04/19] pants-plugins/pack_metadata: register lib and actions/lib in pants python module mapping This makes dependency inference aware of these which may be on the PYTHONPATH. - /lib - /actions/lib --- .../python_rules/python_module_mapper.py | 16 ++- .../python_rules/python_pack_content.py | 102 +++++++++++++++++- 2 files changed, 113 insertions(+), 5 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py index 37ecefbcf1..98fa3efa91 100644 --- a/pants-plugins/pack_metadata/python_rules/python_module_mapper.py +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py @@ -50,9 +50,9 @@ async def map_pack_content_to_python_modules( ResolveName, DefaultDict[str, list[ModuleProvider]] ] = defaultdict(lambda: defaultdict(list)) - pack_content_python_entry_points = await Get( - PackContentPythonEntryPoints, - PackContentPythonEntryPointsRequest(), + pack_content_python_entry_points, pack_python_libs = await MultiGet( + Get(PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest()), + Get(PackPythonLibs, PackPythonLibsRequest()), ) for pack_content in pack_content_python_entry_points: @@ -60,6 +60,16 @@ async def map_pack_content_to_python_modules( pack_content.module ].append(ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL)) + for pack_lib in pack_python_libs: + provider_type = ( + ModuleProviderType.TYPE_STUB + if pack_lib.relative_to_lib.suffix == ".pyi" + else ModuleProviderType.IMPL + ) + resolves_to_modules_to_providers[pack_lib.resolve][pack_lib.module].append( + ModuleProvider(pack_lib.python_address, provider_type) + ) + return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers) diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content.py b/pants-plugins/pack_metadata/python_rules/python_pack_content.py index d73ba9fe8c..3b98ca3571 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content.py @@ -17,21 +17,26 @@ from pathlib import PurePath from typing import DefaultDict +from pants.backend.python.dependency_inference.module_mapper import ( + module_from_stripped_path, +) from pants.backend.python.subsystems.setup import PythonSetup -from pants.backend.python.target_types import PythonResolveField +from pants.backend.python.target_types import PythonResolveField, PythonSourceField from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior -from pants.base.specs import FileLiteralSpec, RawSpecs +from pants.base.specs import FileLiteralSpec, RawSpecs, RecursiveGlobSpec from pants.engine.collection import Collection from pants.engine.fs import DigestContents from pants.engine.internals.native_engine import Address, Digest from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import ( AllTargets, + AllUnexpandedTargets, HydrateSourcesRequest, HydratedSources, Target, Targets, ) +from pants.util.dirutil import fast_relpath from pants.util.logging import LogLevel from pack_metadata.target_types import ( @@ -39,6 +44,7 @@ PackContentResourceTypeField, PackContentResourceTypes, PackMetadata, + PackMetadataSourcesField, ) @@ -88,6 +94,8 @@ class PackContentPythonEntryPointsRequest: def get_possible_modules(path: PurePath) -> list[str]: + if path.name in ("__init__.py", "__init__.pyi"): + path = path.parent module = path.stem if path.suffix == ".py" else path.name modules = [module] @@ -197,5 +205,95 @@ async def find_pack_content_python_entry_points( return PackContentPythonEntryPoints(pack_content_entry_points) +@dataclass(frozen=True) +class PackPythonLib: + pack_path: PurePath + lib_dir: str + relative_to_lib: PurePath + python_address: Address + resolve: str + module: str + + +class PackPythonLibs(Collection[PackPythonLib]): + pass + + +class PackPythonLibsRequest: + pass + + +@rule(desc="Find all Pack lib directory python targets", level=LogLevel.DEBUG) +async def find_python_in_pack_lib_directories( + python_setup: PythonSetup, + all_unexpanded_targets: AllUnexpandedTargets, + _: PackPythonLibsRequest, +) -> PackPythonLibs: + pack_metadata_paths = [ + PurePath(tgt.address.spec_path) + for tgt in all_unexpanded_targets + if tgt.has_field(PackMetadataSourcesField) + ] + pack_lib_directory_targets = await MultiGet( + Get( + Targets, + RawSpecs( + recursive_globs=( + RecursiveGlobSpec(str(path / "lib")), + RecursiveGlobSpec(str(path / "actions" / "lib")), + ), + unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, + description_of_origin="pack_metadata lib directory lookup", + ), + ) + for path in pack_metadata_paths + ) + + # Maybe this should use this to take codegen into account. + # Get(PythonSourceFiles, PythonSourceFilesRequest(targets=lib_directory_targets, include_resources=False) + # For now, just take the targets as they are. + + pack_python_libs: list[PackPythonLib] = [] + + pack_path: PurePath + lib_directory_targets: Targets + for pack_path, lib_directory_targets in zip( + pack_metadata_paths, pack_lib_directory_targets + ): + for tgt in lib_directory_targets: + if not tgt.has_field(PythonSourceField): + # only python targets matter here. + continue + + relative_to_pack = PurePath( + fast_relpath(tgt[PythonSourceField].file_path, str(pack_path)) + ) + if relative_to_pack.parts[0] == "lib": + lib_dir = "lib" + elif relative_to_pack.parts[:2] == ("actions", "lib"): + lib_dir = "actions/lib" + else: + # This should not happen as it is not in the requested glob. + # Use this to tell linters that lib_dir is defined below here. + continue + relative_to_lib = relative_to_pack.relative_to(lib_dir) + + resolve = tgt[PythonResolveField].normalized_value(python_setup) + module = module_from_stripped_path(relative_to_lib) + + pack_python_libs.append( + PackPythonLib( + pack_path=pack_path, + lib_dir=lib_dir, + relative_to_lib=relative_to_lib, + python_address=tgt.address, + resolve=resolve, + module=module, + ) + ) + + return PackPythonLibs(pack_python_libs) + + def rules(): return (*collect_rules(),) From 836766371deb4fb290292b40405c8fdeb257b68c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 17:49:52 -0500 Subject: [PATCH 05/19] pants: Remove /lib and /actions/lib from source roots The pack_metadata plugin now handles identifying these imports for dep inference. Next step, modify the PYTHONPATH as well. --- pants.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pants.toml b/pants.toml index f532780afe..e4a673e2b8 100644 --- a/pants.toml +++ b/pants.toml @@ -90,9 +90,6 @@ root_patterns = [ "/contrib/packs", "/st2tests/testpacks/checks", "/st2tests/testpacks/errorcheck", - # pack common lib directories that ST2 adds to the PATH for actions/sensors - "/contrib/*/lib", - "/contrib/*/actions/lib", # other special-cased pack directories "/contrib/examples/actions/ubuntu_pkg_info", # python script runs via shell expecting cwd in PYTHONPATH # lint plugins From a816565d62124e736aace08166f2da402023b3f8 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 12:23:51 -0500 Subject: [PATCH 06/19] pants-plugins/pack_metadata: Move some logic into PackContent dataclasses --- .../python_rules/python_module_mapper.py | 7 +- .../python_rules/python_pack_content.py | 75 +++++++++++-------- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py index 98fa3efa91..e35e93b997 100644 --- a/pants-plugins/pack_metadata/python_rules/python_module_mapper.py +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py @@ -56,9 +56,10 @@ async def map_pack_content_to_python_modules( ) for pack_content in pack_content_python_entry_points: - resolves_to_modules_to_providers[pack_content.resolve][ - pack_content.module - ].append(ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL)) + for module in pack_content.get_possible_modules(): + resolves_to_modules_to_providers[pack_content.resolve][module].append( + ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL) + ) for pack_lib in pack_python_libs: provider_type = ( diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content.py b/pants-plugins/pack_metadata/python_rules/python_pack_content.py index 3b98ca3571..053b4b6904 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content.py @@ -82,34 +82,44 @@ class PackContentPythonEntryPoint: entry_point: str python_address: Address resolve: str - module: str + @property + def python_file_path(self) -> PurePath: + return PurePath(self.python_address.filename) -class PackContentPythonEntryPoints(Collection[PackContentPythonEntryPoint]): - pass + @staticmethod + def _split_pack_content_path(path: PurePath) -> tuple[PurePath, PurePath]: + content_types = ("actions", "sensors") # only content_types with python content + pack_content_dir = path.parent + while pack_content_dir.name not in content_types: + pack_content_dir = pack_content_dir.parent + relative_to_pack_content_dir = path.relative_to(pack_content_dir) + return pack_content_dir, relative_to_pack_content_dir + def get_possible_modules(self) -> tuple[str, ...]: + """Get module names that could be imported. Mirrors get_possible_paths logic.""" + path = self.python_file_path -class PackContentPythonEntryPointsRequest: - pass + # st2 adds the parent dir of the python file to sys.path at runtime. + module = path.stem if path.suffix == ".py" else path.name + modules = [module] + # By convention, however, just actions/ is on sys.path during tests. + # so, also construct the module name from actions/ to support tests. + _, relative_to_pack_content_dir = self._split_pack_content_path(path) + module = module_from_stripped_path(relative_to_pack_content_dir) + if module not in modules: + modules.append(module) -def get_possible_modules(path: PurePath) -> list[str]: - if path.name in ("__init__.py", "__init__.pyi"): - path = path.parent - module = path.stem if path.suffix == ".py" else path.name - modules = [module] + return tuple(modules) + + +class PackContentPythonEntryPoints(Collection[PackContentPythonEntryPoint]): + pass - try: - start = path.parent.parts.index("actions") + 1 - except ValueError: - start = path.parent.parts.index("sensors") + 1 - # st2 adds the parent dir of the python file to sys.path at runtime. - # by convention, however, just actions/ is on sys.path during tests. - # so, also construct the module name from actions/ to support tests. - if start < len(path.parent.parts): - modules.append(".".join((*path.parent.parts[start:], module))) - return modules +class PackContentPythonEntryPointsRequest: + pass @rule(desc="Find all Pack Content entry_points that are python", level=LogLevel.DEBUG) @@ -190,17 +200,15 @@ async def find_pack_content_python_entry_points( ) in pack_content_entry_points_by_spec[tgt.address.filename]: resolve = tgt[PythonResolveField].normalized_value(python_setup) - for module in get_possible_modules(PurePath(tgt.address.filename)): - pack_content_entry_points.append( - PackContentPythonEntryPoint( - metadata_address=metadata_address, - content_type=content_type, - entry_point=entry_point, - python_address=tgt.address, - resolve=resolve, - module=module, - ) + pack_content_entry_points.append( + PackContentPythonEntryPoint( + metadata_address=metadata_address, + content_type=content_type, + entry_point=entry_point, + python_address=tgt.address, + resolve=resolve, ) + ) return PackContentPythonEntryPoints(pack_content_entry_points) @@ -212,7 +220,10 @@ class PackPythonLib: relative_to_lib: PurePath python_address: Address resolve: str - module: str + + @property + def module(self) -> str: + return module_from_stripped_path(self.relative_to_lib) class PackPythonLibs(Collection[PackPythonLib]): @@ -279,7 +290,6 @@ async def find_python_in_pack_lib_directories( relative_to_lib = relative_to_pack.relative_to(lib_dir) resolve = tgt[PythonResolveField].normalized_value(python_setup) - module = module_from_stripped_path(relative_to_lib) pack_python_libs.append( PackPythonLib( @@ -288,7 +298,6 @@ async def find_python_in_pack_lib_directories( relative_to_lib=relative_to_lib, python_address=tgt.address, resolve=resolve, - module=module, ) ) From 8795c507e72537208abe95d3292fde2939e54e2e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 12:41:16 -0500 Subject: [PATCH 07/19] pants-plugins/pack_metadata: Add implementation notes for python_pack_content and related rules --- .../python_rules/python_pack_content.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content.py b/pants-plugins/pack_metadata/python_rules/python_pack_content.py index 053b4b6904..1ced491181 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content.py @@ -48,6 +48,44 @@ ) +# Implementation Notes: +# +# With pants, we can rely on dependency inference for all the +# st2 components, runners, and other venv bits (st2 venv and pack venv). +# In StackStorm, all of that goes at the end of PYTHONPATH. +# Pants runs things hermetically via pex, so PYTHPNPATH +# changes happen via PEX_EXTRA_SYS_PATH instead. +# +# Actions: +# At runtime, the python_runner creates a PYTHONPATH that includes: +# [pack/lib:]pack_venv/lib/python3.x:pack_venv/lib/python3.x/site-packages:pack/actions/lib:st2_pythonpath +# python_runner runs python_action_wrapper which: +# - injects the action's entry_point's directory in sys.path +# - and then imports the action module and runs it. +# +# Sensors: +# At runtime, ProcessSensorContainer creates PYTHONPATH that includes: +# [pack/lib:]st2_pythonpath +# Then the process_container runs the sensor via sensor_wrapper which: +# - injects the sensor's entry_point's directory in sys.path +# (effectively always "sensors/" as a split("/") assumes only one dir) +# - and then imports the class_name from sensor module and runs it. +# +# For actions, this pants plugin should add this to PEX_EXTRA_SYS_PATH: +# pack/actions/path_to_entry_point:[pack/lib:]pack/actions/lib +# For sensors, this pants plugin should add this to PEX_EXTRA_SYS_PATH: +# pack/sensors:[pack/lib:] +# +# The rules in this file are used by: +# python_module_mapper.py: +# Dependency inference uses pack_metadata's module_mapper to detect any +# python imports that require one of these PYTHONPATH modifications, +# resolving those imports to modules in lib/, actions/, or sensors/. +# python_path_rules.py: +# Then get the relevant python imports from dependencies and +# add their parent directory to a generated PEX_EXTRA_SYS_PATH. + + @dataclass(frozen=True) class PackContentResourceTargetsOfTypeRequest: types: tuple[PackContentResourceTypes, ...] From f5d62ed1fad06b7ba9a6aacf28d22062e898b327 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 12:48:41 -0500 Subject: [PATCH 08/19] pants-plugins/pack_metadata: Add python_path_rules to generate PEX_EXTRA_SYS_PATH for tests This won't work until pants gets support for injecting path entries. --- .../python_rules/python_pack_content.py | 19 +++ .../python_rules/python_path_rules.py | 131 ++++++++++++++++++ pants-plugins/pack_metadata/register.py | 12 ++ pants-plugins/pack_metadata/target_types.py | 18 ++- 4 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 pants-plugins/pack_metadata/python_rules/python_path_rules.py diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content.py b/pants-plugins/pack_metadata/python_rules/python_pack_content.py index 1ced491181..4fbe66ff69 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content.py @@ -151,6 +151,21 @@ def get_possible_modules(self) -> tuple[str, ...]: return tuple(modules) + def get_possible_paths(self) -> tuple[str, ...]: + """Get paths to add to PYTHONPATH and PEX_EXTRA_SYS_PATH. Mirrors get_possible_modules logic.""" + path = self.python_file_path + + # st2 adds the parent dir of the python file to sys.path at runtime. + paths = [path.parent.as_posix()] + + # By convention, however, just actions/ is on sys.path during tests. + # so, also construct the module name from actions/ to support tests. + pack_content_dir, _ = self._split_pack_content_path(path) + if path.parent != pack_content_dir: + paths.append(pack_content_dir.as_posix()) + + return tuple(paths) + class PackContentPythonEntryPoints(Collection[PackContentPythonEntryPoint]): pass @@ -263,6 +278,10 @@ class PackPythonLib: def module(self) -> str: return module_from_stripped_path(self.relative_to_lib) + @property + def lib_path(self) -> PurePath: + return self.pack_path / self.lib_dir + class PackPythonLibs(Collection[PackPythonLib]): pass diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules.py b/pants-plugins/pack_metadata/python_rules/python_path_rules.py new file mode 100644 index 0000000000..314ecd7bf9 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules.py @@ -0,0 +1,131 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Set + +from pants.backend.python.goals.pytest_runner import ( + PytestPluginSetupRequest, + PytestPluginSetup, +) +from pants.engine.internals.native_engine import Address +from pants.engine.rules import collect_rules, Get, MultiGet, rule +from pants.engine.target import Target, TransitiveTargets, TransitiveTargetsRequest +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel +from pants.util.ordered_set import OrderedSet + +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackPythonLibs, + PackPythonLibsRequest, +) +from pack_metadata.target_types import InjectPackPythonPathField + + +@dataclass(frozen=True) +class PackPythonPath: + entries: tuple[str, ...] = () + + +@dataclass(frozen=True) +class PackPythonPathRequest: + address: Address + + +@rule( + desc="Get pack paths that should be added to PYTHONPATH/PEX_EXTRA_SYS_PATH for a target.", + level=LogLevel.DEBUG, +) +async def get_extra_sys_path_for_pack_dependencies( + request: PackPythonPathRequest, +) -> PackPythonPath: + transitive_targets = await Get( + TransitiveTargets, TransitiveTargetsRequest((request.address,)) + ) + + dependency_addresses: Set[Address] = { + tgt.address for tgt in transitive_targets.closure + } + if not dependency_addresses: + return PackPythonPath() + + pack_content_python_entry_points, pack_python_libs = await MultiGet( + Get(PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest()), + Get(PackPythonLibs, PackPythonLibsRequest()), + ) + + # only use addresses of actual dependencies + pack_python_content_addresses: Set[Address] = dependency_addresses & { + pack_content.python_address for pack_content in pack_content_python_entry_points + } + pack_python_lib_addresses: Set[Address] = dependency_addresses & { + pack_lib.python_address for pack_lib in pack_python_libs + } + + if not (pack_python_content_addresses or pack_python_lib_addresses): + return PackPythonPath() + + # filter pack_content_python_entry_points and pack_python_libs + pack_content_python_entry_points = ( + pack_content + for pack_content in pack_content_python_entry_points + if pack_content.python_address in pack_python_content_addresses + ) + pack_python_libs = ( + pack_lib + for pack_lib in pack_python_libs + if pack_lib.python_address in pack_python_lib_addresses + ) + + extra_sys_path_entries = OrderedSet() + for pack_content in pack_content_python_entry_points: + for path_entry in pack_content.get_possible_paths(): + extra_sys_path_entries.add(path_entry) + for pack_lib in pack_python_libs: + extra_sys_path_entries.add(pack_lib.lib_path.as_posix()) + + return PackPythonPath(tuple(extra_sys_path_entries)) + + +class PytestPackTestRequest(PytestPluginSetupRequest): + @classmethod + def is_applicable(cls, target: Target) -> bool: + if not target.has_field(InjectPackPythonPathField): + return False + return bool(target.get(InjectPackPythonPathField).value) + + +@rule( + desc="Inject pack paths in PYTHONPATH/PEX_EXTRA_SYS_PATH for python tests.", + level=LogLevel.DEBUG, +) +async def inject_extra_sys_path_for_pack_tests( + request: PytestPackTestRequest, +) -> PytestPluginSetup: + pack_python_path = await Get( + PackPythonPath, PackPythonPathRequest(request.target.address) + ) + return PytestPluginSetup( + # digest=EMPTY_DIGEST, + extra_sys_path=pack_python_path.entries, + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(PytestPluginSetupRequest, PytestPackTestRequest), + ] diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index de349ee4c8..6cdd7c9f8d 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -12,12 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pants.backend.python.target_types import ( + PythonTestTarget, + PythonTestsGeneratorTarget, +) + from pack_metadata import tailor, target_types_rules from pack_metadata.python_rules import ( python_module_mapper, python_pack_content, + python_path_rules, ) from pack_metadata.target_types import ( + InjectPackPythonPathField, PackContentResourceTarget, PackMetadata, PackMetadataInGitSubmodule, @@ -27,10 +34,15 @@ def rules(): return [ + PythonTestsGeneratorTarget.register_plugin_field( + InjectPackPythonPathField, as_moved_field=True + ), + PythonTestTarget.register_plugin_field(InjectPackPythonPathField), *tailor.rules(), *target_types_rules.rules(), *python_pack_content.rules(), *python_module_mapper.rules(), + *python_path_rules.rules(), ] diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 74fd27d5da..8e3b67fc33 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -16,7 +16,12 @@ from typing import Optional, Sequence, Tuple from pants.engine.internals.native_engine import Address -from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, StringField +from pants.engine.target import ( + BoolField, + COMMON_TARGET_FIELDS, + Dependencies, + StringField, +) from pants.core.target_types import ( ResourceDependenciesField, ResourcesGeneratingSourcesField, @@ -193,3 +198,14 @@ class PacksGlob(GenericTarget): "subdirectories (packs) except those listed with ! in dependencies. " "This is unfortunately needed by tests that use a glob to load pack fixtures." ) + + +class InjectPackPythonPathField(BoolField): + alias = "inject_pack_python_path" + help = ( + "For pack tests, set this to true to make sure /lib or actions/ dirs get " + "added to PYTHONPATH (actually PEX_EXTRA_SYS_PATH). Use `__defaults__` to enable " + "this in the BUILD file where you define pack_metadata, like this: " + "`__defaults__(all=dict(inject_pack_python_path=True))`" + ) + default = False From d310899f42578368f878904af05031948fdcdb9b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 13:17:48 -0500 Subject: [PATCH 09/19] pants: enable pack python injection for pack tests --- contrib/chatops/BUILD | 2 ++ contrib/core/BUILD | 2 ++ contrib/debug/BUILD | 2 ++ contrib/default/BUILD | 2 ++ contrib/examples/BUILD | 2 ++ contrib/hello_st2/BUILD | 2 ++ contrib/linux/BUILD | 2 ++ contrib/packs/BUILD | 2 ++ 8 files changed, 16 insertions(+) diff --git a/contrib/chatops/BUILD b/contrib/chatops/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/chatops/BUILD +++ b/contrib/chatops/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/core/BUILD b/contrib/core/BUILD index 59673bd746..9df7a372c9 100644 --- a/contrib/core/BUILD +++ b/contrib/core/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/debug/BUILD b/contrib/debug/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/debug/BUILD +++ b/contrib/debug/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/default/BUILD b/contrib/default/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/default/BUILD +++ b/contrib/default/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/examples/BUILD b/contrib/examples/BUILD index de3b866405..ab10cd1c85 100644 --- a/contrib/examples/BUILD +++ b/contrib/examples/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/hello_st2/BUILD b/contrib/hello_st2/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/hello_st2/BUILD +++ b/contrib/hello_st2/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/linux/BUILD b/contrib/linux/BUILD index 8a73ff391a..201435eecc 100644 --- a/contrib/linux/BUILD +++ b/contrib/linux/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/packs/BUILD b/contrib/packs/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/packs/BUILD +++ b/contrib/packs/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) From f4aba366842cb94d4603d459abf4a363206c2fe2 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 13:46:37 -0500 Subject: [PATCH 10/19] pants-plugins/uses_services: switch python_targets(uses=...) to a moved field --- pants-plugins/uses_services/register.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 1b5b6e91a2..346f4ecf2e 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -28,7 +28,9 @@ def rules(): return [ - PythonTestsGeneratorTarget.register_plugin_field(UsesServicesField), + PythonTestsGeneratorTarget.register_plugin_field( + UsesServicesField, as_moved_field=True + ), PythonTestTarget.register_plugin_field(UsesServicesField), *platform_rules.rules(), *mongo_rules.rules(), From 31462910b4ab7f25acf64d1640138b564887e3ec Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 14:41:46 -0500 Subject: [PATCH 11/19] pants-plugins/pack_metadata: stub tests for new rules --- .../pack_metadata/python_rules/BUILD | 4 +++ .../python_rules/python_module_mapper_test.py | 17 +++++++++++++ .../python_rules/python_pack_content_test.py | 25 +++++++++++++++++++ .../python_rules/python_path_rules_test.py | 21 ++++++++++++++++ 4 files changed, 67 insertions(+) create mode 100644 pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py create mode 100644 pants-plugins/pack_metadata/python_rules/python_pack_content_test.py create mode 100644 pants-plugins/pack_metadata/python_rules/python_path_rules_test.py diff --git a/pants-plugins/pack_metadata/python_rules/BUILD b/pants-plugins/pack_metadata/python_rules/BUILD index db46e8d6c9..0eea8b1cf1 100644 --- a/pants-plugins/pack_metadata/python_rules/BUILD +++ b/pants-plugins/pack_metadata/python_rules/BUILD @@ -1 +1,5 @@ python_sources() + +python_tests( + name="tests", +) diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py new file mode 100644 index 0000000000..d4b655d9de --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py @@ -0,0 +1,17 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_map_pack_content_to_python_modules() -> None: + pass diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py new file mode 100644 index 0000000000..a6dfc16862 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -0,0 +1,25 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_find_pack_metadata_targets_of_types() -> None: + pass + + +def test_find_pack_content_python_entry_points() -> None: + pass + + +def test_find_python_in_pack_lib_directories() -> None: + pass diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py new file mode 100644 index 0000000000..b887c6e857 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py @@ -0,0 +1,21 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_get_extra_sys_path_for_pack_dependencies() -> None: + pass + + +def test_inject_extra_sys_path_for_pack_tests() -> None: + pass From a9cf6c00c801c1ac6925a4f3fe1059a17f37dc1f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 18:29:15 -0500 Subject: [PATCH 12/19] pants-plugins/pack_metadata: add python_rules.conftest.rule_runner fixture Writes various test scenarios in the test sandbox. This should avoid repeating the setup in every test. --- .../pack_metadata/python_rules/BUILD | 4 + .../pack_metadata/python_rules/conftest.py | 282 ++++++++++++++++++ .../python_rules/python_module_mapper_test.py | 4 +- .../python_rules/python_pack_content_test.py | 8 +- .../python_rules/python_path_rules_test.py | 6 +- 5 files changed, 298 insertions(+), 6 deletions(-) create mode 100644 pants-plugins/pack_metadata/python_rules/conftest.py diff --git a/pants-plugins/pack_metadata/python_rules/BUILD b/pants-plugins/pack_metadata/python_rules/BUILD index 0eea8b1cf1..a172051977 100644 --- a/pants-plugins/pack_metadata/python_rules/BUILD +++ b/pants-plugins/pack_metadata/python_rules/BUILD @@ -3,3 +3,7 @@ python_sources() python_tests( name="tests", ) + +python_test_utils( + name="test_utils", +) diff --git a/pants-plugins/pack_metadata/python_rules/conftest.py b/pants-plugins/pack_metadata/python_rules/conftest.py new file mode 100644 index 0000000000..b5113983f8 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/conftest.py @@ -0,0 +1,282 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from textwrap import dedent + +import pytest +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, +) +from pants.backend.python.goals.pytest_runner import PytestPluginSetup +from pants.backend.python.target_types import ( + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PythonTestTarget, + PythonTestsGeneratorTarget, +) +from pants.backend.python.target_types_rules import rules as python_target_types_rules +from pants.engine.rules import QueryRule +from pants.testutil.python_rule_runner import PythonRuleRunner +from pants.testutil.rule_runner import RuleRunner + +from pack_metadata.python_rules import ( + python_module_mapper, + python_pack_content, + python_path_rules, +) +from pack_metadata.python_rules.python_module_mapper import ( + St2PythonPackContentMappingMarker, +) +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest, + PackPythonLibs, + PackPythonLibsRequest, +) +from pack_metadata.python_rules.python_path_rules import ( + PackPythonPath, + PackPythonPathRequest, + PytestPackTestRequest, +) +from pack_metadata.target_types import ( + InjectPackPythonPathField, + PackContentResourceTarget, + PackMetadata, +) + +# some random pack names +packs = ( + "foo", # imports between actions + "dr_seuss", # imports from /actions/lib + "shards", # imports from /lib + "metals", # imports the action from a subdirectory +) + + +@pytest.fixture +def pack_names() -> tuple[str, ...]: + return packs + + +def write_test_files(rule_runner: RuleRunner): + for pack in packs: + rule_runner.write_files( + { + f"packs/{pack}/BUILD": dedent( + """ + __defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata(name="metadata") + """ + ), + f"packs/{pack}/pack.yaml": dedent( + f""" + --- + name: {pack} + version: 1.0.0 + author: StackStorm + email: info@stackstorm.com + """ + ), + f"packs/{pack}/config.schema.yaml": "", + f"packs/{pack}/{pack}.yaml.example": "", + f"packs/{pack}/icon.png": "", + f"packs/{pack}/README.md": f"# Pack {pack} README", + } + ) + + def action_metadata_file(action: str, entry_point: str = "") -> str: + entry_point = entry_point or f"{action}.py" + return dedent( + f""" + --- + name: {action} + runner_type: python-script + entry_point: {entry_point} + """ + ) + + def test_file(module: str, _object: str) -> str: + return dedent( + f""" + from {module} import {_object} + def test_{module.replace(".", "_")}() -> None: + pass + """ + ) + + rule_runner.write_files( + { + "packs/foo/actions/BUILD": "python_sources()", + "packs/foo/actions/get_bar.yaml": action_metadata_file("get_bar"), + "packs/foo/actions/get_bar.py": dedent( + """ + RESPONSE_CONSTANT = "foobar_key" + class BarAction: + def run(self): + return {RESPONSE_CONSTANT: "bar"} + """ + ), + "packs/foo/actions/get_baz.yaml": action_metadata_file("get_baz"), + "packs/foo/actions/get_baz.py": dedent( + """ + from get_bar import RESPONSE_CONSTANT + class BazAction: + def run(self): + return {RESPONSE_CONSTANT: "baz"} + """ + ), + "packs/foo/tests/BUILD": "python_tests()", + "packs/foo/tests/test_get_bar_action.py": test_file("get_bar", "BarAction"), + "packs/foo/tests/test_get_baz_action.py": test_file("get_baz", "BazAction"), + "packs/dr_seuss/actions/lib/BUILD": "python_sources()", + "packs/dr_seuss/actions/lib/seuss/__init__.py": "", + "packs/dr_seuss/actions/lib/seuss/things.py": dedent( + """ + THING1 = "thing one" + THING2 = "thing two" + """ + ), + "packs/dr_seuss/actions/BUILD": "python_sources()", + "packs/dr_seuss/actions/get_from_actions_lib.yaml": action_metadata_file( + "get_from_actions_lib" + ), + "packs/dr_seuss/actions/get_from_actions_lib.py": dedent( + """ + from seuss.things import THING1, THING2 + class GetFromActionsLibAction: + def run(self): + return {"things": (THING1, THING2)} + """ + ), + "packs/dr_seuss/tests/BUILD": "python_tests()", + "packs/dr_seuss/tests/test_get_from_actions_lib_action.py": test_file( + "get_from_actions_lib", "GetFromActionsLibAction" + ), + "packs/shards/lib/stormlight_archive/BUILD": "python_sources()", + "packs/shards/lib/stormlight_archive/__init__.py": "", + "packs/shards/lib/stormlight_archive/things.py": dedent( + """ + STORM_LIGHT = "Honor" + VOID_LIGHT = "Odium" + LIFE_LIGHT = "Cultivation" + """ + ), + "packs/shards/actions/BUILD": "python_sources()", + "packs/shards/actions/get_from_pack_lib.yaml": action_metadata_file( + "get_from_pack_lib" + ), + "packs/shards/actions/get_from_pack_lib.py": dedent( + """ + from stormlight_archive.things import STORM_LIGHT, VOID_LIGHT, LIFE_LIGHT + class GetFromPackLibAction: + def run(self): + return {"light_sources": (STORM_LIGHT, VOID_LIGHT, LIFE_LIGHT)} + """ + ), + "packs/shards/sensors/BUILD": "python_sources()", + "packs/shards/sensors/horn_eater.yaml": dedent( + """ + --- + name: horn_eater + entry_point: horn_eater.py + class_name: HornEaterSensor + trigger_types: [{name: horn_eater.saw.spren, payload_schema: {type: object}}] + """ + ), + "packs/shards/sensors/horn_eater.py": dedent( + """ + from st2reactor.sensor.base import PollingSensor + from stormlight_archive.things import STORM_LIGHT + class HornEaterSensor(PollingSensor): + def setup(self): pass + def poll(self): + if STORM_LIGHT in self.config: + self.sensor_service.dispatch( + trigger="horn_eater.saw.spren", payload={"spren_type": STORM_LIGHT} + ) + def cleanup(self): pass + def add_trigger(self): pass + def update_trigger(self): pass + def remove_trigger(self): pass + """ + ), + "packs/shards/tests/BUILD": "python_tests()", + "packs/shards/tests/test_get_from_pack_lib_action.py": test_file( + "get_from_pack_lib", "GetFromPackLibAction" + ), + "packs/shards/tests/test_horn_eater_sensor.py": test_file( + "horn_eater", "HornEaterSensor" + ), + "packs/metals/actions/fly.yaml": action_metadata_file( + "fly", "mist_born/fly.py" + ), + "packs/metals/actions/mist_born/BUILD": "python_sources()", + "packs/metals/actions/mist_born/__init__.py": "", + "packs/metals/actions/mist_born/fly.py": dedent( + """ + class FlyAction: + def run(self): + return {"metals": ("steel", "iron")} + """ + ), + "packs/metals/tests/BUILD": "python_tests()", + "packs/metals/tests/test_fly_action.py": test_file( + "mist_born.fly", "FlyAction" + ), + } + ) + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = PythonRuleRunner( + rules=[ + PythonTestsGeneratorTarget.register_plugin_field( + InjectPackPythonPathField, as_moved_field=True + ), + PythonTestTarget.register_plugin_field(InjectPackPythonPathField), + *python_target_types_rules(), + # TODO: not sure if we need a QueryRule for every rule... + *python_pack_content.rules(), + QueryRule( + PackContentResourceTargetsOfType, + (PackContentResourceTargetsOfTypeRequest,), + ), + QueryRule( + PackContentPythonEntryPoints, (PackContentPythonEntryPointsRequest,) + ), + QueryRule(PackPythonLibs, (PackPythonLibsRequest,)), + *python_module_mapper.rules(), + QueryRule( + FirstPartyPythonMappingImpl, (St2PythonPackContentMappingMarker,) + ), + *python_path_rules.rules(), + QueryRule(PackPythonPath, (PackPythonPathRequest,)), + QueryRule(PytestPluginSetup, (PytestPackTestRequest,)), + ], + target_types=[ + PackContentResourceTarget, + PackMetadata, + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PythonTestTarget, + PythonTestsGeneratorTarget, + ], + ) + write_test_files(rule_runner) + args = ["--source-root-patterns=packs/*"] + rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) + return rule_runner diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py index d4b655d9de..0bf6854651 100644 --- a/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pants.testutil.rule_runner import RuleRunner -def test_map_pack_content_to_python_modules() -> None: + +def test_map_pack_content_to_python_modules(rule_runner: RuleRunner) -> None: pass diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py index a6dfc16862..9f7006e225 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pants.testutil.rule_runner import RuleRunner -def test_find_pack_metadata_targets_of_types() -> None: + +def test_find_pack_metadata_targets_of_types(rule_runner: RuleRunner) -> None: pass -def test_find_pack_content_python_entry_points() -> None: +def test_find_pack_content_python_entry_points(rule_runner: RuleRunner) -> None: pass -def test_find_python_in_pack_lib_directories() -> None: +def test_find_python_in_pack_lib_directories(rule_runner: RuleRunner) -> None: pass diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py index b887c6e857..865e6fa443 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pants.testutil.rule_runner import RuleRunner -def test_get_extra_sys_path_for_pack_dependencies() -> None: + +def test_get_extra_sys_path_for_pack_dependencies(rule_runner: RuleRunner) -> None: pass -def test_inject_extra_sys_path_for_pack_tests() -> None: +def test_inject_extra_sys_path_for_pack_tests(rule_runner: RuleRunner) -> None: pass From 55e9b8daccc84a8e19d7121c31138bac7e74d11e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 19:37:27 -0500 Subject: [PATCH 13/19] pants-plugins/pack_metadata: add test for pack content type detection And fix the identified issues. --- .../pack_metadata/python_rules/conftest.py | 2 +- .../python_rules/python_pack_content_test.py | 59 ++++++++++++++++++- pants-plugins/pack_metadata/target_types.py | 14 +++-- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/conftest.py b/pants-plugins/pack_metadata/python_rules/conftest.py index b5113983f8..20b177d776 100644 --- a/pants-plugins/pack_metadata/python_rules/conftest.py +++ b/pants-plugins/pack_metadata/python_rules/conftest.py @@ -91,7 +91,7 @@ def write_test_files(rule_runner: RuleRunner): """ ), f"packs/{pack}/config.schema.yaml": "", - f"packs/{pack}/{pack}.yaml.example": "", + f"packs/{pack}/config.yaml.example": "", f"packs/{pack}/icon.png": "", f"packs/{pack}/README.md": f"# Pack {pack} README", } diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py index 9f7006e225..a9a7026c5f 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -12,11 +12,66 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from pants.testutil.rule_runner import RuleRunner +from pack_metadata.python_rules.python_pack_content import ( + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest, +) +from pack_metadata.target_types import PackContentResourceTypes -def test_find_pack_metadata_targets_of_types(rule_runner: RuleRunner) -> None: - pass + +@pytest.mark.parametrize( + "requested_types,expected_count,expected_file_name", + ( + # one content type + ((PackContentResourceTypes.pack_metadata,), 4, "pack.yaml"), + ((PackContentResourceTypes.pack_config_schema,), 4, "config.schema.yaml"), + ((PackContentResourceTypes.pack_config_example,), 4, "config.yaml.example"), + ((PackContentResourceTypes.pack_icon,), 4, "icon.png"), + ((PackContentResourceTypes.action_metadata,), 5, ".yaml"), + ((PackContentResourceTypes.sensor_metadata,), 1, ".yaml"), + ((PackContentResourceTypes.rule_metadata,), 0, ""), + ((PackContentResourceTypes.policy_metadata,), 0, ""), + ((PackContentResourceTypes.unknown,), 0, ""), + # all content types + ((), 22, ""), + # some content types + ( + ( + PackContentResourceTypes.action_metadata, + PackContentResourceTypes.sensor_metadata, + ), + 6, + "", + ), + ( + ( + PackContentResourceTypes.pack_metadata, + PackContentResourceTypes.pack_config_schema, + PackContentResourceTypes.pack_config_example, + ), + 12, + "", + ), + ), +) +def test_find_pack_metadata_targets_of_types( + rule_runner: RuleRunner, + requested_types: tuple[PackContentResourceTypes, ...], + expected_count: int, + expected_file_name: str, +) -> None: + result = rule_runner.request( + PackContentResourceTargetsOfType, + (PackContentResourceTargetsOfTypeRequest(requested_types),), + ) + assert len(result) == expected_count + if expected_file_name: + for tgt in result: + tgt.address.relative_file_path.endswith(expected_file_name) def test_find_pack_content_python_entry_points(rule_runner: RuleRunner) -> None: diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 8e3b67fc33..01d80c24ad 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -86,17 +86,21 @@ def compute_value( if value is not None: return PackContentResourceTypes(value) path = PurePath(address.relative_file_path) - _yaml_suffixes = ("yaml", "yml") + _yaml_suffixes = (".yaml", ".yml") if len(path.parent.parts) == 0: # in the pack root - if path.name == "pack.yaml": + if path.stem == "pack" and path.suffix in _yaml_suffixes: return PackContentResourceTypes.pack_metadata - if path.stem == "pack.schema" and path.suffix in _yaml_suffixes: + if path.stem == "config.schema" and path.suffix in _yaml_suffixes: return PackContentResourceTypes.pack_config_schema - if path.suffix == "example" and path.suffixes[0] in _yaml_suffixes: + if ( + path.stem.startswith("config.") + and path.suffixes[0] in _yaml_suffixes + and path.suffix == ".example" + ): return PackContentResourceTypes.pack_config_example if path.name == "icon.png": - return PackContentResourceTypes.pack_config_example + return PackContentResourceTypes.pack_icon return PackContentResourceTypes.unknown resource_type = _content_type_by_path_parts.get(path.parent.parts, None) if resource_type is not None: From 6fa9b9893b9e780ac38c57a1bb45e713bd7ffd12 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 3 Aug 2024 22:17:38 -0500 Subject: [PATCH 14/19] pants-plugins/pack_metadata: add tests for entry_point and pack_lib rules And fix the identified mistake in conftest. --- .../pack_metadata/python_rules/conftest.py | 2 +- .../python_rules/python_pack_content_test.py | 80 ++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/conftest.py b/pants-plugins/pack_metadata/python_rules/conftest.py index 20b177d776..d481a1d5d1 100644 --- a/pants-plugins/pack_metadata/python_rules/conftest.py +++ b/pants-plugins/pack_metadata/python_rules/conftest.py @@ -141,7 +141,7 @@ def run(self): "packs/foo/tests/BUILD": "python_tests()", "packs/foo/tests/test_get_bar_action.py": test_file("get_bar", "BarAction"), "packs/foo/tests/test_get_baz_action.py": test_file("get_baz", "BazAction"), - "packs/dr_seuss/actions/lib/BUILD": "python_sources()", + "packs/dr_seuss/actions/lib/seuss/BUILD": "python_sources()", "packs/dr_seuss/actions/lib/seuss/__init__.py": "", "packs/dr_seuss/actions/lib/seuss/things.py": dedent( """ diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py index a9a7026c5f..33c1389bb3 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -14,11 +14,16 @@ import pytest +from pants.engine.addresses import Address from pants.testutil.rule_runner import RuleRunner from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, PackContentResourceTargetsOfType, PackContentResourceTargetsOfTypeRequest, + PackPythonLibs, + PackPythonLibsRequest, ) from pack_metadata.target_types import PackContentResourceTypes @@ -75,8 +80,79 @@ def test_find_pack_metadata_targets_of_types( def test_find_pack_content_python_entry_points(rule_runner: RuleRunner) -> None: - pass + result = rule_runner.request( + PackContentPythonEntryPoints, + (PackContentPythonEntryPointsRequest(),), + ) + assert len(result) == 6 # 5 actions + 1 sensor + assert {res.metadata_address for res in result} == { + Address( + "packs/foo", + relative_file_path="actions/get_bar.yaml", + target_name="metadata", + ), + Address( + "packs/foo", + relative_file_path="actions/get_baz.yaml", + target_name="metadata", + ), + Address( + "packs/dr_seuss", + relative_file_path="actions/get_from_actions_lib.yaml", + target_name="metadata", + ), + Address( + "packs/shards", + relative_file_path="actions/get_from_pack_lib.yaml", + target_name="metadata", + ), + Address( + "packs/shards", + relative_file_path="sensors/horn_eater.yaml", + target_name="metadata", + ), + Address( + "packs/metals", + relative_file_path="actions/fly.yaml", + target_name="metadata", + ), + } + assert {(res.content_type, res.entry_point) for res in result} == { + (PackContentResourceTypes.action_metadata, "get_bar.py"), + (PackContentResourceTypes.action_metadata, "get_baz.py"), + (PackContentResourceTypes.action_metadata, "get_from_actions_lib.py"), + (PackContentResourceTypes.action_metadata, "get_from_pack_lib.py"), + (PackContentResourceTypes.sensor_metadata, "horn_eater.py"), + (PackContentResourceTypes.action_metadata, "mist_born/fly.py"), + } + assert {res.python_address for res in result} == { + Address("packs/foo/actions", relative_file_path="get_bar.py"), + Address("packs/foo/actions", relative_file_path="get_baz.py"), + Address("packs/dr_seuss/actions", relative_file_path="get_from_actions_lib.py"), + Address("packs/shards/actions", relative_file_path="get_from_pack_lib.py"), + Address("packs/shards/sensors", relative_file_path="horn_eater.py"), + Address("packs/metals/actions/mist_born", relative_file_path="fly.py"), + } def test_find_python_in_pack_lib_directories(rule_runner: RuleRunner) -> None: - pass + result = rule_runner.request(PackPythonLibs, (PackPythonLibsRequest(),)) + assert len(result) == 4 + assert {(str(res.pack_path), res.lib_dir) for res in result} == { + ("packs/dr_seuss", "actions/lib"), + ("packs/shards", "lib"), + } + assert {res.python_address for res in result} == { + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="__init__.py"), + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="things.py"), + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="__init__.py" + ), + Address("packs/shards/lib/stormlight_archive", relative_file_path="things.py"), + } + assert {res.module for res in result} == { + "seuss", + "seuss.things", + "stormlight_archive", + "stormlight_archive.things", + } From f43f2d107c4e47b637c12b61a5bdf24f2f2ad8c4 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 5 Aug 2024 11:45:48 -0500 Subject: [PATCH 15/19] pants-plugins/pack_metadata: add test for python module mapper rule --- .../python_rules/python_module_mapper_test.py | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py index 0bf6854651..d1d6d779fb 100644 --- a/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py @@ -12,8 +12,62 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, + ModuleProvider, + ModuleProviderType, +) +from pants.engine.internals.native_engine import Address from pants.testutil.rule_runner import RuleRunner +from pants.util.frozendict import FrozenDict + +from pack_metadata.python_rules.python_module_mapper import ( + St2PythonPackContentMappingMarker, +) def test_map_pack_content_to_python_modules(rule_runner: RuleRunner) -> None: - pass + result = rule_runner.request( + FirstPartyPythonMappingImpl, + (St2PythonPackContentMappingMarker(),), + ) + + def module_provider(spec_path: str, relative_file_path: str) -> ModuleProvider: + return ModuleProvider( + Address(spec_path=spec_path, relative_file_path=relative_file_path), + ModuleProviderType.IMPL, + ) + + expected = { + "": { + "get_bar": (module_provider("packs/foo/actions", "get_bar.py"),), + "get_baz": (module_provider("packs/foo/actions", "get_baz.py"),), + "seuss": ( + module_provider("packs/dr_seuss/actions/lib/seuss", "__init__.py"), + ), + "seuss.things": ( + module_provider("packs/dr_seuss/actions/lib/seuss", "things.py"), + ), + "get_from_actions_lib": ( + module_provider("packs/dr_seuss/actions", "get_from_actions_lib.py"), + ), + "stormlight_archive": ( + module_provider("packs/shards/lib/stormlight_archive", "__init__.py"), + ), + "stormlight_archive.things": ( + module_provider("packs/shards/lib/stormlight_archive", "things.py"), + ), + "get_from_pack_lib": ( + module_provider("packs/shards/actions", "get_from_pack_lib.py"), + ), + "horn_eater": (module_provider("packs/shards/sensors", "horn_eater.py"),), + "fly": (module_provider("packs/metals/actions/mist_born", "fly.py"),), + "mist_born.fly": ( + module_provider("packs/metals/actions/mist_born", "fly.py"), + ), + } + } + assert isinstance(result, FrozenDict) + assert all(isinstance(value, FrozenDict) for value in result.values()) + # pytest reports dict differences better than FrozenDict + assert {resolve: dict(value) for resolve, value in result.items()} == expected From 6d7813665dcd7323503732636157a71439a3be60 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 5 Aug 2024 12:10:22 -0500 Subject: [PATCH 16/19] pants-plugins/pack_metadata: add test for get_extra_sys_path_for_pack_dependencies rule --- .../python_rules/python_path_rules_test.py | 105 +++++++++++++++++- 1 file changed, 103 insertions(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py index 865e6fa443..06bc72e787 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py @@ -12,11 +12,112 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest +from pants.engine.internals.native_engine import Address from pants.testutil.rule_runner import RuleRunner +from pack_metadata.python_rules.python_path_rules import ( + PackPythonPath, + PackPythonPathRequest, +) -def test_get_extra_sys_path_for_pack_dependencies(rule_runner: RuleRunner) -> None: - pass + +@pytest.mark.parametrize( + "address,expected", + ( + ( + Address("packs/foo/actions", relative_file_path="get_bar.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/actions", relative_file_path="get_baz.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_bar_action.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_baz_action.py"), + ("packs/foo/actions",), + ), + ( + Address( + "packs/dr_seuss/actions/lib/seuss", relative_file_path="__init__.py" + ), + ("packs/dr_seuss/actions/lib",), + ), + ( + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="things.py"), + ("packs/dr_seuss/actions/lib",), + ), + ( + Address( + "packs/dr_seuss/actions", relative_file_path="get_from_actions_lib.py" + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/dr_seuss/tests", + relative_file_path="test_get_from_actions_lib_action.py", + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="__init__.py" + ), + ("packs/shards/lib",), + ), + ( + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="things.py" + ), + ("packs/shards/lib",), + ), + ( + Address("packs/shards/actions", relative_file_path="get_from_pack_lib.py"), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address("packs/shards/sensors", relative_file_path="horn_eater.py"), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", + relative_file_path="test_get_from_pack_lib_action.py", + ), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", relative_file_path="test_horn_eater_sensor.py" + ), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address("packs/metals/actions/mist_born", relative_file_path="__init__.py"), + (), # there are no dependencies, and this is not an action entry point. + ), + ( + Address("packs/metals/actions/mist_born", relative_file_path="fly.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ( + Address("packs/metals/tests", relative_file_path="test_fly_action.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ), +) +def test_get_extra_sys_path_for_pack_dependencies( + rule_runner: RuleRunner, address: Address, expected: tuple[str, ...] +) -> None: + pack_python_path = rule_runner.request( + PackPythonPath, (PackPythonPathRequest(address),) + ) + assert pack_python_path.entries == expected def test_inject_extra_sys_path_for_pack_tests(rule_runner: RuleRunner) -> None: From 3601300682b2d637b79683a6561f6952d26fedab Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 5 Aug 2024 12:22:18 -0500 Subject: [PATCH 17/19] pants-plugins/pack_metadata: add test for inject_extra_sys_path_for_pack_tests rule --- .../python_rules/python_path_rules_test.py | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py index 06bc72e787..74ff010b40 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py @@ -13,12 +13,14 @@ # limitations under the License. import pytest -from pants.engine.internals.native_engine import Address +from pants.backend.python.goals.pytest_runner import PytestPluginSetup +from pants.engine.internals.native_engine import Address, EMPTY_DIGEST from pants.testutil.rule_runner import RuleRunner from pack_metadata.python_rules.python_path_rules import ( PackPythonPath, PackPythonPathRequest, + PytestPackTestRequest, ) @@ -120,5 +122,48 @@ def test_get_extra_sys_path_for_pack_dependencies( assert pack_python_path.entries == expected -def test_inject_extra_sys_path_for_pack_tests(rule_runner: RuleRunner) -> None: - pass +@pytest.mark.xfail(raises=AttributeError, reason="Not implemented in pants yet.") +@pytest.mark.parametrize( + "address,expected", + ( + ( + Address("packs/foo/tests", relative_file_path="test_get_bar_action.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_baz_action.py"), + ("packs/foo/actions",), + ), + ( + Address( + "packs/dr_seuss/tests", + relative_file_path="test_get_from_actions_lib_action.py", + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/shards/tests", + relative_file_path="test_get_from_pack_lib_action.py", + ), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", relative_file_path="test_horn_eater_sensor.py" + ), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address("packs/metals/tests", relative_file_path="test_fly_action.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ), +) +def test_inject_extra_sys_path_for_pack_tests( + rule_runner: RuleRunner, address: Address, expected: tuple[str, ...] +) -> None: + target = rule_runner.get_target(address) + result = rule_runner.request(PytestPluginSetup, (PytestPackTestRequest(target),)) + assert result.digest == EMPTY_DIGEST + assert result.extra_sys_path == expected From 5cd5a26fde5bd7d03b0a04bcb8cf7d11f343d7a7 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 18:08:45 -0500 Subject: [PATCH 18/19] pants: ignore pylint error --- contrib/examples/actions/pythonactions/isprime.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/examples/actions/pythonactions/isprime.py b/contrib/examples/actions/pythonactions/isprime.py index 5116831a37..65294a5619 100644 --- a/contrib/examples/actions/pythonactions/isprime.py +++ b/contrib/examples/actions/pythonactions/isprime.py @@ -15,7 +15,8 @@ import math -from environ import get_environ +# TODO: extend pants and pants-plugins/pack_metadata to add lib dirs extra_sys_path for pylint +from environ import get_environ # pylint: disable=E0401 from st2common.runners.base_action import Action From b3d8c4c0dd442257fd12622538cb1cb45bed09eb Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 5 Oct 2024 12:50:34 -0500 Subject: [PATCH 19/19] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 677af5dd79..9b7b381c1f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -65,7 +65,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 - #6254 #6258 #6259 + #6254 #6258 #6259 #6260 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11