chore(hermetic build): changes to take owlbot-hermetic.yaml additions/removals from config

hermetic_build/common/model/gapic_config.py

@@ -39,6 +39,9 @@ def is_stable(self):
     def get_version(self):
         return self.version
 
+    def to_dict(self):
+        return {"proto_path": self.proto_path}
+
     def __parse_version(self) -> Optional[str]:
         version_regex = re.compile(r"^v[1-9]+(p[1-9]+)*(alpha|beta)?.*")
         for directory in self.proto_path.split("/"):

hermetic_build/common/model/generation_config.py

@@ -18,6 +18,11 @@
 from typing import Optional
 from common.model.library_config import LibraryConfig
 from common.model.gapic_config import GapicConfig
+from common.model.owlbot_yaml_config import (
+    OwlbotYamlConfig,
+    OwlbotYamlAdditionRemoval,
+    DeepCopyRegexItem,
+)
 
 REPO_LEVEL_PARAMETER = "Repo level parameter"
 LIBRARY_LEVEL_PARAMETER = "Library level parameter"
@@ -77,6 +82,23 @@ def contains_common_protos(self) -> bool:
                     break
         return self.__contains_common_protos
 
+    def to_dict(self):
+        return {
+            "gapic_generator_version": self.gapic_generator_version,
+            "googleapis_commitish": self.googleapis_commitish,
+            "libraries_bom_version": self.libraries_bom_version,
+            "libraries": [library.to_dict() for library in self.libraries],
+        }
+
+    def write_object_to_yaml(self, file_path):
+        """Writes a Python object to a YAML file."""
+        try:
+            with open(file_path, "w") as file:
+                yaml.dump(self.to_dict(), file, indent=2, sort_keys=False)
+            print(f"Object written to {file_path}")
+        except Exception as e:
+            print(f"Error writing to YAML file: {e}")
+
     @staticmethod
     def __set_generator_version(gapic_generator_version: Optional[str]) -> str:
         if gapic_generator_version is not None:
@@ -131,6 +153,8 @@ def from_yaml(path_to_yaml: str) -> "GenerationConfig":
                 new_gapic = GapicConfig(proto_path)
                 parsed_gapics.append(new_gapic)
 
+            owlbot_yaml = _owlbot_yaml_config_from_yaml(library)
+
             new_library = LibraryConfig(
                 api_shortname=_required(library, "api_shortname"),
                 api_description=_required(library, "api_description"),
@@ -160,6 +184,7 @@ def from_yaml(path_to_yaml: str) -> "GenerationConfig":
                 recommended_package=_optional(library, "recommended_package", None),
                 min_java_version=_optional(library, "min_java_version", None),
                 transport=_optional(library, "transport", None),
+                owlbot_yaml=owlbot_yaml,
             )
             parsed_libraries.append(new_library)
 
@@ -190,3 +215,59 @@ def _optional(config: dict, key: str, default: any):
     if key not in config:
         return default
     return config[key]
+
+
+def _owlbot_yaml_addition_remove_from_yaml(data: dict) -> OwlbotYamlAdditionRemoval:
+    """
+    Parses the addition or remove section from owlbot_yaml data.
+    """
+    deep_copy_regex = _optional(data, "deep_copy_regex", None)
+    deep_remove_regex = _optional(data, "deep_remove_regex", None)
+    deep_preserve_regex = _optional(data, "deep_preserve_regex", None)
+
+    parsed_deep_copy_regex = None
+    if deep_copy_regex:
+        parsed_deep_copy_regex = [
+            _deep_copy_regex_item_from_yaml(item) for item in deep_copy_regex
+        ]
+
+    return OwlbotYamlAdditionRemoval(
+        deep_copy_regex=parsed_deep_copy_regex,
+        deep_remove_regex=deep_remove_regex,
+        deep_preserve_regex=deep_preserve_regex,
+    )
+
+
+def _deep_copy_regex_item_from_yaml(data: dict) -> DeepCopyRegexItem:
+    """
+    Parses a DeepCopyRegexItem from a dictionary.
+    """
+    source = _required(data, "source")
+    dest = _required(data, "dest")
+    return DeepCopyRegexItem(source=source, dest=dest)
+
+
+def _owlbot_yaml_config_from_yaml(
+    library: LibraryConfig,
+) -> Optional["OwlbotYamlConfig"]:
+    """
+    Parses the owlbot_yaml section from a library's data.
+    """
+    owlbot_yaml_data = _optional(library, "owlbot_yaml", None)
+
+    if not owlbot_yaml_data:
+        return None
+
+    addition_data = _optional(owlbot_yaml_data, "addition", None)
+    removal_data = _optional(owlbot_yaml_data, "remove", None)
+
+    additions = None
+    removals = None
+
+    if addition_data:
+        additions = _owlbot_yaml_addition_remove_from_yaml(addition_data)
+
+    if removal_data:
+        removals = _owlbot_yaml_addition_remove_from_yaml(removal_data)
+
+    return OwlbotYamlConfig(additions=additions, removals=removals)

hermetic_build/common/model/library_config.py

@@ -16,6 +16,8 @@
 from typing import Optional
 from common.model.gapic_config import GapicConfig
 from common.model.gapic_inputs import GapicInputs
+from common.model.owlbot_yaml_config import OwlbotYamlConfig
+from collections import OrderedDict
 
 
 MAVEN_COORDINATE_SEPARATOR = ":"
@@ -54,6 +56,7 @@ def __init__(
         recommended_package: Optional[str] = None,
         min_java_version: Optional[int] = None,
         transport: Optional[str] = None,
+        owlbot_yaml: Optional[OwlbotYamlConfig] = None,
     ):
         self.api_shortname = api_shortname
         self.api_description = api_description
@@ -81,6 +84,7 @@ def __init__(
         self.min_java_version = min_java_version
         self.distribution_name = self.__get_distribution_name(distribution_name)
         self.transport = self.__validate_transport(transport)
+        self.owlbot_yaml = owlbot_yaml
 
     def set_gapic_configs(self, gapic_configs: list[GapicConfig]) -> None:
         """
@@ -122,6 +126,61 @@ def get_transport(self, gapic_inputs: GapicInputs) -> str:
         """
         return self.transport if self.transport is not None else gapic_inputs.transport
 
+    def to_dict(self):
+        """Converts the LibraryConfig object to a dictionary with ordered keys."""
+        data = {}
+        data["api_shortname"] = self.api_shortname
+        data["name_pretty"] = self.name_pretty
+        data["product_documentation"] = self.product_documentation
+        data["api_description"] = self.api_description
+        if self.library_type and self.library_type != "GAPIC_AUTO":
+            data["library_type"] = self.library_type
+        if self.release_level:
+            data["release_level"] = self.release_level
+        if self.api_id:
+            data["api_id"] = self.api_id
+        if self.api_reference:
+            data["api_reference"] = self.api_reference
+        if self.codeowner_team:
+            data["codeowner_team"] = self.codeowner_team
+        if self.client_documentation:
+            data["client_documentation"] = self.client_documentation
+        if self.distribution_name:
+            data["distribution_name"] = self.distribution_name
+        if self.excluded_dependencies:
+            data["excluded_dependencies"] = self.excluded_dependencies
+        if self.excluded_poms:
+            data["excluded_poms"] = self.excluded_poms
+        if self.googleapis_commitish:
+            data["googleapis_commitish"] = self.googleapis_commitish
+        if self.group_id and self.group_id != "com.google.cloud":
+            data["group_id"] = self.group_id
+        if self.issue_tracker:
+            data["issue_tracker"] = self.issue_tracker
+        if self.library_name:
+            data["library_name"] = self.library_name
+        if self.rest_documentation:
+            data["rest_documentation"] = self.rest_documentation
+        if self.rpc_documentation:
+            data["rpc_documentation"] = self.rpc_documentation
+        if self.cloud_api is False:  # Only spell out when false
+            data["cloud_api"] = self.cloud_api
+        if self.requires_billing is False:  # Only spell out when false
+            data["requires_billing"] = self.requires_billing
+        if self.extra_versioned_modules:
+            data["extra_versioned_modules"] = self.extra_versioned_modules
+        if self.recommended_package:
+            data["recommended_package"] = self.recommended_package
+        if self.min_java_version:
+            data["min_java_version"] = self.min_java_version
+        if self.transport:
+            data["transport"] = self.transport
+        if self.gapic_configs:
+            data["GAPICs"] = [gc.to_dict() for gc in self.gapic_configs]
+        if self.owlbot_yaml:
+            data["owlbot_yaml"] = self.owlbot_yaml.to_dict()
+        return data
+
     def __get_distribution_name(self, distribution_name: Optional[str]) -> str:
         LibraryConfig.__check_distribution_name(distribution_name)
         if distribution_name:

hermetic_build/common/model/owlbot_yaml_config.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+#  Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import List, Optional, Dict
+
+
+class DeepCopyRegexItem:
+    def __init__(self, source: str, dest: str):
+        self.source = source
+        self.dest = dest
+
+    def to_dict(self):
+        return {
+            "source": self.source,
+            "dest": self.dest,
+        }
+
+
+class OwlbotYamlAdditionRemoval:
+    def __init__(
+        self,
+        deep_copy_regex: Optional[List[DeepCopyRegexItem]] = None,
+        deep_remove_regex: Optional[List[str]] = None,
+        deep_preserve_regex: Optional[List[str]] = None,
+    ):
+        self.deep_copy_regex = deep_copy_regex
+        self.deep_remove_regex = deep_remove_regex
+        self.deep_preserve_regex = deep_preserve_regex
+
+    def to_dict(self):
+        data = {}
+        if self.deep_copy_regex:
+            data["deep_copy_regex"] = [item.to_dict() for item in self.deep_copy_regex]
+        if self.deep_remove_regex:
+            data["deep_remove_regex"] = self.deep_remove_regex
+        if self.deep_preserve_regex:
+            data["deep_preserve_regex"] = self.deep_preserve_regex
+        return data
+
+
+class OwlbotYamlConfig:
+    def __init__(
+        self,
+        additions: Optional[OwlbotYamlAdditionRemoval] = None,
+        removals: Optional[OwlbotYamlAdditionRemoval] = None,
+    ):
+        self.additions = additions
+        self.removals = removals
+
+    def to_dict(self):
+        data = {}
+        if self.additions:
+            data["additions"] = self.additions.to_dict()
+        if self.removals:
+            data["removals"] = self.removals.to_dict()
+        return data

hermetic_build/common/tests/model/gapic_config_unit_tests.py

@@ -54,6 +54,13 @@ def test_is_stable_with_stable_version_returns_true(self):
             GapicConfig(proto_path="example/dir1/dir2/v30").is_stable(),
         )
 
+    def test_to_dict_returns_proto_path_as_dict(self):
+        expected_config_as_dict = {"proto_path": "example/dir1/dir2"}
+        self.assertEqual(
+            expected_config_as_dict,
+            GapicConfig(proto_path="example/dir1/dir2").to_dict(),
+        )
+
     def test_compare_configs_without_a_version(self):
         config_len_3 = GapicConfig(proto_path="example/dir1/dir2")
         config_len_4 = GapicConfig(proto_path="example/dir1/dir2/dir3")

hermetic_build/common/tests/model/generation_config_unit_tests.py

@@ -11,9 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from io import StringIO
 import os
-import unittest
 from pathlib import Path
+import unittest
+from unittest.mock import patch, mock_open
+import yaml
+
 from common.model.generation_config import GenerationConfig
 from common.model.library_config import LibraryConfig
 
@@ -104,6 +109,21 @@ def test_from_yaml_succeeds(self):
         self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path)
         self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path)
 
+        owlbot_yaml_addition = library.owlbot_yaml.additions
+        owlbot_yaml_removal = library.owlbot_yaml.removals
+        self.assertEqual(
+            "/java-asset/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java",
+            owlbot_yaml_removal.deep_preserve_regex[0],
+        )
+        self.assertEqual(
+            "/owl-bot-staging/java-accesscontextmanager/type/proto-google-identity-accesscontextmanager-type/src",
+            owlbot_yaml_addition.deep_copy_regex[0].dest,
+        )
+        self.assertEqual(
+            "/google/identity/accesscontextmanager/type/.*-java/proto-google-.*/src",
+            owlbot_yaml_addition.deep_copy_regex[0].source,
+        )
+
     def test_get_proto_path_to_library_name_success(self):
         paths = GenerationConfig.from_yaml(
             f"{test_config_dir}/generation_config.yaml"
@@ -256,3 +276,65 @@ def test_from_yaml_with_zero_proto_path_raise_exception(self):
             GenerationConfig.from_yaml,
             f"{test_config_dir}/config_without_gapics_value.yaml",
         )
+
+    def test_to_dict_return_correctly(self):
+        config = GenerationConfig(
+            gapic_generator_version="x.y.z",
+            libraries_bom_version="a.b.c",
+            googleapis_commitish="foo",
+            libraries=[library_1],
+        )
+        expect_config_as_dict = {
+            "gapic_generator_version": "x.y.z",
+            "libraries_bom_version": "a.b.c",
+            "googleapis_commitish": "foo",
+            "libraries": [library_1.to_dict()],
+        }
+        self.assertEqual(expect_config_as_dict, config.to_dict())
+
+    @patch("builtins.open", new_callable=mock_open)
+    def test_write_object_to_yaml_success(self, mock_open_file):
+        config = GenerationConfig(
+            gapic_generator_version="x.y.z",
+            libraries_bom_version="a.b.c",
+            googleapis_commitish="foo",
+            libraries=[],
+        )
+
+        file_path = "test_output.yaml"
+        config.write_object_to_yaml(file_path)
+
+        # Assert that open was called with the correct arguments
+        mock_open_file.assert_called_once_with(file_path, "w")
+
+        # Get the handle that was used to write to the file
+        handle = mock_open_file()
+
+        # Get the written YAML data
+        written_data = "".join(call.args[0] for call in handle.write.call_args_list)
+
+        # Load the written data using yaml to verify
+        loaded_data = yaml.safe_load(written_data)
+
+        expected_data = {
+            "gapic_generator_version": "x.y.z",
+            "libraries_bom_version": "a.b.c",
+            "googleapis_commitish": "foo",
+            "libraries": [],
+        }
+
+        self.assertEqual(loaded_data, expected_data)
+
+    @patch("builtins.open", side_effect=Exception("File system error"))
+    @patch("sys.stdout", new_callable=StringIO)
+    def test_write_object_to_yaml_error(self, mock_stdout, mock_open_file):
+        config = GenerationConfig(
+            gapic_generator_version="",
+            googleapis_commitish="",
+            libraries=[library_1],
+        )
+        file_path = "test_output.yaml"
+        config.write_object_to_yaml(file_path)
+
+        # Assert that the error message was printed to stdout
+        self.assertIn("Error writing to YAML file:", mock_stdout.getvalue())