{HDInsight} new api version 2024-08-01-preview (#29752)

* added commands and new features

* Update python SDK version

* Update test session records

* Update SDK version

* fix lint errors

* Fix leak

* Fix Lint error

* Fix lint error

* Fix lint error

* trigger pipeline

* Update SDK version

---------

Co-authored-by: v-yuchenli <[email protected]>

src/azure-cli/azure/cli/command_modules/hdinsight/_help.py

@@ -168,7 +168,7 @@
 
 helps['hdinsight update'] = """
 type: command
-short-summary: Update the tags of the specified HDInsight cluster.
+short-summary: Update the tags or identity of the specified HDInsight cluster.
 examples:
   - name: Update the tags.
     text: |-
@@ -220,6 +220,26 @@
 short-summary: Get the status of Azure Monitor logs integration on an HDInsight cluster.
 """
 
+helps['hdinsight azure-monitor-agent'] = """
+type: group
+short-summary: Manage Azure Monitor Agent logs integration on an HDInsight cluster.
+"""
+
+helps['hdinsight azure-monitor-agent disable'] = """
+type: command
+short-summary: Disable the Azure Monitor Agent logs integration on an HDInsight cluster.
+"""
+
+helps['hdinsight azure-monitor-agent enable'] = """
+type: command
+short-summary: Enable the Azure Monitor Agent logs integration on an HDInsight cluster.
+"""
+
+helps['hdinsight azure-monitor-agent show'] = """
+type: command
+short-summary: Get the status of Azure Monitor Agent logs integration on an HDInsight cluster.
+"""
+
 helps['hdinsight rotate-disk-encryption-key'] = """
 type: command
 short-summary: Rotate the disk encryption key of the specified HDInsight cluster.

src/azure-cli/azure/cli/command_modules/hdinsight/_params.py

@@ -30,7 +30,8 @@ def load_arguments(self, _):
     from ._completers import subnet_completion_list, cluster_admin_account_completion_list, \
         cluster_user_group_completion_list, get_resource_name_completion_list_under_subscription
     from knack.arguments import CLIArgumentType
-    from azure.mgmt.hdinsight.models import Tier, JsonWebKeyEncryptionAlgorithm, ResourceProviderConnection
+    from azure.mgmt.hdinsight.models import Tier, JsonWebKeyEncryptionAlgorithm
+    from azure.mgmt.hdinsight.models import ResourceProviderConnection, OutboundDependenciesManagedType
     from argcomplete.completers import FilesCompleter
     node_size_type = CLIArgumentType(arg_group='Node',
                                      help='The size of the node. See also: https://docs.microsoft.com/azure/'
@@ -227,6 +228,17 @@ def load_arguments(self, _):
                    arg_type=get_enum_type(ResourceProviderConnection), help='The resource provider connection type')
         c.argument('enable_private_link', arg_group='Private Link', arg_type=get_three_state_flag(),
                    help='Indicate whether enable the private link or not.')
+        c.argument('outbound_dependencies_managed_type',
+                   options_list=['--outbound-dependencies-managed-type', '--outbound-managed-type'],
+                   arg_group='Private Link',
+                   arg_type=get_enum_type(OutboundDependenciesManagedType),
+                   help='The direction for the resource provider connection.')
+
+        c.argument('public_ip_tag_type', arg_group='Private Link',
+                   help='Gets or sets the ipTag type: Example FirstPartyUsage.')
+        c.argument('public_ip_tag_value', arg_group='Private Link',
+                   help='Gets or sets value of the IpTag associated with the public IP.'
+                   'Example HDInsight, SQL, Storage etc')
 
         c.argument('private_link_configurations',
                    options_list=['--private-link-config', '--private-link-configurations'],
@@ -252,6 +264,14 @@ def load_arguments(self, _):
             c.argument('target_instance_count', options_list=['--workernode-count', '-c'],
                        help='The target worker node instance count for the operation.', required=True)
 
+        # update
+        with self.argument_context('hdinsight update') as c:
+            c.argument('tags', tags_type)
+            c.argument('assign_identity', arg_group='Managed Service Identity', validator=validate_msi,
+                       completer=get_resource_name_completion_list_under_subscription(
+                           'Microsoft.ManagedIdentity/userAssignedIdentities'),
+                       help="The name or ID of user assigned identity.")
+
         # application
         with self.argument_context('hdinsight application') as c:
             c.argument('cluster_name', options_list=['--cluster-name'],
@@ -317,6 +337,16 @@ def load_arguments(self, _):
                                            'Required when workspace ID is provided.')
             c.ignore('workspace_type')
 
+        # Azure Monitor Agent
+        with self.argument_context('hdinsight azure-monitor-agent') as c:
+            c.argument('workspace', validator=validate_workspace,
+                       completer=get_resource_name_completion_list_under_subscription(
+                           'Microsoft.OperationalInsights/workspaces'),
+                       help='The name, resource ID or workspace ID of Log Analytics workspace.')
+            c.argument('primary_key', help='The certificate for the Log Analytics workspace. '
+                                           'Required when workspace ID is provided.')
+            c.ignore('workspace_type')
+
         with self.argument_context('hdinsight host') as c:
             c.argument('cluster_name', options_list=['--cluster-name'],
                        completer=get_resource_name_completion_list('Microsoft.HDInsight/clusters'),

src/azure-cli/azure/cli/command_modules/hdinsight/commands.py

@@ -106,6 +106,13 @@ def load_command_table(self, _):  # pylint: disable=too-many-statements
         g.custom_command('enable', 'enable_hdi_azure_monitor')
         g.command('disable', 'begin_disable_azure_monitor')
 
+    # New Azure Monitor Agent operations
+    with self.command_group('hdinsight azure-monitor-agent', hdinsight_extensions_sdk,
+                            client_factory=cf_hdinsight_extensions) as g:
+        g.show_command('show', 'get_azure_monitor_agent_status')
+        g.custom_command('enable', 'enable_hdi_azure_monitor_agent')
+        g.command('disable', 'begin_disable_azure_monitor_agent')
+
     # VirtualMachine operations
     with self.command_group('hdinsight host', hdinsight_virtual_machines_sdk,
                             client_factory=cf_hdinsight_virtual_machines) as g:

src/azure-cli/azure/cli/command_modules/hdinsight/custom.py

@@ -36,9 +36,10 @@ def create_cluster(cmd, client, cluster_name, resource_group_name, cluster_type,
                    autoscale_type=None, autoscale_min_workernode_count=None, autoscale_max_workernode_count=None,
                    timezone=None, days=None, time=None, autoscale_workernode_count=None,
                    encryption_at_host=None, esp=False, idbroker=False,
-                   resource_provider_connection=None, enable_private_link=None, enable_compute_isolation=None,
-                   host_sku=None, zones=None, private_link_configurations=None,
-                   no_validation_timeout=False):
+                   resource_provider_connection=None, enable_private_link=None,
+                   public_ip_tag_type=None, public_ip_tag_value=None,
+                   enable_compute_isolation=None, host_sku=None, zones=None, private_link_configurations=None,
+                   no_validation_timeout=False, outbound_dependencies_managed_type=None):
     from .util import build_identities_info, build_virtual_network_profile, parse_domain_name, \
         get_storage_account_endpoint, validate_esp_cluster_create_params, set_vm_size
     from azure.mgmt.hdinsight.models import ClusterCreateParametersExtended, ClusterCreateProperties, OSType, \
@@ -47,7 +48,7 @@ def create_cluster(cmd, client, cluster_name, resource_group_name, cluster_type,
         DiskEncryptionProperties, Tier, SshProfile, SshPublicKey, \
         KafkaRestProperties, ClientGroupInfo, EncryptionInTransitProperties, \
         Autoscale, AutoscaleCapacity, AutoscaleRecurrence, AutoscaleSchedule, AutoscaleTimeAndCapacity, \
-        NetworkProperties, PrivateLink, ComputeIsolationProperties
+        NetworkProperties, IpTag, PrivateLink, ComputeIsolationProperties
 
     validate_esp_cluster_create_params(esp, cluster_name, resource_group_name, cluster_type,
                                        subnet, domain, cluster_admin_account, assign_identity,
@@ -361,6 +362,13 @@ def create_cluster(cmd, client, cluster_name, resource_group_name, cluster_type,
         resource_provider_connection=resource_provider_connection,
         private_link=PrivateLink.enabled if enable_private_link is True else PrivateLink.disabled
     ) if (resource_provider_connection is not None or enable_private_link is not None) else None
+    if outbound_dependencies_managed_type:
+        network_properties.outbound_dependencies_managed_type = outbound_dependencies_managed_type
+    if public_ip_tag_type and public_ip_tag_value:
+        network_properties.public_ip_tag = IpTag(
+            ip_tag_type=public_ip_tag_type,
+            tag=public_ip_tag_value
+        )
 
     # compute isolation
     compute_isolation_properties = ComputeIsolationProperties(
@@ -409,10 +417,16 @@ def list_clusters(cmd, client, resource_group_name=None):  # pylint: disable=unu
     return list(clusters_list)
 
 
-def update_cluster(cmd, client, cluster_name, resource_group_name, tags=None, no_wait=False):
+def update_cluster(cmd, client, cluster_name, resource_group_name, tags=None, assign_identity=None, no_wait=False):
     from azure.mgmt.hdinsight.models import ClusterPatchParameters
+    from .util import build_identities_info
+    assign_identities = []
+    if assign_identity:
+        assign_identities.append(assign_identity)
+
     cluster_patch_parameters = ClusterPatchParameters(
-        tags=tags
+        tags=tags,
+        identity=build_identities_info(assign_identities)
     )
 
     return sdk_no_wait(no_wait, client.update, resource_group_name, cluster_name, cluster_patch_parameters)
@@ -612,6 +626,49 @@ def enable_hdi_azure_monitor(cmd, client, resource_group_name, cluster_name, wor
         cluster_name,
         azure_monitor_request_parameter)
 
+# pylint: disable=unused-argument
+
+
+def enable_hdi_azure_monitor_agent(cmd, client, resource_group_name, cluster_name, workspace, primary_key=None,
+                                   workspace_type='resource_id', no_validation_timeout=False):
+    from azure.mgmt.hdinsight.models import AzureMonitorRequest
+    from msrestazure.tools import parse_resource_id
+    from ._client_factory import cf_log_analytics
+
+    if workspace_type != 'resource_id' and not primary_key:
+        raise RequiredArgumentMissingError('primary key is required when workspace ID is provided.')
+
+    workspace_id = workspace
+    if workspace_type == 'resource_id':
+        parsed_workspace = parse_resource_id(workspace)
+        workspace_resource_group_name = parsed_workspace['resource_group']
+        workspace_name = parsed_workspace['resource_name']
+
+        log_analytics_client = cf_log_analytics(cmd.cli_ctx)
+        log_analytics_workspace = log_analytics_client.workspaces.get(workspace_resource_group_name, workspace_name)
+        if not log_analytics_workspace:
+            raise CLIError('Fails to retrieve workspace by {}'.format(workspace))
+
+        # Only retrieve primary key when not provided
+        if not primary_key:
+            shared_keys = log_analytics_client.shared_keys.get_shared_keys(workspace_resource_group_name,
+                                                                           workspace_name)
+            if not shared_keys:
+                raise CLIError('Fails to retrieve shared key for workspace {}'.format(log_analytics_workspace))
+
+            primary_key = shared_keys.primary_shared_key
+
+        workspace_id = log_analytics_workspace.customer_id
+
+    azure_monitor_agent_request_parameter = AzureMonitorRequest(
+        workspace_id=workspace_id,
+        primary_key=primary_key
+    )
+    return client.begin_enable_azure_monitor_agent(
+        resource_group_name,
+        cluster_name,
+        azure_monitor_agent_request_parameter)
+
 
 # pylint: disable=unused-argument
 def execute_hdi_script_action(cmd, client, resource_group_name, cluster_name,