From b99ad5c5e33c937dc2dde364de2276263eb4fbe9 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Thu, 24 Jun 2021 10:20:45 -0700 Subject: [PATCH 01/89] release prepare-2.4.0.0 (#2280) --- azurelinuxagent/common/cgroupconfigurator.py | 2 +- azurelinuxagent/common/version.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azurelinuxagent/common/cgroupconfigurator.py b/azurelinuxagent/common/cgroupconfigurator.py index d7ebd67f3..b10c9426c 100644 --- a/azurelinuxagent/common/cgroupconfigurator.py +++ b/azurelinuxagent/common/cgroupconfigurator.py @@ -69,7 +69,7 @@ [Service] CPUQuota={0} """ -_AGENT_CPU_QUOTA = 5 +_AGENT_CPU_QUOTA = 100 _AGENT_THROTTLED_TIME_THRESHOLD = 120 # 2 minutes diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 39fdf1ba6..70bd61773 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -200,9 +200,9 @@ def has_logrotate(): # IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise # DCR may test the wrong agent version. # -# When doing a release, be sure to use the actual agent version. Current agent version: 2.3.0.2 +# When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '9.9.9.9' +AGENT_VERSION = '2.4.0.0' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From b6a06246ba9f86c92fe4bce2c5045ca3a0b2166d Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Fri, 25 Jun 2021 14:28:47 -0700 Subject: [PATCH 02/89] Fix bug with dependent extensions with no settings (#2285) --- azurelinuxagent/ga/exthandlers.py | 10 ++++-- ..._conf_dependencies_with_empty_settings.xml | 32 +++++++++++++++++++ tests/ga/test_extension.py | 32 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 tests/data/wire/ext_conf_dependencies_with_empty_settings.xml diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 5a44d5d95..641cf3892 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -574,9 +574,13 @@ def wait_for_handler_completion(handler_i, wait_until, extension=None): Check the status of the extension being handled. Wait until it has a terminal state or times out. :raises: Exception if it is not handled successfully. """ - extension_name = handler_i.get_extension_full_name(extension) + # If the handler had no settings, we should not wait at all for handler to report status. + if extension is None: + logger.info("No settings found for {0}, not waiting for it's status".format(extension_name)) + return + try: ext_completed, status = False, None @@ -1656,7 +1660,9 @@ def get_status_file_path(self, extension=None): def collect_ext_status(self, ext): self.logger.verbose("Collect extension status for {0}".format(self.get_extension_full_name(ext))) seq_no, ext_status_file = self.get_status_file_path(ext) - if seq_no == -1: + + # We should never try to read any status file if the handler has no settings, returning None in that case + if seq_no == -1 or ext is None: return None data = None diff --git a/tests/data/wire/ext_conf_dependencies_with_empty_settings.xml b/tests/data/wire/ext_conf_dependencies_with_empty_settings.xml new file mode 100644 index 000000000..402de6438 --- /dev/null +++ b/tests/data/wire/ext_conf_dependencies_with_empty_settings.xml @@ -0,0 +1,32 @@ + + + + + Prod + + http://mock-goal-state/manifest_of_ga.xml + + + + Test + + http://mock-goal-state/manifest_of_ga.xml + + + + + + + + + + + + + + {"runtimeSettings":[{"handlerSettings":{"protectedSettingsCertThumbprint":"4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3","protectedSettings":"MIICWgYJK","publicSettings":{"foo":"bar"}}}]} + + + https://test.blob.core.windows.net/vhds/test-cs12.test-cs12.test-cs12.status?sr=b&sp=rw&se=9999-01-01&sk=key1&sv=2014-02-14&sig=hfRh7gzUE7sUtYwke78IOlZOrTRCYvkec4hGZ9zZzXo + diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index 4a399bf4e..4432d21ba 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -992,6 +992,38 @@ def test_ext_handler_sequencing(self, *args): (dep_ext_level_5, ExtensionCommandNames.UNINSTALL) ) + def test_it_should_process_sequencing_properly_even_if_no_settings_for_dependent_extension( + self, mock_get, mock_crypt, *args): + test_data_file = DATA_FILE.copy() + test_data_file["ext_conf"] = "wire/ext_conf_dependencies_with_empty_settings.xml" + test_data = mockwiredata.WireProtocolData(test_data_file) + exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) + + ext_1 = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux") + ext_2 = extension_emulator(name="OSTCExtensions.OtherExampleHandlerLinux") + + with enable_invocations(ext_1, ext_2) as invocation_record: + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + + # Ensure no extension status was reported for OtherExampleHandlerLinux as no settings provided for it + self._assert_handler_status(protocol.report_vm_status, "Ready", 0, "1.0.0", + expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") + + # Ensure correct status reported back for the other extension with settings + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="OSTCExtensions.ExampleHandlerLinux") + self._assert_ext_status(protocol.report_vm_status, "success", 0, + expected_handler_name="OSTCExtensions.ExampleHandlerLinux") + + # Ensure the invocation order follows the dependency levels + invocation_record.compare( + (ext_2, ExtensionCommandNames.INSTALL), + (ext_2, ExtensionCommandNames.ENABLE), + (ext_1, ExtensionCommandNames.INSTALL), + (ext_1, ExtensionCommandNames.ENABLE) + ) + def test_ext_handler_sequencing_should_fail_if_handler_failed(self, mock_get, mock_crypt, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SEQUENCING) exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) From 469a168f9c7bef56017aa2f2f9ee2547b74dfd92 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Fri, 9 Jul 2021 12:16:14 -0700 Subject: [PATCH 03/89] update test-requirements to pin pylint. (#2288) (#2299) Co-authored-by: Kevin Clark --- test-requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 85d5d4263..d4d62738e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,4 +6,6 @@ mock==4.0.2; python_version >= '3.6' distro; python_version >= '3.8' nose nose-timer; python_version >= '2.7' -pylint; python_version > '2.6' +pylint; python_version > '2.6' and python_version < '3.6' +pylint==2.8.3; python_version >= '3.6' + From a9f572fac8f430272ffc121466f193ce63249abd Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Fri, 9 Jul 2021 14:08:56 -0700 Subject: [PATCH 04/89] Do not create placeholder status file for AKS extensions (#2298) --- azurelinuxagent/ga/exthandlers.py | 14 ++++- tests/data/wire/ext_conf_aks_extension.xml | 69 ++++++++++++++++++++++ tests/ga/test_extension.py | 50 ++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 tests/data/wire/ext_conf_aks_extension.xml diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 641cf3892..8ff84824c 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -735,7 +735,8 @@ def handle_enable(self, ext_handler_i, extension): # failures back to CRP. If a placeholder for an extension already exists with Transitioning status, we would # not override it, hence we only create a placeholder for enable/disable commands but the extensions have the # data to create their own if needed. - ext_handler_i.create_placeholder_status_file(extension) + if ext_handler_i.should_create_default_placeholder(extension): + ext_handler_i.create_placeholder_status_file(extension) self.__handle_extension(ext_handler_i, extension, uninstall_exit_code) @staticmethod @@ -1398,6 +1399,17 @@ def initialize(self): # Save HandlerEnvironment.json self.create_handler_env() + def should_create_default_placeholder(self, extension=None): + """ + There's a bug in the AKS extension where they dont update the status file if it exists. + This violates the contract we have with extensions. Until they fix their extension, + we're going to skip creating a placeholder for them to ensure they dont have any downtime. + For all other extensions, we should create a + """ + + ignore_extension_regex = r"Microsoft.AKS.Compute.AKS\S*" + return re.match(ignore_extension_regex, self.get_extension_full_name(extension)) is None + def create_placeholder_status_file(self, extension=None, status=ValidHandlerStatus.transitioning, code=0, operation="Enabling Extension", message="Install/Enable is in progress."): _, status_path = self.get_status_file_path(extension) diff --git a/tests/data/wire/ext_conf_aks_extension.xml b/tests/data/wire/ext_conf_aks_extension.xml new file mode 100644 index 000000000..5901c0e44 --- /dev/null +++ b/tests/data/wire/ext_conf_aks_extension.xml @@ -0,0 +1,69 @@ + + + + + Prod + + http://mock-goal-state/manifest_of_ga.xml + + + + Test + + http://mock-goal-state/manifest_of_ga.xml + + + + + + + + + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling non-AKS"} + } + } + ] + } + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling AKSNode"} + } + } + ] + } + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling AKSBilling"} + } + } + ] + } + + + + + https://test.blob.core.windows.net/vhds/test-cs12.test-cs12.test-cs12.status?sr=b&sp=rw&se=9999-01-01&sk=key1&sv=2014-02-14&sig=hfRh7gzUE7sUtYwke78IOlZOrTRCYvkec4hGZ9zZzXo + + + diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index 4432d21ba..add41d61c 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -1664,6 +1664,56 @@ def mock_popen(cmd, *args, **kwargs): expected_msg="Dependent Extension OSTCExtensions.OtherExampleHandlerLinux did not reach a terminal state within the allowed timeout. Last status was {0}".format( ValidHandlerStatus.warning)) + def test_it_should_not_create_placeholder_for_aks_extension(self, mock_http_get, mock_crypt_util, *args): + original_popen = subprocess.Popen + + def mock_popen(cmd, *_, **kwargs): + if 'env' in kwargs: + if ExtensionCommandNames.ENABLE not in cmd: + # To force the test extension to not create a status file on Install, changing command + return original_popen(["echo", "not-enable"], *_, **kwargs) + + seq_no = kwargs['env'][ExtCommandEnvVariable.ExtensionSeqNumber] + ext_path = kwargs['env'][ExtCommandEnvVariable.ExtensionPath] + status_file_name = "{0}.status".format(seq_no) + status_file = os.path.join(ext_path, "status", status_file_name) + if "AKS" in cmd: + self.assertFalse(os.path.exists(status_file), "Placeholder file should not be created for AKS") + else: + self.assertTrue(os.path.exists(status_file), "Placeholder file should be created for all extensions") + + return original_popen(cmd, *_, **kwargs) + + aks_test_mock = DATA_FILE.copy() + aks_test_mock["ext_conf"] = "wire/ext_conf_aks_extension.xml" + + exthandlers_handler, protocol = self._create_mock(mockwiredata.WireProtocolData(aks_test_mock), + mock_http_get, mock_crypt_util, *args) + + with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="OSTCExtensions.ExampleHandlerLinux") + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="Microsoft.AKS.Compute.AKS.Linux.AKSNode") + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="Microsoft.AKS.Compute.AKS-Engine.Linux.Billing") + # Extension without settings + self._assert_handler_status(protocol.report_vm_status, "Ready", 0, "1.0.0", + expected_handler_name="Microsoft.AKS.Compute.AKS.Linux.Billing") + + self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.success, 0, + expected_handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_msg="Enabling non-AKS") + self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.success, 0, + expected_handler_name="Microsoft.AKS.Compute.AKS.Linux.AKSNode", + expected_msg="Enabling AKSNode") + self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.success, 0, + expected_handler_name="Microsoft.AKS.Compute.AKS-Engine.Linux.Billing", + expected_msg="Enabling AKSBilling") + def test_it_should_include_part_of_status_in_ext_handler_message(self, mock_http_get, mock_crypt_util, *args): """ Testing scenario when the status file is invalid, From 1344f619391a992a0b4c61a9edb201caa55b8537 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Thu, 15 Jul 2021 17:47:14 -0700 Subject: [PATCH 05/89] Exception for Linux Patch Extension for creating placeholder status file (#2307) --- azurelinuxagent/ga/exthandlers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 8ff84824c..dac348fa6 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -1401,14 +1401,16 @@ def initialize(self): def should_create_default_placeholder(self, extension=None): """ - There's a bug in the AKS extension where they dont update the status file if it exists. - This violates the contract we have with extensions. Until they fix their extension, + There's a bug in the AKS extension where they dont update the status file if it exists and another bug in + LinuxPatchExtension where they inherently have dependency on creating the status file first. + This violates the contract we have with extensions. Until they fix their extensions, we're going to skip creating a placeholder for them to ensure they dont have any downtime. For all other extensions, we should create a """ - ignore_extension_regex = r"Microsoft.AKS.Compute.AKS\S*" - return re.match(ignore_extension_regex, self.get_extension_full_name(extension)) is None + ignore_extensions_regex = [r"Microsoft.AKS.Compute.AKS\S*", r"Microsoft.CPlat.Core.LinuxPatchExtension\S*"] + return all(re.match(ext_regex, self.get_extension_full_name(extension)) is None + for ext_regex in ignore_extensions_regex) def create_placeholder_status_file(self, extension=None, status=ValidHandlerStatus.transitioning, code=0, operation="Enabling Extension", message="Install/Enable is in progress."): From 92afd7f618ee66bab2d2443a2992fc1177aebec8 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Fri, 16 Jul 2021 10:04:11 -0700 Subject: [PATCH 06/89] update release version (#2308) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 70bd61773..ded3e7608 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -202,7 +202,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '2.4.0.0' +AGENT_VERSION = '2.4.0.1' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 615d48a16a675d92ddffefdb6752ab0216337d05 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Tue, 3 Aug 2021 13:39:47 -0700 Subject: [PATCH 07/89] Dont create default status file for Single-Config extensions (#2318) --- azurelinuxagent/ga/exthandlers.py | 19 +++++-------------- tests/ga/test_extension.py | 13 +++++-------- tests/ga/test_multi_config_extension.py | 9 +++------ 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index dac348fa6..9ccf2669b 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -735,7 +735,11 @@ def handle_enable(self, ext_handler_i, extension): # failures back to CRP. If a placeholder for an extension already exists with Transitioning status, we would # not override it, hence we only create a placeholder for enable/disable commands but the extensions have the # data to create their own if needed. - if ext_handler_i.should_create_default_placeholder(extension): + + # Note: Due to a bug in multiple extensions, we're only creating a default placeholder for Multi-Config extensions. + # A fix will follow soon where we will report transitioning status for extensions by default if no status file + # found instead of reporting an error. + if ext_handler_i.should_perform_multi_config_op(extension): ext_handler_i.create_placeholder_status_file(extension) self.__handle_extension(ext_handler_i, extension, uninstall_exit_code) @@ -1399,19 +1403,6 @@ def initialize(self): # Save HandlerEnvironment.json self.create_handler_env() - def should_create_default_placeholder(self, extension=None): - """ - There's a bug in the AKS extension where they dont update the status file if it exists and another bug in - LinuxPatchExtension where they inherently have dependency on creating the status file first. - This violates the contract we have with extensions. Until they fix their extensions, - we're going to skip creating a placeholder for them to ensure they dont have any downtime. - For all other extensions, we should create a - """ - - ignore_extensions_regex = [r"Microsoft.AKS.Compute.AKS\S*", r"Microsoft.CPlat.Core.LinuxPatchExtension\S*"] - return all(re.match(ext_regex, self.get_extension_full_name(extension)) is None - for ext_regex in ignore_extensions_regex) - def create_placeholder_status_file(self, extension=None, status=ValidHandlerStatus.transitioning, code=0, operation="Enabling Extension", message="Install/Enable is in progress."): _, status_path = self.get_status_file_path(extension) diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index add41d61c..29546a1c7 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -450,7 +450,7 @@ def _assert_handler_status(self, report_vm_status, expected_status, self.assertNotEqual(0, len(vm_status.vmAgent.extensionHandlers)) handler_status = next( status for status in vm_status.vmAgent.extensionHandlers if status.name == expected_handler_name) - self.assertEqual(expected_status, handler_status.status) + self.assertEqual(expected_status, handler_status.status, get_properties(handler_status)) self.assertEqual(expected_handler_name, handler_status.name) self.assertEqual(version, handler_status.version) self.assertEqual(expected_ext_count, len([ext_handler for ext_handler in vm_status.vmAgent.extensionHandlers if @@ -1639,9 +1639,9 @@ def mock_popen(cmd, *args, **kwargs): if "sample.py" in cmd: status_path = os.path.join(kwargs['env'][ExtCommandEnvVariable.ExtensionPath], "status", "{0}.status".format(kwargs['env'][ExtCommandEnvVariable.ExtensionSeqNumber])) + mock_popen.deleted_status_file = status_path if os.path.exists(status_path): os.remove(status_path) - mock_popen.deleted_status_file = status_path return original_popen(["echo", "Yes"], *args, **kwargs) with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): @@ -1664,7 +1664,7 @@ def mock_popen(cmd, *args, **kwargs): expected_msg="Dependent Extension OSTCExtensions.OtherExampleHandlerLinux did not reach a terminal state within the allowed timeout. Last status was {0}".format( ValidHandlerStatus.warning)) - def test_it_should_not_create_placeholder_for_aks_extension(self, mock_http_get, mock_crypt_util, *args): + def test_it_should_not_create_placeholder_for_single_config_extensions(self, mock_http_get, mock_crypt_util, *args): original_popen = subprocess.Popen def mock_popen(cmd, *_, **kwargs): @@ -1677,10 +1677,7 @@ def mock_popen(cmd, *_, **kwargs): ext_path = kwargs['env'][ExtCommandEnvVariable.ExtensionPath] status_file_name = "{0}.status".format(seq_no) status_file = os.path.join(ext_path, "status", status_file_name) - if "AKS" in cmd: - self.assertFalse(os.path.exists(status_file), "Placeholder file should not be created for AKS") - else: - self.assertTrue(os.path.exists(status_file), "Placeholder file should be created for all extensions") + self.assertFalse(os.path.exists(status_file), "Placeholder file should not be created for single config extensions") return original_popen(cmd, *_, **kwargs) @@ -1732,7 +1729,7 @@ def mock_popen(cmd, *args, **kwargs): "{0}.status".format(kwargs['env'][ExtCommandEnvVariable.ExtensionSeqNumber])) invalid_json_path = os.path.join(data_dir, "ext", "sample-status-invalid-json-format.json") - if os.path.exists(status_path): + if 'enable' in cmd: invalid_json = fileutil.read_file(invalid_json_path) fileutil.write_file(status_path,invalid_json) diff --git a/tests/ga/test_multi_config_extension.py b/tests/ga/test_multi_config_extension.py index 00183bf7c..274fe4b11 100644 --- a/tests/ga/test_multi_config_extension.py +++ b/tests/ga/test_multi_config_extension.py @@ -894,7 +894,7 @@ def test_it_should_ignore_disable_errors_for_multi_config_extensions(self): fail_code in kwargs['message'] for args, kwargs in patch_report_event.call_args_list if kwargs['name'] == third_ext.name), "Error not reported") - def test_it_should_always_create_placeholder_for_all_extensions(self): + def test_it_should_always_create_placeholder_for_multi_config_extensions(self): original_popen = subprocess.Popen handler_statuses = {} @@ -903,9 +903,6 @@ def __assert_status_file_in_handlers(): file_name = "{0}.{1}.status".format(handler['runtimeSettingsStatus']['extensionName'], handler['runtimeSettingsStatus']['sequenceNumber']) __assert_status_file(handler['handlerName'], status_file=file_name) - for handler in sc_handler: - file_name = "{0}.status".format(handler['runtimeSettingsStatus']['sequenceNumber']) - __assert_status_file(handler['handlerName'], status_file=file_name) def __assert_status_file(handler_name, status_file): status = handler_statuses["{0}.{1}.enable".format(handler_name, status_file)] @@ -938,7 +935,7 @@ def mock_popen(cmd, *_, **kwargs): "ext_conf_multi_config_no_dependencies.xml") with self._setup_test_env(mock_manifest=True) as (exthandlers_handler, protocol, no_of_extensions): with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): - mc_handlers, sc_handler = self.__run_and_assert_generic_case(exthandlers_handler, protocol, + mc_handlers, _ = self.__run_and_assert_generic_case(exthandlers_handler, protocol, no_of_extensions) # Ensure we dont create a placeholder for Install command @@ -953,7 +950,7 @@ def mock_popen(cmd, *_, **kwargs): __assert_status_file_in_handlers() # Update GS, remove 2 extensions and add 3 - mc_handlers, sc_handler = self.__setup_and_assert_disable_scenario(exthandlers_handler, protocol) + mc_handlers, _ = self.__setup_and_assert_disable_scenario(exthandlers_handler, protocol) __assert_status_file_in_handlers() def test_it_should_report_status_correctly_for_unsupported_goal_state(self): From 11cd6697b213fef8d9c57cf06902e4d23e90c8c2 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Tue, 3 Aug 2021 14:06:11 -0700 Subject: [PATCH 08/89] version update (#2319) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index ded3e7608..b686b8662 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -202,7 +202,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '2.4.0.1' +AGENT_VERSION = '2.4.0.2' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 8bd66bf3d37154feda5b04fb29cbdb4d97a1c35a Mon Sep 17 00:00:00 2001 From: Dhivya Ganesan Date: Wed, 1 Sep 2021 16:09:18 -0700 Subject: [PATCH 09/89] Update Version (#2348) --- azurelinuxagent/common/cgroupconfigurator.py | 2 +- azurelinuxagent/common/version.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azurelinuxagent/common/cgroupconfigurator.py b/azurelinuxagent/common/cgroupconfigurator.py index 358bc6c52..cd12449b7 100644 --- a/azurelinuxagent/common/cgroupconfigurator.py +++ b/azurelinuxagent/common/cgroupconfigurator.py @@ -94,7 +94,7 @@ [Service] CPUQuota={0} """ -_AGENT_CPU_QUOTA = 5 +_AGENT_CPU_QUOTA = 100 _AGENT_THROTTLED_TIME_THRESHOLD = 120 # 2 minutes diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 8371b546c..6d11ef511 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -200,9 +200,9 @@ def has_logrotate(): # IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise # DCR may test the wrong agent version. # -# When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 +# When doing a release, be sure to use the actual agent version. Current agent version: 2.5.0.0 # -AGENT_VERSION = '9.9.9.9' +AGENT_VERSION = '2.5.0.0' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 0eeed5a7cbe597de4e7f3301e123693ca6670b92 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Wed, 1 Sep 2021 17:54:26 -0700 Subject: [PATCH 10/89] Fix bug with dependent extensions with no settings (#2285) (#2349) Co-authored-by: Laveesh Rohra --- azurelinuxagent/ga/exthandlers.py | 10 ++++-- ..._conf_dependencies_with_empty_settings.xml | 32 +++++++++++++++++++ tests/ga/test_extension.py | 32 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 tests/data/wire/ext_conf_dependencies_with_empty_settings.xml diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index eea295c81..29359800d 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -563,9 +563,13 @@ def wait_for_handler_completion(handler_i, wait_until, extension=None): Check the status of the extension being handled. Wait until it has a terminal state or times out. :raises: Exception if it is not handled successfully. """ - extension_name = handler_i.get_extension_full_name(extension) + # If the handler had no settings, we should not wait at all for handler to report status. + if extension is None: + logger.info("No settings found for {0}, not waiting for it's status".format(extension_name)) + return + try: ext_completed, status = False, None @@ -1652,7 +1656,9 @@ def get_status_file_path(self, extension=None): def collect_ext_status(self, ext): self.logger.verbose("Collect extension status for {0}".format(self.get_extension_full_name(ext))) seq_no, ext_status_file = self.get_status_file_path(ext) - if seq_no == -1: + + # We should never try to read any status file if the handler has no settings, returning None in that case + if seq_no == -1 or ext is None: return None data = None diff --git a/tests/data/wire/ext_conf_dependencies_with_empty_settings.xml b/tests/data/wire/ext_conf_dependencies_with_empty_settings.xml new file mode 100644 index 000000000..402de6438 --- /dev/null +++ b/tests/data/wire/ext_conf_dependencies_with_empty_settings.xml @@ -0,0 +1,32 @@ + + + + + Prod + + http://mock-goal-state/manifest_of_ga.xml + + + + Test + + http://mock-goal-state/manifest_of_ga.xml + + + + + + + + + + + + + + {"runtimeSettings":[{"handlerSettings":{"protectedSettingsCertThumbprint":"4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3","protectedSettings":"MIICWgYJK","publicSettings":{"foo":"bar"}}}]} + + + https://test.blob.core.windows.net/vhds/test-cs12.test-cs12.test-cs12.status?sr=b&sp=rw&se=9999-01-01&sk=key1&sv=2014-02-14&sig=hfRh7gzUE7sUtYwke78IOlZOrTRCYvkec4hGZ9zZzXo + diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index cf6988108..e5f110e35 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -994,6 +994,38 @@ def test_ext_handler_sequencing(self, *args): (dep_ext_level_5, ExtensionCommandNames.UNINSTALL) ) + def test_it_should_process_sequencing_properly_even_if_no_settings_for_dependent_extension( + self, mock_get, mock_crypt, *args): + test_data_file = DATA_FILE.copy() + test_data_file["ext_conf"] = "wire/ext_conf_dependencies_with_empty_settings.xml" + test_data = mockwiredata.WireProtocolData(test_data_file) + exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) + + ext_1 = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux") + ext_2 = extension_emulator(name="OSTCExtensions.OtherExampleHandlerLinux") + + with enable_invocations(ext_1, ext_2) as invocation_record: + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + + # Ensure no extension status was reported for OtherExampleHandlerLinux as no settings provided for it + self._assert_handler_status(protocol.report_vm_status, "Ready", 0, "1.0.0", + expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") + + # Ensure correct status reported back for the other extension with settings + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="OSTCExtensions.ExampleHandlerLinux") + self._assert_ext_status(protocol.report_vm_status, "success", 0, + expected_handler_name="OSTCExtensions.ExampleHandlerLinux") + + # Ensure the invocation order follows the dependency levels + invocation_record.compare( + (ext_2, ExtensionCommandNames.INSTALL), + (ext_2, ExtensionCommandNames.ENABLE), + (ext_1, ExtensionCommandNames.INSTALL), + (ext_1, ExtensionCommandNames.ENABLE) + ) + def test_ext_handler_sequencing_should_fail_if_handler_failed(self, mock_get, mock_crypt, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SEQUENCING) exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) From 81f7a1d73aa40aa5c5835841e3c1a64f767139a9 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 20 Sep 2021 10:34:39 -0700 Subject: [PATCH 11/89] =?UTF-8?q?Use=20handler=20status=20if=20extension?= =?UTF-8?q?=20status=20is=20None=20when=20computing=20the=20Ext=E2=80=A6?= =?UTF-8?q?=20(#2358)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use handler status if extension status is None when computing the ExtensionsSummary * Add NotReady to convergence statuses Co-authored-by: narrieta --- azurelinuxagent/ga/exthandlers.py | 20 ++++++++++++++++---- azurelinuxagent/ga/update.py | 7 ++++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 29359800d..ed5fe8d8a 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -87,7 +87,19 @@ # This is the default sequence number we use when there are no settings available for Handlers _DEFAULT_SEQ_NO = "0" + +class HandlerStatus(object): + """ + Statuses for Handlers + """ + ready = "Ready" + not_ready = "NotReady" + + class ValidHandlerStatus(object): + """ + Statuses for Extensions + """ transitioning = "transitioning" warning = "warning" error = "error" @@ -1427,7 +1439,7 @@ def enable(self, extension=None, uninstall_exit_code=None): raise # Even if a single extension is enabled for this handler, set the Handler state as Enabled self.set_handler_state(ExtHandlerState.Enabled) - self.set_handler_status(status="Ready", message="Plugin enabled") + self.set_handler_status(status=HandlerStatus.ready, message="Plugin enabled") def should_perform_multi_config_op(self, extension): return self.supports_multi_config and extension is not None @@ -1480,7 +1492,7 @@ def disable(self, extension=None, ignore_error=False): # For MultiConfig, Set the handler state to Installed only when all extensions have been disabled if not self.supports_multi_config or not any(self.enabled_extensions): self.set_handler_state(ExtHandlerState.Installed) - self.set_handler_status(status="NotReady", message="Plugin disabled") + self.set_handler_status(status=HandlerStatus.not_ready, message="Plugin disabled") def install(self, uninstall_exit_code=None, extension=None): # For Handler level operations, extension just specifies the settings that initiated the install. @@ -1496,7 +1508,7 @@ def install(self, uninstall_exit_code=None, extension=None): self.launch_command(install_cmd, cmd_name="install", timeout=900, extension=extension, extension_error_code=ExtensionErrorCodes.PluginInstallProcessingFailed, env=env) self.set_handler_state(ExtHandlerState.Installed) - self.set_handler_status(status="NotReady", message="Plugin installed but not enabled") + self.set_handler_status(status=HandlerStatus.not_ready, message="Plugin installed but not enabled") def uninstall(self, extension=None): # For Handler level operations, extension just specifies the settings that initiated the uninstall. @@ -2075,7 +2087,7 @@ def __remove_extension_state_files(self, extension): self.report_event(name=extension_name, message=message, is_success=False, log_event=False) self.logger.warn(message) - def set_handler_status(self, status="NotReady", message="", code=0): + def set_handler_status(self, status=HandlerStatus.not_ready, message="", code=0): state_dir = self.get_conf_dir() handler_status = ExtHandlerStatus() diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 6d54d067e..6234a4bc5 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -56,7 +56,7 @@ from azurelinuxagent.ga.env import get_env_handler from azurelinuxagent.ga.collect_telemetry_events import get_collect_telemetry_events_handler -from azurelinuxagent.ga.exthandlers import HandlerManifest, ExtHandlersHandler, list_agent_lib_directory, ValidHandlerStatus +from azurelinuxagent.ga.exthandlers import HandlerManifest, ExtHandlersHandler, list_agent_lib_directory, ValidHandlerStatus, HandlerStatus from azurelinuxagent.ga.monitor import get_monitor_handler from azurelinuxagent.ga.send_telemetry_events import get_send_telemetry_events_handler @@ -105,9 +105,10 @@ def __init__(self, vm_status=None): self.summary = [] self.converged = True else: - self.summary = [(h.extension_status.name, h.extension_status.status) for h in vm_status.vmAgent.extensionHandlers] + # take the name and status of the extension if is it not None, else use the handler's + self.summary = [(o.name, o.status) for o in map(lambda h: h.extension_status if h.extension_status is not None else h, vm_status.vmAgent.extensionHandlers)] self.summary.sort(key=lambda s: s[0]) # sort by extension name to make comparisons easier - self.converged = all(status in (ValidHandlerStatus.success, ValidHandlerStatus.error) for _, status in self.summary) + self.converged = all(status in (ValidHandlerStatus.success, ValidHandlerStatus.error, HandlerStatus.ready, HandlerStatus.not_ready) for _, status in self.summary) def __eq__(self, other): return self.summary == other.summary From 53629f95fd1cf5e7cc46b9703bb2eb30f637e44d Mon Sep 17 00:00:00 2001 From: Dhivya Ganesan Date: Mon, 20 Sep 2021 13:26:30 -0700 Subject: [PATCH 12/89] Release preparation 2.5.0.1 (#2360) --- azurelinuxagent/common/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 6d11ef511..1eb1f5d60 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -200,9 +200,9 @@ def has_logrotate(): # IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise # DCR may test the wrong agent version. # -# When doing a release, be sure to use the actual agent version. Current agent version: 2.5.0.0 +# When doing a release, be sure to use the actual agent version. Current agent version: 2.5.0.1 # -AGENT_VERSION = '2.5.0.0' +AGENT_VERSION = '2.5.0.1' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From a80514218d8ccf6d9f47fb5a3fdf16faecfacfbb Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Thu, 30 Sep 2021 16:33:34 -0700 Subject: [PATCH 13/89] Define ExtensionsSummary.__eq__ (#2371) * define ExtensionsSummary.__eq__ * Fix test name * Fix test name * Fix test name Co-authored-by: narrieta --- azurelinuxagent/ga/update.py | 3 +++ tests/ga/test_update.py | 50 +++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 6234a4bc5..3dd3fc667 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -113,6 +113,9 @@ def __init__(self, vm_status=None): def __eq__(self, other): return self.summary == other.summary + def __ne__(self, other): + return not (self.summary == other.summary) + def __str__(self): return ustr(self.summary) diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index 71ad334d4..3a99a859f 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -43,7 +43,7 @@ from azurelinuxagent.ga.exthandlers import ExtHandlersHandler, ExtHandlerInstance, HandlerEnvironment, ValidHandlerStatus from azurelinuxagent.ga.update import GuestAgent, GuestAgentError, MAX_FAILURE, AGENT_MANIFEST_FILE, \ get_update_handler, ORPHAN_POLL_INTERVAL, AGENT_PARTITION_FILE, AGENT_ERROR_FILE, ORPHAN_WAIT_INTERVAL, \ - CHILD_LAUNCH_RESTART_MAX, CHILD_HEALTH_INTERVAL, GOAL_STATE_PERIOD_EXTENSIONS_DISABLED, UpdateHandler + CHILD_LAUNCH_RESTART_MAX, CHILD_HEALTH_INTERVAL, GOAL_STATE_PERIOD_EXTENSIONS_DISABLED, UpdateHandler, ExtensionsSummary from tests.protocol.mocks import mock_wire_protocol from tests.protocol.mockwiredata import DATA_FILE, DATA_FILE_MULTIPLE_EXT from tests.tools import AgentTestCase, data_dir, DEFAULT, patch, load_bin_data, load_data, Mock, MagicMock, \ @@ -2173,5 +2173,53 @@ def test_update_handler_should_switch_to_the_regular_goal_state_period_when_the_ self.assertEqual(goal_state_period, update_handler._goal_state_period, "Expected the regular goal state period when the goal state does not converge") +class ExtensionsSummaryTestCase(AgentTestCase): + @staticmethod + def _create_extensions_summary(extension_statuses): + """ + Creates an ExtensionsSummary from an array of (extension name, extension status) tuples + """ + vm_status = VMStatus(status="Ready", message="Ready") + vm_status.vmAgent.extensionHandlers = [ExtHandlerStatus()] * len(extension_statuses) + for i in range(len(extension_statuses)): + vm_status.vmAgent.extensionHandlers[i].extension_status = ExtensionStatus(name=extension_statuses[i][0]) + vm_status.vmAgent.extensionHandlers[0].extension_status.status = extension_statuses[i][1] + return ExtensionsSummary(vm_status) + + def test_equality_operator_should_return_true_on_items_with_the_same_value(self): + summary1 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success), ("Extension 2", ValidHandlerStatus.transitioning)]) + summary2 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success), ("Extension 2", ValidHandlerStatus.transitioning)]) + + self.assertTrue(summary1 == summary2, "{0} == {1} should be True".format(summary1, summary2)) + + def test_equality_operator_should_return_false_on_items_with_different_values(self): + summary1 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success), ("Extension 2", ValidHandlerStatus.transitioning)]) + summary2 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success), ("Extension 2", ValidHandlerStatus.success)]) + + self.assertFalse(summary1 == summary2, "{0} == {1} should be False") + + summary1 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success)]) + summary2 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success), ("Extension 2", ValidHandlerStatus.success)]) + + self.assertFalse(summary1 == summary2, "{0} == {1} should be False") + + def test_inequality_operator_should_return_true_on_items_with_different_values(self): + summary1 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success), ("Extension 2", ValidHandlerStatus.transitioning)]) + summary2 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success), ("Extension 2", ValidHandlerStatus.success)]) + + self.assertTrue(summary1 != summary2, "{0} != {1} should be True".format(summary1, summary2)) + + summary1 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success)]) + summary2 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success), ("Extension 2", ValidHandlerStatus.success)]) + + self.assertTrue(summary1 != summary2, "{0} != {1} should be True") + + def test_inequality_operator_should_return_false_on_items_with_same_value(self): + summary1 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success), ("Extension 2", ValidHandlerStatus.transitioning)]) + summary2 = ExtensionsSummaryTestCase._create_extensions_summary([("Extension 1", ValidHandlerStatus.success), ("Extension 2", ValidHandlerStatus.transitioning)]) + + self.assertFalse(summary1 != summary2, "{0} != {1} should be False".format(summary1, summary2)) + + if __name__ == '__main__': unittest.main() From 4488a6216ad5ceb38b7718f16cca3efa9dcbb884 Mon Sep 17 00:00:00 2001 From: Dhivya Ganesan Date: Thu, 30 Sep 2021 16:48:35 -0700 Subject: [PATCH 14/89] Release preparation 2.5.0.2 (#2372) --- azurelinuxagent/common/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 1eb1f5d60..fa10683ee 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -200,9 +200,9 @@ def has_logrotate(): # IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise # DCR may test the wrong agent version. # -# When doing a release, be sure to use the actual agent version. Current agent version: 2.5.0.1 +# When doing a release, be sure to use the actual agent version. Current agent version: 2.5.0.2 # -AGENT_VERSION = '2.5.0.1' +AGENT_VERSION = '2.5.0.2' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 19f27203c6cbc5f28a84f8048da3cf02c71b19d5 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Thu, 28 Oct 2021 22:04:36 +0530 Subject: [PATCH 15/89] update cgroups monitoring expiry date (#2392) --- azurelinuxagent/common/conf.py | 4 ++-- tests/test_agent.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azurelinuxagent/common/conf.py b/azurelinuxagent/common/conf.py index 882ad5c39..9c509a8b6 100644 --- a/azurelinuxagent/common/conf.py +++ b/azurelinuxagent/common/conf.py @@ -159,7 +159,7 @@ def load_conf_from_file(conf_file_path, conf=__conf__): "ResourceDisk.MountOptions": None, "ResourceDisk.Filesystem": "ext3", "AutoUpdate.GAFamily": "Prod", - "Debug.CgroupMonitorExpiryTime": "2021-11-30", + "Debug.CgroupMonitorExpiryTime": "2022-01-31", "Debug.CgroupMonitorExtensionName": "Microsoft.Azure.Monitor.AzureMonitorLinuxAgent", } @@ -551,7 +551,7 @@ def get_cgroup_monitor_expiry_time (conf=__conf__): NOTE: This option is experimental and may be removed in later versions of the Agent. """ - return conf.get("Debug.CgroupMonitorExpiryTime", "2021-11-30") + return conf.get("Debug.CgroupMonitorExpiryTime", "2022-01-31") def get_cgroup_monitor_extension_name (conf=__conf__): """ diff --git a/tests/test_agent.py b/tests/test_agent.py index 9f274183c..f6fc5e3a1 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -34,7 +34,7 @@ Debug.CgroupDisableOnProcessCheckFailure = True Debug.CgroupDisableOnQuotaCheckFailure = True Debug.CgroupLogMetrics = False -Debug.CgroupMonitorExpiryTime = 2021-11-30 +Debug.CgroupMonitorExpiryTime = 2022-01-31 Debug.CgroupMonitorExtensionName = Microsoft.Azure.Monitor.AzureMonitorLinuxAgent Debug.EnableFastTrack = True Debug.EtpCollectionPeriod = 300 From 6118ebaa97cfbadcea8c8593c747879d1c7c8ac2 Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Thu, 28 Oct 2021 09:52:21 -0700 Subject: [PATCH 16/89] Updated Agent Version to 2.6.0.0 (#2393) --- azurelinuxagent/common/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 8371b546c..86afb2956 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -200,9 +200,9 @@ def has_logrotate(): # IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise # DCR may test the wrong agent version. # -# When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 +# When doing a release, be sure to use the actual agent version. Current agent version: 2.6.0.0 # -AGENT_VERSION = '9.9.9.9' +AGENT_VERSION = '2.6.0.0' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 892b5d52294b57b15ae2657e091fdfd4f6e0579a Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 29 Oct 2021 10:43:02 -0700 Subject: [PATCH 17/89] Do not parse status blob (#2394) Co-authored-by: narrieta --- .../common/protocol/extensions_goal_state.py | 14 +++++++++----- tests/protocol/test_extensions_goal_state.py | 5 +++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/azurelinuxagent/common/protocol/extensions_goal_state.py b/azurelinuxagent/common/protocol/extensions_goal_state.py index 39994cc95..6f6200d9e 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state.py @@ -105,10 +105,12 @@ def raise_error(attribute): ) raise GoalStateMismatchError(message) - if from_extensions_config.get_status_upload_blob() != from_vm_settings.get_status_upload_blob(): - raise_error("_status_upload_blob") - if from_extensions_config.get_status_upload_blob_type() != from_vm_settings.get_status_upload_blob_type(): - raise_error("_status_upload_blob_type") + # TODO: HostGAPlugin 112 does not include the status blob; enable these checks once we can ensure a newer version is available. Also enable parsing + # in the _ExtensionsGoalStateFromVmSettings.compare() method + # if from_extensions_config.get_status_upload_blob() != from_vm_settings.get_status_upload_blob(): + # raise_error("_status_upload_blob") + # if from_extensions_config.get_status_upload_blob_type() != from_vm_settings.get_status_upload_blob_type(): + # raise_error("_status_upload_blob_type") if from_extensions_config.get_required_features() != from_vm_settings.get_required_features(): raise_error("_required_features") @@ -546,7 +548,9 @@ def __init__(self, etag, json_text): def _parse_vm_settings(self, json_text): vm_settings = _CaseFoldedDict.from_dict(json.loads(json_text)) - self._parse_status_upload_blob(vm_settings) + # TODO: HostGAPlugin 112 does not include the status blob; enable parsing once we can ensure a newer version is available. Also enable the check on + # the ExtensionsGoalState.compare() method + # self._parse_status_upload_blob(vm_settings) self._parse_required_features(vm_settings) # TODO: Parse all atttributes diff --git a/tests/protocol/test_extensions_goal_state.py b/tests/protocol/test_extensions_goal_state.py index 4791d8283..59c6d9bfd 100644 --- a/tests/protocol/test_extensions_goal_state.py +++ b/tests/protocol/test_extensions_goal_state.py @@ -43,8 +43,9 @@ def test_create_from_vm_settings_should_parse_vm_settiings(self): vm_settings_text = fileutil.read_file(os.path.join(data_dir, "hostgaplugin/vm_settings.json")) vm_settings = ExtensionsGoalState.create_from_vm_settings("123", vm_settings_text) - self.assertEqual("https://dcrcqabsr1.blob.core.windows.net/$system/edpxmal5j1.058b176d-445b-4e75-bd97-4911511b7d96.status?sv=2018-03-28&sr=b&sk=system-1&sig=U4KaLxlyYfgQ%2fie8RCwgMBSXa3E4vlW0ozPYOEHikoc%3d&se=9999-01-01T00%3a00%3a00Z&sp=w", vm_settings.get_status_upload_blob(), 'statusUploadBlob.value was not parsed correctly') - self.assertEqual("BlockBlob", vm_settings.get_status_upload_blob_type(), 'statusBlobType was not parsed correctly') + # TODO: HostGAPlugin 112 does not include the status blob; see TODO in ExtensionsGoalState + # self.assertEqual("https://dcrcqabsr1.blob.core.windows.net/$system/edpxmal5j1.058b176d-445b-4e75-bd97-4911511b7d96.status?sv=2018-03-28&sr=b&sk=system-1&sig=U4KaLxlyYfgQ%2fie8RCwgMBSXa3E4vlW0ozPYOEHikoc%3d&se=9999-01-01T00%3a00%3a00Z&sp=w", vm_settings.get_status_upload_blob(), 'statusUploadBlob.value was not parsed correctly') + # self.assertEqual("BlockBlob", vm_settings.get_status_upload_blob_type(), 'statusBlobType was not parsed correctly') self.assertEqual(["MultipleExtensionsPerHandler"], vm_settings.get_required_features(), 'requiredFeatures was not parsed correctly') From 0d13b058d1925d783de3c6bcd54509fbe7f65ceb Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Fri, 29 Oct 2021 19:46:32 +0000 Subject: [PATCH 18/89] Exclude dcr from setup (#2396) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5c66a44c5..d38d74d6e 100755 --- a/setup.py +++ b/setup.py @@ -300,7 +300,7 @@ def run(self): platforms='Linux', url='https://github.com/Azure/WALinuxAgent', license='Apache License Version 2.0', - packages=find_packages(exclude=["tests*"]), + packages=find_packages(exclude=["tests*", "dcr*"]), py_modules=modules, install_requires=requires, cmdclass={ From ec6911997ea9dc212e33265358522081f61258d9 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 29 Oct 2021 13:29:05 -0700 Subject: [PATCH 19/89] Improve error message for vmSettings errors (#2397) * Improve error message for vmSettings errors * Add etag and correlation id * pylint warning Co-authored-by: narrieta --- azurelinuxagent/common/protocol/wire.py | 7 +++++-- tests/protocol/test_wire.py | 18 +++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 5cbef106a..ab073e942 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -891,6 +891,8 @@ def get_vm_settings(): except ProtocolError: raise + except HttpError as http_error: + raise Exception("{0} [correlation ID: {1} eTag: {2}]".format(ustr(http_error), correlation_id, etag)) except Exception as exception: error = textutil.format_exception(exception) # since this is a generic error, we format the exception to include the traceback raise Exception("Error fetching vmSettings [correlation ID: {0} eTag: {1}]: {2}".format(correlation_id, etag, error)) @@ -1508,11 +1510,12 @@ def __init__(self, operation): self._operation = operation self._error_count = 0 self._next_period = datetime.now() + _ErrorReporter._Period + self._log_message_prefix = "[{0}] [Informational only, the Agent will continue normal operation]".format(self._operation) def report_error(self, error): self._error_count += 1 if self._error_count <= _ErrorReporter._MaxErrors: - logger.info("[{0}] {1}", self._operation, error) + logger.info("{0} {1}", self._log_message_prefix, error) add_event(op=self._operation, message=error, is_success=False, log_event=False) def report_summary(self): @@ -1520,6 +1523,6 @@ def report_summary(self): self._next_period = datetime.now() + _ErrorReporter._Period if self._error_count > 0: message = "{0} errors in the last period".format(self._error_count) - logger.info("[{0}] {1}", self._operation, message) + logger.info("{0} {1}", self._log_message_prefix, message) add_event(op=self._operation, message=message, is_success=False, log_event=False) self._error_count = 0 diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py index 6055a2f5d..200421402 100644 --- a/tests/protocol/test_wire.py +++ b/tests/protocol/test_wire.py @@ -1272,21 +1272,21 @@ def assert_no_exception(test_case, test_function, expected_error): def test_error_in_http_request(test_case, mock_response, expected_error): def do_mock_request(): with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: - def http_get_handler(url, *_, **__): - if self.is_host_plugin_vm_settings_request(url): - if isinstance(mock_response, Exception): - raise mock_response - return mock_response - return None - protocol.set_http_handlers(http_get_handler=http_get_handler) + protocol.do_not_mock = lambda method, url: method == "GET" and self.is_host_plugin_vm_settings_request(url) - protocol.client.update_goal_state() + def http_get_vm_settings(_method, _host, _relative_url, **_): + if isinstance(mock_response, Exception): + raise mock_response + return mock_response + + with patch("azurelinuxagent.common.utils.restutil._http_request", side_effect=http_get_vm_settings): + protocol.client.update_goal_state() assert_no_exception(test_case, do_mock_request, expected_error) # # We test errors different kind of errors; none of them should make update_protocol raise an exception, but all of them should be reported # - test_error_in_http_request("Internal error in the HostGAPlugin", MockHttpResponse(httpclient.BAD_GATEWAY), "[Internal error in HostGAPlugin]") # HostGAPlugin uses 502 for internal errors + test_error_in_http_request("Internal error in the HostGAPlugin", MockHttpResponse(httpclient.BAD_GATEWAY), "Status Code 502") # HostGAPlugin uses 502 for internal errors test_error_in_http_request("Arbitrary error in the request (BAD_REQUEST)", MockHttpResponse(httpclient.BAD_REQUEST), "[HTTP Failed] [400: None]") test_error_in_http_request("ProtocolError during the request", ProtocolError("GENERIC PROTOCOL ERROR"), "GENERIC PROTOCOL ERROR") test_error_in_http_request("Generic error in the request", Exception("GENERIC REQUEST ERROR"), "GENERIC REQUEST ERROR") From 50f35d2d7992fca75d6f8f77b12ec12f48c7d527 Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Fri, 29 Oct 2021 15:18:23 -0700 Subject: [PATCH 20/89] Updated agent version to 2.6.0.1 (#2400) --- azurelinuxagent/common/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 86afb2956..6a32907e9 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -200,9 +200,9 @@ def has_logrotate(): # IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise # DCR may test the wrong agent version. # -# When doing a release, be sure to use the actual agent version. Current agent version: 2.6.0.0 +# When doing a release, be sure to use the actual agent version. Current agent version: 2.6.0.1 # -AGENT_VERSION = '2.6.0.0' +AGENT_VERSION = '2.6.0.1' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From b224fa502a4df89b390c8e393fb2a8fcfe65d21f Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 1 Nov 2021 15:00:11 -0700 Subject: [PATCH 21/89] Verify that extensions goal state from vmsettings has been initialized (#2404) Co-authored-by: narrieta --- azurelinuxagent/common/protocol/wire.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index ab073e942..c6211ffa1 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -823,7 +823,7 @@ def update_goal_state(self, force_update=False): self._extensions_goal_state_from_vm_settings = ExtensionsGoalState.create_from_vm_settings(response_etag, vm_settings) updated = True # If either goal state was updated, compare them - if updated: + if updated and self._extensions_goal_state_from_vm_settings is not None: ExtensionsGoalState.compare(self._extensions_goal_state, self._extensions_goal_state_from_vm_settings) except Exception as error: # TODO: Once FastTrack is stable, these exceptions should be rare. Add a traceback to the error message at that point. From 3c4896530a2df8cf853453e2f0e91c566beeaa6f Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Mon, 1 Nov 2021 15:43:56 -0700 Subject: [PATCH 22/89] Updated agent version to 2.6.0.2 (#2405) --- azurelinuxagent/common/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 6a32907e9..54b4b2406 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -200,9 +200,9 @@ def has_logrotate(): # IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise # DCR may test the wrong agent version. # -# When doing a release, be sure to use the actual agent version. Current agent version: 2.6.0.1 +# When doing a release, be sure to use the actual agent version. Current agent version: 2.6.0.2 # -AGENT_VERSION = '2.6.0.1' +AGENT_VERSION = '2.6.0.2' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 4cff655288c4a02f50577d73eb4b3230073863f8 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 4 Jan 2022 13:17:37 -0800 Subject: [PATCH 23/89] Set Agent version to 2.7.0.0 (#2457) * Set Agent version to 2.7.0.0 * Remove duplicate import Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 4 ++-- tests/ga/test_update.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index e35e28b94..7055bce34 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -207,9 +207,9 @@ def has_logrotate(): # IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise # DCR may test the wrong agent version. # -# When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 +# When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '9.9.9.9' +AGENT_VERSION = '2.7.0.0' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index 25fd59935..6213957b4 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -1344,6 +1344,7 @@ def test_run_latest_creates_only_one_signal_handler(self, mock_signal): self._test_run_latest() self.assertEqual(0, mock_signal.call_count) + @skip_if_predicate_true(lambda: True, "This test has a dependency on the agent version being 9.9.* and breaks when updating the agent version during release") def test_get_latest_agent_should_return_latest_agent_even_on_bad_error_json(self): self.prepare_agents() # Add a malformed error.json file in all existing agents From b597e3576b3ed630959798ec5ac53b227c436304 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 11 Jan 2022 12:33:31 -0800 Subject: [PATCH 24/89] Simplify the logic to update the extensions goal state (#2466) * Simplify the logic to update the extensions goal state * Added telemetry for NotSupported * Added comments * Do not support old hostgaplugin Co-authored-by: narrieta --- azurelinuxagent/common/protocol/wire.py | 116 ++++++++++--------- tests/protocol/test_extensions_goal_state.py | 8 +- 2 files changed, 68 insertions(+), 56 deletions(-) diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 401f9abd0..6f35abd06 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -577,7 +577,7 @@ def __init__(self, endpoint): self._endpoint = endpoint self._goal_state = None self._extensions_goal_state = None # The goal state to use for extensions; can be an ExtensionsGoalStateFromVmSettings or ExtensionsGoalStateFromExtensionsConfig - self._vm_settings_goal_state = None # Cached value of the most recent ExtensionsGoalStateFromVmSettings + self._cached_vm_settings = None # Cached value of the most recent ExtensionsGoalStateFromVmSettings self._host_plugin = None self._host_plugin_version = FlexibleVersion("0.0.0.0") # Version 0 means "unknown" self._host_plugin_supports_vm_settings = False @@ -785,6 +785,13 @@ def update_goal_state(self, force_update=False): Updates the goal state if the incarnation or etag changed or if 'force_update' is True """ try: + # + # The entire goal state needs to be retrieved from the WireServer (via the GoalState class), and the HostGAPlugin + # (via the self._fetch_vm_settings_goal_state method.) + # + # Start by fetching the goal state from the WireServer; if fetch_full_goal_state becomes True this will also + # update the ExtensionsConfig instance in GoalState. + # goal_state = GoalState(self) # Always update the hostgaplugin, since the agent may issue requests to it even if there are no other changes @@ -806,42 +813,35 @@ def update_goal_state(self, force_update=False): self._goal_state = goal_state goal_state_updated = True - vm_settings_goal_state_updated = False - if not conf.get_enable_fast_track(): - # if Fast Track is not enabled use extensionsConfig - self._extensions_goal_state = self._goal_state.extensions_config + # + # Now fetch the vmSettings (which contain the extensions goal state) from the HostGAPlugin + # + vm_settings_goal_state, vm_settings_goal_state_updated = (None, False) + vm_settings_goal_state_is_valid = False + + if conf.get_enable_fast_track(): + try: + vm_settings_goal_state, vm_settings_goal_state_updated = self._fetch_vm_settings_goal_state(force_update=force_update) + + if goal_state_updated or vm_settings_goal_state_updated: + # compare() raises a GoalStateMismatchError if the goal states don't match + ExtensionsGoalState.compare(self._goal_state.extensions_config, vm_settings_goal_state) + + vm_settings_goal_state_is_valid = True + + except Exception as error: + # _fetch_vm_settings_goal_state() does its own detailed error reporting and raises ProtocolError; do not report those + if not isinstance(error, ProtocolError): + self._vm_settings_error_reporter.report_error(format_exception(error)) + self._vm_settings_error_reporter.report_summary() + + # + # If we fetched a valid vmSettings, use that for the extensions goal state. Otherwise use ExtensionsConfig + # + if vm_settings_goal_state_is_valid: + self._extensions_goal_state = vm_settings_goal_state else: - if not self._host_plugin_supports_vm_settings and self._host_plugin_supports_vm_settings_next_check > datetime.now(): - # if vmSettings are not supported use extensionsConfig - self._extensions_goal_state = self._goal_state.extensions_config - else: - try: - vm_settings_goal_state = self._fetch_vm_settings_goal_state(force_update=force_update) - - self._host_plugin_supports_vm_settings = True - - if self._vm_settings_goal_state is None or self._vm_settings_goal_state.etag != vm_settings_goal_state.etag: - self._vm_settings_goal_state = vm_settings_goal_state - self._extensions_goal_state = vm_settings_goal_state - vm_settings_goal_state_updated = True - - if goal_state_updated or vm_settings_goal_state_updated: - # compare() raises a GoalStateMismatchError if the goal states don't match - ExtensionsGoalState.compare(self._goal_state.extensions_config, self._vm_settings_goal_state) - - except _VmSettingsNotSupportedError: - # if vmSettings are not supported use extensionsConfig - self._extensions_goal_state = self._goal_state.extensions_config - # mark vmSettings as not supported for the next 6 hours - self._host_plugin_supports_vm_settings = False - self._host_plugin_supports_vm_settings_next_check = datetime.now() + timedelta(hours=6) - except Exception as error: - # if there is any errors on vmSettings then use extensionsConfig - self._extensions_goal_state = self._goal_state.extensions_config - # _fetch_vm_settings_goal_state() does its own detailed error reporting and raises ProtocolError; do not report those - if not isinstance(error, ProtocolError): - self._vm_settings_error_reporter.report_error(format_exception(error)) - self._vm_settings_error_reporter.report_summary() + self._extensions_goal_state = self._goal_state.extensions_config # If either goal state changed (goal_state or vm_settings_goal_state) save them if goal_state_updated or vm_settings_goal_state_updated: @@ -854,10 +854,25 @@ def update_goal_state(self, force_update=False): def _fetch_vm_settings_goal_state(self, force_update): """ - Queries the vmSettings from the HostGAPlugin and returns an instance of ExtensionsGoalStateFromVmSettings. - Raises _VmSettingsNotSupportedError if the HostGAPlugin does not support FastTrack or ProtocolError on other kinds of errors. + Queries the vmSettings from the HostGAPlugin and returns an (ExtensionsGoalStateFromVmSettings, bool) tuple with the vmSettings and + a boolean indicating if they are an updated (True) or a cached value (False). + + Raises ProtocolError if the request fails for any reason (e.g. not supported, time out, server error) """ - etag = None if force_update or self._vm_settings_goal_state is None else self._vm_settings_goal_state.etag + def raise_not_supported(reset_state=False): + if reset_state: + self._host_plugin_supports_vm_settings = False + self._host_plugin_supports_vm_settings_next_check = datetime.now() + timedelta(hours=6) # check again in 6 hours + # "Not supported" is not considered an error, so don't use self._vm_settings_error_reporter to report it + logger.info("vmSettings is not supported") + add_event(op=WALAEventOperation.HostPlugin, message="vmSettings is not supported", is_success=True) + raise ProtocolError("VmSettings not supported") + + # Raise if VmSettings are not supported but check for periodically since the HostGAPlugin could have been updated since the last check + if not self._host_plugin_supports_vm_settings and self._host_plugin_supports_vm_settings_next_check > datetime.now(): + raise_not_supported() + + etag = None if force_update or self._cached_vm_settings is None else self._cached_vm_settings.etag correlation_id = str(uuid.uuid4()) def format_message(msg): @@ -879,10 +894,10 @@ def get_vm_settings(): response = get_vm_settings() if response.status == httpclient.NOT_FOUND: # the HostGAPlugin does not support FastTrack - raise _VmSettingsNotSupportedError() + raise_not_supported(reset_state=True) if response.status == httpclient.NOT_MODIFIED: # The goal state hasn't changed, return the current instance - return self._vm_settings_goal_state + return self._cached_vm_settings, False if response.status != httpclient.OK: error_description = restutil.read_response_error(response) @@ -923,11 +938,12 @@ def get_vm_settings(): # Don't support HostGAPlugin versions older than 115 if vm_settings.host_ga_plugin_version < FlexibleVersion("1.0.8.115"): - raise _VmSettingsNotSupportedError() + raise_not_supported(reset_state=True) logger.info("Fetched new vmSettings [correlation ID: {0} New eTag: {1}]", correlation_id, vm_settings.etag) - - return vm_settings + self._host_plugin_supports_vm_settings = True + self._cached_vm_settings = vm_settings + return vm_settings, True except ProtocolError: raise @@ -966,11 +982,11 @@ def save_if_not_none(goal_state_property, file_name): text = self._goal_state.extensions_config.get_redacted_text() if text != '': self._save_cache(text, EXT_CONF_FILE_NAME.format(self._goal_state.extensions_config.incarnation)) - # TODO: When Fast Track is fully enabled self._vm_settings_goal_state will go away and this can be deleted - if self._vm_settings_goal_state is not None: - text = self._vm_settings_goal_state.get_redacted_text() + # TODO: When Fast Track is fully enabled self._cached_vm_settings will go away and this can be deleted + if self._cached_vm_settings is not None: + text = self._cached_vm_settings.get_redacted_text() if text != '': - self._save_cache(text, VM_SETTINGS_FILE_NAME.format(self._vm_settings_goal_state.id)) + self._save_cache(text, VM_SETTINGS_FILE_NAME.format(self._cached_vm_settings.id)) # END TODO except Exception as e: @@ -1554,7 +1570,3 @@ def report_summary(self): logger.info("[VmSettingsSummary] {0}", message) self._reset() - - -class _VmSettingsNotSupportedError(ProtocolError): - pass diff --git a/tests/protocol/test_extensions_goal_state.py b/tests/protocol/test_extensions_goal_state.py index eac60f811..235e553fa 100644 --- a/tests/protocol/test_extensions_goal_state.py +++ b/tests/protocol/test_extensions_goal_state.py @@ -19,7 +19,7 @@ class ExtensionsGoalStateTestCase(AgentTestCase): def test_compare_should_succeed_when_extensions_config_and_vm_settings_are_equal(self): with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: from_extensions_config = protocol.client.get_extensions_goal_state() - from_vm_settings = protocol.client._vm_settings_goal_state + from_vm_settings = protocol.client._cached_vm_settings try: ExtensionsGoalState.compare(from_extensions_config, from_vm_settings) @@ -29,7 +29,7 @@ def test_compare_should_succeed_when_extensions_config_and_vm_settings_are_equal def test_compare_should_report_mismatches_between_extensions_config_and_vm_settings(self): with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: from_extensions_config = protocol.client.get_extensions_goal_state() - from_vm_settings = protocol.client._vm_settings_goal_state + from_vm_settings = protocol.client._cached_vm_settings def assert_compare_raises(setup_copy, failing_attribute): from_vm_settings_copy = copy.deepcopy(from_vm_settings) @@ -80,7 +80,7 @@ def test_extension_goal_state_should_parse_requested_version_properly(self): for manifest in fabric_manifests: self.assertEqual(manifest.requested_version_string, "0.0.0.0", "Version should be None") - vm_settings_ga_manifests = protocol.client._vm_settings_goal_state.agent_manifests + vm_settings_ga_manifests = protocol.client._cached_vm_settings.agent_manifests for manifest in vm_settings_ga_manifests: self.assertEqual(manifest.requested_version_string, "0.0.0.0", "Version should be None") @@ -92,6 +92,6 @@ def test_extension_goal_state_should_parse_requested_version_properly(self): for manifest in fabric_manifests: self.assertEqual(manifest.requested_version_string, "9.9.9.10", "Version should be 9.9.9.10") - vm_settings_ga_manifests = protocol.client._vm_settings_goal_state.agent_manifests + vm_settings_ga_manifests = protocol.client._cached_vm_settings.agent_manifests for manifest in vm_settings_ga_manifests: self.assertEqual(manifest.requested_version_string, "9.9.9.9", "Version should be 9.9.9.9") From e6fcf19c2c374160a355032868df79c18db2a269 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Thu, 13 Jan 2022 12:38:06 -0800 Subject: [PATCH 25/89] Retry update_goal_state on GoalStateMismatchError (#2470) * Retry update_goal_state on GoalStateMismatchError * Add sleep before retry * Enable test * Enable test * Update test * Add unit test * Add data file * pylint warning * Add comment; fix typos * fix typo Co-authored-by: narrieta --- .../common/protocol/extensions_goal_state.py | 20 +-- azurelinuxagent/common/protocol/wire.py | 82 ++++++---- .../ext_conf-requested_version.xml | 148 ++++++++++++++++++ .../hostgaplugin/vm_settings-out-of-sync.json | 65 ++++++++ tests/protocol/mockwiredata.py | 19 ++- tests/protocol/test_extensions_goal_state.py | 6 +- tests/protocol/test_wire.py | 27 +++- 7 files changed, 315 insertions(+), 52 deletions(-) create mode 100644 tests/data/hostgaplugin/ext_conf-requested_version.xml create mode 100644 tests/data/hostgaplugin/vm_settings-out-of-sync.json diff --git a/azurelinuxagent/common/protocol/extensions_goal_state.py b/azurelinuxagent/common/protocol/extensions_goal_state.py index 6ce44e2c8..68ce74dc7 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state.py @@ -18,15 +18,15 @@ import datetime import azurelinuxagent.common.logger as logger -from azurelinuxagent.common.future import ustr from azurelinuxagent.common.AgentGlobals import AgentGlobals from azurelinuxagent.common.exception import AgentError from azurelinuxagent.common.utils import textutil class GoalStateMismatchError(AgentError): - def __init__(self, msg): - super(GoalStateMismatchError, self).__init__(msg) + def __init__(self, message, attribute): + super(GoalStateMismatchError, self).__init__(message) + self.attribute = attribute class ExtensionsGoalState(object): @@ -100,9 +100,12 @@ def compare(from_extensions_config, from_vm_settings): context = [] # used to keep track of the attribute that is being compared def compare_goal_states(first, second): + # A mismatch on the timestamp or the activity ID (and maybe also on the correlation ID) most likely indicate that we are comparing two + # different goal states so we check them first (we raise an exception as soon as a mismatch is detected). A mismatch on the other + # attributes likely indicates an actual issue on vmSettings or extensionsConfig). + compare_attributes(first, second, "created_on_timestamp") compare_attributes(first, second, "activity_id") compare_attributes(first, second, "correlation_id") - compare_attributes(first, second, "created_on_timestamp") compare_attributes(first, second, "status_upload_blob") compare_attributes(first, second, "status_upload_blob_type") compare_attributes(first, second, "required_features") @@ -154,14 +157,13 @@ def compare_attributes(first, second, attribute, ignore_order=False): second_value.sort() if first_value != second_value: - raise Exception("[{0}] != [{1}] (Attribute: {2})".format(first_value, second_value, ".".join(context))) + mistmatch = "[{0}] != [{1}] (Attribute: {2})".format(first_value, second_value, ".".join(context)) + message = "Mismatch in Goal States [Incarnation {0}] != [Etag: {1}]: {2}".format(from_extensions_config.id, from_vm_settings.id, mistmatch) + raise GoalStateMismatchError(message, attribute) finally: context.pop() - try: - compare_goal_states(from_extensions_config, from_vm_settings) - except Exception as exception: - raise GoalStateMismatchError("Mismatch in Goal States [Incarnation {0}] != [Etag: {1}]: {2}".format(from_extensions_config.id, from_vm_settings.id, ustr(exception))) + compare_goal_states(from_extensions_config, from_vm_settings) def _do_common_validations(self): """ diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 6f35abd06..abf8b4188 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -35,7 +35,7 @@ from azurelinuxagent.common.exception import ProtocolNotFoundError, \ ResourceGoneError, ExtensionDownloadError, InvalidContainerError, ProtocolError, HttpError from azurelinuxagent.common.future import httpclient, bytebuffer, ustr -from azurelinuxagent.common.protocol.extensions_goal_state import ExtensionsGoalState +from azurelinuxagent.common.protocol.extensions_goal_state import ExtensionsGoalState, GoalStateMismatchError from azurelinuxagent.common.protocol.extensions_goal_state_factory import ExtensionsGoalStateFactory from azurelinuxagent.common.protocol.goal_state import GoalState, TRANSPORT_CERT_FILE_NAME, TRANSPORT_PRV_FILE_NAME from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol @@ -780,55 +780,36 @@ def update_host_plugin_from_goal_state(self): goal_state = GoalState(self) self._update_host_plugin(goal_state.container_id, goal_state.role_config_name) - def update_goal_state(self, force_update=False): + def update_goal_state(self, force_update=False, is_retry=False): """ Updates the goal state if the incarnation or etag changed or if 'force_update' is True """ try: # # The entire goal state needs to be retrieved from the WireServer (via the GoalState class), and the HostGAPlugin - # (via the self._fetch_vm_settings_goal_state method.) + # (via the self._fetch_vm_settings_goal_state method). # - # Start by fetching the goal state from the WireServer; if fetch_full_goal_state becomes True this will also - # update the ExtensionsConfig instance in GoalState. + # We fetch it in 3 parts: + # + # 1) The "main" goal state from the WireServer, which includes the incarnation, container ID, role config, and URLs + # to the rest of the goal state (certificates, remote users, extensions config, etc). We do this first because + # we need to initialize the HostGAPlugin with the container ID and role config. # goal_state = GoalState(self) - # Always update the hostgaplugin, since the agent may issue requests to it even if there are no other changes - # in the goal state (e.g. report status) self._update_host_plugin(goal_state.container_id, goal_state.role_config_name) - # Fetch the full goal state if needed - if force_update: - logger.info("Forcing an update of the goal state..") - - fetch_full_goal_state = force_update or \ - self._goal_state is None or self._extensions_goal_state is None or \ - self._goal_state.incarnation != goal_state.incarnation - - if not fetch_full_goal_state: - goal_state_updated = False - else: - goal_state.fetch_full_goal_state(self) - self._goal_state = goal_state - goal_state_updated = True - # - # Now fetch the vmSettings (which contain the extensions goal state) from the HostGAPlugin + # 2) Then we fetch the vmSettings from the HostGAPlugin. We do this before fetching the rest of the goal state from the + # WireServer to minimize the time between the initial call to the WireServer and the call to the HostGAPlugin (and hence + # reduce the window in which a new goal state may arrive in-between the 2 calls) # vm_settings_goal_state, vm_settings_goal_state_updated = (None, False) - vm_settings_goal_state_is_valid = False if conf.get_enable_fast_track(): try: vm_settings_goal_state, vm_settings_goal_state_updated = self._fetch_vm_settings_goal_state(force_update=force_update) - if goal_state_updated or vm_settings_goal_state_updated: - # compare() raises a GoalStateMismatchError if the goal states don't match - ExtensionsGoalState.compare(self._goal_state.extensions_config, vm_settings_goal_state) - - vm_settings_goal_state_is_valid = True - except Exception as error: # _fetch_vm_settings_goal_state() does its own detailed error reporting and raises ProtocolError; do not report those if not isinstance(error, ProtocolError): @@ -836,14 +817,51 @@ def update_goal_state(self, force_update=False): self._vm_settings_error_reporter.report_summary() # - # If we fetched a valid vmSettings, use that for the extensions goal state. Otherwise use ExtensionsConfig + # 3) Lastly we, fetch the rest of the goal state from the WireServer (but ony if needed: initialization, a "forced" update, or + # a change in the incarnation). Note that if we fetch the full goal state we also update self._goal_state. + # + if force_update: + logger.info("Forcing an update of the goal state..") + + fetch_full_goal_state = force_update or self._goal_state is None or self._goal_state.incarnation != goal_state.incarnation + + if not fetch_full_goal_state: + goal_state_updated = False + else: + goal_state.fetch_full_goal_state(self) + self._goal_state = goal_state + goal_state_updated = True + + # + # If we fetched the vmSettings then compare them against extensionsConfig and use them for the extensions goal state if + # everything matches, otherwise use extensionsConfig. # - if vm_settings_goal_state_is_valid: + use_vm_settings = False + if vm_settings_goal_state is not None: + if not goal_state_updated and not vm_settings_goal_state_updated: # no need to compare them, just use vmSettings + use_vm_settings = True + else: + try: + ExtensionsGoalState.compare(self._goal_state.extensions_config, vm_settings_goal_state) + use_vm_settings = True + except GoalStateMismatchError as mismatch: + if not is_retry and mismatch.attribute in ("created_on_timestamp", "activity_id"): + # this may be OK; a new goal state may have arrived in-between the calls to the HostGAPlugin and the WireServer; + # retry one time after a delay and then report the error if it happens again. + time.sleep(conf.get_goal_state_period()) + self.update_goal_state(is_retry=True) + return + self._vm_settings_error_reporter.report_error(ustr(mismatch)) + self._vm_settings_error_reporter.report_summary() + + if use_vm_settings: self._extensions_goal_state = vm_settings_goal_state else: self._extensions_goal_state = self._goal_state.extensions_config + # # If either goal state changed (goal_state or vm_settings_goal_state) save them + # if goal_state_updated or vm_settings_goal_state_updated: self._save_goal_state() diff --git a/tests/data/hostgaplugin/ext_conf-requested_version.xml b/tests/data/hostgaplugin/ext_conf-requested_version.xml new file mode 100644 index 000000000..c3bd92823 --- /dev/null +++ b/tests/data/hostgaplugin/ext_conf-requested_version.xml @@ -0,0 +1,148 @@ + + + + + Prod + 9.9.9.10 + + https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml + https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml + + + + Test + 9.9.9.10 + + https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml + https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml + + + + CentralUSEUAP + CRP + + + + MultipleExtensionsPerHandler + + +https://dcrcl3a0xs.blob.core.windows.net/$system/edp0plkw2b.86f4ae0a-61f8-48ae-9199-40f402d56864.status?sv=2018-03-28&sr=b&sk=system-1&sig=KNWgC2%3d&se=9999-01-01T00%3a00%3a00Z&sp=w + + + + https://zrdfepirv2cbn09pr02a.blob.core.windows.net/a47f0806d764480a8d989d009c75007d/Microsoft.Azure.Monitor_AzureMonitorLinuxAgent_useast2euap_manifest.xml + + + + + https://zrdfepirv2cbn06prdstr01a.blob.core.windows.net/4ef06ad957494df49c807a5334f2b5d2/Microsoft.Azure.Security.Monitoring_AzureSecurityLinuxAgent_useast2euap_manifest.xml + + + + + https://umsanh4b5rfz0q0p4pwm.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml + + + + + https://umsawqtlsshtn5v2nfgh.blob.core.windows.net/f4086d41-69f9-3103-78e0-8a2c7e789d0f/f4086d41-69f9-3103-78e0-8a2c7e789d0f_manifest.xml + + + + + https://umsah3cwjlctnmhsvzqv.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", + "publicSettings": {"GCS_AUTO_CONFIG":true} + } + } + ] +} + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", + "publicSettings": {"enableGenevaUpload":true} + } + } + ] +} + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"commandToExecute":"echo 'cee174d4-4daa-4b07-9958-53b9649445c2'"} + } + } + ] +} + + + + + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"source":{"script":"echo '4abb1e88-f349-41f8-8442-247d9fdfcac5'"}} + } + } + ] +} + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"source":{"script":"echo 'e865c9bc-a7b3-42c6-9a79-cfa98a1ee8b3'"}} + } + } + ] +} + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"source":{"script":"echo 'f923e416-0340-485c-9243-8b84fb9930c6'"}} + } + } + ] +} + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "protectedSettingsCertThumbprint": "59A10F50FFE2A0408D3F03FE336C8FD5716CF25C", + "protectedSettings": "*** REDACTED ***" + } + } + ] +} + + +https://dcrcl3a0xs.blob.core.windows.net/$system/edp0plkw2b.86f4ae0a-61f8-48ae-9199-40f402d56864.vmSettings?sv=2018-03-28&sr=b&sk=system-1&sig=PaiLic%3d&se=9999-01-01T00%3a00%3a00Z&sp=r + diff --git a/tests/data/hostgaplugin/vm_settings-out-of-sync.json b/tests/data/hostgaplugin/vm_settings-out-of-sync.json new file mode 100644 index 000000000..db755d249 --- /dev/null +++ b/tests/data/hostgaplugin/vm_settings-out-of-sync.json @@ -0,0 +1,65 @@ +{ + "hostGAPluginVersion": "1.0.8.115", + "vmSettingsSchemaVersion": "0.0", + "activityId": "AAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE", + "correlationId": "EEEEEEEE-DDDD-CCCC-BBBB-AAAAAAAAAAAA", + "extensionsLastModifiedTickCount": 637726657000000000, + "extensionGoalStatesSource": "Fabric", + "onHold": true, + "statusUploadBlob": { + "statusBlobType": "BlockBlob", + "value": "https://dcrcl3a0xs.blob.core.windows.net/$system/edp0plkw2b.86f4ae0a-61f8-48ae-9199-40f402d56864.status?sv=2018-03-28&sr=b&sk=system-1&sig=KNWgC2%3d&se=9999-01-01T00%3a00%3a00Z&sp=w" + }, + "inVMMetadata": { + "subscriptionId": "8e037ad4-618f-4466-8bc8-5099d41ac15b", + "resourceGroupName": "rg-dc-86fjzhp", + "vmName": "edp0plkw2b", + "location": "CentralUSEUAP", + "vmId": "86f4ae0a-61f8-48ae-9199-40f402d56864", + "vmSize": "Standard_B2s", + "osType": "Linux" + }, + "requiredFeatures": [ + { + "name": "MultipleExtensionsPerHandler" + } + ], + "gaFamilies": [ + { + "name": "Prod", + "uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml" + ] + }, + { + "name": "Test", + "uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml" + ] + } + ], + "extensionGoalStates": [ + { + "name": "Microsoft.Azure.Monitor.AzureMonitorLinuxAgent", + "version": "1.9.1", + "location": "https://zrdfepirv2cbn04prdstr01a.blob.core.windows.net/a47f0806d764480a8d989d009c75007d/Microsoft.Azure.Monitor_AzureMonitorLinuxAgent_useast2euap_manifest.xml", + "failoverlocation": "https://zrdfepirv2cbn06prdstr01a.blob.core.windows.net/a47f0806d764480a8d989d009c75007d/Microsoft.Azure.Monitor_AzureMonitorLinuxAgent_useast2euap_manifest.xml", + "additionalLocations": ["https://zrdfepirv2cbn09pr02a.blob.core.windows.net/a47f0806d764480a8d989d009c75007d/Microsoft.Azure.Monitor_AzureMonitorLinuxAgent_useast2euap_manifest.xml"], + "state": "enabled", + "autoUpgrade": true, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "settings": [ + { + "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", + "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" + } + ] + } + ] +} diff --git a/tests/protocol/mockwiredata.py b/tests/protocol/mockwiredata.py index b49f91569..b087d8b7a 100644 --- a/tests/protocol/mockwiredata.py +++ b/tests/protocol/mockwiredata.py @@ -38,8 +38,8 @@ "test_ext": "ext/sample_ext-1.3.0.zip", "remote_access": None, "in_vm_artifacts_profile": None, - "vm_settings": "hostgaplugin/vm_settings.json", - "ETag": "1" + "vm_settings": None, + "ETag": None } DATA_FILE_IN_VM_ARTIFACTS_PROFILE = DATA_FILE.copy() @@ -114,6 +114,7 @@ DATA_FILE_VM_SETTINGS = DATA_FILE.copy() DATA_FILE_VM_SETTINGS["vm_settings"] = "hostgaplugin/vm_settings.json" +DATA_FILE_VM_SETTINGS["ETag"] ="1" DATA_FILE_VM_SETTINGS["ext_conf"] = "hostgaplugin/ext_conf.xml" DATA_FILE_VM_SETTINGS["in_vm_artifacts_profile"] = "hostgaplugin/in_vm_artifacts_profile.json" @@ -180,8 +181,11 @@ def reload(self): self.trans_prv = load_data(self.data_files.get("trans_prv")) self.trans_cert = load_data(self.data_files.get("trans_cert")) self.ext = load_bin_data(self.data_files.get("test_ext")) - self.vm_settings = load_data(self.data_files.get("vm_settings")) - self.etag = self.data_files.get("ETag") + + vm_settings = self.data_files.get("vm_settings") + if vm_settings is not None: + self.vm_settings = load_data(self.data_files.get("vm_settings")) + self.etag = self.data_files.get("ETag") remote_access_data_file = self.data_files.get("remote_access") if remote_access_data_file is not None: @@ -229,8 +233,11 @@ def mock_http_get(self, url, *_, **kwargs): content = self.in_vm_artifacts_profile self.call_counts["in_vm_artifacts_profile"] += 1 elif "/vmSettings" in url: - content = self.vm_settings - response_headers = [('ETag', self.etag)] + if self.vm_settings is None: + resp.status = httpclient.NOT_FOUND + else: + content = self.vm_settings + response_headers = [('ETag', self.etag)] self.call_counts["vm_settings"] += 1 else: diff --git a/tests/protocol/test_extensions_goal_state.py b/tests/protocol/test_extensions_goal_state.py index 235e553fa..1138df3a4 100644 --- a/tests/protocol/test_extensions_goal_state.py +++ b/tests/protocol/test_extensions_goal_state.py @@ -75,7 +75,7 @@ def test_create_from_vm_settings_should_assume_block_when_blob_type_is_not_valid self.assertEqual("BlockBlob", extensions_goal_state.status_upload_blob_type, 'Expected BlockBob for an invalid statusBlobType') def test_extension_goal_state_should_parse_requested_version_properly(self): - with mock_wire_protocol(mockwiredata.DATA_FILE) as protocol: + with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: fabric_manifests, _ = protocol.get_vmagent_manifests() for manifest in fabric_manifests: self.assertEqual(manifest.requested_version_string, "0.0.0.0", "Version should be None") @@ -84,9 +84,9 @@ def test_extension_goal_state_should_parse_requested_version_properly(self): for manifest in vm_settings_ga_manifests: self.assertEqual(manifest.requested_version_string, "0.0.0.0", "Version should be None") - data_file = mockwiredata.DATA_FILE.copy() + data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() data_file["vm_settings"] = "hostgaplugin/vm_settings-requested_version.json" - data_file["ext_conf"] = "wire/ext_conf_requested_version.xml" + data_file["ext_conf"] = "hostgaplugin/ext_conf-requested_version.xml" with mock_wire_protocol(data_file) as protocol: fabric_manifests, _ = protocol.get_vmagent_manifests() for manifest in fabric_manifests: diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py index 0ac5fc135..e84ea2a65 100644 --- a/tests/protocol/test_wire.py +++ b/tests/protocol/test_wire.py @@ -52,7 +52,7 @@ from tests.protocol.HttpRequestPredicates import HttpRequestPredicates from tests.protocol.mockwiredata import DATA_FILE_NO_EXT, DATA_FILE from tests.protocol.mockwiredata import WireProtocolData -from tests.tools import Mock, PropertyMock, patch, AgentTestCase +from tests.tools import Mock, PropertyMock, patch, AgentTestCase, load_bin_data, mock_sleep data_with_bom = b'\xef\xbb\xbfhehe' testurl = 'http://foo' @@ -1276,7 +1276,7 @@ def http_get_handler(url, *_, **__): # Lastly, test the goal state comparison def fail_compare(): - error = GoalStateMismatchError("TEST COMPARE FAILED") + error = GoalStateMismatchError("TEST COMPARE FAILED", "dummy_attribute") with patch("azurelinuxagent.common.protocol.extensions_goal_state.ExtensionsGoalState.compare", side_effect=error): with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: protocol.client.update_goal_state() @@ -1371,6 +1371,29 @@ def test_it_should_use_extensions_config_when_vm_settings_do_not_match_extension reported = [kwargs for _, kwargs in add_event_patcher.call_args_list if kwargs['op'] == "VmSettings" and "GoalStateMismatchError" in kwargs['message']] self.assertEqual(1, len(reported), "The goal state mismatch should have been reported exactly once; got: {0}".format([kwargs['message'] for _, kwargs in add_event_patcher.call_args_list])) + def test_it_should_retry_vm_settings_and_extensions_config_do_not_match(self): + def http_get_handler(url, *_, **__): + if self.is_host_plugin_vm_settings_request(url): + response = MockHttpResponse(httpclient.OK) + response.body = load_bin_data("hostgaplugin/vm_settings-out-of-sync.json") + response.headers = [('ETag', "0123456789")] + return response + return None + + data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() + with mock_wire_protocol(data_file) as protocol: + protocol.set_http_handlers(http_get_handler=http_get_handler) + + with patch('time.sleep', side_effect=lambda _: mock_sleep()): # avoid the sleep during retry + with patch("azurelinuxagent.common.protocol.wire.add_event") as add_event: + protocol.client.update_goal_state() + + vm_settings_call_count = len([url for url in protocol.get_tracked_urls() if "vmSettings" in url]) + self.assertEqual(2, vm_settings_call_count, "Expected 2 calls to vmSettings (original and retry)") + + errors = [kwargs["message"] for _, kwargs in add_event.call_args_list if kwargs["op"] == "VmSettings"] + self.assertTrue(any("[GoalStateMismatchError]" in e for e in errors), "Expected GoalStateMismatchError to have been reported. Got: {0}".format(errors)) + def test_it_should_compare_goal_states_when_vm_settings_change(self): with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: protocol.mock_wire_data.set_etag("aNewEtag") From 67f5d756f198780c80c780b82a977bde9e55a6d4 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 14 Jan 2022 07:09:29 -0800 Subject: [PATCH 26/89] Set agent version to 2.7.0.1 (#2473) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 7055bce34..1e9372cd9 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.7.0.0' +AGENT_VERSION = '2.7.0.1' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 966a938596b0812bd678e557f46362b62941ff31 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 19 Jan 2022 10:27:25 -0800 Subject: [PATCH 27/89] Redact settings from mismatch message (#2477) Co-authored-by: narrieta --- .../common/protocol/extensions_goal_state.py | 5 ++++- tests/protocol/test_extensions_goal_state.py | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/azurelinuxagent/common/protocol/extensions_goal_state.py b/azurelinuxagent/common/protocol/extensions_goal_state.py index 68ce74dc7..fcc7e2506 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state.py @@ -157,7 +157,10 @@ def compare_attributes(first, second, attribute, ignore_order=False): second_value.sort() if first_value != second_value: - mistmatch = "[{0}] != [{1}] (Attribute: {2})".format(first_value, second_value, ".".join(context)) + if attribute.lower() in ('protectedsettings', 'publicsettings'): + mistmatch = "[REDACTED] != [REDACTED] (Attribute: {0})".format(".".join(context)) + else: + mistmatch = "[{0}] != [{1}] (Attribute: {2})".format(first_value, second_value, ".".join(context)) message = "Mismatch in Goal States [Incarnation {0}] != [Etag: {1}]: {2}".format(from_extensions_config.id, from_vm_settings.id, mistmatch) raise GoalStateMismatchError(message, attribute) finally: diff --git a/tests/protocol/test_extensions_goal_state.py b/tests/protocol/test_extensions_goal_state.py index 1138df3a4..90b4136d0 100644 --- a/tests/protocol/test_extensions_goal_state.py +++ b/tests/protocol/test_extensions_goal_state.py @@ -31,12 +31,21 @@ def test_compare_should_report_mismatches_between_extensions_config_and_vm_setti from_extensions_config = protocol.client.get_extensions_goal_state() from_vm_settings = protocol.client._cached_vm_settings + mismatch_messages = { + 'publicSettings': None, + 'protectedSettings': None + } + def assert_compare_raises(setup_copy, failing_attribute): from_vm_settings_copy = copy.deepcopy(from_vm_settings) setup_copy(from_vm_settings_copy) - with self.assertRaisesRegexCM(GoalStateMismatchError, re.escape("(Attribute: {0})".format(failing_attribute)), re.DOTALL): + with self.assertRaisesRegexCM(GoalStateMismatchError, re.escape("(Attribute: {0})".format(failing_attribute)), re.DOTALL) as context_manager: ExtensionsGoalState.compare(from_extensions_config, from_vm_settings_copy) + if context_manager.exception.attribute == 'publicSettings': + mismatch_messages['publicSettings'] = str(context_manager.exception) + elif context_manager.exception.attribute == 'protectedSettings': + mismatch_messages['protectedSettings'] = str(context_manager.exception) assert_compare_raises(lambda c: setattr(c, "_activity_id", 'MOCK_ACTIVITY_ID'), "activity_id") assert_compare_raises(lambda c: setattr(c, "_correlation_id", 'MOCK_CORRELATION_ID'), "correlation_id") @@ -63,6 +72,12 @@ def assert_compare_raises(setup_copy, failing_attribute): assert_compare_raises(lambda c: setattr(c.extensions[0].settings[0], "dependencyLevel", 56789), r"extensions[0].settings[0].dependencyLevel") assert_compare_raises(lambda c: setattr(c.extensions[0].settings[0], "state", 'MOCK_STATE'), r"extensions[0].settings[0].state") + expected = r'^\[GoalStateMismatchError\] Mismatch in Goal States \[Incarnation 1\] != \[Etag: 1\]: \[REDACTED\] != \[REDACTED\] \(Attribute: .*\.publicSettings\)$' + self.assertRegex(mismatch_messages['publicSettings'], expected, 'Expected the protected settings to be redacted. Got: "{0}"'.format(mismatch_messages['publicSettings'])) + + expected = r'^\[GoalStateMismatchError\] Mismatch in Goal States \[Incarnation 1\] != \[Etag: 1\]: \[REDACTED\] != \[REDACTED\] \(Attribute: .*\.protectedSettings\)$' + self.assertRegex(mismatch_messages['protectedSettings'], expected, 'Expected the protected settings to be redacted. Got: "{0}"'.format(mismatch_messages['protectedSettings'])) + def test_create_from_extensions_config_should_assume_block_when_blob_type_is_not_valid(self): data_file = mockwiredata.DATA_FILE.copy() data_file["vm_settings"] = "hostgaplugin/ext_conf-invalid_blob_type.xml" From a21d9dd47d3f5e05b10cc576747a133964239177 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 19 Jan 2022 11:13:16 -0800 Subject: [PATCH 28/89] Set Agent Version to 2.7.0.2 (#2478) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 1e9372cd9..db69b8920 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.7.0.1' +AGENT_VERSION = '2.7.0.2' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 83ae9e778cb6420d93bcb0eeb87e9f2d01e57659 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 24 Jan 2022 11:04:30 -0800 Subject: [PATCH 29/89] Improvements in telemetry for vmSettings (#2482) Co-authored-by: narrieta --- .../common/protocol/extensions_goal_state.py | 5 +++-- .../extensions_goal_state_from_vm_settings.py | 10 ++++++---- azurelinuxagent/common/protocol/wire.py | 2 +- azurelinuxagent/ga/exthandlers.py | 5 +++-- tests/protocol/test_extensions_goal_state.py | 16 ++++------------ 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/azurelinuxagent/common/protocol/extensions_goal_state.py b/azurelinuxagent/common/protocol/extensions_goal_state.py index fcc7e2506..ee7079277 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state.py @@ -127,10 +127,11 @@ def compare_extensions(first, second): compare_array(first.settings, second.settings, compare_settings, "settings") def compare_settings(first, second): + # Note that we do not compare protectedSettings since the same settings can be re-encrypted, resulting + # on different encrypted text for the same plain text. compare_attributes(first, second, "name") compare_attributes(first, second, "sequenceNumber") compare_attributes(first, second, "publicSettings") - compare_attributes(first, second, "protectedSettings") compare_attributes(first, second, "certificateThumbprint") compare_attributes(first, second, "dependencyLevel") compare_attributes(first, second, "state") @@ -157,7 +158,7 @@ def compare_attributes(first, second, attribute, ignore_order=False): second_value.sort() if first_value != second_value: - if attribute.lower() in ('protectedsettings', 'publicsettings'): + if attribute.lower() == 'publicsettings': mistmatch = "[REDACTED] != [REDACTED] (Attribute: {0})".format(".".join(context)) else: mistmatch = "[{0}] != [{1}] (Attribute: {2})".format(first_value, second_value, ".".join(context)) diff --git a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py index efc374bec..dcf6c2c2d 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py @@ -53,7 +53,7 @@ def __init__(self, etag, json_text): self._parse_vm_settings(json_text) self._do_common_validations() except Exception as e: - raise VmSettingsError("Error parsing vmSettings (etag: {0}): {1}\n{2}".format(etag, format_exception(e), self.get_redacted_text())) + raise VmSettingsError("Error parsing vmSettings (etag: {0} HGAP: {1}): {2}\n{3}".format(etag, self._host_ga_plugin_version, format_exception(e), self.get_redacted_text())) @property def id(self): @@ -139,14 +139,16 @@ def _parse_simple_attributes(self, vm_settings): # "extensionGoalStatesSource": "Fabric", # ... # } - self._activity_id = self._string_to_id(vm_settings.get("activityId")) - self._correlation_id = self._string_to_id(vm_settings.get("correlationId")) - self._created_on_timestamp = self._ticks_to_utc_timestamp(vm_settings.get("extensionsLastModifiedTickCount")) + # The HGAP version is included in some messages, so parse it first host_ga_plugin_version = vm_settings.get("hostGAPluginVersion") if host_ga_plugin_version is not None: self._host_ga_plugin_version = FlexibleVersion(host_ga_plugin_version) + self._activity_id = self._string_to_id(vm_settings.get("activityId")) + self._correlation_id = self._string_to_id(vm_settings.get("correlationId")) + self._created_on_timestamp = self._ticks_to_utc_timestamp(vm_settings.get("extensionsLastModifiedTickCount")) + schema_version = vm_settings.get("vmSettingsSchemaVersion") if schema_version is not None: self._schema_version = FlexibleVersion(schema_version) diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index abf8b4188..2f80ac212 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -894,7 +894,7 @@ def raise_not_supported(reset_state=False): correlation_id = str(uuid.uuid4()) def format_message(msg): - return "GET vmSettings [correlation ID: {0} eTag: {1}]: {2}".format(correlation_id, etag, msg) + return "GET vmSettings [correlation ID: {0} eTag: {1} HGAP: {2}]: {3}".format(correlation_id, etag, self._host_plugin_version, msg) try: def get_vm_settings(): diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index d1ca9a4fe..d065f8c32 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -300,8 +300,9 @@ def run(self): try: extensions_goal_state = self.protocol.get_extensions_goal_state() - # self.ext_handlers and etag need to be initialized first, since status reporting depends on them - self.ext_handlers = extensions_goal_state.extensions + # self.ext_handlers and etag need to be initialized first, since status reporting depends on them; also + # we make a deep copy of the extensions, since changes are made to self.ext_handlers while processing the extensions + self.ext_handlers = copy.deepcopy(extensions_goal_state.extensions) etag = self.protocol.client.get_goal_state().incarnation if not self._extension_processing_allowed(): diff --git a/tests/protocol/test_extensions_goal_state.py b/tests/protocol/test_extensions_goal_state.py index 90b4136d0..15c3f605a 100644 --- a/tests/protocol/test_extensions_goal_state.py +++ b/tests/protocol/test_extensions_goal_state.py @@ -31,10 +31,7 @@ def test_compare_should_report_mismatches_between_extensions_config_and_vm_setti from_extensions_config = protocol.client.get_extensions_goal_state() from_vm_settings = protocol.client._cached_vm_settings - mismatch_messages = { - 'publicSettings': None, - 'protectedSettings': None - } + public_settings_mismatch = [""] def assert_compare_raises(setup_copy, failing_attribute): from_vm_settings_copy = copy.deepcopy(from_vm_settings) @@ -43,9 +40,7 @@ def assert_compare_raises(setup_copy, failing_attribute): with self.assertRaisesRegexCM(GoalStateMismatchError, re.escape("(Attribute: {0})".format(failing_attribute)), re.DOTALL) as context_manager: ExtensionsGoalState.compare(from_extensions_config, from_vm_settings_copy) if context_manager.exception.attribute == 'publicSettings': - mismatch_messages['publicSettings'] = str(context_manager.exception) - elif context_manager.exception.attribute == 'protectedSettings': - mismatch_messages['protectedSettings'] = str(context_manager.exception) + public_settings_mismatch[0] = str(context_manager.exception) assert_compare_raises(lambda c: setattr(c, "_activity_id", 'MOCK_ACTIVITY_ID'), "activity_id") assert_compare_raises(lambda c: setattr(c, "_correlation_id", 'MOCK_CORRELATION_ID'), "correlation_id") @@ -64,19 +59,16 @@ def assert_compare_raises(setup_copy, failing_attribute): assert_compare_raises(lambda c: setattr(c.extensions[0], "manifest_uris", ['MOCK_URI']), r"extensions[0].manifest_uris") assert_compare_raises(lambda c: setattr(c.extensions[0], "supports_multi_config", True), r"extensions[0].supports_multi_config") + # NOTE: protectedSettings are not compared, so we skip them below assert_compare_raises(lambda c: setattr(c.extensions[0].settings[0], "name", 'MOCK_NAME'), r"extensions[0].settings[0].name") assert_compare_raises(lambda c: setattr(c.extensions[0].settings[0], "sequenceNumber", 98765), r"extensions[0].settings[0].sequenceNumber") assert_compare_raises(lambda c: setattr(c.extensions[0].settings[0], "publicSettings", {'MOCK_NAME': 'MOCK_VALUE'}), r"extensions[0].settings[0].publicSettings") - assert_compare_raises(lambda c: setattr(c.extensions[0].settings[0], "protectedSettings", 'MOCK_SETTINGS'), r"extensions[0].settings[0].protectedSettings") assert_compare_raises(lambda c: setattr(c.extensions[0].settings[0], "certificateThumbprint", 'MOCK_CERT'), r"extensions[0].settings[0].certificateThumbprint") assert_compare_raises(lambda c: setattr(c.extensions[0].settings[0], "dependencyLevel", 56789), r"extensions[0].settings[0].dependencyLevel") assert_compare_raises(lambda c: setattr(c.extensions[0].settings[0], "state", 'MOCK_STATE'), r"extensions[0].settings[0].state") expected = r'^\[GoalStateMismatchError\] Mismatch in Goal States \[Incarnation 1\] != \[Etag: 1\]: \[REDACTED\] != \[REDACTED\] \(Attribute: .*\.publicSettings\)$' - self.assertRegex(mismatch_messages['publicSettings'], expected, 'Expected the protected settings to be redacted. Got: "{0}"'.format(mismatch_messages['publicSettings'])) - - expected = r'^\[GoalStateMismatchError\] Mismatch in Goal States \[Incarnation 1\] != \[Etag: 1\]: \[REDACTED\] != \[REDACTED\] \(Attribute: .*\.protectedSettings\)$' - self.assertRegex(mismatch_messages['protectedSettings'], expected, 'Expected the protected settings to be redacted. Got: "{0}"'.format(mismatch_messages['protectedSettings'])) + self.assertRegex(public_settings_mismatch[0], expected, 'Expected the protected settings to be redacted. Got: "{0}"'.format(public_settings_mismatch[0])) def test_create_from_extensions_config_should_assume_block_when_blob_type_is_not_valid(self): data_file = mockwiredata.DATA_FILE.copy() From b4fa41bf6a824da06da950edc22ff72f89751264 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 24 Jan 2022 11:12:57 -0800 Subject: [PATCH 30/89] Set Agent version to 2.7.0.3 (#2483) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index db69b8920..617c4dc75 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.7.0.2' +AGENT_VERSION = '2.7.0.3' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 7cce03b271cce9bee8d3c09a6eea3ea75cc3526c Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 7 Feb 2022 16:04:13 -0800 Subject: [PATCH 31/89] =?UTF-8?q?Do=20not=20raise=20on=20missing=20status?= =?UTF-8?q?=20blob;=20reduce=20amount=20of=20logging=20for=20vms=E2=80=A6?= =?UTF-8?q?=20(#2492)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Do not raise on missing status blob; reduce amount of logging for vmsettings * remove extra file; fix typo Co-authored-by: narrieta --- .../extensions_goal_state_from_vm_settings.py | 20 +++--- azurelinuxagent/common/protocol/wire.py | 7 +- .../ext_conf-no_status_upload_blob.xml | 39 +++++++++++ .../vm_settings-no_status_upload_blob.json | 65 +++++++++++++++++++ ...sions_goal_state_from_extensions_config.py | 10 +++ ..._extensions_goal_state_from_vm_settings.py | 8 +++ tests/protocol/test_wire.py | 26 +++++--- 7 files changed, 156 insertions(+), 19 deletions(-) create mode 100644 tests/data/hostgaplugin/ext_conf-no_status_upload_blob.xml create mode 100644 tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json diff --git a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py index dcf6c2c2d..12a231732 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py @@ -22,10 +22,10 @@ from azurelinuxagent.common.AgentGlobals import AgentGlobals from azurelinuxagent.common.exception import VmSettingsError +from azurelinuxagent.common.future import ustr from azurelinuxagent.common.protocol.extensions_goal_state import ExtensionsGoalState from azurelinuxagent.common.protocol.restapi import VMAgentManifest, Extension, ExtensionRequestedState, ExtensionSettings from azurelinuxagent.common.utils.flexible_version import FlexibleVersion -from azurelinuxagent.common.utils.textutil import format_exception class ExtensionsGoalStateFromVmSettings(ExtensionsGoalState): @@ -53,7 +53,7 @@ def __init__(self, etag, json_text): self._parse_vm_settings(json_text) self._do_common_validations() except Exception as e: - raise VmSettingsError("Error parsing vmSettings (etag: {0} HGAP: {1}): {2}\n{3}".format(etag, self._host_ga_plugin_version, format_exception(e), self.get_redacted_text())) + raise VmSettingsError("Error parsing vmSettings [HGAP: {0}]: {1}".format(self._host_ga_plugin_version, ustr(e))) @property def id(self): @@ -173,13 +173,15 @@ def _parse_status_upload_blob(self, vm_settings): # } status_upload_blob = vm_settings.get("statusUploadBlob") if status_upload_blob is None: - raise Exception("Missing statusUploadBlob") - self._status_upload_blob = status_upload_blob.get("value") - if self._status_upload_blob is None: - raise Exception("Missing statusUploadBlob.value") - self._status_upload_blob_type = status_upload_blob.get("statusBlobType") - if self._status_upload_blob is None: - raise Exception("Missing statusUploadBlob.statusBlobType") + self._status_upload_blob = None + self._status_upload_blob_type = "BlockBlob" + else: + self._status_upload_blob = status_upload_blob.get("value") + if self._status_upload_blob is None: + raise Exception("Missing statusUploadBlob.value") + self._status_upload_blob_type = status_upload_blob.get("statusBlobType") + if self._status_upload_blob_type is None: + self._status_upload_blob_type = "BlockBlob" def _parse_required_features(self, vm_settings): # Sample: diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 2f80ac212..cda6381f1 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -1537,7 +1537,8 @@ class _VmSettingsError(object): class _VmSettingsErrorReporter(object): - _MaxErrors = 5 # Max number of error reported by period + _MaxLogErrors = 1 # Max number of errors by period reported to the local log + _MaxTelemetryErrors = 3 # Max number of errors by period reported to telemetry _Period = timedelta(hours=1) # How often to report the summary def __init__(self): @@ -1558,8 +1559,10 @@ def report_request(self): def report_error(self, error, category=None): self._error_count += 1 - if self._error_count <= _VmSettingsErrorReporter._MaxErrors: + if self._error_count <= _VmSettingsErrorReporter._MaxLogErrors: logger.info("[VmSettings] [Informational only, the Agent will continue normal operation] {0}", error) + + if self._error_count <= _VmSettingsErrorReporter._MaxTelemetryErrors: add_event(op=WALAEventOperation.VmSettings, message=error, is_success=False, log_event=False) if category == _VmSettingsError.ServerError: diff --git a/tests/data/hostgaplugin/ext_conf-no_status_upload_blob.xml b/tests/data/hostgaplugin/ext_conf-no_status_upload_blob.xml new file mode 100644 index 000000000..5141c0dca --- /dev/null +++ b/tests/data/hostgaplugin/ext_conf-no_status_upload_blob.xml @@ -0,0 +1,39 @@ + + + + + Prod + + https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml + https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml + + + + CentralUSEUAP + CRP + + + + + https://umsanh4b5rfz0q0p4pwm.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml + + + + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"commandToExecute":"echo 'cee174d4-4daa-4b07-9958-53b9649445c2'"} + } + } + ] +} + + +https://dcrcl3a0xs.blob.core.windows.net/$system/edp0plkw2b.86f4ae0a-61f8-48ae-9199-40f402d56864.vmSettings?sv=2018-03-28&sr=b&sk=system-1&sig=PaiLic%3d&se=9999-01-01T00%3a00%3a00Z&sp=r + diff --git a/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json b/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json new file mode 100644 index 000000000..c184454a6 --- /dev/null +++ b/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json @@ -0,0 +1,65 @@ +{ + "hostGAPluginVersion": "1.0.8.115", + "vmSettingsSchemaVersion": "0.0", + "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", + "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", + "extensionsLastModifiedTickCount": 637726657706205217, + "extensionGoalStatesSource": "Fabric", + "onHold": true, + "inVMMetadata": { + "subscriptionId": "8e037ad4-618f-4466-8bc8-5099d41ac15b", + "resourceGroupName": "rg-dc-86fjzhp", + "vmName": "edp0plkw2b", + "location": "CentralUSEUAP", + "vmId": "86f4ae0a-61f8-48ae-9199-40f402d56864", + "vmSize": "Standard_B2s", + "osType": "Linux" + }, + "requiredFeatures": [ + { + "name": "MultipleExtensionsPerHandler" + } + ], + "gaFamilies": [ + { + "name": "Prod", + "uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml" + ] + } + ], + "extensionGoalStates": [ + { + "name": "Microsoft.Azure.Extensions.CustomScript", + "version": "2.1.6", + "location": "https://umsavwggj2v40kvqhc0w.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml", + "failoverlocation": "https://umsafwzhkbm1rfrhl0ws.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml", + "additionalLocations": [ + "https://umsanh4b5rfz0q0p4pwm.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml" + ], + "state": "enabled", + "autoUpgrade": true, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "isMultiConfig": false, + "settings": [ + { + "publicSettings": "{\"commandToExecute\":\"echo 'cee174d4-4daa-4b07-9958-53b9649445c2'\"}" + } + ], + "dependsOn": [ + { + "DependsOnExtension": [ + { + "handler": "Microsoft.Azure.Security.Monitoring.AzureSecurityLinuxAgent" + } + ], + "dependencyLevel": 1 + } + ] + } + ] +} diff --git a/tests/protocol/test_extensions_goal_state_from_extensions_config.py b/tests/protocol/test_extensions_goal_state_from_extensions_config.py index d270f7adc..0fb1ef138 100644 --- a/tests/protocol/test_extensions_goal_state_from_extensions_config.py +++ b/tests/protocol/test_extensions_goal_state_from_extensions_config.py @@ -26,3 +26,13 @@ def test_it_should_use_default_values_when_in_vm_metadata_is_invalid(self): self.assertEqual(AgentGlobals.GUID_ZERO, extensions_goal_state.activity_id, "Incorrect activity Id") self.assertEqual(AgentGlobals.GUID_ZERO, extensions_goal_state.correlation_id, "Incorrect correlation Id") self.assertEqual('1900-01-01T00:00:00.000000Z', extensions_goal_state.created_on_timestamp, "Incorrect GS Creation time") + + def test_it_should_parse_missing_status_upload_blob_as_none(self): + data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() + data_file["ext_conf"] = "hostgaplugin/ext_conf-no_status_upload_blob.xml" + with mock_wire_protocol(data_file) as protocol: + extensions_goal_state = protocol.get_extensions_goal_state() + + self.assertIsNone(extensions_goal_state.status_upload_blob, "Expected status upload blob to be None") + self.assertEqual("BlockBlob", extensions_goal_state.status_upload_blob_type, "Expected status upload blob to be Block") + diff --git a/tests/protocol/test_extensions_goal_state_from_vm_settings.py b/tests/protocol/test_extensions_goal_state_from_vm_settings.py index b8f4deb6e..fe30eff0c 100644 --- a/tests/protocol/test_extensions_goal_state_from_vm_settings.py +++ b/tests/protocol/test_extensions_goal_state_from_vm_settings.py @@ -47,6 +47,14 @@ def assert_property(name, value): # dependency level (multi-config) self.assertEqual(1, vm_settings.extensions[3].settings[1].dependencyLevel, "Incorrect dependency level (multi-config)") + def test_create_from_vm_settings_should_parse_missing_status_upload_blob_as_none(self): + vm_settings_text = fileutil.read_file(os.path.join(data_dir, "hostgaplugin/vm_settings-no_status_upload_blob.json")) + vm_settings = ExtensionsGoalStateFactory.create_from_vm_settings("123", vm_settings_text) + + self.assertIsNone(vm_settings.status_upload_blob, "Expected status upload blob to be None") + self.assertEqual("BlockBlob", vm_settings.status_upload_blob_type, "Expected status upload blob to be Block") + + class CaseFoldedDictionaryTestCase(AgentTestCase): def test_it_should_retrieve_items_ignoring_case(self): dictionary = json.loads('''{ diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py index e84ea2a65..48e6c9b43 100644 --- a/tests/protocol/test_wire.py +++ b/tests/protocol/test_wire.py @@ -1291,25 +1291,35 @@ def http_get_handler(url, *_, **__): return None protocol.set_http_handlers(http_get_handler=http_get_handler) + def get_telemetry_messages(): + return [kwargs["message"] for _, kwargs in add_event.call_args_list if kwargs["op"] == "VmSettings"] + + def get_log_messages(): + return [arg[0][0] for arg in logger_info.call_args_list if "[VmSettings]" in arg[0][0]] + with patch("azurelinuxagent.common.protocol.wire.add_event") as add_event: - for _ in range(_VmSettingsErrorReporter._MaxErrors + 3): - protocol.client.update_goal_state() + with patch('azurelinuxagent.common.logger.info') as logger_info: + for _ in range(_VmSettingsErrorReporter._MaxTelemetryErrors + 3): + protocol.client.update_goal_state() - messages = [kwargs["message"] for _, kwargs in add_event.call_args_list if kwargs["op"] == "VmSettings"] + telemetry_messages = get_telemetry_messages() + self.assertEqual(_VmSettingsErrorReporter._MaxTelemetryErrors, len(telemetry_messages), "The number of errors reported to telemetry is not the max allowed (got: {0})".format(telemetry_messages)) - self.assertEqual(_VmSettingsErrorReporter._MaxErrors, len(messages), "The number of errors reported is not the max allowed (got: {0})".format(messages)) + log_messages = get_log_messages() + self.assertEqual(_VmSettingsErrorReporter._MaxLogErrors, len(log_messages), "The number of errors reported to the local log is not the max allowed (got: {0})".format(telemetry_messages)) # Reset the error reporter and verify that additional errors are reported protocol.client._vm_settings_error_reporter._next_period = datetime.now() protocol.client.update_goal_state() # this triggers the reset with patch("azurelinuxagent.common.protocol.wire.add_event") as add_event: - for _ in range(3): - protocol.client.update_goal_state() + protocol.client.update_goal_state() - messages = [kwargs["message"] for _, kwargs in add_event.call_args_list if kwargs["op"] == "VmSettings"] + telemetry_messages = get_telemetry_messages() + self.assertEqual(1, len(telemetry_messages), "Expected additional errors to be reported to telemetry in the next period (got: {0})".format(telemetry_messages)) - self.assertEqual(3, len(messages), "Expected additional errors to be reported in the next period (got: {0})".format(messages)) + log_messages = get_log_messages() + self.assertEqual(1, len(log_messages), "Expected additional errors to be reported to the local log in the next period (got: {0})".format(telemetry_messages)) def test_it_should_use_vm_settings_by_default(self): with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: From b78f7f2a1e1c7d4151119fdd8bcb6547742bf39b Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 7 Feb 2022 16:15:34 -0800 Subject: [PATCH 32/89] Set agent version to 2.7.0.4 (#2494) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 617c4dc75..d13421568 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.7.0.3' +AGENT_VERSION = '2.7.0.4' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 97906d483ba215e12eae1435dd0a83e8af1ce31c Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 11 Feb 2022 12:41:27 -0800 Subject: [PATCH 33/89] Save vmSettings on parse errors; improve messages in parse errors (#2503) * Save vmSettings on parse errors; improve messages in parse errors * pylint warnings * pylint warnings Co-authored-by: narrieta --- azurelinuxagent/common/exception.py | 4 ++ .../extensions_goal_state_from_vm_settings.py | 26 ++++--- azurelinuxagent/common/protocol/wire.py | 12 +++- tests/common/test_exception.py | 46 ------------- .../ext_conf-empty_depends_on.xml | 56 +++++++++++++++ .../vm_settings-empty_depends_on.json | 68 +++++++++++++++++++ ...sions_goal_state_from_extensions_config.py | 9 +++ tests/protocol/test_wire.py | 17 ++++- 8 files changed, 179 insertions(+), 59 deletions(-) delete mode 100644 tests/common/test_exception.py create mode 100644 tests/data/hostgaplugin/ext_conf-empty_depends_on.xml create mode 100644 tests/data/hostgaplugin/vm_settings-empty_depends_on.json diff --git a/azurelinuxagent/common/exception.py b/azurelinuxagent/common/exception.py index d39b1b959..9e4162fa1 100644 --- a/azurelinuxagent/common/exception.py +++ b/azurelinuxagent/common/exception.py @@ -125,6 +125,10 @@ class VmSettingsError(ExtensionsGoalStateError): """ Error raised when the VmSettings are malformed """ + def __init__(self, message, etag, vm_settings_text, inner=None): + super(VmSettingsError, self).__init__(message, inner) + self.etag = etag + self.vm_settings_text = vm_settings_text class MultiConfigExtensionEnableError(ExtensionError): diff --git a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py index 12a231732..0b0ac107f 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py @@ -23,6 +23,7 @@ from azurelinuxagent.common.AgentGlobals import AgentGlobals from azurelinuxagent.common.exception import VmSettingsError from azurelinuxagent.common.future import ustr +import azurelinuxagent.common.logger as logger from azurelinuxagent.common.protocol.extensions_goal_state import ExtensionsGoalState from azurelinuxagent.common.protocol.restapi import VMAgentManifest, Extension, ExtensionRequestedState, ExtensionSettings from azurelinuxagent.common.utils.flexible_version import FlexibleVersion @@ -53,7 +54,7 @@ def __init__(self, etag, json_text): self._parse_vm_settings(json_text) self._do_common_validations() except Exception as e: - raise VmSettingsError("Error parsing vmSettings [HGAP: {0}]: {1}".format(self._host_ga_plugin_version, ustr(e))) + raise VmSettingsError("Error parsing vmSettings [HGAP: {0}]: {1}".format(self._host_ga_plugin_version, ustr(e)), etag, self.get_redacted_text()) @property def id(self): @@ -197,13 +198,13 @@ def _parse_required_features(self, vm_settings): required_features = vm_settings.get("requiredFeatures") if required_features is not None: if not isinstance(required_features, list): - raise Exception("requiredFeatures should be an array") + raise Exception("requiredFeatures should be an array (got {0})".format(required_features)) def get_required_features_names(): for feature in required_features: name = feature.get("name") if name is None: - raise Exception("A required feature is missing the 'name' property") + raise Exception("A required feature is missing the 'name' property (got {0})".format(feature)) yield name self._required_features.extend(get_required_features_names()) @@ -235,7 +236,7 @@ def _parse_agent_manifests(self, vm_settings): if families is None: return if not isinstance(families, list): - raise Exception("gaFamilies should be an array") + raise Exception("gaFamilies should be an array (got {0})".format(families)) for family in families: name = family["name"] @@ -318,7 +319,7 @@ def _parse_extensions(self, vm_settings): extension_goal_states = vm_settings.get("extensionGoalStates") if extension_goal_states is not None: if not isinstance(extension_goal_states, list): - raise Exception("extension_goal_states should be an array") + raise Exception("extension_goal_states should be an array (got {0})".format(type(extension_goal_states))) # report only the type, since the value may contain secrets for extension_gs in extension_goal_states: extension = Extension() @@ -337,7 +338,7 @@ def _parse_extensions(self, vm_settings): additional_locations = extension_gs.get('additionalLocations') if additional_locations is not None: if not isinstance(additional_locations, list): - raise Exception('additionalLocations should be an array') + raise Exception('additionalLocations should be an array (got {0})'.format(additional_locations)) extension.manifest_uris.extend(additional_locations) # @@ -455,13 +456,18 @@ def _parse_dependency_level(depends_on, extension): # ... # } if not isinstance(depends_on, list): - raise Exception('dependsOn should be an array') + raise Exception('dependsOn should be an array ({0}) (got {1})'.format(extension.name, depends_on)) if not extension.supports_multi_config: # single-config - if len(depends_on) != 1: - raise Exception('dependsOn should be an array with exactly one item for single-config extensions') - extension.settings[0].dependencyLevel = depends_on[0]['dependencyLevel'] + length = len(depends_on) + if length > 1: + raise Exception('dependsOn should be an array with exactly one item for single-config extensions ({0}) (got {1})'.format(extension.name, depends_on)) + elif length == 0: + logger.warn('dependsOn is an empty array for extension {0}; setting the dependency level to 0'.format(extension.name)) + extension.settings[0].dependencyLevel = 0 + else: + extension.settings[0].dependencyLevel = depends_on[0]['dependencyLevel'] else: # multi-config settings_by_name = {} diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index cda6381f1..99d320b65 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -33,7 +33,7 @@ from azurelinuxagent.common.event import add_event, WALAEventOperation, report_event, \ CollectOrReportEventDebugInfo, add_periodic from azurelinuxagent.common.exception import ProtocolNotFoundError, \ - ResourceGoneError, ExtensionDownloadError, InvalidContainerError, ProtocolError, HttpError + ResourceGoneError, ExtensionDownloadError, InvalidContainerError, ProtocolError, HttpError, VmSettingsError from azurelinuxagent.common.future import httpclient, bytebuffer, ustr from azurelinuxagent.common.protocol.extensions_goal_state import ExtensionsGoalState, GoalStateMismatchError from azurelinuxagent.common.protocol.extensions_goal_state_factory import ExtensionsGoalStateFactory @@ -966,6 +966,16 @@ def get_vm_settings(): except ProtocolError: raise except Exception as exception: + if isinstance(exception, VmSettingsError): + message = format_message(ustr(exception)) + self._vm_settings_error_reporter.report_error(message) + try: + # pylint - Instance of 'Exception' has no 'vm_settings_text/etag' member (no-member) + # Disabled; the above check ensures the exception is a VmSettingsError + self._save_cache(exception.vm_settings_text, VM_SETTINGS_FILE_NAME.format(exception.etag)) # pylint: disable=no-member + except Exception as e: + # TODO: Once Fast Track is stable, make this a warning + logger.info("Failed to save vmSettings: {0}", ustr(e)) if isinstance(exception, IOError) and "timed out" in ustr(exception): message = format_message("Timeout") self._vm_settings_error_reporter.report_error(message, _VmSettingsError.Timeout) diff --git a/tests/common/test_exception.py b/tests/common/test_exception.py deleted file mode 100644 index a7acc5d07..000000000 --- a/tests/common/test_exception.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2019 Microsoft Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Requires Python 2.6+ and Openssl 1.0+ -# - -import inspect -import sys - -from azurelinuxagent.common.exception import AgentError -from tests.tools import AgentTestCase - - -class TestAgentError(AgentTestCase): - @classmethod - def setUpClass(cls): - AgentTestCase.setUpClass() - - cls.agent_exceptions = inspect.getmembers( - sys.modules["azurelinuxagent.common.exception"], - lambda member: inspect.isclass(member) and issubclass(member, AgentError)) - - def test_agent_exceptions_should_set_their_error_message(self): - for exception_name, exception_class in TestAgentError.agent_exceptions: - exception_instance = exception_class("A test Message") - - self.assertEqual("[{0}] A test Message".format(exception_name), str(exception_instance)) - - def test_agent_exceptions_should_include_the_inner_exception_in_their_error_message(self): - inner_exception = Exception("The inner exception") - - for exception_name, exception_class in TestAgentError.agent_exceptions: - exception_instance = exception_class("A test Message", inner_exception) - - self.assertEqual("[{0}] A test Message\nInner error: The inner exception".format(exception_name), str(exception_instance)) diff --git a/tests/data/hostgaplugin/ext_conf-empty_depends_on.xml b/tests/data/hostgaplugin/ext_conf-empty_depends_on.xml new file mode 100644 index 000000000..a29424ced --- /dev/null +++ b/tests/data/hostgaplugin/ext_conf-empty_depends_on.xml @@ -0,0 +1,56 @@ + + + + + + Prod + + https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml + https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml + + 2.5.0.2 + + + Test + + https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml + https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml + + 2.5.0.2 + + + CentralUSEUAP + CRP + + + + MultipleExtensionsPerHandler + + + https://dcrcqabsr1.blob.core.windows.net/$system/edpxmal5j1.058b176d-445b-4e75-bd97-4911511b7d96.status?sv=2018-03-28&sr=b&sk=system-1&sig=U4KaLxlyYfgQ%2fie8RCwgMBSXa3E4vlW0ozPYOEHikoc%3d&se=9999-01-01T00%3a00%3a00Z&sp=w + + + + https://umsafwzhkbm1rfrhl0ws.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml + + + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"commandToExecute":"echo '09cd27e9-fbd6-48ad-be86-55f3783e0a23'"} + } + } + ] + } + + + + https://dcrcqabsr1.blob.core.windows.net/$system/edpxmal5j1.058b176d-445b-4e75-bd97-4911511b7d96.vmSettings?sv=2018-03-28&sr=b&sk=system-1&sig=mOMtcUyao4oNPMtcVhQjzMK%2bmGSJS3Y1MIKOJPjqzus%3d&se=9999-01-01T00%3a00%3a00Z&sp=r + + \ No newline at end of file diff --git a/tests/data/hostgaplugin/vm_settings-empty_depends_on.json b/tests/data/hostgaplugin/vm_settings-empty_depends_on.json new file mode 100644 index 000000000..3f31b6f80 --- /dev/null +++ b/tests/data/hostgaplugin/vm_settings-empty_depends_on.json @@ -0,0 +1,68 @@ +{ + "hostGAPluginVersion": "1.0.8.115", + "vmSettingsSchemaVersion": "0.0", + "activityId": "2e7f8b5d-f637-4721-b757-cb190d49b4e9", + "correlationId": "1bef4c48-044e-4225-8f42-1d1eac1eb158", + "extensionsLastModifiedTickCount": 637693267431616449, + "extensionGoalStatesSource": "Fabric", + "StatusUploadBlob": { + "statusBlobType": "BlockBlob", + "value": "https://dcrcqabsr1.blob.core.windows.net/$system/edpxmal5j1.058b176d-445b-4e75-bd97-4911511b7d96.status?sv=2018-03-28&sr=b&sk=system-1&sig=U4KaLxlyYfgQ%2fie8RCwgMBSXa3E4vlW0ozPYOEHikoc%3d&se=9999-01-01T00%3a00%3a00Z&sp=w" + }, + "inVMMetadata": { + "subscriptionId": "8e037ad4-618f-4466-8bc8-5099d41ac15b", + "resourceGroupName": "rg-dc-qphvx25", + "vmName": "edpxmal5j1", + "location": "CentralUSEUAP", + "vmId": "058b176d-445b-4e75-bd97-4911511b7d96", + "vmSize": "Standard_D2s_v3", + "osType": "Linux" + }, + "requiredFeatures": [ + { + "name": "MultipleExtensionsPerHandler" + } + ], + "gaFamilies": [ + { + "Name": "Prod", + "Version": "2.5.0.2", + "Uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml" + ] + }, + { + "Name": "Test", + "Version": "2.5.0.2", + "Uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml" + ] + } + ], + "extensionGoalStates": [ + { + "name": "Microsoft.Azure.Extensions.CustomScript", + "version": "2.1.6", + "location": "https://umsavwggj2v40kvqhc0w.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml", + "failoverlocation": "https://umsanh4b5rfz0q0p4pwm.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml", + "additionalLocations": [ + "https://umsafwzhkbm1rfrhl0ws.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml" + ], + "state": "enabled", + "autoUpgrade": true, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "isMultiConfig": false, + "settings": [ + { + "publicSettings": "{\"commandToExecute\":\"echo '09cd27e9-fbd6-48ad-be86-55f3783e0a23'\"}" + } + ], + "dependsOn": [] + } + ] +} \ No newline at end of file diff --git a/tests/protocol/test_extensions_goal_state_from_extensions_config.py b/tests/protocol/test_extensions_goal_state_from_extensions_config.py index 0fb1ef138..6a7cee966 100644 --- a/tests/protocol/test_extensions_goal_state_from_extensions_config.py +++ b/tests/protocol/test_extensions_goal_state_from_extensions_config.py @@ -36,3 +36,12 @@ def test_it_should_parse_missing_status_upload_blob_as_none(self): self.assertIsNone(extensions_goal_state.status_upload_blob, "Expected status upload blob to be None") self.assertEqual("BlockBlob", extensions_goal_state.status_upload_blob_type, "Expected status upload blob to be Block") + def test_it_should_parse_empty_depends_on_as_dependency_level_0(self): + data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() + data_file["vm_settings"] = "hostgaplugin/vm_settings-empty_depends_on.json" + data_file["ext_conf"] = "hostgaplugin/ext_conf-empty_depends_on.xml" + with mock_wire_protocol(data_file) as protocol: + extensions = protocol.get_extensions_goal_state().extensions + + self.assertEqual(0, extensions[0].settings[0].dependencyLevel, "Incorrect dependencyLevel}") + diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py index 48e6c9b43..351558587 100644 --- a/tests/protocol/test_wire.py +++ b/tests/protocol/test_wire.py @@ -43,7 +43,7 @@ StatusBlob, VMStatus, EXT_CONF_FILE_NAME, _VmSettingsErrorReporter from azurelinuxagent.common.telemetryevent import GuestAgentExtensionEventsSchema, \ TelemetryEventParam, TelemetryEvent -from azurelinuxagent.common.utils import restutil, textutil +from azurelinuxagent.common.utils import restutil, textutil, fileutil from azurelinuxagent.common.version import CURRENT_VERSION, DISTRO_NAME, DISTRO_VERSION from azurelinuxagent.ga.exthandlers import get_exthandlers_handler from tests.ga.test_monitor import random_generator @@ -52,7 +52,7 @@ from tests.protocol.HttpRequestPredicates import HttpRequestPredicates from tests.protocol.mockwiredata import DATA_FILE_NO_EXT, DATA_FILE from tests.protocol.mockwiredata import WireProtocolData -from tests.tools import Mock, PropertyMock, patch, AgentTestCase, load_bin_data, mock_sleep +from tests.tools import Mock, PropertyMock, patch, AgentTestCase, load_bin_data, mock_sleep, load_data data_with_bom = b'\xef\xbb\xbfhehe' testurl = 'http://foo' @@ -1214,6 +1214,19 @@ def test_update_goal_state_should_save_goal_state(self): if e.name in ("Microsoft.Azure.Monitor.AzureMonitorLinuxAgent", "Microsoft.Azure.Security.Monitoring.AzureSecurityLinuxAgent"): self.assertEqual(e.settings[0].protectedSettings, "*** REDACTED ***", "The protected settings for {0} were not redacted".format(e.name)) + def test_update_goal_state_should_save_vm_settings_on_parse_errors(self): + invalid_vm_settings_file = "hostgaplugin/vm_settings-parse_error.json" + data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() + data_file["vm_settings"] = invalid_vm_settings_file + with mock_wire_protocol(data_file): + vm_settings_file = os.path.join(conf.get_lib_dir(), "VmSettings.1.json") + self.assertTrue(os.path.exists(vm_settings_file), "{0} was not saved".format(vm_settings_file)) + + expected = load_data(invalid_vm_settings_file) + actual = fileutil.read_file(vm_settings_file) + + self.assertEqual(expected, actual, "The vmSettings were not saved correctly") + def test_it_should_retry_get_vm_settings_on_resource_gone_error(self): # Requests to the hostgaplugin incude the Container ID and the RoleConfigName as headers; when the hostgaplugin returns GONE (HTTP status 410) the agent # needs to get a new goal state and retry the request with updated values for the Container ID and RoleConfigName headers. From ae5a22244a379841b8169c16ed19fc6124646748 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 11 Feb 2022 12:59:22 -0800 Subject: [PATCH 34/89] Set agent version to 2.7.0.5 (#2504) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index d13421568..7c34da0c2 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.7.0.4' +AGENT_VERSION = '2.7.0.5' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From a728b2e783c1b3c1a5522794646d2cdce2931623 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 11 Feb 2022 13:26:28 -0800 Subject: [PATCH 35/89] Revert "Set agent version to 2.7.0.5 (#2504)" (#2505) This reverts commit ae5a22244a379841b8169c16ed19fc6124646748. Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 7c34da0c2..d13421568 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.7.0.5' +AGENT_VERSION = '2.7.0.4' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 902e1834f75a9fffcbb2d568965996ea5f2530d3 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Fri, 11 Feb 2022 13:26:47 -0800 Subject: [PATCH 36/89] ignore firewall packets reset error, check enable firewall config flag and extend cgroup extension monitoring expiry time (#2502) --- azurelinuxagent/common/conf.py | 6 +- azurelinuxagent/common/osutil/default.py | 5 +- azurelinuxagent/ga/update.py | 3 + tests/common/osutil/test_default.py | 7 ++ tests/ga/test_update.py | 109 ++++++++++++----------- tests/test_agent.py | 2 +- 6 files changed, 73 insertions(+), 59 deletions(-) diff --git a/azurelinuxagent/common/conf.py b/azurelinuxagent/common/conf.py index 65d070309..ad6b939d5 100644 --- a/azurelinuxagent/common/conf.py +++ b/azurelinuxagent/common/conf.py @@ -160,7 +160,7 @@ def load_conf_from_file(conf_file_path, conf=__conf__): "ResourceDisk.MountOptions": None, "ResourceDisk.Filesystem": "ext3", "AutoUpdate.GAFamily": "Prod", - "Debug.CgroupMonitorExpiryTime": "2022-01-31", + "Debug.CgroupMonitorExpiryTime": "2022-03-31", "Debug.CgroupMonitorExtensionName": "Microsoft.Azure.Monitor.AzureMonitorLinuxAgent", } @@ -550,11 +550,11 @@ def get_agent_cpu_quota(conf=__conf__): def get_cgroup_monitor_expiry_time (conf=__conf__): """ - cgroups monitoring disabled after expiry time + cgroups monitoring for pilot extensions disabled after expiry time NOTE: This option is experimental and may be removed in later versions of the Agent. """ - return conf.get("Debug.CgroupMonitorExpiryTime", "2022-01-31") + return conf.get("Debug.CgroupMonitorExpiryTime", "2022-03-31") def get_cgroup_monitor_extension_name (conf=__conf__): """ diff --git a/azurelinuxagent/common/osutil/default.py b/azurelinuxagent/common/osutil/default.py index 1c6042165..4c706a9e7 100644 --- a/azurelinuxagent/common/osutil/default.py +++ b/azurelinuxagent/common/osutil/default.py @@ -179,9 +179,10 @@ def get_firewall_dropped_packets(self, dst_ip=None): return int(m.group(1)) except Exception as e: - if isinstance(e, CommandError) and e.returncode == 3: # pylint: disable=E1101 - # Transient error that we ignore. This code fires every loop + if isinstance(e, CommandError) and (e.returncode == 3 or e.returncode == 4): # pylint: disable=E1101 + # Transient error that we ignore returncode 3. This code fires every loop # of the daemon (60m), so we will get the value eventually. + # ignore returncode 4 as temporary fix (RULE_REPLACE failed (Invalid argument)) return 0 logger.warn("Failed to get firewall packets: {0}", ustr(e)) return -1 diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 293dd2313..cfac6f48c 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -1075,6 +1075,9 @@ def _ensure_firewall_rules_persisted(dst_ip): def _add_accept_tcp_firewall_rule_if_not_enabled(self, dst_ip): + if not conf.enable_firewall(): + return + def _execute_run_command(command): # Helper to execute a run command, returns True if no exception # Here we primarily check if an iptable rule exist. True if it exits , false if not diff --git a/tests/common/osutil/test_default.py b/tests/common/osutil/test_default.py index e34e3d4f4..5e61034a0 100644 --- a/tests/common/osutil/test_default.py +++ b/tests/common/osutil/test_default.py @@ -715,6 +715,13 @@ def test_get_firewall_dropped_packets_should_ignore_transient_errors(self): mock_iptables.set_command(osutil.get_firewall_packets_command(mock_iptables.wait), exit_code=3, output="can't initialize iptables table `security': iptables who? (do you need to insmod?)") self.assertEqual(0, osutil.DefaultOSUtil().get_firewall_dropped_packets()) + def test_get_firewall_dropped_packets_should_ignore_returncode_4(self): + + with TestOSUtil._mock_iptables() as mock_iptables: + with patch.object(osutil, '_enable_firewall', True): + mock_iptables.set_command(osutil.get_firewall_packets_command(mock_iptables.wait), exit_code=4, output="iptables v1.8.2 (nf_tables): RULE_REPLACE failed (Invalid argument): rule in chain OUTPUT") + self.assertEqual(0, osutil.DefaultOSUtil().get_firewall_dropped_packets()) + def test_get_firewall_dropped_packets(self): destination = '168.63.129.16' diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index 6213957b4..2bc627dda 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -1699,73 +1699,76 @@ def test_it_should_set_dns_tcp_iptable_if_drop_available_accept_unavailable(self with TestOSUtil._mock_iptables() as mock_iptables: with _get_update_handler(test_data=DATA_FILE) as (update_handler, _): - with patch.object(osutil, '_enable_firewall', True): - # drop rule is present - mock_iptables.set_command(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=0) - # non root tcp iptable rule is absent - mock_iptables.set_command(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=1) - update_handler.run(debug=True) - - drop_check_command = TestOSUtil._command_to_string(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) - accept_tcp_check_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) - accept_tcp_insert_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.INSERT_COMMAND, mock_iptables.destination)) - - # Filtering the mock iptable command calls with only the once related to this test. - filtered_mock_iptable_calls = [cmd for cmd in mock_iptables.command_calls if cmd in [drop_check_command, accept_tcp_check_rule, accept_tcp_insert_rule]] - - self.assertEqual(len(filtered_mock_iptable_calls), 3, "Incorrect number of calls to iptables: [{0}]".format(mock_iptables.command_calls)) - self.assertEqual(filtered_mock_iptable_calls[0], drop_check_command, - "The first command should check the drop rule") - self.assertEqual(filtered_mock_iptable_calls[1], accept_tcp_check_rule, - "The second command should check the accept rule") - self.assertEqual(filtered_mock_iptable_calls[2], accept_tcp_insert_rule, - "The third command should add the accept rule") + with patch('azurelinuxagent.common.conf.enable_firewall', return_value=True): + with patch.object(osutil, '_enable_firewall', True): + # drop rule is present + mock_iptables.set_command(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=0) + # non root tcp iptable rule is absent + mock_iptables.set_command(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=1) + update_handler.run(debug=True) + + drop_check_command = TestOSUtil._command_to_string(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) + accept_tcp_check_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) + accept_tcp_insert_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.INSERT_COMMAND, mock_iptables.destination)) + + # Filtering the mock iptable command calls with only the once related to this test. + filtered_mock_iptable_calls = [cmd for cmd in mock_iptables.command_calls if cmd in [drop_check_command, accept_tcp_check_rule, accept_tcp_insert_rule]] + + self.assertEqual(len(filtered_mock_iptable_calls), 3, "Incorrect number of calls to iptables: [{0}]".format(mock_iptables.command_calls)) + self.assertEqual(filtered_mock_iptable_calls[0], drop_check_command, + "The first command should check the drop rule") + self.assertEqual(filtered_mock_iptable_calls[1], accept_tcp_check_rule, + "The second command should check the accept rule") + self.assertEqual(filtered_mock_iptable_calls[2], accept_tcp_insert_rule, + "The third command should add the accept rule") def test_it_should_not_set_dns_tcp_iptable_if_drop_unavailable(self): with TestOSUtil._mock_iptables() as mock_iptables: with _get_update_handler(test_data=DATA_FILE) as (update_handler, _): - with patch.object(osutil, '_enable_firewall', True): - # drop rule is not available - mock_iptables.set_command(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=1) + with patch('azurelinuxagent.common.conf.enable_firewall', return_value=True): + with patch.object(osutil, '_enable_firewall', True): + # drop rule is not available + mock_iptables.set_command(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=1) - update_handler.run(debug=True) + update_handler.run(debug=True) - drop_check_command = TestOSUtil._command_to_string(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) - accept_tcp_check_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) - accept_tcp_insert_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.INSERT_COMMAND, mock_iptables.destination)) + drop_check_command = TestOSUtil._command_to_string(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) + accept_tcp_check_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) + accept_tcp_insert_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.INSERT_COMMAND, mock_iptables.destination)) - # Filtering the mock iptable command calls with only the once related to this test. - filtered_mock_iptable_calls = [cmd for cmd in mock_iptables.command_calls if cmd in [drop_check_command, accept_tcp_check_rule, accept_tcp_insert_rule]] + # Filtering the mock iptable command calls with only the once related to this test. + filtered_mock_iptable_calls = [cmd for cmd in mock_iptables.command_calls if cmd in [drop_check_command, accept_tcp_check_rule, accept_tcp_insert_rule]] - self.assertEqual(len(filtered_mock_iptable_calls), 1, "Incorrect number of calls to iptables: [{0}]".format(mock_iptables.command_calls)) - self.assertEqual(filtered_mock_iptable_calls[0], drop_check_command, - "The first command should check the drop rule") + self.assertEqual(len(filtered_mock_iptable_calls), 1, "Incorrect number of calls to iptables: [{0}]".format(mock_iptables.command_calls)) + self.assertEqual(filtered_mock_iptable_calls[0], drop_check_command, + "The first command should check the drop rule") def test_it_should_not_set_dns_tcp_iptable_if_drop_and_accept_available(self): with TestOSUtil._mock_iptables() as mock_iptables: with _get_update_handler(test_data=DATA_FILE) as (update_handler, _): - with patch.object(osutil, '_enable_firewall', True): - # drop rule is available - mock_iptables.set_command(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=0) - # non root tcp iptable rule is available - mock_iptables.set_command(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=0) - - update_handler.run(debug=True) - - drop_check_command = TestOSUtil._command_to_string(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) - accept_tcp_check_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) - accept_tcp_insert_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.INSERT_COMMAND, mock_iptables.destination)) - - # Filtering the mock iptable command calls with only the once related to this test. - filtered_mock_iptable_calls = [cmd for cmd in mock_iptables.command_calls if cmd in [drop_check_command, accept_tcp_check_rule, accept_tcp_insert_rule]] - - self.assertEqual(len(filtered_mock_iptable_calls), 2, "Incorrect number of calls to iptables: [{0}]".format(mock_iptables.command_calls)) - self.assertEqual(filtered_mock_iptable_calls[0], drop_check_command, - "The first command should check the drop rule") - self.assertEqual(filtered_mock_iptable_calls[1], accept_tcp_check_rule, - "The second command should check the accept rule") + with patch('azurelinuxagent.common.conf.enable_firewall', return_value=True): + with patch.object(osutil, '_enable_firewall', True): + # drop rule is available + mock_iptables.set_command(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=0) + # non root tcp iptable rule is available + mock_iptables.set_command(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=0) + + update_handler.run(debug=True) + + drop_check_command = TestOSUtil._command_to_string(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) + accept_tcp_check_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) + accept_tcp_insert_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.INSERT_COMMAND, mock_iptables.destination)) + + # Filtering the mock iptable command calls with only the once related to this test. + filtered_mock_iptable_calls = [cmd for cmd in mock_iptables.command_calls if cmd in [drop_check_command, accept_tcp_check_rule, accept_tcp_insert_rule]] + + self.assertEqual(len(filtered_mock_iptable_calls), 2, "Incorrect number of calls to iptables: [{0}]".format(mock_iptables.command_calls)) + self.assertEqual(filtered_mock_iptable_calls[0], drop_check_command, + "The first command should check the drop rule") + self.assertEqual(filtered_mock_iptable_calls[1], accept_tcp_check_rule, + "The second command should check the accept rule") @contextlib.contextmanager def _setup_test_for_ext_event_dirs_retention(self): diff --git a/tests/test_agent.py b/tests/test_agent.py index 1ce321290..6cf0632b3 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -36,7 +36,7 @@ Debug.CgroupDisableOnProcessCheckFailure = True Debug.CgroupDisableOnQuotaCheckFailure = True Debug.CgroupLogMetrics = False -Debug.CgroupMonitorExpiryTime = 2022-01-31 +Debug.CgroupMonitorExpiryTime = 2022-03-31 Debug.CgroupMonitorExtensionName = Microsoft.Azure.Monitor.AzureMonitorLinuxAgent Debug.EnableFastTrack = True Debug.EnableGAVersioning = False From 0c77fe02066174f1cc7cf515e31c67234e8007d2 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 11 Feb 2022 13:47:43 -0800 Subject: [PATCH 37/89] Set agent version to 2.7.0.5 (#2506) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index d13421568..7c34da0c2 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.7.0.4' +AGENT_VERSION = '2.7.0.5' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 9305e7f2811e24faf9198a93e397ce8023e3e3fb Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Mon, 14 Feb 2022 17:22:39 -0800 Subject: [PATCH 38/89] Handle OOM errors by stopping the periodic log collector (#2510) --- azurelinuxagent/common/logcollector.py | 1 + azurelinuxagent/ga/collect_logs.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/azurelinuxagent/common/logcollector.py b/azurelinuxagent/common/logcollector.py index 84055777b..776f77b5b 100644 --- a/azurelinuxagent/common/logcollector.py +++ b/azurelinuxagent/common/logcollector.py @@ -45,6 +45,7 @@ CGROUPS_UNIT = "collect-logs.scope" +FORCE_KILLED_ERRCODE = -9 INVALID_CGROUPS_ERRCODE = 2 _MUST_COLLECT_FILES = [ diff --git a/azurelinuxagent/ga/collect_logs.py b/azurelinuxagent/ga/collect_logs.py index e494d03d1..b26e260ca 100644 --- a/azurelinuxagent/ga/collect_logs.py +++ b/azurelinuxagent/ga/collect_logs.py @@ -203,6 +203,12 @@ def exec_command(output_file): if e.returncode == logcollector.INVALID_CGROUPS_ERRCODE: # pylint: disable=no-member logger.info("Disabling periodic log collection until service restart due to process error.") self.stop() + + # When the OOM killer is invoked on the log collector process, this error code is + # returned. Stop the periodic operation because it seems to be persistent. + elif e.returncode == logcollector.FORCE_KILLED_ERRCODE: # pylint: disable=no-member + logger.info("Disabling periodic log collection until service restart due to OOM error.") + self.stop() else: logger.info(err_msg) From cb0fb4596d14721d7a5027defe3b68a0060a567b Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Mon, 14 Feb 2022 17:28:54 -0800 Subject: [PATCH 39/89] Set agent version to 2.7.0.6 (#2511) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 7c34da0c2..721cad6d7 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.7.0.5' +AGENT_VERSION = '2.7.0.6' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 47770b71b4965b1b302f02df91350839b7db24f9 Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Mon, 14 Mar 2022 15:34:30 -0700 Subject: [PATCH 40/89] Add keep_alive property to collect_logs (#2533) --- azurelinuxagent/common/interfaces.py | 9 +++++++++ azurelinuxagent/ga/collect_logs.py | 3 +++ azurelinuxagent/ga/update.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/azurelinuxagent/common/interfaces.py b/azurelinuxagent/common/interfaces.py index d6b0753bb..41b16e668 100644 --- a/azurelinuxagent/common/interfaces.py +++ b/azurelinuxagent/common/interfaces.py @@ -30,6 +30,15 @@ def get_thread_name(): def run(self): raise NotImplementedError("run() not implemented") + def keep_alive(self): + """ + Returns true if the thread handler should be restarted when the thread dies + and false when it should remain dead. + + Defaults to True and can be overridden by sub-classes. + """ + return True + def is_alive(self): raise NotImplementedError("is_alive() not implemented") diff --git a/azurelinuxagent/ga/collect_logs.py b/azurelinuxagent/ga/collect_logs.py index b26e260ca..dc62fccf2 100644 --- a/azurelinuxagent/ga/collect_logs.py +++ b/azurelinuxagent/ga/collect_logs.py @@ -107,6 +107,9 @@ def __init__(self): def run(self): self.start() + def keep_alive(self): + return self.should_run + def is_alive(self): return self.event_thread.is_alive() diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index cfac6f48c..2b0e857b4 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -421,7 +421,7 @@ def _check_daemon_running(self, debug): def _check_threads_running(self, all_thread_handlers): # Check that all the threads are still running for thread_handler in all_thread_handlers: - if not thread_handler.is_alive(): + if thread_handler.keep_alive() and not thread_handler.is_alive(): logger.warn("{0} thread died, restarting".format(thread_handler.get_thread_name())) thread_handler.start() From fed399f394a925d08732588d2d5082e848feb2e6 Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Mon, 14 Mar 2022 16:18:44 -0700 Subject: [PATCH 41/89] Set agent version to 2.7.1.0. (#2534) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 721cad6d7..71dfe2937 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.7.0.6' +AGENT_VERSION = '2.7.1.0' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From a5bb8695f84ac7e5aa86bd971f0d97b7472a889c Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 12 Apr 2022 11:14:52 -0700 Subject: [PATCH 42/89] Set agent version to 2.8.0.0 (#2545) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index ff9c903b9..85a0d5624 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -207,9 +207,9 @@ def has_logrotate(): # IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise # DCR may test the wrong agent version. # -# When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 +# When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '9.9.9.9' +AGENT_VERSION = '2.8.0.0' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 50eaac1de1d707e4e0fcd74616cfef80ae54d463 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 13 Apr 2022 13:08:59 -0700 Subject: [PATCH 43/89] Update HGAP minimum required version for FastTrack (#2549) Co-authored-by: narrieta --- azurelinuxagent/common/protocol/hostplugin.py | 5 ++--- .../vm_settings-difference_in_required_features.json | 2 +- tests/data/hostgaplugin/vm_settings-empty_depends_on.json | 2 +- tests/data/hostgaplugin/vm_settings-invalid_blob_type.json | 2 +- .../hostgaplugin/vm_settings-no_extension_manifests.json | 2 +- .../data/hostgaplugin/vm_settings-no_status_upload_blob.json | 2 +- tests/data/hostgaplugin/vm_settings-out-of-sync.json | 2 +- tests/data/hostgaplugin/vm_settings-parse_error.json | 2 +- tests/data/hostgaplugin/vm_settings-requested_version.json | 2 +- tests/data/hostgaplugin/vm_settings.json | 2 +- 10 files changed, 11 insertions(+), 12 deletions(-) diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py index cf254be30..81b906256 100644 --- a/azurelinuxagent/common/protocol/hostplugin.py +++ b/azurelinuxagent/common/protocol/hostplugin.py @@ -547,9 +547,8 @@ def format_message(msg): logger.info(message) add_event(op=WALAEventOperation.HostPlugin, message=message, is_success=True) - # Don't support HostGAPlugin versions older than 123 - # TODO: update the minimum version to 1.0.8.123 before release - if vm_settings.host_ga_plugin_version < FlexibleVersion("1.0.8.117"): + # Don't support HostGAPlugin versions older than 124 + if vm_settings.host_ga_plugin_version < FlexibleVersion("1.0.8.124"): raise_not_supported() self._supports_vm_settings = True diff --git a/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json b/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json index 9cfb42752..9fd2e4f72 100644 --- a/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json +++ b/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings-empty_depends_on.json b/tests/data/hostgaplugin/vm_settings-empty_depends_on.json index 6fa93452c..94d9f0eb1 100644 --- a/tests/data/hostgaplugin/vm_settings-empty_depends_on.json +++ b/tests/data/hostgaplugin/vm_settings-empty_depends_on.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "2e7f8b5d-f637-4721-b757-cb190d49b4e9", "correlationId": "1bef4c48-044e-4225-8f42-1d1eac1eb158", diff --git a/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json b/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json index 62314a403..e7945845a 100644 --- a/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json +++ b/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "2e7f8b5d-f637-4721-b757-cb190d49b4e9", "correlationId": "1bef4c48-044e-4225-8f42-1d1eac1eb158", diff --git a/tests/data/hostgaplugin/vm_settings-no_extension_manifests.json b/tests/data/hostgaplugin/vm_settings-no_extension_manifests.json index 7deff8d5e..b084900b6 100644 --- a/tests/data/hostgaplugin/vm_settings-no_extension_manifests.json +++ b/tests/data/hostgaplugin/vm_settings-no_extension_manifests.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "89d50bf1-fa55-4257-8af3-3db0c9f81ab4", "correlationId": "c143f8f0-a66b-4881-8c06-1efd278b0b02", diff --git a/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json b/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json index 27ebebcef..2f70b5576 100644 --- a/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json +++ b/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings-out-of-sync.json b/tests/data/hostgaplugin/vm_settings-out-of-sync.json index 737350d69..0aae82031 100644 --- a/tests/data/hostgaplugin/vm_settings-out-of-sync.json +++ b/tests/data/hostgaplugin/vm_settings-out-of-sync.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "AAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE", "correlationId": "EEEEEEEE-DDDD-CCCC-BBBB-AAAAAAAAAAAA", diff --git a/tests/data/hostgaplugin/vm_settings-parse_error.json b/tests/data/hostgaplugin/vm_settings-parse_error.json index e817a1e88..bae5de4cb 100644 --- a/tests/data/hostgaplugin/vm_settings-parse_error.json +++ b/tests/data/hostgaplugin/vm_settings-parse_error.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": THIS_IS_A_SYNTAX_ERROR, "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings-requested_version.json b/tests/data/hostgaplugin/vm_settings-requested_version.json index 1b5023b11..49e6a2778 100644 --- a/tests/data/hostgaplugin/vm_settings-requested_version.json +++ b/tests/data/hostgaplugin/vm_settings-requested_version.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings.json b/tests/data/hostgaplugin/vm_settings.json index b67ee0a23..96836e766 100644 --- a/tests/data/hostgaplugin/vm_settings.json +++ b/tests/data/hostgaplugin/vm_settings.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", From 9e0bb7b5f2177eb2ae2a5bd8736f2c6f4489ff50 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 13 Apr 2022 13:18:44 -0700 Subject: [PATCH 44/89] Update agent version to 2.8.0.1 (#2550) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 85a0d5624..1c3b0639a 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.8.0.0' +AGENT_VERSION = '2.8.0.1' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From fa8e3708328de50c899e68b6cfc519c1bda40361 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 19 Apr 2022 16:51:49 -0700 Subject: [PATCH 45/89] Improvements in waagent.log (#2558) * Improvements in waagent.log * fix function call * update comment * typo Co-authored-by: narrieta --- .../extensions_goal_state_from_vm_settings.py | 2 +- azurelinuxagent/common/protocol/goal_state.py | 29 +++++--- azurelinuxagent/common/utils/archive.py | 9 ++- azurelinuxagent/daemon/main.py | 2 +- azurelinuxagent/ga/exthandlers.py | 3 +- azurelinuxagent/ga/update.py | 36 ++++++---- .../ext_conf-requested_version.xml | 4 +- tests/data/hostgaplugin/ext_conf.xml | 4 +- ...tings-difference_in_required_features.json | 4 +- .../vm_settings-missing_cert.json | 68 +++++++++++++++++++ .../hostgaplugin/vm_settings-out-of-sync.json | 2 +- .../vm_settings-requested_version.json | 4 +- tests/data/hostgaplugin/vm_settings.json | 6 +- tests/protocol/test_goal_state.py | 22 +++++- tests/utils/test_archive.py | 3 +- 15 files changed, 154 insertions(+), 44 deletions(-) create mode 100644 tests/data/hostgaplugin/vm_settings-missing_cert.json diff --git a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py index ce99a2607..38cca48f1 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py @@ -289,7 +289,7 @@ def _parse_extensions(self, vm_settings): # "settingsSeqNo": 0, # "settings": [ # { - # "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + # "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", # "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", # "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" # } diff --git a/azurelinuxagent/common/protocol/goal_state.py b/azurelinuxagent/common/protocol/goal_state.py index ae01e4d22..6ec0a5ab6 100644 --- a/azurelinuxagent/common/protocol/goal_state.py +++ b/azurelinuxagent/common/protocol/goal_state.py @@ -18,6 +18,7 @@ import os import re import time +import json import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger @@ -145,6 +146,7 @@ def update(self): return if vm_settings_updated: + logger.info('') logger.info("Fetched new vmSettings [HostGAPlugin correlation ID: {0} eTag: {1} source: {2}]", vm_settings.hostga_plugin_correlation_id, vm_settings.etag, vm_settings.source) # Ignore the vmSettings if their source is Fabric (processing a Fabric goal state may require the tenant certificate and the vmSettings don't include it.) if vm_settings is not None and vm_settings.source == GoalStateSource.Fabric: @@ -184,6 +186,17 @@ def update(self): if self._extensions_goal_state is None or most_recent.created_on_timestamp > self._extensions_goal_state.created_on_timestamp: self._extensions_goal_state = most_recent + # For Fast Track goal states, verify that the required certificates are in the goal state + if self.extensions_goal_state.source == GoalStateSource.FastTrack: + for extension in self.extensions_goal_state.extensions: + for settings in extension.settings: + if settings.protectedSettings is None: + continue + certificates = self.certs.summary + if not any(settings.certificateThumbprint == c['thumbprint'] for c in certificates): + message = "Certificate {0} needed by {1} is missing from the goal state".format(settings.certificateThumbprint, extension.name) + add_event(op=WALAEventOperation.VmSettings, message=message, is_success=False) + def _restore_wire_server_goal_state(self, incarnation, xml_text, xml_doc, vm_settings_support_stopped_error): logger.info('The HGAP stopped supporting vmSettings; will fetched the goal state from the WireServer.') self._history = GoalStateHistory(timeutil.create_timestamp(), incarnation) @@ -270,6 +283,7 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): Returns the value of ExtensionsConfig. """ try: + logger.info('') logger.info('Fetching full goal state from the WireServer [incarnation {0}]', incarnation) role_instance = find(xml_doc, "RoleInstance") @@ -300,9 +314,10 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): certs = None certs_uri = findtext(xml_doc, "Certificates") if certs_uri is not None: - # Note that we do not save the certificates to the goal state history xml_text = self._wire_client.fetch_config(certs_uri, self._wire_client.get_header_for_cert()) certs = Certificates(xml_text) + # Save the certificate summary, which includes only the thumbprint but not the certificate itself, to the goal state history + self._history.save_certificates(json.dumps(certs.summary)) remote_access = None remote_access_uri = findtext(container, "RemoteAccessInfo") @@ -349,6 +364,7 @@ def __init__(self, xml_text): class Certificates(object): def __init__(self, xml_text): self.cert_list = CertList() + self.summary = [] # debugging info # Save the certificates local_file = os.path.join(conf.get_lib_dir(), CERTS_FILE_NAME) @@ -428,18 +444,15 @@ def __init__(self, xml_text): tmp_file = prvs[pubkey] prv = "{0}.prv".format(thumbprint) os.rename(tmp_file, os.path.join(conf.get_lib_dir(), prv)) - logger.info("Found private key matching thumbprint {0}".format(thumbprint)) else: # Since private key has *no* matching certificate, # it will not be named correctly logger.warn("Found NO matching cert/thumbprint for private key!") - # Log if any certificates were found without matching private keys - # This can happen (rarely), and is useful to know for debugging - for pubkey in thumbprints: - if not pubkey in prvs: - msg = "Certificate with thumbprint {0} has no matching private key." - logger.info(msg.format(thumbprints[pubkey])) + for pubkey, thumbprint in thumbprints.items(): + has_private_key = pubkey in prvs + logger.info("Downloaded certificate with thumbprint {0} (has private key: {1})".format(thumbprint, has_private_key)) + self.summary.append({"thumbprint": thumbprint, "hasPrivateKey": has_private_key}) for v1_cert in v1_cert_list: cert = Cert() diff --git a/azurelinuxagent/common/utils/archive.py b/azurelinuxagent/common/utils/archive.py index ed8122e97..880a23a11 100644 --- a/azurelinuxagent/common/utils/archive.py +++ b/azurelinuxagent/common/utils/archive.py @@ -42,13 +42,14 @@ _MAX_ARCHIVED_STATES = 50 _CACHE_PATTERNS = [ - re.compile(r"^VmSettings.\d+\.json$"), + re.compile(r"^VmSettings\.\d+\.json$"), re.compile(r"^(.*)\.(\d+)\.(agentsManifest)$", re.IGNORECASE), re.compile(r"^(.*)\.(\d+)\.(manifest\.xml)$", re.IGNORECASE), re.compile(r"^(.*)\.(\d+)\.(xml)$", re.IGNORECASE), re.compile(r"^SharedConfig\.xml$", re.IGNORECASE), re.compile(r"^HostingEnvironmentConfig\.xml$", re.IGNORECASE), - re.compile(r"^RemoteAccess\.xml$", re.IGNORECASE) + re.compile(r"^RemoteAccess\.xml$", re.IGNORECASE), + re.compile(r"^waagent_status\.\d+\.json$"), ] # @@ -69,6 +70,7 @@ _GOAL_STATE_FILE_NAME = "GoalState.xml" _VM_SETTINGS_FILE_NAME = "VmSettings.json" +_CERTIFICATES_FILE_NAME = "Certificates.json" _HOSTING_ENV_FILE_NAME = "HostingEnvironmentConfig.xml" _SHARED_CONF_FILE_NAME = "SharedConfig.xml" _REMOTE_ACCESS_FILE_NAME = "RemoteAccess.xml" @@ -239,6 +241,9 @@ def save_vm_settings(self, text): def save_remote_access(self, text): self.save(text, _REMOTE_ACCESS_FILE_NAME) + def save_certificates(self, text): + self.save(text, _CERTIFICATES_FILE_NAME) + def save_hosting_env(self, text): self.save(text, _HOSTING_ENV_FILE_NAME) diff --git a/azurelinuxagent/daemon/main.py b/azurelinuxagent/daemon/main.py index 91685bc64..c608768a6 100644 --- a/azurelinuxagent/daemon/main.py +++ b/azurelinuxagent/daemon/main.py @@ -64,7 +64,7 @@ def run(self, child_args=None): # # Be aware that telemetry events emitted before that will not include the Container ID. # - logger.info("{0} Version:{1}", AGENT_LONG_NAME, AGENT_VERSION) + logger.info("{0} Version: {1}", AGENT_LONG_NAME, AGENT_VERSION) logger.info("OS: {0} {1}", DISTRO_NAME, DISTRO_VERSION) logger.info("Python: {0}.{1}.{2}", PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO) diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index cc9d0afc3..3e8dbc23d 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -308,6 +308,7 @@ def run(self): error = None message = "ProcessExtensionsGoalState started [{0} channel: {1} source: {2} activity: {3} correlation {4} created: {5}]".format( egs.id, egs.channel, egs.source, egs.activity_id, egs.correlation_id, egs.created_on_timestamp) + logger.info('') logger.info(message) add_event(op=WALAEventOperation.ExtensionProcessing, message=message) @@ -319,7 +320,7 @@ def run(self): finally: duration = elapsed_milliseconds(utc_start) if error is None: - message = 'ProcessExtensionsGoalState completed [{0} {1} ms]'.format(egs.id, duration) + message = 'ProcessExtensionsGoalState completed [{0} {1} ms]\n'.format(egs.id, duration) logger.info(message) else: message = 'ProcessExtensionsGoalState failed [{0} {1} ms]\n{2}'.format(egs.id, duration, error) diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 2118ca683..9932674ea 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -54,7 +54,7 @@ from azurelinuxagent.common.utils.flexible_version import FlexibleVersion from azurelinuxagent.common.utils.networkutil import AddFirewallRules from azurelinuxagent.common.utils.shellutil import CommandError -from azurelinuxagent.common.version import AGENT_NAME, AGENT_DIR_PATTERN, CURRENT_AGENT, \ +from azurelinuxagent.common.version import AGENT_LONG_NAME, AGENT_NAME, AGENT_DIR_PATTERN, CURRENT_AGENT, \ CURRENT_VERSION, DISTRO_NAME, DISTRO_VERSION, get_lis_version, \ has_logrotate, PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO, get_daemon_version from azurelinuxagent.ga.collect_logs import get_collect_logs_handler, is_log_collection_allowed @@ -324,20 +324,11 @@ def run(self, debug=False): """ try: + logger.info("{0} Version: {1}", AGENT_LONG_NAME, CURRENT_AGENT) + logger.info("OS: {0} {1}", DISTRO_NAME, DISTRO_VERSION) + logger.info("Python: {0}.{1}.{2}", PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO) logger.info(u"Agent {0} is running as the goal state agent", CURRENT_AGENT) - # - # Initialize the goal state; some components depend on information provided by the goal state and this - # call ensures the required info is initialized (e.g. telemetry depends on the container ID.) - # - protocol = self.protocol_util.get_protocol() - - self._initialize_goal_state(protocol) - - # Initialize the common parameters for telemetry events - initialize_event_logger_vminfo_common_parameters(protocol) - - # Log OS-specific info. os_info_msg = u"Distro: {dist_name}-{dist_ver}; "\ u"OSUtil: {util_name}; AgentService: {service_name}; "\ u"Python: {py_major}.{py_minor}.{py_micro}; "\ @@ -351,8 +342,20 @@ def run(self, debug=False): py_micro=PY_VERSION_MICRO, systemd=systemd.is_systemd(), lis_ver=get_lis_version(), has_logrotate=has_logrotate() ) - logger.info(os_info_msg) + + # + # Initialize the goal state; some components depend on information provided by the goal state and this + # call ensures the required info is initialized (e.g. telemetry depends on the container ID.) + # + protocol = self.protocol_util.get_protocol() + + self._initialize_goal_state(protocol) + + # Initialize the common parameters for telemetry events + initialize_event_logger_vminfo_common_parameters(protocol) + + # Send telemetry for the OS-specific info. add_event(AGENT_NAME, op=WALAEventOperation.OSInfo, message=os_info_msg) # @@ -734,7 +737,7 @@ def forward_signal(self, signum, frame): return logger.info( - u"Agent {0} forwarding signal {1} to {2}", + u"Agent {0} forwarding signal {1} to {2}\n", CURRENT_AGENT, signum, self.child_agent.name if self.child_agent is not None else CURRENT_AGENT) @@ -823,6 +826,9 @@ def log_if_op_disabled(name, value): if conf.get_autoupdate_enabled(): log_if_int_changed_from_default("Autoupdate.Frequency", conf.get_autoupdate_frequency()) + if conf.get_enable_fast_track(): + log_if_op_disabled("Debug.EnableFastTrack", conf.get_enable_fast_track()) + if conf.get_lib_dir() != "/var/lib/waagent": log_event("lib dir is in an unexpected location: {0}".format(conf.get_lib_dir())) diff --git a/tests/data/hostgaplugin/ext_conf-requested_version.xml b/tests/data/hostgaplugin/ext_conf-requested_version.xml index c3bd92823..bbb8a20fe 100644 --- a/tests/data/hostgaplugin/ext_conf-requested_version.xml +++ b/tests/data/hostgaplugin/ext_conf-requested_version.xml @@ -60,7 +60,7 @@ "runtimeSettings": [ { "handlerSettings": { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": {"GCS_AUTO_CONFIG":true} } @@ -73,7 +73,7 @@ "runtimeSettings": [ { "handlerSettings": { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": {"enableGenevaUpload":true} } diff --git a/tests/data/hostgaplugin/ext_conf.xml b/tests/data/hostgaplugin/ext_conf.xml index eac5d6364..ebd90aa0b 100644 --- a/tests/data/hostgaplugin/ext_conf.xml +++ b/tests/data/hostgaplugin/ext_conf.xml @@ -58,7 +58,7 @@ "runtimeSettings": [ { "handlerSettings": { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/Microsoft.Azure.Monitor.AzureMonitorLinuxAgent==", "publicSettings": {"GCS_AUTO_CONFIG":true} } @@ -71,7 +71,7 @@ "runtimeSettings": [ { "handlerSettings": { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/Microsoft.Azure.Security.Monitoring.AzureSecurityLinuxAgent==", "publicSettings": {"enableGenevaUpload":true} } diff --git a/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json b/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json index 9fd2e4f72..560126870 100644 --- a/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json +++ b/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json @@ -56,7 +56,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" } @@ -76,7 +76,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": "{\"enableGenevaUpload\":true}" } diff --git a/tests/data/hostgaplugin/vm_settings-missing_cert.json b/tests/data/hostgaplugin/vm_settings-missing_cert.json new file mode 100644 index 000000000..a7192e942 --- /dev/null +++ b/tests/data/hostgaplugin/vm_settings-missing_cert.json @@ -0,0 +1,68 @@ +{ + "hostGAPluginVersion": "1.0.8.124", + "vmSettingsSchemaVersion": "0.0", + "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", + "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", + "inSvdSeqNo": 1, + "extensionsLastModifiedTickCount": 637726657706205299, + "extensionGoalStatesSource": "FastTrack", + "onHold": true, + "statusUploadBlob": { + "statusBlobType": "BlockBlob", + "value": "https://dcrcl3a0xs.blob.core.windows.net/$system/edp0plkw2b.86f4ae0a-61f8-48ae-9199-40f402d56864.status?sv=2018-03-28&sr=b&sk=system-1&sig=KNWgC2%3d&se=9999-01-01T00%3a00%3a00Z&sp=w" + }, + "inVMMetadata": { + "subscriptionId": "8e037ad4-618f-4466-8bc8-5099d41ac15b", + "resourceGroupName": "rg-dc-86fjzhp", + "vmName": "edp0plkw2b", + "location": "CentralUSEUAP", + "vmId": "86f4ae0a-61f8-48ae-9199-40f402d56864", + "vmSize": "Standard_B2s", + "osType": "Linux" + }, + "requiredFeatures": [ + { + "name": "MultipleExtensionsPerHandler" + } + ], + "gaFamilies": [ + { + "name": "Prod", + "uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml" + ] + }, + { + "name": "Test", + "uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml" + ] + } + ], + "extensionGoalStates": [ + { + "name": "Microsoft.OSTCExtensions.VMAccessForLinux", + "version": "1.5.11", + "location": "https://umsasc25p0kjg0c1dg4b.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml", + "failoverlocation": "https://umsamfwlmfshvxx2lsjm.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml", + "additionalLocations": [ + "https://umsah3cwjlctnmhsvzqv.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml" + ], + "state": "enabled", + "autoUpgrade": false, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "isMultiConfig": false, + "settings": [ + { + "protectedSettingsCertThumbprint": "59A10F50FFE2A0408D3F03FE336C8FD5716CF25C", + "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpddesZQewdDBgegkxNzA1BgoJkgergres/Microsoft.OSTCExtensions.VMAccessForLinux==" + } + ] + } + ] +} diff --git a/tests/data/hostgaplugin/vm_settings-out-of-sync.json b/tests/data/hostgaplugin/vm_settings-out-of-sync.json index 0aae82031..1f369ae5b 100644 --- a/tests/data/hostgaplugin/vm_settings-out-of-sync.json +++ b/tests/data/hostgaplugin/vm_settings-out-of-sync.json @@ -56,7 +56,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" } diff --git a/tests/data/hostgaplugin/vm_settings-requested_version.json b/tests/data/hostgaplugin/vm_settings-requested_version.json index 49e6a2778..98959dd4e 100644 --- a/tests/data/hostgaplugin/vm_settings-requested_version.json +++ b/tests/data/hostgaplugin/vm_settings-requested_version.json @@ -56,7 +56,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" } @@ -74,7 +74,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": "{\"enableGenevaUpload\":true}" } diff --git a/tests/data/hostgaplugin/vm_settings.json b/tests/data/hostgaplugin/vm_settings.json index 96836e766..a4ef0f785 100644 --- a/tests/data/hostgaplugin/vm_settings.json +++ b/tests/data/hostgaplugin/vm_settings.json @@ -56,7 +56,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/Microsoft.Azure.Monitor.AzureMonitorLinuxAgent==", "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" } @@ -76,7 +76,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/Microsoft.Azure.Security.Monitoring.AzureSecurityLinuxAgent==", "publicSettings": "{\"enableGenevaUpload\":true}" } @@ -192,7 +192,7 @@ "isMultiConfig": false, "settings": [ { - "protectedSettingsCertThumbprint": "59A10F50FFE2A0408D3F03FE336C8FD5716CF25C", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpddesZQewdDBgegkxNzA1BgoJkgergres/Microsoft.OSTCExtensions.VMAccessForLinux==" } ] diff --git a/tests/protocol/test_goal_state.py b/tests/protocol/test_goal_state.py index 65d050d86..8cfa4842a 100644 --- a/tests/protocol/test_goal_state.py +++ b/tests/protocol/test_goal_state.py @@ -8,6 +8,7 @@ import re import time +from azurelinuxagent.common.event import WALAEventOperation from azurelinuxagent.common.future import httpclient from azurelinuxagent.common.protocol.extensions_goal_state import GoalStateSource, GoalStateChannel from azurelinuxagent.common.protocol.extensions_goal_state_from_extensions_config import ExtensionsGoalStateFromExtensionsConfig @@ -105,7 +106,7 @@ def test_instantiating_goal_state_should_save_the_goal_state_to_the_history_dire self._assert_directory_contents( self._find_history_subdirectory("999-888"), - ["GoalState.xml", "ExtensionsConfig.xml", "VmSettings.json", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) + ["GoalState.xml", "ExtensionsConfig.xml", "VmSettings.json", "Certificates.json", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) def _find_history_subdirectory(self, tag): matches = glob.glob(os.path.join(self.tmp_dir, ARCHIVE_DIRECTORY_NAME, "*_{0}".format(tag))) @@ -128,7 +129,7 @@ def test_update_should_create_new_history_subdirectories(self): goal_state = GoalState(protocol.client) self._assert_directory_contents( self._find_history_subdirectory("123-654"), - ["GoalState.xml", "ExtensionsConfig.xml", "VmSettings.json", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) + ["GoalState.xml", "ExtensionsConfig.xml", "VmSettings.json", "Certificates.json", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) def http_get_handler(url, *_, **__): if HttpRequestPredicates.is_host_plugin_vm_settings_request(url): @@ -140,7 +141,7 @@ def http_get_handler(url, *_, **__): goal_state.update() self._assert_directory_contents( self._find_history_subdirectory("234-654"), - ["GoalState.xml", "ExtensionsConfig.xml", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) + ["GoalState.xml", "ExtensionsConfig.xml", "Certificates.json", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) protocol.mock_wire_data.set_etag(987) protocol.set_http_handlers(http_get_handler=None) @@ -358,3 +359,18 @@ def http_get_handler(url, *_, **__): self._assert_goal_state(goal_state, initial_incarnation, channel=GoalStateChannel.WireServer, source=GoalStateSource.Fabric) self.assertEqual(initial_timestamp, goal_state.extensions_goal_state.created_on_timestamp, "The timestamp of the updated goal state is incorrect") self.assertTrue(goal_state.extensions_goal_state.is_outdated, "The updated goal state should be marked as outdated") + + def test_it_should_report_missing_certificates(self): + data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() + data_file["vm_settings"] = "hostgaplugin/vm_settings-missing_cert.json" + + with mock_wire_protocol(data_file) as protocol: + with patch("azurelinuxagent.common.protocol.goal_state.add_event") as add_event: + _ = GoalState(protocol.client) + + expected_message = "Certificate 59A10F50FFE2A0408D3F03FE336C8FD5716CF25C needed by Microsoft.OSTCExtensions.VMAccessForLinux is missing from the goal state" + events = [kwargs for _, kwargs in add_event.call_args_list if kwargs['op'] == WALAEventOperation.VmSettings and kwargs['message'] == expected_message] + + self.assertTrue( + len(events) == 1, + "Missing certificate 59A10F50FFE2A0408D3F03FE336C8FD5716CF25C was note reported. Telemetry: {0}".format([kwargs['message'] for _, kwargs in add_event.call_args_list])) diff --git a/tests/utils/test_archive.py b/tests/utils/test_archive.py index 466d674c7..61214558c 100644 --- a/tests/utils/test_archive.py +++ b/tests/utils/test_archive.py @@ -135,7 +135,8 @@ def test_purge_legacy_goal_state_history(self): 'Microsoft.Azure.Extensions.CustomScript.1.xml', 'SharedConfig.xml', 'HostingEnvironmentConfig.xml', - 'RemoteAccess.xml' + 'RemoteAccess.xml', + 'waagent_status.1.json' ] legacy_files = [os.path.join(self.tmp_dir, f) for f in legacy_files] for f in legacy_files: From 93a2564fd6509a93ddab1417507a61f40ba56424 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 20 Apr 2022 15:43:45 -0700 Subject: [PATCH 46/89] Change format of history items (#2560) * Change format of history directory * Update message; fix typo * py2 compat * py2 compat Co-authored-by: narrieta --- azurelinuxagent/common/protocol/goal_state.py | 9 +++++---- azurelinuxagent/common/utils/archive.py | 19 +++++++++++-------- azurelinuxagent/common/utils/timeutil.py | 11 ++++++++++- azurelinuxagent/ga/update.py | 5 ++--- tests/protocol/test_goal_state.py | 2 +- tests/utils/test_archive.py | 10 ++++++++-- 6 files changed, 37 insertions(+), 19 deletions(-) diff --git a/azurelinuxagent/common/protocol/goal_state.py b/azurelinuxagent/common/protocol/goal_state.py index 6ec0a5ab6..da0de9a03 100644 --- a/azurelinuxagent/common/protocol/goal_state.py +++ b/azurelinuxagent/common/protocol/goal_state.py @@ -15,6 +15,7 @@ # limitations under the License. # # Requires Python 2.6+ and Openssl 1.0+ +import datetime import os import re import time @@ -31,7 +32,7 @@ from azurelinuxagent.common.protocol.extensions_goal_state import VmSettingsParseError, GoalStateSource from azurelinuxagent.common.protocol.hostplugin import VmSettingsNotSupported, VmSettingsSupportStopped from azurelinuxagent.common.protocol.restapi import Cert, CertList, RemoteAccessUser, RemoteAccessUsersList -from azurelinuxagent.common.utils import fileutil, timeutil +from azurelinuxagent.common.utils import fileutil from azurelinuxagent.common.utils.archive import GoalStateHistory from azurelinuxagent.common.utils.cryptutil import CryptUtil from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, findtext, getattrib @@ -131,7 +132,7 @@ def update(self): # # Fetch the goal state from both the HGAP and the WireServer # - timestamp = timeutil.create_timestamp() + timestamp = datetime.datetime.utcnow() incarnation, xml_text, xml_doc = GoalState._fetch_goal_state(self._wire_client) goal_state_updated = incarnation != self._incarnation @@ -199,7 +200,7 @@ def update(self): def _restore_wire_server_goal_state(self, incarnation, xml_text, xml_doc, vm_settings_support_stopped_error): logger.info('The HGAP stopped supporting vmSettings; will fetched the goal state from the WireServer.') - self._history = GoalStateHistory(timeutil.create_timestamp(), incarnation) + self._history = GoalStateHistory(datetime.datetime.utcnow(), incarnation) self._history.save_goal_state(xml_text) self._extensions_goal_state = self._fetch_full_wire_server_goal_state(incarnation, xml_doc) if self._extensions_goal_state.created_on_timestamp < vm_settings_support_stopped_error.timestamp: @@ -270,7 +271,7 @@ def _fetch_vm_settings(wire_client): except VmSettingsParseError as exception: # ensure we save the vmSettings if there were parsing errors, but save them only once per ETag if not GoalStateHistory.tag_exists(exception.etag): - GoalStateHistory(timeutil.create_timestamp(), exception.etag).save_vm_settings(exception.vm_settings_text) + GoalStateHistory(datetime.datetime.utcnow(), exception.etag).save_vm_settings(exception.vm_settings_text) raise return vm_settings, vm_settings_updated diff --git a/azurelinuxagent/common/utils/archive.py b/azurelinuxagent/common/utils/archive.py index 880a23a11..6123fdb0d 100644 --- a/azurelinuxagent/common/utils/archive.py +++ b/azurelinuxagent/common/utils/archive.py @@ -9,7 +9,7 @@ import azurelinuxagent.common.logger as logger import azurelinuxagent.common.conf as conf -from azurelinuxagent.common.utils import fileutil +from azurelinuxagent.common.utils import fileutil, timeutil # pylint: disable=W0105 @@ -58,13 +58,15 @@ # 2018-04-06T08:21:37.142697.zip # 2018-04-06T08:21:37.142697_incarnation_N # 2018-04-06T08:21:37.142697_incarnation_N.zip +# 2018-04-06T08:21:37.142697_N-M +# 2018-04-06T08:21:37.142697_N-M.zip # # Current names # -# 2018-04-06T08:21:37.142697_N-M -# 2018-04-06T08:21:37.142697_N-M.zip +# 2018-04-06T08-21-37__N-M +# 2018-04-06T08-21-37__N-M.zip # -_ARCHIVE_BASE_PATTERN = r"\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}\.\d+((_incarnation)?_(\d+|status)(-\d+)?)?" +_ARCHIVE_BASE_PATTERN = r"\d{4}\-\d{2}\-\d{2}T\d{2}[:-]\d{2}[:-]\d{2}(\.\d+)?((_incarnation)?_+(\d+|status)(-\d+)?)?" _ARCHIVE_PATTERNS_DIRECTORY = re.compile(r'^{0}$'.format(_ARCHIVE_BASE_PATTERN)) _ARCHIVE_PATTERNS_ZIP = re.compile(r'^{0}\.zip$'.format(_ARCHIVE_BASE_PATTERN)) @@ -163,7 +165,6 @@ def purge(self): newest ones. Also, clean up any legacy history files. """ states = self._get_archive_states() - states.sort(reverse=True) for state in states[_MAX_ARCHIVED_STATES:]: state.delete() @@ -184,7 +185,6 @@ def purge_legacy_goal_state_history(): def archive(self): states = self._get_archive_states() - states.sort(reverse=True) if len(states) > 0: # Skip the most recent goal state, since it may still be in use @@ -203,13 +203,16 @@ def _get_archive_states(self): if match is not None: states.append(StateZip(full_path, match.group(0))) + states.sort(key=lambda state: os.path.getctime(state._path), reverse=True) + return states class GoalStateHistory(object): - def __init__(self, timestamp, tag): + def __init__(self, time, tag): self._errors = False - self._root = os.path.join(conf.get_lib_dir(), ARCHIVE_DIRECTORY_NAME, "{0}_{1}".format(timestamp, tag) if tag is not None else timestamp) + timestamp = timeutil.create_history_timestamp(time) + self._root = os.path.join(conf.get_lib_dir(), ARCHIVE_DIRECTORY_NAME, "{0}__{1}".format(timestamp, tag) if tag is not None else timestamp) @staticmethod def tag_exists(tag): diff --git a/azurelinuxagent/common/utils/timeutil.py b/azurelinuxagent/common/utils/timeutil.py index c4dd755a0..c8fa37647 100644 --- a/azurelinuxagent/common/utils/timeutil.py +++ b/azurelinuxagent/common/utils/timeutil.py @@ -5,7 +5,7 @@ def create_timestamp(dt=None): """ - Returns a string with the given datetime iso format. If no datetime is given as parameter, it + Returns a string with the given datetime in iso format. If no datetime is given as parameter, it uses datetime.utcnow(). """ if dt is None: @@ -13,6 +13,15 @@ def create_timestamp(dt=None): return dt.isoformat() +def create_history_timestamp(dt=None): + """ + Returns a string with the given datetime formatted as a timestamp for the agent's history folder + """ + if dt is None: + dt = datetime.datetime.utcnow() + return dt.strftime('%Y-%m-%dT%H-%M-%S') + + def datetime_to_ticks(dt): """ Converts 'dt', a datetime, to the number of ticks (1 tick == 1/10000000 sec) since datetime.min (0001-01-01 00:00:00). diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 9932674ea..bf0b53993 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -54,7 +54,7 @@ from azurelinuxagent.common.utils.flexible_version import FlexibleVersion from azurelinuxagent.common.utils.networkutil import AddFirewallRules from azurelinuxagent.common.utils.shellutil import CommandError -from azurelinuxagent.common.version import AGENT_LONG_NAME, AGENT_NAME, AGENT_DIR_PATTERN, CURRENT_AGENT, \ +from azurelinuxagent.common.version import AGENT_LONG_NAME, AGENT_NAME, AGENT_DIR_PATTERN, CURRENT_AGENT, AGENT_VERSION, \ CURRENT_VERSION, DISTRO_NAME, DISTRO_VERSION, get_lis_version, \ has_logrotate, PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO, get_daemon_version from azurelinuxagent.ga.collect_logs import get_collect_logs_handler, is_log_collection_allowed @@ -324,10 +324,9 @@ def run(self, debug=False): """ try: - logger.info("{0} Version: {1}", AGENT_LONG_NAME, CURRENT_AGENT) + logger.info("{0} (Goal State Agent version {1})", AGENT_LONG_NAME, AGENT_VERSION) logger.info("OS: {0} {1}", DISTRO_NAME, DISTRO_VERSION) logger.info("Python: {0}.{1}.{2}", PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO) - logger.info(u"Agent {0} is running as the goal state agent", CURRENT_AGENT) os_info_msg = u"Distro: {dist_name}-{dist_ver}; "\ u"OSUtil: {util_name}; AgentService: {service_name}; "\ diff --git a/tests/protocol/test_goal_state.py b/tests/protocol/test_goal_state.py index 8cfa4842a..c54a65f9f 100644 --- a/tests/protocol/test_goal_state.py +++ b/tests/protocol/test_goal_state.py @@ -373,4 +373,4 @@ def test_it_should_report_missing_certificates(self): self.assertTrue( len(events) == 1, - "Missing certificate 59A10F50FFE2A0408D3F03FE336C8FD5716CF25C was note reported. Telemetry: {0}".format([kwargs['message'] for _, kwargs in add_event.call_args_list])) + "Missing certificate 59A10F50FFE2A0408D3F03FE336C8FD5716CF25C was not reported. Telemetry: {0}".format([kwargs['message'] for _, kwargs in add_event.call_args_list])) diff --git a/tests/utils/test_archive.py b/tests/utils/test_archive.py index 61214558c..0c649c9e2 100644 --- a/tests/utils/test_archive.py +++ b/tests/utils/test_archive.py @@ -67,7 +67,10 @@ def test_archive_should_zip_all_but_the_latest_goal_state_in_the_history_folder( test_directories.append(directory) test_subject = StateArchiver(self.tmp_dir) - test_subject.archive() + # NOTE: StateArchiver sorts the state directories by creation time, but the test files are created too fast and the + # time resolution is too coarse, so instead we mock getctime to simply return the path of the file + with patch("azurelinuxagent.common.utils.archive.os.path.getctime", side_effect=lambda path: path): + test_subject.archive() for directory in test_directories[0:2]: zip_file = directory + ".zip" @@ -110,7 +113,10 @@ def test_archive02(self): self.assertEqual(total, len(os.listdir(self.history_dir))) test_subject = StateArchiver(self.tmp_dir) - test_subject.purge() + # NOTE: StateArchiver sorts the state directories by creation time, but the test files are created too fast and the + # time resolution is too coarse, so instead we mock getctime to simply return the path of the file + with patch("azurelinuxagent.common.utils.archive.os.path.getctime", side_effect=lambda path: path): + test_subject.purge() archived_entries = os.listdir(self.history_dir) self.assertEqual(_MAX_ARCHIVED_STATES, len(archived_entries)) From a3d761f6c8187ab6f2f2deda839eb31419043ab3 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 20 Apr 2022 16:31:13 -0700 Subject: [PATCH 47/89] Update agent version to 2.8.0.2 (#2561) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 1c3b0639a..b6fb5f228 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.8.0.1' +AGENT_VERSION = '2.8.0.2' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From cd03ff2c22c572fcafb74d645ed5a437a2c3bf5d Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 25 Apr 2022 13:00:51 -0700 Subject: [PATCH 48/89] Refresh goal state when certificates are missing (#2562) * Refresh goal state when certificates are missing * Improve error reporting * Fix assert message Co-authored-by: narrieta --- azurelinuxagent/common/logger.py | 4 + azurelinuxagent/common/protocol/goal_state.py | 100 ++++++++++++------ azurelinuxagent/common/protocol/hostplugin.py | 4 +- azurelinuxagent/common/protocol/wire.py | 12 +-- azurelinuxagent/ga/update.py | 52 +++++---- tests/data/wire/certs-2.xml | 85 +++++++++++++++ tests/data/wire/goal_state.xml | 12 +-- tests/data/wire/goal_state_no_ext.xml | 10 +- tests/data/wire/goal_state_remote_access.xml | 13 +-- tests/ga/test_update.py | 48 ++++----- tests/protocol/HttpRequestPredicates.py | 16 +++ tests/protocol/mockwiredata.py | 28 ++--- tests/protocol/test_goal_state.py | 56 ++++++++-- tests/protocol/test_hostplugin.py | 12 +-- tests/protocol/test_wire.py | 2 +- 15 files changed, 323 insertions(+), 131 deletions(-) create mode 100644 tests/data/wire/certs-2.xml diff --git a/azurelinuxagent/common/logger.py b/azurelinuxagent/common/logger.py index 07e3f2393..3d0dc617d 100644 --- a/azurelinuxagent/common/logger.py +++ b/azurelinuxagent/common/logger.py @@ -45,6 +45,7 @@ def __init__(self, logger=None, prefix=None): self.logger = self if logger is None else logger self.periodic_messages = {} self.prefix = prefix + self.silent = False def reset_periodic(self): self.logger.periodic_messages = {} @@ -124,6 +125,9 @@ def write_log(log_appender): # pylint: disable=W0612 finally: log_appender.appender_lock = False + if self.silent: + return + # if msg_format is not unicode convert it to unicode if type(msg_format) is not ustr: msg_format = ustr(msg_format, errors="backslashreplace") diff --git a/azurelinuxagent/common/protocol/goal_state.py b/azurelinuxagent/common/protocol/goal_state.py index da0de9a03..edfd9d14f 100644 --- a/azurelinuxagent/common/protocol/goal_state.py +++ b/azurelinuxagent/common/protocol/goal_state.py @@ -48,8 +48,16 @@ _GET_GOAL_STATE_MAX_ATTEMPTS = 6 +class GoalStateInconsistentError(ProtocolError): + """ + Indicates an inconsistency in the goal state (e.g. missing tenant certificate) + """ + def __init__(self, msg, inner=None): + super(GoalStateInconsistentError, self).__init__(msg, inner) + + class GoalState(object): - def __init__(self, wire_client): + def __init__(self, wire_client, silent=False): """ Fetches the goal state using the given wire client. @@ -64,6 +72,8 @@ def __init__(self, wire_client): self._wire_client = wire_client self._history = None self._extensions_goal_state = None # populated from vmSettings or extensionsConfig + self.logger = logger.Logger(logger.DEFAULT_LOGGER) + self.logger.silent = silent # These properties hold the goal state from the WireServer and are initialized by self._fetch_full_wire_server_goal_state() self._incarnation = None @@ -75,8 +85,10 @@ def __init__(self, wire_client): self._certs = None self._remote_access = None - self.update() + self.update(silent=silent) + except ProtocolError: + raise except Exception as exception: # We don't log the error here since fetching the goal state is done every few seconds raise ProtocolError(msg="Error fetching goal state", inner=exception) @@ -125,34 +137,47 @@ def update_host_plugin_headers(wire_client): # Fetching the goal state updates the HostGAPlugin so simply trigger the request GoalState._fetch_goal_state(wire_client) - def update(self): + def update(self, silent=False): """ Updates the current GoalState instance fetching values from the WireServer/HostGAPlugin as needed """ + self.logger.silent = silent + + try: + self._update(force_update=False) + except GoalStateInconsistentError as e: + self.logger.warn("Detected an inconsistency in the goal state: {0}", ustr(e)) + self._update(force_update=True) + self.logger.info("The goal state is consistent") + + def _update(self, force_update): # # Fetch the goal state from both the HGAP and the WireServer # timestamp = datetime.datetime.utcnow() + if force_update: + self.logger.info("Refreshing goal state and vmSettings") + incarnation, xml_text, xml_doc = GoalState._fetch_goal_state(self._wire_client) - goal_state_updated = incarnation != self._incarnation + goal_state_updated = force_update or incarnation != self._incarnation if goal_state_updated: - logger.info('Fetched a new incarnation for the WireServer goal state [incarnation {0}]', incarnation) + self.logger.info('Fetched a new incarnation for the WireServer goal state [incarnation {0}]', incarnation) vm_settings, vm_settings_updated = None, False try: - vm_settings, vm_settings_updated = GoalState._fetch_vm_settings(self._wire_client) + vm_settings, vm_settings_updated = GoalState._fetch_vm_settings(self._wire_client, force_update=force_update) except VmSettingsSupportStopped as exception: # If the HGAP stopped supporting vmSettings, we need to use the goal state from the WireServer self._restore_wire_server_goal_state(incarnation, xml_text, xml_doc, exception) return if vm_settings_updated: - logger.info('') - logger.info("Fetched new vmSettings [HostGAPlugin correlation ID: {0} eTag: {1} source: {2}]", vm_settings.hostga_plugin_correlation_id, vm_settings.etag, vm_settings.source) + self.logger.info('') + self.logger.info("Fetched new vmSettings [HostGAPlugin correlation ID: {0} eTag: {1} source: {2}]", vm_settings.hostga_plugin_correlation_id, vm_settings.etag, vm_settings.source) # Ignore the vmSettings if their source is Fabric (processing a Fabric goal state may require the tenant certificate and the vmSettings don't include it.) if vm_settings is not None and vm_settings.source == GoalStateSource.Fabric: if vm_settings_updated: - logger.info("The vmSettings originated via Fabric; will ignore them.") + self.logger.info("The vmSettings originated via Fabric; will ignore them.") vm_settings, vm_settings_updated = None, False # If neither goal state has changed we are done with the update @@ -187,19 +212,30 @@ def update(self): if self._extensions_goal_state is None or most_recent.created_on_timestamp > self._extensions_goal_state.created_on_timestamp: self._extensions_goal_state = most_recent - # For Fast Track goal states, verify that the required certificates are in the goal state + # + # For Fast Track goal states, verify that the required certificates are in the goal state. + # + # Some scenarios can produce inconsistent goal states. For example, during hibernation/resume, the Fabric goal state changes (the + # tenant certificate is re-generated when the VM is restarted) *without* the incarnation necessarily changing (e.g. if the incarnation + # is 1 before the hibernation; on resume the incarnation is set to 1 even though the goal state has a new certificate). If a Fast + # Track goal state comes after that, the extensions will need the new certificate. The Agent needs to refresh the goal state in that + # case, to ensure it fetches the new certificate. + # if self.extensions_goal_state.source == GoalStateSource.FastTrack: - for extension in self.extensions_goal_state.extensions: - for settings in extension.settings: - if settings.protectedSettings is None: - continue - certificates = self.certs.summary - if not any(settings.certificateThumbprint == c['thumbprint'] for c in certificates): - message = "Certificate {0} needed by {1} is missing from the goal state".format(settings.certificateThumbprint, extension.name) - add_event(op=WALAEventOperation.VmSettings, message=message, is_success=False) + self._check_certificates() + + def _check_certificates(self): + for extension in self.extensions_goal_state.extensions: + for settings in extension.settings: + if settings.protectedSettings is None: + continue + certificates = self.certs.summary + if not any(settings.certificateThumbprint == c['thumbprint'] for c in certificates): + message = "Certificate {0} needed by {1} is missing from the goal state".format(settings.certificateThumbprint, extension.name) + raise GoalStateInconsistentError(message) def _restore_wire_server_goal_state(self, incarnation, xml_text, xml_doc, vm_settings_support_stopped_error): - logger.info('The HGAP stopped supporting vmSettings; will fetched the goal state from the WireServer.') + self.logger.info('The HGAP stopped supporting vmSettings; will fetched the goal state from the WireServer.') self._history = GoalStateHistory(datetime.datetime.utcnow(), incarnation) self._history.save_goal_state(xml_text) self._extensions_goal_state = self._fetch_full_wire_server_goal_state(incarnation, xml_doc) @@ -207,7 +243,7 @@ def _restore_wire_server_goal_state(self, incarnation, xml_text, xml_doc, vm_set self._extensions_goal_state.is_outdated = True msg = "Fetched a Fabric goal state older than the most recent FastTrack goal state; will skip it.\nFabric: {0}\nFastTrack: {1}".format( self._extensions_goal_state.created_on_timestamp, vm_settings_support_stopped_error.timestamp) - logger.info(msg) + self.logger.info(msg) add_event(op=WALAEventOperation.VmSettings, message=msg, is_success=True) def save_to_history(self, data, file_name): @@ -249,7 +285,7 @@ def _fetch_goal_state(wire_client): return incarnation, xml_text, xml_doc @staticmethod - def _fetch_vm_settings(wire_client): + def _fetch_vm_settings(wire_client, force_update=False): """ Issues an HTTP request (HostGAPlugin) for the vm settings and returns the response as an ExtensionsGoalState. """ @@ -258,11 +294,11 @@ def _fetch_vm_settings(wire_client): if conf.get_enable_fast_track(): try: try: - vm_settings, vm_settings_updated = wire_client.get_host_plugin().fetch_vm_settings() + vm_settings, vm_settings_updated = wire_client.get_host_plugin().fetch_vm_settings(force_update=force_update) except ResourceGoneError: # retry after refreshing the HostGAPlugin GoalState.update_host_plugin_headers(wire_client) - vm_settings, vm_settings_updated = wire_client.get_host_plugin().fetch_vm_settings() + vm_settings, vm_settings_updated = wire_client.get_host_plugin().fetch_vm_settings(force_update=force_update) except VmSettingsSupportStopped: raise @@ -284,8 +320,8 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): Returns the value of ExtensionsConfig. """ try: - logger.info('') - logger.info('Fetching full goal state from the WireServer [incarnation {0}]', incarnation) + self.logger.info('') + self.logger.info('Fetching full goal state from the WireServer [incarnation {0}]', incarnation) role_instance = find(xml_doc, "RoleInstance") role_instance_id = findtext(role_instance, "InstanceId") @@ -317,7 +353,11 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): if certs_uri is not None: xml_text = self._wire_client.fetch_config(certs_uri, self._wire_client.get_header_for_cert()) certs = Certificates(xml_text) - # Save the certificate summary, which includes only the thumbprint but not the certificate itself, to the goal state history + # Log and save the certificates summary (i.e. the thumbprint but not the certificate itself) to the goal state history + for c in certs.summary: + logger.info("Downloaded certificate {0}".format(c)) + if len(certs.warnings) > 0: + logger.warn(certs.warnings) self._history.save_certificates(json.dumps(certs.summary)) remote_access = None @@ -339,10 +379,10 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): return extensions_config except Exception as exception: - logger.warn("Fetching the goal state failed: {0}", ustr(exception)) + self.logger.warn("Fetching the goal state failed: {0}", ustr(exception)) raise ProtocolError(msg="Error fetching goal state", inner=exception) finally: - logger.info('Fetch goal state completed') + self.logger.info('Fetch goal state completed') class HostingEnv(object): @@ -366,6 +406,7 @@ class Certificates(object): def __init__(self, xml_text): self.cert_list = CertList() self.summary = [] # debugging info + self.warnings = [] # Save the certificates local_file = os.path.join(conf.get_lib_dir(), CERTS_FILE_NAME) @@ -448,11 +489,10 @@ def __init__(self, xml_text): else: # Since private key has *no* matching certificate, # it will not be named correctly - logger.warn("Found NO matching cert/thumbprint for private key!") + self.warnings.append("Found NO matching cert/thumbprint for private key!") for pubkey, thumbprint in thumbprints.items(): has_private_key = pubkey in prvs - logger.info("Downloaded certificate with thumbprint {0} (has private key: {1})".format(thumbprint, has_private_key)) self.summary.append({"thumbprint": thumbprint, "hasPrivateKey": has_private_key}) for v1_cert in v1_cert_list: diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py index 81b906256..5f795dca3 100644 --- a/azurelinuxagent/common/protocol/hostplugin.py +++ b/azurelinuxagent/common/protocol/hostplugin.py @@ -453,7 +453,7 @@ def get_fast_track_timestamp(): HostPluginProtocol._get_fast_track_state_file(), ustr(e)) return timeutil.create_timestamp(datetime.datetime.utcnow()) - def fetch_vm_settings(self): + def fetch_vm_settings(self, force_update=False): """ Queries the vmSettings from the HostGAPlugin and returns an (ExtensionsGoalState, bool) tuple with the vmSettings and a boolean indicating if they are an updated (True) or a cached value (False). @@ -491,7 +491,7 @@ def format_message(msg): # Raise VmSettingsNotSupported directly instead of using raise_not_supported() to avoid resetting the timestamp for the next check raise VmSettingsNotSupported() - etag = None if self._cached_vm_settings is None else self._cached_vm_settings.etag + etag = None if force_update or self._cached_vm_settings is None else self._cached_vm_settings.etag correlation_id = str(uuid.uuid4()) self._vm_settings_error_reporter.report_request() diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 40e58cc0f..7923bea75 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -83,8 +83,8 @@ def detect(self): logger.info('Initializing goal state during protocol detection') self.client.update_goal_state(force_update=True) - def update_goal_state(self): - self.client.update_goal_state() + def update_goal_state(self, silent=False): + self.client.update_goal_state(silent=silent) def update_host_plugin_from_goal_state(self): self.client.update_host_plugin_from_goal_state() @@ -759,18 +759,18 @@ def update_host_plugin(self, container_id, role_config_name): self._host_plugin.update_container_id(container_id) self._host_plugin.update_role_config_name(role_config_name) - def update_goal_state(self, force_update=False): + def update_goal_state(self, force_update=False, silent=False): """ Updates the goal state if the incarnation or etag changed or if 'force_update' is True """ try: if force_update: - logger.info("Forcing an update of the goal state..") + logger.info("Forcing an update of the goal state.") if self._goal_state is None or force_update: - self._goal_state = GoalState(self) + self._goal_state = GoalState(self, silent=silent) else: - self._goal_state.update() + self._goal_state.update(silent=silent) except ProtocolError: raise diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index bf0b53993..cbf0ac38b 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -168,7 +168,8 @@ def __init__(self): # these members are used to avoid reporting errors too frequently self._heartbeat_update_goal_state_error_count = 0 - self._last_try_update_goal_state_failed = False + self._update_goal_state_error_count = 0 + self._update_goal_state_last_error_report = datetime.min self._report_status_last_failed_goal_state = None # incarnation of the last goal state that has been fully processed @@ -481,13 +482,16 @@ def _try_update_goal_state(self, protocol): Attempts to update the goal state and returns True on success or False on failure, sending telemetry events about the failures. """ try: - protocol.update_goal_state() + max_errors_to_log = 3 + + protocol.update_goal_state(silent=self._update_goal_state_error_count >= max_errors_to_log) self._goal_state = protocol.get_goal_state() - if self._last_try_update_goal_state_failed: - self._last_try_update_goal_state_failed = False - message = u"Retrieving the goal state recovered from previous errors" + if self._update_goal_state_error_count > 0: + self._update_goal_state_error_count = 0 + message = u"Fetching the goal state recovered from previous errors. Fetched {0} (certificates: {1})".format( + self._goal_state.extensions_goal_state.id, self._goal_state.certs.summary) add_event(AGENT_NAME, op=WALAEventOperation.FetchGoalState, version=CURRENT_VERSION, is_success=True, message=message, log_event=False) logger.info(message) @@ -497,15 +501,21 @@ def _try_update_goal_state(self, protocol): self._supports_fast_track = False except Exception as e: - if not self._last_try_update_goal_state_failed: - self._last_try_update_goal_state_failed = True - message = u"An error occurred while retrieving the goal state: {0}".format(textutil.format_exception(e)) - logger.warn(message) - add_event(AGENT_NAME, op=WALAEventOperation.FetchGoalState, version=CURRENT_VERSION, is_success=False, message=message, log_event=False) - message = u"Attempts to retrieve the goal state are failing: {0}".format(ustr(e)) - logger.periodic_warn(logger.EVERY_SIX_HOURS, "[PERIODIC] {0}".format(message)) + self._update_goal_state_error_count += 1 self._heartbeat_update_goal_state_error_count += 1 + if self._update_goal_state_error_count <= max_errors_to_log: + message = u"Error fetching the goal state: {0}".format(textutil.format_exception(e)) + logger.error(message) + add_event(op=WALAEventOperation.FetchGoalState, is_success=False, message=message, log_event=False) + self._update_goal_state_last_error_report = datetime.now() + else: + if self._update_goal_state_last_error_report + timedelta(hours=6) > datetime.now(): + self._update_goal_state_last_error_report = datetime.now() + message = u"Fetching the goal state is still failing: {0}".format(textutil.format_exception(e)) + logger.error(message) + add_event(op=WALAEventOperation.FetchGoalState, is_success=False, message=message, log_event=False) return False + return True def __update_guest_agent(self, protocol): @@ -559,8 +569,8 @@ def handle_updates_for_requested_version(): raise AgentUpgradeExitException( "Exiting current process to {0} to the request Agent version {1}".format(prefix, requested_version)) - # Ignore new agents if updating is disabled - if not conf.get_autoupdate_enabled(): + # Skip the update if there is no goal state yet or auto-update is disabled + if self._goal_state is None or not conf.get_autoupdate_enabled(): return False if self._download_agent_if_upgrade_available(protocol): @@ -600,11 +610,14 @@ def _process_goal_state(self, exthandlers_handler, remote_access_handler): protocol = exthandlers_handler.protocol # update self._goal_state - self._try_update_goal_state(protocol) - - # Update the Guest Agent if a new version is available - if self._goal_state is not None: + if not self._try_update_goal_state(protocol): + # agent updates and status reporting should be done even when the goal state is not updated self.__update_guest_agent(protocol) + self._report_status(exthandlers_handler) + return + + # check for agent updates + self.__update_guest_agent(protocol) if self._processing_new_extensions_goal_state(): if not self._extensions_summary.converged: @@ -616,8 +629,7 @@ def _process_goal_state(self, exthandlers_handler, remote_access_handler): self._extensions_summary = ExtensionsSummary() exthandlers_handler.run() - # always report status, even if the goal state did not change - # do it before processing the remote access, since that operation can take a long time + # report status before processing the remote access, since that operation can take a long time self._report_status(exthandlers_handler) if self._processing_new_incarnation(): diff --git a/tests/data/wire/certs-2.xml b/tests/data/wire/certs-2.xml new file mode 100644 index 000000000..66a231ee8 --- /dev/null +++ b/tests/data/wire/certs-2.xml @@ -0,0 +1,85 @@ + + + 2012-11-30 + 5 + Pkcs7BlobWithPfxContents + MIIOgwYJKoZIhvcNAQcDoIIOdDCCDnACAQIxggEwMIIBLAIBAoAUiF8ZYMs9mMa8 +QOEMxDaIhGza+0IwDQYJKoZIhvcNAQEBBQAEggEAQW7GyeRVEhHSU1/dzV0IndH0 +rDQk+27MvlsWTcpNcgGFtfRYxu5bzmp0+DoimX3pRBlSFOpMJ34jpg4xs78EsSWH +FRhCf3EGuEUBHo6yR8FhXDTuS7kZ0UmquiCI2/r8j8gbaGBNeP8IRizcAYrPMA5S +E8l1uCrw7DHuLscbVni/7UglGaTfFS3BqS5jYbiRt2Qh3p+JPUfm51IG3WCIw/WS +2QHebmHxvMFmAp8AiBWSQJizQBEJ1lIfhhBMN4A7NadMWAe6T2DRclvdrQhJX32k +amOiogbW4HJsL6Hphn7Frrw3CENOdWMAvgQBvZ3EjAXgsJuhBA1VIrwofzlDljCC +DTUGCSqGSIb3DQEHATAUBggqhkiG9w0DBwQIxcvw9qx4y0qAgg0QrINXpC23BWT2 +Fb9N8YS3Be9eO3fF8KNdM6qGf0kKR16l/PWyP2L+pZxCcCPk83d070qPdnJK9qpJ +6S1hI80Y0oQnY9VBFrdfkc8fGZHXqm5jNS9G32v/AxYpJJC/qrAQnWuOdLtOZaGL +94GEh3XRagvz1wifv8SRI8B1MzxrpCimeMxHkL3zvJFg9FjLGdrak868feqhr6Nb +pqH9zL7bMq8YP788qTRELUnL72aDzGAM7HEj7V4yu2uD3i3Ryz3bqWaj9IF38Sa0 +6rACBkiNfZBPgExoMUm2GNVyx8hTis2XKRgz4NLh29bBkKrArK9sYDncE9ocwrrX +AQ99yn03Xv6TH8bRp0cSj4jzBXc5RFsUQG/LxzJVMjvnkDbwNE41DtFiYz5QVcv1 +cMpTH16YfzSL34a479eNq/4+JAs/zcb2wjBskJipMUU4hNx5fhthvfKwDOQbLTqN +HcP23iPQIhjdUXf6gpu5RGu4JZ0dAMHMHFKvNL6TNejwx/H6KAPp6rCRsYi6QhAb +42SXdZmhAyQsFpGD9U5ieJApqeCHfj9Xhld61GqLJA9+WLVhDPADjqHoAVvrOkKH +OtPegId/lWnCB7p551klAjiEA2/DKxFBIAEhqZpiLl+juZfMXovkdmGxMP4gvNNF +gbS2k5A0IJ8q51gZcH1F56smdAmi5kvhPnFdy/9gqeI/F11F1SkbPVLImP0mmrFi +zQD5JGfEu1psUYvhpOdaYDkmAK5qU5xHSljqZFz5hXNt4ebvSlurHAhunJb2ln3g +AJUHwtZnVBrtYMB0w6fdwYqMxXi4vLeqUiHtIQtbOq32zlSryNPQqG9H0iP9l/G1 +t7oUfr9woI/B0kduaY9jd5Qtkqs1DoyfNMSaPNohUK/CWOTD51qOadzSvK0hJ+At +033PFfv9ilaX6GmzHdEVEanrn9a+BoBCnGnuysHk/8gdswj9OzeCemyIFJD7iObN +rNex3SCf3ucnAejJOA0awaLx88O1XTteUjcFn26EUji6DRK+8JJiN2lXSyQokNeY +ox6Z4hFQDmw/Q0k/iJqe9/Dq4zA0l3Krkpra0DZoWh5kzYUA0g5+Yg6GmRNRa8YG +tuuD6qK1SBEzmCYff6ivjgsXV5+vFBSjEpx2dPEaKdYxtHMOjkttuTi1mr+19dVf +hSltbzfISbV9HafX76dhwZJ0QwsUx+aOW6OrnK8zoQc5AFOXpe9BrrOuEX01qrM0 +KX5tS8Zx5HqDLievjir194oi3r+nAiG14kYlGmOTHshu7keGCgJmzJ0iVG/i+TnV +ZSLyd8OqV1F6MET1ijgR3OPL3kt81Zy9lATWk/DgKbGBkkKAnXO2HUw9U34JFyEy +vEc81qeHci8sT5QKSFHiP3r8EcK8rT5k9CHpnbFmg7VWSMVD0/wRB/C4BiIw357a +xyJ/q1NNvOZVAyYzIzf9TjwREtyeHEo5kS6hyWSn7fbFf3sNGO2I30veWOvE6kFA +HMtF3NplOrTYcM7fAK5zJCBK20oU645TxI8GsICMog7IFidFMdRn4MaXpwAjEZO4 +44m2M+4XyeRCAZhp1Fu4mDiHGqgd44mKtwvLACVF4ygWZnACDpI17X88wMnwL4uU +vgehLZdAE89gvukSCsET1inVBnn/hVenCRbbZ++IGv2XoYvRfeezfOoNUcJXyawQ +JFqN0CRB5pliuCesTO2urn4HSwGGoeBd507pGWZmOAjbNjGswlJJXF0NFnNW/zWw +UFYy+BI9axuhWTSnCXbNbngdNQKHznKe1Lwit6AI3U9jS33pM3W+pwUAQegVdtpG +XT01YgiMCBX+b8B/xcWTww0JbeUwKXudzKsPhQmaA0lubAo04JACMfON8jSZCeRV +TyIzgacxGU6YbEKH4PhYTGl9srcWIT9iGSYD53V7Kyvjumd0Y3Qc3JLnuWZT6Oe3 +uJ4xz9jJtoaTDvPJQNK3igscjZnWZSP8XMJo1/f7vbvD57pPt1Hqdirp1EBQNshk +iX9CUh4fuGFFeHf6MtGxPofbXmvA2GYcFsOez4/2eOTEmo6H3P4Hrya97XHS0dmD +zFSAjzAlacTrn1uuxtxFTikdOwvdmQJJEfyYWCB1lqWOZi97+7nzqyXMLvMgmwug +ZF/xHFMhFTR8Wn7puuwf36JpPQiM4oQ/Lp66zkS4UlKrVsmSXIXudLMg8SQ5WqK8 +DjevEZwsHHaMtfDsnCAhAdRc2jCpyHKKnmhCDdkcdJJEymWKILUJI5PJ3XtiMHnR +Sa35OOICS0lTq4VwhUdkGwGjRoY1GsriPHd6LOt1aom14yJros1h7ta604hSCn4k +zj9p7wY9gfgkXWXNfmarrZ9NNwlHxzgSva+jbJcLmE4GMX5OFHHGlRj/9S1xC2Wf +MY9orzlooGM74NtmRi4qNkFj3dQCde8XRR4wh2IvPUCsr4j+XaoCoc3R5Rn/yNJK +zIkccJ2K14u9X/A0BLXHn5Gnd0tBYcVOqP6dQlW9UWdJC/Xooh7+CVU5cZIxuF/s +Vvg+Xwiv3XqekJRu3cMllJDp5rwe5EWZSmnoAiGKjouKAIszlevaRiD/wT6Zra3c +Wn/1U/sGop6zRscHR7pgI99NSogzpVGThUs+ez7otDBIdDbLpMjktahgWoi1Vqhc +fNZXjA6ob4zTWY/16Ys0YWxHO+MtyWTMP1dnsqePDfYXGUHe8yGxylbcjfrsVYta +4H6eYR86eU3eXB+MpS/iA4jBq4QYWR9QUkd6FDfmRGgWlMXhisPv6Pfnj384NzEV +Emeg7tW8wzWR64EON9iGeGYYa2BBl2FVaayMEoUhthhFcDM1r3/Mox5xF0qnlys4 +goWkMzqbzA2t97bC0KDGzkcHT4wMeiJBLDZ7S2J2nDAEhcTLY0P2zvOB4879pEWx +Bd15AyG1DvNssA5ooaDzKi/Li6NgDuMJ8W7+tmsBwDvwuf2N3koqBeXfKhR4rTqu +Wg1k9fX3+8DzDf0EjtDZJdfWZAynONi1PhZGbNbaMKsQ+6TflkCACInRdOADR5GM +rL7JtrgF1a9n0HD9vk2WGZqKI71tfS8zODkOZDD8aAusD2DOSmVZl48HX/t4i4Wc +3dgi/gkCMrfK3wOujb8tL4zjnlVkM7kzKk0MgHuA1w81zFjeMFvigHes4IWhQVcz +ek3l4bGifI2kzU7bGIi5e/019ppJzGsVcrOE/3z4GS0DJVk6fy7MEMIFx0LhJPlL +T+9HMH85sSYb97PTiMWpfBvNw3FSC7QQT9FC3L8d/XtMY3NvZoc7Fz7cSGaj7NXG +1OgVnAzMunPa3QaduoxMF9346s+4a+FrpRxL/3bb4skojjmmLqP4dsbD1uz0fP9y +xSifnTnrtjumYWMVi+pEb5kR0sTHl0XS7qKRi3SEfv28uh72KdvcufonIA5rnEb5 ++yqAZiqW2OxVsRoVLVODPswP4VIDiun2kCnfkQygPzxlZUeDZur0mmZ3vwC81C1Q +dZcjlukZcqUaxybUloUilqfNeby+2Uig0krLh2+AM4EqR63LeZ/tk+zCitHeRBW0 +wl3Bd7ShBFg6kN5tCJlHf/G6suIJVr+A9BXfwekO9+//CutKakCwmJTUiNWbQbtN +q3aNCnomyD3WjvUbitVO0CWYjZrmMLIsPtzyLQydpT7tjXpHgvwm5GYWdUGnNs4y +NbA262sUl7Ku/GDw1CnFYXbxl+qxbucLtCdSIFR2xUq3rEO1MXlD/txdTxn6ANax +hi9oBg8tHzuGYJFiCDCvbVVTHgWUSnm/EqfclpJzGmxt8g7vbaohW7NMmMQrLBFP +G6qBypgvotx1iJWaHVLNNiXvyqQwTtelNPAUweRoNawBp/5KTwwy/tHeF0gsVQ7y +mFX4umub9YT34Lpe7qUPKNxXzFcUgAf1SA6vyZ20UI7p42S2OT2PrahJ+uO6LQVD ++REhtN0oyS3G6HzAmKkBgw7LcV3XmAr39iSR7mdmoHSJuI9bjveAPhniK+N6uuln +xf17Qnw5NWfr9MXcLli7zqwMglU/1bNirkwVqf/ogi/zQ3JYCo6tFGf/rnGQAORJ +hvOq2SEYXnizPPIH7VrpE16+jUXwgpiQ8TDyeLPmpZVuhXTXiCaJO5lIwmLQqkmg +JqNiT9V44sksNFTGNKgZo5O9rEqfqX4dLjfv6pGJL+MFXD9if4f1JQiXJfhcRcDh +Ff9B6HukgbJ1H96eLUUNj8sL1+WPOqawkS4wg7tVaERE8CW7mqk15dCysn9shSut +I+7JU7+dZsxpj0ownrxuPAFuT8ZlcBPrFzPUwTlW1G0CbuEco8ijfy5IfbyGCn5s +K/0bOfAuNVGoOpLZ1dMki2bGdBwQOQlkLKhAxYcCVQ0/urr1Ab+VXU9kBsIU8ssN +GogKngYpuUV0PHmpzmobielOHLjNqA2v9vQSV3Ed48wRy5OCwLX1+vYmYlggMDGt +wfl+7QbXYf+k5WnELf3IqYvh8ZWexa0= + + \ No newline at end of file diff --git a/tests/data/wire/goal_state.xml b/tests/data/wire/goal_state.xml index 579b5e87a..0ccff211c 100644 --- a/tests/data/wire/goal_state.xml +++ b/tests/data/wire/goal_state.xml @@ -15,12 +15,12 @@ b61f93d0-e1ed-40b2-b067-22c243233448.MachineRole_IN_0 Started - http://168.63.129.16:80/hostingenvuri/ - http://168.63.129.16:80/sharedconfiguri/ - http://168.63.129.16:80/certificatesuri/ - http://168.63.129.16:80/extensionsconfiguri/ - http://168.63.129.16:80/fullconfiguri/ - b61f93d0-e1ed-40b2-b067-22c243233448.1.b61f93d0-e1ed-40b2-b067-22c243233448.2.MachineRole_IN_0.xml + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=hostingEnvironmentConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=sharedConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=extensionsConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=fullConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=certificates&incarnation=1 + bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5.bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5._canary.1.xml diff --git a/tests/data/wire/goal_state_no_ext.xml b/tests/data/wire/goal_state_no_ext.xml index ef7e3989e..e9048daf6 100644 --- a/tests/data/wire/goal_state_no_ext.xml +++ b/tests/data/wire/goal_state_no_ext.xml @@ -15,11 +15,11 @@ b61f93d0-e1ed-40b2-b067-22c243233448.MachineRole_IN_0 Started - http://168.63.129.16:80/hostingenvuri/ - http://168.63.129.16:80/sharedconfiguri/ - http://168.63.129.16:80/certificatesuri/ - http://168.63.129.16:80/fullconfiguri/ - b61f93d0-e1ed-40b2-b067-22c243233448.1.b61f93d0-e1ed-40b2-b067-22c243233448.2.MachineRole_IN_0.xml + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=hostingEnvironmentConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=sharedConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=fullConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=certificates&incarnation=1 + bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5.bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5._canary.1.xml diff --git a/tests/data/wire/goal_state_remote_access.xml b/tests/data/wire/goal_state_remote_access.xml index c2840645f..279006f21 100644 --- a/tests/data/wire/goal_state_remote_access.xml +++ b/tests/data/wire/goal_state_remote_access.xml @@ -17,12 +17,13 @@ b61f93d0-e1ed-40b2-b067-22c243233448.MachineRole_IN_0 Started - http://168.63.129.16:80/hostingenvuri/ - http://168.63.129.16:80/sharedconfiguri/ - http://168.63.129.16:80/certificatesuri/ - http://168.63.129.16:80/extensionsconfiguri/ - http://168.63.129.16:80/fullconfiguri/ - b61f93d0-e1ed-40b2-b067-22c243233448.1.b61f93d0-e1ed-40b2-b067-22c243233448.2.MachineRole_IN_0.xml + b61f93d0-e1ed-40b2-b067-22c243233448.1.b61f93d0-e1ed-40b2-b067-22c243233448.2.MachineRole_IN_0.xml + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=hostingEnvironmentConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=sharedConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=extensionsConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=fullConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=certificates&incarnation=1 + bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5.bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5._canary.1.xml diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index 8a648a8c4..91545db2e 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -1898,7 +1898,7 @@ def update_goal_state_and_run_handler(): def test_it_should_wait_to_fetch_first_goal_state(self): with _get_update_handler() as (update_handler, protocol): - with patch("azurelinuxagent.common.logger.warn") as patch_warn: + with patch("azurelinuxagent.common.logger.error") as patch_error: with patch("azurelinuxagent.common.logger.info") as patch_info: # Fail GS fetching for the 1st 5 times the agent asks for it update_handler._fail_gs_count = 5 @@ -1914,13 +1914,13 @@ def get_handler(url, **kwargs): self.assertTrue(update_handler.exit_mock.called, "The process should have exited") exit_args, _ = update_handler.exit_mock.call_args - self.assertEqual(exit_args[0], 0, "Exit code should be 0; List of all warnings logged by the agent: {0}".format( - patch_warn.call_args_list)) - warn_msgs = [args[0] for (args, _) in patch_warn.call_args_list if - "An error occurred while retrieving the goal state" in args[0]] - self.assertTrue(len(warn_msgs) > 0, "Error should've been reported when failed to retrieve GS") + self.assertEqual(exit_args[0], 0, "Exit code should be 0; List of all errors logged by the agent: {0}".format( + patch_error.call_args_list)) + error_msgs = [args[0] for (args, _) in patch_error.call_args_list if + "Error fetching the goal state" in args[0]] + self.assertTrue(len(error_msgs) > 0, "Error should've been reported when failed to retrieve GS") info_msgs = [args[0] for (args, _) in patch_info.call_args_list if - "Retrieving the goal state recovered from previous errors" in args[0]] + "Fetching the goal state recovered from previous errors." in args[0]] self.assertTrue(len(info_msgs) > 0, "Agent should've logged a message when recovered from GS errors") def test_it_should_reset_legacy_blacklisted_agents_on_process_start(self): @@ -2684,9 +2684,9 @@ def create_log_and_telemetry_mocks(): calls_to_strings = lambda calls: (str(c) for c in calls) filter_calls = lambda calls, regex=None: (c for c in calls_to_strings(calls) if regex is None or re.match(regex, c)) logger_calls = lambda regex=None: [m for m in filter_calls(logger.method_calls, regex)] # pylint: disable=used-before-assignment,unnecessary-comprehension - warnings = lambda: logger_calls(r'call.warn\(.*An error occurred while retrieving the goal state.*') - periodic_warnings = lambda: logger_calls(r'call.periodic_warn\(.*Attempts to retrieve the goal state are failing.*') - success_messages = lambda: logger_calls(r'call.info\(.*Retrieving the goal state recovered from previous errors.*') + errors = lambda: logger_calls(r'call.error\(.*Error fetching the goal state.*') + periodic_errors = lambda: logger_calls(r'call.error\(.*Fetching the goal state is still failing*') + success_messages = lambda: logger_calls(r'call.info\(.*Fetching the goal state recovered from previous errors.*') telemetry_calls = lambda regex=None: [m for m in filter_calls(add_event.mock_calls, regex)] # pylint: disable=used-before-assignment,unnecessary-comprehension goal_state_events = lambda: telemetry_calls(r".*op='FetchGoalState'.*") @@ -2711,10 +2711,8 @@ def create_log_and_telemetry_mocks(): with create_log_and_telemetry_mocks() as (logger, add_event): update_handler._try_update_goal_state(protocol) - w = warnings() - pw = periodic_warnings() - self.assertEqual(1, len(w), "A failure should have produced a warning: [{0}]".format(w)) - self.assertEqual(1, len(pw), "A failure should have produced a periodic warning: [{0}]".format(pw)) + e = errors() + self.assertEqual(1, len(e), "A failure should have produced an error: [{0}]".format(e)) gs = goal_state_events() self.assertTrue(len(gs) == 1 and 'is_success=False' in gs[0], "A failure should produce a telemetry event (success=false): [{0}]".format(gs)) @@ -2723,17 +2721,17 @@ def create_log_and_telemetry_mocks(): # ... and errors continue happening... # with create_log_and_telemetry_mocks() as (logger, add_event): - update_handler._try_update_goal_state(protocol) - update_handler._try_update_goal_state(protocol) - update_handler._try_update_goal_state(protocol) + for _ in range(5): + update_handler._update_goal_state_last_error_report = datetime.now() + timedelta(days=1) + update_handler._try_update_goal_state(protocol) - w = warnings() - pw = periodic_warnings() - self.assertTrue(len(w) == 0, "Subsequent failures should not produce warnings: [{0}]".format(w)) - self.assertEqual(len(pw), 3, "Subsequent failures should produce periodic warnings: [{0}]".format(pw)) + e = errors() + pe = periodic_errors() + self.assertEqual(2, len(e), "Two additional errors should have been reported: [{0}]".format(e)) + self.assertEqual(len(pe), 3, "Subsequent failures should produce periodic errors: [{0}]".format(pe)) tc = telemetry_calls() - self.assertTrue(len(tc) == 0, "Subsequent failures should not produce any telemetry events: [{0}]".format(tc)) + self.assertTrue(len(tc) == 5, "The failures should have produced telemetry events. Got: [{0}]".format(tc)) # # ... until we finally succeed @@ -2743,10 +2741,10 @@ def create_log_and_telemetry_mocks(): update_handler._try_update_goal_state(protocol) s = success_messages() - w = warnings() - pw = periodic_warnings() + e = errors() + pe = periodic_errors() self.assertEqual(len(s), 1, "Recovering after failures should have produced an info message: [{0}]".format(s)) - self.assertTrue(len(w) == 0 and len(pw) == 0, "Recovering after failures should have not produced any warnings: [{0}] [{1}]".format(w, pw)) + self.assertTrue(len(e) == 0 and len(pe) == 0, "Recovering after failures should have not produced any errors: [{0}] [{1}]".format(e, pe)) gs = goal_state_events() self.assertTrue(len(gs) == 1 and 'is_success=True' in gs[0], "Recovering after failures should produce a telemetry event (success=true): [{0}]".format(gs)) diff --git a/tests/protocol/HttpRequestPredicates.py b/tests/protocol/HttpRequestPredicates.py index 39243d543..db3ab8b2a 100644 --- a/tests/protocol/HttpRequestPredicates.py +++ b/tests/protocol/HttpRequestPredicates.py @@ -11,6 +11,22 @@ class HttpRequestPredicates(object): def is_goal_state_request(url): return url.lower() == 'http://{0}/machine/?comp=goalstate'.format(restutil.KNOWN_WIRESERVER_IP) + @staticmethod + def is_certificates_request(url): + return re.match(r'http://{0}(:80)?/machine/.*?comp=certificates'.format(restutil.KNOWN_WIRESERVER_IP), url, re.IGNORECASE) + + @staticmethod + def is_extensions_config_request(url): + return re.match(r'http://{0}(:80)?/machine/.*?comp=config&type=extensionsConfig'.format(restutil.KNOWN_WIRESERVER_IP), url, re.IGNORECASE) + + @staticmethod + def is_hosting_environment_config_request(url): + return re.match(r'http://{0}(:80)?/machine/.*?comp=config&type=hostingEnvironmentConfig'.format(restutil.KNOWN_WIRESERVER_IP), url, re.IGNORECASE) + + @staticmethod + def is_shared_config_request(url): + return re.match(r'http://{0}(:80)?/machine/.*?comp=config&type=sharedConfig'.format(restutil.KNOWN_WIRESERVER_IP), url, re.IGNORECASE) + @staticmethod def is_telemetry_request(url): return url.lower() == 'http://{0}/machine?comp=telemetrydata'.format(restutil.KNOWN_WIRESERVER_IP) diff --git a/tests/protocol/mockwiredata.py b/tests/protocol/mockwiredata.py index 218bd2937..7ec311af4 100644 --- a/tests/protocol/mockwiredata.py +++ b/tests/protocol/mockwiredata.py @@ -135,10 +135,10 @@ def __init__(self, data_files=None): "/HealthService": 0, "/vmAgentLog": 0, "goalstate": 0, - "hostingenvuri": 0, - "sharedconfiguri": 0, - "certificatesuri": 0, - "extensionsconfiguri": 0, + "hostingEnvironmentConfig": 0, + "sharedConfig": 0, + "certificates": 0, + "extensionsConfig": 0, "remoteaccessinfouri": 0, "extensionArtifact": 0, "agentArtifact": 0, @@ -198,6 +198,10 @@ def reload(self): if in_vm_artifacts_profile_file is not None: self.in_vm_artifacts_profile = load_data(in_vm_artifacts_profile_file) + def reset_call_counts(self): + for counter in self.call_counts: + self.call_counts[counter] = 0 + def mock_http_get(self, url, *_, **kwargs): content = '' response_headers = [] @@ -217,18 +221,18 @@ def mock_http_get(self, url, *_, **kwargs): elif "goalstate" in url: content = self.goal_state self.call_counts["goalstate"] += 1 - elif "hostingenvuri" in url: + elif HttpRequestPredicates.is_hosting_environment_config_request(url): content = self.hosting_env - self.call_counts["hostingenvuri"] += 1 - elif "sharedconfiguri" in url: + self.call_counts["hostingEnvironmentConfig"] += 1 + elif HttpRequestPredicates.is_shared_config_request(url): content = self.shared_config - self.call_counts["sharedconfiguri"] += 1 - elif "certificatesuri" in url: + self.call_counts["sharedConfig"] += 1 + elif HttpRequestPredicates.is_certificates_request(url): content = self.certs - self.call_counts["certificatesuri"] += 1 - elif "extensionsconfiguri" in url: + self.call_counts["certificates"] += 1 + elif HttpRequestPredicates.is_extensions_config_request(url): content = self.ext_conf - self.call_counts["extensionsconfiguri"] += 1 + self.call_counts["extensionsConfig"] += 1 elif "remoteaccessinfouri" in url: content = self.remote_access self.call_counts["remoteaccessinfouri"] += 1 diff --git a/tests/protocol/test_goal_state.py b/tests/protocol/test_goal_state.py index c54a65f9f..c77417159 100644 --- a/tests/protocol/test_goal_state.py +++ b/tests/protocol/test_goal_state.py @@ -8,13 +8,12 @@ import re import time -from azurelinuxagent.common.event import WALAEventOperation from azurelinuxagent.common.future import httpclient from azurelinuxagent.common.protocol.extensions_goal_state import GoalStateSource, GoalStateChannel from azurelinuxagent.common.protocol.extensions_goal_state_from_extensions_config import ExtensionsGoalStateFromExtensionsConfig from azurelinuxagent.common.protocol.extensions_goal_state_from_vm_settings import ExtensionsGoalStateFromVmSettings from azurelinuxagent.common.protocol import hostplugin -from azurelinuxagent.common.protocol.goal_state import GoalState, _GET_GOAL_STATE_MAX_ATTEMPTS +from azurelinuxagent.common.protocol.goal_state import GoalState, GoalStateInconsistentError, _GET_GOAL_STATE_MAX_ATTEMPTS from azurelinuxagent.common.exception import ProtocolError from azurelinuxagent.common.utils import fileutil from azurelinuxagent.common.utils.archive import ARCHIVE_DIRECTORY_NAME @@ -360,17 +359,54 @@ def http_get_handler(url, *_, **__): self.assertEqual(initial_timestamp, goal_state.extensions_goal_state.created_on_timestamp, "The timestamp of the updated goal state is incorrect") self.assertTrue(goal_state.extensions_goal_state.is_outdated, "The updated goal state should be marked as outdated") - def test_it_should_report_missing_certificates(self): + def test_it_should_raise_when_the_tenant_certificate_is_missing(self): data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() - data_file["vm_settings"] = "hostgaplugin/vm_settings-missing_cert.json" with mock_wire_protocol(data_file) as protocol: - with patch("azurelinuxagent.common.protocol.goal_state.add_event") as add_event: + data_file["vm_settings"] = "hostgaplugin/vm_settings-missing_cert.json" + protocol.mock_wire_data.reload() + + with self.assertRaises(GoalStateInconsistentError) as context: _ = GoalState(protocol.client) - expected_message = "Certificate 59A10F50FFE2A0408D3F03FE336C8FD5716CF25C needed by Microsoft.OSTCExtensions.VMAccessForLinux is missing from the goal state" - events = [kwargs for _, kwargs in add_event.call_args_list if kwargs['op'] == WALAEventOperation.VmSettings and kwargs['message'] == expected_message] + expected_message = "Certificate 59A10F50FFE2A0408D3F03FE336C8FD5716CF25C needed by Microsoft.OSTCExtensions.VMAccessForLinux is missing from the goal state" + self.assertIn(expected_message, str(context.exception)) + + def test_it_should_refresh_the_goal_state_when_it_is_inconsistent(self): + # + # Some scenarios can produce inconsistent goal states. For example, during hibernation/resume, the Fabric goal state changes (the + # tenant certificate is re-generated when the VM is restarted) *without* the incarnation changing. If a Fast Track goal state + # comes after that, the extensions will need the new certificate. This test simulates that scenario by mocking the certificates + # request and returning first a set of certificates (certs-2.xml) that do not match those needed by the extensions, and then a + # set (certs.xml) that does match. The test then ensures that the goal state was refreshed and the correct certificates were + # fetched. + # + data_files = [ + "wire/certs-2.xml", + "wire/certs.xml" + ] + + def http_get_handler(url, *_, **__): + if HttpRequestPredicates.is_certificates_request(url): + http_get_handler.certificate_requests += 1 + if http_get_handler.certificate_requests < len(data_files): + data = load_data(data_files[http_get_handler.certificate_requests - 1]) + return MockHttpResponse(status=200, body=data.encode('utf-8')) + return None + http_get_handler.certificate_requests = 0 + + with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: + protocol.set_http_handlers(http_get_handler=http_get_handler) + protocol.mock_wire_data.reset_call_counts() + + goal_state = GoalState(protocol.client) + + self.assertEqual(2, protocol.mock_wire_data.call_counts['goalstate'], "There should have been exactly 2 requests for the goal state (original + refresh)") + self.assertEqual(2, http_get_handler.certificate_requests, "There should have been exactly 2 requests for the goal state certificates (original + refresh)") + + thumbprints = [c.thumbprint for c in goal_state.certs.cert_list.certificates] - self.assertTrue( - len(events) == 1, - "Missing certificate 59A10F50FFE2A0408D3F03FE336C8FD5716CF25C was not reported. Telemetry: {0}".format([kwargs['message'] for _, kwargs in add_event.call_args_list])) + for extension in goal_state.extensions_goal_state.extensions: + for settings in extension.settings: + if settings.protectedSettings is not None: + self.assertIn(settings.certificateThumbprint, thumbprints, "Certificate is missing from the goal state.") diff --git a/tests/protocol/test_hostplugin.py b/tests/protocol/test_hostplugin.py index 16bb7ef0b..9f96f7d55 100644 --- a/tests/protocol/test_hostplugin.py +++ b/tests/protocol/test_hostplugin.py @@ -257,9 +257,8 @@ def test_default_channel(self, patch_put, patch_upload, _): # assert host plugin route is called self.assertEqual(1, patch_put.call_count, "Host plugin was not used") - # assert update goal state is only called once, non-forced + # assert update goal state is only called once self.assertEqual(1, wire_protocol.client.update_goal_state.call_count, "Unexpected call count") - self.assertEqual(0, len(wire_protocol.client.update_goal_state.call_args[1]), "Unexpected parameters") # ensure the correct url is used self.assertEqual(sas_url, patch_put.call_args[0][0]) @@ -291,9 +290,8 @@ def test_fallback_channel_503(self, patch_put, patch_upload, _): # assert host plugin route is called self.assertEqual(1, patch_put.call_count, "Host plugin was not used") - # assert update goal state is only called once, non-forced + # assert update goal state is only called once self.assertEqual(1, wire_protocol.client.update_goal_state.call_count, "Update goal state unexpected call count") - self.assertEqual(0, len(wire_protocol.client.update_goal_state.call_args[1]), "Update goal state unexpected call count") # ensure the correct url is used self.assertEqual(sas_url, patch_put.call_args[0][0]) @@ -326,9 +324,8 @@ def test_fallback_channel_410(self, patch_refresh_host_plugin, patch_put, patch_ # assert host plugin route is called self.assertEqual(1, patch_put.call_count, "Host plugin was not used") - # assert update goal state is called with no arguments (forced=False), then update_host_plugin_from_goal_state is called + # assert update goal state is called, then update_host_plugin_from_goal_state is called self.assertEqual(1, wire_protocol.client.update_goal_state.call_count, "Update goal state unexpected call count") - self.assertEqual(0, len(wire_protocol.client.update_goal_state.call_args[1]), "Update goal state unexpected argument count") self.assertEqual(1, patch_refresh_host_plugin.call_count, "Refresh host plugin unexpected call count") # ensure the correct url is used @@ -361,9 +358,8 @@ def test_fallback_channel_failure(self, patch_put, patch_upload, _): # assert host plugin route is called self.assertEqual(1, patch_put.call_count, "Host plugin was not used") - # assert update goal state is called twice, forced=True on the second + # assert update goal state is called twice self.assertEqual(1, wire_protocol.client.update_goal_state.call_count, "Update goal state unexpected call count") - self.assertEqual(0, len(wire_protocol.client.update_goal_state.call_args[1]), "Update goal state unexpected call count") # ensure the correct url is used self.assertEqual(sas_url, patch_put.call_args[0][0]) diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py index 0cc8a01e9..c564af721 100644 --- a/tests/protocol/test_wire.py +++ b/tests/protocol/test_wire.py @@ -160,7 +160,7 @@ def test_getters_with_stale_goal_state(self, patch_report, *args): # -- Tracking calls to retrieve GoalState is problematic since it is # fetched often; however, the dependent documents, such as the # HostingEnvironmentConfig, will be retrieved the expected number - self.assertEqual(1, test_data.call_counts["hostingenvuri"]) + self.assertEqual(1, test_data.call_counts["hostingEnvironmentConfig"]) self.assertEqual(1, patch_report.call_count) def test_call_storage_kwargs(self, *args): # pylint: disable=unused-argument From a4836c54da9a69ff5c03c7b1224d88de8cf4ffe3 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 25 Apr 2022 15:50:55 -0700 Subject: [PATCH 49/89] Update agent version to 2.8.0.3 (#2563) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index b6fb5f228..3e465b3cf 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.8.0.2' +AGENT_VERSION = '2.8.0.3' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 4bc921590409e610d225387afb13f9b7c821777a Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 27 Apr 2022 15:22:34 -0700 Subject: [PATCH 50/89] Do not mark goal state as processed when goal state fails to update (#2569) Co-authored-by: narrieta --- azurelinuxagent/common/protocol/goal_state.py | 4 ++-- azurelinuxagent/ga/update.py | 20 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/azurelinuxagent/common/protocol/goal_state.py b/azurelinuxagent/common/protocol/goal_state.py index edfd9d14f..3301d783d 100644 --- a/azurelinuxagent/common/protocol/goal_state.py +++ b/azurelinuxagent/common/protocol/goal_state.py @@ -209,7 +209,7 @@ def _update(self, force_update): else: # vm_settings_updated most_recent = vm_settings - if self._extensions_goal_state is None or most_recent.created_on_timestamp > self._extensions_goal_state.created_on_timestamp: + if self._extensions_goal_state is None or most_recent.created_on_timestamp >= self._extensions_goal_state.created_on_timestamp: self._extensions_goal_state = most_recent # @@ -221,7 +221,7 @@ def _update(self, force_update): # Track goal state comes after that, the extensions will need the new certificate. The Agent needs to refresh the goal state in that # case, to ensure it fetches the new certificate. # - if self.extensions_goal_state.source == GoalStateSource.FastTrack: + if self._extensions_goal_state.source == GoalStateSource.FastTrack: self._check_certificates() def _check_certificates(self): diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index cbf0ac38b..f393aef8a 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -606,19 +606,19 @@ def _processing_new_extensions_goal_state(self): return self._goal_state is not None and egs.id != self._last_extensions_gs_id and not egs.is_outdated def _process_goal_state(self, exthandlers_handler, remote_access_handler): - try: - protocol = exthandlers_handler.protocol - - # update self._goal_state - if not self._try_update_goal_state(protocol): - # agent updates and status reporting should be done even when the goal state is not updated - self.__update_guest_agent(protocol) - self._report_status(exthandlers_handler) - return + protocol = exthandlers_handler.protocol - # check for agent updates + # update self._goal_state + if not self._try_update_goal_state(protocol): + # agent updates and status reporting should be done even when the goal state is not updated self.__update_guest_agent(protocol) + self._report_status(exthandlers_handler) + return + + # check for agent updates + self.__update_guest_agent(protocol) + try: if self._processing_new_extensions_goal_state(): if not self._extensions_summary.converged: message = "A new goal state was received, but not all the extensions in the previous goal state have completed: {0}".format(self._extensions_summary) From 8d7237f1e5b5cd2a7f3066797dad995114f818de Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 27 Apr 2022 15:27:54 -0700 Subject: [PATCH 51/89] Update agent version to 2.8.0.4 (#2570) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 3e465b3cf..99eb6473f 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.8.0.3' +AGENT_VERSION = '2.8.0.4' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 8e9c1b5b514ffccb10f664f9f1b7c9edee1ddced Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Fri, 29 Apr 2022 14:03:49 -0700 Subject: [PATCH 52/89] Bug fix for fetching a goal state with empty certificates property (#2575) --- azurelinuxagent/common/protocol/goal_state.py | 9 +++- tests/data/wire/goal_state_no_certs.xml | 27 +++++++++++ tests/ga/test_update.py | 47 +++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 tests/data/wire/goal_state_no_certs.xml diff --git a/azurelinuxagent/common/protocol/goal_state.py b/azurelinuxagent/common/protocol/goal_state.py index 3301d783d..8b508f61a 100644 --- a/azurelinuxagent/common/protocol/goal_state.py +++ b/azurelinuxagent/common/protocol/goal_state.py @@ -82,7 +82,7 @@ def __init__(self, wire_client, silent=False): self._container_id = None self._hosting_env = None self._shared_conf = None - self._certs = None + self._certs = EmptyCertificates() self._remote_access = None self.update(silent=silent) @@ -348,7 +348,7 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): shared_conf = SharedConfig(xml_text) self._history.save_shared_conf(xml_text) - certs = None + certs = EmptyCertificates() certs_uri = findtext(xml_doc, "Certificates") if certs_uri is not None: xml_text = self._wire_client.fetch_config(certs_uri, self._wire_client.get_header_for_cert()) @@ -506,6 +506,11 @@ def _write_to_tmp_file(index, suffix, buf): fileutil.write_file(file_name, "".join(buf)) return file_name +class EmptyCertificates: + def __init__(self): + self.cert_list = CertList() + self.summary = [] # debugging info + self.warnings = [] class RemoteAccess(object): """ diff --git a/tests/data/wire/goal_state_no_certs.xml b/tests/data/wire/goal_state_no_certs.xml new file mode 100644 index 000000000..1ab7fa217 --- /dev/null +++ b/tests/data/wire/goal_state_no_certs.xml @@ -0,0 +1,27 @@ + + + 2010-12-15 + 1 + + Started + + 16001 + + + + c6d5526c-5ac2-4200-b6e2-56f2b70c5ab2 + + + b61f93d0-e1ed-40b2-b067-22c243233448.MachineRole_IN_0 + Started + + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=hostingEnvironmentConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=sharedConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=extensionsConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=fullConfig&incarnation=1 + bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5.bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5._canary.1.xml + + + + + diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index 91545db2e..4ac1ba645 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -1564,6 +1564,53 @@ def _get_test_ext_handler_instance(protocol, name="OSTCExtensions.ExampleHandler eh = Extension(name=name) eh.version = version return ExtHandlerInstance(eh, protocol) + + def test_update_handler_recovers_from_error_with_no_certs(self): + data = DATA_FILE.copy() + data['goal_state'] = 'wire/goal_state_no_certs.xml' + + def fail_gs_fetch(url, *_, **__): + if HttpRequestPredicates.is_goal_state_request(url): + return MockHttpResponse(status=500) + return None + + with mock_wire_protocol(data) as protocol: + + def fail_fetch_on_second_iter(iteration): + if iteration == 2: + protocol.set_http_handlers(http_get_handler=fail_gs_fetch) + if iteration > 2: # Zero out the fail handler for subsequent iterations. + protocol.set_http_handlers(http_get_handler=None) + + with mock_update_handler(protocol, 3, on_new_iteration=fail_fetch_on_second_iter) as update_handler: + with patch("azurelinuxagent.ga.update.logger.error") as patched_error: + with patch("azurelinuxagent.ga.update.logger.info") as patched_info: + def match_unexpected_errors(): + unexpected_msg_fragment = "Error fetching the goal state:" + + matching_errors = [] + for (args, _) in filter(lambda a: len(a) > 0, patched_error.call_args_list): + if unexpected_msg_fragment in args[0]: + matching_errors.append(args[0]) + + if len(matching_errors) > 1: + self.fail("Guest Agent did not recover, with new error(s): {}"\ + .format(matching_errors[1:])) + + def match_expected_info(): + expected_msg_fragment = "Fetching the goal state recovered from previous errors" + + for (call_args, _) in filter(lambda a: len(a) > 0, patched_info.call_args_list): + if expected_msg_fragment in call_args[0]: + break + else: + self.fail("Expected the guest agent to recover with '{}', but it didn't"\ + .format(expected_msg_fragment)) + + update_handler.run(debug=True) + match_unexpected_errors() # Match on errors first, they can provide more info. + match_expected_info() + def test_it_should_recreate_handler_env_on_service_startup(self): iterations = 5 From dee5fef9b98bf3e50b8fce213f67bd0d0b288c21 Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Fri, 29 Apr 2022 14:48:26 -0700 Subject: [PATCH 53/89] Move error counter reset down to end of block. (#2576) --- azurelinuxagent/ga/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index f393aef8a..469b7ece6 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -489,11 +489,11 @@ def _try_update_goal_state(self, protocol): self._goal_state = protocol.get_goal_state() if self._update_goal_state_error_count > 0: - self._update_goal_state_error_count = 0 message = u"Fetching the goal state recovered from previous errors. Fetched {0} (certificates: {1})".format( self._goal_state.extensions_goal_state.id, self._goal_state.certs.summary) add_event(AGENT_NAME, op=WALAEventOperation.FetchGoalState, version=CURRENT_VERSION, is_success=True, message=message, log_event=False) logger.info(message) + self._update_goal_state_error_count = 0 try: self._supports_fast_track = conf.get_enable_fast_track() and protocol.client.get_host_plugin().check_vm_settings_support() From 35fed83afb5da1e93d0e207f4b54a863f188ec6c Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Fri, 29 Apr 2022 19:23:02 -0700 Subject: [PATCH 54/89] Bug Fix: Change fast track timestamp default from None to datetime.min (#2577) --- azurelinuxagent/common/protocol/hostplugin.py | 4 +- .../vm_settings-fabric-no_thumbprints.json | 192 ++++++++++++++++++ tests/ga/test_update.py | 55 ++++- 3 files changed, 245 insertions(+), 6 deletions(-) create mode 100644 tests/data/hostgaplugin/vm_settings-fabric-no_thumbprints.json diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py index 5f795dca3..f79076f8e 100644 --- a/azurelinuxagent/common/protocol/hostplugin.py +++ b/azurelinuxagent/common/protocol/hostplugin.py @@ -95,7 +95,7 @@ def __init__(self, endpoint): if not os.path.exists(self._get_fast_track_state_file()): self._supports_vm_settings = False self._supports_vm_settings_next_check = datetime.datetime.now() - self._fast_track_timestamp = None + self._fast_track_timestamp = timeutil.create_timestamp(datetime.datetime.min) else: self._supports_vm_settings = True self._supports_vm_settings_next_check = datetime.datetime.now() @@ -443,7 +443,7 @@ def get_fast_track_timestamp(): goal state was Fabric or fetch_vm_settings() has not been invoked. """ if not os.path.exists(HostPluginProtocol._get_fast_track_state_file()): - return None + return timeutil.create_timestamp(datetime.datetime.min) try: with open(HostPluginProtocol._get_fast_track_state_file(), "r") as file_: diff --git a/tests/data/hostgaplugin/vm_settings-fabric-no_thumbprints.json b/tests/data/hostgaplugin/vm_settings-fabric-no_thumbprints.json new file mode 100644 index 000000000..bbd945933 --- /dev/null +++ b/tests/data/hostgaplugin/vm_settings-fabric-no_thumbprints.json @@ -0,0 +1,192 @@ +{ + "hostGAPluginVersion": "1.0.8.124", + "vmSettingsSchemaVersion": "0.0", + "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", + "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", + "inSvdSeqNo": 1, + "extensionsLastModifiedTickCount": 637726657706205299, + "extensionGoalStatesSource": "Fabric", + "onHold": true, + "statusUploadBlob": { + "statusBlobType": "BlockBlob", + "value": "https://dcrcl3a0xs.blob.core.windows.net/$system/edp0plkw2b.86f4ae0a-61f8-48ae-9199-40f402d56864.status?sv=2018-03-28&sr=b&sk=system-1&sig=KNWgC2%3d&se=9999-01-01T00%3a00%3a00Z&sp=w" + }, + "inVMMetadata": { + "subscriptionId": "8e037ad4-618f-4466-8bc8-5099d41ac15b", + "resourceGroupName": "rg-dc-86fjzhp", + "vmName": "edp0plkw2b", + "location": "CentralUSEUAP", + "vmId": "86f4ae0a-61f8-48ae-9199-40f402d56864", + "vmSize": "Standard_B2s", + "osType": "Linux" + }, + "requiredFeatures": [ + { + "name": "MultipleExtensionsPerHandler" + } + ], + "gaFamilies": [ + { + "name": "Prod", + "uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml" + ] + }, + { + "name": "Test", + "uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml" + ] + } + ], + "extensionGoalStates": [ + { + "name": "Microsoft.Azure.Monitor.AzureMonitorLinuxAgent", + "version": "1.9.1", + "location": "https://zrdfepirv2cbn04prdstr01a.blob.core.windows.net/a47f0806d764480a8d989d009c75007d/Microsoft.Azure.Monitor_AzureMonitorLinuxAgent_useast2euap_manifest.xml", + "failoverlocation": "https://zrdfepirv2cbn06prdstr01a.blob.core.windows.net/a47f0806d764480a8d989d009c75007d/Microsoft.Azure.Monitor_AzureMonitorLinuxAgent_useast2euap_manifest.xml", + "additionalLocations": ["https://zrdfepirv2cbn09pr02a.blob.core.windows.net/a47f0806d764480a8d989d009c75007d/Microsoft.Azure.Monitor_AzureMonitorLinuxAgent_useast2euap_manifest.xml"], + "state": "enabled", + "autoUpgrade": true, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "settings": [ + { + "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" + } + ] + }, + { + "name": "Microsoft.Azure.Security.Monitoring.AzureSecurityLinuxAgent", + "version": "2.15.112", + "location": "https://zrdfepirv2cbn04prdstr01a.blob.core.windows.net/4ef06ad957494df49c807a5334f2b5d2/Microsoft.Azure.Security.Monitoring_AzureSecurityLinuxAgent_useast2euap_manifest.xml", + "failoverlocation": "https://zrdfepirv2cbz06prdstr01a.blob.core.windows.net/4ef06ad957494df49c807a5334f2b5d2/Microsoft.Azure.Security.Monitoring_AzureSecurityLinuxAgent_useast2euap_manifest.xml", + "additionalLocations": ["https://zrdfepirv2cbn06prdstr01a.blob.core.windows.net/4ef06ad957494df49c807a5334f2b5d2/Microsoft.Azure.Security.Monitoring_AzureSecurityLinuxAgent_useast2euap_manifest.xml"], + "state": "enabled", + "autoUpgrade": true, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "settings": [ + { + "publicSettings": "{\"enableGenevaUpload\":true}" + } + ] + }, + { + "name": "Microsoft.Azure.Extensions.CustomScript", + "version": "2.1.6", + "location": "https://umsavwggj2v40kvqhc0w.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml", + "failoverlocation": "https://umsafwzhkbm1rfrhl0ws.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml", + "additionalLocations": [ + "https://umsanh4b5rfz0q0p4pwm.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml" + ], + "state": "enabled", + "autoUpgrade": true, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "isMultiConfig": false, + "settings": [ + { + "publicSettings": "{\"commandToExecute\":\"echo 'cee174d4-4daa-4b07-9958-53b9649445c2'\"}" + } + ], + "dependsOn": [ + { + "DependsOnExtension": [ + { + "handler": "Microsoft.Azure.Security.Monitoring.AzureSecurityLinuxAgent" + } + ], + "dependencyLevel": 1 + } + ] + }, + { + "name": "Microsoft.CPlat.Core.RunCommandHandlerLinux", + "version": "1.2.0", + "location": "https://umsavbvncrpzbnxmxzmr.blob.core.windows.net/f4086d41-69f9-3103-78e0-8a2c7e789d0f/f4086d41-69f9-3103-78e0-8a2c7e789d0f_manifest.xml", + "failoverlocation": "https://umsajbjtqrb3zqjvgb2z.blob.core.windows.net/f4086d41-69f9-3103-78e0-8a2c7e789d0f/f4086d41-69f9-3103-78e0-8a2c7e789d0f_manifest.xml", + "additionalLocations": [ + "https://umsawqtlsshtn5v2nfgh.blob.core.windows.net/f4086d41-69f9-3103-78e0-8a2c7e789d0f/f4086d41-69f9-3103-78e0-8a2c7e789d0f_manifest.xml" + ], + "state": "enabled", + "autoUpgrade": true, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "isMultiConfig": true, + "settings": [ + { + "publicSettings": "{\"source\":{\"script\":\"echo '4abb1e88-f349-41f8-8442-247d9fdfcac5'\"}}", + "seqNo": 0, + "extensionName": "MCExt1", + "extensionState": "enabled" + }, + { + "publicSettings": "{\"source\":{\"script\":\"echo 'e865c9bc-a7b3-42c6-9a79-cfa98a1ee8b3'\"}}", + "seqNo": 0, + "extensionName": "MCExt2", + "extensionState": "enabled" + }, + { + "publicSettings": "{\"source\":{\"script\":\"echo 'f923e416-0340-485c-9243-8b84fb9930c6'\"}}", + "seqNo": 0, + "extensionName": "MCExt3", + "extensionState": "enabled" + } + ], + "dependsOn": [ + { + "dependsOnExtension": [ + { + "extension": "...", + "handler": "..." + }, + { + "extension": "...", + "handler": "..." + } + ], + "dependencyLevel": 2, + "name": "MCExt1" + }, + { + "dependsOnExtension": [ + { + "extension": "...", + "handler": "..." + } + ], + "dependencyLevel": 1, + "name": "MCExt2" + } + ] + }, + { + "name": "Microsoft.OSTCExtensions.VMAccessForLinux", + "version": "1.5.11", + "location": "https://umsasc25p0kjg0c1dg4b.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml", + "failoverlocation": "https://umsamfwlmfshvxx2lsjm.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml", + "additionalLocations": [ + "https://umsah3cwjlctnmhsvzqv.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml" + ], + "state": "enabled", + "autoUpgrade": false, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "isMultiConfig": false, + "settings": [ ] + } + ] +} diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index 4ac1ba645..cd5595569 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -40,7 +40,7 @@ VMAgentUpdateStatuses from azurelinuxagent.common.protocol.util import ProtocolUtil from azurelinuxagent.common.protocol.wire import WireProtocol -from azurelinuxagent.common.utils import fileutil, restutil, textutil +from azurelinuxagent.common.utils import fileutil, restutil, textutil, timeutil from azurelinuxagent.common.utils.archive import ARCHIVE_DIRECTORY_NAME, AGENT_STATUS_FILE from azurelinuxagent.common.utils.flexible_version import FlexibleVersion from azurelinuxagent.common.utils.networkutil import FirewallCmdDirectCommands, AddFirewallRules @@ -54,7 +54,7 @@ READONLY_FILE_GLOBS, ExtensionsSummary, AgentUpgradeType from tests.ga.mocks import mock_update_handler from tests.protocol.mocks import mock_wire_protocol, MockHttpResponse -from tests.protocol.mockwiredata import DATA_FILE, DATA_FILE_MULTIPLE_EXT +from tests.protocol.mockwiredata import DATA_FILE, DATA_FILE_MULTIPLE_EXT, DATA_FILE_VM_SETTINGS from tests.tools import AgentTestCase, AgentTestCaseWithGetVmSizeMock, data_dir, DEFAULT, patch, load_bin_data, Mock, MagicMock, \ clear_singleton_instances, mock_sleep from tests.protocol import mockwiredata @@ -2919,7 +2919,7 @@ def test_it_should_mark_outdated_goal_states_on_service_restart_when_host_ga_plu def test_it_should_clear_the_timestamp_for_the_most_recent_fast_track_goal_state(self): data_file = self._prepare_fast_track_goal_state() - if HostPluginProtocol.get_fast_track_timestamp() is None: + if HostPluginProtocol.get_fast_track_timestamp() == timeutil.create_timestamp(datetime.min): raise Exception("The test setup did not save the Fast Track state") with patch("azurelinuxagent.common.conf.get_enable_fast_track", return_value=False): @@ -2927,8 +2927,55 @@ def test_it_should_clear_the_timestamp_for_the_most_recent_fast_track_goal_state with mock_update_handler(protocol) as update_handler: update_handler.run() - self.assertIsNone(HostPluginProtocol.get_fast_track_timestamp(), "The Fast Track state was not cleared") + self.assertEqual(HostPluginProtocol.get_fast_track_timestamp(), timeutil.create_timestamp(datetime.min), + "The Fast Track state was not cleared") + + def test_it_should_default_fast_track_timestamp_to_datetime_min(self): + data = DATA_FILE_VM_SETTINGS.copy() + # TODO: Currently, there's a limitation in the mocks where bumping the incarnation but the goal + # state will cause the agent to error out while trying to write the certificates to disk. These + # files have no dependencies on certs, so using them does not present that issue. + # + # Note that the scenario this test is representing does not depend on certificates at all, and + # can be changed to use the default files when the above limitation is addressed. + data["vm_settings"] = "hostgaplugin/vm_settings-fabric-no_thumbprints.json" + data['goal_state'] = 'wire/goal_state_no_certs.xml' + + def vm_settings_no_change(url, *_, **__): + if HttpRequestPredicates.is_host_plugin_vm_settings_request(url): + return MockHttpResponse(httpclient.NOT_MODIFIED) + return None + + def vm_settings_not_supported(url, *_, **__): + if HttpRequestPredicates.is_host_plugin_vm_settings_request(url): + return MockHttpResponse(404) + return None + + with mock_wire_protocol(data) as protocol: + + def mock_live_migration(iteration): + if iteration == 1: + protocol.mock_wire_data.set_incarnation(2) + protocol.set_http_handlers(http_get_handler=vm_settings_no_change) + elif iteration == 2: + protocol.mock_wire_data.set_incarnation(3) + protocol.set_http_handlers(http_get_handler=vm_settings_not_supported) + + with mock_update_handler(protocol, 3, on_new_iteration=mock_live_migration) as update_handler: + with patch("azurelinuxagent.ga.update.logger.error") as patched_error: + def check_for_errors(): + msg_fragment = "Error fetching the goal state:" + + for (args, _) in filter(lambda a: len(a) > 0, patched_error.call_args_list): + if msg_fragment in args[0]: + self.fail("Found error: {}".format(args[0])) + update_handler.run(debug=True) + check_for_errors() + + timestamp = protocol.client.get_host_plugin()._fast_track_timestamp + self.assertEqual(timestamp, timeutil.create_timestamp(datetime.min), + "Expected fast track time stamp to be set to {0}, got {1}".format(datetime.min, timestamp)) class HeartbeatTestCase(AgentTestCase): From 9895809483361a28a60322d8dc9be74771a9c28b Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Mon, 2 May 2022 13:37:19 -0700 Subject: [PATCH 55/89] Update agent version to 2.8.0.5. (#2580) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 99eb6473f..398367e6c 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.8.0.4' +AGENT_VERSION = '2.8.0.5' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 7b92617d5fb155b4c56a03313b00a77382db4d3c Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Thu, 19 May 2022 16:44:19 -0700 Subject: [PATCH 56/89] Create placeholder GoalState.*.xml file (#2594) Co-authored-by: narrieta --- azurelinuxagent/common/utils/archive.py | 22 ++++++++++++++++++++++ tests/utils/test_archive.py | 8 ++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/azurelinuxagent/common/utils/archive.py b/azurelinuxagent/common/utils/archive.py index 6123fdb0d..0be1544c5 100644 --- a/azurelinuxagent/common/utils/archive.py +++ b/azurelinuxagent/common/utils/archive.py @@ -39,6 +39,10 @@ ARCHIVE_DIRECTORY_NAME = 'history' +# TODO: See comment in GoalStateHistory._save_placeholder and remove this code when no longer needed +_PLACEHOLDER_FILE_NAME = 'GoalState.1.xml' +# END TODO + _MAX_ARCHIVED_STATES = 50 _CACHE_PATTERNS = [ @@ -173,6 +177,10 @@ def purge(self): def purge_legacy_goal_state_history(): lib_dir = conf.get_lib_dir() for current_file in os.listdir(lib_dir): + # TODO: See comment in GoalStateHistory._save_placeholder and remove this code when no longer needed + if current_file == _PLACEHOLDER_FILE_NAME: + return + # END TODO full_path = os.path.join(lib_dir, current_file) for pattern in _CACHE_PATTERNS: match = pattern.match(current_file) @@ -232,8 +240,22 @@ def save(self, data, file_name): self._errors = True logger.warn("Failed to save {0} to the goal state history: {1} [no additional errors saving the goal state will be reported]".format(file_name, e)) + @staticmethod + def _save_placeholder(): + """ + Some internal components took a dependency in the legacy GoalState.*.xml file. We create it here while those components are updated to remove the dependency. + When removing this code, also remove the check in StateArchiver.purge_legacy_goal_state_history, and the definition of _PLACEHOLDER_FILE_NAME + """ + try: + placeholder = os.path.join(conf.get_lib_dir(), _PLACEHOLDER_FILE_NAME) + with open(placeholder, "w") as handle: + handle.write("empty placeholder file") + except Exception as e: + logger.warn("Failed to save placeholder file ({0}): {1}".format(_PLACEHOLDER_FILE_NAME, e)) + def save_goal_state(self, text): self.save(text, _GOAL_STATE_FILE_NAME) + self._save_placeholder() def save_extensions_config(self, text): self.save(text, _EXT_CONF_FILE_NAME) diff --git a/tests/utils/test_archive.py b/tests/utils/test_archive.py index 0c649c9e2..5eee67c7d 100644 --- a/tests/utils/test_archive.py +++ b/tests/utils/test_archive.py @@ -134,10 +134,10 @@ def test_archive02(self): def test_purge_legacy_goal_state_history(self): with patch("azurelinuxagent.common.conf.get_lib_dir", return_value=self.tmp_dir): legacy_files = [ - 'GoalState.1.xml', - 'VmSettings.1.json', - 'Prod.1.manifest.xml', - 'ExtensionsConfig.1.xml', + 'GoalState.2.xml', + 'VmSettings.2.json', + 'Prod.2.manifest.xml', + 'ExtensionsConfig.2.xml', 'Microsoft.Azure.Extensions.CustomScript.1.xml', 'SharedConfig.xml', 'HostingEnvironmentConfig.xml', From fcc81a1c3c2a7167201d7e27852c615d15168302 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Thu, 19 May 2022 17:01:50 -0700 Subject: [PATCH 57/89] Update version to 2.8.0.6 (#2595) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 398367e6c..4ada1d135 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.8.0.5' +AGENT_VERSION = '2.8.0.6' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 5e4a9d8a1a8f0a02c9e1f9c009b44d2c740e0241 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Tue, 14 Jun 2022 13:50:55 -0700 Subject: [PATCH 58/89] fix network interface restart in RHEL9 (#2592) (#2612) (cherry picked from commit b8ca4323d91fd2fac54fbcb7aab0f6988b4cd0e5) --- azurelinuxagent/common/osutil/factory.py | 5 ++++- azurelinuxagent/common/osutil/redhat.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py index b8f4291b3..104a180bc 100644 --- a/azurelinuxagent/common/osutil/factory.py +++ b/azurelinuxagent/common/osutil/factory.py @@ -34,7 +34,7 @@ from .nsbsd import NSBSDOSUtil from .openbsd import OpenBSDOSUtil from .openwrt import OpenWRTOSUtil -from .redhat import RedhatOSUtil, Redhat6xOSUtil +from .redhat import RedhatOSUtil, Redhat6xOSUtil, RedhatOSModernUtil from .suse import SUSEOSUtil, SUSE11OSUtil from .photonos import PhotonOSUtil from .ubuntu import UbuntuOSUtil, Ubuntu12OSUtil, Ubuntu14OSUtil, \ @@ -106,6 +106,9 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) if Version(distro_version) < Version("7"): return Redhat6xOSUtil() + if Version(distro_version) == Version("8.6") or Version(distro_version) > Version("9"): + return RedhatOSModernUtil() + return RedhatOSUtil() if distro_name == "euleros": diff --git a/azurelinuxagent/common/osutil/redhat.py b/azurelinuxagent/common/osutil/redhat.py index 9759d1136..312dd1608 100644 --- a/azurelinuxagent/common/osutil/redhat.py +++ b/azurelinuxagent/common/osutil/redhat.py @@ -142,3 +142,25 @@ def get_dhcp_lease_endpoint(self): endpoint = self.get_endpoint_from_leases_path('/var/lib/NetworkManager/dhclient-*.lease') return endpoint + + +class RedhatOSModernUtil(RedhatOSUtil): + def __init__(self): # pylint: disable=W0235 + super(RedhatOSModernUtil, self).__init__() + + def restart_if(self, ifname, retries=3, wait=5): + """ + Restart an interface by bouncing the link. systemd-networkd observes + this event, and forces a renew of DHCP. + """ + retry_limit = retries + 1 + for attempt in range(1, retry_limit): + return_code = shellutil.run("ip link set {0} down && ip link set {0} up".format(ifname)) + if return_code == 0: + return + logger.warn("failed to restart {0}: return code {1}".format(ifname, return_code)) + if attempt < retry_limit: + logger.info("retrying in {0} seconds".format(wait)) + time.sleep(wait) + else: + logger.warn("exceeded restart retries") From dde20d2d1d6024065533848d48fc49c616f4074b Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Tue, 14 Jun 2022 15:00:10 -0700 Subject: [PATCH 59/89] set agent version to 2.7.2.0 (#2613) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 71dfe2937..3cec53bc6 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.7.1.0' +AGENT_VERSION = '2.7.2.0' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 7e39ec80f27f6fa04b3780b23b7e4a4332c2d0b0 Mon Sep 17 00:00:00 2001 From: narrieta Date: Fri, 17 Jun 2022 12:54:26 -0700 Subject: [PATCH 60/89] Parse missing agent manifests as empty --- .../protocol/extensions_goal_state_from_vm_settings.py | 4 +++- ...on_manifests.json => vm_settings-no_manifests.json} | 7 +------ .../test_extensions_goal_state_from_vm_settings.py | 10 +++++++++- 3 files changed, 13 insertions(+), 8 deletions(-) rename tests/data/hostgaplugin/{vm_settings-no_extension_manifests.json => vm_settings-no_manifests.json} (83%) diff --git a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py index 38cca48f1..10e036c9c 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py @@ -266,7 +266,9 @@ def _parse_agent_manifests(self, vm_settings): for family in families: name = family["name"] version = family.get("version") - uris = family["uris"] + uris = family.get("uris") + if uris is None: + uris = [] manifest = VMAgentManifest(name, version) for u in uris: manifest.uris.append(u) diff --git a/tests/data/hostgaplugin/vm_settings-no_extension_manifests.json b/tests/data/hostgaplugin/vm_settings-no_manifests.json similarity index 83% rename from tests/data/hostgaplugin/vm_settings-no_extension_manifests.json rename to tests/data/hostgaplugin/vm_settings-no_manifests.json index b084900b6..7ec3a5c3d 100644 --- a/tests/data/hostgaplugin/vm_settings-no_extension_manifests.json +++ b/tests/data/hostgaplugin/vm_settings-no_manifests.json @@ -27,12 +27,7 @@ }, "gaFamilies": [ { - "name": "Prod", - "uris": [ - "https://zrdfepirv2dz5prdstr07a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentral_manifest.xml", - "https://rdfepirv2dm1prdstr09.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentral_manifest.xml", - "https://zrdfepirv2dm5prdstr06a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentral_manifest.xml" - ] + "name": "Prod" } ], "extensionGoalStates": [ diff --git a/tests/protocol/test_extensions_goal_state_from_vm_settings.py b/tests/protocol/test_extensions_goal_state_from_vm_settings.py index 9bcba5ece..8cdfa81bf 100644 --- a/tests/protocol/test_extensions_goal_state_from_vm_settings.py +++ b/tests/protocol/test_extensions_goal_state_from_vm_settings.py @@ -69,9 +69,17 @@ def test_it_should_parse_missing_status_upload_blob_as_none(self): self.assertIsNone(extensions_goal_state.status_upload_blob, "Expected status upload blob to be None") self.assertEqual("BlockBlob", extensions_goal_state.status_upload_blob_type, "Expected status upload blob to be Block") + def test_it_should_parse_missing_agent_manifests_as_empty(self): + data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() + data_file["vm_settings"] = "hostgaplugin/vm_settings-no_manifests.json" + with mock_wire_protocol(data_file) as protocol: + extensions_goal_state = protocol.get_goal_state().extensions_goal_state + self.assertEqual(1, len(extensions_goal_state.agent_manifests), "Expected exactly one agent manifest. Got: {0}".format(extensions_goal_state.agent_manifests)) + self.assertListEqual([], extensions_goal_state.agent_manifests[0].uris, "Expected an empty list of agent manifests") + def test_it_should_parse_missing_extension_manifests_as_empty(self): data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() - data_file["vm_settings"] = "hostgaplugin/vm_settings-no_extension_manifests.json" + data_file["vm_settings"] = "hostgaplugin/vm_settings-no_manifests.json" with mock_wire_protocol(data_file) as protocol: extensions_goal_state = protocol.get_goal_state().extensions_goal_state From 40b3c50910683cda9287a4d3fe4777936a23d51b Mon Sep 17 00:00:00 2001 From: narrieta Date: Fri, 17 Jun 2022 13:18:11 -0700 Subject: [PATCH 61/89] Set agent version to 2.8.0.7 --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 4ada1d135..43769a02f 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.8.0.6' +AGENT_VERSION = '2.8.0.7' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From dbc82d3a948ff423529e8ea75d5ba9465709d67c Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 5 Jul 2022 13:21:32 -0700 Subject: [PATCH 62/89] Retry HGAP's extensionsArtifact requests on BAD_REQUEST status (#2621) * Retry HGAP's extensionsArtifact requests on BAD_REQUEST status * python 2.6 compat Co-authored-by: narrieta --- ...ensions_goal_state_from_extensions_config.py | 3 ++- azurelinuxagent/common/protocol/wire.py | 17 ++++++++++------- azurelinuxagent/common/utils/restutil.py | 9 +++++++++ azurelinuxagent/ga/update.py | 6 +++--- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py b/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py index c7e01dd20..8dce261ce 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py @@ -25,6 +25,7 @@ from azurelinuxagent.common.future import ustr from azurelinuxagent.common.protocol.extensions_goal_state import ExtensionsGoalState, GoalStateChannel, GoalStateSource from azurelinuxagent.common.protocol.restapi import ExtensionSettings, Extension, VMAgentManifest, ExtensionState, InVMGoalStateMetaData +from azurelinuxagent.common.utils import restutil from azurelinuxagent.common.utils.textutil import parse_doc, parse_json, findall, find, findtext, getattrib, gettext, format_exception, \ is_str_none_or_whitespace, is_str_empty @@ -99,7 +100,7 @@ def fetch_direct(): def fetch_through_host(): host = wire_client.get_host_plugin() uri, headers = host.get_artifact_request(artifacts_profile_blob) - content, _ = wire_client.fetch(uri, headers, use_proxy=False) + content, _ = wire_client.fetch(uri, headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES) return content logger.verbose("Retrieving the artifacts profile") diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 7923bea75..a57355e07 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -130,7 +130,7 @@ def get_goal_state(self): def _download_ext_handler_pkg_through_host(self, uri, destination): host = self.client.get_host_plugin() uri, headers = host.get_artifact_request(uri, host.manifest_uri) - success = self.client.stream(uri, destination, headers=headers, use_proxy=False, max_retry=1) + success = self.client.stream(uri, destination, headers=headers, use_proxy=False, max_retry=1) # set max_retry to 1 because extension packages already have a retry loop (see ExtHandlerInstance.download()) return success def download_ext_handler_pkg(self, uri, destination, headers=None, use_proxy=True): # pylint: disable=W0613 @@ -626,7 +626,7 @@ def call_storage_service(http_req, *args, **kwargs): def fetch_manifest_through_host(self, uri): host = self.get_host_plugin() uri, headers = host.get_artifact_request(uri) - response, _ = self.fetch(uri, headers, use_proxy=False, max_retry=1) + response, _ = self.fetch(uri, headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES) return response def fetch_manifest(self, version_uris, timeout_in_minutes=5, timeout_in_ms=0): @@ -649,9 +649,11 @@ def fetch_manifest(self, version_uris, timeout_in_minutes=5, timeout_in_ms=0): logger.verbose('The specified manifest URL is empty, ignored.') continue - direct_func = lambda: self.fetch(version_uri, max_retry=1)[0] # pylint: disable=W0640 + # Disable W0640: OK to use version_uri in a lambda within the loop's body + direct_func = lambda: self.fetch(version_uri)[0] # pylint: disable=W0640 # NOTE: the host_func may be called after refreshing the goal state, be careful about any goal state data # in the lambda. + # Disable W0640: OK to use version_uri in a lambda within the loop's body host_func = lambda: self.fetch_manifest_through_host(version_uri) # pylint: disable=W0640 try: @@ -690,7 +692,7 @@ def stream(self, uri, destination, headers=None, use_proxy=None, max_retry=None) return success - def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None, ok_codes=None): + def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None, retry_codes=None, ok_codes=None): """ max_retry indicates the maximum number of retries for the HTTP request; None indicates that the default value should be used @@ -699,14 +701,14 @@ def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None, logger.verbose("Fetch [{0}] with headers [{1}]", uri, headers) content = None response_headers = None - response = self._fetch_response(uri, headers, use_proxy, max_retry=max_retry, ok_codes=ok_codes) + response = self._fetch_response(uri, headers, use_proxy, max_retry=max_retry, retry_codes=retry_codes, ok_codes=ok_codes) if response is not None and not restutil.request_failed(response, ok_codes=ok_codes): response_content = response.read() content = self.decode_config(response_content) if decode else response_content response_headers = response.getheaders() return content, response_headers - def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, ok_codes=None): + def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, retry_codes=None, ok_codes=None): """ max_retry indicates the maximum number of retries for the HTTP request; None indicates that the default value should be used """ @@ -717,7 +719,8 @@ def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, ok_ uri, headers=headers, use_proxy=use_proxy, - max_retry=max_retry) + max_retry=max_retry, + retry_codes=retry_codes) host_plugin = self.get_host_plugin() diff --git a/azurelinuxagent/common/utils/restutil.py b/azurelinuxagent/common/utils/restutil.py index 0c6d6d9ad..8c2fc4e4e 100644 --- a/azurelinuxagent/common/utils/restutil.py +++ b/azurelinuxagent/common/utils/restutil.py @@ -56,6 +56,15 @@ 429, # Request Rate Limit Exceeded ] +# +# Currently the HostGAPlugin has an issue its cache that may produce a BAD_REQUEST failure for valid URIs when using the extensionArtifact API. +# Add this status to the retryable codes, but use it only when requesting downloads via the HostGAPlugin. The retry logic in the download code +# would give enough time to the HGAP to refresh its cache. Once the fix to address that issue is deployed, consider removing the use of +# HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES. +# +HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES = RETRY_CODES[:] # make a copy of RETRY_CODES +HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES.append(httpclient.BAD_REQUEST) + RESOURCE_GONE_CODES = [ httpclient.GONE ] diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 469b7ece6..583f38943 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -1614,7 +1614,7 @@ def _download(self): uri, headers = self.host.get_artifact_request(uri, self.host.manifest_uri) try: - if self._fetch(uri, headers=headers, use_proxy=False): + if self._fetch(uri, headers=headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES): if not HostPluginProtocol.is_default_channel: logger.verbose("Setting host plugin as default channel") HostPluginProtocol.is_default_channel = True @@ -1641,12 +1641,12 @@ def _download(self): message=msg) raise UpdateError(msg) - def _fetch(self, uri, headers=None, use_proxy=True): + def _fetch(self, uri, headers=None, use_proxy=True, retry_codes=None): package = None try: is_healthy = True error_response = '' - resp = restutil.http_get(uri, use_proxy=use_proxy, headers=headers, max_retry=1) + resp = restutil.http_get(uri, use_proxy=use_proxy, headers=headers, max_retry=3, retry_codes=retry_codes) # Use only 3 retries, since there are usually 5 or 6 URIs and we try all of them if restutil.request_succeeded(resp): package = resp.read() fileutil.write_file(self.get_agent_pkg_path(), From ee630fd722c82ff7c5166470cdf3c0fb6f8dfb25 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 5 Jul 2022 13:58:11 -0700 Subject: [PATCH 63/89] Retry HGAP's extensionsArtifact requests on BAD_REQUEST status (#2621) (#2622) * Retry HGAP's extensionsArtifact requests on BAD_REQUEST status * python 2.6 compat Co-authored-by: narrieta (cherry picked from commit dbc82d3a948ff423529e8ea75d5ba9465709d67c) --- ...ensions_goal_state_from_extensions_config.py | 3 ++- azurelinuxagent/common/protocol/wire.py | 17 ++++++++++------- azurelinuxagent/common/utils/restutil.py | 9 +++++++++ azurelinuxagent/ga/update.py | 6 +++--- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py b/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py index 7f8aeea3d..7c19532e6 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py @@ -25,6 +25,7 @@ from azurelinuxagent.common.future import ustr from azurelinuxagent.common.protocol.extensions_goal_state import ExtensionsGoalState from azurelinuxagent.common.protocol.restapi import ExtensionSettings, Extension, VMAgentManifest, ExtensionState, InVMGoalStateMetaData +from azurelinuxagent.common.utils import restutil from azurelinuxagent.common.utils.textutil import parse_doc, parse_json, findall, find, findtext, getattrib, gettext, format_exception, \ is_str_none_or_whitespace, is_str_empty @@ -98,7 +99,7 @@ def fetch_direct(): def fetch_through_host(): host = wire_client.get_host_plugin() uri, headers = host.get_artifact_request(artifacts_profile_blob) - content, _ = wire_client.fetch(uri, headers, use_proxy=False) + content, _ = wire_client.fetch(uri, headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES) return content logger.verbose("Retrieving the artifacts profile") diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 99d320b65..c61fb6c0c 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -147,7 +147,7 @@ def get_extensions_goal_state(self): def _download_ext_handler_pkg_through_host(self, uri, destination): host = self.client.get_host_plugin() uri, headers = host.get_artifact_request(uri, host.manifest_uri) - success = self.client.stream(uri, destination, headers=headers, use_proxy=False, max_retry=1) + success = self.client.stream(uri, destination, headers=headers, use_proxy=False, max_retry=1) # set max_retry to 1 because extension packages already have a retry loop (see ExtHandlerInstance.download()) return success def download_ext_handler_pkg(self, uri, destination, headers=None, use_proxy=True): # pylint: disable=W0613 @@ -652,7 +652,7 @@ def call_storage_service(http_req, *args, **kwargs): def fetch_manifest_through_host(self, uri): host = self.get_host_plugin() uri, headers = host.get_artifact_request(uri) - response, _ = self.fetch(uri, headers, use_proxy=False, max_retry=1) + response, _ = self.fetch(uri, headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES) return response def fetch_manifest(self, version_uris, timeout_in_minutes=5, timeout_in_ms=0): @@ -675,9 +675,11 @@ def fetch_manifest(self, version_uris, timeout_in_minutes=5, timeout_in_ms=0): logger.verbose('The specified manifest URL is empty, ignored.') continue - direct_func = lambda: self.fetch(version_uri, max_retry=1)[0] # pylint: disable=W0640 + # Disable W0640: OK to use version_uri in a lambda within the loop's body + direct_func = lambda: self.fetch(version_uri)[0] # pylint: disable=W0640 # NOTE: the host_func may be called after refreshing the goal state, be careful about any goal state data # in the lambda. + # Disable W0640: OK to use version_uri in a lambda within the loop's body host_func = lambda: self.fetch_manifest_through_host(version_uri) # pylint: disable=W0640 try: @@ -716,7 +718,7 @@ def stream(self, uri, destination, headers=None, use_proxy=None, max_retry=None) return success - def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None, ok_codes=None): + def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None, retry_codes=None, ok_codes=None): """ max_retry indicates the maximum number of retries for the HTTP request; None indicates that the default value should be used @@ -725,14 +727,14 @@ def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None, logger.verbose("Fetch [{0}] with headers [{1}]", uri, headers) content = None response_headers = None - response = self._fetch_response(uri, headers, use_proxy, max_retry=max_retry, ok_codes=ok_codes) + response = self._fetch_response(uri, headers, use_proxy, max_retry=max_retry, retry_codes=retry_codes, ok_codes=ok_codes) if response is not None and not restutil.request_failed(response, ok_codes=ok_codes): response_content = response.read() content = self.decode_config(response_content) if decode else response_content response_headers = response.getheaders() return content, response_headers - def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, ok_codes=None): + def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, retry_codes=None, ok_codes=None): """ max_retry indicates the maximum number of retries for the HTTP request; None indicates that the default value should be used """ @@ -743,7 +745,8 @@ def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, ok_ uri, headers=headers, use_proxy=use_proxy, - max_retry=max_retry) + max_retry=max_retry, + retry_codes=retry_codes) host_plugin = self.get_host_plugin() diff --git a/azurelinuxagent/common/utils/restutil.py b/azurelinuxagent/common/utils/restutil.py index 0c6d6d9ad..8c2fc4e4e 100644 --- a/azurelinuxagent/common/utils/restutil.py +++ b/azurelinuxagent/common/utils/restutil.py @@ -56,6 +56,15 @@ 429, # Request Rate Limit Exceeded ] +# +# Currently the HostGAPlugin has an issue its cache that may produce a BAD_REQUEST failure for valid URIs when using the extensionArtifact API. +# Add this status to the retryable codes, but use it only when requesting downloads via the HostGAPlugin. The retry logic in the download code +# would give enough time to the HGAP to refresh its cache. Once the fix to address that issue is deployed, consider removing the use of +# HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES. +# +HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES = RETRY_CODES[:] # make a copy of RETRY_CODES +HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES.append(httpclient.BAD_REQUEST) + RESOURCE_GONE_CODES = [ httpclient.GONE ] diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 2b0e857b4..f35338cee 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -1317,7 +1317,7 @@ def _download(self): uri, headers = self.host.get_artifact_request(uri, self.host.manifest_uri) try: - if self._fetch(uri, headers=headers, use_proxy=False): + if self._fetch(uri, headers=headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES): if not HostPluginProtocol.is_default_channel: logger.verbose("Setting host plugin as default channel") HostPluginProtocol.is_default_channel = True @@ -1344,12 +1344,12 @@ def _download(self): message=msg) raise UpdateError(msg) - def _fetch(self, uri, headers=None, use_proxy=True): + def _fetch(self, uri, headers=None, use_proxy=True, retry_codes=None): package = None try: is_healthy = True error_response = '' - resp = restutil.http_get(uri, use_proxy=use_proxy, headers=headers, max_retry=1) + resp = restutil.http_get(uri, use_proxy=use_proxy, headers=headers, max_retry=3, retry_codes=retry_codes) # Use only 3 retries, since there are usually 5 or 6 URIs and we try all of them if restutil.request_succeeded(resp): package = resp.read() fileutil.write_file(self.get_agent_pkg_path(), From e54072819d985059caf40fc59816dbb81f80050f Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Tue, 5 Jul 2022 14:39:06 -0700 Subject: [PATCH 64/89] fix if command in rhel v8.6+ (#2624) --- azurelinuxagent/common/osutil/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py index 104a180bc..5b7db4249 100644 --- a/azurelinuxagent/common/osutil/factory.py +++ b/azurelinuxagent/common/osutil/factory.py @@ -106,7 +106,7 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) if Version(distro_version) < Version("7"): return Redhat6xOSUtil() - if Version(distro_version) == Version("8.6") or Version(distro_version) > Version("9"): + if Version(distro_version) >= Version("8.6"): return RedhatOSModernUtil() return RedhatOSUtil() From e84fbe11947fe495625a272959ff51d05353d3cb Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 5 Jul 2022 15:08:42 -0700 Subject: [PATCH 65/89] Set agent version to 2.7.3.0 (#2625) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 3cec53bc6..6b608e7d2 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.7.2.0' +AGENT_VERSION = '2.7.3.0' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 4a304a878fe1c618dad48d922ad12158bad6acee Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 6 Jul 2022 16:48:10 -0700 Subject: [PATCH 66/89] Set Agent version to 2.8.0.8 (#2627) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 43769a02f..b65a1d79c 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.8.0.7' +AGENT_VERSION = '2.8.0.8' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 3866c691e9fb1b8dcaae3a559608e56cb07e6848 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 11 Jul 2022 11:38:26 -0700 Subject: [PATCH 67/89] fix network interface restart in RHEL9 (#2592) (#2629) (cherry picked from commit b8ca4323d91fd2fac54fbcb7aab0f6988b4cd0e5) Co-authored-by: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> --- azurelinuxagent/common/osutil/factory.py | 5 ++++- azurelinuxagent/common/osutil/redhat.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py index 2ed4be78b..61c2e6d20 100644 --- a/azurelinuxagent/common/osutil/factory.py +++ b/azurelinuxagent/common/osutil/factory.py @@ -34,7 +34,7 @@ from .nsbsd import NSBSDOSUtil from .openbsd import OpenBSDOSUtil from .openwrt import OpenWRTOSUtil -from .redhat import RedhatOSUtil, Redhat6xOSUtil +from .redhat import RedhatOSUtil, Redhat6xOSUtil, RedhatOSModernUtil from .suse import SUSEOSUtil, SUSE11OSUtil from .photonos import PhotonOSUtil from .ubuntu import UbuntuOSUtil, Ubuntu12OSUtil, Ubuntu14OSUtil, \ @@ -107,6 +107,9 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) if Version(distro_version) < Version("7"): return Redhat6xOSUtil() + if Version(distro_version) == Version("8.6") or Version(distro_version) > Version("9"): + return RedhatOSModernUtil() + return RedhatOSUtil() if distro_name == "euleros": diff --git a/azurelinuxagent/common/osutil/redhat.py b/azurelinuxagent/common/osutil/redhat.py index 9759d1136..312dd1608 100644 --- a/azurelinuxagent/common/osutil/redhat.py +++ b/azurelinuxagent/common/osutil/redhat.py @@ -142,3 +142,25 @@ def get_dhcp_lease_endpoint(self): endpoint = self.get_endpoint_from_leases_path('/var/lib/NetworkManager/dhclient-*.lease') return endpoint + + +class RedhatOSModernUtil(RedhatOSUtil): + def __init__(self): # pylint: disable=W0235 + super(RedhatOSModernUtil, self).__init__() + + def restart_if(self, ifname, retries=3, wait=5): + """ + Restart an interface by bouncing the link. systemd-networkd observes + this event, and forces a renew of DHCP. + """ + retry_limit = retries + 1 + for attempt in range(1, retry_limit): + return_code = shellutil.run("ip link set {0} down && ip link set {0} up".format(ifname)) + if return_code == 0: + return + logger.warn("failed to restart {0}: return code {1}".format(ifname, return_code)) + if attempt < retry_limit: + logger.info("retrying in {0} seconds".format(wait)) + time.sleep(wait) + else: + logger.warn("exceeded restart retries") From 3a9fc4524ef9343f058c33afdcf2f759607c2770 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 11 Jul 2022 12:58:37 -0700 Subject: [PATCH 68/89] fix if command in rhel v8.6+ (#2624) (#2630) (cherry picked from commit e54072819d985059caf40fc59816dbb81f80050f) Co-authored-by: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> --- azurelinuxagent/common/osutil/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py index 61c2e6d20..f14fdfbb5 100644 --- a/azurelinuxagent/common/osutil/factory.py +++ b/azurelinuxagent/common/osutil/factory.py @@ -107,7 +107,7 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) if Version(distro_version) < Version("7"): return Redhat6xOSUtil() - if Version(distro_version) == Version("8.6") or Version(distro_version) > Version("9"): + if Version(distro_version) >= Version("8.6"): return RedhatOSModernUtil() return RedhatOSUtil() From 672dbf32f565a14632bcfd081c3c553c821fca77 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 11 Jul 2022 13:18:52 -0700 Subject: [PATCH 69/89] Set Agent version to 2.8.0.9 (#2631) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index b65a1d79c..75820d69b 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.8.0.8' +AGENT_VERSION = '2.8.0.9' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From ac56d0e1395db01e23c5e4a3aeefcc702752dd84 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 22 Jul 2022 16:19:14 -0700 Subject: [PATCH 70/89] Cleanup history directory when creating new subdirectories (#2633) * Cleanup history directory when creating new subdirectories * Review feedback Co-authored-by: narrieta --- azurelinuxagent/common/protocol/goal_state.py | 10 ++-- azurelinuxagent/common/protocol/wire.py | 10 ++-- azurelinuxagent/common/utils/archive.py | 51 ++++++++++++---- azurelinuxagent/ga/update.py | 7 +-- tests/utils/test_archive.py | 58 +++---------------- 5 files changed, 63 insertions(+), 73 deletions(-) diff --git a/azurelinuxagent/common/protocol/goal_state.py b/azurelinuxagent/common/protocol/goal_state.py index 8b508f61a..4d354e567 100644 --- a/azurelinuxagent/common/protocol/goal_state.py +++ b/azurelinuxagent/common/protocol/goal_state.py @@ -352,12 +352,12 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): certs_uri = findtext(xml_doc, "Certificates") if certs_uri is not None: xml_text = self._wire_client.fetch_config(certs_uri, self._wire_client.get_header_for_cert()) - certs = Certificates(xml_text) + certs = Certificates(xml_text, self.logger) # Log and save the certificates summary (i.e. the thumbprint but not the certificate itself) to the goal state history for c in certs.summary: - logger.info("Downloaded certificate {0}".format(c)) + self.logger.info("Downloaded certificate {0}".format(c)) if len(certs.warnings) > 0: - logger.warn(certs.warnings) + self.logger.warn(certs.warnings) self._history.save_certificates(json.dumps(certs.summary)) remote_access = None @@ -403,7 +403,7 @@ def __init__(self, xml_text): class Certificates(object): - def __init__(self, xml_text): + def __init__(self, xml_text, my_logger): self.cert_list = CertList() self.summary = [] # debugging info self.warnings = [] @@ -421,7 +421,7 @@ def __init__(self, xml_text): # if the certificates format is not Pkcs7BlobWithPfxContents do not parse it certificateFormat = findtext(xml_doc, "Format") if certificateFormat and certificateFormat != "Pkcs7BlobWithPfxContents": - logger.warn("The Format is not Pkcs7BlobWithPfxContents. Format is " + certificateFormat) + my_logger.warn("The Format is not Pkcs7BlobWithPfxContents. Format is " + certificateFormat) return cryptutil = CryptUtil(conf.get_openssl_cmd()) diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index a57355e07..b8b05c98b 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -767,7 +767,7 @@ def update_goal_state(self, force_update=False, silent=False): Updates the goal state if the incarnation or etag changed or if 'force_update' is True """ try: - if force_update: + if force_update and not silent: logger.info("Forcing an update of the goal state.") if self._goal_state is None or force_update: @@ -970,11 +970,13 @@ def upload_status_blob(self): if extensions_goal_state.status_upload_blob is None: # the status upload blob is in ExtensionsConfig so force a full goal state refresh - self.update_goal_state(force_update=True) + self.update_goal_state(force_update=True, silent=True) extensions_goal_state = self.get_goal_state().extensions_goal_state - if extensions_goal_state.status_upload_blob is None: - raise ProtocolNotFoundError("Status upload uri is missing") + if extensions_goal_state.status_upload_blob is None: + raise ProtocolNotFoundError("Status upload uri is missing") + + logger.info("Refreshed the goal state to get the status upload blob. New Goal State ID: {0}", extensions_goal_state.id) blob_type = extensions_goal_state.status_upload_blob_type diff --git a/azurelinuxagent/common/utils/archive.py b/azurelinuxagent/common/utils/archive.py index 0be1544c5..b624d1742 100644 --- a/azurelinuxagent/common/utils/archive.py +++ b/azurelinuxagent/common/utils/archive.py @@ -162,17 +162,6 @@ def __init__(self, lib_dir): if exception.errno != errno.EEXIST: logger.warn("{0} : {1}", self._source, exception.strerror) - def purge(self): - """ - Delete "old" archive directories and .zip archives. Old - is defined as any directories or files older than the X - newest ones. Also, clean up any legacy history files. - """ - states = self._get_archive_states() - - for state in states[_MAX_ARCHIVED_STATES:]: - state.delete() - @staticmethod def purge_legacy_goal_state_history(): lib_dir = conf.get_lib_dir() @@ -222,6 +211,8 @@ def __init__(self, time, tag): timestamp = timeutil.create_history_timestamp(time) self._root = os.path.join(conf.get_lib_dir(), ARCHIVE_DIRECTORY_NAME, "{0}__{1}".format(timestamp, tag) if tag is not None else timestamp) + GoalStateHistory._purge() + @staticmethod def tag_exists(tag): """ @@ -240,6 +231,44 @@ def save(self, data, file_name): self._errors = True logger.warn("Failed to save {0} to the goal state history: {1} [no additional errors saving the goal state will be reported]".format(file_name, e)) + _purge_error_count = 0 + + @staticmethod + def _purge(): + """ + Delete "old" history directories and .zip archives. Old is defined as any directories or files older than the X newest ones. + """ + try: + history_root = os.path.join(conf.get_lib_dir(), ARCHIVE_DIRECTORY_NAME) + + if not os.path.exists(history_root): + return + + items = [] + for current_item in os.listdir(history_root): + full_path = os.path.join(history_root, current_item) + items.append(full_path) + items.sort(key=os.path.getctime, reverse=True) + + for current_item in items[_MAX_ARCHIVED_STATES:]: + if os.path.isfile(current_item): + os.remove(current_item) + else: + shutil.rmtree(current_item) + + if GoalStateHistory._purge_error_count > 0: + GoalStateHistory._purge_error_count = 0 + # Log a success message when we are recovering from errors. + logger.info("Successfully cleaned up the goal state history directory") + + except Exception as e: + GoalStateHistory._purge_error_count += 1 + if GoalStateHistory._purge_error_count < 5: + logger.warn("Failed to clean up the goal state history directory: {0}".format(e)) + elif GoalStateHistory._purge_error_count == 5: + logger.warn("Failed to clean up the goal state history directory [will stop reporting these errors]: {0}".format(e)) + + @staticmethod def _save_placeholder(): """ diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 583f38943..66b0de5d4 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -635,9 +635,9 @@ def _process_goal_state(self, exthandlers_handler, remote_access_handler): if self._processing_new_incarnation(): remote_access_handler.run() - # lastly, cleanup the goal state history (but do it only on new goal states - no need to do it on every iteration) + # lastly, archive the goal state history (but do it only on new goal states - no need to do it on every iteration) if self._processing_new_extensions_goal_state(): - UpdateHandler._cleanup_goal_state_history() + UpdateHandler._archive_goal_state_history() finally: if self._goal_state is not None: @@ -645,10 +645,9 @@ def _process_goal_state(self, exthandlers_handler, remote_access_handler): self._last_extensions_gs_id = self._goal_state.extensions_goal_state.id @staticmethod - def _cleanup_goal_state_history(): + def _archive_goal_state_history(): try: archiver = StateArchiver(conf.get_lib_dir()) - archiver.purge() archiver.archive() except Exception as exception: logger.warn("Error cleaning up the goal state history: {0}", ustr(exception)) diff --git a/tests/utils/test_archive.py b/tests/utils/test_archive.py index 5eee67c7d..ce97d65fd 100644 --- a/tests/utils/test_archive.py +++ b/tests/utils/test_archive.py @@ -6,8 +6,9 @@ from datetime import datetime, timedelta import azurelinuxagent.common.logger as logger +from azurelinuxagent.common import conf from azurelinuxagent.common.utils import fileutil, timeutil -from azurelinuxagent.common.utils.archive import StateArchiver, _MAX_ARCHIVED_STATES +from azurelinuxagent.common.utils.archive import GoalStateHistory, StateArchiver, _MAX_ARCHIVED_STATES, ARCHIVE_DIRECTORY_NAME from tests.tools import AgentTestCase, patch debug = False @@ -28,7 +29,7 @@ def setUp(self): self.tmp_dir = tempfile.mkdtemp(prefix=prefix) def _write_file(self, filename, contents=None): - full_name = os.path.join(self.tmp_dir, filename) + full_name = os.path.join(conf.get_lib_dir(), filename) fileutil.mkdir(os.path.dirname(full_name)) with open(full_name, 'w') as file_handler: @@ -38,7 +39,7 @@ def _write_file(self, filename, contents=None): @property def history_dir(self): - return os.path.join(self.tmp_dir, 'history') + return os.path.join(conf.get_lib_dir(), ARCHIVE_DIRECTORY_NAME) @staticmethod def _parse_archive_name(name): @@ -66,7 +67,7 @@ def test_archive_should_zip_all_but_the_latest_goal_state_in_the_history_folder( self._write_file(os.path.join(directory, current_file)) test_directories.append(directory) - test_subject = StateArchiver(self.tmp_dir) + test_subject = StateArchiver(conf.get_lib_dir()) # NOTE: StateArchiver sorts the state directories by creation time, but the test files are created too fast and the # time resolution is too coarse, so instead we mock getctime to simply return the path of the file with patch("azurelinuxagent.common.utils.archive.os.path.getctime", side_effect=lambda path: path): @@ -83,9 +84,9 @@ def test_archive_should_zip_all_but_the_latest_goal_state_in_the_history_folder( self.assertTrue(os.path.exists(test_directories[2]), "{0}, the latest goal state, should not have being removed".format(test_directories[2])) - def test_archive02(self): + def test_goal_state_history_init_should_purge_old_items(self): """ - StateArchiver should purge the MAX_ARCHIVED_STATES oldest files + GoalStateHistory.__init__ should _purge the MAX_ARCHIVED_STATES oldest files or directories. The oldest timestamps are purged first. This test case creates a mixture of archive files and directories. @@ -112,11 +113,10 @@ def test_archive02(self): self.assertEqual(total, len(os.listdir(self.history_dir))) - test_subject = StateArchiver(self.tmp_dir) - # NOTE: StateArchiver sorts the state directories by creation time, but the test files are created too fast and the + # NOTE: The purge method sorts the items by creation time, but the test files are created too fast and the # time resolution is too coarse, so instead we mock getctime to simply return the path of the file with patch("azurelinuxagent.common.utils.archive.os.path.getctime", side_effect=lambda path: path): - test_subject.purge() + GoalStateHistory(datetime.utcnow(), 'test') archived_entries = os.listdir(self.history_dir) self.assertEqual(_MAX_ARCHIVED_STATES, len(archived_entries)) @@ -153,46 +153,6 @@ def test_purge_legacy_goal_state_history(self): for f in legacy_files: self.assertFalse(os.path.exists(f), "Legacy file {0} was not removed".format(f)) - def test_archive03(self): - """ - All archives should be purged, both with the legacy naming (with incarnation number) and with the new naming. - """ - start = datetime.now() - timestamp1 = start + timedelta(seconds=5) - timestamp2 = start + timedelta(seconds=10) - timestamp3 = start + timedelta(seconds=10) - - dir_old = timestamp1.isoformat() - dir_new = "{0}_incarnation_1".format(timestamp2.isoformat()) - - archive_old = "{0}.zip".format(timestamp1.isoformat()) - archive_new = "{0}_incarnation_1.zip".format(timestamp2.isoformat()) - - status = "{0}.zip".format(timestamp3.isoformat()) - - self._write_file(os.path.join("history", dir_old, "Prod.manifest.xml")) - self._write_file(os.path.join("history", dir_new, "Prod.manifest.xml")) - self._write_file(os.path.join("history", archive_old)) - self._write_file(os.path.join("history", archive_new)) - self._write_file(os.path.join("history", status)) - - self.assertEqual(5, len(os.listdir(self.history_dir)), "Not all entries were archived!") - - test_subject = StateArchiver(self.tmp_dir) - with patch("azurelinuxagent.common.utils.archive._MAX_ARCHIVED_STATES", 0): - test_subject.purge() - - archived_entries = os.listdir(self.history_dir) - self.assertEqual(0, len(archived_entries), "Not all entries were purged!") - - def test_archive04(self): - """ - The archive directory is created if it does not exist. - - This failure was caught when .purge() was called before .archive(). - """ - test_subject = StateArchiver(os.path.join(self.tmp_dir, 'does-not-exist')) - test_subject.purge() @staticmethod def parse_isoformat(timestamp_str): From 0312e9584341a7529a62de717b3503138fdd5f69 Mon Sep 17 00:00:00 2001 From: narrieta Date: Mon, 1 Aug 2022 15:45:11 -0700 Subject: [PATCH 71/89] Set agent version to 2.8.0.10 --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 75820d69b..891e1c395 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.8.0.9' +AGENT_VERSION = '2.8.0.10' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 6f9c516ac9ee20413ab3c90acdf125acfd3e9976 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 15 Aug 2022 16:03:06 -0700 Subject: [PATCH 72/89] Save sharedconfig to disk (#2649) * Save sharedconfig to disk * Update tests * pylint warnings Co-authored-by: narrieta --- azurelinuxagent/common/protocol/goal_state.py | 12 +++++++++--- azurelinuxagent/common/utils/archive.py | 11 +++++++---- azurelinuxagent/ga/update.py | 3 +-- tests/protocol/test_goal_state.py | 11 ++++++++++- tests/utils/test_archive.py | 9 +++++++-- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/azurelinuxagent/common/protocol/goal_state.py b/azurelinuxagent/common/protocol/goal_state.py index 4d354e567..97ae270f8 100644 --- a/azurelinuxagent/common/protocol/goal_state.py +++ b/azurelinuxagent/common/protocol/goal_state.py @@ -33,7 +33,7 @@ from azurelinuxagent.common.protocol.hostplugin import VmSettingsNotSupported, VmSettingsSupportStopped from azurelinuxagent.common.protocol.restapi import Cert, CertList, RemoteAccessUser, RemoteAccessUsersList from azurelinuxagent.common.utils import fileutil -from azurelinuxagent.common.utils.archive import GoalStateHistory +from azurelinuxagent.common.utils.archive import GoalStateHistory, SHARED_CONF_FILE_NAME from azurelinuxagent.common.utils.cryptutil import CryptUtil from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, findtext, getattrib @@ -345,8 +345,14 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): shared_conf_uri = findtext(xml_doc, "SharedConfig") xml_text = self._wire_client.fetch_config(shared_conf_uri, self._wire_client.get_header()) - shared_conf = SharedConfig(xml_text) + shared_config = SharedConfig(xml_text) self._history.save_shared_conf(xml_text) + # SharedConfig.xml is used by other components (Azsec and Singularity/HPC Infiniband), so save it to the agent's root directory as well + shared_config_file = os.path.join(conf.get_lib_dir(), SHARED_CONF_FILE_NAME) + try: + fileutil.write_file(shared_config_file, xml_text) + except Exception as e: + logger.warn("Failed to save {0}: {1}".format(shared_config, e)) certs = EmptyCertificates() certs_uri = findtext(xml_doc, "Certificates") @@ -372,7 +378,7 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): self._role_config_name = role_config_name self._container_id = container_id self._hosting_env = hosting_env - self._shared_conf = shared_conf + self._shared_conf = shared_config self._certs = certs self._remote_access = remote_access diff --git a/azurelinuxagent/common/utils/archive.py b/azurelinuxagent/common/utils/archive.py index b624d1742..1f8fdd931 100644 --- a/azurelinuxagent/common/utils/archive.py +++ b/azurelinuxagent/common/utils/archive.py @@ -46,11 +46,13 @@ _MAX_ARCHIVED_STATES = 50 _CACHE_PATTERNS = [ + # + # Note that SharedConfig.xml is not included here; this file is used by other components (Azsec and Singularity/HPC Infiniband) + # re.compile(r"^VmSettings\.\d+\.json$"), re.compile(r"^(.*)\.(\d+)\.(agentsManifest)$", re.IGNORECASE), re.compile(r"^(.*)\.(\d+)\.(manifest\.xml)$", re.IGNORECASE), re.compile(r"^(.*)\.(\d+)\.(xml)$", re.IGNORECASE), - re.compile(r"^SharedConfig\.xml$", re.IGNORECASE), re.compile(r"^HostingEnvironmentConfig\.xml$", re.IGNORECASE), re.compile(r"^RemoteAccess\.xml$", re.IGNORECASE), re.compile(r"^waagent_status\.\d+\.json$"), @@ -78,12 +80,12 @@ _VM_SETTINGS_FILE_NAME = "VmSettings.json" _CERTIFICATES_FILE_NAME = "Certificates.json" _HOSTING_ENV_FILE_NAME = "HostingEnvironmentConfig.xml" -_SHARED_CONF_FILE_NAME = "SharedConfig.xml" _REMOTE_ACCESS_FILE_NAME = "RemoteAccess.xml" _EXT_CONF_FILE_NAME = "ExtensionsConfig.xml" _MANIFEST_FILE_NAME = "{0}.manifest.xml" AGENT_STATUS_FILE = "waagent_status.json" +SHARED_CONF_FILE_NAME = "SharedConfig.xml" # TODO: use @total_ordering once RHEL/CentOS and SLES 11 are EOL. # @total_ordering first appeared in Python 2.7 and 3.2 @@ -166,9 +168,10 @@ def __init__(self, lib_dir): def purge_legacy_goal_state_history(): lib_dir = conf.get_lib_dir() for current_file in os.listdir(lib_dir): + # Don't remove the placeholder goal state file. # TODO: See comment in GoalStateHistory._save_placeholder and remove this code when no longer needed if current_file == _PLACEHOLDER_FILE_NAME: - return + continue # END TODO full_path = os.path.join(lib_dir, current_file) for pattern in _CACHE_PATTERNS: @@ -302,4 +305,4 @@ def save_hosting_env(self, text): self.save(text, _HOSTING_ENV_FILE_NAME) def save_shared_conf(self, text): - self.save(text, _SHARED_CONF_FILE_NAME) + self.save(text, SHARED_CONF_FILE_NAME) diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 66b0de5d4..d17fff6a4 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -378,6 +378,7 @@ def run(self, debug=False): self._ensure_firewall_rules_persisted(dst_ip=protocol.get_endpoint()) self._add_accept_tcp_firewall_rule_if_not_enabled(dst_ip=protocol.get_endpoint()) self._reset_legacy_blacklisted_agents() + self._cleanup_legacy_goal_state_history() # Get all thread handlers telemetry_handler = get_send_telemetry_events_handler(self.protocol_util) @@ -396,8 +397,6 @@ def run(self, debug=False): logger.info("Goal State Period: {0} sec. This indicates how often the agent checks for new goal states and reports status.", self._goal_state_period) - self._cleanup_legacy_goal_state_history() - while self.is_running: self._check_daemon_running(debug) self._check_threads_running(all_thread_handlers) diff --git a/tests/protocol/test_goal_state.py b/tests/protocol/test_goal_state.py index c77417159..87a1db50e 100644 --- a/tests/protocol/test_goal_state.py +++ b/tests/protocol/test_goal_state.py @@ -8,6 +8,7 @@ import re import time +from azurelinuxagent.common import conf from azurelinuxagent.common.future import httpclient from azurelinuxagent.common.protocol.extensions_goal_state import GoalStateSource, GoalStateChannel from azurelinuxagent.common.protocol.extensions_goal_state_from_extensions_config import ExtensionsGoalStateFromExtensionsConfig @@ -96,7 +97,15 @@ def test_fetch_goal_state_should_raise_on_incomplete_goal_state(self): GoalState(protocol.client) self.assertEqual(_GET_GOAL_STATE_MAX_ATTEMPTS, mock_sleep.call_count, "Unexpected number of retries") - def test_instantiating_goal_state_should_save_the_goal_state_to_the_history_directory(self): + def test_fetching_the_goal_state_should_save_the_shared_config(self): + # SharedConfig.xml is used by other components (Azsec and Singularity/HPC Infiniband); verify that we do not delete it + with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: + _ = GoalState(protocol.client) + + shared_config = os.path.join(conf.get_lib_dir(), 'SharedConfig.xml') + self.assertTrue(os.path.exists(shared_config), "{0} should have been created".format(shared_config)) + + def test_fetching_the_goal_state_should_save_the_goal_state_to_the_history_directory(self): with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: protocol.mock_wire_data.set_incarnation(999) protocol.mock_wire_data.set_etag(888) diff --git a/tests/utils/test_archive.py b/tests/utils/test_archive.py index ce97d65fd..54766862f 100644 --- a/tests/utils/test_archive.py +++ b/tests/utils/test_archive.py @@ -133,27 +133,32 @@ def test_goal_state_history_init_should_purge_old_items(self): def test_purge_legacy_goal_state_history(self): with patch("azurelinuxagent.common.conf.get_lib_dir", return_value=self.tmp_dir): + # SharedConfig.xml is used by other components (Azsec and Singularity/HPC Infiniband); verify that we do not delete it + shared_config = os.path.join(self.tmp_dir, 'SharedConfig.xml') + legacy_files = [ 'GoalState.2.xml', 'VmSettings.2.json', 'Prod.2.manifest.xml', 'ExtensionsConfig.2.xml', 'Microsoft.Azure.Extensions.CustomScript.1.xml', - 'SharedConfig.xml', 'HostingEnvironmentConfig.xml', 'RemoteAccess.xml', 'waagent_status.1.json' ] legacy_files = [os.path.join(self.tmp_dir, f) for f in legacy_files] + + self._write_file(shared_config) for f in legacy_files: self._write_file(f) StateArchiver.purge_legacy_goal_state_history() + self.assertTrue(os.path.exists(shared_config), "{0} should not have been removed".format(shared_config)) + for f in legacy_files: self.assertFalse(os.path.exists(f), "Legacy file {0} was not removed".format(f)) - @staticmethod def parse_isoformat(timestamp_str): return datetime.strptime(timestamp_str, '%Y-%m-%dT%H:%M:%S.%f') From 1b08441e3d4432bbc4641cec0d31b65e810334ff Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Mon, 15 Aug 2022 16:15:37 -0700 Subject: [PATCH 73/89] Set Agent version to 2.8.0.11 (#2650) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 891e1c395..099ebc542 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '2.8.0.10' +AGENT_VERSION = '2.8.0.11' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 0e42320ec393e7af6fe7d55c9fd24db56343d86c Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Tue, 13 Sep 2022 15:44:25 -0700 Subject: [PATCH 74/89] set agent version to 2.9.0.0 (#2664) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index ff9c903b9..20d2c605d 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '9.9.9.9' +AGENT_VERSION = '2.9.0.0' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 8a79ea77371859bde3ae1e0884c59f7530ea0d0d Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 20 Sep 2022 07:06:53 -0700 Subject: [PATCH 75/89] Add logging statements for mrseq migration during update (#2667) Co-authored-by: narrieta --- azurelinuxagent/ga/exthandlers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 99f380944..c01fc15bc 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -1210,7 +1210,10 @@ def copy_status_files(self, old_ext_handler_i): old_ext_mrseq_file = os.path.join(old_ext_dir, "mrseq") if os.path.isfile(old_ext_mrseq_file): + logger.info("Migrating {0} to {1}.", old_ext_mrseq_file, new_ext_dir) shutil.copy2(old_ext_mrseq_file, new_ext_dir) + else: + logger.info("{0} does not exist, no migration is needed.", old_ext_mrseq_file) old_ext_status_dir = old_ext_handler_i.get_status_dir() new_ext_status_dir = self.get_status_dir() From 3928dbd1796d9d9a898f6516b43c35aa0d080dd0 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 20 Sep 2022 11:46:51 -0700 Subject: [PATCH 76/89] test fixes --- azurelinuxagent/common/conf.py | 4 -- tests/ga/test_update.py | 49 ------------------- ..._extensions_goal_state_from_vm_settings.py | 8 --- 3 files changed, 61 deletions(-) diff --git a/azurelinuxagent/common/conf.py b/azurelinuxagent/common/conf.py index 8664d77b8..bd101f617 100644 --- a/azurelinuxagent/common/conf.py +++ b/azurelinuxagent/common/conf.py @@ -559,10 +559,6 @@ def get_cgroup_monitor_expiry_time(conf=__conf__): """ cgroups monitoring for pilot extensions disabled after expiry time - NOTE: This option is experimental and may be removed in later versions of the Agent. - """ - cgroups monitoring for pilot extensions disabled after expiry time - NOTE: This option is experimental and may be removed in later versions of the Agent. """ return conf.get("Debug.CgroupMonitorExpiryTime", "2022-03-31") diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index 7489ed9ad..cd5595569 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -1322,7 +1322,6 @@ def test_run_latest_creates_only_one_signal_handler(self, mock_signal): self._test_run_latest() self.assertEqual(0, mock_signal.call_count) - # @skip_if_predicate_true(lambda: True, "This test has a dependency on the agent version being 9.9.* and breaks when updating the agent version during release") def test_get_latest_agent_should_return_latest_agent_even_on_bad_error_json(self): dst_ver = self.prepare_agents() # Add a malformed error.json file in all existing agents @@ -1709,7 +1708,6 @@ def test_it_should_set_dns_tcp_iptable_if_drop_available_accept_unavailable(self with patch('azurelinuxagent.common.conf.enable_firewall', return_value=True): with patch.object(osutil, '_enable_firewall', True): # drop rule is present -# <<<<<<< HEAD mock_iptables.set_command( AddFirewallRules.get_wire_non_root_drop_rule(AddFirewallRules.CHECK_COMMAND, mock_iptables.destination, @@ -1743,21 +1741,6 @@ def test_it_should_set_dns_tcp_iptable_if_drop_available_accept_unavailable(self self.assertEqual(len(filtered_mock_iptable_calls), 3, "Incorrect number of calls to iptables: [{0}]".format( mock_iptables.command_calls)) -# ======= -# mock_iptables.set_command(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=0) -# # non root tcp iptable rule is absent -# mock_iptables.set_command(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=1) -# update_handler.run(debug=True) -# -# drop_check_command = TestOSUtil._command_to_string(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) -# accept_tcp_check_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) -# accept_tcp_insert_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.INSERT_COMMAND, mock_iptables.destination)) -# -# # Filtering the mock iptable command calls with only the once related to this test. -# filtered_mock_iptable_calls = [cmd for cmd in mock_iptables.command_calls if cmd in [drop_check_command, accept_tcp_check_rule, accept_tcp_insert_rule]] -# -# self.assertEqual(len(filtered_mock_iptable_calls), 3, "Incorrect number of calls to iptables: [{0}]".format(mock_iptables.command_calls)) -# >>>>>>> master self.assertEqual(filtered_mock_iptable_calls[0], drop_check_command, "The first command should check the drop rule") self.assertEqual(filtered_mock_iptable_calls[1], accept_tcp_check_rule, @@ -1772,7 +1755,6 @@ def test_it_should_not_set_dns_tcp_iptable_if_drop_unavailable(self): with patch('azurelinuxagent.common.conf.enable_firewall', return_value=True): with patch.object(osutil, '_enable_firewall', True): # drop rule is not available -# <<<<<<< HEAD mock_iptables.set_command( AddFirewallRules.get_wire_non_root_drop_rule(AddFirewallRules.CHECK_COMMAND, mock_iptables.destination, @@ -1801,20 +1783,6 @@ def test_it_should_not_set_dns_tcp_iptable_if_drop_unavailable(self): self.assertEqual(len(filtered_mock_iptable_calls), 1, "Incorrect number of calls to iptables: [{0}]".format( mock_iptables.command_calls)) -# ======= -# mock_iptables.set_command(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=1) -# -# update_handler.run(debug=True) -# -# drop_check_command = TestOSUtil._command_to_string(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) -# accept_tcp_check_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) -# accept_tcp_insert_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.INSERT_COMMAND, mock_iptables.destination)) -# -# # Filtering the mock iptable command calls with only the once related to this test. -# filtered_mock_iptable_calls = [cmd for cmd in mock_iptables.command_calls if cmd in [drop_check_command, accept_tcp_check_rule, accept_tcp_insert_rule]] -# -# self.assertEqual(len(filtered_mock_iptable_calls), 1, "Incorrect number of calls to iptables: [{0}]".format(mock_iptables.command_calls)) -# >>>>>>> master self.assertEqual(filtered_mock_iptable_calls[0], drop_check_command, "The first command should check the drop rule") @@ -1825,7 +1793,6 @@ def test_it_should_not_set_dns_tcp_iptable_if_drop_and_accept_available(self): with patch('azurelinuxagent.common.conf.enable_firewall', return_value=True): with patch.object(osutil, '_enable_firewall', True): # drop rule is available -# <<<<<<< HEAD mock_iptables.set_command( AddFirewallRules.get_wire_non_root_drop_rule(AddFirewallRules.CHECK_COMMAND, mock_iptables.destination, @@ -1859,22 +1826,6 @@ def test_it_should_not_set_dns_tcp_iptable_if_drop_and_accept_available(self): self.assertEqual(len(filtered_mock_iptable_calls), 2, "Incorrect number of calls to iptables: [{0}]".format( mock_iptables.command_calls)) -# ======= -# mock_iptables.set_command(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=0) -# # non root tcp iptable rule is available -# mock_iptables.set_command(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination), exit_code=0) -# -# update_handler.run(debug=True) -# -# drop_check_command = TestOSUtil._command_to_string(osutil.get_firewall_drop_command(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) -# accept_tcp_check_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.CHECK_COMMAND, mock_iptables.destination)) -# accept_tcp_insert_rule = TestOSUtil._command_to_string(osutil.get_accept_tcp_rule(mock_iptables.wait, AddFirewallRules.INSERT_COMMAND, mock_iptables.destination)) -# -# # Filtering the mock iptable command calls with only the once related to this test. -# filtered_mock_iptable_calls = [cmd for cmd in mock_iptables.command_calls if cmd in [drop_check_command, accept_tcp_check_rule, accept_tcp_insert_rule]] -# -# self.assertEqual(len(filtered_mock_iptable_calls), 2, "Incorrect number of calls to iptables: [{0}]".format(mock_iptables.command_calls)) -# >>>>>>> master self.assertEqual(filtered_mock_iptable_calls[0], drop_check_command, "The first command should check the drop rule") self.assertEqual(filtered_mock_iptable_calls[1], accept_tcp_check_rule, diff --git a/tests/protocol/test_extensions_goal_state_from_vm_settings.py b/tests/protocol/test_extensions_goal_state_from_vm_settings.py index 8a7d4c89a..8cdfa81bf 100644 --- a/tests/protocol/test_extensions_goal_state_from_vm_settings.py +++ b/tests/protocol/test_extensions_goal_state_from_vm_settings.py @@ -108,14 +108,6 @@ def test_its_source_channel_should_be_host_ga_plugin(self): self.assertEqual(GoalStateChannel.HostGAPlugin, extensions_goal_state.channel, "The channel is incorrect") - def test_create_from_vm_settings_should_parse_missing_status_upload_blob_as_none(self): - vm_settings_text = fileutil.read_file(os.path.join(data_dir, "hostgaplugin/vm_settings-no_status_upload_blob.json")) - vm_settings = ExtensionsGoalStateFactory.create_from_vm_settings("123", vm_settings_text) - - self.assertIsNone(vm_settings.status_upload_blob, "Expected status upload blob to be None") - self.assertEqual("BlockBlob", vm_settings.status_upload_blob_type, "Expected status upload blob to be Block") - - class CaseFoldedDictionaryTestCase(AgentTestCase): def test_it_should_retrieve_items_ignoring_case(self): dictionary = json.loads('''{ From a65135f7577faa605cb31ccae7d1f50766b46dd8 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 30 Sep 2022 11:26:46 -0700 Subject: [PATCH 77/89] Require HostGAPlugin >= 133 for Fast Track (#2673) Co-authored-by: narrieta --- azurelinuxagent/common/protocol/hostplugin.py | 6 +++--- .../vm_settings-difference_in_required_features.json | 2 +- tests/data/hostgaplugin/vm_settings-empty_depends_on.json | 2 +- .../hostgaplugin/vm_settings-fabric-no_thumbprints.json | 2 +- tests/data/hostgaplugin/vm_settings-invalid_blob_type.json | 2 +- tests/data/hostgaplugin/vm_settings-missing_cert.json | 2 +- tests/data/hostgaplugin/vm_settings-no_manifests.json | 2 +- .../hostgaplugin/vm_settings-no_status_upload_blob.json | 2 +- tests/data/hostgaplugin/vm_settings-out-of-sync.json | 2 +- tests/data/hostgaplugin/vm_settings-parse_error.json | 2 +- tests/data/hostgaplugin/vm_settings-requested_version.json | 2 +- tests/data/hostgaplugin/vm_settings.json | 2 +- 12 files changed, 14 insertions(+), 14 deletions(-) diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py index 55997d649..0aaff2184 100644 --- a/azurelinuxagent/common/protocol/hostplugin.py +++ b/azurelinuxagent/common/protocol/hostplugin.py @@ -491,7 +491,7 @@ def format_message(msg): try: # Raise if VmSettings are not supported, but check again periodically since the HostGAPlugin could have been updated since the last check # Note that self._host_plugin_supports_vm_settings can be None, so we need to compare against False - if self._supports_vm_settings == False and self._supports_vm_settings_next_check > datetime.datetime.now(): + if not self._supports_vm_settings and self._supports_vm_settings_next_check > datetime.datetime.now(): # Raise VmSettingsNotSupported directly instead of using raise_not_supported() to avoid resetting the timestamp for the next check raise VmSettingsNotSupported() @@ -551,8 +551,8 @@ def format_message(msg): logger.info(message) add_event(op=WALAEventOperation.HostPlugin, message=message, is_success=True) - # Don't support HostGAPlugin versions older than 124 - if vm_settings.host_ga_plugin_version < FlexibleVersion("1.0.8.124"): + # Don't support HostGAPlugin versions older than 133 + if vm_settings.host_ga_plugin_version < FlexibleVersion("1.0.8.133"): raise_not_supported() self._supports_vm_settings = True diff --git a/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json b/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json index a17776828..71cdbf5c5 100644 --- a/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json +++ b/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.124", + "hostGAPluginVersion": "1.0.8.133", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings-empty_depends_on.json b/tests/data/hostgaplugin/vm_settings-empty_depends_on.json index 94d9f0eb1..2295ae85c 100644 --- a/tests/data/hostgaplugin/vm_settings-empty_depends_on.json +++ b/tests/data/hostgaplugin/vm_settings-empty_depends_on.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.124", + "hostGAPluginVersion": "1.0.8.133", "vmSettingsSchemaVersion": "0.0", "activityId": "2e7f8b5d-f637-4721-b757-cb190d49b4e9", "correlationId": "1bef4c48-044e-4225-8f42-1d1eac1eb158", diff --git a/tests/data/hostgaplugin/vm_settings-fabric-no_thumbprints.json b/tests/data/hostgaplugin/vm_settings-fabric-no_thumbprints.json index bbd945933..a98f0affe 100644 --- a/tests/data/hostgaplugin/vm_settings-fabric-no_thumbprints.json +++ b/tests/data/hostgaplugin/vm_settings-fabric-no_thumbprints.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.124", + "hostGAPluginVersion": "1.0.8.133", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json b/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json index e7945845a..ef63166df 100644 --- a/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json +++ b/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.124", + "hostGAPluginVersion": "1.0.8.133", "vmSettingsSchemaVersion": "0.0", "activityId": "2e7f8b5d-f637-4721-b757-cb190d49b4e9", "correlationId": "1bef4c48-044e-4225-8f42-1d1eac1eb158", diff --git a/tests/data/hostgaplugin/vm_settings-missing_cert.json b/tests/data/hostgaplugin/vm_settings-missing_cert.json index a7192e942..4ce8a20cf 100644 --- a/tests/data/hostgaplugin/vm_settings-missing_cert.json +++ b/tests/data/hostgaplugin/vm_settings-missing_cert.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.124", + "hostGAPluginVersion": "1.0.8.133", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings-no_manifests.json b/tests/data/hostgaplugin/vm_settings-no_manifests.json index 7ec3a5c3d..c8d2bf138 100644 --- a/tests/data/hostgaplugin/vm_settings-no_manifests.json +++ b/tests/data/hostgaplugin/vm_settings-no_manifests.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.124", + "hostGAPluginVersion": "1.0.8.133", "vmSettingsSchemaVersion": "0.0", "activityId": "89d50bf1-fa55-4257-8af3-3db0c9f81ab4", "correlationId": "c143f8f0-a66b-4881-8c06-1efd278b0b02", diff --git a/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json b/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json index 2f70b5576..502d9a99c 100644 --- a/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json +++ b/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.124", + "hostGAPluginVersion": "1.0.8.133", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings-out-of-sync.json b/tests/data/hostgaplugin/vm_settings-out-of-sync.json index c35fdb5a3..0d4806af9 100644 --- a/tests/data/hostgaplugin/vm_settings-out-of-sync.json +++ b/tests/data/hostgaplugin/vm_settings-out-of-sync.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.124", + "hostGAPluginVersion": "1.0.8.133", "vmSettingsSchemaVersion": "0.0", "activityId": "AAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE", "correlationId": "EEEEEEEE-DDDD-CCCC-BBBB-AAAAAAAAAAAA", diff --git a/tests/data/hostgaplugin/vm_settings-parse_error.json b/tests/data/hostgaplugin/vm_settings-parse_error.json index bae5de4cb..da82fda78 100644 --- a/tests/data/hostgaplugin/vm_settings-parse_error.json +++ b/tests/data/hostgaplugin/vm_settings-parse_error.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.124", + "hostGAPluginVersion": "1.0.8.133", "vmSettingsSchemaVersion": THIS_IS_A_SYNTAX_ERROR, "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings-requested_version.json b/tests/data/hostgaplugin/vm_settings-requested_version.json index e7e5135f9..0f73cb255 100644 --- a/tests/data/hostgaplugin/vm_settings-requested_version.json +++ b/tests/data/hostgaplugin/vm_settings-requested_version.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.124", + "hostGAPluginVersion": "1.0.8.133", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings.json b/tests/data/hostgaplugin/vm_settings.json index 3018616ab..1f6d44deb 100644 --- a/tests/data/hostgaplugin/vm_settings.json +++ b/tests/data/hostgaplugin/vm_settings.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.124", + "hostGAPluginVersion": "1.0.8.133", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", From e7641bd3321ed15a41114238fd348f85d6e7ced2 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 30 Sep 2022 13:00:45 -0700 Subject: [PATCH 78/89] Additional telemetry for goal state (#2675) * Additional telemetry for goal state * add success message Co-authored-by: narrieta --- azurelinuxagent/common/protocol/goal_state.py | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/azurelinuxagent/common/protocol/goal_state.py b/azurelinuxagent/common/protocol/goal_state.py index c2b95cfb2..ef4730503 100644 --- a/azurelinuxagent/common/protocol/goal_state.py +++ b/azurelinuxagent/common/protocol/goal_state.py @@ -192,7 +192,9 @@ def _update(self, force_update): incarnation, xml_text, xml_doc = GoalState._fetch_goal_state(self._wire_client) goal_state_updated = force_update or incarnation != self._incarnation if goal_state_updated: - self.logger.info('Fetched a new incarnation for the WireServer goal state [incarnation {0}]', incarnation) + message = 'Fetched a new incarnation for the WireServer goal state [incarnation {0}]'.format(incarnation) + self.logger.info(message) + add_event(op=WALAEventOperation.GoalState, message=message) vm_settings, vm_settings_updated = None, False try: @@ -203,11 +205,15 @@ def _update(self, force_update): if vm_settings_updated: self.logger.info('') - self.logger.info("Fetched new vmSettings [HostGAPlugin correlation ID: {0} eTag: {1} source: {2}]", vm_settings.hostga_plugin_correlation_id, vm_settings.etag, vm_settings.source) + message = "Fetched new vmSettings [HostGAPlugin correlation ID: {0} eTag: {1} source: {2}]".format(vm_settings.hostga_plugin_correlation_id, vm_settings.etag, vm_settings.source) + self.logger.info(message) + add_event(op=WALAEventOperation.GoalState, message=message) # Ignore the vmSettings if their source is Fabric (processing a Fabric goal state may require the tenant certificate and the vmSettings don't include it.) if vm_settings is not None and vm_settings.source == GoalStateSource.Fabric: if vm_settings_updated: - self.logger.info("The vmSettings originated via Fabric; will ignore them.") + message = "The vmSettings originated via Fabric; will ignore them." + self.logger.info(message) + add_event(op=WALAEventOperation.GoalState, message=message) vm_settings, vm_settings_updated = None, False # If neither goal state has changed we are done with the update @@ -265,7 +271,9 @@ def _check_certificates(self): raise GoalStateInconsistentError(message) def _restore_wire_server_goal_state(self, incarnation, xml_text, xml_doc, vm_settings_support_stopped_error): - self.logger.info('The HGAP stopped supporting vmSettings; will fetched the goal state from the WireServer.') + msg = 'The HGAP stopped supporting vmSettings; will fetched the goal state from the WireServer.' + self.logger.info(msg) + add_event(op=WALAEventOperation.VmSettings, message=msg) self._history = GoalStateHistory(datetime.datetime.utcnow(), incarnation) self._history.save_goal_state(xml_text) self._extensions_goal_state = self._fetch_full_wire_server_goal_state(incarnation, xml_doc) @@ -274,7 +282,7 @@ def _restore_wire_server_goal_state(self, incarnation, xml_text, xml_doc, vm_set msg = "Fetched a Fabric goal state older than the most recent FastTrack goal state; will skip it.\nFabric: {0}\nFastTrack: {1}".format( self._extensions_goal_state.created_on_timestamp, vm_settings_support_stopped_error.timestamp) self.logger.info(msg) - add_event(op=WALAEventOperation.VmSettings, message=msg, is_success=True) + add_event(op=WALAEventOperation.VmSettings, message=msg) def save_to_history(self, data, file_name): self._history.save(data, file_name) @@ -351,7 +359,9 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): """ try: self.logger.info('') - self.logger.info('Fetching full goal state from the WireServer [incarnation {0}]', incarnation) + message = 'Fetching full goal state from the WireServer [incarnation {0}]'.format(incarnation) + self.logger.info(message) + add_event(op=WALAEventOperation.GoalState, message=message) role_instance = find(xml_doc, "RoleInstance") role_instance_id = findtext(role_instance, "InstanceId") @@ -391,9 +401,12 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): certs = Certificates(xml_text, self.logger) # Log and save the certificates summary (i.e. the thumbprint but not the certificate itself) to the goal state history for c in certs.summary: - self.logger.info("Downloaded certificate {0}".format(c)) + message = "Downloaded certificate {0}".format(c) + self.logger.info(message) + add_event(op=WALAEventOperation.GoalState, message=message) if len(certs.warnings) > 0: self.logger.warn(certs.warnings) + add_event(op=WALAEventOperation.GoalState, message=certs.warnings) self._history.save_certificates(json.dumps(certs.summary)) remote_access = None @@ -418,7 +431,9 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): self.logger.warn("Fetching the goal state failed: {0}", ustr(exception)) raise ProtocolError(msg="Error fetching goal state", inner=exception) finally: - self.logger.info('Fetch goal state completed') + message = 'Fetch goal state completed' + self.logger.info(message) + add_event(op=WALAEventOperation.GoalState, message=message) class HostingEnv(object): @@ -455,9 +470,11 @@ def __init__(self, xml_text, my_logger): return # if the certificates format is not Pkcs7BlobWithPfxContents do not parse it - certificateFormat = findtext(xml_doc, "Format") - if certificateFormat and certificateFormat != "Pkcs7BlobWithPfxContents": - my_logger.warn("The Format is not Pkcs7BlobWithPfxContents. Format is " + certificateFormat) + certificate_format = findtext(xml_doc, "Format") + if certificate_format and certificate_format != "Pkcs7BlobWithPfxContents": + message = "The Format is not Pkcs7BlobWithPfxContents. Format is {0}".format(certificate_format) + my_logger.warn(message) + add_event(op=WALAEventOperation.GoalState, message=message) return cryptutil = CryptUtil(conf.get_openssl_cmd()) From d72201dd416c27a2842c945a9aba405a4c73f0b5 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Fri, 30 Sep 2022 14:37:57 -0700 Subject: [PATCH 79/89] version update to 2.9.0.1 (#2678) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 20d2c605d..79d32f1e3 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '2.9.0.0' +AGENT_VERSION = '2.9.0.1' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 60f5b4dd465b3921ca50ec2e5c9fc4f1987d90b8 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Mon, 17 Oct 2022 15:01:09 -0700 Subject: [PATCH 80/89] drop cgroup support for rhel (#2685) --- azurelinuxagent/common/cgroupapi.py | 2 +- tests/common/test_cgroupapi.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/azurelinuxagent/common/cgroupapi.py b/azurelinuxagent/common/cgroupapi.py index 2935e2516..d7040747a 100644 --- a/azurelinuxagent/common/cgroupapi.py +++ b/azurelinuxagent/common/cgroupapi.py @@ -60,7 +60,7 @@ def cgroups_supported(): except ValueError: return False return ((distro_name.lower() == 'ubuntu' and distro_version.major >= 16) or - (distro_name.lower() in ("centos", "redhat") and + (distro_name.lower() == "centos" and ((distro_version.major == 7 and distro_version.minor >= 4) or distro_version.major >= 8))) @staticmethod diff --git a/tests/common/test_cgroupapi.py b/tests/common/test_cgroupapi.py index 3b70214d3..ca380ed2c 100644 --- a/tests/common/test_cgroupapi.py +++ b/tests/common/test_cgroupapi.py @@ -57,13 +57,13 @@ def test_cgroups_should_be_supported_only_on_ubuntu16_centos7dot4_redhat7dot4_an (['ubuntu', '20.04', 'focal'], True), (['ubuntu', '20.10', 'groovy'], True), (['centos', '7.8', 'Source'], True), - (['redhat', '7.8', 'Maipo'], True), - (['redhat', '7.9.1908', 'Core'], True), + (['redhat', '7.8', 'Maipo'], False), + (['redhat', '7.9.1908', 'Core'], False), (['centos', '8.1', 'Source'], True), - (['redhat', '8.2', 'Maipo'], True), - (['redhat', '8.2.2111', 'Core'], True), + (['redhat', '8.2', 'Maipo'], False), + (['redhat', '8.2.2111', 'Core'], False), (['centos', '7.4', 'Source'], True), - (['redhat', '7.4', 'Maipo'], True), + (['redhat', '7.4', 'Maipo'], False), (['centos', '7.5', 'Source'], True), (['centos', '7.3', 'Maipo'], False), (['redhat', '7.2', 'Maipo'], False), From 70bb383c8f873d859a6eee6670b014c2599f1787 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Mon, 17 Oct 2022 15:20:07 -0700 Subject: [PATCH 81/89] version update to 2.9.0.2 (#2686) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 79d32f1e3..0e6270b0c 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '2.9.0.1' +AGENT_VERSION = '2.9.0.2' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 2fa01e53c5de85c831e2cf01ae39113ac643f164 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Mon, 24 Oct 2022 12:01:19 -0700 Subject: [PATCH 82/89] drop cgroup support for centos (#2689) --- azurelinuxagent/common/cgroupapi.py | 4 +--- tests/common/test_cgroupapi.py | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/azurelinuxagent/common/cgroupapi.py b/azurelinuxagent/common/cgroupapi.py index d7040747a..66e893ef6 100644 --- a/azurelinuxagent/common/cgroupapi.py +++ b/azurelinuxagent/common/cgroupapi.py @@ -59,9 +59,7 @@ def cgroups_supported(): distro_version = FlexibleVersion(distro_info[1]) except ValueError: return False - return ((distro_name.lower() == 'ubuntu' and distro_version.major >= 16) or - (distro_name.lower() == "centos" and - ((distro_version.major == 7 and distro_version.minor >= 4) or distro_version.major >= 8))) + return distro_name.lower() == 'ubuntu' and distro_version.major >= 16 @staticmethod def track_cgroups(extension_cgroups): diff --git a/tests/common/test_cgroupapi.py b/tests/common/test_cgroupapi.py index ca380ed2c..a31d57d72 100644 --- a/tests/common/test_cgroupapi.py +++ b/tests/common/test_cgroupapi.py @@ -56,15 +56,15 @@ def test_cgroups_should_be_supported_only_on_ubuntu16_centos7dot4_redhat7dot4_an (['ubuntu', '18.10', 'cosmic'], True), (['ubuntu', '20.04', 'focal'], True), (['ubuntu', '20.10', 'groovy'], True), - (['centos', '7.8', 'Source'], True), + (['centos', '7.8', 'Source'], False), (['redhat', '7.8', 'Maipo'], False), (['redhat', '7.9.1908', 'Core'], False), - (['centos', '8.1', 'Source'], True), + (['centos', '8.1', 'Source'], False), (['redhat', '8.2', 'Maipo'], False), (['redhat', '8.2.2111', 'Core'], False), - (['centos', '7.4', 'Source'], True), + (['centos', '7.4', 'Source'], False), (['redhat', '7.4', 'Maipo'], False), - (['centos', '7.5', 'Source'], True), + (['centos', '7.5', 'Source'], False), (['centos', '7.3', 'Maipo'], False), (['redhat', '7.2', 'Maipo'], False), (['bigip', '15.0.1', 'Final'], False), From 43cf6a2ac3a40e977e92cca8d8b0ce4c32b6c01f Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Mon, 24 Oct 2022 15:55:49 -0700 Subject: [PATCH 83/89] update version to 2.9.0.3 (#2690) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 0e6270b0c..6f08cd9b3 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '2.9.0.2' +AGENT_VERSION = '2.9.0.3' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 49d4f7cbb38c4ef9e1e6b7450c62cf2b7235b157 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Mon, 31 Oct 2022 11:31:16 -0700 Subject: [PATCH 84/89] reset the quotas when agent drop the cgroup support (#2693) * reset the quotas when agent drop the cgroup support * address comments * log removed file names * remove only agent created files * fix pylint error * fix agent drop in path str --- azurelinuxagent/common/cgroupconfigurator.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/azurelinuxagent/common/cgroupconfigurator.py b/azurelinuxagent/common/cgroupconfigurator.py index 47f1da35a..b22a26bcd 100644 --- a/azurelinuxagent/common/cgroupconfigurator.py +++ b/azurelinuxagent/common/cgroupconfigurator.py @@ -149,6 +149,25 @@ def initialize(self): try: if self._initialized: return + # This check is to reset the quotas if agent goes from cgroup supported to unsupported distros later in time. + if not CGroupsApi.cgroups_supported(): + agent_drop_in_path = systemd.get_agent_drop_in_path() + try: + if os.path.exists(agent_drop_in_path) and os.path.isdir(agent_drop_in_path): + files_to_cleanup = [] + agent_drop_in_file_slice = os.path.join(agent_drop_in_path, _AGENT_DROP_IN_FILE_SLICE) + agent_drop_in_file_cpu_accounting = os.path.join(agent_drop_in_path, + _DROP_IN_FILE_CPU_ACCOUNTING) + agent_drop_in_file_memory_accounting = os.path.join(agent_drop_in_path, + _DROP_IN_FILE_MEMORY_ACCOUNTING) + agent_drop_in_file_cpu_quota = os.path.join(agent_drop_in_path, _DROP_IN_FILE_CPU_QUOTA) + files_to_cleanup.extend([agent_drop_in_file_slice, agent_drop_in_file_cpu_accounting, + agent_drop_in_file_memory_accounting, agent_drop_in_file_cpu_quota]) + self.__cleanup_all_files(files_to_cleanup) + self.__reload_systemd_config() + logger.info("Agent reset the quotas if distro: {0} goes from supported to unsupported list", get_distro()) + except Exception as err: + logger.warn("Unable to delete Agent drop-in files while resetting the quotas: {0}".format(err)) # check whether cgroup monitoring is supported on the current distro self._cgroups_supported = CGroupsApi.cgroups_supported() From 045dbd3b9bf2ed74af69654ccecdb41d6f89676c Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Mon, 31 Oct 2022 13:02:42 -0700 Subject: [PATCH 85/89] version update to 2.9.0.4 (#2694) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 6f08cd9b3..20e11cb3a 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '2.9.0.3' +AGENT_VERSION = '2.9.0.4' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 44701b8d52325e345fba875f2896d1c702c0f9ca Mon Sep 17 00:00:00 2001 From: "microsoft-github-policy-service[bot]" <77245923+microsoft-github-policy-service[bot]@users.noreply.github.com> Date: Wed, 1 Feb 2023 09:05:42 -0800 Subject: [PATCH 86/89] Microsoft mandatory file (#2737) Co-authored-by: microsoft-github-policy-service[bot] <77245923+microsoft-github-policy-service[bot]@users.noreply.github.com> --- SECURITY.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..e138ec5d6 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). + + From 8d0ca20fe040bc16b004dec96161f69daf21b530 Mon Sep 17 00:00:00 2001 From: maddieford <93676569+maddieford@users.noreply.github.com> Date: Fri, 7 Apr 2023 13:46:45 -0700 Subject: [PATCH 87/89] Update version to 2.9.1.0 (#2800) * Update version to dummy 1.0.0.0' * Revert version change * Update version to 2.9.1.0 * Trigger unit tests --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index ff9c903b9..a77f35e25 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '9.9.9.9' +AGENT_VERSION = '2.9.1.0' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 66e8b3d782fdf2ebc443212bbb731a89599201f6 Mon Sep 17 00:00:00 2001 From: maddieford <93676569+maddieford@users.noreply.github.com> Date: Fri, 12 May 2023 16:32:31 -0700 Subject: [PATCH 88/89] Add vm arch to heartbeat telemetry (#2818) * Add VM Arch to heartbeat telemetry * Remove outdated vmsize heartbeat tesT * Remove unused import * Use platform to get vmarch --- azurelinuxagent/ga/update.py | 12 ++++++---- tests/ga/test_update.py | 43 ------------------------------------ 2 files changed, 8 insertions(+), 47 deletions(-) diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index cd758b972..2b0975b05 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -19,6 +19,7 @@ import glob import json import os +import platform import re import shutil import signal @@ -462,6 +463,9 @@ def _get_vm_size(self, protocol): return self._vm_size + def _get_vm_arch(self): + return platform.machine() + def _check_daemon_running(self, debug): # Check that the parent process (the agent's daemon) is still running if not debug and self._is_orphaned: @@ -1265,13 +1269,13 @@ def _send_heartbeat_telemetry(self, protocol): if datetime.utcnow() >= (self._last_telemetry_heartbeat + UpdateHandler.TELEMETRY_HEARTBEAT_PERIOD): dropped_packets = self.osutil.get_firewall_dropped_packets(protocol.get_endpoint()) auto_update_enabled = 1 if conf.get_autoupdate_enabled() else 0 - # Include VMSize in the heartbeat message because the kusto table does not have - # a separate column for it (or architecture). - vmsize = self._get_vm_size(protocol) + # Include vm architecture in the heartbeat message because the kusto table does not have + # a separate column for it. + vmarch = self._get_vm_arch() telemetry_msg = "{0};{1};{2};{3};{4};{5}".format(self._heartbeat_counter, self._heartbeat_id, dropped_packets, self._heartbeat_update_goal_state_error_count, - auto_update_enabled, vmsize) + auto_update_enabled, vmarch) debug_log_msg = "[DEBUG HeartbeatCounter: {0};HeartbeatId: {1};DroppedPackets: {2};" \ "UpdateGSErrors: {3};AutoUpdate: {4}]".format(self._heartbeat_counter, self._heartbeat_id, dropped_packets, diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index 1b84d6f1c..e5f15fbd0 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -20,7 +20,6 @@ from datetime import datetime, timedelta from threading import current_thread -from azurelinuxagent.common.protocol.imds import ComputeInfo from tests.common.osutil.test_default import TestOSUtil import azurelinuxagent.common.osutil.default as osutil @@ -2773,48 +2772,6 @@ def test_telemetry_heartbeat_creates_event(self, patch_add_event, patch_info, *_ self.assertTrue(any(call_args[0] == "[HEARTBEAT] Agent {0} is running as the goal state agent {1}" for call_args in patch_info.call_args), "The heartbeat was not written to the agent's log") - @patch("azurelinuxagent.ga.update.add_event") - @patch("azurelinuxagent.common.protocol.imds.ImdsClient") - def test_telemetry_heartbeat_retries_failed_vm_size_fetch(self, mock_imds_factory, patch_add_event, *_): - - def validate_single_heartbeat_event_matches_vm_size(vm_size): - heartbeat_event_kwargs = [ - kwargs for _, kwargs in patch_add_event.call_args_list - if kwargs.get('op', None) == WALAEventOperation.HeartBeat - ] - - self.assertEqual(1, len(heartbeat_event_kwargs), "Expected exactly one HeartBeat event, got {0}"\ - .format(heartbeat_event_kwargs)) - - telemetry_message = heartbeat_event_kwargs[0].get("message", "") - self.assertTrue(telemetry_message.endswith(vm_size), - "Expected HeartBeat message ('{0}') to end with the test vmSize value, {1}."\ - .format(telemetry_message, vm_size)) - - with mock_wire_protocol(mockwiredata.DATA_FILE) as mock_protocol: - update_handler = get_update_handler() - update_handler.protocol_util.get_protocol = Mock(return_value=mock_protocol) - - # Zero out the _vm_size parameter for test resiliency - update_handler._vm_size = None - - mock_imds_client = mock_imds_factory.return_value = Mock() - - # First force a vmSize retrieval failure - mock_imds_client.get_compute.side_effect = HttpError(msg="HTTP Test Failure") - update_handler._last_telemetry_heartbeat = datetime.utcnow() - timedelta(hours=1) - update_handler._send_heartbeat_telemetry(mock_protocol) - - validate_single_heartbeat_event_matches_vm_size("unknown") - patch_add_event.reset_mock() - - # Now provide a vmSize - mock_imds_client.get_compute = lambda: ComputeInfo(vmSize="TestVmSizeValue") - update_handler._last_telemetry_heartbeat = datetime.utcnow() - timedelta(hours=1) - update_handler._send_heartbeat_telemetry(mock_protocol) - - validate_single_heartbeat_event_matches_vm_size("TestVmSizeValue") - class AgentMemoryCheckTestCase(AgentTestCase): From 362f6859662598347ddf4c7066e3e9c3136539d7 Mon Sep 17 00:00:00 2001 From: maddieford <93676569+maddieford@users.noreply.github.com> Date: Fri, 12 May 2023 16:44:05 -0700 Subject: [PATCH 89/89] Increment version to 2.9.1.1 for retake (#2819) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index a77f35e25..8e12eff5f 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -209,7 +209,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '2.9.1.0' +AGENT_VERSION = '2.9.1.1' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux