From 3e71269a439fe392b5385120d6db9b7c52f3e7d7 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Tue, 27 Apr 2021 17:10:48 -0700 Subject: [PATCH 01/35] Disable consider-using-with in pylint (#2231) --- azurelinuxagent/agent.py | 5 +++-- azurelinuxagent/common/utils/archive.py | 16 ++++++++++------ azurelinuxagent/daemon/scvmm.py | 6 +++--- azurelinuxagent/pa/rdma/ubuntu.py | 12 ++++++------ ci/2.7.pylintrc | 1 + ci/3.6.pylintrc | 1 + tests/ga/test_update.py | 9 +++++---- tests/utils/test_archive.py | 16 ++++++++++------ tests/utils/test_extension_process_util.py | 2 ++ 9 files changed, 41 insertions(+), 27 deletions(-) diff --git a/azurelinuxagent/agent.py b/azurelinuxagent/agent.py index b5ee2f3e9..ad789334a 100644 --- a/azurelinuxagent/agent.py +++ b/azurelinuxagent/agent.py @@ -371,11 +371,12 @@ def start(conf_file_path=None): Start agent daemon in a background process and set stdout/stderr to /dev/null """ - devnull = open(os.devnull, 'w') args = [sys.argv[0], '-daemon'] if conf_file_path is not None: args.append('-configuration-path:{0}'.format(conf_file_path)) - subprocess.Popen(args, stdout=devnull, stderr=devnull) + + with open(os.devnull, 'w') as devnull: + subprocess.Popen(args, stdout=devnull, stderr=devnull) if __name__ == '__main__' : diff --git a/azurelinuxagent/common/utils/archive.py b/azurelinuxagent/common/utils/archive.py index b87f337d5..e87c50ee7 100644 --- a/azurelinuxagent/common/utils/archive.py +++ b/azurelinuxagent/common/utils/archive.py @@ -192,12 +192,16 @@ def archive(self): fn_tmp = "{0}.zip.tmp".format(self._path) filename = "{0}.zip".format(self._path) - ziph = zipfile.ZipFile(fn_tmp, 'w') - for current_file in os.listdir(self._path): - full_path = os.path.join(self._path, current_file) - ziph.write(full_path, current_file, zipfile.ZIP_DEFLATED) - - ziph.close() + ziph = None + try: + # contextmanager for zipfile.ZipFile doesn't exist for py2.6, manually closing it + ziph = zipfile.ZipFile(fn_tmp, 'w') + for current_file in os.listdir(self._path): + full_path = os.path.join(self._path, current_file) + ziph.write(full_path, current_file, zipfile.ZIP_DEFLATED) + finally: + if ziph is not None: + ziph.close() os.rename(fn_tmp, filename) shutil.rmtree(self._path) diff --git a/azurelinuxagent/daemon/scvmm.py b/azurelinuxagent/daemon/scvmm.py index a264f30c5..fe1342190 100644 --- a/azurelinuxagent/daemon/scvmm.py +++ b/azurelinuxagent/daemon/scvmm.py @@ -63,9 +63,9 @@ def start_scvmm_agent(self, mount_point=None): if mount_point is None: mount_point = conf.get_dvd_mount_point() startup_script = os.path.join(mount_point, VMM_STARTUP_SCRIPT_NAME) - devnull = open(os.devnull, 'w') - subprocess.Popen(["/bin/bash", startup_script, "-p " + mount_point], - stdout=devnull, stderr=devnull) + with open(os.devnull, 'w') as devnull: + subprocess.Popen(["/bin/bash", startup_script, "-p " + mount_point], + stdout=devnull, stderr=devnull) def run(self): if self.detect_scvmm_env(): diff --git a/azurelinuxagent/pa/rdma/ubuntu.py b/azurelinuxagent/pa/rdma/ubuntu.py index c789f3f17..a56a4be4e 100644 --- a/azurelinuxagent/pa/rdma/ubuntu.py +++ b/azurelinuxagent/pa/rdma/ubuntu.py @@ -108,15 +108,15 @@ def update_modprobed_conf(self, nd_version): if not os.path.isfile(modprobed_file): logger.info("RDMA: %s not found, it will be created" % modprobed_file) else: - f = open(modprobed_file, 'r') - lines = f.read() - f.close() + with open(modprobed_file, 'r') as f: + lines = f.read() + r = re.search('alias hv_network_direct hv_network_direct_\S+', lines) # pylint: disable=W1401 if r: lines = re.sub('alias hv_network_direct hv_network_direct_\S+', 'alias hv_network_direct hv_network_direct_%s' % nd_version, lines) # pylint: disable=W1401 else: lines += '\nalias hv_network_direct hv_network_direct_%s\n' % nd_version - f = open('/etc/modprobe.d/vmbus-rdma.conf', 'w') - f.write(lines) - f.close() + with open('/etc/modprobe.d/vmbus-rdma.conf', 'w') as f: + f.write(lines) + logger.info("RDMA: hv_network_direct alias updated to ND %s" % nd_version) diff --git a/ci/2.7.pylintrc b/ci/2.7.pylintrc index bfcdeffd3..1f66c00dd 100644 --- a/ci/2.7.pylintrc +++ b/ci/2.7.pylintrc @@ -7,6 +7,7 @@ disable=C, # (C) convention, for programming standard violation consider-using-dict-comprehension, # (R1717): *Consider using a dictionary comprehension* consider-using-in, # (R1714): *Consider merging these comparisons with "in" to %r* consider-using-set-comprehension, # (R1718): *Consider using a set comprehension* + consider-using-with, # (R1732): *Emitted if a resource-allocating assignment or call may be replaced by a 'with' block* duplicate-code, # (R0801): *Similar lines in %s files* no-init, # (W0232): Class has no __init__ method no-else-break, # (R1723): *Unnecessary "%s" after "break"* diff --git a/ci/3.6.pylintrc b/ci/3.6.pylintrc index 4c6497f74..2699bcf7f 100644 --- a/ci/3.6.pylintrc +++ b/ci/3.6.pylintrc @@ -6,6 +6,7 @@ disable=C, # (C) convention, for programming standard violation consider-using-dict-comprehension, # (R1717): *Consider using a dictionary comprehension* consider-using-in, # (R1714): *Consider merging these comparisons with "in" to %r* consider-using-set-comprehension, # (R1718): *Consider using a set comprehension* + consider-using-with, # (R1732): *Emitted if a resource-allocating assignment or call may be replaced by a 'with' block* duplicate-code, # (R0801): *Similar lines in %s files* no-init, # (W0232): Class has no __init__ method no-else-break, # (R1723): *Unnecessary "%s" after "break"* diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index 0733a7946..e4d04118e 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -174,13 +174,14 @@ def agent_versions(self): v.sort(reverse=True) return v + @contextlib.contextmanager def get_error_file(self, error_data=None): if error_data is None: error_data = NO_ERROR - fp = tempfile.NamedTemporaryFile(mode="w") - json.dump(error_data if error_data is not None else NO_ERROR, fp) - fp.seek(0) - return fp + with tempfile.NamedTemporaryFile(mode="w") as fp: + json.dump(error_data if error_data is not None else NO_ERROR, fp) + fp.seek(0) + yield fp def create_error(self, error_data=None): if error_data is None: diff --git a/tests/utils/test_archive.py b/tests/utils/test_archive.py index d857065ce..85d08746b 100644 --- a/tests/utils/test_archive.py +++ b/tests/utils/test_archive.py @@ -237,9 +237,13 @@ def assert_datetime_close_to(self, time1, time2, within): self.fail("the timestamps are outside of the tolerance of by {0} seconds".format(secs)) def assert_zip_contains(self, zip_filename, files): - ziph = zipfile.ZipFile(zip_filename, 'r') - zip_files = [x.filename for x in ziph.filelist] - for current_file in files: - self.assertTrue(current_file in zip_files, "'{0}' was not found in {1}".format(current_file, zip_filename)) - - ziph.close() + ziph = None + try: + # contextmanager for zipfile.ZipFile doesn't exist for py2.6, manually closing it + ziph = zipfile.ZipFile(zip_filename, 'r') + zip_files = [x.filename for x in ziph.filelist] + for current_file in files: + self.assertTrue(current_file in zip_files, "'{0}' was not found in {1}".format(current_file, zip_filename)) + finally: + if ziph is not None: + ziph.close() diff --git a/tests/utils/test_extension_process_util.py b/tests/utils/test_extension_process_util.py index 139bde2a4..11ad1fdce 100644 --- a/tests/utils/test_extension_process_util.py +++ b/tests/utils/test_extension_process_util.py @@ -37,6 +37,8 @@ def setUp(self): self.stderr.write("The five boxing wizards jump quickly.".encode("utf-8")) def tearDown(self): + self.stderr.close() + self.stdout.close() if self.tmp_dir is not None: shutil.rmtree(self.tmp_dir) From 5a97dec88873a22da93327866ef66389584da7a8 Mon Sep 17 00:00:00 2001 From: Dhivya Ganesan Date: Wed, 28 Apr 2021 10:19:49 -0700 Subject: [PATCH 02/35] Add User (#2229) Co-authored-by: Laveesh Rohra --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 46196ee32..3602a09ae 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -20,4 +20,4 @@ # # Linux Agent team # -* @narrieta @larohra @kevinclark19a @ZhidongPeng +* @narrieta @larohra @kevinclark19a @ZhidongPeng @dhivyaganesan From c468d0ade6d8b33e29565cf484dc1845d8424769 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Fri, 30 Apr 2021 16:53:41 -0700 Subject: [PATCH 03/35] Fix utf-encoding for reporting firewall-setup logs (#2233) --- azurelinuxagent/common/persist_firewall_rules.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/azurelinuxagent/common/persist_firewall_rules.py b/azurelinuxagent/common/persist_firewall_rules.py index 44a801c45..31f0899ca 100644 --- a/azurelinuxagent/common/persist_firewall_rules.py +++ b/azurelinuxagent/common/persist_firewall_rules.py @@ -17,6 +17,7 @@ # import os import sys +import traceback import azurelinuxagent.common.conf as conf from azurelinuxagent.common import logger @@ -247,15 +248,15 @@ def __log_network_setup_service_logs(self): service_failed = self.__verify_network_setup_service_failed() try: stdout = shellutil.run_command(cmd) - msg = "Logs from the {0} since system boot:\n {1}".format(self._network_setup_service_name, stdout) + msg = ustr("Logs from the {0} since system boot:\n {1}").format(self._network_setup_service_name, stdout) logger.info(msg) except CommandError as error: msg = "Unable to fetch service logs, Command: {0} failed with ExitCode: {1}\nStdout: {2}\nStderr: {3}".format( ' '.join(cmd), error.returncode, error.stdout, error.stderr) logger.warn(msg) - except Exception as error: + except Exception: msg = "Ran into unexpected error when getting logs for {0} service. Error: {1}".format( - self._network_setup_service_name, ustr(error)) + self._network_setup_service_name, traceback.format_exc()) logger.warn(msg) # Log service status and logs if we can fetch them from journalctl and send it to Kusto, From 6309ad968baed4068cb9f2bbab5ac72fe7ffb39d Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 5 May 2021 13:39:08 -0700 Subject: [PATCH 04/35] Use venv for Python 2.6 unit tests (#2242) Co-authored-by: narrieta --- .github/workflows/ci_pr.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci_pr.yml b/.github/workflows/ci_pr.yml index bceb1d949..a6c395a4c 100644 --- a/.github/workflows/ci_pr.yml +++ b/.github/workflows/ci_pr.yml @@ -22,26 +22,14 @@ jobs: - name: Install Python 2.6 run: | - sudo add-apt-repository ppa:deadsnakes/ppa -y - sudo apt-get update -y - sudo apt-get install python2.6 python2.6-dev -y - sudo ln -sf /usr/bin/python2.6 /usr/bin/python - PATH="/usr/bin/${PATH:+:${PATH}}" - echo $PATH - echo "Installed python2.6" - python -V - curl "https://bootstrap.pypa.io/pip/2.6/get-pip.py" -o get-pip.py - sudo env "PATH=$PATH" python get-pip.py + curl https://dcrdata.blob.core.windows.net/python/python-2.6.tar.bz2 -o python-2.6.tar.bz2 + sudo tar xjvf python-2.6.tar.bz2 --directory / - uses: actions/checkout@v2 - - name: Install dependencies - run: | - sudo env "PATH=$PATH" pip install -r requirements.txt - sudo env "PATH=$PATH" pip install -r test-requirements.txt - - name: Test with nosetests run: | + source /home/waagent/virtualenv/python2.6.9/bin/activate ./ci/nosetests.sh exit $? From a3a79a3be79a83a5b6a6a67663fcf14901599295 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Wed, 5 May 2021 15:37:16 -0700 Subject: [PATCH 05/35] Fix bad logging (#2241) --- azurelinuxagent/ga/collect_telemetry_events.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azurelinuxagent/ga/collect_telemetry_events.py b/azurelinuxagent/ga/collect_telemetry_events.py index 87b83f52e..c3a17ca1f 100644 --- a/azurelinuxagent/ga/collect_telemetry_events.py +++ b/azurelinuxagent/ga/collect_telemetry_events.py @@ -234,17 +234,17 @@ def _ensure_all_events_directories_empty(extension_events_directories): if not os.path.exists(event_dir_path): return - err = None + log_err = True # Delete any residue files in the events directory for residue_file in os.listdir(event_dir_path): try: os.remove(os.path.join(event_dir_path, residue_file)) except Exception as error: - # Only log the first error once per handler per run if unable to clean off residue files - err = ustr(error) if err is None else err - - if err is not None: - logger.error("Failed to completely clear the {0} directory. Exception: {1}", event_dir_path, err) + # Only log the first error once per handler per run to keep the logfile clean + if log_err: + logger.error("Failed to completely clear the {0} directory. Exception: {1}", event_dir_path, + ustr(error)) + log_err = False def _enqueue_events_and_get_count(self, handler_name, event_file_path, captured_events_count, dropped_events_with_error_count): From e5d75bcc93c91341e4a6626e3665a7b41e37bce2 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 5 May 2021 20:35:20 -0700 Subject: [PATCH 06/35] Update test matrix and support (#2234) --- README.md | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 25fd4a825..3785df884 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,15 @@ # Microsoft Azure Linux Agent -## Develop branch status +## Linux distributions support + +Our daily automation tests most of the [Linux distributions supported by Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/endorsed-distros); the Agent can be +used on other distributions as well, but development, testing and support for those are done by the open source community. + +Testing is done using the develop branch, which can be unstable. For a stable build please use the master branch instead. [![CodeCov](https://codecov.io/gh/Azure/WALinuxAgent/branch/develop/graph/badge.svg)](https://codecov.io/gh/Azure/WALinuxAgent/branch/develop) -Each badge below represents our basic validation tests for an image, which are executed several times each day. These include provisioning, user account, disk, extension and networking scenarios. - -Note: These badges represent testing to our develop branch which might not be stable. For a stable build please use master branch instead. - -Image | Status | -------|--------| -Canonical UbuntuServer 14.04.5-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_14.04.5-LTS__agent--bvt.svg) -Canonical UbuntuServer 14.04.5-DAILY-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_14.04.5-DAILY-LTS__agent--bvt.svg) -Canonical UbuntuServer 16.04-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_16.04-LTS__agent--bvt.svg) -Canonical UbuntuServer 16.04-DAILY-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_16.04-DAILY-LTS__agent--bvt.svg) -Canonical UbuntuServer 18.04-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_18.04-LTS__agent--bvt.svg) -Canonical UbuntuServer 18.04-DAILY-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_18.04-DAILY-LTS__agent--bvt.svg) -Credativ Debian 8|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Credativ_Debian_8__agent--bvt.svg) -Credativ Debian 9|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Credativ_Debian_9__agent--bvt.svg) -OpenLogic CentOS 6.9|![badge](https://dcrbadges.blob.core.windows.net/scenarios/OpenLogic_CentOS_6.9__agent--bvt.svg) -OpenLogic CentOS 7.4|![badge](https://dcrbadges.blob.core.windows.net/scenarios/OpenLogic_CentOS_7.4__agent--bvt.svg) -RedHat RHEL 6.9|![badge](https://dcrbadges.blob.core.windows.net/scenarios/RedHat_RHEL_6.9__agent--bvt.svg) -RedHat RHEL 7-RAW|![badge](https://dcrbadges.blob.core.windows.net/scenarios/RedHat_RHEL_7-RAW__agent--bvt.svg) -SUSE SLES 12-SP5|![badge](https://dcrbadges.blob.core.windows.net/scenarios/SUSE_SLES_12-SP5__agent--bvt.svg) ## Introduction From 8c2fc0827e616500799091d4da957f6eec182ae6 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 5 May 2021 20:39:00 -0700 Subject: [PATCH 07/35] Remove unused telemetry event (#2235) --- azurelinuxagent/common/event.py | 1 - 1 file changed, 1 deletion(-) diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py index 74f0713e2..51ea35392 100644 --- a/azurelinuxagent/common/event.py +++ b/azurelinuxagent/common/event.py @@ -73,7 +73,6 @@ class WALAEventOperation: CGroupsCleanUp = "CGroupsCleanUp" CGroupsDisabled = "CGroupsDisabled" CGroupsInfo = "CGroupsInfo" - CGroupsLimitsCrossed = "CGroupsLimitsCrossed" CollectEventErrors = "CollectEventErrors" CollectEventUnicodeErrors = "CollectEventUnicodeErrors" ConfigurationChange = "ConfigurationChange" From 5815f2b6fc97c92e0fa2687fe4ffbc138c56aa58 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Fri, 7 May 2021 15:21:29 -0700 Subject: [PATCH 08/35] Merge Multi config to develop (#2245) * Add support for multi config (#2230) * Add support for depends-on extensions for MultiConfig (#2243) * Fail handler if inconsistent data from CRP and HandlerManifest.json (#2244) --- azurelinuxagent/common/exception.py | 6 + azurelinuxagent/common/protocol/goal_state.py | 17 +- azurelinuxagent/common/protocol/restapi.py | 37 +- azurelinuxagent/common/protocol/wire.py | 61 +- azurelinuxagent/ga/exthandlers.py | 1136 ++++++++++----- tests/data/ext/sample_ext-1.3.0.zip | Bin 1083 -> 1770 bytes tests/data/ext/sample_ext-1.3.0/sample.py | 103 +- .../ext_conf_mc_disabled_extensions.xml | 84 ++ .../ext_conf_mc_update_extensions.xml | 75 + .../ext_conf_multi_config_no_dependencies.xml | 75 + .../ext_conf_with_disabled_multi_config.xml | 4 +- .../ext_conf_with_multi_config.xml | 4 +- ...xt_conf_with_multi_config_dependencies.xml | 99 ++ tests/ga/extension_emulator.py | 119 +- tests/ga/test_extension.py | 701 +++++----- tests/ga/test_exthandlers.py | 40 +- tests/ga/test_multi_config_extension.py | 1214 +++++++++++++++++ tests/protocol/mockwiredata.py | 8 + 18 files changed, 2903 insertions(+), 880 deletions(-) create mode 100644 tests/data/wire/multi-config/ext_conf_mc_disabled_extensions.xml create mode 100644 tests/data/wire/multi-config/ext_conf_mc_update_extensions.xml create mode 100644 tests/data/wire/multi-config/ext_conf_multi_config_no_dependencies.xml create mode 100644 tests/data/wire/multi-config/ext_conf_with_multi_config_dependencies.xml create mode 100644 tests/ga/test_multi_config_extension.py diff --git a/azurelinuxagent/common/exception.py b/azurelinuxagent/common/exception.py index c4d0a6853..aa3737ea6 100644 --- a/azurelinuxagent/common/exception.py +++ b/azurelinuxagent/common/exception.py @@ -100,6 +100,12 @@ class ExtensionConfigError(ExtensionError): """ +class MultiConfigExtensionEnableError(ExtensionError): + """ + Error raised when enable for a Multi-Config extension is failing. + """ + + class ProvisionError(AgentError): """ When provision failed diff --git a/azurelinuxagent/common/protocol/goal_state.py b/azurelinuxagent/common/protocol/goal_state.py index ec38972be..97066335c 100644 --- a/azurelinuxagent/common/protocol/goal_state.py +++ b/azurelinuxagent/common/protocol/goal_state.py @@ -32,7 +32,7 @@ from azurelinuxagent.common.future import ustr from azurelinuxagent.common.protocol.restapi import Cert, CertList, Extension, ExtHandler, ExtHandlerList, \ ExtHandlerVersionUri, RemoteAccessUser, RemoteAccessUsersList, VMAgentManifest, VMAgentManifestList, \ - VMAgentManifestUri, InVMGoalStateMetaData, RequiredFeature + VMAgentManifestUri, InVMGoalStateMetaData, RequiredFeature, ExtensionState from azurelinuxagent.common.utils import fileutil from azurelinuxagent.common.utils.cryptutil import CryptUtil from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, findtext, getattrib, gettext @@ -526,7 +526,7 @@ def to_lower(str_to_change): return str_to_change.lower() if str_to_change is no runtime_settings_nodes = findall(plugin_settings_node, "RuntimeSettings") extension_runtime_settings_nodes = findall(plugin_settings_node, "ExtensionRuntimeSettings") - if (runtime_settings_nodes != []) and (extension_runtime_settings_nodes != []): + if any(runtime_settings_nodes) and any(extension_runtime_settings_nodes): # There can only be a single RuntimeSettings node or multiple ExtensionRuntimeSettings nodes per Plugin msg = "Both RuntimeSettings and ExtensionRuntimeSettings found for the same handler: {0} and version: {1}".format( handler_name, version) @@ -652,27 +652,34 @@ def __parse_extension_runtime_settings(plugin_settings_node, extension_runtime_s dependency_level = ExtensionsConfig.__get_dependency_level_from_node(depends_on_node, extension_name) dependency_levels[extension_name] = dependency_level + ext_handler.supports_multi_config = True for extension_runtime_setting_node in extension_runtime_settings_nodes: # Name and State will only be set for ExtensionRuntimeSettings for Multi-Config extension_name = getattrib(extension_runtime_setting_node, "name") if extension_name in (None, ""): raise ExtensionConfigError("Extension Name not specified for ExtensionRuntimeSettings for MultiConfig!") - # State can either be `enabled` (default) or `disabled` + # State can either be `ExtensionState.Enabled` (default) or `ExtensionState.Disabled` state = getattrib(extension_runtime_setting_node, "state") - state = state if state not in (None, "") else "enabled" + state = ustr(state.lower()) if state not in (None, "") else ExtensionState.Enabled ExtensionsConfig.__parse_and_add_extension_settings(extension_runtime_setting_node, extension_name, ext_handler, dependency_levels[extension_name], state=state) @staticmethod - def __parse_and_add_extension_settings(settings_node, name, ext_handler, depends_on_level, state="enabled"): + def __parse_and_add_extension_settings(settings_node, name, ext_handler, depends_on_level, state=ExtensionState.Enabled): seq_no = getattrib(settings_node, "seqNo") if seq_no in (None, ""): raise ExtensionConfigError("SeqNo not specified for the Extension: {0}".format(name)) + try: runtime_settings = json.loads(gettext(settings_node)) except ValueError as error: logger.error("Invalid extension settings: {0}", ustr(error)) + # Incase of invalid/no settings, add the name and seqNo of the Extension and treat it as an extension with + # no settings since we were able to successfully parse those data properly. Without this, we wont report + # anything for that sequence number and CRP would eventually have to timeout rather than fail fast. + ext_handler.properties.extensions.append( + Extension(name=name, sequenceNumber=seq_no, state=state, dependencyLevel=depends_on_level)) return for plugin_settings_list in runtime_settings["runtimeSettings"]: diff --git a/azurelinuxagent/common/protocol/restapi.py b/azurelinuxagent/common/protocol/restapi.py index 5f085c730..2f960b4ac 100644 --- a/azurelinuxagent/common/protocol/restapi.py +++ b/azurelinuxagent/common/protocol/restapi.py @@ -89,6 +89,22 @@ def __init__(self, name, value=None): self.value = value +class ExtensionState(object): + Enabled = ustr("enabled") + Disabled = ustr("disabled") + + +class ExtHandlerRequestedState(object): + """ + This is the state of the Handler as requested by the Goal State. + CRP only supports 2 states as of now - Enabled and Uninstall + Disabled was used for older XML extensions and we keep it to support backward compatibility. + """ + Enabled = ustr("enabled") + Disabled = ustr("disabled") + Uninstall = ustr("uninstall") + + class Extension(DataContract): """ The runtime settings associated with a Handler @@ -105,7 +121,7 @@ def __init__(self, protectedSettings=None, certificateThumbprint=None, dependencyLevel=0, - state="enabled"): + state=ExtensionState.Enabled): self.name = name self.sequenceNumber = sequenceNumber self.publicSettings = publicSettings @@ -114,6 +130,16 @@ def __init__(self, self.dependencyLevel = dependencyLevel self.state = state + def dependency_level_sort_key(self, handler_state): + level = self.dependencyLevel + # Process uninstall or disabled before enabled, in reverse order + # Prioritize Handler state and Extension state both when sorting extensions + # remap 0 to -1, 1 to -2, 2 to -3, etc + if handler_state != ExtHandlerRequestedState.Enabled or self.state != ExtensionState.Enabled: + level = (0 - level) - 1 + + return level + class ExtHandlerProperties(DataContract): def __init__(self): @@ -139,6 +165,7 @@ def __init__(self, name=None): self.properties = ExtHandlerProperties() self.versionUris = DataContractList(ExtHandlerVersionUri) self.__invalid_handler_setting_reason = None + self.supports_multi_config = False @property def is_invalid_setting(self): @@ -152,7 +179,7 @@ def invalid_setting_reason(self): def invalid_setting_reason(self, value): self.__invalid_handler_setting_reason = value - def sort_key(self): + def dependency_level_sort_key(self): levels = [e.dependencyLevel for e in self.properties.extensions] if len(levels) == 0: level = 0 @@ -243,12 +270,14 @@ def __init__(self, name=None, status=None, code=None, message=None): class ExtensionStatus(DataContract): def __init__(self, + name=None, configurationAppliedTime=None, operation=None, status=None, seq_no=None, code=None, message=None): + self.name = name self.configurationAppliedTime = configurationAppliedTime self.operation = operation self.status = status @@ -270,7 +299,8 @@ def __init__(self, self.status = status self.code = code self.message = message - self.extensions = DataContractList(ustr) + self.supports_multi_config = False + self.extension_status = None class VMAgentStatus(DataContract): @@ -318,4 +348,3 @@ def __init__(self, name, encrypted_password, expiration): class RemoteAccessUsersList(DataContract): def __init__(self): self.users = DataContractList(RemoteAccessUser) - diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 107eeb85a..2ed0d8faa 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -37,7 +37,7 @@ from azurelinuxagent.common.future import httpclient, bytebuffer, ustr from azurelinuxagent.common.protocol.goal_state import GoalState, TRANSPORT_CERT_FILE_NAME, TRANSPORT_PRV_FILE_NAME from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol -from azurelinuxagent.common.protocol.restapi import DataContract, ExtensionStatus, ExtHandlerPackage, \ +from azurelinuxagent.common.protocol.restapi import DataContract, ExtHandlerPackage, \ ExtHandlerPackageList, ExtHandlerVersionUri, ProvisionStatus, VMInfo, VMStatus from azurelinuxagent.common.telemetryevent import GuestAgentExtensionEventsSchema from azurelinuxagent.common.utils import fileutil, restutil @@ -192,10 +192,6 @@ def report_vm_status(self, vm_status): self.client.status_blob.set_vm_status(vm_status) self.client.upload_status_blob() - def report_ext_status(self, ext_handler_name, ext_name, ext_status): # pylint: disable=W0613 - validate_param("ext_status", ext_status, ExtensionStatus) - self.client.status_blob.set_ext_status(ext_handler_name, ext_status) - def report_event(self, events_iterator): self.client.report_event(events_iterator) @@ -323,14 +319,14 @@ def ext_substatus_to_v1(sub_status_list): return status_list -def ext_status_to_v1(ext_name, ext_status): +def ext_status_to_v1(ext_status): if ext_status is None: return None timestamp = _get_utc_timestamp_for_status_reporting() v1_sub_status = ext_substatus_to_v1(ext_status.substatusList) v1_ext_status = { "status": { - "name": ext_name, + "name": ext_status.name, "configurationAppliedTime": ext_status.configurationAppliedTime, "operation": ext_status.operation, "status": ext_status.status, @@ -345,27 +341,28 @@ def ext_status_to_v1(ext_name, ext_status): return v1_ext_status -def ext_handler_status_to_v1(handler_status, ext_statuses): +def ext_handler_status_to_v1(ext_handler_status): v1_handler_status = { - 'handlerVersion': handler_status.version, - 'handlerName': handler_status.name, - 'status': handler_status.status, - 'code': handler_status.code, + 'handlerVersion': ext_handler_status.version, + 'handlerName': ext_handler_status.name, + 'status': ext_handler_status.status, + 'code': ext_handler_status.code, 'useExactVersion': True } - if handler_status.message is not None: - v1_handler_status["formattedMessage"] = __get_formatted_msg_for_status_reporting(handler_status.message) - - if len(handler_status.extensions) > 0: - # Currently, no more than one extension per handler - ext_name = handler_status.extensions[0] - ext_status = ext_statuses.get(ext_name) - v1_ext_status = ext_status_to_v1(ext_name, ext_status) - if ext_status is not None and v1_ext_status is not None: - v1_handler_status["runtimeSettingsStatus"] = { - 'settingsStatus': v1_ext_status, - 'sequenceNumber': ext_status.sequenceNumber - } + if ext_handler_status.message is not None: + v1_handler_status["formattedMessage"] = __get_formatted_msg_for_status_reporting(ext_handler_status.message) + + v1_ext_status = ext_status_to_v1(ext_handler_status.extension_status) + if ext_handler_status.extension_status is not None and v1_ext_status is not None: + v1_handler_status["runtimeSettingsStatus"] = { + 'settingsStatus': v1_ext_status, + 'sequenceNumber': ext_handler_status.extension_status.sequenceNumber + } + + # Add extension name if Handler supports MultiConfig + if ext_handler_status.supports_multi_config: + v1_handler_status["runtimeSettingsStatus"]["extensionName"] = ext_handler_status.extension_status.name + return v1_handler_status @@ -388,7 +385,7 @@ def vm_artifacts_aggregate_status_to_v1(vm_artifacts_aggregate_status): return v1_artifact_aggregate_status -def vm_status_to_v1(vm_status, ext_statuses): +def vm_status_to_v1(vm_status): timestamp = _get_utc_timestamp_for_status_reporting() v1_ga_guest_info = ga_status_to_guest_info(vm_status.vmAgent) @@ -397,10 +394,7 @@ def vm_status_to_v1(vm_status, ext_statuses): vm_status.vmAgent.vm_artifacts_aggregate_status) v1_handler_status_list = [] for handler_status in vm_status.vmAgent.extensionHandlers: - v1_handler_status = ext_handler_status_to_v1(handler_status, - ext_statuses) - if v1_handler_status is not None: - v1_handler_status_list.append(v1_handler_status) + v1_handler_status_list.append(ext_handler_status_to_v1(handler_status)) v1_agg_status = { 'guestAgentStatus': v1_ga_status, @@ -434,7 +428,6 @@ def vm_status_to_v1(vm_status, ext_statuses): class StatusBlob(object): def __init__(self, client): self.vm_status = None - self.ext_statuses = {} self.client = client self.type = None self.data = None @@ -443,12 +436,8 @@ def set_vm_status(self, vm_status): validate_param("vmAgent", vm_status, VMStatus) self.vm_status = vm_status - def set_ext_status(self, ext_handler_name, ext_status): - validate_param("extensionStatus", ext_status, ExtensionStatus) - self.ext_statuses[ext_handler_name] = ext_status - def to_json(self): - report = vm_status_to_v1(self.vm_status, self.ext_statuses) + report = vm_status_to_v1(self.vm_status) return json.dumps(report) __storage_version__ = "2014-02-14" diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 0295afcae..59fcc4c59 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -20,7 +20,6 @@ import datetime import glob import json -import operator import os import random import re @@ -31,6 +30,8 @@ import time import traceback import zipfile +from collections import defaultdict +from functools import partial import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger @@ -41,13 +42,14 @@ from azurelinuxagent.common.cgroupconfigurator import CGroupConfigurator from azurelinuxagent.common.datacontract import get_properties, set_properties from azurelinuxagent.common.errorstate import ErrorState -from azurelinuxagent.common.event import add_event, elapsed_milliseconds, report_event, WALAEventOperation, \ +from azurelinuxagent.common.event import add_event, elapsed_milliseconds, WALAEventOperation, \ add_periodic, EVENTS_DIRECTORY from azurelinuxagent.common.exception import ExtensionDownloadError, ExtensionError, ExtensionErrorCodes, \ - ExtensionOperationError, ExtensionUpdateError, ProtocolError, ProtocolNotFoundError, ExtensionConfigError, GoalStateAggregateStatusCodes + ExtensionOperationError, ExtensionUpdateError, ProtocolError, ProtocolNotFoundError, ExtensionConfigError, \ + GoalStateAggregateStatusCodes, MultiConfigExtensionEnableError from azurelinuxagent.common.future import ustr, is_file_not_found_error from azurelinuxagent.common.protocol.restapi import ExtensionStatus, ExtensionSubStatus, ExtHandler, ExtHandlerStatus, \ - VMStatus, GoalStateAggregateStatus, Extension + VMStatus, GoalStateAggregateStatus, ExtensionState, ExtHandlerRequestedState, Extension from azurelinuxagent.common.utils.flexible_version import FlexibleVersion from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION, DISTRO_NAME, DISTRO_VERSION, \ GOAL_STATE_AGENT_VERSION, PY_VERSION_MAJOR, PY_VERSION_MICRO, PY_VERSION_MINOR @@ -83,6 +85,9 @@ _NUM_OF_STATUS_FILE_RETRIES = 5 _STATUS_FILE_RETRY_DELAY = 2 # seconds +# This is the default sequence number we use when there are no settings available for Handlers +_DEFAULT_SEQ_NO = "0" + class ValidHandlerStatus(object): transitioning = "transitioning" @@ -98,10 +103,12 @@ class ValidHandlerStatus(object): class ExtCommandEnvVariable(object): Prefix = "AZURE_GUEST_AGENT" DisableReturnCode = "{0}_DISABLE_CMD_EXIT_CODE".format(Prefix) + DisableReturnCodeMultipleExtensions = "{0}_DISABLE_CMD_EXIT_CODES_MULTIPLE_EXTENSIONS".format(Prefix) UninstallReturnCode = "{0}_UNINSTALL_CMD_EXIT_CODE".format(Prefix) ExtensionPath = "{0}_EXTENSION_PATH".format(Prefix) ExtensionVersion = "{0}_EXTENSION_VERSION".format(Prefix) ExtensionSeqNumber = "ConfigSequenceNumber" # At par with Windows Guest Agent + ExtensionName = "ConfigExtensionName" UpdatingFromVersion = "{0}_UPDATING_FROM_VERSION".format(Prefix) WireProtocolAddress = "{0}_WIRE_PROTOCOL_ADDRESS".format(Prefix) ExtensionSupportedFeatures = "{0}_EXTENSION_SUPPORTED_FEATURES".format(Prefix) @@ -232,17 +239,6 @@ class ExtHandlerState(object): FailedUpgrade = "FailedUpgrade" -class ExtensionRequestedState(object): - """ - This is the state of the Extension as requested by the Goal State. - CRP only supports 2 states as of now - Enabled and Uninstall - Disabled was used for older XML extensions and we keep it to support backward compatibility. - """ - Enabled = u"enabled" - Disabled = u"disabled" - Uninstall = u"uninstall" - - class GoalStateStatus(object): """ This is an Enum to define the State of the GoalState as a whole. This is reported as part of the @@ -281,7 +277,7 @@ def __init__(self, protocol): self.log_process = False # The GoalState Aggregate status needs to report the last status of the GoalState. Since we only process # extensions on incarnation change, we need to maintain its state. - # Setting the status as Initialize here. This would be overridden as soon as the first GoalState is processed + # Setting the status to None here. This would be overridden as soon as the first GoalState is processed # (once self._extension_processing_allowed() is True). self.__gs_aggregate_status = None @@ -463,193 +459,296 @@ def _extension_processing_allowed(self): return True + @staticmethod + def __get_dependency_level(tup): + (extension, handler) = tup + if extension is not None: + return extension.dependency_level_sort_key(handler.properties.state) + return handler.dependency_level_sort_key() + + def __get_sorted_extensions_for_processing(self): + all_extensions = [] + for handler in self.ext_handlers.extHandlers: + if any(handler.properties.extensions): + all_extensions.extend([(ext, handler) for ext in handler.properties.extensions]) + else: + # We need to process the Handler even if no settings specified from CRP (legacy behavior) + logger.info("No extension/run-time settings settings found for {0}".format(handler.name)) + all_extensions.append((None, handler)) + + all_extensions.sort(key=self.__get_dependency_level) + + return all_extensions + def handle_ext_handlers(self, etag=None): if not self.ext_handlers.extHandlers: logger.info("No extension handlers found, not processing anything.") return wait_until = datetime.datetime.utcnow() + datetime.timedelta(minutes=_DEFAULT_EXT_TIMEOUT_MINUTES) - max_dep_level = max([handler.sort_key() for handler in self.ext_handlers.extHandlers]) - self.ext_handlers.extHandlers.sort(key=operator.methodcaller('sort_key')) - for ext_handler in self.ext_handlers.extHandlers: - handler_success = self.handle_ext_handler(ext_handler, etag) + all_extensions = self.__get_sorted_extensions_for_processing() + # Since all_extensions are sorted based on sort_key, the last element would be the maximum based on the sort_key + max_dep_level = self.__get_dependency_level(all_extensions[-1]) if any(all_extensions) else 0 + + depends_on_err_msg = None + for extension, ext_handler in all_extensions: + + handler_i = ExtHandlerInstance(ext_handler, self.protocol, extension=extension) + + # In case of depends-on errors, we skip processing extensions if there was an error processing dependent extensions. + # But CRP is still waiting for some status back for the skipped extensions. In order to propagate the status back to CRP, + # we will report status back here with the relevant error message for each of the dependent extension. + if depends_on_err_msg is not None: - # Wait for the extension installation until it is handled. - # This is done for the install and enable. Not for the uninstallation. - # If handled successfully, proceed with the current handler. - # Otherwise, skip the rest of the extension installation. - dep_level = ext_handler.sort_key() + # For MC extensions, report the HandlerStatus as is and create a new placeholder per extension if doesnt exist + if handler_i.should_perform_multi_config_op(extension): + # Ensure some handler status exists for the Handler, if not, set it here + if handler_i.get_handler_status() is None: + handler_i.set_handler_status(message=depends_on_err_msg, code=-1) + + handler_i.create_placeholder_status_file(extension, status=ValidHandlerStatus.error, code=-1, + operation=WALAEventOperation.ExtensionProcessing, + message=depends_on_err_msg) + + # For SC extensions, overwrite the HandlerStatus with the relevant message + else: + handler_i.set_handler_status(message=depends_on_err_msg, code=-1) + + continue + + # Process extensions and get if it was successfully executed or not + extension_success = self.handle_ext_handler(handler_i, extension, etag) + + dep_level = self.__get_dependency_level((extension, ext_handler)) if 0 <= dep_level < max_dep_level: + extension_full_name = handler_i.get_extension_full_name(extension) + try: + # Do no wait for extension status if the handler failed + if not extension_success: + raise Exception("Skipping processing of extensions since execution of dependent extension {0} failed".format( + extension_full_name)) - # Do no wait for extension status if the handler failed - if not handler_success: - msg = "Handler: {0} processing failed, will skip processing the rest of the extensions".format( - ext_handler.name) - add_event(AGENT_NAME, - version=CURRENT_VERSION, + # Wait for the extension installation until it is handled. + # This is done for the install and enable. Not for the uninstallation. + # If handled successfully, proceed with the current handler. + # Otherwise, skip the rest of the extension installation. + self.wait_for_handler_completion(handler_i, wait_until, extension=extension) + + except Exception as error: + logger.warn( + "Dependent extension {0} failed or timed out, will skip processing the rest of the extensions".format( + extension_full_name)) + depends_on_err_msg = ustr(error) + add_event(name=extension_full_name, + version=handler_i.ext_handler.properties.version, op=WALAEventOperation.ExtensionProcessing, is_success=False, - message=msg, - log_event=True) - break + message=depends_on_err_msg) - if not self.wait_for_handler_completion(ext_handler, wait_until): - logger.warn("An extension failed or timed out, will skip processing the rest of the extensions") - break - - def wait_for_handler_completion(self, ext_handler, wait_until): + @staticmethod + def wait_for_handler_completion(handler_i, wait_until, extension=None): """ - Check the status of the extension being handled. - Wait until it has a terminal state or times out. - Return True if it is handled successfully. False if not. + Check the status of the extension being handled. Wait until it has a terminal state or times out. + :raises: Exception if it is not handled successfully. """ - def _report_error_event_and_return_false(error_message): - # In case of error, return False so that the processing of dependent extensions can be skipped (fail fast) - add_event(AGENT_NAME, - version=CURRENT_VERSION, - op=WALAEventOperation.ExtensionProcessing, - is_success=False, - message=error_message, - log_event=True) - return False + extension_name = handler_i.get_extension_full_name(extension) try: - handler_i = ExtHandlerInstance(ext_handler, self.protocol) - - # Loop through all settings of the Handler and verify all extensions reported success status in status file - # Currently, we only support 1 extension (runtime-settings) per handler ext_completed, status = False, None - for ext in ext_handler.properties.extensions: - # Keep polling for the extension status until it succeeds or times out - while datetime.datetime.utcnow() <= wait_until: - ext_completed, status = handler_i.is_ext_handling_complete(ext) - if ext_completed: - break - time.sleep(5) - - # In case of timeout or terminal error state, we log it and return false - # Incase extension reported status at the last sec, we should prioritize reporting status over timeout - if not ext_completed and datetime.datetime.utcnow() > wait_until: - msg = "Extension {0} did not reach a terminal state within the allowed timeout. Last status was {1}".format( - ext.name, status) - return _report_error_event_and_return_false(msg) + # Keep polling for the extension status until it succeeds or times out + while datetime.datetime.utcnow() <= wait_until: + ext_completed, status = handler_i.is_ext_handling_complete(extension) + if ext_completed: + break + time.sleep(5) - if status != ValidHandlerStatus.success: - msg = "Extension {0} did not succeed. Status was {1}".format(ext.name, status) - return _report_error_event_and_return_false(msg) + except Exception: + msg = "Failed to wait for Handler completion due to unknown error. Marking the dependent extension as failed: {0}, {1}".format( + extension_name, traceback.format_exc()) + raise Exception(msg) - except Exception as error: - msg = "Failed to wait for Handler completion due to unknown error. Marking the extension as failed: {0}, {1}".format( - ustr(error), traceback.format_exc()) - return _report_error_event_and_return_false(msg) + # In case of timeout or terminal error state, we log it and raise + # Incase extension reported status at the last sec, we should prioritize reporting status over timeout + if not ext_completed and datetime.datetime.utcnow() > wait_until: + msg = "Dependent Extension {0} did not reach a terminal state within the allowed timeout. Last status was {1}".format( + extension_name, status) + raise Exception(msg) - return True + if status != ValidHandlerStatus.success: + msg = "Dependent Extension {0} did not succeed. Status was {1}".format(extension_name, status) + raise Exception(msg) - def handle_ext_handler(self, ext_handler, etag): + def handle_ext_handler(self, ext_handler_i, extension, etag): """ Execute the requested command for the handler and return if success - :param ext_handler: The ExtHandler to execute the command on + :param ext_handler_i: The ExtHandlerInstance object to execute the command on + :param extension: The extension settings on which to run the command on :param etag: Current incarnation of the GoalState :return: True if the operation was successful, False if not """ - ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol) + try: # Ensure the extension config was valid - if ext_handler.is_invalid_setting: - raise ExtensionConfigError(ext_handler.invalid_setting_reason) + if ext_handler_i.ext_handler.is_invalid_setting: + raise ExtensionConfigError(ext_handler_i.ext_handler.invalid_setting_reason) - state = ext_handler.properties.state + handler_state = ext_handler_i.ext_handler.properties.state # The Guest Agent currently only supports 1 installed version per extension on the VM. # If the extension version is unregistered and the customers wants to uninstall the extension, # we should let it go through even if the installed version doesnt exist in Handler manifest (PIR) anymore. # If target state is enabled and version not found in manifest, do not process the extension. - if ext_handler_i.decide_version(target_state=state) is None and state == ExtensionRequestedState.Enabled: + if ext_handler_i.decide_version(target_state=handler_state, + extension=extension) is None and handler_state == ExtHandlerRequestedState.Enabled: handler_version = ext_handler_i.ext_handler.properties.version name = ext_handler_i.ext_handler.name err_msg = "Unable to find version {0} in manifest for extension {1}".format(handler_version, name) ext_handler_i.set_operation(WALAEventOperation.Download) - ext_handler_i.set_handler_status(message=ustr(err_msg), code=-1) - ext_handler_i.report_event(message=ustr(err_msg), is_success=False) - return False - - ext_handler_i.logger.info("Target handler state: {0} [incarnation {1}]", state, etag) - if state == ExtensionRequestedState.Enabled: - self.handle_enable(ext_handler_i) - elif state == ExtensionRequestedState.Disabled: - self.handle_disable(ext_handler_i) - elif state == ExtensionRequestedState.Uninstall: - self.handle_uninstall(ext_handler_i) + raise ExtensionError(msg=err_msg) + + # Handle everything on an extension level rather than Handler level + ext_handler_i.logger.info("Target handler state: {0} [incarnation {1}]", handler_state, etag) + if handler_state == ExtHandlerRequestedState.Enabled: + self.handle_enable(ext_handler_i, extension) + elif handler_state == ExtHandlerRequestedState.Disabled: + self.handle_disable(ext_handler_i, extension) + elif handler_state == ExtHandlerRequestedState.Uninstall: + self.handle_uninstall(ext_handler_i, extension=extension) else: - message = u"Unknown ext handler state:{0}".format(state) + message = u"Unknown ext handler state:{0}".format(handler_state) raise ExtensionError(message) return True - + except MultiConfigExtensionEnableError as error: + ext_name = ext_handler_i.get_extension_full_name(extension) + err_msg = "Error processing MultiConfig extension {0}: {1}".format(ext_name, ustr(error)) + # This error is only thrown for enable operation on MultiConfig extension. + # Since these are maintained by the extensions, the expectation here is that they would update their status files appropriately with their errors. + # The extensions should already have a placeholder status file, but incase they dont, setting one here to fail fast. + ext_handler_i.create_placeholder_status_file(extension, status=ValidHandlerStatus.error, code=error.code, + operation=ext_handler_i.operation, message=err_msg) + add_event(name=ext_name, version=ext_handler_i.ext_handler.properties.version, op=ext_handler_i.operation, + is_success=False, log_event=True, message=err_msg) except ExtensionConfigError as error: # Catch and report Invalid ExtensionConfig errors here to fail fast rather than timing out after 90 min err_msg = "Ran into config errors: {0}. \nPlease retry again as another operation with updated settings".format( ustr(error)) self.__handle_and_report_ext_handler_errors(ext_handler_i, error, report_op=WALAEventOperation.InvalidExtensionConfig, - message=err_msg) + message=err_msg, extension=extension) except ExtensionUpdateError as error: # Not reporting the error as it has already been reported from the old version - self.handle_ext_handler_error(ext_handler_i, error, error.code, report_telemetry_event=False) + self.__handle_and_report_ext_handler_errors(ext_handler_i, error, ext_handler_i.operation, ustr(error), + report=False, extension=extension) except ExtensionDownloadError as error: msg = "Failed to download artifacts: {0}".format(ustr(error)) self.__handle_and_report_ext_handler_errors(ext_handler_i, error, report_op=WALAEventOperation.Download, - message=msg) + message=msg, extension=extension) except ExtensionError as error: - self.handle_ext_handler_error(ext_handler_i, error, error.code) + self.__handle_and_report_ext_handler_errors(ext_handler_i, error, ext_handler_i.operation, ustr(error), + extension=extension) except Exception as error: - self.handle_ext_handler_error(ext_handler_i, error) + error.code = -1 + self.__handle_and_report_ext_handler_errors(ext_handler_i, error, ext_handler_i.operation, ustr(error), + extension=extension) return False @staticmethod - def handle_ext_handler_error(ext_handler_i, error, code=-1, report_telemetry_event=True): - msg = ustr(error) - ext_handler_i.set_handler_status(message=msg, code=code) - - if report_telemetry_event: - ext_handler_i.report_event(message=msg, is_success=False, log_event=True) - - @staticmethod - def __handle_and_report_ext_handler_errors(ext_handler_i, error, report_op, message): + def __handle_and_report_ext_handler_errors(ext_handler_i, error, report_op, message, report=True, extension=None): + # This function is only called for Handler level errors, we capture MultiConfig errors separately, + # so report only HandlerStatus here. ext_handler_i.set_handler_status(message=message, code=error.code) - report_event(op=report_op, is_success=False, log_event=True, message=message) - def handle_enable(self, ext_handler_i): + # If the handler supports multi-config, create a status file with failed status if no status file exists. + # This is for correctly reporting errors back to CRP for failed Handler level operations for MultiConfig extensions. + # In case of Handler failures, we will retry each time for each extension, so we need to create a status + # file with failure since the extensions wont be called where they can create their status files. + # This way we guarantee reporting back to CRP + if ext_handler_i.should_perform_multi_config_op(extension): + ext_handler_i.create_placeholder_status_file(extension, status=ValidHandlerStatus.error, code=error.code, + operation=report_op, message=message) + + if report: + name = ext_handler_i.get_extension_full_name(extension) + handler_version = ext_handler_i.ext_handler.properties.version + add_event(name=name, version=handler_version, op=report_op, is_success=False, log_event=True, + message=message) + + def handle_enable(self, ext_handler_i, extension): + """ + 1- Ensure the handler is installed + 2- Check if extension is enabled or disabled and then process accordingly + """ self.log_process = True uninstall_exit_code = None old_ext_handler_i = ext_handler_i.get_installed_ext_handler() - handler_state = ext_handler_i.get_handler_state() - ext_handler_i.logger.info("[Enable] current handler state is: {0}", - handler_state.lower()) + current_handler_state = ext_handler_i.get_handler_state() + ext_handler_i.logger.info("[Enable] current handler state is: {0}", current_handler_state.lower()) # We go through the entire process of downloading and initializing the extension if it's either a fresh # extension or if it's a retry of a previously failed upgrade. - if handler_state == ExtHandlerState.NotInstalled or handler_state == ExtHandlerState.FailedUpgrade: - ext_handler_i.set_handler_state(ExtHandlerState.NotInstalled) - ext_handler_i.download() - ext_handler_i.initialize() - ext_handler_i.update_settings() + if current_handler_state == ExtHandlerState.NotInstalled or current_handler_state == ExtHandlerState.FailedUpgrade: + self.__setup_new_handler(ext_handler_i, extension) + if old_ext_handler_i is None: - ext_handler_i.install() + ext_handler_i.install(extension=extension) elif ext_handler_i.version_ne(old_ext_handler_i): + # This is a special case, we need to update the handler version here but to do that we need to also + # disable each enabled extension of this handler. uninstall_exit_code = ExtHandlersHandler._update_extension_handler_and_return_if_failed( - old_ext_handler_i, ext_handler_i) + old_ext_handler_i, ext_handler_i, extension) else: - ext_handler_i.update_settings() + ext_handler_i.ensure_consistent_data_for_mc() + ext_handler_i.update_settings(extension) + + # Always create a placeholder file for enable/disable command if not exists + # We don't create a placeholder for other commands because we use status file as a way to report Handler level + # failures back to CRP. If a placeholder for an extension already exists with Transitioning status, we would + # not override it, hence we only create a placeholder for enable/disable commands but the extensions have the + # data to create their own if needed. + ext_handler_i.create_placeholder_status_file(extension) + self.__handle_extension(ext_handler_i, extension, uninstall_exit_code) - ext_handler_i.enable(uninstall_exit_code=uninstall_exit_code) + @staticmethod + def __setup_new_handler(ext_handler_i, extension): + ext_handler_i.set_handler_state(ExtHandlerState.NotInstalled) + ext_handler_i.download() + ext_handler_i.initialize() + ext_handler_i.update_settings(extension) + + @staticmethod + def __handle_extension(ext_handler_i, extension, uninstall_exit_code): + # Check if extension level settings provided for the handler, if not, call enable for the handler. + # This is legacy behavior, we can have handlers with no settings. + if extension is None: + ext_handler_i.enable() + return + + # MultiConfig: Handle extension level ops here + ext_handler_i.logger.info("{0} requested state: {1}", ext_handler_i.get_extension_full_name(extension), + extension.state) + + if extension.state == ExtensionState.Enabled: + ext_handler_i.enable(extension, uninstall_exit_code=uninstall_exit_code) + elif extension.state == ExtensionState.Disabled: + # Only disable extension if the requested state == Disabled and current state is != Disabled + if ext_handler_i.get_extension_state(extension) != ExtensionState.Disabled: + # Extensions can only be disabled for Multi Config extensions. Disable operation for extension is + # tantamount to uninstalling Handler so ignoring errors incase of Disable failure and deleting state. + ext_handler_i.disable(extension, ignore_error=True) + else: + ext_handler_i.logger.info( + "{0} already disabled, not doing anything".format(ext_handler_i.get_extension_full_name(extension))) + else: + raise ExtensionConfigError( + "Unknown requested state for Extension {0}: {1}".format(extension.name, extension.state)) @staticmethod - def _update_extension_handler_and_return_if_failed(old_ext_handler_i, ext_handler_i): + def _update_extension_handler_and_return_if_failed(old_ext_handler_i, ext_handler_i, extension=None): def execute_old_handler_command_and_return_if_succeeds(func): """ @@ -678,52 +777,124 @@ def execute_old_handler_command_and_return_if_succeeds(func): logger.info("Continue on Update failure flag is set, proceeding with update") return exit_code - disable_exit_code = NOT_RUN - # We only want to disable the old handler if it is currently enabled; no - # other state makes sense. + disable_exit_codes = defaultdict(lambda: NOT_RUN) + # We only want to disable the old handler if it is currently enabled; no other state makes sense. if old_ext_handler_i.get_handler_state() == ExtHandlerState.Enabled: - disable_exit_code = execute_old_handler_command_and_return_if_succeeds( - func=lambda: old_ext_handler_i.disable()) # pylint: disable=W0108 + + # Corner case - If the old handler is a Single config Handler with no extensions at all, + # we should just disable the handler + if not old_ext_handler_i.supports_multi_config and not any(old_ext_handler_i.extensions): + disable_exit_codes[ + old_ext_handler_i.ext_handler.name] = execute_old_handler_command_and_return_if_succeeds( + func=partial(old_ext_handler_i.disable, extension=None)) + + # Else we disable all enabled extensions of this handler + # Note: If MC is supported this will disable only enabled_extensions else it will disable all extensions + for old_ext in old_ext_handler_i.enabled_extensions: + disable_exit_codes[old_ext.name] = execute_old_handler_command_and_return_if_succeeds( + func=partial(old_ext_handler_i.disable, extension=old_ext)) ext_handler_i.copy_status_files(old_ext_handler_i) if ext_handler_i.version_gt(old_ext_handler_i): - ext_handler_i.update(disable_exit_code=disable_exit_code, - updating_from_version=old_ext_handler_i.ext_handler.properties.version) + ext_handler_i.update(disable_exit_codes=disable_exit_codes, + updating_from_version=old_ext_handler_i.ext_handler.properties.version, + extension=extension) else: updating_from_version = ext_handler_i.ext_handler.properties.version - old_ext_handler_i.update(version=updating_from_version, - disable_exit_code=disable_exit_code, updating_from_version=updating_from_version) + old_ext_handler_i.update(handler_version=updating_from_version, + disable_exit_codes=disable_exit_codes, updating_from_version=updating_from_version, + extension=extension) uninstall_exit_code = execute_old_handler_command_and_return_if_succeeds( - func=lambda: old_ext_handler_i.uninstall()) # pylint: disable=W0108 + func=partial(old_ext_handler_i.uninstall, extension=extension)) old_ext_handler_i.remove_ext_handler() - ext_handler_i.update_with_install(uninstall_exit_code=uninstall_exit_code) + ext_handler_i.update_with_install(uninstall_exit_code=uninstall_exit_code, extension=extension) return uninstall_exit_code - def handle_disable(self, ext_handler_i): + def handle_disable(self, ext_handler_i, extension=None): + """ + Disable is a legacy behavior, CRP doesn't support it, its only for XML based extensions. + In case we get a disable request, just disable that extension. + """ self.log_process = True handler_state = ext_handler_i.get_handler_state() - ext_handler_i.logger.info("[Disable] current handler state is: {0}", - handler_state.lower()) + ext_handler_i.logger.info("[Disable] current handler state is: {0}", handler_state.lower()) if handler_state == ExtHandlerState.Enabled: - ext_handler_i.disable() + ext_handler_i.disable(extension) - def handle_uninstall(self, ext_handler_i): + def handle_uninstall(self, ext_handler_i, extension): + """ + To Uninstall the handler, first ensure all extensions are disabled + 1- Disable all enabled extensions first if Handler is Enabled and then Disable the handler + (disabled extensions wont have any extensions dependent on them so we can just go + ahead and remove all of them at once if HandlerState==Uninstall. + CRP will only set the HandlerState to Uninstall if all its extensions are set to be disabled) + 2- Finally uninstall the handler + """ self.log_process = True handler_state = ext_handler_i.get_handler_state() - ext_handler_i.logger.info("[Uninstall] current handler state is: {0}", - handler_state.lower()) + ext_handler_i.logger.info("[Uninstall] current handler state is: {0}", handler_state.lower()) if handler_state != ExtHandlerState.NotInstalled: if handler_state == ExtHandlerState.Enabled: - ext_handler_i.disable() + # Corner case - Single config Handler with no extensions at all + # If there are no extension settings for Handler, we should just disable the handler + if not ext_handler_i.supports_multi_config and not any(ext_handler_i.extensions): + ext_handler_i.disable() + + # If Handler is Enabled, there should be atleast 1 enabled extension for the handler + # Note: If MC is supported this will disable only enabled_extensions else it will disable all extensions + for enabled_ext in ext_handler_i.enabled_extensions: + ext_handler_i.disable(enabled_ext) # Try uninstalling the extension and swallow any exceptions in case of failures after logging them try: - ext_handler_i.uninstall() + ext_handler_i.uninstall(extension=extension) except ExtensionError as e: ext_handler_i.report_event(message=ustr(e), is_success=False) ext_handler_i.remove_ext_handler() + def __get_handlers_on_file_system(self, incarnation_changed): + handlers_to_report = [] + for item, path in list_agent_lib_directory(skip_agent_package=True): + try: + handler_instance = ExtHandlersHandler.get_ext_handler_instance_from_path(name=item, + path=path, + protocol=self.protocol) + if handler_instance is not None: + ext_handler = handler_instance.ext_handler + # For each handler we need to add extensions to report their status. + # For Single Config, we just need to add one extension with name as Handler Name + # For Multi Config, walk the config directory and find all unique extension names + # and add them as extensions to the handler. + extensions_names = set() + # Settings for Multi Config are saved as ..settings. + # Use this pattern to determine if Handler supports Multi Config or not and add extensions + for settings_path in glob.iglob(os.path.join(handler_instance.get_conf_dir(), "*.*.settings")): + match = re.search("(?P\\w+)\\.\\d+\\.settings", settings_path) + if match is not None: + extensions_names.add(match.group("extname")) + ext_handler.supports_multi_config = True + + # If nothing found with that pattern then its a Single Config, add an extension with Handler Name + if not any(extensions_names): + extensions_names.add(ext_handler.name) + + for ext_name in extensions_names: + ext = Extension(name=ext_name) + # Fetch the last modified sequence number + seq_no, _ = handler_instance.get_status_file_path(ext) + ext.sequenceNumber = seq_no + # Append extension to the list of extensions for the handler + ext_handler.properties.extensions.append(ext) + + handlers_to_report.append(ext_handler) + except Exception as error: + # Log error once per incarnation + if incarnation_changed: + logger.warn("Can't fetch ExtHandler from path: {0}; Error: {1}".format(path, ustr(error))) + + return handlers_to_report + def report_ext_handlers_status(self, incarnation_changed=False): """ Go through handler_state dir, collect and report status @@ -735,21 +906,7 @@ def report_ext_handlers_status(self, incarnation_changed=False): # Incase of Unsupported error, report the status of the handlers in the VM if self.__last_gs_unsupported(): - for item, path in list_agent_lib_directory(skip_agent_package=True): - try: - handler_instance = ExtHandlersHandler.get_ext_handler_instance_from_path(name=item, - path=path, - protocol=self.protocol) - if handler_instance is not None: - ext_handler = handler_instance.ext_handler - # We need to add one extension for the agent to report its status (Single config only expects - # a single extension per Handler). Without this no extensions would be reported for this Handler - ext_handler.properties.extensions.append(Extension(name=ext_handler.name)) - handlers_to_report.append(ext_handler) - except Exception as error: - # Log error once per incarnation - if incarnation_changed: - logger.warn("Can't fetch ExtHandler from path: {0}; Error: {1}".format(path, ustr(error))) + handlers_to_report = self.__get_handlers_on_file_system(incarnation_changed) # If GoalState supported, report the status of extension handlers that were requested by the GoalState elif not self.__last_gs_unsupported() and self.ext_handlers is not None: @@ -757,7 +914,7 @@ def report_ext_handlers_status(self, incarnation_changed=False): for ext_handler in handlers_to_report: try: - self.report_ext_handler_status(vm_status, ext_handler) + self.report_ext_handler_status(vm_status, ext_handler, incarnation_changed) except ExtensionError as error: add_event(op=WALAEventOperation.ExtensionProcessing, is_success=False, message=ustr(error)) @@ -817,7 +974,7 @@ def write_ext_handlers_status_to_info_file(vm_status): try: handler_status.pop('code', None) handler_status.pop('message', None) - handler_status.pop('extensions', None) + handler_status.pop('extension_status', None) except KeyError: pass @@ -826,34 +983,103 @@ def write_ext_handlers_status_to_info_file(vm_status): fileutil.write_file(status_path, agent_details_json) - def report_ext_handler_status(self, vm_status, ext_handler): + def report_ext_handler_status(self, vm_status, ext_handler, incarnation_changed): ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol) handler_status = ext_handler_i.get_handler_status() + + # If nothing available, skip reporting if handler_status is None: + # We should always have some handler status if requested state != Uninstall irrespective of single or + # multi-config. If state is != Uninstall, report error + if ext_handler.properties.state != ExtHandlerRequestedState.Uninstall: + msg = "No handler status found for {0}. Not reporting anything for it.".format(ext_handler.name) + ext_handler_i.report_error_on_incarnation_change(incarnation_changed, log_msg=msg, event_msg=msg) return handler_state = ext_handler_i.get_handler_state() - if handler_state != ExtHandlerState.NotInstalled: - try: - active_exts = ext_handler_i.report_ext_status() - handler_status.extensions.extend(active_exts) - except ExtensionError as e: - ext_handler_i.set_handler_status(message=ustr(e), code=e.code) + ext_handler_statuses = [] + # For MultiConfig, we need to report status per extension even for Handler level failures. + # If we have HandlerStatus for a MultiConfig handler and GS is requesting for it, we would report status per + # extension even if HandlerState == NotInstalled (Sample scenario: ExtensionConfigError, DecideVersionError, etc) + if handler_state != ExtHandlerState.NotInstalled or ext_handler.supports_multi_config: + + # Since we require reading the Manifest for reading the heartbeat, this would fail if HandlerManifest not found. + # Only try to read heartbeat if HandlerState != NotInstalled. + if handler_state != ExtHandlerState.NotInstalled: + # Heartbeat is a handler level thing only, so we dont need to modify this + try: + heartbeat = ext_handler_i.collect_heartbeat() + if heartbeat is not None: + handler_status.status = heartbeat.get('status') + except ExtensionError as e: + ext_handler_i.set_handler_status(message=ustr(e), code=e.code) - try: - heartbeat = ext_handler_i.collect_heartbeat() - if heartbeat is not None: - handler_status.status = heartbeat.get('status') - except ExtensionError as e: - ext_handler_i.set_handler_status(message=ustr(e), code=e.code) + ext_handler_statuses = ext_handler_i.get_extension_handler_statuses(handler_status, incarnation_changed) - vm_status.vmAgent.extensionHandlers.append(handler_status) + # If not any extension status reported, report the Handler status + if not any(ext_handler_statuses): + ext_handler_statuses.append(handler_status) + + vm_status.vmAgent.extensionHandlers.extend(ext_handler_statuses) class ExtHandlerInstance(object): - def __truncate_file_head(self, filename, max_size): + def __init__(self, ext_handler, protocol, execution_log_max_size=(10 * 1024 * 1024), extension=None): + self.ext_handler = ext_handler + self.protocol = protocol + self.operation = None + self.pkg = None + self.pkg_file = None + self.logger = None + self.set_logger(extension=extension, execution_log_max_size=execution_log_max_size) + + @property + def supports_multi_config(self): + return self.ext_handler.supports_multi_config + + @property + def extensions(self): + return self.ext_handler.properties.extensions + + @property + def enabled_extensions(self): + """ + In case of Single config, just return all the extensions of the handler + (expectation being that there'll only be a single extension per handler). + We will not be maintaining extension level state for Single config Handlers + """ + if self.supports_multi_config: + return [ext for ext in self.extensions if self.get_extension_state(ext) == ExtensionState.Enabled] + return self.extensions + + def get_extension_full_name(self, extension=None): + """ + Get the full name of the extension .. + :param extension: The requested extension + :return: if MultiConfig not supported or extension == None, else . + """ + if self.should_perform_multi_config_op(extension): + return "{0}.{1}".format(self.ext_handler.name, extension.name) + + return self.ext_handler.name + + def __set_command_execution_log(self, extension, execution_log_max_size): + try: + fileutil.mkdir(self.get_log_dir(), mode=0o755) + except IOError as e: + self.logger.error(u"Failed to create extension log dir: {0}", e) + else: + log_file_name = "CommandExecution.log" if not self.should_perform_multi_config_op( + extension) else "CommandExecution_{0}.log".format(extension.name) + + log_file = os.path.join(self.get_log_dir(), log_file_name) + self.__truncate_file_head(log_file, execution_log_max_size, self.get_extension_full_name(extension)) + self.logger.add_appender(logger.AppenderType.FILE, logger.LogLevel.INFO, log_file) + + @staticmethod + def __truncate_file_head(filename, max_size, extension_name): try: if os.stat(filename).st_size <= max_size: return @@ -874,8 +1100,9 @@ def __truncate_file_head(self, filename, max_size): # installed. return - logger.error("Exception occurred while attempting to truncate CommandExecution.log " - "for extension {0}. Exception is: {1}", self.ext_handler.name, e) + logger.error( + "Exception occurred while attempting to truncate {0} for extension {1}. Exception is: {2}", + filename, extension_name, ustr(e)) for f in (filename, filename + ".tmp"): try: @@ -884,31 +1111,10 @@ def __truncate_file_head(self, filename, max_size): if is_file_not_found_error(cleanup_exception): logger.info("File '{0}' does not exist.", f) else: - logger.warn("Exception occurred while attempting " - "to remove file '{0}': {1}", f, cleanup_exception) - - - def __init__(self, ext_handler, protocol, execution_log_max_size=(10 * 1024 * 1024)): - self.ext_handler = ext_handler - self.protocol = protocol - self.operation = None - self.pkg = None - self.pkg_file = None - self.logger = None - self.set_logger() - - try: - fileutil.mkdir(self.get_log_dir(), mode=0o755) - except IOError as e: - self.logger.error(u"Failed to create extension log dir: {0}", e) - else: - log_file = os.path.join(self.get_log_dir(), "CommandExecution.log") - - self.__truncate_file_head(log_file, execution_log_max_size) + logger.warn("Exception occurred while attempting to remove file '{0}': {1}", f, + cleanup_exception) - self.logger.add_appender(logger.AppenderType.FILE, logger.LogLevel.INFO, log_file) - - def decide_version(self, target_state=None): + def decide_version(self, target_state=None, extension=None): self.logger.verbose("Decide which version to use") try: pkg_list = self.protocol.get_ext_handler_pkgs(self.ext_handler) @@ -921,9 +1127,7 @@ def decide_version(self, target_state=None): # Determine the desired and installed versions requested_version = FlexibleVersion(str(self.ext_handler.properties.version)) installed_version_string = self.get_installed_version() - installed_version = requested_version \ - if installed_version_string is None \ - else FlexibleVersion(installed_version_string) + installed_version = requested_version if installed_version_string is None else FlexibleVersion(installed_version_string) # Divide packages # - Find the installed package (its version must exactly match) @@ -943,7 +1147,7 @@ def decide_version(self, target_state=None): # Note: # - A downgrade, which will be bound to the same major version, # is allowed if the installed version is no longer available - if target_state in (ExtensionRequestedState.Uninstall, ExtensionRequestedState.Disabled): + if target_state in (ExtHandlerRequestedState.Uninstall, ExtHandlerRequestedState.Disabled): if installed_pkg is None: msg = "Failed to find installed version: {0} of Handler: {1} in handler manifest to uninstall.".format( installed_version, self.ext_handler.name) @@ -958,12 +1162,17 @@ def decide_version(self, target_state=None): if self.pkg is not None: self.logger.verbose("Use version: {0}", self.pkg.version) - self.set_logger() + + # We reset the logger here incase the handler version changes + if not requested_version.matches(FlexibleVersion(self.ext_handler.properties.version)): + self.set_logger(extension=extension) + return self.pkg - def set_logger(self): + def set_logger(self, execution_log_max_size=(10 * 1024 * 1024), extension=None): prefix = "[{0}]".format(self.get_full_name()) self.logger = logger.Logger(logger.DEFAULT_LOGGER, prefix) + self.__set_command_execution_log(extension, execution_log_max_size) def version_gt(self, other): self_version = self.ext_handler.properties.version @@ -998,8 +1207,7 @@ def get_installed_version(self): if not os.path.exists(state_path) or fileutil.read_file(state_path) == ExtHandlerState.NotInstalled \ or fileutil.read_file(state_path) == ExtHandlerState.FailedUpgrade: - logger.verbose("Ignoring version of uninstalled or failed extension: " - "{0}".format(path)) + logger.verbose("Ignoring version of uninstalled or failed extension: {0}".format(path)) continue if latest_version is None or latest_version < version_from_path: @@ -1028,9 +1236,10 @@ def copy_status_files(self, old_ext_handler_i): def set_operation(self, op): self.operation = op - def report_event(self, message="", is_success=True, duration=0, log_event=True): + def report_event(self, name=None, message="", is_success=True, duration=0, log_event=True): ext_handler_version = self.ext_handler.properties.version - add_event(name=self.ext_handler.name, version=ext_handler_version, message=message, + name = self.ext_handler.name if name is None else name + add_event(name=name, version=ext_handler_version, message=message, op=self.operation, is_success=is_success, duration=duration, log_event=log_event) def _download_extension_package(self, source_uri, target_file): @@ -1105,6 +1314,16 @@ def download(self): self.pkg_file = destination + def ensure_consistent_data_for_mc(self): + # If CRP expects Handler to support MC, ensure the HandlerManifest also reflects that. + # Even though the HandlerManifest.json is not expected to change once the extension is installed, + # CRP can wrongfully request send a Multi-Config GoalState even if the Handler supports only Single Config. + # Checking this only if HandlerState == Enable. In case of Uninstall, we dont care. + if self.supports_multi_config and not self.load_manifest().supports_multiple_extensions(): + raise ExtensionConfigError( + "Handler {0} does not support MultiConfig but CRP expects it, failing due to inconsistent data".format( + self.ext_handler.name)) + def initialize(self): self.logger.info("Initializing extension {0}".format(self.get_full_name())) @@ -1125,6 +1344,8 @@ def initialize(self): fileutil.clean_ioerror(e, paths=[self.get_base_dir(), self.pkg_file]) raise ExtensionDownloadError(u"Failed to save HandlerManifest.json", e) + self.ensure_consistent_data_for_mc() + # Create status and config dir try: status_dir = self.get_status_dir() @@ -1136,27 +1357,6 @@ def initialize(self): if get_supported_feature_by_name(SupportedFeatureNames.ExtensionTelemetryPipeline).is_supported: fileutil.mkdir(self.get_extension_events_dir(), mode=0o700) - seq_no, status_path = self.get_status_file_path() # pylint: disable=W0612 - if status_path is not None: - now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - status = [ - { - "version": 1.0, - "timestampUTC": now, - "status": { - "name": self.ext_handler.name, - "operation": "Enabling Handler", - "status": "transitioning", - "code": 0, - "formattedMessage": { - "lang": "en-US", - "message": "Install/Enable is in progress." - } - } - } - ] - fileutil.write_file(status_path, json.dumps(status)) - except IOError as e: fileutil.clean_ioerror(e, paths=[self.get_base_dir(), self.pkg_file]) raise ExtensionDownloadError(u"Failed to initialize extension '{0}'".format(self.get_full_name()), e) @@ -1164,30 +1364,102 @@ def initialize(self): # Save HandlerEnvironment.json self.create_handler_env() - def enable(self, uninstall_exit_code=None): + def create_placeholder_status_file(self, extension=None, status=ValidHandlerStatus.transitioning, code=0, + operation="Enabling Extension", message="Install/Enable is in progress."): + _, status_path = self.get_status_file_path(extension) + if status_path is not None and not os.path.exists(status_path): + now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + status = [ + { + "version": 1.0, + "timestampUTC": now, + "status": { + "name": self.get_extension_full_name(extension), + "operation": operation, + "status": status, + "code": code, + "formattedMessage": { + "lang": "en-US", + "message": message + } + } + } + ] + # Create status directory if not exists. This is needed in the case where the Handler fails before even + # initializing the directories (ExtensionConfigError, Version deleted from PIR error, etc) + if not os.path.exists(os.path.dirname(status_path)): + fileutil.mkdir(os.path.dirname(status_path), mode=0o700) + fileutil.write_file(status_path, json.dumps(status)) + + def enable(self, extension=None, uninstall_exit_code=None): + try: + self._enable_extension(extension, uninstall_exit_code) + except ExtensionError as error: + if self.should_perform_multi_config_op(extension): + raise MultiConfigExtensionEnableError(error) + raise + # Even if a single extension is enabled for this handler, set the Handler state as Enabled + self.set_handler_state(ExtHandlerState.Enabled) + self.set_handler_status(status="Ready", message="Plugin enabled") + + def should_perform_multi_config_op(self, extension): + return self.supports_multi_config and extension is not None + + def _enable_extension(self, extension, uninstall_exit_code): uninstall_exit_code = str(uninstall_exit_code) if uninstall_exit_code is not None else NOT_RUN - env = {ExtCommandEnvVariable.UninstallReturnCode: uninstall_exit_code} + + env = { + ExtCommandEnvVariable.UninstallReturnCode: uninstall_exit_code + } self.set_operation(WALAEventOperation.Enable) man = self.load_manifest() enable_cmd = man.get_enable_command() - self.logger.info("Enable extension [{0}]".format(enable_cmd)) + self.logger.info("Enable extension {0}: [{1}]".format(self.get_extension_full_name(extension), enable_cmd)) self.launch_command(enable_cmd, timeout=300, - extension_error_code=ExtensionErrorCodes.PluginEnableProcessingFailed, env=env) - self.set_handler_state(ExtHandlerState.Enabled) - self.set_handler_status(status="Ready", message="Plugin enabled") + extension_error_code=ExtensionErrorCodes.PluginEnableProcessingFailed, env=env, + extension=extension) + + if self.should_perform_multi_config_op(extension): + # Only save extension state if MC supported + self.__set_extension_state(extension, ExtensionState.Enabled) - def disable(self): + def _disable_extension(self, extension=None): self.set_operation(WALAEventOperation.Disable) man = self.load_manifest() disable_cmd = man.get_disable_command() - self.logger.info("Disable extension [{0}]".format(disable_cmd)) + self.logger.info("Disable extension {0}: [{1}]".format(self.get_extension_full_name(extension), disable_cmd)) self.launch_command(disable_cmd, timeout=900, - extension_error_code=ExtensionErrorCodes.PluginDisableProcessingFailed) - self.set_handler_state(ExtHandlerState.Installed) - self.set_handler_status(status="NotReady", message="Plugin disabled") + extension_error_code=ExtensionErrorCodes.PluginDisableProcessingFailed, + extension=extension) - def install(self, uninstall_exit_code=None): + def disable(self, extension=None, ignore_error=False): + try: + self._disable_extension(extension) + except ExtensionError as error: + if not ignore_error: + raise + + msg = "[Ignored Error] Ran into error disabling extension {0}:{1}".format( + self.get_extension_full_name(extension), ustr(error)) + self.logger.info(msg) + self.report_event(name=self.get_extension_full_name(extension), message=msg, is_success=False, + log_event=False) + + # Clean extension state For Multi Config extensions on Disable + if self.should_perform_multi_config_op(extension): + self.__remove_extension_state_files(extension) + + # For Single config, dont check enabled_extensions because no extension state is maintained. + # For MultiConfig, Set the handler state to Installed only when all extensions have been disabled + if not self.supports_multi_config or not any(self.enabled_extensions): + self.set_handler_state(ExtHandlerState.Installed) + self.set_handler_status(status="NotReady", message="Plugin disabled") + + def install(self, uninstall_exit_code=None, extension=None): + # For Handler level operations, extension just specifies the settings that initiated the install. + # This is needed to provide the sequence number and extension name in case the extension needs to report + # failure/status using status file. uninstall_exit_code = str(uninstall_exit_code) if uninstall_exit_code is not None else NOT_RUN env = {ExtCommandEnvVariable.UninstallReturnCode: uninstall_exit_code} @@ -1195,16 +1467,20 @@ def install(self, uninstall_exit_code=None): install_cmd = man.get_install_command() self.logger.info("Install extension [{0}]".format(install_cmd)) self.set_operation(WALAEventOperation.Install) - self.launch_command(install_cmd, timeout=900, + self.launch_command(install_cmd, timeout=900, extension=extension, extension_error_code=ExtensionErrorCodes.PluginInstallProcessingFailed, env=env) self.set_handler_state(ExtHandlerState.Installed) + self.set_handler_status(status="NotReady", message="Plugin installed but not enabled") - def uninstall(self): + def uninstall(self, extension=None): + # For Handler level operations, extension just specifies the settings that initiated the uninstall. + # This is needed to provide the sequence number and extension name in case the extension needs to report + # failure/status using status file. self.set_operation(WALAEventOperation.UnInstall) man = self.load_manifest() uninstall_cmd = man.get_uninstall_command() self.logger.info("Uninstall extension [{0}]".format(uninstall_cmd)) - self.launch_command(uninstall_cmd) + self.launch_command(uninstall_cmd, extension=extension) def remove_ext_handler(self): try: @@ -1229,13 +1505,29 @@ def on_rmtree_error(_, __, exc_info): self.report_event(message=message, is_success=False) self.logger.warn(message) - def update(self, version=None, disable_exit_code=None, updating_from_version=None): # pylint: disable=W0621 - if version is None: - version = self.ext_handler.properties.version + def update(self, handler_version=None, disable_exit_codes=None, updating_from_version=None, extension=None): + # For Handler level operations, extension just specifies the settings that initiated the update. + # This is needed to provide the sequence number and extension name in case the extension needs to report + # failure/status using status file. + if handler_version is None: + handler_version = self.ext_handler.properties.version + + env = { + 'VERSION': handler_version, + ExtCommandEnvVariable.UpdatingFromVersion: updating_from_version + } - disable_exit_code = str(disable_exit_code) if disable_exit_code is not None else NOT_RUN - env = {'VERSION': version, ExtCommandEnvVariable.DisableReturnCode: disable_exit_code, - ExtCommandEnvVariable.UpdatingFromVersion: updating_from_version} + if not self.supports_multi_config: + # For single config, extension.name == ext_handler.name + env[ExtCommandEnvVariable.DisableReturnCode] = ustr(disable_exit_codes.get(self.ext_handler.name)) + else: + disable_codes = [] + for ext in self.extensions: + disable_codes.append({ + "extensionName": ext.name, + "exitCode": ustr(disable_exit_codes.get(ext.name)) + }) + env[ExtCommandEnvVariable.DisableReturnCodeMultipleExtensions] = json.dumps(disable_codes) try: self.set_operation(WALAEventOperation.Update) @@ -1245,31 +1537,33 @@ def update(self, version=None, disable_exit_code=None, updating_from_version=Non self.launch_command(update_cmd, timeout=900, extension_error_code=ExtensionErrorCodes.PluginUpdateProcessingFailed, - env=env) + env=env, extension=extension) except ExtensionError: # Mark the handler as Failed so we don't clean it up and can keep reporting its status self.set_handler_state(ExtHandlerState.FailedUpgrade) raise - def update_with_install(self, uninstall_exit_code=None): + def update_with_install(self, uninstall_exit_code=None, extension=None): man = self.load_manifest() if man.is_update_with_install(): - self.install(uninstall_exit_code=uninstall_exit_code) + self.install(uninstall_exit_code=uninstall_exit_code, extension=extension) else: self.logger.info("UpdateWithInstall not set. " "Skip install during upgrade.") self.set_handler_state(ExtHandlerState.Installed) - def _get_last_modified_seq_no_from_config_files(self): + def _get_last_modified_seq_no_from_config_files(self, extension): """ The sequence number is not guaranteed to always be strictly increasing. To ensure we always get the latest one, fetching the sequence number from config file that was last modified (and not necessarily the largest). :return: Last modified Sequence number or -1 on errors - - Note: This function is going to be deprecated soon. We should only rely on seqNo from GoalState rather than file system. """ seq_no = -1 + if self.supports_multi_config and (extension is None or extension.name is None): + # If no extension name is provided for Multi Config, don't try to parse any sequence number from filesystem + return seq_no + try: largest_modified_time = 0 conf_dir = self.get_conf_dir() @@ -1278,9 +1572,14 @@ def _get_last_modified_seq_no_from_config_files(self): if not os.path.isfile(item_path): continue try: - separator = item.rfind(".") - if separator > 0 and item[separator + 1:] == 'settings': - curr_seq_no = int(item.split('.')[0]) + # Settings file for Multi Config look like - ..settings + # Settings file for Single Config look like - .settings + match = re.search("((?P\\w+)\\.)*(?P\\d+)\\.settings", item_path) + if match is not None: + ext_name = match.group('ext_name') + if self.supports_multi_config and extension.name != ext_name: + continue + curr_seq_no = int(match.group("seq_no")) curr_modified_time = os.path.getmtime(item_path) if curr_modified_time > largest_modified_time: seq_no = curr_seq_no @@ -1295,44 +1594,45 @@ def _get_last_modified_seq_no_from_config_files(self): return seq_no def get_status_file_path(self, extension=None): + """ + We should technically only fetch the sequence number from GoalState and not rely on the filesystem at all, + But there are certain scenarios where we need to fetch the latest sequence number from the filesystem + (For example when we need to report the status for extensions of previous GS if the current GS is Unsupported). + Always prioritizing sequence number from extensions but falling back to filesystem + :param extension: Extension for which the sequence number is required + :return: Sequence number for the extension, Status file path or -1, None + """ path = None - # Todo: Remove check on filesystem for fetching sequence number (legacy behaviour). - # We should technically only fetch the sequence number from GoalState and not rely on the filesystem at all, - # But since we still have Kusto data from the operation below (~0.000065% VMs are still reporting - # WALAEventOperation.SequenceNumberMismatch), keeping this as is with modified logic for fetching - # sequence number from filesystem. Based on the new data we will eventually phase this out. - seq_no = self._get_last_modified_seq_no_from_config_files() - - # Issue 1116: use the sequence number from goal state where possible + seq_no = None if extension is not None and extension.sequenceNumber is not None: try: - gs_seq_no = int(extension.sequenceNumber) - - if gs_seq_no != seq_no: - add_event(AGENT_NAME, version=CURRENT_VERSION, op=WALAEventOperation.SequenceNumberMismatch, - is_success=False, message="Goal state: {0}, disk: {1}".format(gs_seq_no, seq_no), - log_event=False) - - seq_no = gs_seq_no + seq_no = int(extension.sequenceNumber) except ValueError: logger.error('Sequence number [{0}] does not appear to be valid'.format(extension.sequenceNumber)) - if seq_no > -1: - path = os.path.join( - self.get_status_dir(), - "{0}.status".format(seq_no)) + if seq_no is None: + # If we're unable to fetch Sequence number from Extension for any reason, + # try fetching it from the last modified Settings file. + seq_no = self._get_last_modified_seq_no_from_config_files(extension) - return seq_no, path + if seq_no is not None and seq_no > -1: + if self.should_perform_multi_config_op(extension) and extension is not None and extension.name is not None: + path = os.path.join(self.get_status_dir(), "{0}.{1}.status".format(extension.name, seq_no)) + elif not self.supports_multi_config: + path = os.path.join(self.get_status_dir(), "{0}.status").format(seq_no) + + return seq_no if seq_no is not None else -1, path def collect_ext_status(self, ext): - self.logger.verbose("Collect extension status for {0}".format(ext.name)) + self.logger.verbose("Collect extension status for {0}".format(self.get_extension_full_name(ext))) seq_no, ext_status_file = self.get_status_file_path(ext) if seq_no == -1: return None data = None data_str = None - ext_status = ExtensionStatus(seq_no=seq_no) + # Extension.name contains the extension name in case of MC and Handler name in case of Single Config. + ext_status = ExtensionStatus(name=ext.name, seq_no=seq_no) try: data_str, data = self._read_and_parse_json_status_file(ext_status_file) @@ -1340,20 +1640,21 @@ def collect_ext_status(self, ext): msg = "" if e.code == ExtensionStatusError.CouldNotReadStatusFile: ext_status.code = ExtensionErrorCodes.PluginUnknownFailure - msg = u"We couldn't read any status for {0}-{1} extension, for the sequence number {2}. It failed due" \ - u" to {3}".format(ext.name, self.ext_handler.properties.version, seq_no, e) + msg = u"We couldn't read any status for {0} extension, for the sequence number {1}. It failed due" \ + u" to {2}".format(self.get_full_name(ext), seq_no, ustr(e)) elif ExtensionStatusError.InvalidJsonFile: ext_status.code = ExtensionErrorCodes.PluginSettingsStatusInvalid - msg = u"The status reported by the extension {0}-{1}(Sequence number {2}), was in an " \ - u"incorrect format and the agent could not parse it correctly. Failed due to {3}" \ - .format(ext.name, self.ext_handler.properties.version, seq_no, e) + msg = u"The status reported by the extension {0}(Sequence number {1}), was in an " \ + u"incorrect format and the agent could not parse it correctly. Failed due to {2}" \ + .format(self.get_full_name(ext), seq_no, ustr(e)) # This log is periodic due to the verbose nature of the status check. Please make sure that the message # constructed above does not change very frequently and includes important info such as sequence number, # extension name to make sure that the log reflects changes in the extension sequence for which the # status is being sent. logger.periodic_warn(logger.EVERY_HALF_HOUR, u"[PERIODIC] " + msg) - add_periodic(delta=logger.EVERY_HALF_HOUR, name=ext.name, version=self.ext_handler.properties.version, + add_periodic(delta=logger.EVERY_HALF_HOUR, name=self.get_extension_full_name(ext), + version=self.ext_handler.properties.version, op=WALAEventOperation.StatusProcessing, is_success=False, message=msg, log_event=False) @@ -1367,17 +1668,18 @@ def collect_ext_status(self, ext): try: parse_ext_status(ext_status, data) if len(data_str) > _MAX_STATUS_FILE_SIZE_IN_BYTES: - raise ExtensionStatusError(msg="For Extension Handler {0}-{1} for the sequence number {2}, the status " - "file {3} of size {4} bytes is too big. Max Limit allowed is {5} bytes" - .format(ext.name, self.ext_handler.properties.version, seq_no, + raise ExtensionStatusError(msg="For Extension Handler {0} for the sequence number {1}, the status " + "file {2} of size {3} bytes is too big. Max Limit allowed is {4} bytes" + .format(self.get_full_name(ext), seq_no, ext_status_file, len(data_str), _MAX_STATUS_FILE_SIZE_IN_BYTES), code=ExtensionStatusError.MaxSizeExceeded) except ExtensionStatusError as e: - msg = u"For Extension Handler {0}-{1} for the sequence number {2}, the status file {3}. " \ - u"Encountered the following error: {4}".format(ext.name, self.ext_handler.properties.version, seq_no, + msg = u"For Extension Handler {0} for the sequence number {1}, the status file {2}. " \ + u"Encountered the following error: {3}".format(self.get_full_name(ext), seq_no, ext_status_file, ustr(e)) logger.periodic_warn(logger.EVERY_DAY, u"[PERIODIC] " + msg) - add_periodic(delta=logger.EVERY_HALF_HOUR, name=ext.name, version=self.ext_handler.properties.version, + add_periodic(delta=logger.EVERY_HALF_HOUR, name=self.get_extension_full_name(ext), + version=self.ext_handler.properties.version, op=WALAEventOperation.StatusProcessing, is_success=False, message=msg, log_event=False) if e.code == ExtensionStatusError.MaxSizeExceeded: @@ -1385,9 +1687,8 @@ def collect_ext_status(self, ext): ext_status.substatusList = self._process_substatus_list(ext_status.substatusList, field_size) elif e.code == ExtensionStatusError.StatusFileMalformed: - ext_status.message = "Could not get a valid status from the extension {0}-{1}. Encountered the " \ - "following error: {2}".format(ext.name, self.ext_handler.properties.version, - ustr(e)) + ext_status.message = "Could not get a valid status from the extension {0}. Encountered the " \ + "following error: {1}".format(self.get_full_name(ext), ustr(e)) ext_status.code = ExtensionErrorCodes.PluginSettingsStatusInvalid ext_status.status = ValidHandlerStatus.error @@ -1424,20 +1725,77 @@ def is_ext_handling_complete(self, ext): # Extension completed, return its status return True, status - def report_ext_status(self): - active_exts = [] + def report_error_on_incarnation_change(self, incarnation_changed, log_msg, event_msg, extension=None, + op=WALAEventOperation.ReportStatus): + # Since this code is called on a loop, logging as a warning only on incarnation change, else logging it + # as verbose + if incarnation_changed: + logger.warn(log_msg) + add_event(name=self.get_extension_full_name(extension), version=self.ext_handler.properties.version, + op=op, message=event_msg, is_success=False, log_event=False) + else: + logger.verbose(log_msg) + + def get_extension_handler_statuses(self, handler_status, incarnation_changed): + """ + Get the list of ExtHandlerStatus objects corresponding to each extension in the Handler. Each object might have + its own status for the Extension status but the Handler status would be the same for each extension in a Handle + :return: List of ExtHandlerStatus objects for each extension in the Handler + """ + ext_handler_statuses = [] # TODO Refactor or remove this common code pattern (for each extension subordinate to an ext_handler, do X). - for ext in self.ext_handler.properties.extensions: - ext_status = self.collect_ext_status(ext) - if ext_status is None: + for ext in self.extensions: + # In MC, for disabled extensions we dont need to report status. Skip reporting if disabled and state == disabled + # Extension.state corresponds to the state requested by CRP, self.__get_extension_state() corresponds to the + # state of the extension on the VM. Skip reporting only if both are Disabled + if self.should_perform_multi_config_op(ext) and \ + ext.state == ExtensionState.Disabled and self.get_extension_state(ext) == ExtensionState.Disabled: continue + + # Breaking off extension reporting in 2 parts, one which is Handler dependent and the other that is Extension dependent + try: + ext_handler_status = ExtHandlerStatus() + set_properties("ExtHandlerStatus", ext_handler_status, get_properties(handler_status)) + except Exception as error: + msg = "Something went wrong when trying to get a copy of the Handler status for {0}: {1}".format( + self.get_extension_full_name(), ustr(error)) + self.report_error_on_incarnation_change(incarnation_changed, event_msg=msg, + log_msg="{0}.\nStack Trace: {1}".format( + msg, traceback.format_exc())) + # Since this is a Handler level error and we need to do it per extension, breaking here and logging + # error since we wont be able to report error anyways and saving it as a handler status (legacy behavior) + self.set_handler_status(message=msg, code=-1) + break + + # For the extension dependent stuff, if there's some unhandled error, we will report it back to CRP as an extension error. try: - self.protocol.report_ext_status(self.ext_handler.name, ext.name, - ext_status) - active_exts.append(ext.name) - except ProtocolError as e: - self.logger.error(u"Failed to report extension status: {0}", e) - return active_exts + ext_status = self.collect_ext_status(ext) + if ext_status is not None: + ext_handler_status.extension_status = ext_status + ext_handler_statuses.append(ext_handler_status) + except ExtensionError as error: + + msg = "Unknown error when trying to fetch status from extension {0}: {1}".format( + self.get_extension_full_name(ext), ustr(error)) + self.report_error_on_incarnation_change(incarnation_changed, event_msg=msg, + log_msg="{0}.\nStack Trace: {1}".format( + msg, traceback.format_exc()), + extension=ext) + + # Unexpected error, for single config, keep the behavior as is + if not self.should_perform_multi_config_op(ext): + self.set_handler_status(message=ustr(error), code=error.code) + break + + # For MultiConfig, create a custom ExtensionStatus object with the error details and attach it to the Handler. + # This way the error would be reported back to CRP and the failure would be propagated instantly as compared to CRP eventually timing it out. + ext_status = ExtensionStatus(name=ext.name, seq_no=ext.sequenceNumber, + code=ExtensionErrorCodes.PluginUnknownFailure, + status=ValidHandlerStatus.error, message=msg) + ext_handler_status.extension_status = ext_status + ext_handler_statuses.append(ext_handler_status) + + return ext_handler_statuses def collect_heartbeat(self): # pylint: disable=R1710 man = self.load_manifest() @@ -1475,7 +1833,7 @@ def is_responsive(heartbeat_file): return last_update <= 600 def launch_command(self, cmd, timeout=300, extension_error_code=ExtensionErrorCodes.PluginProcessingError, - env=None): + env=None, extension=None): begin_utc = datetime.datetime.utcnow() self.logger.verbose("Launch command: [{0}]", cmd) @@ -1485,15 +1843,22 @@ def launch_command(self, cmd, timeout=300, extension_error_code=ExtensionErrorCo with tempfile.TemporaryFile(dir=base_dir, mode="w+b") as stderr: if env is None: env = {} - env.update(os.environ) + # Always add Extension Path and version to the current launch_command (Ask from publishers) env.update({ ExtCommandEnvVariable.ExtensionPath: base_dir, ExtCommandEnvVariable.ExtensionVersion: str(self.ext_handler.properties.version), - ExtCommandEnvVariable.ExtensionSeqNumber: str(self.get_seq_no()), - ExtCommandEnvVariable.WireProtocolAddress: self.protocol.get_endpoint() + ExtCommandEnvVariable.WireProtocolAddress: self.protocol.get_endpoint(), + + # Setting sequence number to 0 incase no settings provided to keep in accordance with the empty + # 0.settings file that we create for such extensions. + ExtCommandEnvVariable.ExtensionSeqNumber: str( + extension.sequenceNumber) if extension is not None else _DEFAULT_SEQ_NO }) + if self.should_perform_multi_config_op(extension): + env[ExtCommandEnvVariable.ExtensionName] = extension.name + supported_features = [] for _, feature in get_agent_supported_features_list_for_extensions().items(): supported_features.append( @@ -1503,18 +1868,20 @@ def launch_command(self, cmd, timeout=300, extension_error_code=ExtensionErrorCo } ) if supported_features: - env.update({ - ExtCommandEnvVariable.ExtensionSupportedFeatures: json.dumps(supported_features) - }) + env[ExtCommandEnvVariable.ExtensionSupportedFeatures] = json.dumps(supported_features) try: # Some extensions erroneously begin cmd with a slash; don't interpret those # as root-relative. (Issue #1170) - full_path = os.path.join(base_dir, cmd.lstrip(os.path.sep)) + command_full_path = os.path.join(base_dir, cmd.lstrip(os.path.sep)) + self.logger.info("Executing command: {0} with environment variables: {1}".format(command_full_path, + json.dumps(env))) + # Add the os environment variables before executing command + env.update(os.environ) process_output = CGroupConfigurator.get_instance().start_extension_command( - extension_name=self.get_full_name(), - command=full_path, + extension_name=self.get_full_name(extension), + command=command_full_path, timeout=timeout, shell=True, cwd=base_dir, @@ -1524,14 +1891,15 @@ def launch_command(self, cmd, timeout=300, extension_error_code=ExtensionErrorCo error_code=extension_error_code) except OSError as e: - raise ExtensionError("Failed to launch '{0}': {1}".format(full_path, e.strerror), + raise ExtensionError("Failed to launch '{0}': {1}".format(command_full_path, e.strerror), code=extension_error_code) duration = elapsed_milliseconds(begin_utc) - log_msg = "{0}\n{1}".format(cmd, "\n".join([line for line in process_output.split('\n') if line != ""])) - - self.logger.verbose(log_msg) - self.report_event(message=log_msg, duration=duration, log_event=False) + ext_name = self.get_extension_full_name(extension) + log_msg = "Extension: {0}; Command: {1}\n{2}".format(ext_name, cmd, "\n".join( + [line for line in process_output.split('\n') if line != ""])) + self.logger.info(log_msg) + self.report_event(name=ext_name, message=log_msg, duration=duration, log_event=False) return process_output @@ -1557,29 +1925,30 @@ def update_settings_file(self, settings_file, settings): paths=[settings_file]) raise ExtensionError(u"Failed to update settings file", e) - def update_settings(self): - if self.ext_handler.properties.extensions is None or \ - len(self.ext_handler.properties.extensions) == 0: + def update_settings(self, extension): + if self.extensions is None or len(self.extensions) == 0 or extension is None: # This is the behavior of waagent 2.0.x # The new agent has to be consistent with the old one. self.logger.info("Extension has no settings, write empty 0.settings") - self.update_settings_file("0.settings", "") + self.update_settings_file("{0}.settings".format(_DEFAULT_SEQ_NO), "") return - for ext in self.ext_handler.properties.extensions: - settings = { - 'publicSettings': ext.publicSettings, - 'protectedSettings': ext.protectedSettings, - 'protectedSettingsCertThumbprint': ext.certificateThumbprint - } - ext_settings = { - "runtimeSettings": [{ - "handlerSettings": settings - }] - } - settings_file = "{0}.settings".format(ext.sequenceNumber) - self.logger.info("Update settings file: {0}", settings_file) - self.update_settings_file(settings_file, json.dumps(ext_settings)) + settings = { + 'publicSettings': extension.publicSettings, + 'protectedSettings': extension.protectedSettings, + 'protectedSettingsCertThumbprint': extension.certificateThumbprint + } + ext_settings = { + "runtimeSettings": [{ + "handlerSettings": settings + }] + } + # MultiConfig: change the name to ..settings for MC and .settings for SC + settings_file = "{0}.{1}.settings".format(extension.name, extension.sequenceNumber) if \ + self.should_perform_multi_config_op(extension) else "{0}.settings".format(extension.sequenceNumber) + + self.logger.info("Update settings file: {0}", settings_file) + self.update_settings_file(settings_file, json.dumps(ext_settings)) def create_handler_env(self): handler_env = { @@ -1604,28 +1973,62 @@ def create_handler_env(self): paths=[self.get_base_dir(), self.pkg_file]) raise ExtensionDownloadError(u"Failed to save handler environment", e) + def __get_handler_state_file_name(self, extension=None): + if self.should_perform_multi_config_op(extension): + return "{0}.HandlerState".format(extension.name) + return "HandlerState" + def set_handler_state(self, handler_state): + self.__set_state(name=self.__get_handler_state_file_name(), value=handler_state) + + def get_handler_state(self): + return self.__get_state(name=self.__get_handler_state_file_name(), default=ExtHandlerState.NotInstalled) + + def __set_extension_state(self, extension, extension_state): + self.__set_state(name=self.__get_handler_state_file_name(extension), value=extension_state) + + def get_extension_state(self, extension=None): + return self.__get_state(name=self.__get_handler_state_file_name(extension), default=ExtensionState.Disabled) + + def __set_state(self, name, value): state_dir = self.get_conf_dir() - state_file = os.path.join(state_dir, "HandlerState") + state_file = os.path.join(state_dir, name) try: if not os.path.exists(state_dir): fileutil.mkdir(state_dir, mode=0o700) - fileutil.write_file(state_file, handler_state) + fileutil.write_file(state_file, value) except IOError as e: fileutil.clean_ioerror(e, paths=[state_file]) self.logger.error("Failed to set state: {0}", e) - def get_handler_state(self): + def __get_state(self, name, default=None): state_dir = self.get_conf_dir() - state_file = os.path.join(state_dir, "HandlerState") + state_file = os.path.join(state_dir, name) if not os.path.isfile(state_file): - return ExtHandlerState.NotInstalled + return default try: return fileutil.read_file(state_file) except IOError as e: self.logger.error("Failed to get state: {0}", e) - return ExtHandlerState.NotInstalled + return default + + def __remove_extension_state_files(self, extension): + try: + # MultiConfig: Remove all config/.*.settings, status/.*.status and config/.HandlerState files + files_to_delete = [ + os.path.join(self.get_conf_dir(), "{0}.*.settings".format(extension.name)), + os.path.join(self.get_status_dir(), "{0}.*.status".format(extension.name)), + os.path.join(self.get_conf_dir(), self.__get_handler_state_file_name(extension)) + ] + + fileutil.rm_files(*files_to_delete) + + except Exception as error: + extension_name = self.get_extension_full_name(extension) + message = "Failed to remove extension state files for {0}: {1}".format(extension_name, ustr(error)) + self.report_event(name=extension_name, message=message, is_success=False, log_event=False) + self.logger.warn(message) def set_handler_status(self, status="NotReady", message="", code=0): state_dir = self.get_conf_dir() @@ -1636,6 +2039,7 @@ def set_handler_status(self, status="NotReady", message="", code=0): handler_status.message = message handler_status.code = code handler_status.status = status + handler_status.supports_multi_config = self.ext_handler.supports_multi_config status_file = os.path.join(state_dir, "HandlerStatus") try: @@ -1646,8 +2050,7 @@ def set_handler_status(self, status="NotReady", message="", code=0): fileutil.write_file(status_file, handler_status_json) else: self.logger.error("Failed to create JSON document of handler status for {0} version {1}".format( - self.ext_handler.name, - self.ext_handler.properties.version)) + self.ext_handler.name, self.ext_handler.properties.version)) except (IOError, ValueError, ProtocolError) as error: fileutil.clean_ioerror(error, paths=[status_file]) self.logger.error("Failed to save handler status: {0}, {1}", ustr(error), traceback.format_exc()) @@ -1686,9 +2089,12 @@ def get_extension_package_zipfile_name(self): self.ext_handler.properties.version, HANDLER_PKG_EXT) - def get_full_name(self): - return "{0}-{1}".format(self.ext_handler.name, - self.ext_handler.properties.version) + def get_full_name(self, extension=None): + """ + :return: - if extension is None or Handler does not support Multi Config, + else then return - .- + """ + return "{0}-{1}".format(self.get_extension_full_name(extension), self.ext_handler.properties.version) def get_base_dir(self): return os.path.join(conf.get_lib_dir(), self.get_full_name()) @@ -1714,17 +2120,6 @@ def get_env_file(self): def get_log_dir(self): return os.path.join(conf.get_ext_log_dir(), self.ext_handler.name) - def get_seq_no(self): - runtime_settings = self.ext_handler.properties.extensions - # If no runtime_settings available for this ext_handler, then return 0 (this is the behavior we follow - # for update_settings) - if not runtime_settings or len(runtime_settings) == 0: - return "0" - # Currently for every runtime settings we use the same sequence number - # (Check : def parse_plugin_settings(self, ext_handler, plugin_settings) in wire.py) - # Will have to revisit once the feature to enable multiple runtime settings is rolled out by CRP - return self.ext_handler.properties.extensions[0].sequenceNumber - @staticmethod def _read_and_parse_json_status_file(ext_status_file): failed_to_read = False @@ -1835,6 +2230,9 @@ def is_update_with_install(self): def is_continue_on_update_failure(self): return self.data['handlerManifest'].get('continueOnUpdateFailure', False) + def supports_multiple_extensions(self): + return self.data['handlerManifest'].get('supportsMultipleExtensions', False) + class ExtensionStatusError(ExtensionError): """ diff --git a/tests/data/ext/sample_ext-1.3.0.zip b/tests/data/ext/sample_ext-1.3.0.zip index 19a68c5e533a3ab3086cf230e3d15b6423c16f1f..2b7e69417013a7cf654e7a032f9593393a912d5b 100644 GIT binary patch delta 1424 zcmZvcdpOg39LIn2n=z>na!XUWoRlKBE({TpTQ-_T+AN&Ntu%AmI6@K5Hu0npJIKg= z(V@mvLJEo6!CWTO#r-aH%=7e|e|o;p_w#(7@8|ixKd--Db&_AUD!Dj-P%Y*~1}NU@XJ=?s2%4d-j7Xt#?mh zl}@4>$=?Mr)}jx^sJ!kwotK_B47FU3N;s56GOeq%R(~V7|G{>2X1JCI{$*ozX1o?Au`4E!bw^IiOPp(CHY&#ej>J(Q$YXy=wMu)Vn`U4=sSL}o8pKkE z8D_nOm+kKCnW`}L(df90gOxJOP8OoTz!283G~^fjx|Y3mOd%-quoG|_(cEb`(4Esi zdgy4$d@7v36R)+=O}! zSDl{w@AD%}uHwx8Hs9Wy4AtUsA6a$6sk|bDzZpYgs+oM77TMQCJdwbOG!0BveE9&? zqlpv<$>g`~en-#UMHD+rk1qO4HBB&k2h5!s3_n7BXkqF0aRXKE4cVNgE8Y}8QeAb> z__B~2MSPFPsUUL@VnzmU(@f(d&hKdN89V`Ir+e%2_lD1**FNvtrs}iTX3}K$=kMdT zX?tu2m)dS(1G;_N;tFnYgLgHc(H7p9$Yw`Qg0&B-%S%gR+v}~b>n6>!7t})rS5RtM z6^oK(b`b{ZQ0eAE1yU%}mQ4BcrY~=(_4n2n(Uy!3Zl1a^<)G==JV(BmY9@kQkt9dM z1FQG?lg-9JgOW(^){i=QwJD*rbeAc);X=3X#bu`td<|(iT&{^7nlB-kf`Md6)BCp6bXgNe&+ae zQb`Scp@U&D-R3T<@pOCobH5uVwC(l0jF`q(xRru-L~pcnb?8n1DmLv?#10-iRP#r* zH**|X#zhMsS)pXhizyG7-6(FZ#jeE&SS?6b7oVX7j#hrlbZ@$^-&Ue*d*<*F6qVqc z%%Bk*oiOP(#@FdxN`m-S*uET2Knh!lli$tI?Fs-t1CEHl`lonnujiaEA%|)9Zb|$k|!Bw z*YqB+c{K-L8-bdBHtBsc6a67y#TwyN-CtT@qZ10RJ<^2~L^x8ha+7g}>|;9gTn>Kd zNhSr(5Tvc5_4OJeUQ}f(vv0Q>XM|1DVaxGzckL8^fFcJ;iR=Ol7EL+Xt<8I>Z1gG{ z2ao{N00J@Ks$RH18U+VHLkm5xVA@q5ItQ5$|SC?t@e7ZQm^i38v-gy-TQiu)C!i17cowV0$0L5g^#H-R_OL`-xN es0tN9(vhGFI0^-*yLL+aoGC(g!}#JNjQ;?xd3Oi^ delta 742 zcmaFGyPJbAz?+#xgn@y9gTbZlna|`eESenhpX{SQmPp1-R%P8?k1YE@`cuP9Mg|5h z76t}R1{sFp#N2|MRK0@A&=5`rW{|c82h?IpE4UdLSza(RFo22DptD7{6$JKu7MT)v zL1p6H)#-1a{$NtO*ahvo-blfnDxQ7zS4K|shh{RZ?E#CyH^3xes zaefPB4bbMX`r^T($YOm@D2~T@?#nI)6gvT$%H98rr=0^-SgS+w~W zndF#}5= 0 and item[separator + 1:] == "settings": - sequence = int(item[0: separator]) - if sequence > latest_seq: - latest_seq = sequence + match = re.search("((?P\\w+)\\.)*(?P\\d+)\\.settings", item_path) + if match is not None: + ext_name = match.group('ext_name') + if requested_ext_name is not None and ext_name != requested_ext_name: + continue + curr_seq_no = int(match.group("seq_no")) + curr_modified_time = os.path.getmtime(item_path) + if curr_modified_time > largest_modified_time: + latest_seq = curr_seq_no + largest_modified_time = curr_modified_time + return latest_seq -succeed_status = """ -[{ - "status": { - "status": "success" - } -}] -""" +def get_extension_state_prefix(): + requested_ext_name = None if 'ConfigExtensionName' not in os.environ else os.environ['ConfigExtensionName'] + seq = get_seq(requested_ext_name) + if seq >= 0: + if requested_ext_name is not None: + seq = "{0}.{1}".format(requested_ext_name, seq) + return seq + + return None + + +def read_settings_file(seq_prefix): + settings_file = os.path.join(os.getcwd(), "config", "{0}.settings".format(seq_prefix)) + if not os.path.exists(settings_file): + print("No settings found for {0}".format(settings_file)) + return None + + with open(settings_file, "rb") as file_: + return json.loads(file_.read().decode("utf-8")) + + +def report_status(seq_prefix, status="success", message=None): + status_path = os.path.join(os.getcwd(), "status") + if not os.path.exists(status_path): + os.makedirs(status_path) + status_file = os.path.join(status_path, "{0}.status".format(seq_prefix)) + with open(status_file, "w+") as status_: + status_to_report = { + "status": { + "status": status + } + } + if message is not None: + status_to_report['status']["formattedMessage"] = { + "lang": "en-US", + "message": message + } + status_.write(json.dumps([status_to_report])) + if __name__ == "__main__": - seq = get_seq() - if seq >= 0: - status_path = os.path.join(os.getcwd(), "status") - if not os.path.exists(status_path): - os.makedirs(status_path) - status_file = os.path.join(status_path, "{0}.status".format(seq)) - with open(status_file, "w+") as status: - status.write(succeed_status) + prefix = get_extension_state_prefix() + if prefix is None: + print("No sequence number found!") + sys.exit(-1) + + try: + settings = read_settings_file(prefix) + except Exception as error: + msg = "Error when trying to fetch settings {0}.settings: {1}".format(prefix, error) + print(msg) + report_status(prefix, status="error", message=msg) + else: + status_msg = None + if settings is not None: + print(settings) + try: + status_msg = settings['runtimeSettings'][0]['handlerSettings']['publicSettings']['message'] + except Exception: + # Settings might not contain the message. Ignore error if not found + pass + + report_status(prefix, message=status_msg) diff --git a/tests/data/wire/multi-config/ext_conf_mc_disabled_extensions.xml b/tests/data/wire/multi-config/ext_conf_mc_disabled_extensions.xml new file mode 100644 index 000000000..268927933 --- /dev/null +++ b/tests/data/wire/multi-config/ext_conf_mc_disabled_extensions.xml @@ -0,0 +1,84 @@ + + + + + Prod + + http://mock-goal-state/manifest_of_ga.xml + + + + Test + + http://mock-goal-state/manifest_of_ga.xml + + + + eastus + CRP + + + + MultipleExtensionsPerHandler + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.status?sv=2018-03-28&sr=b&sk=system-1&sig=1%2b%2f4nL3kZJyUb7EKxSVGQ%2fHLpXBZxCU8Zo4diPFPv5o%3d&se=9999-01-01T00%3a00%3a00Z&sp=w + + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"source":{"script":"Write-Host First: Hello World 1234!"},"parameters":[{"name":"extensionName","value":"firstRunCommand"}],"timeoutInSeconds":120} + } + } + ] +} + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Disabling secondExtension"} + } + } + ] +} + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling thirdExtension"} + } + } + ] +} + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling fourthExtension"} + } + } + ] +} + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling SingleConfig Extension"} + } + } + ] +} + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.vmSettings?sv=2018-03-28&sr=b&sk=system-1&sig=8YHwmibhasT0r9MZgL09QmFwL7ZV%2bg%2b49QP5Zwe4ksY%3d&se=9999-01-01T00%3a00%3a00Z&sp=r + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.vmHealth?sv=2018-03-28&sr=b&sk=system-1&sig=DQSxfPRZEoGBGIFl%2f4bFZ0LM9RNr9DbUEmmtkiQkWkE%3d&se=9999-01-01T00%3a00%3a00Z&sp=rw \ No newline at end of file diff --git a/tests/data/wire/multi-config/ext_conf_mc_update_extensions.xml b/tests/data/wire/multi-config/ext_conf_mc_update_extensions.xml new file mode 100644 index 000000000..cdbd62618 --- /dev/null +++ b/tests/data/wire/multi-config/ext_conf_mc_update_extensions.xml @@ -0,0 +1,75 @@ + + + + + Prod + + http://mock-goal-state/manifest_of_ga.xml + + + + Test + + http://mock-goal-state/manifest_of_ga.xml + + + + eastus + CRP + + + + MultipleExtensionsPerHandler + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.status?sv=2018-03-28&sr=b&sk=system-1&sig=1%2b%2f4nL3kZJyUb7EKxSVGQ%2fHLpXBZxCU8Zo4diPFPv5o%3d&se=9999-01-01T00%3a00%3a00Z&sp=w + + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Disabling firstExtension"} + } + } + ] +} + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Disabling secondExtension"} + } + } + ] +} + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling thirdExtension"} + } + } + ] +} + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling SingleConfig extension"} + } + } + ] +} + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.vmSettings?sv=2018-03-28&sr=b&sk=system-1&sig=8YHwmibhasT0r9MZgL09QmFwL7ZV%2bg%2b49QP5Zwe4ksY%3d&se=9999-01-01T00%3a00%3a00Z&sp=r + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.vmHealth?sv=2018-03-28&sr=b&sk=system-1&sig=DQSxfPRZEoGBGIFl%2f4bFZ0LM9RNr9DbUEmmtkiQkWkE%3d&se=9999-01-01T00%3a00%3a00Z&sp=rw \ No newline at end of file diff --git a/tests/data/wire/multi-config/ext_conf_multi_config_no_dependencies.xml b/tests/data/wire/multi-config/ext_conf_multi_config_no_dependencies.xml new file mode 100644 index 000000000..52868d044 --- /dev/null +++ b/tests/data/wire/multi-config/ext_conf_multi_config_no_dependencies.xml @@ -0,0 +1,75 @@ + + + + + Prod + + http://mock-goal-state/manifest_of_ga.xml + + + + Test + + http://mock-goal-state/manifest_of_ga.xml + + + + eastus + CRP + + + + MultipleExtensionsPerHandler + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.status?sv=2018-03-28&sr=b&sk=system-1&sig=1%2b%2f4nL3kZJyUb7EKxSVGQ%2fHLpXBZxCU8Zo4diPFPv5o%3d&se=9999-01-01T00%3a00%3a00Z&sp=w + + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling firstExtension"} + } + } + ] +} + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling secondExtension"} + } + } + ] +} + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling thirdExtension"} + } + } + ] +} + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling SingleConfig extension"} + } + } + ] +} + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.vmSettings?sv=2018-03-28&sr=b&sk=system-1&sig=8YHwmibhasT0r9MZgL09QmFwL7ZV%2bg%2b49QP5Zwe4ksY%3d&se=9999-01-01T00%3a00%3a00Z&sp=r + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.vmHealth?sv=2018-03-28&sr=b&sk=system-1&sig=DQSxfPRZEoGBGIFl%2f4bFZ0LM9RNr9DbUEmmtkiQkWkE%3d&se=9999-01-01T00%3a00%3a00Z&sp=rw \ No newline at end of file diff --git a/tests/data/wire/multi-config/ext_conf_with_disabled_multi_config.xml b/tests/data/wire/multi-config/ext_conf_with_disabled_multi_config.xml index 0f29e1ce2..4b4de2854 100644 --- a/tests/data/wire/multi-config/ext_conf_with_disabled_multi_config.xml +++ b/tests/data/wire/multi-config/ext_conf_with_disabled_multi_config.xml @@ -24,14 +24,14 @@ https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.status?sv=2018-03-28&sr=b&sk=system-1&sig=1%2b%2f4nL3kZJyUb7EKxSVGQ%2fHLpXBZxCU8Zo4diPFPv5o%3d&se=9999-01-01T00%3a00%3a00Z&sp=w - + - + diff --git a/tests/data/wire/multi-config/ext_conf_with_multi_config.xml b/tests/data/wire/multi-config/ext_conf_with_multi_config.xml index 2a714c241..f9863c876 100644 --- a/tests/data/wire/multi-config/ext_conf_with_multi_config.xml +++ b/tests/data/wire/multi-config/ext_conf_with_multi_config.xml @@ -24,14 +24,14 @@ https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.status?sv=2018-03-28&sr=b&sk=system-1&sig=1%2b%2f4nL3kZJyUb7EKxSVGQ%2fHLpXBZxCU8Zo4diPFPv5o%3d&se=9999-01-01T00%3a00%3a00Z&sp=w - + - + diff --git a/tests/data/wire/multi-config/ext_conf_with_multi_config_dependencies.xml b/tests/data/wire/multi-config/ext_conf_with_multi_config_dependencies.xml new file mode 100644 index 000000000..a7e9e83a1 --- /dev/null +++ b/tests/data/wire/multi-config/ext_conf_with_multi_config_dependencies.xml @@ -0,0 +1,99 @@ + + + + + Prod + + http://mock-goal-state/manifest_of_ga.xml + + + + Test + + http://mock-goal-state/manifest_of_ga.xml + + + + eastus + CRP + + + + MultipleExtensionsPerHandler + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.status?sv=2018-03-28&sr=b&sk=system-1&sig=1%2b%2f4nL3kZJyUb7EKxSVGQ%2fHLpXBZxCU8Zo4diPFPv5o%3d&se=9999-01-01T00%3a00%3a00Z&sp=w + + + + + + + + + + + + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling firstExtension"} + } + } + ] +} + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling secondExtension"} + } + } + ] +} + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling thirdExtension"} + } + } + ] +} + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling dependent SingleConfig extension"} + } + } + ] +} + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling independent SingleConfig extension"} + } + } + ] +} + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.vmSettings?sv=2018-03-28&sr=b&sk=system-1&sig=8YHwmibhasT0r9MZgL09QmFwL7ZV%2bg%2b49QP5Zwe4ksY%3d&se=9999-01-01T00%3a00%3a00Z&sp=r + + +https://test.blob.core.windows.net/$system/lrwinmcdn_0.0f3bfecf-f14f-4c7d-8275-9dee7310fe8c.vmHealth?sv=2018-03-28&sr=b&sk=system-1&sig=DQSxfPRZEoGBGIFl%2f4bFZ0LM9RNr9DbUEmmtkiQkWkE%3d&se=9999-01-01T00%3a00%3a00Z&sp=rw \ No newline at end of file diff --git a/tests/ga/extension_emulator.py b/tests/ga/extension_emulator.py index ac7fbb421..aa496a54f 100644 --- a/tests/ga/extension_emulator.py +++ b/tests/ga/extension_emulator.py @@ -25,7 +25,7 @@ import azurelinuxagent.common.conf as conf from azurelinuxagent.common.future import ustr from azurelinuxagent.common.utils import fileutil -from azurelinuxagent.ga.exthandlers import ExtHandlerInstance +from azurelinuxagent.ga.exthandlers import ExtHandlerInstance, ExtCommandEnvVariable from tests.tools import Mock, patch from tests.protocol.mocks import HttpRequestPredicates @@ -67,18 +67,21 @@ def fail_action(*_, **__): def extension_emulator(name="OSTCExtensions.ExampleHandlerLinux", version="1.0.0", - update_mode="UpdateWithInstall", report_heartbeat=False, continue_on_update_failure=False, - install_action=Actions.succeed_action, uninstall_action=Actions.succeed_action, - enable_action=Actions.succeed_action, disable_action=Actions.succeed_action, - update_action=Actions.succeed_action): + update_mode="UpdateWithInstall", report_heartbeat=False, continue_on_update_failure=False, + supports_multiple_extensions=False, install_action=Actions.succeed_action, + uninstall_action=Actions.succeed_action, + enable_action=Actions.succeed_action, disable_action=Actions.succeed_action, + update_action=Actions.succeed_action): """ Factory method for ExtensionEmulator objects with sensible defaults. """ # Linter reports too many arguments, but this isn't an issue because all are defaulted; # no caller will have to actually provide all of the arguments listed. - + return ExtensionEmulator(name, version, update_mode, report_heartbeat, continue_on_update_failure, - install_action, uninstall_action, enable_action, disable_action, update_action) + supports_multiple_extensions, install_action, uninstall_action, enable_action, + disable_action, update_action) + @contextlib.contextmanager def enable_invocations(*emulators): @@ -98,6 +101,7 @@ def enable_invocations(*emulators): with patch("subprocess.Popen", patched_popen): yield invocation_record + def generate_put_handler(*emulators): """ Create a HTTP handler to store status blobs for each provided emulator. @@ -125,6 +129,7 @@ def mock_put_handler(url, *args, **_): return mock_put_handler + class InvocationRecord: def __init__(self): @@ -152,13 +157,14 @@ def compare(self, *expected_cmds): except IndexError: raise Exception("No more invocations recorded. Expected ({0}, {1}, {2}).".format(expected_ext_emulator.name, - expected_ext_emulator.version, command_name)) + expected_ext_emulator.version, command_name)) if self._queue: raise Exception("Invocation recorded, but not expected: ({0}, {1}, {2})".format( *self._queue[0] )) + def _first_matching_emulator(emulators, name, version): for ext in emulators: if ext.matches(name, version): @@ -166,17 +172,19 @@ def _first_matching_emulator(emulators, name, version): raise StopIteration + class ExtensionEmulator: """ A wrapper class for the possible actions and options that an extension might support. """ def __init__(self, name, version, - update_mode, report_heartbeat, - continue_on_update_failure, - install_action, uninstall_action, - enable_action, disable_action, - update_action): + update_mode, report_heartbeat, + continue_on_update_failure, + supports_multiple_extensions, + install_action, uninstall_action, + enable_action, disable_action, + update_action): # Linter reports too many arguments, but this constructor has its access mediated by # a factory method; the calls affected by the number of arguments here is very # limited in scope. @@ -187,6 +195,7 @@ def __init__(self, name, version, self.update_mode = update_mode self.report_heartbeat = report_heartbeat self.continue_on_update_failure = continue_on_update_failure + self.supports_multiple_extensions = supports_multiple_extensions self._actions = { ExtensionCommandNames.INSTALL: ExtensionEmulator._extend_func(install_action), @@ -225,30 +234,23 @@ def _extend_func(func): def wrapped_func(cmd, *args, **kwargs): return_value = func(cmd, *args, **kwargs) - config_dir = os.path.join(os.path.dirname(cmd), "config") + prefix = kwargs['env'][ExtCommandEnvVariable.ExtensionSeqNumber] + if ExtCommandEnvVariable.ExtensionName in kwargs['env']: + prefix = "{0}.{1}".format(kwargs['env'][ExtCommandEnvVariable.ExtensionName], prefix) - regex = r'{directory}{sep}(?P{sequence})\.settings'.format( - directory=config_dir, sep=os.path.sep, sequence=r'[0-9]+' - ) - - seq = 0 - for config_file in map(lambda filename: os.path.join(config_dir, filename), os.listdir(config_dir)): - if not os.path.isfile(config_file): - continue - - match = re.match(regex, config_file) - if not match: - continue - - if seq < int(match.group("seq")): - seq = int(match.group("seq")) - - status_file = os.path.join(os.path.dirname(cmd), "status", "{seq}.status".format(seq=seq)) + status_file = os.path.join(os.path.dirname(cmd), "status", "{seq}.status".format(seq=prefix)) if return_value == 0: status_contents = [{ "status": {"status": "success"} }] else: - status_contents = [{ "status": {"status": "error", "substatus": {"exit_code": return_value}} }] + try: + ec = int(return_value) + except Exception: + # Error when trying to parse return_value, probably not an integer. + # Failing with -1 and passing the return_value as message + ec = -1 + status_contents = [{"status": {"status": "error", "code": ec, + "formattedMessage": {"message": return_value, "lang": "en-US"}}}] fileutil.write_file(status_file, json.dumps(status_contents)) @@ -259,11 +261,11 @@ def wrapped_func(cmd, *args, **kwargs): # Wrap the function in a mock to allow invocation reflection a la .assert_not_called(), etc. return Mock(wraps=wrapped_func) - - + def matches(self, name, version): return self.name == name and self.version == version + def generate_patched_popen(invocation_record, *emulators): """ Create a mock popen function able to invoke the proper action for an extension @@ -274,54 +276,73 @@ def generate_patched_popen(invocation_record, *emulators): def patched_popen(cmd, *args, **kwargs): try: - ext_name, ext_version, command_name = _extract_extension_info_from_command(cmd) - invocation_record.add(ext_name, ext_version, command_name) + handler_name, handler_version, command_name = extract_extension_info_from_command(cmd) except ValueError: return original_popen(cmd, *args, **kwargs) try: - matching_ext = _first_matching_emulator(emulators, ext_name, ext_version) + name = handler_name + # MultiConfig scenario, search for full name - . + if ExtCommandEnvVariable.ExtensionName in kwargs['env']: + name = "{0}.{1}".format(handler_name, kwargs['env'][ExtCommandEnvVariable.ExtensionName]) + + invocation_record.add(name, handler_version, command_name) + matching_ext = _first_matching_emulator(emulators, name, handler_version) return matching_ext.actions[command_name](cmd, *args, **kwargs) except StopIteration: raise Exception("Extension('{name}', '{version}') not listed as a parameter. Is it being emulated?".format( - name=ext_name, version=ext_version + name=handler_name, version=handler_version )) return patched_popen + def generate_mock_load_manifest(*emulators): original_load_manifest = ExtHandlerInstance.load_manifest def mock_load_manifest(self): + matching_emulator = None + names = [self.ext_handler.name] + # Incase of MC, search for full names - . + if self.supports_multi_config: + names = [self.get_extension_full_name(ext) for ext in self.extensions] - try: - matching_emulator = _first_matching_emulator(emulators, self.ext_handler.name, self.ext_handler.properties.version) - except StopIteration: - raise Exception("Extension('{name}', '{version}') not listed as a parameter. Is it being emulated?".format( - name=self.ext_handler.name, version=self.ext_handler.properties.version - )) + for name in names: + try: + matching_emulator = _first_matching_emulator(emulators, name, self.ext_handler.properties.version) + except StopIteration: + continue + else: + break + + if matching_emulator is None: + raise Exception( + "Extension('{name}', '{version}') not listed as a parameter. Is it being emulated?".format( + name=self.ext_handler.name, version=self.ext_handler.properties.version)) base_manifest = original_load_manifest(self) base_manifest.data["handlerManifest"].update({ "continueOnUpdateFailure": matching_emulator.continue_on_update_failure, "reportHeartbeat": matching_emulator.report_heartbeat, - "updateMode": matching_emulator.update_mode + "updateMode": matching_emulator.update_mode, + "supportsMultipleExtensions": matching_emulator.supports_multiple_extensions }) return base_manifest return mock_load_manifest -def _extract_extension_info_from_command(command): + +def extract_extension_info_from_command(command): """ Parse a command into a tuple of extension info. """ if not isinstance(command, (str, ustr)): - raise Exception("Cannot extract extension info from non-string commands") + raise ValueError("Cannot extract extension info from non-string commands") # Group layout of the expected command; this lets us grab what we want after a match template = r'(?<={base_dir}/)(?P{ext_name})-(?P{ext_ver})(?:/{script_file} -)(?P{ext_cmd})' @@ -329,8 +350,8 @@ def _extract_extension_info_from_command(command): base_dir_regex = conf.get_lib_dir() script_file_regex = r'[^\s]+' ext_cmd_regex = r'[a-zA-Z]+' - ext_name_regex = r'[a-zA-Z]+(?:\.[a-zA-Z]+)?' - ext_ver_regex = r'[0-9]+(?:\.[0-9]+)*' + ext_name_regex = r'[a-zA-Z]+(?:[.a-zA-Z]+)?' + ext_ver_regex = r'[0-9]+(?:[.0-9]+)*' full_regex = template.format( ext_name=ext_name_regex, diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index a909fe8bb..037a21171 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -27,8 +27,6 @@ import unittest import uuid -import datetime - from azurelinuxagent.common import conf from azurelinuxagent.common.cgroupconfigurator import CGroupConfigurator from azurelinuxagent.common.datacontract import get_properties @@ -41,13 +39,13 @@ from azurelinuxagent.common.exception import ResourceGoneError, ExtensionDownloadError, ProtocolError, \ ExtensionErrorCodes, ExtensionError, GoalStateAggregateStatusCodes from azurelinuxagent.common.protocol.restapi import Extension, ExtHandler, ExtHandlerStatus, \ - ExtensionStatus + ExtensionStatus, ExtHandlerRequestedState from azurelinuxagent.common.protocol.wire import WireProtocol, InVMArtifactsProfile from azurelinuxagent.common.utils.restutil import KNOWN_WIRESERVER_IP -from azurelinuxagent.ga.exthandlers import ExtHandlersHandler, ExtHandlerInstance, migrate_handler_state, \ +from azurelinuxagent.ga.exthandlers import ExtHandlerInstance, migrate_handler_state, \ get_exthandlers_handler, AGENT_STATUS_FILE, ExtCommandEnvVariable, HandlerManifest, NOT_RUN, \ - ValidHandlerStatus, HANDLER_COMPLETE_NAME_PATTERN, HandlerEnvironment, ExtensionRequestedState, GoalStateStatus + ValidHandlerStatus, HANDLER_COMPLETE_NAME_PATTERN, HandlerEnvironment, GoalStateStatus from tests.protocol import mockwiredata from tests.protocol.mocks import mock_wire_protocol, HttpRequestPredicates, MockHttpResponse @@ -153,7 +151,7 @@ def test_cleanup_removes_uninstalled_extensions(self): # Update incarnation and extension config protocol.mock_wire_data.set_incarnation(2) - protocol.mock_wire_data.set_extensions_config_state(ExtensionRequestedState.Uninstall) + protocol.mock_wire_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) protocol.client.update_goal_state() exthandlers_handler.run() @@ -200,7 +198,7 @@ def mock_fail_popen(*args, **kwargs): # pylint: disable=unused-argument # Update incarnation and extension config to uninstall the extension, this should delete the extension protocol.mock_wire_data.set_incarnation(2) - protocol.mock_wire_data.set_extensions_config_state(ExtensionRequestedState.Uninstall) + protocol.mock_wire_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) protocol.client.update_goal_state() exthandlers_handler.run() @@ -432,18 +430,20 @@ def setUp(self): def _assert_handler_status(self, report_vm_status, expected_status, expected_ext_count, version, - expected_handler_name="OSTCExtensions.ExampleHandlerLinux"): + expected_handler_name="OSTCExtensions.ExampleHandlerLinux", expected_msg=None): self.assertTrue(report_vm_status.called) args, kw = report_vm_status.call_args # pylint: disable=unused-variable vm_status = args[0] self.assertNotEqual(0, len(vm_status.vmAgent.extensionHandlers)) - handler_status = vm_status.vmAgent.extensionHandlers[0] + handler_status = next(status for status in vm_status.vmAgent.extensionHandlers if status.name == expected_handler_name) self.assertEqual(expected_status, handler_status.status) - self.assertEqual(expected_handler_name, - handler_status.name) + self.assertEqual(expected_handler_name, handler_status.name) self.assertEqual(version, handler_status.version) - self.assertEqual(expected_ext_count, len(handler_status.extensions)) - return + self.assertEqual(expected_ext_count, len([ext_handler for ext_handler in vm_status.vmAgent.extensionHandlers if + ext_handler.name == expected_handler_name and ext_handler.extension_status is not None])) + + if expected_msg is not None: + self.assertIn(expected_msg, handler_status.message) def _assert_ext_pkg_file_status(self, expected_to_be_present=True, extension_version="1.0.0", extension_handler_name="OSTCExtensions.ExampleHandlerLinux"): @@ -460,14 +460,14 @@ def _assert_no_handler_status(self, report_vm_status): self.assertEqual(0, len(vm_status.vmAgent.extensionHandlers)) return - def _create_mock(self, test_data, mock_http_get, MockCryptUtil, *args): # pylint: disable=unused-argument + @staticmethod + def _create_mock(test_data, mock_http_get, mock_crypt_util, *_): # Mock protocol to return test data mock_http_get.side_effect = test_data.mock_http_get - MockCryptUtil.side_effect = test_data.mock_crypt_util + mock_crypt_util.side_effect = test_data.mock_crypt_util protocol = WireProtocol(KNOWN_WIRESERVER_IP) protocol.detect() - protocol.report_ext_status = MagicMock() protocol.report_vm_status = MagicMock() handler = get_exthandlers_handler(protocol) @@ -490,7 +490,7 @@ def _set_up_update_test_and_update_gs(self, patch_command, *args): self.assertEqual(0, patch_command.call_count) self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Next incarnation, update version test_data.set_incarnation(2) @@ -516,7 +516,7 @@ def test_ext_handler(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Test goal state not changed exthandlers_handler.run() @@ -531,7 +531,7 @@ def test_ext_handler(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 1) + self._assert_ext_status(protocol.report_vm_status, "success", 1) # Test hotfix test_data.set_incarnation(3) @@ -542,7 +542,7 @@ def test_ext_handler(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.1") - self._assert_ext_status(protocol.report_ext_status, "success", 2) + self._assert_ext_status(protocol.report_vm_status, "success", 2) # Test upgrade test_data.set_incarnation(4) @@ -553,11 +553,11 @@ def test_ext_handler(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.2.0") - self._assert_ext_status(protocol.report_ext_status, "success", 3) + self._assert_ext_status(protocol.report_vm_status, "success", 3) # Test disable test_data.set_incarnation(5) - test_data.set_extensions_config_state(ExtensionRequestedState.Disabled) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Disabled) protocol.update_goal_state() exthandlers_handler.run() @@ -566,7 +566,7 @@ def test_ext_handler(self, *args): # Test uninstall test_data.set_incarnation(6) - test_data.set_extensions_config_state(ExtensionRequestedState.Uninstall) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) protocol.update_goal_state() exthandlers_handler.run() @@ -584,7 +584,7 @@ def test_it_should_only_download_extension_manifest_once_per_goal_state(self, *a def _assert_handler_status_and_manifest_download_count(protocol, test_data, manifest_count): self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) self.assertEqual(test_data.call_counts['manifest.xml'], manifest_count, "We should have downloaded extension manifest {0} times".format(manifest_count)) @@ -621,13 +621,14 @@ def test_it_should_fail_handler_on_bad_extension_config_and_report_error(self, m test_data = mockwiredata.WireProtocolData(bad_conf) exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt_util, *args) - with patch('azurelinuxagent.common.event.add_event') as patch_add_event: + with patch('azurelinuxagent.ga.exthandlers.add_event') as patch_add_event: exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "NotReady", 0, "1.0.0") invalid_config_errors = [kw for _, kw in patch_add_event.call_args_list if kw['op'] == WALAEventOperation.InvalidExtensionConfig] - self.assertEqual(1, len(invalid_config_errors), "Error not logged and reported to Kusto") + self.assertEqual(1, len(invalid_config_errors), + "Error not logged and reported to Kusto for {0}".format(bad_config_file_path)) def test_it_should_process_valid_extensions_if_present(self, mock_get, mock_crypt_util, *args): @@ -648,7 +649,9 @@ def test_it_should_process_valid_extensions_if_present(self, mock_get, mock_cryp self.assertEqual(expected_status, handler.status, "Invalid status") self.assertIn(handler.name, expected_handlers, "Handler not found") self.assertEqual("1.0.0", handler.version, "Incorrect handler version") - self.assertEqual(expected_ext_count, len(handler.extensions), "Incorrect extensions enabled") + self.assertEqual(expected_ext_count, len([ext for ext in vm_status.vmAgent.extensionHandlers if + ext.name == handler.name and ext.extension_status is not None]), + "Incorrect extensions enabled") expected_handlers.remove(handler.name) self.assertEqual(0, len(expected_handlers), "All handlers not reported status") @@ -660,7 +663,7 @@ def test_ext_zip_file_packages_removed_in_update_case(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) self._assert_ext_pkg_file_status(expected_to_be_present=True, extension_version="1.0.0") # Update the package @@ -672,7 +675,7 @@ def test_ext_zip_file_packages_removed_in_update_case(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.0") - self._assert_ext_status(protocol.report_ext_status, "success", 1) + self._assert_ext_status(protocol.report_vm_status, "success", 1) self._assert_ext_pkg_file_status(expected_to_be_present=False, extension_version="1.0.0") self._assert_ext_pkg_file_status(expected_to_be_present=True, extension_version="1.1.0") @@ -685,7 +688,7 @@ def test_ext_zip_file_packages_removed_in_update_case(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.2.0") - self._assert_ext_status(protocol.report_ext_status, "success", 2) + self._assert_ext_status(protocol.report_vm_status, "success", 2) self._assert_ext_pkg_file_status(expected_to_be_present=False, extension_version="1.1.0") self._assert_ext_pkg_file_status(expected_to_be_present=True, extension_version="1.2.0") @@ -698,12 +701,12 @@ def test_ext_zip_file_packages_removed_in_uninstall_case(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, extension_version) - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) self._assert_ext_pkg_file_status(expected_to_be_present=True, extension_version=extension_version) # Test uninstall test_data.set_incarnation(2) - test_data.set_extensions_config_state(ExtensionRequestedState.Uninstall) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) protocol.update_goal_state() exthandlers_handler.run() @@ -719,7 +722,7 @@ def test_ext_zip_file_packages_removed_in_update_and_uninstall_case(self, *args) exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) self._assert_ext_pkg_file_status(expected_to_be_present=True, extension_version="1.0.0") # Update the package @@ -731,7 +734,7 @@ def test_ext_zip_file_packages_removed_in_update_and_uninstall_case(self, *args) exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.0") - self._assert_ext_status(protocol.report_ext_status, "success", 1) + self._assert_ext_status(protocol.report_vm_status, "success", 1) self._assert_ext_pkg_file_status(expected_to_be_present=False, extension_version="1.0.0") self._assert_ext_pkg_file_status(expected_to_be_present=True, extension_version="1.1.0") @@ -744,13 +747,13 @@ def test_ext_zip_file_packages_removed_in_update_and_uninstall_case(self, *args) exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.2.0") - self._assert_ext_status(protocol.report_ext_status, "success", 2) + self._assert_ext_status(protocol.report_vm_status, "success", 2) self._assert_ext_pkg_file_status(expected_to_be_present=False, extension_version="1.1.0") self._assert_ext_pkg_file_status(expected_to_be_present=True, extension_version="1.2.0") # Test uninstall test_data.set_incarnation(4) - test_data.set_extensions_config_state(ExtensionRequestedState.Uninstall) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) protocol.update_goal_state() exthandlers_handler.run() @@ -779,7 +782,9 @@ def test_it_should_ignore_case_when_parsing_plugin_settings(self, mock_get, mock self.assertEqual("Ready", handler_status.status, "Handler is not Ready") self.assertIn(handler_status.name, expected_ext_handlers, "Handler not reported") self.assertEqual("1.0.0", handler_status.version, "Handler version not matching") - self.assertEqual(1, len(handler_status.extensions), "No settings were found for this extension") + self.assertEqual(1, len( + [status for status in vm_status.vmAgent.extensionHandlers if status.name == handler_status.name]), + "No settings were found for this extension") expected_ext_handlers.remove(handler_status.name) self.assertEqual(0, len(expected_ext_handlers), "All handlers not reported") @@ -788,8 +793,29 @@ def test_ext_handler_no_settings(self, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_NO_SETTINGS) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter - exthandlers_handler.run() - self._assert_handler_status(protocol.report_vm_status, "Ready", 0, "1.0.0") + test_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux") + with enable_invocations(test_ext) as invocation_record: + exthandlers_handler.run() + self._assert_handler_status(protocol.report_vm_status, "Ready", 0, "1.0.0") + invocation_record.compare( + (test_ext, ExtensionCommandNames.INSTALL), + (test_ext, ExtensionCommandNames.ENABLE) + ) + + # Uninstall the Plugin and make sure Disable called + test_data.set_incarnation(2) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) + protocol.update_goal_state() + + with enable_invocations(test_ext) as invocation_record: + exthandlers_handler.run() + self.assertTrue(protocol.report_vm_status.called) + args, _ = protocol.report_vm_status.call_args + self.assertEqual(0, len(args[0].vmAgent.extensionHandlers)) + invocation_record.compare( + (test_ext, ExtensionCommandNames.DISABLE), + (test_ext, ExtensionCommandNames.UNINSTALL) + ) def test_ext_handler_no_public_settings(self, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_NO_PUBLIC) @@ -811,18 +837,33 @@ def test_ext_handler_sequencing(self, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SEQUENCING) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter - exthandlers_handler.run() + dep_ext_level_2 = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux") + dep_ext_level_1 = extension_emulator(name="OSTCExtensions.OtherExampleHandlerLinux") - self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + with enable_invocations(dep_ext_level_2, dep_ext_level_1) as invocation_record: + exthandlers_handler.run() + + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") + self._assert_ext_status(protocol.report_vm_status, "success", 0, expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") - self._assert_ext_status(protocol.report_ext_status, "success", 0) - # check handler list - self.assertTrue(exthandlers_handler.ext_handlers is not None) - self.assertTrue(exthandlers_handler.ext_handlers.extHandlers is not None) - self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2) - self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 1) - self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 2) + # check handler list and dependency levels + self.assertTrue(exthandlers_handler.ext_handlers is not None) + self.assertTrue(exthandlers_handler.ext_handlers.extHandlers is not None) + self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2) + self.assertEqual(1, next(handler for handler in exthandlers_handler.ext_handlers.extHandlers if + handler.name == dep_ext_level_1.name).properties.extensions[0].dependencyLevel) + self.assertEqual(2, next(handler for handler in exthandlers_handler.ext_handlers.extHandlers if + handler.name == dep_ext_level_2.name).properties.extensions[0].dependencyLevel) + + # Ensure the invocation order follows the dependency levels + invocation_record.compare( + (dep_ext_level_1, ExtensionCommandNames.INSTALL), + (dep_ext_level_1, ExtensionCommandNames.ENABLE), + (dep_ext_level_2, ExtensionCommandNames.INSTALL), + (dep_ext_level_2, ExtensionCommandNames.ENABLE) + ) # Test goal state not changed exthandlers_handler.run() @@ -833,53 +874,84 @@ def test_ext_handler_sequencing(self, *args): test_data.set_incarnation(2) test_data.set_extensions_config_sequence_number(1) # Swap the dependency ordering + dep_ext_level_3 = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux") + dep_ext_level_4 = extension_emulator(name="OSTCExtensions.OtherExampleHandlerLinux") test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"2\"", "dependencyLevel=\"3\"") test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"1\"", "dependencyLevel=\"4\"") protocol.update_goal_state() - exthandlers_handler.run() + with enable_invocations(dep_ext_level_3, dep_ext_level_4) as invocation_record: + exthandlers_handler.run() - self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 1) + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") + self._assert_ext_status(protocol.report_vm_status, "success", 1) - self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2) - self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 3) - self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 4) + self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2) + self.assertEqual(3, next(handler for handler in exthandlers_handler.ext_handlers.extHandlers if + handler.name == dep_ext_level_3.name).properties.extensions[0].dependencyLevel) + self.assertEqual(4, next(handler for handler in exthandlers_handler.ext_handlers.extHandlers if + handler.name == dep_ext_level_4.name).properties.extensions[0].dependencyLevel) + + # Ensure the invocation order follows the dependency levels + invocation_record.compare( + (dep_ext_level_3, ExtensionCommandNames.ENABLE), + (dep_ext_level_4, ExtensionCommandNames.ENABLE) + ) # Test disable # In the case of disable, the last extension to be enabled should be # the first extension disabled. The first extension enabled should be # the last one disabled. test_data.set_incarnation(3) - test_data.set_extensions_config_state(ExtensionRequestedState.Disabled) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Disabled) protocol.update_goal_state() - exthandlers_handler.run() + with enable_invocations(dep_ext_level_3, dep_ext_level_4) as invocation_record: + exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "NotReady", 1, "1.0.0", expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") - self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2) - self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 4) - self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 3) + + self.assertEqual(3, next(handler for handler in exthandlers_handler.ext_handlers.extHandlers if + handler.name == dep_ext_level_3.name).properties.extensions[0].dependencyLevel) + self.assertEqual(4, next(handler for handler in exthandlers_handler.ext_handlers.extHandlers if + handler.name == dep_ext_level_4.name).properties.extensions[0].dependencyLevel) + + # Ensure the invocation order follows the dependency levels + invocation_record.compare( + (dep_ext_level_4, ExtensionCommandNames.DISABLE), + (dep_ext_level_3, ExtensionCommandNames.DISABLE) + ) # Test uninstall # In the case of uninstall, the last extension to be installed should be # the first extension uninstalled. The first extension installed # should be the last one uninstalled. test_data.set_incarnation(4) - test_data.set_extensions_config_state(ExtensionRequestedState.Uninstall) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) # Swap the dependency ordering AGAIN + dep_ext_level_5 = extension_emulator(name="OSTCExtensions.OtherExampleHandlerLinux") + dep_ext_level_6 = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux") test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"3\"", "dependencyLevel=\"6\"") test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"4\"", "dependencyLevel=\"5\"") protocol.update_goal_state() - exthandlers_handler.run() + with enable_invocations(dep_ext_level_5, dep_ext_level_6) as invocation_record: + exthandlers_handler.run() - self._assert_no_handler_status(protocol.report_vm_status) - self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2) - self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 6) - self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 5) + self._assert_no_handler_status(protocol.report_vm_status) + self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2) + self.assertEqual(5, next(handler for handler in exthandlers_handler.ext_handlers.extHandlers if + handler.name == dep_ext_level_5.name).properties.extensions[0].dependencyLevel) + self.assertEqual(6, next(handler for handler in exthandlers_handler.ext_handlers.extHandlers if + handler.name == dep_ext_level_6.name).properties.extensions[0].dependencyLevel) + + # Ensure the invocation order follows the dependency levels + invocation_record.compare( + (dep_ext_level_6, ExtensionCommandNames.UNINSTALL), + (dep_ext_level_5, ExtensionCommandNames.UNINSTALL) + ) def test_ext_handler_sequencing_should_fail_if_handler_failed(self, mock_get, mock_crypt, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SEQUENCING) @@ -887,9 +959,9 @@ def test_ext_handler_sequencing_should_fail_if_handler_failed(self, mock_get, mo original_popen = subprocess.Popen - def _assert_event_reported_only_on_incarnation_change(patch_add_event, expected_count=1): + def _assert_event_reported_only_on_incarnation_change(expected_count=1): handler_seq_reporting = [kwargs for _, kwargs in patch_add_event.call_args_list if kwargs[ - 'op'] == WALAEventOperation.ExtensionProcessing and "will skip processing the rest of the extensions" in + 'op'] == WALAEventOperation.ExtensionProcessing and "Skipping processing of extensions since execution of dependent extension" in kwargs['message']] self.assertEqual(len(handler_seq_reporting), expected_count, "Error should be reported only on incarnation change") @@ -899,7 +971,6 @@ def mock_fail_extension_commands(args, **kwargs): return original_popen("fail_this_command", **kwargs) return original_popen(args, **kwargs) - with patch("subprocess.Popen", mock_fail_extension_commands): with patch('azurelinuxagent.ga.exthandlers.add_event') as patch_add_event: exthandlers_handler.run() @@ -907,18 +978,18 @@ def mock_fail_extension_commands(args, **kwargs): self._assert_handler_status(protocol.report_vm_status, "NotReady", 0, "1.0.0", expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") - _assert_event_reported_only_on_incarnation_change(patch_add_event, expected_count=1) + _assert_event_reported_only_on_incarnation_change(expected_count=1) # Assert that on rerun it should not report errors unless incarnation changes for _ in range(5): exthandlers_handler.run() - _assert_event_reported_only_on_incarnation_change(patch_add_event, expected_count=1) + _assert_event_reported_only_on_incarnation_change(expected_count=1) test_data.set_incarnation(2) protocol.update_goal_state() exthandlers_handler.run() # We should report error again on incarnation change - _assert_event_reported_only_on_incarnation_change(patch_add_event, expected_count=2) + _assert_event_reported_only_on_incarnation_change(expected_count=2) # Test it recovers on a new goal state if Handler succeeds test_data.set_incarnation(3) @@ -928,14 +999,33 @@ def mock_fail_extension_commands(args, **kwargs): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") - self._assert_ext_status(protocol.report_ext_status, "success", 1) + self._assert_ext_status(protocol.report_vm_status, "success", 1, + expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") + + # Update incarnation to confirm extension invocation order + test_data.set_incarnation(4) + protocol.update_goal_state() + + dep_ext_level_2 = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux") + dep_ext_level_1 = extension_emulator(name="OSTCExtensions.OtherExampleHandlerLinux") - # check handler list and dependency levels - self.assertTrue(exthandlers_handler.ext_handlers is not None) - self.assertTrue(exthandlers_handler.ext_handlers.extHandlers is not None) - self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2) - self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 1) - self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 2) + with enable_invocations(dep_ext_level_2, dep_ext_level_1) as invocation_record: + exthandlers_handler.run() + + # check handler list and dependency levels + self.assertTrue(exthandlers_handler.ext_handlers is not None) + self.assertTrue(exthandlers_handler.ext_handlers.extHandlers is not None) + self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2) + self.assertEqual(1, next(handler for handler in exthandlers_handler.ext_handlers.extHandlers if + handler.name == dep_ext_level_1.name).properties.extensions[0].dependencyLevel) + self.assertEqual(2, next(handler for handler in exthandlers_handler.ext_handlers.extHandlers if + handler.name == dep_ext_level_2.name).properties.extensions[0].dependencyLevel) + + # Ensure the invocation order follows the dependency levels + invocation_record.compare( + (dep_ext_level_1, ExtensionCommandNames.ENABLE), + (dep_ext_level_2, ExtensionCommandNames.ENABLE) + ) def test_ext_handler_sequencing_default_dependency_level(self, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) @@ -973,27 +1063,32 @@ def test_ext_handler_reporting_status_file(self, *args): {{ "name": "OSTCExtensions.ExampleHandlerLinux", "version": "1.0.0", - "status": "Ready" + "status": "Ready", + "supports_multi_config": false }}, {{ "name": "Microsoft.Powershell.ExampleExtension", "version": "1.0.0", - "status": "Ready" + "status": "Ready", + "supports_multi_config": false }}, {{ "name": "Microsoft.EnterpriseCloud.Monitoring.ExampleHandlerLinux", "version": "1.0.0", - "status": "Ready" + "status": "Ready", + "supports_multi_config": false }}, {{ "name": "Microsoft.CPlat.Core.ExampleExtensionLinux", "version": "1.0.0", - "status": "Ready" + "status": "Ready", + "supports_multi_config": false }}, {{ "name": "Microsoft.OSTCExtensions.Edp.ExampleExtensionLinuxInTest", "version": "1.0.0", - "status": "Ready" + "status": "Ready", + "supports_multi_config": false }} ] }}'''.format(agent_name=AGENT_NAME, @@ -1022,7 +1117,7 @@ def test_ext_handler_rollingupgrade(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Test goal state changed test_data.set_incarnation(2) @@ -1031,7 +1126,7 @@ def test_ext_handler_rollingupgrade(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Test minor version bump test_data.set_incarnation(3) @@ -1041,7 +1136,7 @@ def test_ext_handler_rollingupgrade(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Test hotfix version bump test_data.set_incarnation(4) @@ -1051,11 +1146,11 @@ def test_ext_handler_rollingupgrade(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.1") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Test disable test_data.set_incarnation(5) - test_data.set_extensions_config_state(ExtensionRequestedState.Disabled) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Disabled) protocol.update_goal_state() exthandlers_handler.run() @@ -1064,7 +1159,7 @@ def test_ext_handler_rollingupgrade(self, *args): # Test uninstall test_data.set_incarnation(6) - test_data.set_extensions_config_state(ExtensionRequestedState.Uninstall) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) protocol.update_goal_state() exthandlers_handler.run() @@ -1081,13 +1176,13 @@ def test_ext_handler_rollingupgrade(self, *args): # Test re-install test_data.set_incarnation(8) - test_data.set_extensions_config_state(ExtensionRequestedState.Enabled) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Enabled) protocol.update_goal_state() exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.1") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Test version bump post-re-install test_data.set_incarnation(9) @@ -1097,7 +1192,7 @@ def test_ext_handler_rollingupgrade(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.2.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Test rollback test_data.set_incarnation(10) @@ -1107,19 +1202,7 @@ def test_ext_handler_rollingupgrade(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) - - @patch('azurelinuxagent.ga.exthandlers.add_event') - def test_ext_handler_download_failure_transient(self, mock_add_event, *args): - original_sleep = time.sleep # pylint: disable=unused-variable - - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter - protocol.download_ext_handler_pkg = Mock(side_effect=ProtocolError) - - exthandlers_handler.run() - - self.assertEqual(0, mock_add_event.call_count) + self._assert_ext_status(protocol.report_vm_status, "success", 0) def test_it_should_create_extension_events_dir_and_set_handler_environment_only_if_extension_telemetry_enabled(self, *args): @@ -1134,7 +1217,7 @@ def test_it_should_create_extension_events_dir_and_set_handler_environment_only_ exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) for ext_handler in exthandlers_handler.ext_handlers.extHandlers: ehi = ExtHandlerInstance(ext_handler, protocol) @@ -1164,13 +1247,13 @@ def test_it_should_not_delete_extension_events_directory_on_extension_uninstall( with patch("azurelinuxagent.common.agent_supported_feature._ETPFeature.is_supported", True): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) ehi = ExtHandlerInstance(exthandlers_handler.ext_handlers.extHandlers[0], protocol) self.assertTrue(os.path.exists(ehi.get_extension_events_dir()), "Events directory should exist") # Uninstall extensions now - test_data.set_extensions_config_state(ExtensionRequestedState.Uninstall) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) test_data.set_incarnation(2) protocol.update_goal_state() exthandlers_handler.run() @@ -1184,7 +1267,7 @@ def test_it_should_uninstall_unregistered_extensions_properly(self, *args): # Update version and set it to uninstall. That is how it would be propagated by CRP if a version 1.0.0 is # unregistered in PIR and a new version 1.0.1 is published. - test_data.set_extensions_config_state(ExtensionRequestedState.Uninstall) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) test_data.set_extensions_config_version("1.0.1") # Since the installed version is not in PIR anymore, we need to also remove it from manifest file test_data.manifest = test_data.manifest.replace("1.0.0", "9.9.9") @@ -1225,7 +1308,7 @@ def test_ext_handler_report_status_resource_gone(self, mock_add_event, *args): self.assertEqual("ExtensionProcessing", kw['op']) @patch('azurelinuxagent.common.errorstate.ErrorState.is_triggered') - @patch('azurelinuxagent.ga.exthandlers.ExtHandlerInstance.report_event') + @patch('azurelinuxagent.ga.exthandlers.add_event') def test_ext_handler_download_failure_permanent_ProtocolError(self, mock_add_event, mock_error_state, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter @@ -1234,14 +1317,14 @@ def test_ext_handler_download_failure_permanent_ProtocolError(self, mock_add_eve mock_error_state.return_value = True exthandlers_handler.run() + event_occurrences = [kw for _, kw in mock_add_event.call_args_list if + "[ExtensionError] Failed to get ext handler pkgs" in kw['message']] + self.assertEqual(1, len(event_occurrences)) + self.assertFalse(event_occurrences[0]['is_success']) + self.assertTrue("Failed to get ext handler pkgs" in event_occurrences[0]['message']) + self.assertTrue("ProtocolError" in event_occurrences[0]['message']) - self.assertEqual(1, mock_add_event.call_count) - args, kw = mock_add_event.call_args_list[0] - self.assertEqual(False, kw['is_success']) - self.assertTrue("Failed to get ext handler pkgs" in kw['message']) - self.assertTrue("ProtocolError" in kw['message']) - - @patch('azurelinuxagent.common.event.add_event') + @patch('azurelinuxagent.ga.exthandlers.add_event') def test_ext_handler_download_errors_should_be_reported_only_on_new_goal_state(self, mock_add_event, *args): def _assert_mock_add_event_call(expected_download_failed_event_count, err_msg_guid): @@ -1341,45 +1424,42 @@ def mock_popen(*args, **kwargs): # 1 expected call count for Enable command assert_extensions_called(exthandlers_handler, expected_call_count=1) - def test_handle_ext_handlers_on_hold_true(self, *args): - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter - protocol.get_artifacts_profile = MagicMock() - - # Disable extension handling blocking - exthandlers_handler._extension_processing_allowed = Mock(return_value=False) - with patch.object(ExtHandlersHandler, 'handle_ext_handlers') as patch_handle_ext_handlers: - exthandlers_handler.run() - self.assertEqual(0, patch_handle_ext_handlers.call_count) - - # enable extension handling blocking - exthandlers_handler._extension_processing_allowed = Mock(return_value=True) - with patch.object(ExtHandlersHandler, 'handle_ext_handlers') as patch_handle_ext_handlers: - exthandlers_handler.run() - self.assertEqual(1, patch_handle_ext_handlers.call_count) - - def test_handle_ext_handlers_on_hold_false(self, *args): + def test_it_should_process_extensions_appropriately_on_artifact_hold(self, mock_get, mock_crypt, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter - exthandlers_handler.ext_handlers, exthandlers_handler.last_etag = protocol.get_ext_handlers() - exthandlers_handler.protocol = protocol + exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) # enable extension handling blocking conf.get_enable_overprovisioning = Mock(return_value=True) - - # Test when is_on_hold returns False mock_in_vm_artifacts_profile = InVMArtifactsProfile(MagicMock()) + + # Test when is_on_hold returns True - should not work + mock_in_vm_artifacts_profile.is_on_hold = Mock(return_value=True) + protocol.get_artifacts_profile = Mock(return_value=mock_in_vm_artifacts_profile) + exthandlers_handler.run() + vm_agent_status = protocol.report_vm_status.call_args[0][0].vmAgent + self.assertEqual(vm_agent_status.status, "Ready", "Agent should report ready") + self.assertEqual(0, len(vm_agent_status.extensionHandlers), + "No extensions should be reported as on_hold is True") + self.assertIsNone(vm_agent_status.vm_artifacts_aggregate_status.goal_state_aggregate_status, + "No GS Aggregate status should be reported") + + # Test when is_on_hold returns False - should work mock_in_vm_artifacts_profile.is_on_hold = Mock(return_value=False) protocol.get_artifacts_profile = Mock(return_value=mock_in_vm_artifacts_profile) - with patch.object(ExtHandlersHandler, 'handle_ext_handler') as patch_handle_ext_handler: - exthandlers_handler.handle_ext_handlers() - self.assertEqual(1, patch_handle_ext_handler.call_count) + exthandlers_handler.run() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") + self.assertEqual("1", protocol.report_vm_status.call_args[0][ + 0].vmAgent.vm_artifacts_aggregate_status.goal_state_aggregate_status.in_svd_seq_no, "Incarnation mismatch") - # Test when in_vm_artifacts_profile is not available + # Test when in_vm_artifacts_profile is not available - Should work protocol.get_artifacts_profile = Mock(return_value=None) - with patch.object(ExtHandlersHandler, 'handle_ext_handler') as patch_handle_ext_handler: - exthandlers_handler.handle_ext_handlers() - self.assertEqual(1, patch_handle_ext_handler.call_count) + # Update GoalState + test_data.set_incarnation(2) + protocol.update_goal_state() + exthandlers_handler.run() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") + self.assertEqual("2", protocol.report_vm_status.call_args[0][ + 0].vmAgent.vm_artifacts_aggregate_status.goal_state_aggregate_status.in_svd_seq_no, "Incarnation mismatch") def test_last_etag_on_extension_processing(self, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) @@ -1423,14 +1503,31 @@ def test_it_should_parse_in_vm_metadata_properly(self, mock_get, mock_crypt, *ar self.assertEqual(correlation_id, "NA", "Correlation Id should be NA") self.assertEqual(gs_creation_time, "NA", "GS Creation time should be NA") - def _assert_ext_status(self, report_ext_status, expected_status, - expected_seq_no): - self.assertTrue(report_ext_status.called) - args, kw = report_ext_status.call_args # pylint: disable=unused-variable - ext_status = args[-1] + def _assert_ext_status(self, vm_agent_status, expected_status, + expected_seq_no, expected_handler_name="OSTCExtensions.ExampleHandlerLinux", expected_msg=None): + + self.assertTrue(vm_agent_status.called) + args, _ = vm_agent_status.call_args + vm_status = args[0] + ext_status = next(handler_status.extension_status for handler_status in vm_status.vmAgent.extensionHandlers if + handler_status.name == expected_handler_name) self.assertEqual(expected_status, ext_status.status) self.assertEqual(expected_seq_no, ext_status.sequenceNumber) + if expected_msg is not None: + self.assertIn(expected_msg, ext_status.message) + + def test_it_should_initialise_and_use_command_execution_log_for_extensions(self, mock_get, mock_crypt_util, *args): + test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) + exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt_util, *args) + exthandlers_handler.run() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") + + command_execution_log = os.path.join(conf.get_ext_log_dir(), "OSTCExtensions.ExampleHandlerLinux", + "CommandExecution.log") + self.assertTrue(os.path.exists(command_execution_log), "CommandExecution.log file not found") + self.assertGreater(os.path.getsize(command_execution_log), 0, "The file should not be empty") + def test_ext_handler_no_reporting_status(self, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter @@ -1446,88 +1543,94 @@ def test_ext_handler_no_reporting_status(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, ValidHandlerStatus.error, 0) + self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.error, 0) - def test_wait_for_handler_completion_empty_exts(self, *args): + def test_wait_for_handler_completion_no_status(self, mock_http_get, mock_crypt_util, *args): """ - Testing wait_for_handler_completion() when there is no extension in a handler. - Expected to return True. + Testing depends-on scenario when there is no status file reported by the extension. + Expected to retry and eventually report failure for all dependent extensions. """ - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=unused-variable,no-value-for-parameter - - handler = ExtHandler(name="handler") - ExtHandlerInstance(handler, protocol).set_handler_status("Ready") + exthandlers_handler, protocol = self._create_mock( + mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SEQUENCING), mock_http_get, mock_crypt_util, *args) - ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=None) - self.assertTrue(exthandlers_handler.wait_for_handler_completion(handler, datetime.datetime.utcnow())) + original_popen = subprocess.Popen - def _helper_wait_for_handler_completion(self, exthandlers_handler): - """ - Call wait_for_handler_completion() passing a handler with an extension. - Override the wait time to be 5 seconds to minimize the timout duration. - Return the value returned by wait_for_handler_completion(). - """ - handler_name = "Handler" - exthandler = ExtHandler(name=handler_name) - extension = Extension(name=handler_name) - exthandler.properties.extensions.append(extension) + def mock_popen(cmd, *args, **kwargs): + # For the purpose of this test, deleting the placeholder status file created by the agent + if "sample.py" in cmd: + status_path = os.path.join(kwargs['env'][ExtCommandEnvVariable.ExtensionPath], "status", + "{0}.status".format(kwargs['env'][ExtCommandEnvVariable.ExtensionSeqNumber])) + if os.path.exists(status_path): + os.remove(status_path) + mock_popen.deleted_status_file = status_path + return original_popen(["echo", "Yes"], *args, **kwargs) - # Override the timeout value to minimize the test duration - wait_until = datetime.datetime.utcnow() + datetime.timedelta(seconds=0.1) - ExtHandlerInstance(exthandler, Mock()).set_handler_status("Ready") - return exthandlers_handler.wait_for_handler_completion(exthandler, wait_until) + with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): + with patch('azurelinuxagent.ga.exthandlers._DEFAULT_EXT_TIMEOUT_MINUTES', 0.001): + exthandlers_handler.run() - def test_wait_for_handler_completion_no_status(self, *args): - """ - Testing wait_for_handler_completion() when there is no status file or seq_no is negative. - Expected to return False. - """ - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=unused-variable,no-value-for-parameter + # The Handler Status for the base extension should be ready as it was executed successfully by the agent + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") + # The extension status reported by the Handler should be an error since no status file was found + self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.error, 0, + expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux", + expected_msg="No such file or directory: '{0}'".format(mock_popen.deleted_status_file)) - ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=None) - self.assertFalse(self._helper_wait_for_handler_completion(exthandlers_handler)) + # The Handler Status for the dependent extension should be NotReady as it was not executed at all + # And since it was not executed, it should not report any extension status either + self._assert_handler_status(protocol.report_vm_status, "NotReady", 0, "1.0.0", + expected_msg="Dependent Extension OSTCExtensions.OtherExampleHandlerLinux did not reach a terminal state within the allowed timeout. Last status was {0}".format( + ValidHandlerStatus.warning)) - def test_wait_for_handler_completion_success_status(self, *args): + def test_wait_for_handler_completion_success_status(self, mock_http_get, mock_crypt_util, *args): """ - Testing wait_for_handler_successful_completion() when there is successful status. - Expected to return True. + Testing depends-on scenario on a successful case. Expected to report the status for both extensions properly. """ - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=unused-variable,no-value-for-parameter + exthandlers_handler, protocol = self._create_mock( + mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SEQUENCING), mock_http_get, mock_crypt_util, *args) - status = "success" + exthandlers_handler.run() - ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=status) - self.assertTrue(self._helper_wait_for_handler_completion(exthandlers_handler)) + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux", + expected_msg='Plugin enabled') + # The extension status reported by the Handler should be an error since no status file was found + self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.success, 0, + expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") - def test_wait_for_handler_completion_error_status(self, *args): + # The Handler Status for the dependent extension should be NotReady as it was not executed at all + # And since it was not executed, it should not report any extension status either + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", expected_msg='Plugin enabled') + self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.success, 0) + + def test_wait_for_handler_completion_error_status(self, mock_http_get, mock_crypt_util, *args): """ Testing wait_for_handler_completion() when there is error status. Expected to return False. """ - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=unused-variable,no-value-for-parameter + exthandlers_handler, protocol = self._create_mock( + mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SEQUENCING), mock_http_get, mock_crypt_util, *args) - status = "error" + original_popen = subprocess.Popen - ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=status) - self.assertFalse(self._helper_wait_for_handler_completion(exthandlers_handler)) + def mock_popen(cmd, *args, **kwargs): + # For the purpose of this test, deleting the placeholder status file created by the agent + if "sample.py" in cmd: + return original_popen(["/fail/this/command"], *args, **kwargs) + return original_popen(cmd, *args, **kwargs) - def test_wait_for_handler_completion_timeout(self, *args): - """ - Testing wait_for_handler_successful_completion() when there is non terminal status. - Expected to return False. - """ - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=unused-variable,no-value-for-parameter + with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): + exthandlers_handler.run() - # Choose a non-terminal status - status = "warning" + # The Handler Status for the base extension should be NotReady as it failed + self._assert_handler_status(protocol.report_vm_status, "NotReady", 0, "1.0.0", + expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") - ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=status) - self.assertFalse(self._helper_wait_for_handler_completion(exthandlers_handler)) + # The Handler Status for the dependent extension should be NotReady as it was not executed at all + # And since it was not executed, it should not report any extension status either + self._assert_handler_status(protocol.report_vm_status, "NotReady", 0, "1.0.0", + expected_msg='Skipping processing of extensions since execution of dependent extension OSTCExtensions.OtherExampleHandlerLinux failed') def test_get_ext_handling_status(self, *args): """ @@ -1698,7 +1801,7 @@ def test_extensions_deleted(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Update incarnation, simulate new extension version and old one deleted test_data.set_incarnation(2) @@ -1710,7 +1813,7 @@ def test_extensions_deleted(self, *args): exthandlers_handler.run() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.1") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) @patch('azurelinuxagent.ga.exthandlers.ExtHandlerInstance.install', side_effect=ExtHandlerInstance.install, autospec=True) @@ -1735,10 +1838,8 @@ def test_install_failure(self, patch_get_install_command, patch_install, *args): self.assertEqual(1, patch_install.call_count) self.assertEqual(2, protocol.report_vm_status.call_count) - @patch('azurelinuxagent.ga.exthandlers.ExtHandlersHandler.handle_ext_handler_error') @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_install_command') - def test_install_failure_check_exception_handling(self, patch_get_install_command, patch_handle_ext_handler_error, - *args): + def test_install_failure_check_exception_handling(self, patch_get_install_command, *args): """ When extension install fails, the operation should be reported to our telemetry service. """ @@ -1750,7 +1851,8 @@ def test_install_failure_check_exception_handling(self, patch_get_install_comman exthandlers_handler.run() self.assertEqual(1, protocol.report_vm_status.call_count) - self.assertEqual(1, patch_handle_ext_handler_error.call_count) + self._assert_handler_status(protocol.report_vm_status, expected_status="NotReady", expected_ext_count=0, + version="1.0.0") @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_enable_command') def test_enable_failure(self, patch_get_enable_command, *args): @@ -1773,10 +1875,8 @@ def test_enable_failure(self, patch_get_enable_command, *args): self.assertEqual(1, patch_get_enable_command.call_count) self.assertEqual(2, protocol.report_vm_status.call_count) - @patch('azurelinuxagent.ga.exthandlers.ExtHandlersHandler.handle_ext_handler_error') @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_enable_command') - def test_enable_failure_check_exception_handling(self, patch_get_enable_command, - patch_handle_ext_handler_error, *args): + def test_enable_failure_check_exception_handling(self, patch_get_enable_command, *args): """ When extension enable fails, the operation should be reported. """ @@ -1790,7 +1890,7 @@ def test_enable_failure_check_exception_handling(self, patch_get_enable_command, self.assertEqual(1, patch_get_enable_command.call_count) self.assertEqual(1, protocol.report_vm_status.call_count) - self.assertEqual(1, patch_handle_ext_handler_error.call_count) + self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.0") @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_disable_command') def test_disable_failure(self, patch_get_disable_command, *args): @@ -1808,11 +1908,11 @@ def test_disable_failure(self, patch_get_disable_command, *args): self.assertEqual(0, patch_get_disable_command.call_count) self.assertEqual(1, protocol.report_vm_status.call_count) self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Next incarnation, disable extension test_data.set_incarnation(2) - test_data.set_extensions_config_state(ExtensionRequestedState.Disabled) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Disabled) protocol.update_goal_state() exthandlers_handler.run() @@ -1828,10 +1928,9 @@ def test_disable_failure(self, patch_get_disable_command, *args): self.assertEqual(3, protocol.report_vm_status.call_count) self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.0") - @patch('azurelinuxagent.ga.exthandlers.ExtHandlersHandler.handle_ext_handler_error') @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_disable_command') def test_disable_failure_with_exception_handling(self, patch_get_disable_command, - patch_handle_ext_handler_error, *args): + *args): """ When extension disable fails, the operation should be reported. """ @@ -1846,18 +1945,18 @@ def test_disable_failure_with_exception_handling(self, patch_get_disable_command self.assertEqual(0, patch_get_disable_command.call_count) self.assertEqual(1, protocol.report_vm_status.call_count) self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Next incarnation, disable extension test_data.set_incarnation(2) - test_data.set_extensions_config_state(ExtensionRequestedState.Disabled) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Disabled) protocol.update_goal_state() exthandlers_handler.run() self.assertEqual(1, patch_get_disable_command.call_count) self.assertEqual(2, protocol.report_vm_status.call_count) - self.assertEqual(1, patch_handle_ext_handler_error.call_count) + self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.0") @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_uninstall_command') def test_uninstall_failure(self, patch_get_uninstall_command, *args): @@ -1875,11 +1974,11 @@ def test_uninstall_failure(self, patch_get_uninstall_command, *args): self.assertEqual(0, patch_get_uninstall_command.call_count) self.assertEqual(1, protocol.report_vm_status.call_count) self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.0") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) # Next incarnation, disable extension test_data.set_incarnation(2) - test_data.set_extensions_config_state(ExtensionRequestedState.Uninstall) + test_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) protocol.update_goal_state() exthandlers_handler.run() @@ -2073,10 +2172,8 @@ def test_old_handler_reports_failure_on_disable_fail_on_update(self, patch_get_d # This is ensuring that the error status is being written to the new version self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=0, version=new_version) - @patch('azurelinuxagent.ga.exthandlers.ExtHandlersHandler.handle_ext_handler_error') @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_update_command') - def test_upgrade_failure_with_exception_handling(self, patch_get_update_command, - patch_handle_ext_handler_error, *args): + def test_upgrade_failure_with_exception_handling(self, patch_get_update_command, *args): """ Extension upgrade failure should not be retried """ @@ -2085,7 +2182,7 @@ def test_upgrade_failure_with_exception_handling(self, patch_get_update_command, exthandlers_handler.run() self.assertEqual(1, patch_get_update_command.call_count) - self.assertEqual(1, patch_handle_ext_handler_error.call_count) + self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.1") @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_disable_command') def test_extension_upgrade_should_pass_when_continue_on_update_failure_is_true_and_prev_version_disable_fails( @@ -2103,7 +2200,7 @@ def test_extension_upgrade_should_pass_when_continue_on_update_failure_is_true_a # Ensure the handler status and ext_status is successful self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.1") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_uninstall_command') def test_extension_upgrade_should_pass_when_continue_on_update_failue_is_true_and_prev_version_uninstall_fails( @@ -2121,7 +2218,7 @@ def test_extension_upgrade_should_pass_when_continue_on_update_failue_is_true_an # Ensure the handler status and ext_status is successful self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.1") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_disable_command') def test_extension_upgrade_should_fail_when_continue_on_update_failure_is_false_and_prev_version_disable_fails( @@ -2225,7 +2322,7 @@ def test_uninstall_rc_env_var_should_report_not_run_for_non_update_calls_to_exth # Ensure the handler status and ext_status is successful self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.1") - self._assert_ext_status(protocol.report_ext_status, "success", 0) + self._assert_ext_status(protocol.report_vm_status, "success", 0) def test_ext_path_and_version_env_variables_set_for_ever_operation(self, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SINGLE) @@ -2459,7 +2556,6 @@ def _create_mock(self, mock_http_get, MockCryptUtil): protocol = WireProtocol(KNOWN_WIRESERVER_IP) protocol.detect() - protocol.report_ext_status = MagicMock() protocol.report_vm_status = MagicMock() protocol.get_artifacts_profile = MagicMock() @@ -2467,16 +2563,9 @@ def _create_mock(self, mock_http_get, MockCryptUtil): handler.ext_handlers, handler.last_etag = protocol.get_ext_handlers() conf.get_enable_overprovisioning = Mock(return_value=False) - def wait_for_handler_completion(prev_handler, _): - return orig_wait_for_handler_completion(prev_handler, - datetime.datetime.utcnow() + datetime.timedelta( - seconds=5)) - def reset_etag(): handler.last_etag = 0 - orig_wait_for_handler_completion = handler.wait_for_handler_completion - handler.wait_for_handler_completion = wait_for_handler_completion handler.reset_etag = reset_etag return handler @@ -2490,7 +2579,7 @@ def _set_dependency_levels(self, dependency_levels, exthandlers_handler): if handler_map.get(handler_name) is None: handler = ExtHandler(name=handler_name) extension = Extension(name=handler_name) - handler.properties.state = ExtensionRequestedState.Enabled + handler.properties.state = ExtHandlerRequestedState.Enabled handler.properties.extensions.append(extension) handler_map[handler_name] = handler all_handlers.append(handler) @@ -2504,7 +2593,7 @@ def _set_dependency_levels(self, dependency_levels, exthandlers_handler): exthandlers_handler.ext_handlers.extHandlers.append(handler) def _validate_extension_sequence(self, expected_sequence, exthandlers_handler): - installed_extensions = [a[0].name for a, k in exthandlers_handler.handle_ext_handler.call_args_list] # pylint: disable=unused-variable + installed_extensions = [a[0].ext_handler.name for a, _ in exthandlers_handler.handle_ext_handler.call_args_list] self.assertListEqual(expected_sequence, installed_extensions, "Expected and actual list of extensions are not equal") @@ -2525,8 +2614,9 @@ def get_ext_handling_status(ext): with patch.object(ExtHandlerInstance, "get_ext_handling_status", side_effect=get_ext_handling_status): with patch.object(ExtHandlerInstance, "get_handler_status", ExtHandlerStatus): - exthandlers_handler.run() - self._validate_extension_sequence(expected_sequence, exthandlers_handler) + with patch('azurelinuxagent.ga.exthandlers._DEFAULT_EXT_TIMEOUT_MINUTES', 0.01): + exthandlers_handler.run() + self._validate_extension_sequence(expected_sequence, exthandlers_handler) def test_handle_ext_handlers(self, *args): """ @@ -3215,7 +3305,7 @@ def manifest_location_handler(url, **kwargs): return None - + with mock_wire_protocol(self.test_data, http_get_handler=manifest_location_handler) as protocol: exthandlers_handler = get_exthandlers_handler(protocol) exthandlers_handler.run() @@ -3256,117 +3346,6 @@ def manifest_location_handler(url, **kwargs): protocol.client.fetch_manifest(ext_handlers.extHandlers[0].versionUris, timeout_in_minutes=0, timeout_in_ms=200) -class TestMultiConfigExtensions(AgentTestCase): - - _MULTI_CONFIG_TEST_DATA = os.path.join("wire", "multi-config") - - def setUp(self): - AgentTestCase.setUp(self) - self.mock_sleep = patch("time.sleep", lambda *_: mock_sleep(0.0001)) - self.mock_sleep.start() - self.test_data = DATA_FILE.copy() - - def tearDown(self): - self.mock_sleep.stop() - AgentTestCase.tearDown(self) - - class _TestExtHandlerObject: - def __init__(self, name, version, state="enabled"): - self.name = name - self.version = version - self.state = state - self.is_invalid_setting = False - self.extensions = dict() - - class _TestExtensionObject: - def __init__(self, name, seq_no, dependency_level="0", state="enabled"): - self.name = name - self.seq_no = seq_no - self.dependency_level = int(dependency_level) - self.state = state - - def _mock_and_assert_ext_handlers(self, expected_handlers): - with mock_wire_protocol(self.test_data) as protocol: - ext_handlers, _ = protocol.get_ext_handlers() - for ext_handler in ext_handlers.extHandlers: - if ext_handler.name not in expected_handlers: - continue - expected_handler = expected_handlers.pop(ext_handler.name) - self.assertEqual(expected_handler.state, ext_handler.properties.state) - self.assertEqual(expected_handler.version, ext_handler.properties.version) - self.assertEqual(expected_handler.is_invalid_setting, ext_handler.is_invalid_setting) - self.assertEqual(len(expected_handler.extensions), len(ext_handler.properties.extensions)) - - for extension in ext_handler.properties.extensions: - self.assertIn(extension.name, expected_handler.extensions) - expected_extension = expected_handler.extensions.pop(extension.name) - self.assertEqual(expected_extension.seq_no, extension.sequenceNumber) - self.assertEqual(expected_extension.state, extension.state) - self.assertEqual(expected_extension.dependency_level, extension.dependencyLevel) - - self.assertEqual(0, len(expected_handler.extensions), "All extensions not verified for handler") - - self.assertEqual(0, len(expected_handlers), "All handlers not verified") - - def _get_mock_expected_handler_data(self, rc_extensions, vmaccess_extensions, geneva_extensions): - # Set expected handler data - run_command_test_handler = self._TestExtHandlerObject("Microsoft.CPlat.Core.RunCommandHandlerWindows", "2.0.2") - run_command_test_handler.extensions.update(rc_extensions) - - vm_access_test_handler = self._TestExtHandlerObject("Microsoft.Compute.VMAccessAgent", "2.4.7") - vm_access_test_handler.extensions.update(vmaccess_extensions) - - geneva_test_handler = self._TestExtHandlerObject("Microsoft.Azure.Geneva.GenevaMonitoring", "2.20.0.1") - geneva_test_handler.extensions.update(geneva_extensions) - - expected_handlers = { - run_command_test_handler.name: run_command_test_handler, - vm_access_test_handler.name: vm_access_test_handler, - geneva_test_handler.name: geneva_test_handler - } - return expected_handlers - - def test_it_should_parse_multi_config_settings_properly(self): - self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, "ext_conf_with_multi_config.xml") - - rc_extensions = dict() - rc_extensions["firstRunCommand"] = self._TestExtensionObject(name="firstRunCommand", seq_no="2") - rc_extensions["secondRunCommand"] = self._TestExtensionObject(name="secondRunCommand", seq_no="2", - dependency_level="3") - rc_extensions["thirdRunCommand"] = self._TestExtensionObject(name="thirdRunCommand", seq_no="1", - dependency_level="4") - - vmaccess_extensions = { - "Microsoft.Compute.VMAccessAgent": self._TestExtensionObject(name="Microsoft.Compute.VMAccessAgent", - seq_no="1", dependency_level="2")} - - geneva_extensions = {"Microsoft.Azure.Geneva.GenevaMonitoring": self._TestExtensionObject( - name="Microsoft.Azure.Geneva.GenevaMonitoring", seq_no="1")} - - expected_handlers = self._get_mock_expected_handler_data(rc_extensions, vmaccess_extensions, geneva_extensions) - self._mock_and_assert_ext_handlers(expected_handlers) - - def test_it_should_parse_multi_config_with_disable_state_properly(self): - self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, - "ext_conf_with_disabled_multi_config.xml") - - rc_extensions = dict() - rc_extensions["firstRunCommand"] = self._TestExtensionObject(name="firstRunCommand", seq_no="3") - rc_extensions["secondRunCommand"] = self._TestExtensionObject(name="secondRunCommand", seq_no="3", - dependency_level="1") - rc_extensions["thirdRunCommand"] = self._TestExtensionObject(name="thirdRunCommand", seq_no="1", - dependency_level="4", state="disabled") - - vmaccess_extensions = { - "Microsoft.Compute.VMAccessAgent": self._TestExtensionObject(name="Microsoft.Compute.VMAccessAgent", - seq_no="2", dependency_level="2")} - - geneva_extensions = {"Microsoft.Azure.Geneva.GenevaMonitoring": self._TestExtensionObject( - name="Microsoft.Azure.Geneva.GenevaMonitoring", seq_no="2")} - - expected_handlers = self._get_mock_expected_handler_data(rc_extensions, vmaccess_extensions, geneva_extensions) - self._mock_and_assert_ext_handlers(expected_handlers) - if __name__ == '__main__': unittest.main() diff --git a/tests/ga/test_exthandlers.py b/tests/ga/test_exthandlers.py index 4893dca3f..d2ee2de75 100644 --- a/tests/ga/test_exthandlers.py +++ b/tests/ga/test_exthandlers.py @@ -31,7 +31,7 @@ from azurelinuxagent.common.utils.extensionprocessutil import TELEMETRY_MESSAGE_MAX_LEN, format_stdout_stderr, \ read_output from azurelinuxagent.ga.exthandlers import parse_ext_status, ExtHandlerInstance, ExtCommandEnvVariable, \ - ExtensionStatusError + ExtensionStatusError, _DEFAULT_SEQ_NO from tests.protocol import mockwiredata from tests.protocol.mocks import mock_wire_protocol from tests.tools import AgentTestCase, patch, mock_sleep, clear_singleton_instances @@ -209,14 +209,11 @@ def test_parse_extension_status_with_empty_status(self): self.assertEqual(0, ext_status.sequenceNumber) self.assertEqual(0, len(ext_status.substatusList)) - @patch('azurelinuxagent.common.event.EventLogger.add_event') @patch('azurelinuxagent.ga.exthandlers.ExtHandlerInstance._get_last_modified_seq_no_from_config_files') - def assert_extension_sequence_number(self, - patch_get_largest_seq, - patch_add_event, - goal_state_sequence_number, - disk_sequence_number, - expected_sequence_number): + def assert_extension_sequence_number(self, patch_get_largest_seq=None, + goal_state_sequence_number=None, + disk_sequence_number=None, + expected_sequence_number=None): ext = Extension() ext.sequenceNumber = goal_state_sequence_number patch_get_largest_seq.return_value = disk_sequence_number @@ -229,23 +226,6 @@ def assert_extension_sequence_number(self, instance = ExtHandlerInstance(ext_handler=ext_handler, protocol=None) seq, path = instance.get_status_file_path(ext) - try: - gs_seq_int = int(goal_state_sequence_number) - gs_int = True - except ValueError: - gs_int = False - - if gs_int and gs_seq_int != disk_sequence_number: - self.assertEqual(1, patch_add_event.call_count) - args, kw_args = patch_add_event.call_args # pylint: disable=unused-variable - self.assertEqual('SequenceNumberMismatch', kw_args['op']) - self.assertEqual(False, kw_args['is_success']) - self.assertEqual('Goal state: {0}, disk: {1}' - .format(gs_seq_int, disk_sequence_number), - kw_args['message']) - else: - self.assertEqual(0, patch_add_event.call_count) - self.assertEqual(expected_sequence_number, seq) if seq > -1: self.assertTrue(path.endswith('/foo-1.2.3/status/{0}.status'.format(expected_sequence_number))) @@ -253,19 +233,19 @@ def assert_extension_sequence_number(self, self.assertIsNone(path) def test_extension_sequence_number(self): - self.assert_extension_sequence_number(goal_state_sequence_number="12", # pylint: disable=no-value-for-parameter + self.assert_extension_sequence_number(goal_state_sequence_number="12", disk_sequence_number=366, expected_sequence_number=12) - self.assert_extension_sequence_number(goal_state_sequence_number=" 12 ", # pylint: disable=no-value-for-parameter + self.assert_extension_sequence_number(goal_state_sequence_number=" 12 ", disk_sequence_number=366, expected_sequence_number=12) - self.assert_extension_sequence_number(goal_state_sequence_number=" foo", # pylint: disable=no-value-for-parameter + self.assert_extension_sequence_number(goal_state_sequence_number=" foo", disk_sequence_number=3, expected_sequence_number=3) - self.assert_extension_sequence_number(goal_state_sequence_number="-1", # pylint: disable=no-value-for-parameter + self.assert_extension_sequence_number(goal_state_sequence_number="-1", disk_sequence_number=3, expected_sequence_number=-1) @@ -700,7 +680,7 @@ def test_it_should_contain_all_helper_environment_variables(self): wire_ip = str(uuid.uuid4()) ext_handler_instance = ExtHandlerInstance(ext_handler=self.ext_handler, protocol=WireProtocol(wire_ip)) - helper_env_vars = {ExtCommandEnvVariable.ExtensionSeqNumber: ext_handler_instance.get_seq_no(), + helper_env_vars = {ExtCommandEnvVariable.ExtensionSeqNumber: _DEFAULT_SEQ_NO, ExtCommandEnvVariable.ExtensionPath: self.tmp_dir, ExtCommandEnvVariable.ExtensionVersion: ext_handler_instance.ext_handler.properties.version, ExtCommandEnvVariable.WireProtocolAddress: wire_ip} diff --git a/tests/ga/test_multi_config_extension.py b/tests/ga/test_multi_config_extension.py new file mode 100644 index 000000000..a6a61345e --- /dev/null +++ b/tests/ga/test_multi_config_extension.py @@ -0,0 +1,1214 @@ +import contextlib +import json +import os.path +import subprocess +import uuid + +from azurelinuxagent.common import conf +from azurelinuxagent.common.event import WALAEventOperation +from azurelinuxagent.common.exception import GoalStateAggregateStatusCodes +from azurelinuxagent.common.future import ustr +from azurelinuxagent.common.protocol.restapi import ExtHandlerRequestedState, ExtensionState, ExtensionStatus +from azurelinuxagent.common.utils import fileutil +from azurelinuxagent.ga.exthandlers import get_exthandlers_handler, ValidHandlerStatus, ExtCommandEnvVariable, \ + parse_ext_status, GoalStateStatus, ExtHandlerInstance +from tests.ga.extension_emulator import enable_invocations, extension_emulator, ExtensionCommandNames, Actions, \ + extract_extension_info_from_command +from tests.protocol.mocks import mock_wire_protocol, HttpRequestPredicates, MockHttpResponse +from tests.protocol.mockwiredata import DATA_FILE, WireProtocolData +from tests.tools import AgentTestCase, mock_sleep, patch + + +class TestMultiConfigExtensionsConfigParsing(AgentTestCase): + + _MULTI_CONFIG_TEST_DATA = os.path.join("wire", "multi-config") + + def setUp(self): + AgentTestCase.setUp(self) + self.mock_sleep = patch("time.sleep", lambda *_: mock_sleep(0.0001)) + self.mock_sleep.start() + self.test_data = DATA_FILE.copy() + + def tearDown(self): + self.mock_sleep.stop() + AgentTestCase.tearDown(self) + + class _TestExtHandlerObject: + def __init__(self, name, version, state="enabled"): + self.name = name + self.version = version + self.state = state + self.is_invalid_setting = False + self.extensions = dict() + + class _TestExtensionObject: + def __init__(self, name, seq_no, dependency_level="0", state="enabled"): + self.name = name + self.seq_no = seq_no + self.dependency_level = int(dependency_level) + self.state = state + + def _mock_and_assert_ext_handlers(self, expected_handlers): + with mock_wire_protocol(self.test_data) as protocol: + ext_handlers, _ = protocol.get_ext_handlers() + for ext_handler in ext_handlers.extHandlers: + if ext_handler.name not in expected_handlers: + continue + expected_handler = expected_handlers.pop(ext_handler.name) + self.assertEqual(expected_handler.state, ext_handler.properties.state) + self.assertEqual(expected_handler.version, ext_handler.properties.version) + self.assertEqual(expected_handler.is_invalid_setting, ext_handler.is_invalid_setting) + self.assertEqual(len(expected_handler.extensions), len(ext_handler.properties.extensions)) + + for extension in ext_handler.properties.extensions: + self.assertIn(extension.name, expected_handler.extensions) + expected_extension = expected_handler.extensions.pop(extension.name) + self.assertEqual(expected_extension.seq_no, extension.sequenceNumber) + self.assertEqual(expected_extension.state, extension.state) + self.assertEqual(expected_extension.dependency_level, extension.dependencyLevel) + + self.assertEqual(0, len(expected_handler.extensions), "All extensions not verified for handler") + + self.assertEqual(0, len(expected_handlers), "All handlers not verified") + + def _get_mock_expected_handler_data(self, rc_extensions, vmaccess_extensions, geneva_extensions): + # Set expected handler data + run_command_test_handler = self._TestExtHandlerObject("Microsoft.CPlat.Core.RunCommandHandlerWindows", "2.3.0") + run_command_test_handler.extensions.update(rc_extensions) + + vm_access_test_handler = self._TestExtHandlerObject("Microsoft.Compute.VMAccessAgent", "2.4.7") + vm_access_test_handler.extensions.update(vmaccess_extensions) + + geneva_test_handler = self._TestExtHandlerObject("Microsoft.Azure.Geneva.GenevaMonitoring", "2.20.0.1") + geneva_test_handler.extensions.update(geneva_extensions) + + expected_handlers = { + run_command_test_handler.name: run_command_test_handler, + vm_access_test_handler.name: vm_access_test_handler, + geneva_test_handler.name: geneva_test_handler + } + return expected_handlers + + def test_it_should_parse_multi_config_settings_properly(self): + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, "ext_conf_with_multi_config.xml") + + rc_extensions = dict() + rc_extensions["firstRunCommand"] = self._TestExtensionObject(name="firstRunCommand", seq_no="2") + rc_extensions["secondRunCommand"] = self._TestExtensionObject(name="secondRunCommand", seq_no="2", + dependency_level="3") + rc_extensions["thirdRunCommand"] = self._TestExtensionObject(name="thirdRunCommand", seq_no="1", + dependency_level="4") + + vmaccess_extensions = { + "Microsoft.Compute.VMAccessAgent": self._TestExtensionObject(name="Microsoft.Compute.VMAccessAgent", + seq_no="1", dependency_level="2")} + + geneva_extensions = {"Microsoft.Azure.Geneva.GenevaMonitoring": self._TestExtensionObject( + name="Microsoft.Azure.Geneva.GenevaMonitoring", seq_no="1")} + + expected_handlers = self._get_mock_expected_handler_data(rc_extensions, vmaccess_extensions, geneva_extensions) + self._mock_and_assert_ext_handlers(expected_handlers) + + def test_it_should_parse_multi_config_with_disable_state_properly(self): + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_with_disabled_multi_config.xml") + + rc_extensions = dict() + rc_extensions["firstRunCommand"] = self._TestExtensionObject(name="firstRunCommand", seq_no="3") + rc_extensions["secondRunCommand"] = self._TestExtensionObject(name="secondRunCommand", seq_no="3", + dependency_level="1") + rc_extensions["thirdRunCommand"] = self._TestExtensionObject(name="thirdRunCommand", seq_no="1", + dependency_level="4", state="disabled") + + vmaccess_extensions = { + "Microsoft.Compute.VMAccessAgent": self._TestExtensionObject(name="Microsoft.Compute.VMAccessAgent", + seq_no="2", dependency_level="2")} + + geneva_extensions = {"Microsoft.Azure.Geneva.GenevaMonitoring": self._TestExtensionObject( + name="Microsoft.Azure.Geneva.GenevaMonitoring", seq_no="2")} + + expected_handlers = self._get_mock_expected_handler_data(rc_extensions, vmaccess_extensions, geneva_extensions) + self._mock_and_assert_ext_handlers(expected_handlers) + + +class _MultiConfigBaseTestClass(AgentTestCase): + _MULTI_CONFIG_TEST_DATA = os.path.join("wire", "multi-config") + + def setUp(self): + AgentTestCase.setUp(self) + self.mock_sleep = patch("time.sleep", lambda *_: mock_sleep(0.01)) + self.mock_sleep.start() + self.test_data = DATA_FILE.copy() + + def tearDown(self): + self.mock_sleep.stop() + AgentTestCase.tearDown(self) + + @contextlib.contextmanager + def _setup_test_env(self, mock_manifest=False): + + with mock_wire_protocol(self.test_data) as protocol: + def mock_http_put(url, *args, **_): + if HttpRequestPredicates.is_host_plugin_status_request(url): + # Skip reading the HostGA request data as its encoded + return MockHttpResponse(status=500) + protocol.aggregate_status = json.loads(args[0]) + return MockHttpResponse(status=201) + + with patch("azurelinuxagent.common.agent_supported_feature._MultiConfigFeature.is_supported", True): + protocol.aggregate_status = None + protocol.set_http_handlers(http_put_handler=mock_http_put) + exthandlers_handler = get_exthandlers_handler(protocol) + no_of_extensions = protocol.mock_wire_data.get_no_of_extensions_in_config() + + if mock_manifest: + with patch('azurelinuxagent.ga.exthandlers.HandlerManifest.supports_multiple_extensions', + return_value=True): + yield exthandlers_handler, protocol, no_of_extensions + else: + yield exthandlers_handler, protocol, no_of_extensions + + def _assert_and_get_handler_status(self, aggregate_status, handler_name="OSTCExtensions.ExampleHandlerLinux", + handler_version="1.0.0", status="Ready", expected_count=1, message=None): + self.assertIsNotNone(aggregate_status['aggregateStatus'], "No aggregate status found") + handlers = [handler for handler in aggregate_status['aggregateStatus']['handlerAggregateStatus'] if + handler_name == handler['handlerName'] and handler_version == handler['handlerVersion']] + self.assertEqual(expected_count, len(handlers), "Unexpected extension count") + self.assertTrue(all(handler['status'] == status for handler in handlers), + "Unexpected Status reported for handler {0}".format(handler_name)) + if message is not None: + self.assertTrue(all(message in handler['formattedMessage']['message'] for handler in handlers), + "Status Message mismatch") + return handlers + + def _assert_extension_status(self, handler_statuses, expected_ext_status, multi_config=False): + for ext_name, settings_status in expected_ext_status.items(): + ext_status = next(handler for handler in handler_statuses if + handler['runtimeSettingsStatus']['settingsStatus']['status']['name'] == ext_name) + ext_runtime_status = ext_status['runtimeSettingsStatus'] + self.assertIsNotNone(ext_runtime_status, "Extension not found") + self.assertEqual(settings_status['seq_no'], ext_runtime_status['sequenceNumber'], "Sequence no mismatch") + self.assertEqual(settings_status['status'], ext_runtime_status['settingsStatus']['status']['status'], + "status mismatch") + + if 'message' in settings_status and settings_status['message'] is not None: + self.assertIn(settings_status['message'], + ext_runtime_status['settingsStatus']['status']['formattedMessage']['message'], + "message mismatch") + + if multi_config: + self.assertEqual(ext_name, ext_runtime_status['extensionName'], "ext name mismatch") + else: + self.assertNotIn('extensionName', ext_runtime_status, "Extension name should not be reported for SC") + + handler_statuses.remove(ext_status) + + self.assertEqual(0, len(handler_statuses), "Unexpected extensions left for handler") + + +class TestMultiConfigExtensions(_MultiConfigBaseTestClass): + + def __assert_extension_not_present(self, handlers, extensions): + for ext_name in extensions: + self.assertFalse(all( + 'runtimeSettingsStatus' in handler and 'extensionName' in handler['runtimeSettingsStatus'] + and handler['runtimeSettingsStatus']['extensionName'] == ext_name for + handler in handlers), "Extension status found") + + def __run_and_assert_generic_case(self, exthandlers_handler, protocol, no_of_extensions, with_message=True): + + def get_message(msg): + return msg if with_message else None + + exthandlers_handler.run() + self.assertEqual(no_of_extensions, + len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), + "incorrect extensions reported") + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=3) + expected_extensions = { + "firstExtension": {"status": ValidHandlerStatus.success, "seq_no": 1, + "message": get_message("Enabling firstExtension")}, + "secondExtension": {"status": ValidHandlerStatus.success, "seq_no": 2, + "message": get_message("Enabling secondExtension")}, + "thirdExtension": {"status": ValidHandlerStatus.success, "seq_no": 3, + "message": get_message("Enabling thirdExtension")}, + } + self._assert_extension_status(mc_handlers[:], expected_extensions, multi_config=True) + + sc_handler = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="Microsoft.Powershell.ExampleExtension") + expected_extensions = { + "Microsoft.Powershell.ExampleExtension": {"status": ValidHandlerStatus.success, "seq_no": 9, + "message": get_message("Enabling SingleConfig extension")} + } + self._assert_extension_status(sc_handler[:], expected_extensions) + return mc_handlers, sc_handler + + def __setup_and_assert_disable_scenario(self, exthandlers_handler, protocol): + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, 'ext_conf_mc_disabled_extensions.xml') + protocol.mock_wire_data = WireProtocolData(self.test_data) + protocol.mock_wire_data.set_incarnation(2) + protocol.update_goal_state() + exthandlers_handler.run() + + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + status="Ready", expected_count=2) + expected_extensions = { + "thirdExtension": {"status": ValidHandlerStatus.success, "seq_no": 99, "message": None}, + "fourthExtension": {"status": ValidHandlerStatus.success, "seq_no": 101, "message": None}, + } + self.__assert_extension_not_present(mc_handlers[:], ["firstExtension", "secondExtension"]) + self._assert_extension_status(mc_handlers[:], expected_extensions, multi_config=True) + sc_handler = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="Microsoft.Powershell.ExampleExtension", + status="Ready") + expected_extensions = { + "Microsoft.Powershell.ExampleExtension": {"status": ValidHandlerStatus.success, "seq_no": 10, + "message": None} + } + self._assert_extension_status(sc_handler[:], expected_extensions) + return mc_handlers, sc_handler + + @contextlib.contextmanager + def __setup_generic_test_env(self): + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_multi_config_no_dependencies.xml") + + first_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.firstExtension") + second_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.secondExtension") + third_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.thirdExtension") + fourth_ext = extension_emulator(name="Microsoft.Powershell.ExampleExtension") + + with self._setup_test_env(mock_manifest=True) as (exthandlers_handler, protocol, no_of_extensions): + with enable_invocations(first_ext, second_ext, third_ext, fourth_ext) as invocation_record: + exthandlers_handler.run() + self.assertEqual(no_of_extensions, + len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), + "incorrect extensions reported") + invocation_record.compare( + (first_ext, ExtensionCommandNames.INSTALL), + (first_ext, ExtensionCommandNames.ENABLE), + (second_ext, ExtensionCommandNames.ENABLE), + (third_ext, ExtensionCommandNames.ENABLE), + (fourth_ext, ExtensionCommandNames.INSTALL), + (fourth_ext, ExtensionCommandNames.ENABLE) + ) + + self.__run_and_assert_generic_case(exthandlers_handler, protocol, no_of_extensions, with_message=False) + yield exthandlers_handler, protocol, [first_ext, second_ext, third_ext, fourth_ext] + + def test_it_should_execute_and_report_multi_config_extensions_properly(self): + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_multi_config_no_dependencies.xml") + with self._setup_test_env(mock_manifest=True) as (exthandlers_handler, protocol, no_of_extensions): + + # Case 1: Install and enable Single and MultiConfig extensions + self.__run_and_assert_generic_case(exthandlers_handler, protocol, no_of_extensions) + + # Case 2: Disable 2 multi-config extensions and add another for enable + self.__setup_and_assert_disable_scenario(exthandlers_handler, protocol) + + # Case 3: Uninstall Multi-config handler (with enabled extensions) and single config extension + protocol.mock_wire_data.set_incarnation(3) + protocol.mock_wire_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) + protocol.update_goal_state() + exthandlers_handler.run() + self.assertEqual(0, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), + "No handler/extension status should be reported") + + def test_it_should_report_unregistered_version_error_per_extension(self): + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_multi_config_no_dependencies.xml") + with self._setup_test_env() as (exthandlers_handler, protocol, no_of_extensions): + # Set a random failing extension + failing_version = "19.12.1221" + protocol.mock_wire_data.set_extensions_config_version(failing_version) + protocol.mock_wire_data.set_incarnation(2) + protocol.update_goal_state() + exthandlers_handler.run() + self.assertEqual(no_of_extensions, + len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), + "incorrect extensions reported") + error_msg_format = '[ExtensionError] Unable to find version {0} in manifest for extension {1}' + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + handler_version=failing_version, status="NotReady", + expected_count=3, + message=error_msg_format.format(failing_version, + "OSTCExtensions.ExampleHandlerLinux")) + self.assertTrue(all( + handler['runtimeSettingsStatus']['settingsStatus']['status']['operation'] == WALAEventOperation.Download and + handler['runtimeSettingsStatus']['settingsStatus']['status']['status'] == ValidHandlerStatus.error for + handler in mc_handlers), "Incorrect data reported") + sc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="Microsoft.Powershell.ExampleExtension", + handler_version=failing_version, status="NotReady", + message=error_msg_format.format(failing_version, + "Microsoft.Powershell.ExampleExtension")) + self.assertFalse(all("runtimeSettingsStatus" in handler for handler in sc_handlers), "Incorrect status") + + def test_it_should_not_install_handler_again_if_installed(self): + + with self.__setup_generic_test_env() as (_, _, _): + # Everything is already asserted in the context manager + pass + + def test_it_should_retry_handler_installation_per_extension_if_failed(self): + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_multi_config_no_dependencies.xml") + with self._setup_test_env() as (exthandlers_handler, protocol, no_of_extensions): + fail_code, fail_action = Actions.generate_unique_fail() + first_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.firstExtension", + install_action=fail_action, supports_multiple_extensions=True) + second_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.secondExtension", + supports_multiple_extensions=True) + third_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.thirdExtension", + supports_multiple_extensions=True) + sc_ext = extension_emulator(name="Microsoft.Powershell.ExampleExtension", install_action=fail_action) + with enable_invocations(first_ext, second_ext, third_ext, sc_ext) as invocation_record: + exthandlers_handler.run() + self.assertEqual(no_of_extensions, + len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), + "incorrect extensions reported") + invocation_record.compare( + (first_ext, ExtensionCommandNames.INSTALL), + # Should try installation again if first time failed + (second_ext, ExtensionCommandNames.INSTALL), + (second_ext, ExtensionCommandNames.ENABLE), + (third_ext, ExtensionCommandNames.ENABLE), + (sc_ext, ExtensionCommandNames.INSTALL) + ) + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=3, status="Ready") + expected_extensions = { + "firstExtension": {"status": ValidHandlerStatus.error, "seq_no": 1, "message": fail_code}, + "secondExtension": {"status": ValidHandlerStatus.success, "seq_no": 2, "message": None}, + "thirdExtension": {"status": ValidHandlerStatus.success, "seq_no": 3, "message": None}, + } + self._assert_extension_status(mc_handlers, expected_extensions, multi_config=True) + + sc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="Microsoft.Powershell.ExampleExtension", + status="NotReady", message=fail_code) + self.assertFalse(all("runtimeSettingsStatus" in handler for handler in sc_handlers), "Incorrect status") + + def test_it_should_only_disable_enabled_extensions_on_update(self): + with self.__setup_generic_test_env() as (exthandlers_handler, protocol, old_exts): + + # Update extensions + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, 'ext_conf_mc_update_extensions.xml') + protocol.mock_wire_data = WireProtocolData(self.test_data) + protocol.mock_wire_data.set_incarnation(2) + protocol.update_goal_state() + + new_version = "1.1.0" + new_first_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.firstExtension", + version=new_version, supports_multiple_extensions=True) + new_second_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.secondExtension", + version=new_version, supports_multiple_extensions=True) + new_third_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.thirdExtension", + version=new_version, supports_multiple_extensions=True) + new_sc_ext = extension_emulator(name="Microsoft.Powershell.ExampleExtension", version=new_version) + with enable_invocations(new_first_ext, new_second_ext, new_third_ext, new_sc_ext, *old_exts) as invocation_record: + exthandlers_handler.run() + old_first, old_second, old_third, old_fourth = old_exts + invocation_record.compare( + # Disable all enabled commands for MC before updating the Handler + (old_first, ExtensionCommandNames.DISABLE), + (old_second, ExtensionCommandNames.DISABLE), + (old_third, ExtensionCommandNames.DISABLE), + (new_first_ext, ExtensionCommandNames.UPDATE), + (old_first, ExtensionCommandNames.UNINSTALL), + (new_first_ext, ExtensionCommandNames.INSTALL), + # No enable for First and Second extension as their state is Disabled in GoalState, + # only enabled the ThirdExtension + (new_third_ext, ExtensionCommandNames.ENABLE), + # Follow the normal update pattern for Single config handlers + (old_fourth, ExtensionCommandNames.DISABLE), + (new_sc_ext, ExtensionCommandNames.UPDATE), + (old_fourth, ExtensionCommandNames.UNINSTALL), + (new_sc_ext, ExtensionCommandNames.INSTALL), + (new_sc_ext, ExtensionCommandNames.ENABLE) + ) + + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=1, handler_version=new_version) + expected_extensions = { + "thirdExtension": {"status": ValidHandlerStatus.success, "seq_no": 99, "message": None} + } + self._assert_extension_status(mc_handlers, expected_extensions, multi_config=True) + self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, handler_version=new_version, + handler_name="Microsoft.Powershell.ExampleExtension") + + def test_it_should_retry_update_sequence_per_extension_if_previous_failed(self): + with self.__setup_generic_test_env() as (exthandlers_handler, protocol, old_exts): + # Update extensions + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, 'ext_conf_mc_update_extensions.xml') + protocol.mock_wire_data = WireProtocolData(self.test_data) + protocol.mock_wire_data.set_incarnation(2) + protocol.update_goal_state() + + new_version = "1.1.0" + _, fail_action = Actions.generate_unique_fail() + # Fail Uninstall of the secondExtension + old_exts[1] = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.secondExtension", + uninstall_action=fail_action, supports_multiple_extensions=True) + # Fail update of the first extension + new_first_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.firstExtension", + version=new_version, update_action=fail_action, + supports_multiple_extensions=True) + new_second_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.secondExtension", + version=new_version, supports_multiple_extensions=True) + new_third_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.thirdExtension", + version=new_version, supports_multiple_extensions=True) + new_sc_ext = extension_emulator(name="Microsoft.Powershell.ExampleExtension", version=new_version) + + with enable_invocations(new_first_ext, new_second_ext, new_third_ext, new_sc_ext, + *old_exts) as invocation_record: + exthandlers_handler.run() + old_first, old_second, old_third, old_fourth = old_exts + invocation_record.compare( + # Disable all enabled commands for MC before updating the Handler + (old_first, ExtensionCommandNames.DISABLE), + (old_second, ExtensionCommandNames.DISABLE), + (old_third, ExtensionCommandNames.DISABLE), + (new_first_ext, ExtensionCommandNames.UPDATE), + # Since the extensions have been disabled before, we won't disable them again for Update scenario + (new_second_ext, ExtensionCommandNames.UPDATE), + # This will fail too as per the mock above + (old_second, ExtensionCommandNames.UNINSTALL), + (new_third_ext, ExtensionCommandNames.UPDATE), + (old_third, ExtensionCommandNames.UNINSTALL), + (new_third_ext, ExtensionCommandNames.INSTALL), + # No enable for First and Second extension as their state is Disabled in GoalState, + # only enabled the ThirdExtension + (new_third_ext, ExtensionCommandNames.ENABLE), + # Follow the normal update pattern for Single config handlers + (old_fourth, ExtensionCommandNames.DISABLE), + (new_sc_ext, ExtensionCommandNames.UPDATE), + (old_fourth, ExtensionCommandNames.UNINSTALL), + (new_sc_ext, ExtensionCommandNames.INSTALL), + (new_sc_ext, ExtensionCommandNames.ENABLE) + ) + + # Since firstExtension and secondExtension are Disabled, we won't report their status + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=1, handler_version=new_version) + expected_extensions = { + "thirdExtension": {"status": ValidHandlerStatus.success, "seq_no": 99, "message": None} + } + self._assert_extension_status(mc_handlers, expected_extensions, multi_config=True) + sc_handler = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_version=new_version, + handler_name="Microsoft.Powershell.ExampleExtension") + expected_extensions = { + "Microsoft.Powershell.ExampleExtension": {"status": ValidHandlerStatus.success, "seq_no": 10, + "message": None} + } + self._assert_extension_status(sc_handler, expected_extensions) + + def test_it_should_report_disabled_extension_errors_if_update_failed(self): + with self.__setup_generic_test_env() as (exthandlers_handler, protocol, old_exts): + # Update extensions + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, 'ext_conf_mc_update_extensions.xml') + protocol.mock_wire_data = WireProtocolData(self.test_data) + protocol.mock_wire_data.set_incarnation(2) + protocol.update_goal_state() + + new_version = "1.1.0" + fail_code, fail_action = Actions.generate_unique_fail() + # Fail Disable of the firstExtension + old_exts[0] = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.firstExtension", + disable_action=fail_action, supports_multiple_extensions=True) + new_first_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.firstExtension", + version=new_version, supports_multiple_extensions=True) + new_second_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.secondExtension", + version=new_version, supports_multiple_extensions=True) + new_third_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.thirdExtension", + version=new_version, supports_multiple_extensions=True) + new_fourth_ext = extension_emulator(name="Microsoft.Powershell.ExampleExtension", version=new_version) + + with enable_invocations(new_first_ext, new_second_ext, new_third_ext, new_fourth_ext, + *old_exts) as invocation_record: + exthandlers_handler.run() + old_first, _, _, old_fourth = old_exts + invocation_record.compare( + # Disable for firstExtension should fail 3 times, i.e., once per extension which tries to update the Handler + (old_first, ExtensionCommandNames.DISABLE), + (old_first, ExtensionCommandNames.DISABLE), + (old_first, ExtensionCommandNames.DISABLE), + # Since Disable fails for the firstExtension and continueOnUpdate = False, Update should not go through + # Follow the normal update pattern for Single config handlers + (old_fourth, ExtensionCommandNames.DISABLE), + (new_fourth_ext, ExtensionCommandNames.UPDATE), + (old_fourth, ExtensionCommandNames.UNINSTALL), + (new_fourth_ext, ExtensionCommandNames.INSTALL), + (new_fourth_ext, ExtensionCommandNames.ENABLE) + ) + + # Since firstExtension and secondExtension are Disabled, we won't report their status + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=1, handler_version=new_version, + status="NotReady", message=fail_code) + expected_extensions = { + "thirdExtension": {"status": ValidHandlerStatus.error, "seq_no": 99, "message": fail_code} + } + self._assert_extension_status(mc_handlers, expected_extensions, multi_config=True) + sc_handler = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_version=new_version, + handler_name="Microsoft.Powershell.ExampleExtension") + expected_extensions = { + "Microsoft.Powershell.ExampleExtension": {"status": ValidHandlerStatus.success, "seq_no": 10, + "message": None} + } + self._assert_extension_status(sc_handler, expected_extensions) + + def test_it_should_report_extension_status_properly(self): + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_multi_config_no_dependencies.xml") + with self._setup_test_env(mock_manifest=True) as (exthandlers_handler, protocol, no_of_extensions): + self.__run_and_assert_generic_case(exthandlers_handler, protocol, no_of_extensions) + + def test_it_should_handle_and_report_enable_errors_properly(self): + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_multi_config_no_dependencies.xml") + with self._setup_test_env() as (exthandlers_handler, protocol, no_of_extensions): + fail_code, fail_action = Actions.generate_unique_fail() + first_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.firstExtension", + supports_multiple_extensions=True) + second_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.secondExtension", + supports_multiple_extensions=True) + third_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.thirdExtension", + enable_action=fail_action, supports_multiple_extensions=True) + fourth_ext = extension_emulator(name="Microsoft.Powershell.ExampleExtension", enable_action=fail_action) + with enable_invocations(first_ext, second_ext, third_ext, fourth_ext) as invocation_record: + exthandlers_handler.run() + self.assertEqual(no_of_extensions, + len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), + "incorrect extensions reported") + invocation_record.compare( + (first_ext, ExtensionCommandNames.INSTALL), + (first_ext, ExtensionCommandNames.ENABLE), + (second_ext, ExtensionCommandNames.ENABLE), + (third_ext, ExtensionCommandNames.ENABLE), + (fourth_ext, ExtensionCommandNames.INSTALL), + (fourth_ext, ExtensionCommandNames.ENABLE) + ) + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=3, status="Ready") + expected_extensions = { + "firstExtension": {"status": ValidHandlerStatus.success, "seq_no": 1, "message": None}, + "secondExtension": {"status": ValidHandlerStatus.success, "seq_no": 2, "message": None}, + "thirdExtension": {"status": ValidHandlerStatus.error, "seq_no": 3, "message": fail_code}, + } + self._assert_extension_status(mc_handlers, expected_extensions, multi_config=True) + + sc_handler = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="Microsoft.Powershell.ExampleExtension", + status="NotReady", message=fail_code) + expected_extensions = { + "Microsoft.Powershell.ExampleExtension": {"status": ValidHandlerStatus.error, "seq_no": 9, + "message": fail_code} + } + self._assert_extension_status(sc_handler, expected_extensions) + + def test_it_should_cleanup_extension_state_on_disable(self): + + def __assert_state_file(handler_name, handler_version, extensions, state, not_present=None): + config_path = os.path.join(self.tmp_dir, "{0}-{1}".format(handler_name, handler_version), "config") + config_files = os.listdir(config_path) + + for ext_name in extensions: + self.assertIn("{0}.settings".format(ext_name), config_files, "settings not found") + self.assertEqual( + fileutil.read_file(os.path.join(config_path, "{0}.HandlerState".format(ext_name.split(".")[0]))), + state, "Invalid state") + + if not_present is not None: + for ext_name in not_present: + self.assertNotIn("{0}.HandlerState".format(ext_name), config_files, "Wrongful state found") + + with self.__setup_generic_test_env() as (ext_handler, protocol, _): + __assert_state_file("OSTCExtensions.ExampleHandlerLinux", "1.0.0", + ["firstExtension.1", "secondExtension.2", "thirdExtension.3"], ExtensionState.Enabled) + + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, 'ext_conf_mc_disabled_extensions.xml') + protocol.mock_wire_data = WireProtocolData(self.test_data) + protocol.mock_wire_data.set_incarnation(2) + protocol.update_goal_state() + + ext_handler.run() + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=2, status="Ready") + expected_extensions = { + "thirdExtension": {"status": ValidHandlerStatus.success, "seq_no": 99, "message": "Enabling thirdExtension"}, + "fourthExtension": {"status": ValidHandlerStatus.success, "seq_no": 101, "message": "Enabling fourthExtension"}, + } + self._assert_extension_status(mc_handlers, expected_extensions, multi_config=True) + __assert_state_file("OSTCExtensions.ExampleHandlerLinux", "1.0.0", + ["thirdExtension.99", "fourthExtension.101"], ExtensionState.Enabled, + not_present=["firstExtension", "secondExtension"]) + + sc_handler = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="Microsoft.Powershell.ExampleExtension") + expected_extensions = { + "Microsoft.Powershell.ExampleExtension": {"status": ValidHandlerStatus.success, "seq_no": 10, + "message": "Enabling SingleConfig Extension"} + } + self._assert_extension_status(sc_handler, expected_extensions) + + def test_it_should_create_command_execution_log_per_extension(self): + with self.__setup_generic_test_env() as (_, _, _): + sc_handler_path = os.path.join(conf.get_ext_log_dir(), "Microsoft.Powershell.ExampleExtension") + mc_handler_path = os.path.join(conf.get_ext_log_dir(), "OSTCExtensions.ExampleHandlerLinux") + self.assertIn("CommandExecution_firstExtension.log", os.listdir(mc_handler_path), + "Command Execution file not found") + self.assertGreater(os.path.getsize(os.path.join(mc_handler_path, "CommandExecution_firstExtension.log")), 0, + "Log file not being used") + self.assertIn("CommandExecution_secondExtension.log", os.listdir(mc_handler_path), + "Command Execution file not found") + self.assertGreater(os.path.getsize(os.path.join(mc_handler_path, "CommandExecution_secondExtension.log")), 0, + "Log file not being used") + self.assertIn("CommandExecution_thirdExtension.log", os.listdir(mc_handler_path), + "Command Execution file not found") + self.assertGreater(os.path.getsize(os.path.join(mc_handler_path, "CommandExecution_thirdExtension.log")), 0, + "Log file not being used") + self.assertIn("CommandExecution.log", os.listdir(sc_handler_path), "Command Execution file not found") + self.assertGreater(os.path.getsize(os.path.join(sc_handler_path, "CommandExecution.log")), 0, + "Log file not being used") + + def test_it_should_set_relevant_environment_variables_for_mc(self): + original_popen = subprocess.Popen + handler_envs = {} + + def __assert_env_variables(handler_name, handler_version="1.0.0", seq_no="1", ext_name=None, expected_vars=None, + not_expected=None): + original_env_vars = { + ExtCommandEnvVariable.ExtensionPath: os.path.join(self.tmp_dir, "{0}-{1}".format(handler_name, handler_version)), + ExtCommandEnvVariable.ExtensionVersion: handler_version, + ExtCommandEnvVariable.ExtensionSeqNumber: ustr(seq_no), + ExtCommandEnvVariable.WireProtocolAddress: '168.63.129.16', + ExtCommandEnvVariable.ExtensionSupportedFeatures: json.dumps([{"Key": "ExtensionTelemetryPipeline", + "Value": "1.0"}]) + + } + + full_name = handler_name + if ext_name is not None: + original_env_vars[ExtCommandEnvVariable.ExtensionName] = ext_name + full_name = "{0}.{1}".format(handler_name, ext_name) + + self.assertIn(full_name, handler_envs, "Handler/ext combo not called") + for commands in handler_envs[full_name]: + expected_environment_variables = original_env_vars.copy() + if expected_vars is not None and commands['command'] in expected_vars: + for name, val in expected_vars[commands['command']].items(): + expected_environment_variables[name] = val + + self.assertTrue(all( + env_var in commands['data'] and env_val == commands['data'][env_var] for env_var, env_val in + expected_environment_variables.items()), + "Incorrect data for environment variable for {0}-{1}, incorrect: {2}".format( + full_name, commands['command'], + [(env_var, env_val) for env_var, env_val in expected_environment_variables.items() if + env_var not in commands['data'] or env_val != commands['data'][env_var]])) + + if not_expected is not None and commands['command'] in not_expected: + self.assertFalse(any(env_var in commands['data'] for env_var in not_expected), "Unwanted env variable found") + + def mock_popen(cmd, *_, **kwargs): + if 'env' in kwargs: + handler_name, __, command = extract_extension_info_from_command(cmd) + name = handler_name + if ExtCommandEnvVariable.ExtensionName in kwargs['env']: + name = "{0}.{1}".format(handler_name, kwargs['env'][ExtCommandEnvVariable.ExtensionName]) + + data = { + "command": command, + "data": kwargs['env'] + } + if name in handler_envs: + handler_envs[name].append(data) + else: + handler_envs[name] = [data] + return original_popen(cmd, *_, **kwargs) + + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_multi_config_no_dependencies.xml") + with self._setup_test_env(mock_manifest=True) as (exthandlers_handler, protocol, no_of_extensions): + with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): + # Case 1: Check normal scenario - Install/Enable + mc_handlers, sc_handler = self.__run_and_assert_generic_case(exthandlers_handler, protocol, + no_of_extensions) + + for handler in mc_handlers: + __assert_env_variables(handler['handlerName'], + ext_name=handler['runtimeSettingsStatus']['extensionName'], + seq_no=handler['runtimeSettingsStatus']['sequenceNumber']) + for handler in sc_handler: + __assert_env_variables(handler['handlerName'], + seq_no=handler['runtimeSettingsStatus']['sequenceNumber']) + + # Case 2: Check Update Scenario + # Clear old test case state + handler_envs = {} + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + 'ext_conf_mc_update_extensions.xml') + protocol.mock_wire_data = WireProtocolData(self.test_data) + protocol.mock_wire_data.set_incarnation(2) + protocol.update_goal_state() + exthandlers_handler.run() + + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=1, handler_version="1.1.0") + expected_extensions = { + "thirdExtension": {"status": ValidHandlerStatus.success, "seq_no": 99, + "message": "Enabling thirdExtension"}, + } + self._assert_extension_status(mc_handlers[:], expected_extensions, multi_config=True) + + sc_handler = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="Microsoft.Powershell.ExampleExtension", + handler_version="1.1.0") + expected_extensions = { + "Microsoft.Powershell.ExampleExtension": {"status": ValidHandlerStatus.success, "seq_no": 10, + "message": "Enabling SingleConfig extension"} + } + self._assert_extension_status(sc_handler[:], expected_extensions) + + for handler in mc_handlers: + __assert_env_variables(handler['handlerName'], + handler_version="1.1.0", + ext_name=handler['runtimeSettingsStatus']['extensionName'], + seq_no=handler['runtimeSettingsStatus']['sequenceNumber'], + expected_vars={ + "disable": { + ExtCommandEnvVariable.ExtensionPath: os.path.join(self.tmp_dir, "{0}-{1}".format(handler['handlerName'], "1.0.0")), + ExtCommandEnvVariable.ExtensionVersion: '1.0.0' + }}) + + # Assert the environment variables were present even for disabled/uninstalled commands + first_ext_expected_vars = { + "disable": { + ExtCommandEnvVariable.ExtensionPath: os.path.join(self.tmp_dir, "{0}-{1}".format(handler['handlerName'], "1.0.0")), + ExtCommandEnvVariable.ExtensionVersion: '1.0.0' + }, + "uninstall": { + ExtCommandEnvVariable.ExtensionPath: os.path.join(self.tmp_dir, "{0}-{1}".format(handler['handlerName'], "1.0.0")), + ExtCommandEnvVariable.ExtensionVersion: '1.0.0' + }, + "update": { + ExtCommandEnvVariable.UpdatingFromVersion: "1.0.0", + ExtCommandEnvVariable.DisableReturnCodeMultipleExtensions: + json.dumps([ + {"extensionName": "firstExtension", "exitCode": "0"}, + {"extensionName": "secondExtension", "exitCode": "0"}, + {"extensionName": "thirdExtension", "exitCode": "0"} + ]) + } + } + __assert_env_variables(handler['handlerName'], ext_name="firstExtension", + expected_vars=first_ext_expected_vars, handler_version="1.1.0", seq_no="1", + not_expected={ + "update": [ExtCommandEnvVariable.DisableReturnCode] + }) + __assert_env_variables(handler['handlerName'], ext_name="secondExtension", seq_no="2") + + for handler in sc_handler: + sc_expected_vars = { + "disable": { + ExtCommandEnvVariable.ExtensionPath: os.path.join(self.tmp_dir, "{0}-{1}".format(handler['handlerName'], "1.0.0")), + ExtCommandEnvVariable.ExtensionVersion: '1.0.0' + }, + "uninstall": { + ExtCommandEnvVariable.ExtensionPath: os.path.join(self.tmp_dir, "{0}-{1}".format(handler['handlerName'], "1.0.0")), + ExtCommandEnvVariable.ExtensionVersion: '1.0.0' + }, + "update": { + ExtCommandEnvVariable.UpdatingFromVersion: "1.0.0", + ExtCommandEnvVariable.DisableReturnCode: "0" + } + } + __assert_env_variables(handler['handlerName'], handler_version="1.1.0", + seq_no=handler['runtimeSettingsStatus']['sequenceNumber'], + expected_vars=sc_expected_vars, not_expected={ + "update": [ExtCommandEnvVariable.DisableReturnCodeMultipleExtensions] + }) + + def test_it_should_ignore_disable_errors_for_multi_config_extensions(self): + fail_code, fail_action = Actions.generate_unique_fail() + + with self.__setup_generic_test_env() as (exthandlers_handler, protocol, exts): + + # Fail disable of 1st and 2nd extension + exts[0] = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.firstExtension", + disable_action=fail_action) + exts[1] = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.secondExtension", + disable_action=fail_action) + fourth_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.fourthExtension") + + with patch.object(ExtHandlerInstance, "report_event", autospec=True) as patch_report_event: + with enable_invocations(fourth_ext, *exts) as invocation_record: + # Assert even though 2 extensions are failing, we clean their state up properly and enable the + # remaining extensions + self.__setup_and_assert_disable_scenario(exthandlers_handler, protocol) + first_ext, second_ext, third_ext, sc_ext = exts + invocation_record.compare( + (first_ext, ExtensionCommandNames.DISABLE), + (second_ext, ExtensionCommandNames.DISABLE), + (third_ext, ExtensionCommandNames.ENABLE), + (fourth_ext, ExtensionCommandNames.ENABLE), + (sc_ext, ExtensionCommandNames.ENABLE) + ) + + self.assertTrue(all( + fail_code in kwargs['message'] for args, kwargs in patch_report_event.call_args_list if + kwargs['name'] == first_ext.name), "Error not reported") + self.assertTrue(all( + fail_code in kwargs['message'] for args, kwargs in patch_report_event.call_args_list if + kwargs['name'] == second_ext.name), "Error not reported") + # Make sure fail code is not reported for any other extension + self.assertFalse(all( + fail_code in kwargs['message'] for args, kwargs in patch_report_event.call_args_list if + kwargs['name'] == third_ext.name), "Error not reported") + + def test_it_should_always_create_placeholder_for_all_extensions(self): + original_popen = subprocess.Popen + handler_statuses = {} + + def __assert_status_file_in_handlers(): + for handler in mc_handlers: + file_name = "{0}.{1}.status".format(handler['runtimeSettingsStatus']['extensionName'], + handler['runtimeSettingsStatus']['sequenceNumber']) + __assert_status_file(handler['handlerName'], status_file=file_name) + for handler in sc_handler: + file_name = "{0}.status".format(handler['runtimeSettingsStatus']['sequenceNumber']) + __assert_status_file(handler['handlerName'], status_file=file_name) + + def __assert_status_file(handler_name, status_file): + status = handler_statuses["{0}.{1}.enable".format(handler_name, status_file)] + self.assertIsNotNone(status, "No status found") + # Assert the format of the placeholder is correct + ext_status = ExtensionStatus() + # If the format is wrong or unexpected, this would throw and fail the test + parse_ext_status(ext_status, status) + self.assertIn(ext_status.status, [ValidHandlerStatus.success, ValidHandlerStatus.transitioning], + "Incorrect status") + + def mock_popen(cmd, *_, **kwargs): + if 'env' in kwargs: + handler_name, handler_version, command_name = extract_extension_info_from_command(cmd) + ext_name = None + if ExtCommandEnvVariable.ExtensionName in kwargs['env']: + ext_name = kwargs['env'][ExtCommandEnvVariable.ExtensionName] + seq_no = kwargs['env'][ExtCommandEnvVariable.ExtensionSeqNumber] + status_file_name = "{0}.status".format(seq_no) + status_file_name = "{0}.{1}".format(ext_name, status_file_name) if ext_name is not None else status_file_name + status_file = os.path.join(self.tmp_dir, "{0}-{1}".format(handler_name, handler_version), "status", status_file_name) + contents = None + if os.path.exists(status_file): + contents = json.loads(fileutil.read_file(status_file)) + handler_statuses["{0}.{1}.{2}".format(handler_name, status_file_name, command_name)] = contents + + return original_popen(cmd, *_, **kwargs) + + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_multi_config_no_dependencies.xml") + with self._setup_test_env(mock_manifest=True) as (exthandlers_handler, protocol, no_of_extensions): + with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): + mc_handlers, sc_handler = self.__run_and_assert_generic_case(exthandlers_handler, protocol, + no_of_extensions) + + # Ensure we dont create a placeholder for Install command + self.assertTrue( + all(handler_statuses[status] is None for status in handler_statuses if "install" in status), + "Incorrect status file found for install") + + # Ensure we create a valid status file for Enable + # Note: As part of our test, the sample-ext creates a status file after install due to which a placeholder + # is not created. We will verify a valid status file exists for all extensions instead since that's the + # main scenario. + __assert_status_file_in_handlers() + + # Update GS, remove 2 extensions and add 3 + mc_handlers, sc_handler = self.__setup_and_assert_disable_scenario(exthandlers_handler, protocol) + __assert_status_file_in_handlers() + + def test_it_should_report_status_correctly_for_unsupported_goal_state(self): + with self.__setup_generic_test_env() as (exthandlers_handler, protocol, _): + + # Update GS with an ExtensionConfig with 3 Required features to force GA to mark it as unsupported + self.test_data['ext_conf'] = "wire/ext_conf_required_features.xml" + protocol.mock_wire_data = WireProtocolData(self.test_data) + protocol.mock_wire_data.set_incarnation(2) + protocol.update_goal_state() + # Assert the extension status is the same as we reported for Incarnation 1. + self.__run_and_assert_generic_case(exthandlers_handler, protocol, no_of_extensions=4, with_message=False) + + # Assert the GS was reported as unsupported + gs_aggregate_status = protocol.aggregate_status['aggregateStatus']['vmArtifactsAggregateStatus'][ + 'goalStateAggregateStatus'] + self.assertEqual(gs_aggregate_status['status'], GoalStateStatus.Failed, "Incorrect status") + self.assertEqual(gs_aggregate_status['code'], + GoalStateAggregateStatusCodes.GoalStateUnsupportedRequiredFeatures, "Incorrect code") + self.assertEqual(gs_aggregate_status['inSvdSeqNo'], '2', "Incorrect incarnation reported") + self.assertEqual(gs_aggregate_status['formattedMessage']['message'], + 'Failing GS incarnation: 2 as Unsupported features found: TestRequiredFeature1, TestRequiredFeature2, TestRequiredFeature3', + "Incorrect error message reported") + + def test_it_should_fail_handler_if_handler_does_not_support_mc(self): + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_multi_config_no_dependencies.xml") + + first_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.firstExtension") + second_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.secondExtension") + third_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.thirdExtension") + fourth_ext = extension_emulator(name="Microsoft.Powershell.ExampleExtension") + + with self._setup_test_env() as (exthandlers_handler, protocol, no_of_extensions): + with enable_invocations(first_ext, second_ext, third_ext, fourth_ext) as invocation_record: + exthandlers_handler.run() + self.assertEqual(no_of_extensions, + len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), + "incorrect extensions reported") + + invocation_record.compare( + # Since we raise a ConfigError, we shouldn't process any of the MC extensions at all + (fourth_ext, ExtensionCommandNames.INSTALL), + (fourth_ext, ExtensionCommandNames.ENABLE) + ) + + err_msg = 'Handler OSTCExtensions.ExampleHandlerLinux does not support MultiConfig but CRP expects it, failing due to inconsistent data' + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=3, status="NotReady", message=err_msg) + expected_extensions = { + "firstExtension": {"status": ValidHandlerStatus.error, "seq_no": 1, "message": err_msg}, + "secondExtension": {"status": ValidHandlerStatus.error, "seq_no": 2, "message": err_msg}, + "thirdExtension": {"status": ValidHandlerStatus.error, "seq_no": 3, "message": err_msg}, + } + self._assert_extension_status(mc_handlers[:], expected_extensions, multi_config=True) + + sc_handler = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="Microsoft.Powershell.ExampleExtension") + expected_extensions = { + "Microsoft.Powershell.ExampleExtension": {"status": ValidHandlerStatus.success, "seq_no": 9} + } + self._assert_extension_status(sc_handler[:], expected_extensions) + + def test_it_should_check_every_time_if_handler_supports_mc(self): + with self.__setup_generic_test_env() as (exthandlers_handler, protocol, old_exts): + + protocol.mock_wire_data.set_incarnation(2) + protocol.update_goal_state() + + # Mock manifest to not support multiple extensions + with patch('azurelinuxagent.ga.exthandlers.HandlerManifest.supports_multiple_extensions', return_value=False): + with enable_invocations(*old_exts) as invocation_record: + (_, _, _, fourth_ext) = old_exts + exthandlers_handler.run() + self.assertEqual(4, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), + "incorrect extensions reported") + + invocation_record.compare( + # Since we raise a ConfigError, we shouldn't process any of the MC extensions at all + (fourth_ext, ExtensionCommandNames.ENABLE) + ) + + err_msg = 'Handler OSTCExtensions.ExampleHandlerLinux does not support MultiConfig but CRP expects it, failing due to inconsistent data' + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=3, status="NotReady", message=err_msg) + + # Since the extensions were not even executed, their status file should reflect the last status + # (Handler status above should always report the error though) + expected_extensions = { + "firstExtension": {"status": ValidHandlerStatus.success, "seq_no": 1}, + "secondExtension": {"status": ValidHandlerStatus.success, "seq_no": 2}, + "thirdExtension": {"status": ValidHandlerStatus.success, "seq_no": 3}, + } + self._assert_extension_status(mc_handlers[:], expected_extensions, multi_config=True) + + sc_handler = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="Microsoft.Powershell.ExampleExtension") + expected_extensions = { + "Microsoft.Powershell.ExampleExtension": {"status": ValidHandlerStatus.success, "seq_no": 9} + } + self._assert_extension_status(sc_handler[:], expected_extensions) + + +class TestMultiConfigExtensionSequencing(_MultiConfigBaseTestClass): + + @contextlib.contextmanager + def __setup_test_and_get_exts(self): + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_with_multi_config_dependencies.xml") + + first_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.firstExtension", supports_multiple_extensions=True) + second_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.secondExtension", supports_multiple_extensions=True) + third_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.thirdExtension", supports_multiple_extensions=True) + dependent_sc_ext = extension_emulator(name="Microsoft.Powershell.ExampleExtension") + independent_sc_ext = extension_emulator(name="Microsoft.Azure.Geneva.GenevaMonitoring", version="1.1.0") + + with self._setup_test_env() as (exthandlers_handler, protocol, no_of_extensions): + yield exthandlers_handler, protocol, no_of_extensions, first_ext, second_ext, third_ext, dependent_sc_ext, independent_sc_ext + + def test_it_should_process_dependency_chain_extensions_properly(self): + with self.__setup_test_and_get_exts() as ( + exthandlers_handler, protocol, no_of_extensions, first_ext, second_ext, third_ext, dependent_sc_ext, + independent_sc_ext): + with enable_invocations(first_ext, second_ext, third_ext, dependent_sc_ext, independent_sc_ext) as invocation_record: + exthandlers_handler.run() + self.assertEqual(no_of_extensions, + len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), + "incorrect extensions reported") + invocation_record.compare( + (first_ext, ExtensionCommandNames.INSTALL), + (first_ext, ExtensionCommandNames.ENABLE), + (independent_sc_ext, ExtensionCommandNames.INSTALL), + (independent_sc_ext, ExtensionCommandNames.ENABLE), + (dependent_sc_ext, ExtensionCommandNames.INSTALL), + (dependent_sc_ext, ExtensionCommandNames.ENABLE), + (second_ext, ExtensionCommandNames.ENABLE), + (third_ext, ExtensionCommandNames.ENABLE) + ) + + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=3) + expected_extensions = { + "firstExtension": {"status": ValidHandlerStatus.success, "seq_no": 2}, + "secondExtension": {"status": ValidHandlerStatus.success, "seq_no": 2}, + "thirdExtension": {"status": ValidHandlerStatus.success, "seq_no": 1}, + } + self._assert_extension_status(mc_handlers[:], expected_extensions, multi_config=True) + + sc_dependent_handler = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="Microsoft.Powershell.ExampleExtension") + expected_extensions = { + "Microsoft.Powershell.ExampleExtension": {"status": ValidHandlerStatus.success, "seq_no": 2} + } + self._assert_extension_status(sc_dependent_handler[:], expected_extensions) + sc_independent_handler = self._assert_and_get_handler_status( + aggregate_status=protocol.aggregate_status, handler_name="Microsoft.Azure.Geneva.GenevaMonitoring", + handler_version="1.1.0") + expected_extensions = { + "Microsoft.Azure.Geneva.GenevaMonitoring": {"status": ValidHandlerStatus.success, "seq_no": 1} + } + self._assert_extension_status(sc_independent_handler[:], expected_extensions) + + def __assert_invalid_status_scenario(self, protocol, fail_code, mc_status="NotReady", + mc_message="Plugin installed but not enabled", err_msg=None): + mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_count=3, status=mc_status, + message=mc_message) + + expected_extensions = { + "firstExtension": {"status": ValidHandlerStatus.error, "seq_no": 2, "message": fail_code}, + "secondExtension": {"status": ValidHandlerStatus.error, "seq_no": 2, "message": err_msg}, + "thirdExtension": {"status": ValidHandlerStatus.error, "seq_no": 1, "message": err_msg}, + } + self._assert_extension_status(mc_handlers[:], expected_extensions, multi_config=True) + + sc_dependent_handler = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, + handler_name="Microsoft.Powershell.ExampleExtension", + status="NotReady", message=err_msg) + self.assertTrue(all('runtimeSettingsStatus' not in handler for handler in sc_dependent_handler)) + + sc_independent_handler = self._assert_and_get_handler_status( + aggregate_status=protocol.aggregate_status, handler_name="Microsoft.Azure.Geneva.GenevaMonitoring", + handler_version="1.1.0", status="NotReady", message=err_msg) + self.assertTrue(all('runtimeSettingsStatus' not in handler for handler in sc_independent_handler)) + + def test_it_should_report_extension_status_failures_for_all_dependent_extensions(self): + with self.__setup_test_and_get_exts() as ( + exthandlers_handler, protocol, no_of_extensions, first_ext, second_ext, third_ext, dependent_sc_ext, + independent_sc_ext): + + # Fail the enable for firstExtension. + fail_code, fail_action = Actions.generate_unique_fail() + first_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux.firstExtension", + enable_action=fail_action, supports_multiple_extensions=True) + + with enable_invocations(first_ext, second_ext, third_ext, dependent_sc_ext, + independent_sc_ext) as invocation_record: + exthandlers_handler.run() + self.assertEqual(no_of_extensions, + len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), + "incorrect extensions reported") + + # Since firstExtension is high up on the dependency chain, no other extensions should be executed + invocation_record.compare( + (first_ext, ExtensionCommandNames.INSTALL), + (first_ext, ExtensionCommandNames.ENABLE) + ) + + err_msg = 'Skipping processing of extensions since execution of dependent extension OSTCExtensions.ExampleHandlerLinux.firstExtension failed' + self.__assert_invalid_status_scenario(protocol, fail_code, err_msg=err_msg) + + def test_it_should_stop_execution_if_status_file_contains_errors(self): + # This test tests the scenario where the extensions exit with a success exit code but fail subsequently with an + # error in the status file + self.test_data['ext_conf'] = os.path.join(self._MULTI_CONFIG_TEST_DATA, + "ext_conf_with_multi_config_dependencies.xml") + + original_popen = subprocess.Popen + invocation_records = [] + fail_code = str(uuid.uuid4()) + + def mock_popen(cmd, *_, **kwargs): + + try: + handler_name, handler_version, command_name = extract_extension_info_from_command(cmd) + except ValueError: + return original_popen(cmd, *_, **kwargs) + + if 'env' in kwargs: + env = kwargs['env'] + if ExtCommandEnvVariable.ExtensionName in env: + full_name = "{0}.{1}".format(handler_name, env[ExtCommandEnvVariable.ExtensionName]) + status_file = "{0}.{1}.status".format(env[ExtCommandEnvVariable.ExtensionName], + env[ExtCommandEnvVariable.ExtensionSeqNumber]) + + status_contents = [{"status": {"status": ValidHandlerStatus.error, "code": fail_code, + "formattedMessage": {"message": fail_code, "lang": "en-US"}}}] + fileutil.write_file(os.path.join(env[ExtCommandEnvVariable.ExtensionPath], "status", status_file), + json.dumps(status_contents)) + + invocation_records.append((full_name, handler_version, command_name)) + # The return code is 0 but the status file should have the error, this it to test the scenario + # where the extensions return a success code but fail later. + return original_popen(['echo', "works"], *_, **kwargs) + + invocation_records.append((handler_name, handler_version, command_name)) + return original_popen(cmd, *_, **kwargs) + + with self._setup_test_env(mock_manifest=True) as (exthandlers_handler, protocol, no_of_extensions): + with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): + exthandlers_handler.run() + + self.assertEqual(no_of_extensions, + len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), + "incorrect extensions reported") + + # Since we're writing error status for firstExtension, only the firstExtension should be invoked and + # everything else should be skipped + expected_invocations = [ + ('OSTCExtensions.ExampleHandlerLinux.firstExtension', '1.0.0', ExtensionCommandNames.INSTALL), + ('OSTCExtensions.ExampleHandlerLinux.firstExtension', '1.0.0', ExtensionCommandNames.ENABLE)] + self.assertEqual(invocation_records, expected_invocations, "Invalid invocations found") + + err_msg = 'Dependent Extension OSTCExtensions.ExampleHandlerLinux.firstExtension did not succeed. Status was error' + self.__assert_invalid_status_scenario(protocol, fail_code, mc_status="Ready", + mc_message="Plugin enabled", err_msg=err_msg) diff --git a/tests/protocol/mockwiredata.py b/tests/protocol/mockwiredata.py index dc453cd23..c36867e54 100644 --- a/tests/protocol/mockwiredata.py +++ b/tests/protocol/mockwiredata.py @@ -298,6 +298,14 @@ def get_no_of_plugins_in_extension_config(self): plugins_list = find(ext_config_doc, "Plugins") return len(findall(plugins_list, "Plugin")) + def get_no_of_extensions_in_config(self): + if self.ext_conf is None: + return 0 + ext_config_doc = parse_doc(self.ext_conf) + plugin_settings = find(ext_config_doc, "PluginSettings") + return len(findall(plugin_settings, "ExtensionRuntimeSettings")) + len( + findall(plugin_settings, "RuntimeSettings")) + # # Having trouble reading the regular expressions below? you are not alone! # From 991924dc87704948bcb01a98245bc316f2ee30c8 Mon Sep 17 00:00:00 2001 From: Long Li Date: Wed, 12 May 2021 09:36:50 -0700 Subject: [PATCH 09/35] Match IPoIB interface with any alphanumeric characters (#2239) --- azurelinuxagent/common/rdma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/rdma.py b/azurelinuxagent/common/rdma.py index c6d7e78ce..299b1a8a5 100644 --- a/azurelinuxagent/common/rdma.py +++ b/azurelinuxagent/common/rdma.py @@ -369,7 +369,7 @@ def update_iboip_interfaces(self, mac_ip_array): for nic in nics: # look for IBoIP interface of format ibXXX - if not re.match(r"ib\d+", nic): + if not re.match(r"ib\w+", nic): continue mac_addr = None From 290c7ae5f223f49e53af765eb6c6b5e44705a5f6 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Wed, 12 May 2021 17:43:59 -0700 Subject: [PATCH 10/35] Improved logging for MC (#2246) --- azurelinuxagent/ga/exthandlers.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 59fcc4c59..b8d027ae7 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -729,8 +729,7 @@ def __handle_extension(ext_handler_i, extension, uninstall_exit_code): return # MultiConfig: Handle extension level ops here - ext_handler_i.logger.info("{0} requested state: {1}", ext_handler_i.get_extension_full_name(extension), - extension.state) + ext_handler_i.logger.info("Requested extension state: {0}", extension.state) if extension.state == ExtensionState.Enabled: ext_handler_i.enable(extension, uninstall_exit_code=uninstall_exit_code) @@ -741,8 +740,7 @@ def __handle_extension(ext_handler_i, extension, uninstall_exit_code): # tantamount to uninstalling Handler so ignoring errors incase of Disable failure and deleting state. ext_handler_i.disable(extension, ignore_error=True) else: - ext_handler_i.logger.info( - "{0} already disabled, not doing anything".format(ext_handler_i.get_extension_full_name(extension))) + ext_handler_i.logger.info("Extension already disabled, not doing anything") else: raise ExtensionConfigError( "Unknown requested state for Extension {0}: {1}".format(extension.name, extension.state)) @@ -1170,7 +1168,7 @@ def decide_version(self, target_state=None, extension=None): return self.pkg def set_logger(self, execution_log_max_size=(10 * 1024 * 1024), extension=None): - prefix = "[{0}]".format(self.get_full_name()) + prefix = "[{0}]".format(self.get_full_name(extension)) self.logger = logger.Logger(logger.DEFAULT_LOGGER, prefix) self.__set_command_execution_log(extension, execution_log_max_size) @@ -1369,7 +1367,7 @@ def create_placeholder_status_file(self, extension=None, status=ValidHandlerStat _, status_path = self.get_status_file_path(extension) if status_path is not None and not os.path.exists(status_path): now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - status = [ + status_contents = [ { "version": 1.0, "timestampUTC": now, @@ -1389,7 +1387,8 @@ def create_placeholder_status_file(self, extension=None, status=ValidHandlerStat # initializing the directories (ExtensionConfigError, Version deleted from PIR error, etc) if not os.path.exists(os.path.dirname(status_path)): fileutil.mkdir(os.path.dirname(status_path), mode=0o700) - fileutil.write_file(status_path, json.dumps(status)) + self.logger.info("Creating a placeholder status file {0} with status: {1}".format(status_path, status)) + fileutil.write_file(status_path, json.dumps(status_contents)) def enable(self, extension=None, uninstall_exit_code=None): try: @@ -1415,7 +1414,7 @@ def _enable_extension(self, extension, uninstall_exit_code): self.set_operation(WALAEventOperation.Enable) man = self.load_manifest() enable_cmd = man.get_enable_command() - self.logger.info("Enable extension {0}: [{1}]".format(self.get_extension_full_name(extension), enable_cmd)) + self.logger.info("Enable extension: [{0}]".format(enable_cmd)) self.launch_command(enable_cmd, timeout=300, extension_error_code=ExtensionErrorCodes.PluginEnableProcessingFailed, env=env, extension=extension) @@ -1428,7 +1427,7 @@ def _disable_extension(self, extension=None): self.set_operation(WALAEventOperation.Disable) man = self.load_manifest() disable_cmd = man.get_disable_command() - self.logger.info("Disable extension {0}: [{1}]".format(self.get_extension_full_name(extension), disable_cmd)) + self.logger.info("Disable extension: [{0}]".format(disable_cmd)) self.launch_command(disable_cmd, timeout=900, extension_error_code=ExtensionErrorCodes.PluginDisableProcessingFailed, extension=extension) @@ -1440,8 +1439,7 @@ def disable(self, extension=None, ignore_error=False): if not ignore_error: raise - msg = "[Ignored Error] Ran into error disabling extension {0}:{1}".format( - self.get_extension_full_name(extension), ustr(error)) + msg = "[Ignored Error] Ran into error disabling extension:{0}".format(ustr(error)) self.logger.info(msg) self.report_event(name=self.get_extension_full_name(extension), message=msg, is_success=False, log_event=False) @@ -1896,7 +1894,7 @@ def launch_command(self, cmd, timeout=300, extension_error_code=ExtensionErrorCo duration = elapsed_milliseconds(begin_utc) ext_name = self.get_extension_full_name(extension) - log_msg = "Extension: {0}; Command: {1}\n{2}".format(ext_name, cmd, "\n".join( + log_msg = "Command: {0}\n{1}".format(cmd, "\n".join( [line for line in process_output.split('\n') if line != ""])) self.logger.info(log_msg) self.report_event(name=ext_name, message=log_msg, duration=duration, log_event=False) @@ -2014,6 +2012,7 @@ def __get_state(self, name, default=None): return default def __remove_extension_state_files(self, extension): + self.logger.info("Removing states files for disabled extension: {0}".format(extension.name)) try: # MultiConfig: Remove all config/.*.settings, status/.*.status and config/.HandlerState files files_to_delete = [ From f4849c601a8c6e4d7415d2d4d3e0a2c2edd10f0b Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 18 May 2021 09:33:01 -0700 Subject: [PATCH 11/35] Merge release-2.2.54 into develop (#2249) * Fix for invalid gs createdOnTicks values (#2210) * Increase Agent Version to 2.2.54.2 (#2212) Co-authored-by: Kevin Clark --- azurelinuxagent/common/version.py | 4 +-- azurelinuxagent/ga/exthandlers.py | 26 +++++++++++++---- .../wire/ext_conf_invalid_vm_metadata.xml | 29 +++++++++++++++++++ tests/ga/test_extension.py | 12 +++++++- tests/protocol/mockwiredata.py | 3 ++ 5 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 tests/data/wire/ext_conf_invalid_vm_metadata.xml diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 45f2a0f06..399dc2ca1 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -196,9 +196,9 @@ def has_logrotate(): AGENT_NAME = "WALinuxAgent" AGENT_LONG_NAME = "Azure Linux Agent" -# Setting the version to 9.9.9.9 to ensure DCR always uses this version and never auto-updates. +# Setting the version to 9.9.9.9 for testing purposes. # Replace this with the actual agent version on release. -# Current Agent Version = 2.2.54.1 +# Current Agent Version = 2.2.54.2 AGENT_VERSION = '9.9.9.9' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index b8d027ae7..c340fea35 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -300,15 +300,29 @@ def get_goal_state_debug_metadata(self): :return: Tuple (activity_id, correlation_id, gs_created_timestamp) or "NA" for any property that's not available """ - def check_empty(value): return value if value not in (None, "") else "NA" + def format_value(parse_fn, value): + + try: + if value not in (None, ""): + return parse_fn(value) + except Exception as e: + # A failure here isn't a fatal error, because the info we're + # trying to retrieve is debug only on linux. + error_msg = u"Couldn't parse debug metadata value: {0}".format(e) + logger.verbose(error_msg) + + return "NA" + + to_utc = lambda time: time.strftime(logger.Logger.LogTimeFormatInUTC) + identity = lambda value: value in_vm_gs_metadata = self.protocol.get_in_vm_gs_metadata() - gs_creation_time = check_empty(in_vm_gs_metadata.created_on_ticks) - gs_creation_time = gs_creation_time.strftime( - logger.Logger.LogTimeFormatInUTC) if gs_creation_time != "NA" else gs_creation_time - return check_empty(in_vm_gs_metadata.activity_id), check_empty( - in_vm_gs_metadata.correlation_id), gs_creation_time + gs_creation_time = format_value(to_utc, in_vm_gs_metadata.created_on_ticks) + activity_id = format_value(identity, in_vm_gs_metadata.activity_id) + correlation_id = format_value(identity, in_vm_gs_metadata.correlation_id) + + return activity_id, correlation_id, gs_creation_time def run(self): diff --git a/tests/data/wire/ext_conf_invalid_vm_metadata.xml b/tests/data/wire/ext_conf_invalid_vm_metadata.xml new file mode 100644 index 000000000..3c407021e --- /dev/null +++ b/tests/data/wire/ext_conf_invalid_vm_metadata.xml @@ -0,0 +1,29 @@ + + + + + + Prod + + http://mock-goal-state/manifest_of_ga.xml + + + + Test + + http://mock-goal-state/manifest_of_ga.xml + + + + + + + + + + {"runtimeSettings":[{"handlerSettings":{"protectedSettingsCertThumbprint":"4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3","protectedSettings":"MIICWgYJK","publicSettings":{"foo":"bar"}}}]} + + + https://test.blob.core.windows.net/vhds/test-cs12.test-cs12.test-cs12.status?sr=b&sp=rw&se=9999-01-01&sk=key1&sv=2014-02-14&sig=hfRh7gzUE7sUtYwke78IOlZOrTRCYvkec4hGZ9zZzXo + + \ No newline at end of file diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index 037a21171..22de84faa 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -1480,7 +1480,7 @@ def test_last_etag_on_extension_processing(self, *args): self.assertEqual(etag, exthandlers_handler.last_etag, "Last etag and etag should be same if extension processing is enabled") - def test_it_should_parse_in_vm_metadata_properly(self, mock_get, mock_crypt, *args): + def test_it_should_parse_valid_in_vm_metadata_properly(self, mock_get, mock_crypt, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_IN_VM_META_DATA) exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) @@ -1492,6 +1492,8 @@ def test_it_should_parse_in_vm_metadata_properly(self, mock_get, mock_crypt, *ar self.assertEqual(correlation_id, "400de90b-522e-491f-9d89-ec944661f531", "Incorrect correlation Id") self.assertEqual(gs_creation_time, '2020-11-09T17:48:50.412125Z', "Incorrect GS Creation time") + def test_it_should_process_goal_state_even_if_metadata_missing(self, mock_get, mock_crypt, *args): + # If the data is not provided in ExtensionConfig, it should just be None test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) @@ -1502,6 +1504,14 @@ def test_it_should_parse_in_vm_metadata_properly(self, mock_get, mock_crypt, *ar self.assertEqual(activity_id, "NA", "Activity Id should be NA") self.assertEqual(correlation_id, "NA", "Correlation Id should be NA") self.assertEqual(gs_creation_time, "NA", "GS Creation time should be NA") + + def test_it_should_process_goal_state_even_if_metadata_invalid(self, mock_get, mock_crypt, *args): + + test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_INVALID_VM_META_DATA) + exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) + + exthandlers_handler.run() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") def _assert_ext_status(self, vm_agent_status, expected_status, expected_seq_no, expected_handler_name="OSTCExtensions.ExampleHandlerLinux", expected_msg=None): diff --git a/tests/protocol/mockwiredata.py b/tests/protocol/mockwiredata.py index c36867e54..337b678d7 100644 --- a/tests/protocol/mockwiredata.py +++ b/tests/protocol/mockwiredata.py @@ -46,6 +46,9 @@ DATA_FILE_IN_VM_META_DATA = DATA_FILE.copy() DATA_FILE_IN_VM_META_DATA["ext_conf"] = "wire/ext_conf_in_vm_metadata.xml" +DATA_FILE_INVALID_VM_META_DATA = DATA_FILE.copy() +DATA_FILE_INVALID_VM_META_DATA["ext_conf"] = "wire/ext_conf_invalid_vm_metadata.xml" + DATA_FILE_NO_EXT = DATA_FILE.copy() DATA_FILE_NO_EXT["goal_state"] = "wire/goal_state_no_ext.xml" DATA_FILE_NO_EXT["ext_conf"] = None From 82be0ff2109df951b436eb492de57cb4a1472e64 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 18 May 2021 09:50:38 -0700 Subject: [PATCH 12/35] Revert "Merge release-2.2.54 into develop (#2249)" (#2251) This reverts commit f4849c601a8c6e4d7415d2d4d3e0a2c2edd10f0b. --- azurelinuxagent/common/version.py | 4 +-- azurelinuxagent/ga/exthandlers.py | 26 ++++------------- .../wire/ext_conf_invalid_vm_metadata.xml | 29 ------------------- tests/ga/test_extension.py | 12 +------- tests/protocol/mockwiredata.py | 3 -- 5 files changed, 9 insertions(+), 65 deletions(-) delete mode 100644 tests/data/wire/ext_conf_invalid_vm_metadata.xml diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 399dc2ca1..45f2a0f06 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -196,9 +196,9 @@ def has_logrotate(): AGENT_NAME = "WALinuxAgent" AGENT_LONG_NAME = "Azure Linux Agent" -# Setting the version to 9.9.9.9 for testing purposes. +# Setting the version to 9.9.9.9 to ensure DCR always uses this version and never auto-updates. # Replace this with the actual agent version on release. -# Current Agent Version = 2.2.54.2 +# Current Agent Version = 2.2.54.1 AGENT_VERSION = '9.9.9.9' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index c340fea35..b8d027ae7 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -300,29 +300,15 @@ def get_goal_state_debug_metadata(self): :return: Tuple (activity_id, correlation_id, gs_created_timestamp) or "NA" for any property that's not available """ - def format_value(parse_fn, value): - - try: - if value not in (None, ""): - return parse_fn(value) - except Exception as e: - # A failure here isn't a fatal error, because the info we're - # trying to retrieve is debug only on linux. - error_msg = u"Couldn't parse debug metadata value: {0}".format(e) - logger.verbose(error_msg) - - return "NA" - - to_utc = lambda time: time.strftime(logger.Logger.LogTimeFormatInUTC) - identity = lambda value: value + def check_empty(value): return value if value not in (None, "") else "NA" in_vm_gs_metadata = self.protocol.get_in_vm_gs_metadata() + gs_creation_time = check_empty(in_vm_gs_metadata.created_on_ticks) + gs_creation_time = gs_creation_time.strftime( + logger.Logger.LogTimeFormatInUTC) if gs_creation_time != "NA" else gs_creation_time - gs_creation_time = format_value(to_utc, in_vm_gs_metadata.created_on_ticks) - activity_id = format_value(identity, in_vm_gs_metadata.activity_id) - correlation_id = format_value(identity, in_vm_gs_metadata.correlation_id) - - return activity_id, correlation_id, gs_creation_time + return check_empty(in_vm_gs_metadata.activity_id), check_empty( + in_vm_gs_metadata.correlation_id), gs_creation_time def run(self): diff --git a/tests/data/wire/ext_conf_invalid_vm_metadata.xml b/tests/data/wire/ext_conf_invalid_vm_metadata.xml deleted file mode 100644 index 3c407021e..000000000 --- a/tests/data/wire/ext_conf_invalid_vm_metadata.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - Prod - - http://mock-goal-state/manifest_of_ga.xml - - - - Test - - http://mock-goal-state/manifest_of_ga.xml - - - - - - - - - - {"runtimeSettings":[{"handlerSettings":{"protectedSettingsCertThumbprint":"4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3","protectedSettings":"MIICWgYJK","publicSettings":{"foo":"bar"}}}]} - - - https://test.blob.core.windows.net/vhds/test-cs12.test-cs12.test-cs12.status?sr=b&sp=rw&se=9999-01-01&sk=key1&sv=2014-02-14&sig=hfRh7gzUE7sUtYwke78IOlZOrTRCYvkec4hGZ9zZzXo - - \ No newline at end of file diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index 22de84faa..037a21171 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -1480,7 +1480,7 @@ def test_last_etag_on_extension_processing(self, *args): self.assertEqual(etag, exthandlers_handler.last_etag, "Last etag and etag should be same if extension processing is enabled") - def test_it_should_parse_valid_in_vm_metadata_properly(self, mock_get, mock_crypt, *args): + def test_it_should_parse_in_vm_metadata_properly(self, mock_get, mock_crypt, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_IN_VM_META_DATA) exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) @@ -1492,8 +1492,6 @@ def test_it_should_parse_valid_in_vm_metadata_properly(self, mock_get, mock_cryp self.assertEqual(correlation_id, "400de90b-522e-491f-9d89-ec944661f531", "Incorrect correlation Id") self.assertEqual(gs_creation_time, '2020-11-09T17:48:50.412125Z', "Incorrect GS Creation time") - def test_it_should_process_goal_state_even_if_metadata_missing(self, mock_get, mock_crypt, *args): - # If the data is not provided in ExtensionConfig, it should just be None test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) @@ -1504,14 +1502,6 @@ def test_it_should_process_goal_state_even_if_metadata_missing(self, mock_get, m self.assertEqual(activity_id, "NA", "Activity Id should be NA") self.assertEqual(correlation_id, "NA", "Correlation Id should be NA") self.assertEqual(gs_creation_time, "NA", "GS Creation time should be NA") - - def test_it_should_process_goal_state_even_if_metadata_invalid(self, mock_get, mock_crypt, *args): - - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_INVALID_VM_META_DATA) - exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) - - exthandlers_handler.run() - self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") def _assert_ext_status(self, vm_agent_status, expected_status, expected_seq_no, expected_handler_name="OSTCExtensions.ExampleHandlerLinux", expected_msg=None): diff --git a/tests/protocol/mockwiredata.py b/tests/protocol/mockwiredata.py index 337b678d7..c36867e54 100644 --- a/tests/protocol/mockwiredata.py +++ b/tests/protocol/mockwiredata.py @@ -46,9 +46,6 @@ DATA_FILE_IN_VM_META_DATA = DATA_FILE.copy() DATA_FILE_IN_VM_META_DATA["ext_conf"] = "wire/ext_conf_in_vm_metadata.xml" -DATA_FILE_INVALID_VM_META_DATA = DATA_FILE.copy() -DATA_FILE_INVALID_VM_META_DATA["ext_conf"] = "wire/ext_conf_invalid_vm_metadata.xml" - DATA_FILE_NO_EXT = DATA_FILE.copy() DATA_FILE_NO_EXT["goal_state"] = "wire/goal_state_no_ext.xml" DATA_FILE_NO_EXT["ext_conf"] = None From 38261f26e19cd377f654234444068032a9f715e9 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 18 May 2021 11:18:14 -0700 Subject: [PATCH 13/35] Resolve merge conflict: remove unused variable --- azurelinuxagent/ga/exthandlers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index d691caacd..c340fea35 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -87,7 +87,6 @@ # This is the default sequence number we use when there are no settings available for Handlers _DEFAULT_SEQ_NO = "0" -_ENABLE_EXTENSION_TELEMETRY_PIPELINE = True class ValidHandlerStatus(object): From 1e34825c72a2a6874f0bd65385b60f8818b6d188 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Thu, 20 May 2021 16:48:28 -0700 Subject: [PATCH 14/35] Cleanup history folder every 30 min (#2258) * Cleanup history folder every 30 min * fix unit test Co-authored-by: narrieta --- README.md | 2 +- azurelinuxagent/common/conf.py | 4 ++-- config/waagent.conf | 4 ++-- tests/test_agent.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3785df884..35bd081f5 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ A sample configuration file is shown below: ```yml Extensions.Enabled=y Extensions.GoalStatePeriod=6 -Extensions.GoalStateHistoryCleanupPeriod=86400 +Extensions.GoalStateHistoryCleanupPeriod=1800 Provisioning.Agent=auto Provisioning.DeleteRootPassword=n Provisioning.RegenerateSshHostKeyPair=y diff --git a/azurelinuxagent/common/conf.py b/azurelinuxagent/common/conf.py index 5b6b9778d..67798a47f 100644 --- a/azurelinuxagent/common/conf.py +++ b/azurelinuxagent/common/conf.py @@ -145,7 +145,7 @@ def load_conf_from_file(conf_file_path, conf=__conf__): __INTEGER_OPTIONS__ = { "Extensions.GoalStatePeriod": 6, - "Extensions.GoalStateHistoryCleanupPeriod": 86400, + "Extensions.GoalStateHistoryCleanupPeriod": 1800, "OS.EnableFirewallPeriod": 30, "OS.RemovePersistentNetRulesPeriod": 30, "OS.RootDeviceScsiTimeoutPeriod": 30, @@ -346,7 +346,7 @@ def get_goal_state_period(conf=__conf__): def get_goal_state_history_cleanup_period(conf=__conf__): - return conf.get_int("Extensions.GoalStateHistoryCleanupPeriod", 86400) + return conf.get_int("Extensions.GoalStateHistoryCleanupPeriod", 1800) def get_allow_reset_sys_user(conf=__conf__): diff --git a/config/waagent.conf b/config/waagent.conf index 87e201e98..75a79b06e 100644 --- a/config/waagent.conf +++ b/config/waagent.conf @@ -9,8 +9,8 @@ Extensions.Enabled=y # How often (in seconds) to poll for new goal states Extensions.GoalStatePeriod=6 -# How often (in seconds) to clean up the goal state history. The default value is 24 hrs -Extensions.GoalStateHistoryCleanupPeriod=86400 +# How often (in seconds) to clean up the goal state history. The default value is 30 min +Extensions.GoalStateHistoryCleanupPeriod=1800 # Which provisioning agent to use. Supported values are "auto" (default), "waagent", # "cloud-init", or "disabled". diff --git a/tests/test_agent.py b/tests/test_agent.py index 285710bd6..558532f82 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -35,7 +35,7 @@ EnableOverProvisioning = True Extension.LogDir = /var/log/azure Extensions.Enabled = True -Extensions.GoalStateHistoryCleanupPeriod = 86400 +Extensions.GoalStateHistoryCleanupPeriod = 1800 Extensions.GoalStatePeriod = 6 HttpProxy.Host = None HttpProxy.Port = None From dfcd8354e7a9f3e7bdcdd6fc33bb596f5c6f5e17 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 21 May 2021 09:55:37 -0700 Subject: [PATCH 15/35] Set CPUQuota to 5% (#2259) Co-authored-by: narrieta --- azurelinuxagent/common/cgroupconfigurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/cgroupconfigurator.py b/azurelinuxagent/common/cgroupconfigurator.py index b10c9426c..d7ebd67f3 100644 --- a/azurelinuxagent/common/cgroupconfigurator.py +++ b/azurelinuxagent/common/cgroupconfigurator.py @@ -69,7 +69,7 @@ [Service] CPUQuota={0} """ -_AGENT_CPU_QUOTA = 100 +_AGENT_CPU_QUOTA = 5 _AGENT_THROTTLED_TIME_THRESHOLD = 120 # 2 minutes From f19525b0e64d5c33541c14c2b87da5f34ec784f6 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Tue, 25 May 2021 13:03:26 -0700 Subject: [PATCH 16/35] Enable MultiConfig + Minor fixes (#2261) --- .../common/agent_supported_feature.py | 2 +- azurelinuxagent/common/utils/archive.py | 8 +- azurelinuxagent/ga/exthandlers.py | 14 ++- tests/ga/test_extension.py | 95 +++++++++---------- tests/protocol/test_wire.py | 13 ++- 5 files changed, 74 insertions(+), 58 deletions(-) diff --git a/azurelinuxagent/common/agent_supported_feature.py b/azurelinuxagent/common/agent_supported_feature.py index ed2044d6c..e10abfecb 100644 --- a/azurelinuxagent/common/agent_supported_feature.py +++ b/azurelinuxagent/common/agent_supported_feature.py @@ -51,7 +51,7 @@ class _MultiConfigFeature(AgentSupportedFeature): __NAME = SupportedFeatureNames.MultiConfig __VERSION = "1.0" - __SUPPORTED = False + __SUPPORTED = True def __init__(self): super(_MultiConfigFeature, self).__init__(name=_MultiConfigFeature.__NAME, diff --git a/azurelinuxagent/common/utils/archive.py b/azurelinuxagent/common/utils/archive.py index e87c50ee7..e18b1b06a 100644 --- a/azurelinuxagent/common/utils/archive.py +++ b/azurelinuxagent/common/utils/archive.py @@ -36,7 +36,7 @@ """ # pylint: enable=W0105 -_ARCHIVE_DIRECTORY_NAME = 'history' +ARCHIVE_DIRECTORY_NAME = 'history' _MAX_ARCHIVED_STATES = 50 @@ -59,7 +59,7 @@ class StateFlusher(object): def __init__(self, lib_dir): self._source = lib_dir - directory = os.path.join(self._source, _ARCHIVE_DIRECTORY_NAME) + directory = os.path.join(self._source, ARCHIVE_DIRECTORY_NAME) if not os.path.exists(directory): try: fileutil.mkdir(directory) @@ -82,7 +82,7 @@ def flush(self): self._purge(files) def history_dir(self, name): - return os.path.join(self._source, _ARCHIVE_DIRECTORY_NAME, name) + return os.path.join(self._source, ARCHIVE_DIRECTORY_NAME, name) @staticmethod def _get_archive_name(files): @@ -209,7 +209,7 @@ def archive(self): class StateArchiver(object): def __init__(self, lib_dir): - self._source = os.path.join(lib_dir, _ARCHIVE_DIRECTORY_NAME) + self._source = os.path.join(lib_dir, ARCHIVE_DIRECTORY_NAME) if not os.path.isdir(self._source): try: diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index c340fea35..af99cfcfd 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -50,6 +50,7 @@ from azurelinuxagent.common.future import ustr, is_file_not_found_error from azurelinuxagent.common.protocol.restapi import ExtensionStatus, ExtensionSubStatus, ExtHandler, ExtHandlerStatus, \ VMStatus, GoalStateAggregateStatus, ExtensionState, ExtHandlerRequestedState, Extension +from azurelinuxagent.common.utils.archive import ARCHIVE_DIRECTORY_NAME from azurelinuxagent.common.utils.flexible_version import FlexibleVersion from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION, DISTRO_NAME, DISTRO_VERSION, \ GOAL_STATE_AGENT_VERSION, PY_VERSION_MAJOR, PY_VERSION_MICRO, PY_VERSION_MINOR @@ -257,11 +258,14 @@ def get_exthandlers_handler(protocol): return ExtHandlersHandler(protocol) -def list_agent_lib_directory(skip_agent_package=True): +def list_agent_lib_directory(skip_agent_package=True, ignore_names=None): lib_dir = conf.get_lib_dir() for name in os.listdir(lib_dir): path = os.path.join(lib_dir, name) + if ignore_names is not None and any(ignore_names) and name in ignore_names: + continue + if skip_agent_package and (version.is_agent_package(path) or version.is_agent_path(path)): continue @@ -867,7 +871,9 @@ def handle_uninstall(self, ext_handler_i, extension): def __get_handlers_on_file_system(self, incarnation_changed): handlers_to_report = [] - for item, path in list_agent_lib_directory(skip_agent_package=True): + # Ignoring the `history` and `events` directories as they're not handlers and are agent-generated + for item, path in list_agent_lib_directory(skip_agent_package=True, + ignore_names=[EVENTS_DIRECTORY, ARCHIVE_DIRECTORY_NAME]): try: handler_instance = ExtHandlersHandler.get_ext_handler_instance_from_path(name=item, path=path, @@ -973,7 +979,9 @@ def write_ext_handlers_status_to_info_file(vm_status): "goal_state_version": str(GOAL_STATE_AGENT_VERSION), "distro_details": "{0}:{1}".format(DISTRO_NAME, DISTRO_VERSION), "last_successful_status_upload_time": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - "python_version": "Python: {0}.{1}.{2}".format(PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO) + "python_version": "Python: {0}.{1}.{2}".format(PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO), + "crp_supported_features": [name for name, _ in get_agent_supported_features_list_for_crp().items()], + "extension_supported_features": [name for name, _ in get_agent_supported_features_list_for_extensions().items()] } # Convert VMStatus class to Dict. diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index 22de84faa..a6ed9cbd3 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -28,6 +28,8 @@ import uuid from azurelinuxagent.common import conf +from azurelinuxagent.common.agent_supported_feature import get_agent_supported_features_list_for_crp, \ + get_agent_supported_features_list_for_extensions from azurelinuxagent.common.cgroupconfigurator import CGroupConfigurator from azurelinuxagent.common.datacontract import get_properties from azurelinuxagent.common.event import WALAEventOperation @@ -1051,54 +1053,49 @@ def test_ext_handler_sequencing_invalid_dependency_level(self, *args): @patch('time.gmtime', MagicMock(return_value=time.gmtime(0))) def test_ext_handler_reporting_status_file(self, *args): - expected_status = ''' -{{ - "agent_name": "{agent_name}", - "current_version": "{current_version}", - "goal_state_version": "{goal_state_version}", - "distro_details": "{distro_details}", - "last_successful_status_upload_time": "{last_successful_status_upload_time}", - "python_version": "{python_version}", - "extensions_status": [ - {{ - "name": "OSTCExtensions.ExampleHandlerLinux", - "version": "1.0.0", - "status": "Ready", - "supports_multi_config": false - }}, - {{ - "name": "Microsoft.Powershell.ExampleExtension", - "version": "1.0.0", - "status": "Ready", - "supports_multi_config": false - }}, - {{ - "name": "Microsoft.EnterpriseCloud.Monitoring.ExampleHandlerLinux", - "version": "1.0.0", - "status": "Ready", - "supports_multi_config": false - }}, - {{ - "name": "Microsoft.CPlat.Core.ExampleExtensionLinux", - "version": "1.0.0", - "status": "Ready", - "supports_multi_config": false - }}, - {{ - "name": "Microsoft.OSTCExtensions.Edp.ExampleExtensionLinuxInTest", - "version": "1.0.0", - "status": "Ready", - "supports_multi_config": false - }} - ] -}}'''.format(agent_name=AGENT_NAME, - current_version=str(CURRENT_VERSION), - goal_state_version=str(GOAL_STATE_AGENT_VERSION), - distro_details="{0}:{1}".format(DISTRO_NAME, DISTRO_VERSION), - last_successful_status_upload_time=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - python_version="Python: {0}.{1}.{2}".format(PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO)) - - expected_status_json = json.loads(expected_status) + + expected_status = { + "agent_name": AGENT_NAME, + "current_version": str(CURRENT_VERSION), + "goal_state_version": str(GOAL_STATE_AGENT_VERSION), + "distro_details": "{0}:{1}".format(DISTRO_NAME, DISTRO_VERSION), + "last_successful_status_upload_time": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "python_version": "Python: {0}.{1}.{2}".format(PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO), + "crp_supported_features": [name for name, _ in get_agent_supported_features_list_for_crp().items()], + "extension_supported_features": [name for name, _ in get_agent_supported_features_list_for_extensions().items()], + "extensions_status": [ + { + "name": "OSTCExtensions.ExampleHandlerLinux", + "version": "1.0.0", + "status": "Ready", + "supports_multi_config": False + }, + { + "name": "Microsoft.Powershell.ExampleExtension", + "version": "1.0.0", + "status": "Ready", + "supports_multi_config": False + }, + { + "name": "Microsoft.EnterpriseCloud.Monitoring.ExampleHandlerLinux", + "version": "1.0.0", + "status": "Ready", + "supports_multi_config": False + }, + { + "name": "Microsoft.CPlat.Core.ExampleExtensionLinux", + "version": "1.0.0", + "status": "Ready", + "supports_multi_config": False + }, + { + "name": "Microsoft.OSTCExtensions.Edp.ExampleExtensionLinuxInTest", + "version": "1.0.0", + "status": "Ready", + "supports_multi_config": False + } + ] + } test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_MULTIPLE_EXT) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=unused-variable,no-value-for-parameter @@ -1107,7 +1104,7 @@ def test_ext_handler_reporting_status_file(self, *args): status_path = os.path.join(conf.get_lib_dir(), AGENT_STATUS_FILE) actual_status_json = json.loads(fileutil.read_file(status_path)) - self.assertEqual(expected_status_json, actual_status_json) + self.assertEqual(expected_status, actual_status_json) def test_ext_handler_rollingupgrade(self, *args): # Test enable scenario. diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py index f26ab1b6b..5efe0d46e 100644 --- a/tests/protocol/test_wire.py +++ b/tests/protocol/test_wire.py @@ -360,11 +360,22 @@ def test_report_vm_status(self, *args): # pylint: disable=unused-argument 'guestAgentStatus': v1_ga_status, 'handlerAggregateStatus': [] } + + supported_features = [] + for _, feature in get_agent_supported_features_list_for_crp().items(): + supported_features.append( + { + "Key": feature.name, + "Value": feature.version + } + ) + v1_vm_status = { 'version': '1.1', 'timestampUTC': timestamp, 'aggregateStatus': v1_agg_status, - 'guestOSInfo': v1_ga_guest_info + 'guestOSInfo': v1_ga_guest_info, + 'supportedFeatures': supported_features } self.assertEqual(json.dumps(v1_vm_status), actual.to_json()) From 77f0110b3d8821d583eeb10eefe0c2ae3aece7cc Mon Sep 17 00:00:00 2001 From: Dhivya Ganesan Date: Tue, 25 May 2021 16:04:43 -0700 Subject: [PATCH 17/35] Updated _read_status_file to include a fragment of status file in the exception (#2257) * Add User * Updated _read_and_parse_json_status_file function to include a fragment of status file * Updated _read_and_parse_json_status_file function to include a fragment of status file * Updated _read_and_parse_json_status_file function to include a fragment of status file * Fixing Pylint error * Undo some cosmetic diffs Pycharm introduced * Addressed PR comments and deleted unused file * Fixed failed tests * Fixed failed tests * Addressed PR comments Co-authored-by: Laveesh Rohra --- azurelinuxagent/ga/exthandlers.py | 46 +++++----- .../sample-status-invalid-json-format.json | 37 ++++++++ tests/ga/test_extension.py | 85 +++++++++++++++---- 3 files changed, 128 insertions(+), 40 deletions(-) create mode 100644 tests/data/ext/sample-status-invalid-json-format.json diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index af99cfcfd..1003def12 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -1655,7 +1655,7 @@ def collect_ext_status(self, ext): ext_status = ExtensionStatus(name=ext.name, seq_no=seq_no) try: - data_str, data = self._read_and_parse_json_status_file(ext_status_file) + data_str, data = self._read_status_file(ext_status_file) except ExtensionStatusError as e: msg = "" if e.code == ExtensionStatusError.CouldNotReadStatusFile: @@ -2142,34 +2142,32 @@ def get_log_dir(self): return os.path.join(conf.get_ext_log_dir(), self.ext_handler.name) @staticmethod - def _read_and_parse_json_status_file(ext_status_file): - failed_to_read = False - failed_to_parse_json = False - raised_exception = None - data_str = None - data = None - - for attempt in range(_NUM_OF_STATUS_FILE_RETRIES): # pylint: disable=W0612 + def _read_status_file(ext_status_file): + err_count = 0 + while True: try: - data_str = fileutil.read_file(ext_status_file) - data = json.loads(data_str) - break - except IOError as e: - failed_to_read = True - raised_exception = e - except (ValueError, TypeError) as e: - failed_to_parse_json = True - raised_exception = e + return ExtHandlerInstance._read_and_parse_json_status_file(ext_status_file) + except Exception: + err_count += 1 + if err_count >= _NUM_OF_STATUS_FILE_RETRIES: + raise time.sleep(_STATUS_FILE_RETRY_DELAY) - if failed_to_read: - raise ExtensionStatusError(msg=ustr(raised_exception), inner=raised_exception, + @staticmethod + def _read_and_parse_json_status_file(ext_status_file): + + try: + data_str = fileutil.read_file(ext_status_file) + except IOError as e: + raise ExtensionStatusError(msg=ustr(e), inner=e, code=ExtensionStatusError.CouldNotReadStatusFile) - elif failed_to_parse_json: - raise ExtensionStatusError(msg=ustr(raised_exception), inner=raised_exception, + try: + data = json.loads(data_str) + except (ValueError, TypeError) as e: + raise ExtensionStatusError(msg="{0} \n First 2000 Bytes of status file:\n {1}".format(ustr(e), ustr(data_str)[:2000]), + inner=e, code=ExtensionStatusError.InvalidJsonFile) - else: - return data_str, data + return data_str, data def _process_substatus_list(self, substatus_list, current_status_size=0): processed_substatus = [] diff --git a/tests/data/ext/sample-status-invalid-json-format.json b/tests/data/ext/sample-status-invalid-json-format.json new file mode 100644 index 000000000..9bbea3462 --- /dev/null +++ b/tests/data/ext/sample-status-invalid-json-format.json @@ -0,0 +1,37 @@ +[ + { + "_comment": "This is an invalid status file, it's missing a brace at line 37", + "status": { + "status": "success", + "code": 1, + "snapshotInfo": null, + "name": "Microsoft.Azure.Extension.VMExtension", + "commandStartTimeUTCTicks": "636953997844977993", + "taskId": "e5e5602b-48a6-4c35-9f96-752043777af1", + "formattedMessage": { + "lang": "en-US", + "message": "Aenean semper nunc nisl, vitae sollicitudin felis consequat at. In lobortis elementum sapien, non commodo odio semper ac." + }, + "uniqueMachineId": "e5e5602b-48a6-4c35-9f96-752043777af1", + "vmHealthInfo": null, + "storageDetails": { + "totalUsedSizeInBytes": 10000000000, + "partitionCount": 3, + "isSizeComputationFailed": false, + "isStoragespacePresent": false + }, + "telemetryData": null, + "substatus": [ + { + "status": "success", + "formattedMessage": null, + "code": "0", + "name": "[{\"status\": {\"status\": \"success\", \"code\": \"1\", \"snapshotInfo\": [{\"snapshotUri\": \"https://md-rvr1sst1m0s0.blob.core.windows.net/p3w4cdkggwwl/abcd?snapshot=2019-06-06T23:53:14.9090608Z\", \"errorMessage\": \"\", \"isSuccessful\": \"true\"}, {\"snapshotUri\": \"https://md-rvr1sst1m0s0.blob.core.windows.net/l0z0cjhf0fbr/abcd?snapshot=2019-06-06T23:53:14.9083776Z\", \"errorMessage\": \"\", \"isSuccessful\": \"true\"}, {\"snapshotUri\": \"https://md-rvr1sst1m0s0.blob.core.windows.net/lxqfz15mlw0s/abcd?snapshot=2019-06-06T23:53:14.9137572Z\", \"errorMessage\": \"\", \"isSuccessful\": \"true\"}, {\"snapshotUri\": \"https://md-rvr1sst1m0s0.blob.core.windows.net/m04dplcjltlt/abcd?snapshot=2019-06-06T23:53:14.9087358Z\", \"errorMessage\": \"\", \"isSuccessful\": \"true\"}, {\"snapshotUri\": \"https://md-rvr1sst1m0s0.blob.core.windows.net/nkx4dljgcppt/abcd?snapshot=2019-06-06T23:53:14.9089608Z\", \"errorMessage\": \"\", \"isSuccessful\": \"true\"}, {\"snapshotUri\": \"https://md-sqfzxrbqwwkg.blob.core.windows.net/sl2lt5k20wwx/abcd?snapshot=2019-06-06T23:53:14.8338051Z\", \"errorMessage\": \"\", \"isSuccessful\": \"true\"}], \"name\": \"Microsoft.Azure.RecoveryServices.VMSnapshotLinux\", \"commandStartTimeUTCTicks\": \"636953997844977993\", \"taskId\": \"767baff0-3f1e-4363-a974-5d801189250d\", \"formattedMessage\": {\"lang\": \"en-US\", \"message\": \" statusBlobUploadError=true, snapshotCreator=backupHostService, hostStatusCodeDoSnapshot=200, \"}, \"uniqueMachineId\": \"cf2545fd-fef4-250b-d167-f2edb86da1ac\", \"vmHealthInfo\": {\"vmHealthStatusCode\": 1, \"vmHealthState\": 0}, \"storageDetails\": {\"totalUsedSizeInBytes\": 10795593728, \"partitionCount\": 3, \"isSizeComputationFailed\": false, \"isStoragespacePresent\": false}, \"telemetryData\": [{\"Key\": \"kernelVersion\", \"Value\": \"4.4.0-145-generic\"}, {\"Key\": \"networkFSTypePresentInMount\", \"Value\": \"True\"}, {\"Key\": \"extensionVersion\", \"Value\": \"1.0.9150.0\"}, {\"Key\": \"guestAgentVersion\", \"Value\": \"2.2.32.2\"}, {\"Key\": \"FreezeTime\", \"Value\": \"0:00:00.258313\"}, {\"Key\": \"ramDisksSize\", \"Value\": \"626944\"}, {\"Key\": \"platformArchitecture\", \"Value\": \"64bit\"}, {\"Key\": \"pythonVersion\", \"Value\": \"2.7.12\"}, {\"Key\": \"snapshotCreator\", \"Value\": \"backupHostService\"}, {\"Key\": \"tempDisksSize\", \"Value\": \"60988\"}, {\"Key\": \"statusBlobUploadError\", \"Value\": \"true\"}, {\"Key\": \"ThawTime\", \"Value\": \"0:00:00.143658\"}, {\"Key\": \"extErrorCode\", \"Value\": \"success\"}, {\"Key\": \"osVersion\", \"Value\": \"Ubuntu-16.04\"}, {\"Key\": \"hostStatusCodeDoSnapshot\", \"Value\": \"200\"}, {\"Key\": \"snapshotTimeTaken\", \"Value\": \"0:00:00.325422\"}], \"substatus\": [], \"operation\": \"Enable\"}, \"version\": \"1.0\", \"timestampUTC\": \"\\/Date(1559865179380)\\/\"}]" + } + ], + "operation": "Enable" + }, + "version": "1.0", + "timestampUTC": "2019-06-06T23:52:59Z" + +] \ No newline at end of file diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index a6ed9cbd3..9b134a95e 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -437,7 +437,8 @@ def _assert_handler_status(self, report_vm_status, expected_status, args, kw = report_vm_status.call_args # pylint: disable=unused-variable vm_status = args[0] self.assertNotEqual(0, len(vm_status.vmAgent.extensionHandlers)) - handler_status = next(status for status in vm_status.vmAgent.extensionHandlers if status.name == expected_handler_name) + handler_status = next( + status for status in vm_status.vmAgent.extensionHandlers if status.name == expected_handler_name) self.assertEqual(expected_status, handler_status.status) self.assertEqual(expected_handler_name, handler_status.name) self.assertEqual(version, handler_status.version) @@ -1328,6 +1329,8 @@ def _assert_mock_add_event_call(expected_download_failed_event_count, err_msg_gu event_occurrences = [kw for _, kw in mock_add_event.call_args_list if "Failed to download artifacts: [ExtensionDownloadError] {0}".format(err_msg_guid) in kw['message']] self.assertEqual(expected_download_failed_event_count, len(event_occurrences), "Call count do not match") + + self.assertFalse(any(kw['is_success'] for kw in event_occurrences), "The events should have failed") self.assertEqual(expected_download_failed_event_count, len([kw['op'] for kw in event_occurrences]), "Incorrect Operation, all events should be a download errors") @@ -1590,6 +1593,42 @@ def mock_popen(cmd, *args, **kwargs): expected_msg="Dependent Extension OSTCExtensions.OtherExampleHandlerLinux did not reach a terminal state within the allowed timeout. Last status was {0}".format( ValidHandlerStatus.warning)) + def test_it_should_include_part_of_status_in_ext_handler_message(self, mock_http_get, mock_crypt_util, *args): + """ + Testing scenario when the status file is invalid, + The extension status reported by the Handler should contain a fragment of status file for + debugging. + """ + exthandlers_handler, protocol = self._create_mock( + mockwiredata.WireProtocolData(mockwiredata.DATA_FILE), mock_http_get, mock_crypt_util, *args) + + original_popen = subprocess.Popen + + def mock_popen(cmd, *args, **kwargs): + # For the purpose of this test, replacing the status file with file that could not be parsed + if "sample.py" in cmd: + status_path = os.path.join(kwargs['env'][ExtCommandEnvVariable.ExtensionPath], "status", + "{0}.status".format(kwargs['env'][ExtCommandEnvVariable.ExtensionSeqNumber])) + invalid_json_path = os.path.join(data_dir, "ext", "sample-status-invalid-json-format.json") + + if os.path.exists(status_path): + invalid_json = fileutil.read_file(invalid_json_path) + fileutil.write_file(status_path,invalid_json) + + return original_popen(["echo", "Yes"], *args, **kwargs) + + with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): + exthandlers_handler.run() + + # The Handler Status for the base extension should be ready as it was executed successfully by the agent + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="OSTCExtensions.ExampleHandlerLinux") + # The extension status reported by the Handler should contain a fragment of status file for + # debugging. The uniqueMachineId tag comes from status file + self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.error, 0, + expected_handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_msg="\"uniqueMachineId\": \"e5e5602b-48a6-4c35-9f96-752043777af1\"") + def test_wait_for_handler_completion_success_status(self, mock_http_get, mock_crypt_util, *args): """ Testing depends-on scenario on a successful case. Expected to report the status for both extensions properly. @@ -2271,7 +2310,6 @@ def test_extension_upgrade_should_fail_when_continue_on_update_failure_is_true_a as mock_continue_on_update_failure: with patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_enable_command', return_value="exit 1")\ as patch_get_enable: - # These are just testing the mocks have been called and asserting the test conditions have been met exthandlers_handler.run() self.assertEqual(1, patch_get_disable_command.call_count) @@ -2778,7 +2816,6 @@ def _do_upgrade_scenario_and_get_order(first_ext, upgraded_ext): """ with mock_wire_protocol(DATA_FILE, http_put_handler=generate_put_handler(first_ext, upgraded_ext)) as protocol: - exthandlers_handler = get_exthandlers_handler(protocol) with enable_invocations(first_ext, upgraded_ext) as invocation_record: @@ -2800,9 +2837,7 @@ def _do_upgrade_scenario_and_get_order(first_ext, upgraded_ext): return invocation_record - def test_non_enabled_ext_should_not_be_disabled_at_ver_update(self): - _, enable_action = Actions.generate_unique_fail() first_ext = extension_emulator(enable_action=enable_action) @@ -2838,7 +2873,6 @@ def test_disable_failed_env_variable_should_be_set_for_update_cmd_when_continue_ self.assertEqual(kwargs["env"][ExtCommandEnvVariable.DisableReturnCode], exit_code, "DisableAction's return code should be in updateAction's env.") - def test_uninstall_failed_env_variable_should_set_for_install_when_continue_on_update_failure_is_true(self): exit_code, uninstall_action = Actions.generate_unique_fail() @@ -3018,16 +3052,13 @@ def test_correct_exit_code_should_set_on_disable_cmd_failure(self): self.assertEqual(update_kwargs["env"][ExtCommandEnvVariable.DisableReturnCode], exit_code, "DisableAction's return code should be present in UpdateAction's env.") - def test_timeout_code_should_set_on_cmd_timeout(self): - # Return None to every poll, forcing a timeout after 900 seconds (actually very quick because sleep(*) is mocked) force_timeout = lambda *args, **kwargs: None first_ext = extension_emulator(disable_action=force_timeout, uninstall_action=force_timeout) second_ext = extension_emulator(version="1.1.0", continue_on_update_failure=True) - with patch("os.killpg"): with patch("os.getpgid"): invocation_record = TestExtensionUpdateOnFailure._do_upgrade_scenario_and_get_order(first_ext, second_ext) @@ -3124,6 +3155,28 @@ def test_collect_ext_status(self, mock_lib_dir, *args): self.assertEqual(sub_status.message, None) self.assertEqual(sub_status.status, ValidHandlerStatus.success) + @patch("azurelinuxagent.common.conf.get_lib_dir") + def test_collect_ext_status_for_invalid_json(self, mock_lib_dir, *args): + """ + This test validates that collect_ext_status correctly picks up the status file (sample-status-invalid-json-format.json) + and then since the Json cannot be parsed correctly it extension status message should include 2000 bytes of status file + and the line number in which it failed to parse. The uniqueMachineId tag comes from status file. + """ + ext_handler_i, extension = self._setup_extension_for_validating_collect_ext_status(mock_lib_dir, + "sample-status-invalid-json-format.json", *args) + ext_status = ext_handler_i.collect_ext_status(extension) + + self.assertEqual(ext_status.code, ExtensionErrorCodes.PluginSettingsStatusInvalid) + self.assertEqual(ext_status.configurationAppliedTime, None) + self.assertEqual(ext_status.operation, None) + self.assertEqual(ext_status.sequenceNumber, 0) + self.assertRegex(ext_status.message, r".*The status reported by the extension TestHandler-1.0.0\(Sequence number 0\), " + r"was in an incorrect format and the agent could not parse it correctly." + r" Failed due to.*") + self.assertIn("\"uniqueMachineId\": \"e5e5602b-48a6-4c35-9f96-752043777af1\"",ext_status.message) + self.assertEqual(ext_status.status, ValidHandlerStatus.error) + self.assertEqual(len(ext_status.substatusList), 0) + @patch("azurelinuxagent.common.conf.get_lib_dir") def test_it_should_collect_ext_status_even_when_config_dir_deleted(self, mock_lib_dir, *args): @@ -3282,7 +3335,7 @@ def test_additional_locations_node_is_consumed(self): .format(r'(location)|(failoverlocation)|(additionalLocation)') location_uri_regex = re.compile(location_uri_pattern) - manifests_used = [ ('location', '1'), ('failoverlocation', '2'), + manifests_used = [ ('location', '1'), ('failoverlocation', '2'), ('additionalLocation', '3'), ('additionalLocation', '4') ] def manifest_location_handler(url, **kwargs): @@ -3294,7 +3347,7 @@ def manifest_location_handler(url, **kwargs): if wrapped_url and location_uri_regex.match(wrapped_url): return Exception("Ignoring host plugin requests for testing purposes.") - + return None location_type, manifest_num = url_match.group("location_type", "manifest_num") @@ -3318,7 +3371,7 @@ def manifest_location_handler(url, **kwargs): exthandlers_handler.run() def test_fetch_manifest_timeout_is_respected(self): - + location_uri_pattern = r'https?://mock-goal-state/(?P{0})/(?P\d)/manifest.xml'\ .format(r'(location)|(failoverlocation)|(additionalLocation)') location_uri_regex = re.compile(location_uri_pattern) @@ -3332,17 +3385,17 @@ def manifest_location_handler(url, **kwargs): if wrapped_url and location_uri_regex.match(wrapped_url): return Exception("Ignoring host plugin requests for testing purposes.") - + return None - + if manifest_location_handler.num_times_called == 0: time.sleep(.3) manifest_location_handler.num_times_called += 1 return Exception("Failing manifest fetch from uri '{0}' for testing purposes."\ .format(url)) - + return None - + manifest_location_handler.num_times_called = 0 From b80efb66d3139a13a70546ede408e4d881974326 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Tue, 25 May 2021 16:57:43 -0700 Subject: [PATCH 18/35] Intermediary merge release-2.3.0.0 into develop (#2262) --- .../common/persist_firewall_rules.py | 68 ++++++++++-- tests/common/test_persist_firewall_rules.py | 102 ++++++++++++++---- tests/ga/test_extension.py | 2 +- tests/ga/test_update.py | 8 +- 4 files changed, 146 insertions(+), 34 deletions(-) diff --git a/azurelinuxagent/common/persist_firewall_rules.py b/azurelinuxagent/common/persist_firewall_rules.py index 31f0899ca..ac336cdae 100644 --- a/azurelinuxagent/common/persist_firewall_rules.py +++ b/azurelinuxagent/common/persist_firewall_rules.py @@ -32,13 +32,14 @@ class PersistFirewallRulesHandler(object): __SERVICE_FILE_CONTENT = """ -# This unit file was created by the Azure VM Agent. +# This unit file (Version={version}) was created by the Azure VM Agent. # Do not edit. [Unit] Description=Setup network rules for WALinuxAgent Before=network-pre.target Wants=network-pre.target DefaultDependencies=no +ConditionPathExists={binary_path} [Service] Type=oneshot @@ -67,6 +68,10 @@ class PersistFirewallRulesHandler(object): _FIREWALLD_RUNNING_CMD = ["firewall-cmd", "--state"] + # The current version of the unit file; Update it whenever the unit file is modified to ensure Agent can dynamically + # modify the unit file on VM too + _UNIT_VERSION = "1.2" + @staticmethod def get_service_file_path(): osutil = get_osutil() @@ -113,6 +118,13 @@ def setup(self): # In case of a failure, this would throw. In such a case, we don't need to try to setup our custom service # because on system reboot, all iptable rules are reset by firewalld.service so it would be a no-op. self._setup_permanent_firewalld_rules() + + # Remove custom service if exists to avoid problems with firewalld + try: + fileutil.rm_files(*[self.get_service_file_path(), os.path.join(conf.get_lib_dir(), self.BINARY_FILE_NAME)]) + except Exception as error: + logger.info( + "Unable to delete existing service {0}: {1}".format(self._network_setup_service_name, ustr(error))) return logger.info( @@ -163,12 +175,19 @@ def _setup_network_setup_service(self): # the service is always run from the most latest agent. self.__setup_binary_file() - if self.__verify_network_setup_service_enabled(): + network_service_enabled = self.__verify_network_setup_service_enabled() + if network_service_enabled and not self.__unit_file_version_modified(): logger.info("Service: {0} already enabled. No change needed.".format(self._network_setup_service_name)) self.__log_network_setup_service_logs() else: - logger.info("Service: {0} not enabled. Adding it now".format(self._network_setup_service_name)) + if not network_service_enabled: + logger.info("Service: {0} not enabled. Adding it now".format(self._network_setup_service_name)) + else: + logger.info( + "Unit file {0} version modified to {1}, setting it up again".format(self.get_service_file_path(), + self._UNIT_VERSION)) + # Create unit file with default values self.__set_service_unit_file() # Reload systemd configurations when we setup the service for the first time to avoid systemctl warnings @@ -199,7 +218,8 @@ def __set_service_unit_file(self): try: fileutil.write_file(service_unit_file, self.__SERVICE_FILE_CONTENT.format(binary_path=binary_path, - py_path=sys.executable)) + py_path=sys.executable, + version=self._UNIT_VERSION)) fileutil.chmod(service_unit_file, 0o644) # Finally enable the service. This is needed to ensure the service is started on system boot @@ -207,7 +227,8 @@ def __set_service_unit_file(self): try: shellutil.run_command(cmd) except CommandError as error: - msg = "Unable to enable service: {0}; deleting service file: {1}. Command: {2}, Exit-code: {3}.\nstdout: {4}\nstderr: {5}".format( + msg = ustr( + "Unable to enable service: {0}; deleting service file: {1}. Command: {2}, Exit-code: {3}.\nstdout: {4}\nstderr: {5}").format( self._network_setup_service_name, service_unit_file, ' '.join(cmd), error.returncode, error.stdout, error.stderr) raise Exception(msg) @@ -244,7 +265,7 @@ def __verify_network_setup_service_failed(self): def __log_network_setup_service_logs(self): # Get logs from journalctl - https://www.freedesktop.org/software/systemd/man/journalctl.html - cmd = ["journalctl", "-u", self._network_setup_service_name, "-b"] + cmd = ["journalctl", "-u", self._network_setup_service_name, "-b", "--utc"] service_failed = self.__verify_network_setup_service_failed() try: stdout = shellutil.run_command(cmd) @@ -273,3 +294,38 @@ def __reload_systemd_conf(self): shellutil.run_command(["systemctl", "daemon-reload"]) except Exception as exception: logger.warn("Unable to reload systemctl configurations: {0}".format(ustr(exception))) + + def __get_unit_file_version(self): + if not os.path.exists(self.get_service_file_path()): + raise OSError("{0} not found".format(self.get_service_file_path())) + + match = fileutil.findre_in_file(self.get_service_file_path(), + line_re="This unit file \\(Version=([\\d.]+)\\) was created by the Azure VM Agent.") + if match is None: + raise ValueError("Version tag not found in the unit file") + + return match.group(1).strip() + + def __unit_file_version_modified(self): + """ + Check if the unit file version changed from the expected version + :return: True if unit file version changed else False + """ + + try: + unit_file_version = self.__get_unit_file_version() + except Exception as error: + logger.info("Unable to determine version of unit file: {0}, overwriting unit file".format(ustr(error))) + # Since we can't determine the version, marking the file as modified to overwrite the unit file + return True + + if unit_file_version != self._UNIT_VERSION: + logger.info( + "Unit file version: {0} does not match with expected version: {1}, overwriting unit file".format( + unit_file_version, self._UNIT_VERSION)) + return True + + logger.info( + "Unit file version matches with expected version: {0}, not overwriting unit file".format(unit_file_version)) + return False + diff --git a/tests/common/test_persist_firewall_rules.py b/tests/common/test_persist_firewall_rules.py index a5d832a79..29d845d2b 100644 --- a/tests/common/test_persist_firewall_rules.py +++ b/tests/common/test_persist_firewall_rules.py @@ -161,6 +161,31 @@ def __mock_network_setup_service_disabled(cmd): return True, ["echo", "not enabled"] return False, [] + @staticmethod + def __mock_firewalld_running_and_not_applied(cmd): + if cmd == PersistFirewallRulesHandler._FIREWALLD_RUNNING_CMD: + return True, ["echo", "running"] + # This is to fail the check if firewalld-rules are already applied + cmds_to_fail = ["firewall-cmd", FirewallCmdDirectCommands.QueryPassThrough, "conntrack"] + if all(cmd_to_fail in cmd for cmd_to_fail in cmds_to_fail): + return True, ["exit", "1"] + if "firewall-cmd" in cmd: + return True, ["echo", "enabled"] + return False, [] + + def __setup_and_assert_network_service_setup_scenario(self, handler, mock_popen=None): + mock_popen = TestPersistFirewallRulesHandler.__mock_network_setup_service_disabled if mock_popen is None else mock_popen + self.__replace_popen_cmd = mock_popen + handler.setup() + + self.__assert_systemctl_called(cmd="is-enabled", validate_command_called=True) + self.__assert_systemctl_called(cmd="enable", validate_command_called=True) + self.__assert_systemctl_reloaded(validate_command_called=True) + self.__assert_firewall_cmd_running_called(validate_command_called=True) + self.__assert_firewall_called(cmd=FirewallCmdDirectCommands.QueryPassThrough, validate_command_called=False) + self.__assert_firewall_called(cmd=FirewallCmdDirectCommands.PassThrough, validate_command_called=False) + self.assertTrue(os.path.exists(handler.get_service_file_path()), "Service unit file not found") + def test_it_should_skip_setup_if_firewalld_already_enabled(self): self.__replace_popen_cmd = lambda cmd: ("firewall-cmd" in cmd, ["echo", "running"]) with self._get_persist_firewall_rules_handler() as handler: @@ -173,19 +198,27 @@ def test_it_should_skip_setup_if_firewalld_already_enabled(self): # Assert no commands for systemctl were called self.assertFalse(any("systemctl" in cmd for cmd in self.__executed_commands), "Systemctl shouldn't be called") - def test_it_should_skip_setup_if_agent_network_setup_service_already_enabled(self): - self.__replace_popen_cmd = TestPersistFirewallRulesHandler.__mock_network_setup_service_enabled + def test_it_should_skip_setup_if_agent_network_setup_service_already_enabled_and_version_same(self): + with self._get_persist_firewall_rules_handler() as handler: + # 1st time should setup the service + self.__setup_and_assert_network_service_setup_scenario(handler) + + # 2nd time setup should do nothing as service is enabled and no version updated + self.__replace_popen_cmd = TestPersistFirewallRulesHandler.__mock_network_setup_service_enabled + # Reset state + self.__executed_commands = [] handler.setup() - self.__assert_systemctl_called(cmd="is-enabled", validate_command_called=True) - self.__assert_systemctl_called(cmd="enabled", validate_command_called=False) - self.__assert_systemctl_reloaded(validate_command_called=False) - self.__assert_firewall_cmd_running_called(validate_command_called=True) - self.__assert_firewall_called(cmd=FirewallCmdDirectCommands.QueryPassThrough, validate_command_called=False) - self.__assert_firewall_called(cmd=FirewallCmdDirectCommands.PassThrough, validate_command_called=False) + self.__assert_systemctl_called(cmd="is-enabled", validate_command_called=True) + self.__assert_systemctl_called(cmd="enable", validate_command_called=False) + self.__assert_systemctl_reloaded(validate_command_called=False) + self.__assert_firewall_cmd_running_called(validate_command_called=True) + self.__assert_firewall_called(cmd=FirewallCmdDirectCommands.QueryPassThrough, validate_command_called=False) + self.__assert_firewall_called(cmd=FirewallCmdDirectCommands.PassThrough, validate_command_called=False) + self.assertTrue(os.path.exists(handler.get_service_file_path()), "Service unit file not found") - def test_it_should_always_replace_only_drop_in_file_if_using_custom_network_service(self): + def test_it_should_always_replace_binary_file_only_if_using_custom_network_service(self): def _find_in_file(file_name, line_str): try: @@ -207,7 +240,7 @@ def _find_in_file(file_name, line_str): handler.setup() orig_service_file_contents = "ExecStart={py_path} {binary_path}".format(py_path=sys.executable, - binary_path=self._binary_file) + binary_path=self._binary_file) self.__assert_systemctl_called(cmd="is-enabled", validate_command_called=True) self.__assert_firewall_called(cmd=FirewallCmdDirectCommands.PassThrough, validate_command_called=False) self.assertTrue(os.path.exists(self._binary_file), "Binary file should be there") @@ -226,7 +259,7 @@ def _find_in_file(file_name, line_str): # The service should say its enabled now self.__replace_popen_cmd = TestPersistFirewallRulesHandler.__mock_network_setup_service_enabled with self._get_persist_firewall_rules_handler() as handler: - # The Drop-in file should be available on the 2nd run + # The Binary file should be available on the 2nd run self.assertTrue(os.path.exists(self._binary_file), "Binary file should be there") handler.setup() @@ -241,18 +274,7 @@ def _find_in_file(file_name, line_str): def test_it_should_use_firewalld_if_available(self): - def __mock_firewalld_running_and_not_applied(cmd): - if cmd == PersistFirewallRulesHandler._FIREWALLD_RUNNING_CMD: - return True, ["echo", "running"] - # This is to fail the check if firewalld-rules are already applied - cmds_to_fail = ["firewall-cmd", FirewallCmdDirectCommands.QueryPassThrough, "conntrack"] - if all(cmd_to_fail in cmd for cmd_to_fail in cmds_to_fail): - return True, ["exit", "1"] - if "firewall-cmd" in cmd: - return True, ["echo", "enabled"] - return False, [] - - self.__replace_popen_cmd = __mock_firewalld_running_and_not_applied + self.__replace_popen_cmd = self.__mock_firewalld_running_and_not_applied with self._get_persist_firewall_rules_handler() as handler: handler.setup() @@ -321,3 +343,37 @@ def test_it_should_not_fail_if_egg_not_found(self): expected_str = "{0} file not found, skipping execution of firewall execution setup for this boot".format( os.path.join(os.getcwd(), test_str)) self.assertIn(expected_str, output, "Unexpected output") + + def test_it_should_delete_custom_service_files_if_firewalld_enabled(self): + with self._get_persist_firewall_rules_handler() as handler: + # 1st run - Setup the Custom Service + self.__setup_and_assert_network_service_setup_scenario(handler) + + # 2nd run - Enable Firewalld and ensure the agent sets firewall rules using firewalld and deletes custom service + self.__executed_commands = [] + self.__replace_popen_cmd = self.__mock_firewalld_running_and_not_applied + handler.setup() + + self.__assert_firewall_cmd_running_called(validate_command_called=True) + self.__assert_firewall_called(cmd=FirewallCmdDirectCommands.QueryPassThrough, validate_command_called=True) + self.__assert_firewall_called(cmd=FirewallCmdDirectCommands.PassThrough, validate_command_called=True) + self.__assert_systemctl_called(cmd="is-enabled", validate_command_called=False) + self.__assert_systemctl_called(cmd="enable", validate_command_called=False) + self.__assert_systemctl_reloaded(validate_command_called=False) + self.assertFalse(os.path.exists(handler.get_service_file_path()), "Service unit file found") + self.assertFalse(os.path.exists(os.path.join(conf.get_lib_dir(), handler.BINARY_FILE_NAME)), "Binary file found") + + def test_it_should_reset_service_unit_files_if_version_changed(self): + with self._get_persist_firewall_rules_handler() as handler: + # 1st step - Setup the service with old Version + test_ver = str(uuid.uuid4()) + with patch.object(handler, "_UNIT_VERSION", test_ver): + self.__setup_and_assert_network_service_setup_scenario(handler) + self.assertIn(test_ver, fileutil.read_file(handler.get_service_file_path()), "Test version not found") + + # 2nd step - Re-run the setup and ensure the service file set up again even if service enabled + self.__executed_commands = [] + self.__setup_and_assert_network_service_setup_scenario(handler, + mock_popen=self.__mock_network_setup_service_enabled) + self.assertNotIn(test_ver, fileutil.read_file(handler.get_service_file_path()), + "Test version found incorrectly") diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index 9b134a95e..e72c93a9a 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -1504,7 +1504,7 @@ def test_it_should_process_goal_state_even_if_metadata_missing(self, mock_get, m self.assertEqual(activity_id, "NA", "Activity Id should be NA") self.assertEqual(correlation_id, "NA", "Correlation Id should be NA") self.assertEqual(gs_creation_time, "NA", "GS Creation time should be NA") - + def test_it_should_process_goal_state_even_if_metadata_invalid(self, mock_get, mock_crypt, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_INVALID_VM_META_DATA) diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index e4d04118e..d2bda25ae 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -144,8 +144,8 @@ def _get_agent_version(): def agent_bin(self, version, suffix): return "bin/{0}-{1}{2}.egg".format(AGENT_NAME, version, suffix) - def rename_agent_bin(self, path, src_v, dst_v): - src_bin = glob.glob(os.path.join(path, self.agent_bin(src_v, '*')))[0] + def rename_agent_bin(self, path, dst_v): + src_bin = glob.glob(os.path.join(path, self.agent_bin("*.*.*.*", '*')))[0] dst_bin = os.path.join(path, self.agent_bin(dst_v, '')) shutil.move(src_bin, dst_bin) @@ -220,7 +220,7 @@ def prepare_agent(self, version): if from_path != to_path: shutil.move(from_path + ".zip", to_path + ".zip") shutil.move(from_path, to_path) - self.rename_agent_bin(to_path, src_v, dst_v) + self.rename_agent_bin(to_path, dst_v) return def prepare_agents(self, @@ -267,7 +267,7 @@ def replicate_agents(self, to_path = self.agent_dir(dst_v) shutil.copyfile(from_path + ".zip", to_path + ".zip") shutil.copytree(from_path, to_path) - self.rename_agent_bin(to_path, src_v, dst_v) + self.rename_agent_bin(to_path, dst_v) if not is_available: GuestAgent(to_path).mark_failure(is_fatal=True) return dst_v From 334365b9df0917af18d11eb76d6b376c5c303a3d Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Thu, 27 May 2021 08:17:07 -0700 Subject: [PATCH 19/35] Adding nag's username to code owners (#2264) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 3602a09ae..9801efc18 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -20,4 +20,4 @@ # # Linux Agent team # -* @narrieta @larohra @kevinclark19a @ZhidongPeng @dhivyaganesan +* @narrieta @larohra @kevinclark19a @ZhidongPeng @dhivyaganesan @nagworld9 From 0e30e0fd1a7e574ccc1ef890bc08f4bdcb7672b1 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Thu, 27 May 2021 11:47:40 -0700 Subject: [PATCH 20/35] fixed logging of PeriodicOperation (#2263) --- azurelinuxagent/ga/monitor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/azurelinuxagent/ga/monitor.py b/azurelinuxagent/ga/monitor.py index 17220b827..a4afea98a 100644 --- a/azurelinuxagent/ga/monitor.py +++ b/azurelinuxagent/ga/monitor.py @@ -181,6 +181,7 @@ def _operation(self): is_success=False, message=msg, log_event=False) + raise class SendImdsHeartbeat(PeriodicOperation): @@ -220,6 +221,7 @@ def _operation(self): is_success=False, message=msg, log_event=False) + raise class MonitorHandler(ThreadHandlerInterface): @@ -282,9 +284,6 @@ def daemon(self): try: for op in periodic_operations: op.run() - - except Exception as e: - logger.error("An error occurred in the monitor thread main loop; will skip the current iteration.\n{0}", ustr(e)) finally: PeriodicOperation.sleep_until_next_operation(periodic_operations) except Exception as e: From 0a27a260b693c5ad674db4600520207e103c3b65 Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Thu, 10 Jun 2021 11:41:09 -0700 Subject: [PATCH 21/35] Initial commit for str -> ustr to fix pfr setup (#2268) --- azurelinuxagent/common/event.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py index 51ea35392..0cb654b86 100644 --- a/azurelinuxagent/common/event.py +++ b/azurelinuxagent/common/event.py @@ -483,11 +483,11 @@ def add_event(self, name, op=WALAEventOperation.Unknown, is_success=True, durati _log_event(name, op, message, duration, is_success=is_success) event = TelemetryEvent(TELEMETRY_EVENT_EVENT_ID, TELEMETRY_EVENT_PROVIDER_ID) - event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Name, str(name))) - event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Version, str(version))) - event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Operation, str(op))) + event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Name, ustr(name))) + event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Version, ustr(version))) + event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Operation, ustr(op))) event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.OperationSuccess, bool(is_success))) - event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Message, str(message))) + event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Message, ustr(message))) event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Duration, int(duration))) self.add_common_event_parameters(event, datetime.utcnow()) From e079933e3a30edae9ce964b42340f776fc75c350 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 11 Jun 2021 14:57:06 -0700 Subject: [PATCH 22/35] Set agent version to 9.9.9.9 for testing purposes (#2273) Co-authored-by: narrieta --- azurelinuxagent/common/version.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 0717d51d6..39fdf1ba6 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -196,9 +196,13 @@ def has_logrotate(): AGENT_NAME = "WALinuxAgent" AGENT_LONG_NAME = "Azure Linux Agent" -# Setting the version to 9.9.9.9 to ensure DCR always uses this version and never auto-updates. -# Replace this with the actual agent version on release. -AGENT_VERSION = '2.3.0.2' # (current agent version = 2.3.0.2) +# +# IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise +# DCR may test the wrong agent version. +# +# When doing a release, be sure to use the actual agent version. Current agent version: 2.3.0.2 +# +AGENT_VERSION = '9.9.9.9' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From ca5a78885c5d2ac32b9caeea672ad32c63d9cc26 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Wed, 16 Jun 2021 14:26:11 -0700 Subject: [PATCH 23/35] Update history cleanup period in readme (#2266) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35bd081f5..61ba98f78 100644 --- a/README.md +++ b/README.md @@ -246,7 +246,7 @@ setting affects how fast the agent starts executing extensions. #### __Extensions.GoalStateHistoryCleanupPeriod__ _Type: Integer_ -_Default: 86400 (24 hours)_ +_Default: 1800 (30 minutes)_ How often to clean up the history folder of the agent. The agent keeps past goal states on this folder, each goal state represented with a set of small files. The From 9da99cc6e19c54c6b3a4afc48bcdd1cdfac183e4 Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Wed, 16 Jun 2021 18:01:06 -0700 Subject: [PATCH 24/35] Log collector broken pipe fix (#2267) --- azurelinuxagent/common/future.py | 21 ++++++++ azurelinuxagent/ga/collect_logs.py | 81 +++++++++++++++++------------- tests/ga/test_collect_logs.py | 9 +++- 3 files changed, 76 insertions(+), 35 deletions(-) diff --git a/azurelinuxagent/common/future.py b/azurelinuxagent/common/future.py index b7e42eabd..0c0e016ee 100644 --- a/azurelinuxagent/common/future.py +++ b/azurelinuxagent/common/future.py @@ -1,3 +1,4 @@ +import contextlib import platform import sys import os @@ -35,6 +36,10 @@ from collections import OrderedDict # pylint: disable=W0611 from queue import Queue, Empty # pylint: disable=W0611,import-error + # unused-import Disabled: python2.7 doesn't have subprocess.DEVNULL + # so this import is only used by python3. + import subprocess # pylint: disable=unused-import + elif sys.version_info[0] == 2: import httplib as httpclient # pylint: disable=E0401,W0611 from urlparse import urlparse # pylint: disable=E0401 @@ -143,6 +148,22 @@ def is_file_not_found_error(exception): return isinstance(exception, FileNotFoundError) +@contextlib.contextmanager +def subprocess_dev_null(): + + if sys.version_info[0] == 3: + # Suppress no-member errors on python2.7 + yield subprocess.DEVNULL # pylint: disable=no-member + else: + try: + devnull = open(os.devnull, "a+") + yield devnull + except Exception: + yield None + finally: + if devnull is not None: + devnull.close() + def array_to_bytes(buff): # Python 3.9 removed the tostring() method on arrays, the new alias is tobytes() if sys.version_info[0] == 2: diff --git a/azurelinuxagent/ga/collect_logs.py b/azurelinuxagent/ga/collect_logs.py index b2fb59110..ad5befdf5 100644 --- a/azurelinuxagent/ga/collect_logs.py +++ b/azurelinuxagent/ga/collect_logs.py @@ -26,7 +26,7 @@ import azurelinuxagent.common.conf as conf from azurelinuxagent.common import logger from azurelinuxagent.common.event import elapsed_milliseconds, add_event, WALAEventOperation -from azurelinuxagent.common.future import ustr +from azurelinuxagent.common.future import subprocess_dev_null, ustr from azurelinuxagent.common.interfaces import ThreadHandlerInterface from azurelinuxagent.common.logcollector import COMPRESSED_ARCHIVE_PATH from azurelinuxagent.common.osutil import systemd @@ -161,41 +161,54 @@ def _collect_logs(): collect_logs_cmd = [sys.executable, "-u", sys.argv[0], "-collect-logs"] final_command = systemd_cmd + resource_limits + collect_logs_cmd - start_time = datetime.datetime.utcnow() - success = False - msg = None + def exec_command(output_file): + start_time = datetime.datetime.utcnow() + success = False + msg = None + try: + # TODO: Remove track_process (and its implementation) when the log collector is moved to the agent's cgroup + shellutil.run_command(final_command, log_error=False, track_process=False, + stdout=output_file, stderr=output_file) + duration = elapsed_milliseconds(start_time) + archive_size = os.path.getsize(COMPRESSED_ARCHIVE_PATH) + + msg = "Successfully collected logs. Archive size: {0} b, elapsed time: {1} ms.".format(archive_size, + duration) + logger.info(msg) + success = True + + return True + except Exception as e: + duration = elapsed_milliseconds(start_time) + + if isinstance(e, CommandError): + exception_message = ustr("[stderr] %s", e.stderr) # pylint: disable=no-member + else: + exception_message = ustr(e) + + msg = "Failed to collect logs. Elapsed time: {0} ms. Error: {1}".format(duration, exception_message) + # No need to log to the local log since we ran run_command with logging errors as enabled + + return False + finally: + add_event( + name=AGENT_NAME, + version=CURRENT_VERSION, + op=WALAEventOperation.LogCollection, + is_success=success, + message=msg, + log_event=False) + try: - # TODO: Remove track_process (and its implementation) when the log collector is moved to the agent's cgroup - shellutil.run_command(final_command, log_error=True, track_process=False) - duration = elapsed_milliseconds(start_time) - archive_size = os.path.getsize(COMPRESSED_ARCHIVE_PATH) - - msg = "Successfully collected logs. Archive size: {0} b, elapsed time: {1} ms.".format(archive_size, - duration) - logger.info(msg) - success = True - - return True - except Exception as e: - duration = elapsed_milliseconds(start_time) - - if isinstance(e, CommandError): - exception_message = ustr("[stderr] %s", e.stderr) # pylint: disable=no-member - else: - exception_message = ustr(e) - - msg = "Failed to collect logs. Elapsed time: {0} ms. Error: {1}".format(duration, exception_message) - # No need to log to the local log since we ran run_command with logging errors as enabled - - return False + logfile = open(conf.get_agent_log_file(), "a+") + except Exception: + with subprocess_dev_null() as DEVNULL: + return exec_command(DEVNULL) + else: + return exec_command(logfile) finally: - add_event( - name=AGENT_NAME, - version=CURRENT_VERSION, - op=WALAEventOperation.LogCollection, - is_success=success, - message=msg, - log_event=False) + if logfile is not None: + logfile.close() def _send_logs(self): msg = None diff --git a/tests/ga/test_collect_logs.py b/tests/ga/test_collect_logs.py index 45e13b48d..6e4e4e43a 100644 --- a/tests/ga/test_collect_logs.py +++ b/tests/ga/test_collect_logs.py @@ -17,7 +17,7 @@ import contextlib import os -from azurelinuxagent.common import logger +from azurelinuxagent.common import logger, conf from azurelinuxagent.common.logger import Logger from azurelinuxagent.common.protocol.util import ProtocolUtil from azurelinuxagent.ga.collect_logs import get_collect_logs_handler, is_log_collection_allowed @@ -69,6 +69,10 @@ def setUp(self): self.mock_archive_path = patch("azurelinuxagent.ga.collect_logs.COMPRESSED_ARCHIVE_PATH", self.archive_path) self.mock_archive_path.start() + self.logger_path = os.path.join(self.tmp_dir, "waagent.log") + self.mock_logger_path = patch.object(conf, "get_agent_log_file", return_value=self.logger_path) + self.mock_logger_path.start() + # Since ProtocolUtil is a singleton per thread, we need to clear it to ensure that the test cases do not # reuse a previous state clear_singleton_instances(ProtocolUtil) @@ -77,6 +81,9 @@ def tearDown(self): if os.path.exists(self.archive_path): os.remove(self.archive_path) self.mock_archive_path.stop() + if os.path.exists(self.logger_path): + os.remove(self.logger_path) + self.mock_logger_path.stop() AgentTestCase.tearDown(self) def _create_dummy_archive(self, size=1024): From 68559c56550ce8d7f480d24ca6811e40799981fe Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Thu, 17 Jun 2021 12:06:19 -0700 Subject: [PATCH 25/35] Support sles 15 sp2 distro (#2272) * support sles 15 sp2 distro * copy of waagent file * setting bin file --- bin/py3/waagent | 53 +++++++++++++++++++++++++++++++++++++++ init/sles/waagent.service | 16 ++++++++++++ setup.py | 24 ++++++++++++++++-- 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100755 bin/py3/waagent create mode 100644 init/sles/waagent.service diff --git a/bin/py3/waagent b/bin/py3/waagent new file mode 100755 index 000000000..516993de7 --- /dev/null +++ b/bin/py3/waagent @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# +# Azure Linux Agent +# +# Copyright 2015 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Requires Python 2.6 and Openssl 1.0+ +# +# Implements parts of RFC 2131, 1541, 1497 and +# http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx +# http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx +# + +import os +import sys + +if sys.version_info[0] == 2: + import imp +else: + import importlib + +if __name__ == '__main__' : + import azurelinuxagent.agent as agent + """ + Invoke main method of agent + """ + agent.main() + +if __name__ == 'waagent': + """ + Load waagent2.0 to support old version of extensions + """ + if sys.version_info[0] == 3: + raise ImportError("waagent2.0 doesn't support python3") + bin_path = os.path.dirname(os.path.abspath(__file__)) + agent20_path = os.path.join(bin_path, "waagent2.0") + if not os.path.isfile(agent20_path): + raise ImportError("Can't load waagent") + agent20 = imp.load_source('waagent', agent20_path) + __all__ = dir(agent20) + diff --git a/init/sles/waagent.service b/init/sles/waagent.service new file mode 100644 index 000000000..39d01d1c4 --- /dev/null +++ b/init/sles/waagent.service @@ -0,0 +1,16 @@ +[Unit] +Description=Azure Linux Agent +Wants=network-online.target sshd.service sshd-keygen.service +After=network-online.target + +ConditionFileIsExecutable=/usr/sbin/waagent +ConditionPathExists=/etc/waagent.conf + +[Service] +Type=simple +ExecStart=/usr/bin/python3 -u /usr/sbin/waagent -daemon +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/setup.py b/setup.py index c258e4b87..23680ed86 100755 --- a/setup.py +++ b/setup.py @@ -43,7 +43,6 @@ def set_bin_files(data_files, dest, src=None): src = ["bin/waagent", "bin/waagent2.0"] data_files.append((dest, src)) - def set_conf_files(data_files, dest="/etc", src=None): if src is None: src = ["config/waagent.conf"] @@ -96,8 +95,8 @@ def get_data_files(name, version, fullname): # pylint: disable=R0912 systemd_dir_path = osutil.get_systemd_unit_file_install_path() agent_bin_path = osutil.get_agent_bin_path() - set_bin_files(data_files, dest=agent_bin_path) if name == 'redhat' or name == 'centos': # pylint: disable=R1714 + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files) set_logrotate_files(data_files) set_udev_files(data_files) @@ -110,11 +109,13 @@ def get_data_files(name, version, fullname): # pylint: disable=R0912 # TODO this is a mitigation to systemctl bug on 7.1 set_sysv_files(data_files) elif name == 'arch': + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files, src=["config/arch/waagent.conf"]) set_udev_files(data_files) set_systemd_files(data_files, dest=systemd_dir_path, src=["init/arch/waagent.service"]) elif name == 'coreos': + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files, dest="/usr/share/oem", src=["config/coreos/waagent.conf"]) set_logrotate_files(data_files) @@ -122,16 +123,19 @@ def get_data_files(name, version, fullname): # pylint: disable=R0912 set_files(data_files, dest="/usr/share/oem", src=["init/coreos/cloud-config.yml"]) elif "Clear Linux" in fullname: + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files, dest="/usr/share/defaults/waagent", src=["config/clearlinux/waagent.conf"]) set_systemd_files(data_files, dest=systemd_dir_path, src=["init/clearlinux/waagent.service"]) elif name == 'mariner': + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files, dest="/etc", src=["config/mariner/waagent.conf"]) set_systemd_files(data_files, dest=systemd_dir_path, src=["init/mariner/waagent.service"]) elif name == 'ubuntu': + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files, src=["config/ubuntu/waagent.conf"]) set_logrotate_files(data_files) set_udev_files(data_files) @@ -153,6 +157,7 @@ def get_data_files(name, version, fullname): # pylint: disable=R0912 "init/ubuntu/azure-vmextensions.slice" ]) elif name == 'suse' or name == 'opensuse': # pylint: disable=R1714 + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files, src=["config/suse/waagent.conf"]) set_logrotate_files(data_files) set_udev_files(data_files) @@ -165,19 +170,32 @@ def get_data_files(name, version, fullname): # pylint: disable=R0912 else: # sles 12+ and openSUSE 13.2+ use systemd set_systemd_files(data_files, dest=systemd_dir_path) + elif name == 'sles': # sles 15+ distro named as sles + set_bin_files(data_files, dest=agent_bin_path, + src = ["bin/py3/waagent", "bin/waagent2.0"]) + set_conf_files(data_files, src=["config/suse/waagent.conf"]) + set_logrotate_files(data_files) + set_udev_files(data_files) + # sles 15+ uses systemd and python3 + set_systemd_files(data_files, dest=systemd_dir_path, + src=["init/sles/waagent.service"]) elif name == 'freebsd': + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files, src=["config/freebsd/waagent.conf"]) set_freebsd_rc_files(data_files) elif name == 'openbsd': + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files, src=["config/openbsd/waagent.conf"]) set_openbsd_rc_files(data_files) elif name == 'debian': + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files, src=["config/debian/waagent.conf"]) set_logrotate_files(data_files) set_udev_files(data_files, dest="/lib/udev/rules.d") if debian_has_systemd(): set_systemd_files(data_files, dest=systemd_dir_path) elif name == 'iosxe': + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files, src=["config/iosxe/waagent.conf"]) set_logrotate_files(data_files) set_udev_files(data_files) @@ -186,11 +204,13 @@ def get_data_files(name, version, fullname): # pylint: disable=R0912 # TODO this is a mitigation to systemctl bug on 7.1 set_sysv_files(data_files) elif name == 'openwrt': + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files) set_logrotate_files(data_files) set_sysv_files(data_files, dest='/etc/init.d', src=["init/openwrt/waagent"]) else: # Use default setting + set_bin_files(data_files, dest=agent_bin_path) set_conf_files(data_files) set_logrotate_files(data_files) set_udev_files(data_files) From 8f92fc7c885f2a78f78ca3038ead02622fa8fcb8 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 22 Jun 2021 10:12:41 -0700 Subject: [PATCH 26/35] Refactoring of Agent's main loop (#2275) * Refactoring main loop * Add unit test for UpdateHandler._process_goal_state * Remove reference to incarnation from RemoteAccessHandler * review feedback * Add duration to completed message * Remove obsolete test Co-authored-by: narrieta --- azurelinuxagent/common/event.py | 1 - azurelinuxagent/common/exception.py | 9 + azurelinuxagent/ga/exthandlers.py | 158 ++++++----- azurelinuxagent/ga/remoteaccess.py | 10 +- azurelinuxagent/ga/update.py | 121 ++++---- tests/ga/test_extension.py | 361 ++++++++++++------------ tests/ga/test_multi_config_extension.py | 17 ++ tests/ga/test_remoteaccess_handler.py | 2 +- tests/ga/test_update.py | 190 +++++++------ tests/protocol/test_wire.py | 4 + 10 files changed, 470 insertions(+), 403 deletions(-) diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py index 0cb654b86..4f94ec488 100644 --- a/azurelinuxagent/common/event.py +++ b/azurelinuxagent/common/event.py @@ -105,7 +105,6 @@ class WALAEventOperation: PersistFirewallRules = "PersistFirewallRules" PluginSettingsVersionMismatch = "PluginSettingsVersionMismatch" InvalidExtensionConfig = "InvalidExtensionConfig" - ProcessGoalState = "ProcessGoalState" Provision = "Provision" ProvisionGuestAgent = "ProvisionGuestAgent" RemoteAccessHandling = "RemoteAccessHandling" diff --git a/azurelinuxagent/common/exception.py b/azurelinuxagent/common/exception.py index aa3737ea6..c0421d003 100644 --- a/azurelinuxagent/common/exception.py +++ b/azurelinuxagent/common/exception.py @@ -22,6 +22,15 @@ """ +class ExitException(BaseException): + """ + Used to exit the agent's process + """ + def __init__(self, reason): + super(ExitException, self).__init__() + self.reason = reason + + class AgentError(Exception): """ Base class of agent error. diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 1003def12..5a44d5d95 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -276,9 +276,6 @@ class ExtHandlersHandler(object): def __init__(self, protocol): self.protocol = protocol self.ext_handlers = None - self.last_etag = None - self.log_report = False - self.log_process = False # The GoalState Aggregate status needs to report the last status of the GoalState. Since we only process # extensions on incarnation change, we need to maintain its state. # Setting the status to None here. This would be overridden as soon as the first GoalState is processed @@ -287,10 +284,6 @@ def __init__(self, protocol): self.report_status_error_state = ErrorState() - def _incarnation_changed(self, etag): - # Skip processing if GoalState incarnation did not change - return self.last_etag != etag - def __last_gs_unsupported(self): # Return if the last GoalState was unsupported @@ -329,37 +322,48 @@ def format_value(parse_fn, value): return activity_id, correlation_id, gs_creation_time def run(self): + etag, activity_id, correlation_id, gs_creation_time = None, None, None, None try: + # self.ext_handlers needs to be initialized first, since status reporting depends on it self.ext_handlers, etag = self.protocol.get_ext_handlers() - msg = u"Handle extensions updates for incarnation {0}".format(etag) - logger.verbose(msg) - # Log status report success on new config - self.log_report = True - incarnation_changed = self._incarnation_changed(etag) - if self._extension_processing_allowed() and incarnation_changed: - activity_id, correlation_id, gs_creation_time = self.get_goal_state_debug_metadata() - - logger.info( - "ProcessGoalState started [Incarnation: {0}; Activity Id: {1}; Correlation Id: {2}; GS Creation Time: {3}]".format( - etag, activity_id, correlation_id, gs_creation_time)) - - self.__process_and_handle_extensions(etag) - self.last_etag = etag - - self.report_ext_handlers_status(incarnation_changed=incarnation_changed) - self._cleanup_outdated_handlers() + + if not self._extension_processing_allowed(): + return + + activity_id, correlation_id, gs_creation_time = self.get_goal_state_debug_metadata() except Exception as error: - msg = u"Exception processing extension handlers: {0}".format(ustr(error)) - detailed_msg = '{0} {1}'.format(msg, traceback.extract_tb(get_traceback(error))) + msg = u"ProcessExtensionsInGoalState - Exception processing extension handlers: {0}\n{1}".format(ustr(error), traceback.extract_tb(get_traceback(error))) logger.warn(msg) - add_event(AGENT_NAME, - version=CURRENT_VERSION, - op=WALAEventOperation.ExtensionProcessing, - is_success=False, - message=detailed_msg) + add_event(op=WALAEventOperation.ExtensionProcessing, is_success=False, message=msg, log_event=False) return + def goal_state_debug_info(duration=None): + if duration is None: + return "[Incarnation: {0}; Activity Id: {1}; Correlation Id: {2}; GS Creation Time: {3}]".format(etag, activity_id, correlation_id, gs_creation_time) + else: + return "[Incarnation: {0}; {1} ms; Activity Id: {2}; Correlation Id: {3}; GS Creation Time: {4}]".format(etag, duration, activity_id, correlation_id, gs_creation_time) + + utc_start = datetime.datetime.utcnow() + error = None + message = "ProcessExtensionsInGoalState started {0}".format(goal_state_debug_info()) + logger.info(message) + add_event(op=WALAEventOperation.ExtensionProcessing, message=message) + try: + self.__process_and_handle_extensions(etag) + self._cleanup_outdated_handlers() + except Exception as error: + error = u"ProcessExtensionsInGoalState - Exception processing extension handlers: {0}\n{1}".format(ustr(error), traceback.extract_tb(get_traceback(error))) + finally: + duration = elapsed_milliseconds(utc_start) + if error is None: + message = 'ProcessExtensionsInGoalState completed {0}'.format(goal_state_debug_info(duration=duration)) + logger.info(message) + else: + message = 'ProcessExtensionsInGoalState failed {0}\nError:{1}'.format(goal_state_debug_info(duration=duration), error) + logger.warn(message) + add_event(op=WALAEventOperation.ExtensionProcessing, is_success=(error is None), message=message, log_event=False, duration=duration) + def __get_unsupported_features(self): required_features = self.protocol.get_required_features() supported_features = get_agent_supported_features_list_for_crp() @@ -701,7 +705,6 @@ def handle_enable(self, ext_handler_i, extension): 1- Ensure the handler is installed 2- Check if extension is enabled or disabled and then process accordingly """ - self.log_process = True uninstall_exit_code = None old_ext_handler_i = ext_handler_i.get_installed_ext_handler() @@ -831,7 +834,6 @@ def handle_disable(self, ext_handler_i, extension=None): Disable is a legacy behavior, CRP doesn't support it, its only for XML based extensions. In case we get a disable request, just disable that extension. """ - self.log_process = True handler_state = ext_handler_i.get_handler_state() ext_handler_i.logger.info("[Disable] current handler state is: {0}", handler_state.lower()) if handler_state == ExtHandlerState.Enabled: @@ -846,7 +848,6 @@ def handle_uninstall(self, ext_handler_i, extension): CRP will only set the HandlerState to Uninstall if all its extensions are set to be disabled) 2- Finally uninstall the handler """ - self.log_process = True handler_state = ext_handler_i.get_handler_state() ext_handler_i.logger.info("[Uninstall] current handler state is: {0}", handler_state.lower()) if handler_state != ExtHandlerState.NotInstalled: @@ -917,57 +918,66 @@ def report_ext_handlers_status(self, incarnation_changed=False): """ Go through handler_state dir, collect and report status """ - vm_status = VMStatus(status="Ready", message="Guest Agent is running", - gs_aggregate_status=self.__gs_aggregate_status) + try: + vm_status = VMStatus(status="Ready", message="Guest Agent is running", + gs_aggregate_status=self.__gs_aggregate_status) - handlers_to_report = [] + handlers_to_report = [] - # Incase of Unsupported error, report the status of the handlers in the VM - if self.__last_gs_unsupported(): - handlers_to_report = self.__get_handlers_on_file_system(incarnation_changed) + # In case of Unsupported error, report the status of the handlers in the VM + if self.__last_gs_unsupported(): + handlers_to_report = self.__get_handlers_on_file_system(incarnation_changed) - # If GoalState supported, report the status of extension handlers that were requested by the GoalState - elif not self.__last_gs_unsupported() and self.ext_handlers is not None: - handlers_to_report = self.ext_handlers.extHandlers + # If GoalState supported, report the status of extension handlers that were requested by the GoalState + elif not self.__last_gs_unsupported() and self.ext_handlers is not None: + handlers_to_report = self.ext_handlers.extHandlers - for ext_handler in handlers_to_report: + for ext_handler in handlers_to_report: + try: + self.report_ext_handler_status(vm_status, ext_handler, incarnation_changed) + except ExtensionError as error: + add_event(op=WALAEventOperation.ExtensionProcessing, is_success=False, message=ustr(error)) + + logger.verbose("Report vm agent status") try: - self.report_ext_handler_status(vm_status, ext_handler, incarnation_changed) - except ExtensionError as error: - add_event(op=WALAEventOperation.ExtensionProcessing, is_success=False, message=ustr(error)) + self.protocol.report_vm_status(vm_status) + logger.verbose("Completed vm agent status report successfully") + self.report_status_error_state.reset() + except ProtocolNotFoundError as error: + self.report_status_error_state.incr() + message = "Failed to report vm agent status: {0}".format(error) + logger.verbose(message) + except ProtocolError as error: + self.report_status_error_state.incr() + message = "Failed to report vm agent status: {0}".format(error) + add_event(AGENT_NAME, + version=CURRENT_VERSION, + op=WALAEventOperation.ExtensionProcessing, + is_success=False, + message=message) - logger.verbose("Report vm agent status") - try: - self.protocol.report_vm_status(vm_status) - if self.log_report: - logger.verbose("Completed vm agent status report") - self.report_status_error_state.reset() - except ProtocolNotFoundError as error: - self.report_status_error_state.incr() - message = "Failed to report vm agent status: {0}".format(error) - logger.verbose(message) - except ProtocolError as error: - self.report_status_error_state.incr() - message = "Failed to report vm agent status: {0}".format(error) - add_event(AGENT_NAME, - version=CURRENT_VERSION, - op=WALAEventOperation.ExtensionProcessing, - is_success=False, - message=message) + if self.report_status_error_state.is_triggered(): + message = "Failed to report vm agent status for more than {0}" \ + .format(self.report_status_error_state.min_timedelta) - if self.report_status_error_state.is_triggered(): - message = "Failed to report vm agent status for more than {0}" \ - .format(self.report_status_error_state.min_timedelta) + add_event(AGENT_NAME, + version=CURRENT_VERSION, + op=WALAEventOperation.ReportStatusExtended, + is_success=False, + message=message) + + self.report_status_error_state.reset() + self.write_ext_handlers_status_to_info_file(vm_status) + + except Exception as error: + msg = u"Failed to report status: {0}\n{1}".format(ustr(error), traceback.extract_tb(get_traceback(error))) + logger.warn(msg) add_event(AGENT_NAME, version=CURRENT_VERSION, - op=WALAEventOperation.ReportStatusExtended, + op=WALAEventOperation.ReportStatus, is_success=False, - message=message) - - self.report_status_error_state.reset() - - self.write_ext_handlers_status_to_info_file(vm_status) + message=msg) @staticmethod def write_ext_handlers_status_to_info_file(vm_status): diff --git a/azurelinuxagent/ga/remoteaccess.py b/azurelinuxagent/ga/remoteaccess.py index f18f46e27..f12fd0d49 100644 --- a/azurelinuxagent/ga/remoteaccess.py +++ b/azurelinuxagent/ga/remoteaccess.py @@ -48,18 +48,14 @@ def __init__(self, protocol): self._protocol = protocol self._cryptUtil = CryptUtil(conf.get_openssl_cmd()) self._remote_access = None - self._incarnation = 0 self._check_existing_jit_users = True def run(self): try: if self._os_util.jit_enabled: - current_incarnation = self._protocol.get_incarnation() - if self._incarnation != current_incarnation: - # something changed. Handle remote access if any. - self._incarnation = current_incarnation - self._remote_access = self._protocol.client.get_remote_access() - self._handle_remote_access() + # Handle remote access if any. + self._remote_access = self._protocol.client.get_remote_access() + self._handle_remote_access() except Exception as e: msg = u"Exception processing goal state for remote access users: {0} {1}".format(ustr(e), traceback.format_exc()) add_event(AGENT_NAME, diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 3ff2bf18f..c656388d2 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -43,8 +43,8 @@ from azurelinuxagent.common.cgroupconfigurator import CGroupConfigurator from azurelinuxagent.common.event import add_event, initialize_event_logger_vminfo_common_parameters, \ - elapsed_milliseconds, WALAEventOperation, EVENTS_DIRECTORY -from azurelinuxagent.common.exception import ResourceGoneError, UpdateError + WALAEventOperation, EVENTS_DIRECTORY +from azurelinuxagent.common.exception import ResourceGoneError, UpdateError, ExitException from azurelinuxagent.common.future import ustr from azurelinuxagent.common.osutil import get_osutil, systemd from azurelinuxagent.common.protocol.util import get_protocol_util @@ -100,7 +100,7 @@ def __init__(self): self.osutil = get_osutil() self.protocol_util = get_protocol_util() - self.running = True + self._is_running = True self.last_attempt_time = None self.agents = [] @@ -117,6 +117,8 @@ def __init__(self): self._heartbeat_counter = 0 self._heartbeat_update_goal_state_error_count = 0 + self.last_incarnation = None + def run_latest(self, child_args=None): """ This method is called from the daemon to find and launch the most @@ -225,7 +227,7 @@ def run_latest(self, child_args=None): except Exception as e: # Ignore child errors during termination - if self.running: + if self.is_running: msg = u"Agent {0} launched with command '{1}' failed with exception: {2}".format( agent_name, agent_cmd, @@ -317,64 +319,15 @@ def run(self, debug=False): goal_state_interval = conf.get_goal_state_period() if conf.get_extensions_enabled() else GOAL_STATE_INTERVAL_DISABLED - while self.running: - # - # Check that the parent process (the agent's daemon) is still running - # - if not debug and self._is_orphaned: - logger.info("Agent {0} is an orphan -- exiting", CURRENT_AGENT) - break - - # - # Check that all the threads are still running - # - for thread_handler in all_thread_handlers: - if not thread_handler.is_alive(): - logger.warn("{0} thread died, restarting".format(thread_handler.get_thread_name())) - thread_handler.start() - - # - # Process the goal state - # - if not protocol.try_update_goal_state(): - self._heartbeat_update_goal_state_error_count += 1 - else: - if self._upgrade_available(protocol): - available_agent = self.get_latest_agent() - if available_agent is None: - logger.info( - "Agent {0} is reverting to the installed agent -- exiting", - CURRENT_AGENT) - else: - logger.info( - u"Agent {0} discovered update {1} -- exiting", - CURRENT_AGENT, - available_agent.name) - break - - utc_start = datetime.utcnow() - - last_etag = exthandlers_handler.last_etag - exthandlers_handler.run() - - remote_access_handler.run() - - if last_etag != exthandlers_handler.last_etag: - self._ensure_readonly_files() - duration = elapsed_milliseconds(utc_start) - activity_id, correlation_id, gs_creation_time = exthandlers_handler.get_goal_state_debug_metadata() - msg = 'ProcessGoalState completed [Incarnation: {0}; {1} ms; Activity Id: {2}; Correlation Id: {3}; GS Creation Time: {4}]'.format( - exthandlers_handler.last_etag, duration, activity_id, correlation_id, gs_creation_time) - logger.info(msg) - add_event( - AGENT_NAME, - op=WALAEventOperation.ProcessGoalState, - duration=duration, - message=msg) - + while self.is_running: + self._check_daemon_running(debug) + self._check_threads_running(all_thread_handlers) + self._process_goal_state(protocol, exthandlers_handler, remote_access_handler) self._send_heartbeat_telemetry(protocol) time.sleep(goal_state_interval) + except ExitException as exitException: + logger.info(exitException.reason) except Exception as error: msg = u"Agent {0} failed with exception: {1}".format(CURRENT_AGENT, ustr(error)) self._set_sentinel(msg=msg) @@ -387,6 +340,46 @@ def run(self, debug=False): self._shutdown() sys.exit(0) + def _check_daemon_running(self, debug): + # Check that the parent process (the agent's daemon) is still running + if not debug and self._is_orphaned: + raise ExitException("Agent {0} is an orphan -- exiting".format(CURRENT_AGENT)) + + def _check_threads_running(self, all_thread_handlers): + # Check that all the threads are still running + for thread_handler in all_thread_handlers: + if not thread_handler.is_alive(): + logger.warn("{0} thread died, restarting".format(thread_handler.get_thread_name())) + thread_handler.start() + + def _process_goal_state(self, protocol, exthandlers_handler, remote_access_handler): + if not protocol.try_update_goal_state(): + self._heartbeat_update_goal_state_error_count += 1 + return + + if self._upgrade_available(protocol): + available_agent = self.get_latest_agent() + if available_agent is None: + reason = "Agent {0} is reverting to the installed agent -- exiting".format(CURRENT_AGENT) + else: + reason = "Agent {0} discovered update {1} -- exiting".format(CURRENT_AGENT, available_agent.name) + raise ExitException(reason) + + incarnation = protocol.get_incarnation() + + try: + if incarnation != self.last_incarnation: + exthandlers_handler.run() + + # report status always, even if the goal state did not change + # do it before processing the remote access, since that operation can take a long time + exthandlers_handler.report_ext_handlers_status(incarnation_changed=incarnation != self.last_incarnation) + + if incarnation != self.last_incarnation: + remote_access_handler.run() + finally: + self.last_incarnation = incarnation + def forward_signal(self, signum, frame): if signum == signal.SIGTERM: self._shutdown() @@ -586,6 +579,14 @@ def _get_pid_files(self): pid_files.sort(key=lambda f: int(pid_re.match(os.path.basename(f)).group(1))) return pid_files + @property + def is_running(self): + return self._is_running + + @is_running.setter + def is_running(self, value): + self._is_running = value + @property def _is_clean_start(self): return not os.path.isfile(self._sentinel_file_path()) @@ -678,7 +679,7 @@ def _sentinel_file_path(self): def _shutdown(self): # Todo: Ensure all threads stopped when shutting down the main extension handler to ensure that the state of # all threads is clean. - self.running = False + self.is_running = False if not os.path.isfile(self._sentinel_file_path()): return diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index e72c93a9a..4a399bf4e 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -25,7 +25,6 @@ import tempfile import time import unittest -import uuid from azurelinuxagent.common import conf from azurelinuxagent.common.agent_supported_feature import get_agent_supported_features_list_for_crp, \ @@ -136,6 +135,8 @@ def mock_http_put(url, *args, **_): def test_cleanup_leaves_installed_extensions(self): with self._setup_test_env(mockwiredata.DATA_FILE_MULTIPLE_EXT) as (exthandlers_handler, protocol, no_of_exts): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(no_of_exts, TestExtensionCleanup._count_packages(), "No of extensions in config doesn't match the packages") self.assertEqual(no_of_exts, TestExtensionCleanup._count_extension_directories(), @@ -146,6 +147,7 @@ def test_cleanup_leaves_installed_extensions(self): def test_cleanup_removes_uninstalled_extensions(self): with self._setup_test_env(mockwiredata.DATA_FILE_MULTIPLE_EXT) as (exthandlers_handler, protocol, no_of_exts): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(no_of_exts, TestExtensionCleanup._count_packages(), "No of extensions in config doesn't match the packages") self._assert_ext_handler_status(protocol.aggregate_status, "Ready", expected_ext_handler_count=no_of_exts, @@ -157,6 +159,7 @@ def test_cleanup_removes_uninstalled_extensions(self): protocol.client.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(0, TestExtensionCleanup._count_packages(), "All packages must be deleted") self._assert_ext_handler_status(protocol.aggregate_status, "Ready", expected_ext_handler_count=0, @@ -178,6 +181,8 @@ def test_cleanup_removes_orphaned_packages(self): self.assertEqual(no_of_orphaned_packages, TestExtensionCleanup._count_extension_directories(), "Test Setup error - Not enough extension directories") exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(no_of_exts, TestExtensionCleanup._count_extension_directories(), "There should be no extension directories in FS") self.assertIsNone(protocol.aggregate_status, @@ -192,6 +197,8 @@ def mock_fail_popen(*args, **kwargs): # pylint: disable=unused-argument with self._setup_test_env(mockwiredata.DATA_FILE_EXT_SINGLE) as (exthandlers_handler, protocol, no_of_exts): with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen", mock_fail_popen): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_ext_handler_status(protocol.aggregate_status, "NotReady", expected_ext_handler_count=no_of_exts, version="1.0.0", verify_ext_reported=False) @@ -204,6 +211,7 @@ def mock_fail_popen(*args, **kwargs): # pylint: disable=unused-argument protocol.client.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(0, TestExtensionCleanup._count_packages(), "All packages must be deleted") self.assertEqual(0, TestExtensionCleanup._count_extension_directories(), @@ -227,6 +235,7 @@ def assert_extension_seq_no(expected_seq_no): with self._setup_test_env(mockwiredata.DATA_FILE_MULTIPLE_EXT) as (exthandlers_handler, protocol, orig_no_of_exts): # Run 1 - GS has no required features and contains 5 extensions exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(orig_no_of_exts, TestExtensionCleanup._count_packages(), "No of extensions in config doesn't match the packages") self.assertEqual(orig_no_of_exts, TestExtensionCleanup._count_extension_directories(), @@ -244,6 +253,7 @@ def assert_extension_seq_no(expected_seq_no): protocol.mock_wire_data.set_extensions_config_sequence_number(random.randint(10, 100)) protocol.client.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertGreater(orig_no_of_exts, 1, "No of extensions to check should be > 1") self.assertEqual(orig_no_of_exts, TestExtensionCleanup._count_packages(), "No of extensions should not be changed") @@ -269,6 +279,7 @@ def assert_extension_seq_no(expected_seq_no): protocol.mock_wire_data.set_extensions_config_sequence_number(extension_seq_no) protocol.client.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(1, TestExtensionCleanup._count_packages(), "No of extensions should not be changed") self.assertEqual(1, TestExtensionCleanup._count_extension_directories(), @@ -490,6 +501,7 @@ def _set_up_update_test_and_update_gs(self, patch_command, *args): # Ensure initial install and enable is successful exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(0, patch_command.call_count) self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.0") @@ -517,12 +529,14 @@ def test_ext_handler(self, *args): exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self._assert_ext_status(protocol.report_vm_status, "success", 0) # Test goal state not changed exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") @@ -532,6 +546,8 @@ def test_ext_handler(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self._assert_ext_status(protocol.report_vm_status, "success", 1) @@ -543,6 +559,7 @@ def test_ext_handler(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.1") self._assert_ext_status(protocol.report_vm_status, "success", 2) @@ -554,6 +571,7 @@ def test_ext_handler(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.2.0") self._assert_ext_status(protocol.report_vm_status, "success", 3) @@ -564,6 +582,7 @@ def test_ext_handler(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "NotReady", 1, "1.2.0") @@ -573,6 +592,7 @@ def test_ext_handler(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_no_handler_status(protocol.report_vm_status) @@ -581,6 +601,8 @@ def test_ext_handler(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_no_handler_status(protocol.report_vm_status) def test_it_should_only_download_extension_manifest_once_per_goal_state(self, *args): @@ -594,25 +616,17 @@ def _assert_handler_status_and_manifest_download_count(protocol, test_data, mani test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() _assert_handler_status_and_manifest_download_count(protocol, test_data, 1) - for _ in range(5): - exthandlers_handler.run() - # The extension manifest should only be downloaded once as incarnation did not change - _assert_handler_status_and_manifest_download_count(protocol, test_data, 1) - # Update Incarnation test_data.set_incarnation(2) protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() _assert_handler_status_and_manifest_download_count(protocol, test_data, 2) - for _ in range(5): - exthandlers_handler.run() - # The extension manifest should be downloaded twice now as incarnation changed once - _assert_handler_status_and_manifest_download_count(protocol, test_data, 2) - def test_it_should_fail_handler_on_bad_extension_config_and_report_error(self, mock_get, mock_crypt_util, *args): invalid_config_dir = os.path.join(data_dir, "wire", "invalid_config") @@ -626,6 +640,7 @@ def test_it_should_fail_handler_on_bad_extension_config_and_report_error(self, m with patch('azurelinuxagent.ga.exthandlers.add_event') as patch_add_event: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "NotReady", 0, "1.0.0") invalid_config_errors = [kw for _, kw in patch_add_event.call_args_list if @@ -641,6 +656,7 @@ def test_it_should_process_valid_extensions_if_present(self, mock_get, mock_cryp exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt_util, *args) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertTrue(protocol.report_vm_status.called) args, _ = protocol.report_vm_status.call_args vm_status = args[0] @@ -664,6 +680,7 @@ def test_ext_zip_file_packages_removed_in_update_case(self, *args): exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -676,6 +693,7 @@ def test_ext_zip_file_packages_removed_in_update_case(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.0") self._assert_ext_status(protocol.report_vm_status, "success", 1) @@ -689,6 +707,7 @@ def test_ext_zip_file_packages_removed_in_update_case(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.2.0") self._assert_ext_status(protocol.report_vm_status, "success", 2) @@ -702,6 +721,7 @@ def test_ext_zip_file_packages_removed_in_uninstall_case(self, *args): extension_version = "1.0.0" exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, extension_version) self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -713,6 +733,7 @@ def test_ext_zip_file_packages_removed_in_uninstall_case(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_no_handler_status(protocol.report_vm_status) self._assert_ext_pkg_file_status(expected_to_be_present=False, extension_version=extension_version) @@ -723,6 +744,7 @@ def test_ext_zip_file_packages_removed_in_update_and_uninstall_case(self, *args) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -735,6 +757,7 @@ def test_ext_zip_file_packages_removed_in_update_and_uninstall_case(self, *args) protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.0") self._assert_ext_status(protocol.report_vm_status, "success", 1) @@ -748,6 +771,7 @@ def test_ext_zip_file_packages_removed_in_update_and_uninstall_case(self, *args) protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.2.0") self._assert_ext_status(protocol.report_vm_status, "success", 2) @@ -760,6 +784,7 @@ def test_ext_zip_file_packages_removed_in_update_and_uninstall_case(self, *args) protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_no_handler_status(protocol.report_vm_status) self._assert_ext_pkg_file_status(expected_to_be_present=False, extension_version="1.2.0") @@ -769,6 +794,7 @@ def test_it_should_ignore_case_when_parsing_plugin_settings(self, mock_get, mock exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt_util, *args) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() expected_ext_handlers = ["OSTCExtensions.ExampleHandlerLinux", "Microsoft.Powershell.ExampleExtension", "Microsoft.EnterpriseCloud.Monitoring.ExampleHandlerLinux", @@ -799,6 +825,7 @@ def test_ext_handler_no_settings(self, *args): test_ext = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux") with enable_invocations(test_ext) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 0, "1.0.0") invocation_record.compare( (test_ext, ExtensionCommandNames.INSTALL), @@ -812,6 +839,7 @@ def test_ext_handler_no_settings(self, *args): with enable_invocations(test_ext) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertTrue(protocol.report_vm_status.called) args, _ = protocol.report_vm_status.call_args self.assertEqual(0, len(args[0].vmAgent.extensionHandlers)) @@ -825,6 +853,7 @@ def test_ext_handler_no_public_settings(self, *args): exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") def test_ext_handler_no_ext(self, *args): @@ -833,6 +862,7 @@ def test_ext_handler_no_ext(self, *args): # Assert no extension handler status exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_no_handler_status(protocol.report_vm_status) def test_ext_handler_sequencing(self, *args): @@ -845,6 +875,7 @@ def test_ext_handler_sequencing(self, *args): with enable_invocations(dep_ext_level_2, dep_ext_level_1) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") @@ -870,6 +901,8 @@ def test_ext_handler_sequencing(self, *args): # Test goal state not changed exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") @@ -885,6 +918,7 @@ def test_ext_handler_sequencing(self, *args): with enable_invocations(dep_ext_level_3, dep_ext_level_4) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self._assert_ext_status(protocol.report_vm_status, "success", 1) @@ -911,6 +945,7 @@ def test_ext_handler_sequencing(self, *args): with enable_invocations(dep_ext_level_3, dep_ext_level_4) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "NotReady", 1, "1.0.0", expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") @@ -942,6 +977,7 @@ def test_ext_handler_sequencing(self, *args): with enable_invocations(dep_ext_level_5, dep_ext_level_6) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_no_handler_status(protocol.report_vm_status) self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2) @@ -977,20 +1013,19 @@ def mock_fail_extension_commands(args, **kwargs): with patch("subprocess.Popen", mock_fail_extension_commands): with patch('azurelinuxagent.ga.exthandlers.add_event') as patch_add_event: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "NotReady", 0, "1.0.0", expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") _assert_event_reported_only_on_incarnation_change(expected_count=1) - # Assert that on rerun it should not report errors unless incarnation changes - for _ in range(5): - exthandlers_handler.run() - _assert_event_reported_only_on_incarnation_change(expected_count=1) - test_data.set_incarnation(2) protocol.update_goal_state() + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + # We should report error again on incarnation change _assert_event_reported_only_on_incarnation_change(expected_count=2) @@ -1000,6 +1035,8 @@ def mock_fail_extension_commands(args, **kwargs): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") self._assert_ext_status(protocol.report_vm_status, "success", 1, @@ -1014,6 +1051,7 @@ def mock_fail_extension_commands(args, **kwargs): with enable_invocations(dep_ext_level_2, dep_ext_level_1) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() # check handler list and dependency levels self.assertTrue(exthandlers_handler.ext_handlers is not None) @@ -1034,6 +1072,8 @@ def test_ext_handler_sequencing_default_dependency_level(self, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=unused-variable,no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0) self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0) @@ -1048,6 +1088,7 @@ def test_ext_handler_sequencing_invalid_dependency_level(self, *args): exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=unused-variable,no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0) self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0) @@ -1101,6 +1142,8 @@ def test_ext_handler_reporting_status_file(self, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_MULTIPLE_EXT) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=unused-variable,no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + status_path = os.path.join(conf.get_lib_dir(), AGENT_STATUS_FILE) actual_status_json = json.loads(fileutil.read_file(status_path)) @@ -1113,6 +1156,7 @@ def test_ext_handler_rollingupgrade(self, *args): exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -1122,6 +1166,7 @@ def test_ext_handler_rollingupgrade(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -1132,6 +1177,7 @@ def test_ext_handler_rollingupgrade(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.0") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -1142,6 +1188,7 @@ def test_ext_handler_rollingupgrade(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.1") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -1152,6 +1199,7 @@ def test_ext_handler_rollingupgrade(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "NotReady", 1, "1.1.1") @@ -1161,6 +1209,7 @@ def test_ext_handler_rollingupgrade(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_no_handler_status(protocol.report_vm_status) @@ -1169,6 +1218,7 @@ def test_ext_handler_rollingupgrade(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_no_handler_status(protocol.report_vm_status) @@ -1178,6 +1228,7 @@ def test_ext_handler_rollingupgrade(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.1") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -1188,6 +1239,7 @@ def test_ext_handler_rollingupgrade(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.2.0") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -1198,6 +1250,7 @@ def test_ext_handler_rollingupgrade(self, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.1.0") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -1214,6 +1267,8 @@ def test_it_should_create_extension_events_dir_and_set_handler_environment_only_ exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -1244,6 +1299,8 @@ def test_it_should_not_delete_extension_events_directory_on_extension_uninstall( with patch("azurelinuxagent.common.agent_supported_feature._ETPFeature.is_supported", True): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -1254,13 +1311,18 @@ def test_it_should_not_delete_extension_events_directory_on_extension_uninstall( test_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) test_data.set_incarnation(2) protocol.update_goal_state() + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertTrue(os.path.exists(ehi.get_extension_events_dir()), "Events directory should still exist") def test_it_should_uninstall_unregistered_extensions_properly(self, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") # Update version and set it to uninstall. That is how it would be propagated by CRP if a version 1.0.0 is @@ -1271,7 +1333,10 @@ def test_it_should_uninstall_unregistered_extensions_properly(self, *args): test_data.manifest = test_data.manifest.replace("1.0.0", "9.9.9") test_data.set_incarnation(2) protocol.update_goal_state() + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + args, _ = protocol.report_vm_status.call_args vm_status = args[0] self.assertEqual(0, len(vm_status.vmAgent.extensionHandlers), @@ -1285,8 +1350,10 @@ def test_ext_handler_report_status_permanent(self, mock_add_event, mock_error_st protocol.report_vm_status = Mock(side_effect=ProtocolError) mock_error_state.return_value = True + exthandlers_handler.run() - self.assertEqual(5, mock_add_event.call_count) + exthandlers_handler.report_ext_handlers_status() + args, kw = mock_add_event.call_args self.assertEqual(False, kw['is_success']) self.assertTrue("Failed to report vm agent status" in kw['message']) @@ -1299,11 +1366,12 @@ def test_ext_handler_report_status_resource_gone(self, mock_add_event, *args): protocol.report_vm_status = Mock(side_effect=ResourceGoneError) exthandlers_handler.run() - self.assertEqual(4, mock_add_event.call_count) + exthandlers_handler.report_ext_handlers_status() + args, kw = mock_add_event.call_args self.assertEqual(False, kw['is_success']) self.assertTrue("ResourceGoneError" in kw['message']) - self.assertEqual("ExtensionProcessing", kw['op']) + self.assertEqual("ReportStatus", kw['op']) @patch('azurelinuxagent.common.errorstate.ErrorState.is_triggered') @patch('azurelinuxagent.ga.exthandlers.add_event') @@ -1315,6 +1383,8 @@ def test_ext_handler_download_failure_permanent_ProtocolError(self, mock_add_eve mock_error_state.return_value = True exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + event_occurrences = [kw for _, kw in mock_add_event.call_args_list if "[ExtensionError] Failed to get ext handler pkgs" in kw['message']] self.assertEqual(1, len(event_occurrences)) @@ -1322,48 +1392,16 @@ def test_ext_handler_download_failure_permanent_ProtocolError(self, mock_add_eve self.assertTrue("Failed to get ext handler pkgs" in event_occurrences[0]['message']) self.assertTrue("ProtocolError" in event_occurrences[0]['message']) - @patch('azurelinuxagent.ga.exthandlers.add_event') - def test_ext_handler_download_errors_should_be_reported_only_on_new_goal_state(self, mock_add_event, *args): - - def _assert_mock_add_event_call(expected_download_failed_event_count, err_msg_guid): - event_occurrences = [kw for _, kw in mock_add_event.call_args_list if - "Failed to download artifacts: [ExtensionDownloadError] {0}".format(err_msg_guid) in kw['message']] - self.assertEqual(expected_download_failed_event_count, len(event_occurrences), "Call count do not match") - - - self.assertFalse(any(kw['is_success'] for kw in event_occurrences), "The events should have failed") - self.assertEqual(expected_download_failed_event_count, len([kw['op'] for kw in event_occurrences]), - "Incorrect Operation, all events should be a download errors") - - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter - unique_error_message_guid = str(uuid.uuid4()) - protocol.get_ext_handler_pkgs = Mock(side_effect=ExtensionDownloadError(unique_error_message_guid)) - - exthandlers_handler.run() - _assert_mock_add_event_call(expected_download_failed_event_count=1, err_msg_guid=unique_error_message_guid) - self._assert_handler_status(protocol.report_vm_status, "NotReady", 0, "1.0.0") - - # Re-run exthandler.run without updating the GS and ensure we dont report error - exthandlers_handler.run() - _assert_mock_add_event_call(expected_download_failed_event_count=1, err_msg_guid=unique_error_message_guid) - self._assert_handler_status(protocol.report_vm_status, "NotReady", 0, "1.0.0") - - # Change incarnation and then re-check we report error - test_data.set_incarnation(2) - protocol.update_goal_state() - - exthandlers_handler.run() - _assert_mock_add_event_call(expected_download_failed_event_count=2, err_msg_guid=unique_error_message_guid) - self._assert_handler_status(protocol.report_vm_status, "NotReady", 0, "1.0.0") - @patch('azurelinuxagent.ga.exthandlers.fileutil') def test_ext_handler_io_error(self, mock_fileutil, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=unused-variable,no-value-for-parameter mock_fileutil.write_file.return_value = IOError("Mock IO Error") + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + def test_it_should_process_extensions_only_if_allowed(self, mock_get, mock_crypt, *args): @@ -1379,6 +1417,8 @@ def mock_popen(*args, **kwargs): with patch('subprocess.Popen', side_effect=mock_popen): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(expected_call_count, len(extension_calls), "Call counts dont match") test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) @@ -1388,9 +1428,6 @@ def mock_popen(*args, **kwargs): assert_extensions_called(exthandlers_handler, expected_call_count=2) self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") - # We should not re-handle the extensions if GoalState didn't change - assert_extensions_called(exthandlers_handler, expected_call_count=0) - # Update GoalState test_data.set_incarnation(2) protocol.update_goal_state() @@ -1435,7 +1472,10 @@ def test_it_should_process_extensions_appropriately_on_artifact_hold(self, mock_ # Test when is_on_hold returns True - should not work mock_in_vm_artifacts_profile.is_on_hold = Mock(return_value=True) protocol.get_artifacts_profile = Mock(return_value=mock_in_vm_artifacts_profile) + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + vm_agent_status = protocol.report_vm_status.call_args[0][0].vmAgent self.assertEqual(vm_agent_status.status, "Ready", "Agent should report ready") self.assertEqual(0, len(vm_agent_status.extensionHandlers), @@ -1447,6 +1487,8 @@ def test_it_should_process_extensions_appropriately_on_artifact_hold(self, mock_ mock_in_vm_artifacts_profile.is_on_hold = Mock(return_value=False) protocol.get_artifacts_profile = Mock(return_value=mock_in_vm_artifacts_profile) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self.assertEqual("1", protocol.report_vm_status.call_args[0][ 0].vmAgent.vm_artifacts_aggregate_status.goal_state_aggregate_status.in_svd_seq_no, "Incarnation mismatch") @@ -1456,36 +1498,22 @@ def test_it_should_process_extensions_appropriately_on_artifact_hold(self, mock_ # Update GoalState test_data.set_incarnation(2) protocol.update_goal_state() + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self.assertEqual("2", protocol.report_vm_status.call_args[0][ 0].vmAgent.vm_artifacts_aggregate_status.goal_state_aggregate_status.in_svd_seq_no, "Incarnation mismatch") - def test_last_etag_on_extension_processing(self, *args): - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter - exthandlers_handler.ext_handlers, etag = protocol.get_ext_handlers() - exthandlers_handler.protocol = protocol - - # Disable extension handling blocking in the first run and enable in the 2nd run - with patch.object(exthandlers_handler, '_extension_processing_allowed', side_effect=[False, True]): - exthandlers_handler.run() - self.assertIsNone(exthandlers_handler.last_etag, - "The last etag should be None initially as extension_processing is False") - self.assertNotEqual(etag, exthandlers_handler.last_etag, - "Last etag and etag should not be same if extension processing is disabled") - exthandlers_handler.run() - self.assertIsNotNone(exthandlers_handler.last_etag, - "Last etag should not be none if extension processing is allowed") - self.assertEqual(etag, exthandlers_handler.last_etag, - "Last etag and etag should be same if extension processing is enabled") - def test_it_should_parse_valid_in_vm_metadata_properly(self, mock_get, mock_crypt, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_IN_VM_META_DATA) exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") activity_id, correlation_id, gs_creation_time = exthandlers_handler.get_goal_state_debug_metadata() self.assertEqual(activity_id, "555e551c-600e-4fb4-90ba-8ab8ec28eccc", "Incorrect activity Id") @@ -1499,6 +1527,8 @@ def test_it_should_process_goal_state_even_if_metadata_missing(self, mock_get, m exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") activity_id, correlation_id, gs_creation_time = exthandlers_handler.get_goal_state_debug_metadata() self.assertEqual(activity_id, "NA", "Activity Id should be NA") @@ -1511,6 +1541,8 @@ def test_it_should_process_goal_state_even_if_metadata_invalid(self, mock_get, m exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") def _assert_ext_status(self, vm_agent_status, expected_status, @@ -1531,6 +1563,8 @@ def test_it_should_initialise_and_use_command_execution_log_for_extensions(self, test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt_util, *args) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") command_execution_log = os.path.join(conf.get_ext_log_dir(), "OSTCExtensions.ExampleHandlerLinux", @@ -1542,6 +1576,8 @@ def test_ext_handler_no_reporting_status(self, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") # Remove status file and re-run collecting extension status @@ -1551,7 +1587,8 @@ def test_ext_handler_no_reporting_status(self, *args): self.assertTrue(os.path.isfile(status_file)) os.remove(status_file) - exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.error, 0) @@ -1578,6 +1615,8 @@ def mock_popen(cmd, *args, **kwargs): with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): with patch('azurelinuxagent.ga.exthandlers._DEFAULT_EXT_TIMEOUT_MINUTES', 0.001): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + # The Handler Status for the base extension should be ready as it was executed successfully by the agent self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", @@ -1619,6 +1658,7 @@ def mock_popen(cmd, *args, **kwargs): with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() # The Handler Status for the base extension should be ready as it was executed successfully by the agent self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", @@ -1637,6 +1677,7 @@ def test_wait_for_handler_completion_success_status(self, mock_http_get, mock_cr mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SEQUENCING), mock_http_get, mock_crypt_util, *args) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux", @@ -1668,6 +1709,7 @@ def mock_popen(cmd, *args, **kwargs): with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() # The Handler Status for the base extension should be NotReady as it failed self._assert_handler_status(protocol.report_vm_status, "NotReady", 0, "1.0.0", @@ -1830,13 +1872,18 @@ def test_extensions_disabled(self, _, *args): # test status is reported for no extensions test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_NO_EXT) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_no_handler_status(protocol.report_vm_status) # test status is reported, but extensions are not processed test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE) exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_no_handler_status(protocol.report_vm_status) def test_extensions_deleted(self, *args): @@ -1845,6 +1892,7 @@ def test_extensions_deleted(self, *args): exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -1857,6 +1905,7 @@ def test_extensions_deleted(self, *args): # Ensure new extension can be enabled exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.1") self._assert_ext_status(protocol.report_vm_status, "success", 0) @@ -1873,17 +1922,14 @@ def test_install_failure(self, patch_get_install_command, patch_install, *args): # Ensure initial install is unsuccessful patch_get_install_command.return_value = "exit.sh 1" + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(1, patch_install.call_count) self.assertEqual(1, protocol.report_vm_status.call_count) self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=0, version="1.0.0") - # Ensure subsequent no further retries are made - exthandlers_handler.run() - self.assertEqual(1, patch_install.call_count) - self.assertEqual(2, protocol.report_vm_status.call_count) - @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_install_command') def test_install_failure_check_exception_handling(self, patch_get_install_command, *args): """ @@ -1895,32 +1941,12 @@ def test_install_failure_check_exception_handling(self, patch_get_install_comman # Ensure install is unsuccessful patch_get_install_command.return_value = "exit.sh 1" exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(1, protocol.report_vm_status.call_count) self._assert_handler_status(protocol.report_vm_status, expected_status="NotReady", expected_ext_count=0, version="1.0.0") - @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_enable_command') - def test_enable_failure(self, patch_get_enable_command, *args): - """ - When extension enable fails, the operation should not be retried. - """ - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SINGLE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter - - # Ensure initial install is successful, but enable fails - patch_get_enable_command.call_count = 0 - patch_get_enable_command.return_value = "exit.sh 1" - exthandlers_handler.run() - - self.assertEqual(1, patch_get_enable_command.call_count) - self.assertEqual(1, protocol.report_vm_status.call_count) - self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.0") - - exthandlers_handler.run() - self.assertEqual(1, patch_get_enable_command.call_count) - self.assertEqual(2, protocol.report_vm_status.call_count) - @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_enable_command') def test_enable_failure_check_exception_handling(self, patch_get_enable_command, *args): """ @@ -1933,47 +1959,12 @@ def test_enable_failure_check_exception_handling(self, patch_get_enable_command, patch_get_enable_command.call_count = 0 patch_get_enable_command.return_value = "exit.sh 1" exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(1, patch_get_enable_command.call_count) self.assertEqual(1, protocol.report_vm_status.call_count) self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.0") - @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_disable_command') - def test_disable_failure(self, patch_get_disable_command, *args): - """ - When extension disable fails, the operation should not be retried. - """ - # Ensure initial install and enable is successful, but disable fails - test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SINGLE) - exthandlers_handler, protocol = self._create_mock(test_data, *args) # pylint: disable=no-value-for-parameter - patch_get_disable_command.call_count = 0 - patch_get_disable_command.return_value = "exit.sh 1" - - exthandlers_handler.run() - - self.assertEqual(0, patch_get_disable_command.call_count) - self.assertEqual(1, protocol.report_vm_status.call_count) - self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.0") - self._assert_ext_status(protocol.report_vm_status, "success", 0) - - # Next incarnation, disable extension - test_data.set_incarnation(2) - test_data.set_extensions_config_state(ExtHandlerRequestedState.Disabled) - protocol.update_goal_state() - - exthandlers_handler.run() - - self.assertEqual(1, patch_get_disable_command.call_count) - self.assertEqual(2, protocol.report_vm_status.call_count) - self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.0") - - # Ensure there are no further retries - exthandlers_handler.run() - - self.assertEqual(1, patch_get_disable_command.call_count) - self.assertEqual(3, protocol.report_vm_status.call_count) - self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.0") - @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_disable_command') def test_disable_failure_with_exception_handling(self, patch_get_disable_command, *args): @@ -1987,6 +1978,7 @@ def test_disable_failure_with_exception_handling(self, patch_get_disable_command patch_get_disable_command.return_value = "exit 1" exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(0, patch_get_disable_command.call_count) self.assertEqual(1, protocol.report_vm_status.call_count) @@ -1999,6 +1991,7 @@ def test_disable_failure_with_exception_handling(self, patch_get_disable_command protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(1, patch_get_disable_command.call_count) self.assertEqual(2, protocol.report_vm_status.call_count) @@ -2016,6 +2009,7 @@ def test_uninstall_failure(self, patch_get_uninstall_command, *args): patch_get_uninstall_command.return_value = "exit 1" exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(0, patch_get_uninstall_command.call_count) self.assertEqual(1, protocol.report_vm_status.call_count) @@ -2028,6 +2022,7 @@ def test_uninstall_failure(self, patch_get_uninstall_command, *args): protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(1, patch_get_uninstall_command.call_count) self.assertEqual(2, protocol.report_vm_status.call_count) @@ -2036,6 +2031,7 @@ def test_uninstall_failure(self, patch_get_uninstall_command, *args): # Ensure there are no further retries exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(1, patch_get_uninstall_command.call_count) self.assertEqual(3, protocol.report_vm_status.call_count) @@ -2064,6 +2060,8 @@ def mock_popen(*args, **kwargs): with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + update_command_count = len([extension_call for extension_call in extension_calls if patch_get_update_command.return_value in extension_call]) enable_command_count = len([extension_call for extension_call in extension_calls @@ -2075,24 +2073,13 @@ def mock_popen(*args, **kwargs): # We report the failure of the new extension version self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.1") - # Ensure we are processing the same goal state only once - loop_run = 5 - for x in range(loop_run): # pylint: disable=unused-variable - exthandlers_handler.run() - - update_command_count = len([extension_call for extension_call in extension_calls - if patch_get_update_command.return_value in extension_call]) - enable_command_count = len([extension_call for extension_call in extension_calls - if "-enable" in extension_call]) - self.assertEqual(1, update_command_count) - self.assertEqual(0, enable_command_count) - # If the incarnation number changes (there's a new goal state), ensure we go through the entire upgrade # process again. test_data.set_incarnation(3) protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() update_command_count = len([extension_call for extension_call in extension_calls if patch_get_update_command.return_value in extension_call]) @@ -2110,6 +2097,7 @@ def test_extension_upgrade_failure_when_prev_version_disable_fails(self, patch_g with patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_enable_command') as patch_get_enable_command: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() # When the previous version's disable fails, we expect the upgrade scenario to fail, so the enable # for the new version is not called and the new version handler's status is reported as not ready. @@ -2117,14 +2105,6 @@ def test_extension_upgrade_failure_when_prev_version_disable_fails(self, patch_g self.assertEqual(0, patch_get_enable_command.call_count) self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=0, version="1.0.1") - # Ensure we are processing the same goal state only once - loop_run = 5 - for x in range(loop_run): # pylint: disable=unused-variable - exthandlers_handler.run() - - self.assertEqual(1, patch_get_disable_command.call_count) - self.assertEqual(0, patch_get_enable_command.call_count) - @patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_disable_command') def test_extension_upgrade_failure_when_prev_version_disable_fails_and_recovers_on_next_incarnation(self, patch_get_disable_command, *args): @@ -2133,6 +2113,7 @@ def test_extension_upgrade_failure_when_prev_version_disable_fails_and_recovers_ with patch('azurelinuxagent.ga.exthandlers.HandlerManifest.get_enable_command') as patch_get_enable_command: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() # When the previous version's disable fails, we expect the upgrade scenario to fail, so the enable # for the new version is not called and the new version handler's status is reported as not ready. @@ -2140,14 +2121,6 @@ def test_extension_upgrade_failure_when_prev_version_disable_fails_and_recovers_ self.assertEqual(0, patch_get_enable_command.call_count) self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=0, version="1.0.1") - # Ensure we are processing the same goal state only once - loop_run = 5 - for x in range(loop_run): # pylint: disable=unused-variable - exthandlers_handler.run() - - self.assertEqual(1, patch_get_disable_command.call_count) - self.assertEqual(0, patch_get_enable_command.call_count) - # Force a new goal state incarnation, only then will we attempt the upgrade again test_data.set_incarnation(3) protocol.update_goal_state() @@ -2155,6 +2128,8 @@ def test_extension_upgrade_failure_when_prev_version_disable_fails_and_recovers_ # Ensure disable won't fail by making launch_command a no-op with patch('azurelinuxagent.ga.exthandlers.ExtHandlerInstance.launch_command') as patch_launch_command: # pylint: disable=unused-variable exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(2, patch_get_disable_command.call_count) self.assertEqual(1, patch_get_enable_command.call_count) self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.1") @@ -2173,6 +2148,8 @@ def test_extension_upgrade_failure_when_prev_version_disable_fails_incorrect_zip patch_zipfile_extractall.side_effect = raise_ioerror # The zipfile was corrupt and the upgrade sequence failed exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + # We never called the disable of the old version due to the failure when unzipping the new version, # nor the enable of the new version @@ -2183,6 +2160,7 @@ def test_extension_upgrade_failure_when_prev_version_disable_fails_incorrect_zip loop_run = 5 for x in range(loop_run): # pylint: disable=unused-variable exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(0, patch_get_disable_command.call_count) self.assertEqual(0, patch_get_enable_command.call_count) @@ -2194,7 +2172,9 @@ def test_old_handler_reports_failure_on_disable_fail_on_update(self, patch_get_d *args) with patch.object(ExtHandlerInstance, "report_event", autospec=True) as patch_report_event: - exthandlers_handler.run() # Download the new update the first time, and then we patch the download method. + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(1, patch_get_disable_command.call_count) old_version_args, old_version_kwargs = patch_report_event.call_args @@ -2227,6 +2207,8 @@ def test_upgrade_failure_with_exception_handling(self, patch_get_update_command, *args) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(1, patch_get_update_command.call_count) self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.1") @@ -2240,6 +2222,8 @@ def test_extension_upgrade_should_pass_when_continue_on_update_failure_is_true_a as mock_continue_on_update_failure: # These are just testing the mocks have been called and asserting the test conditions have been met exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(1, patch_get_disable_command.call_count) self.assertEqual(2, mock_continue_on_update_failure.call_count, "This should be called twice, for both disable and uninstall") @@ -2258,6 +2242,8 @@ def test_extension_upgrade_should_pass_when_continue_on_update_failue_is_true_an as mock_continue_on_update_failure: # These are just testing the mocks have been called and asserting the test conditions have been met exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(1, patch_get_uninstall_command.call_count) self.assertEqual(2, mock_continue_on_update_failure.call_count, "This should be called twice, for both disable and uninstall") @@ -2276,6 +2262,8 @@ def test_extension_upgrade_should_fail_when_continue_on_update_failure_is_false_ as mock_continue_on_update_failure: # These are just testing the mocks have been called and asserting the test conditions have been met exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(1, patch_get_disable_command.call_count) self.assertEqual(1, mock_continue_on_update_failure.call_count, "The first call would raise an exception") @@ -2293,6 +2281,8 @@ def test_extension_upgrade_should_fail_when_continue_on_update_failure_is_false_ as mock_continue_on_update_failure: # These are just testing the mocks have been called and asserting the test conditions have been met exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(1, patch_get_uninstall_command.call_count) self.assertEqual(2, mock_continue_on_update_failure.call_count, "The second call would raise an exception") @@ -2312,6 +2302,8 @@ def test_extension_upgrade_should_fail_when_continue_on_update_failure_is_true_a as patch_get_enable: # These are just testing the mocks have been called and asserting the test conditions have been met exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertEqual(1, patch_get_disable_command.call_count) self.assertEqual(2, mock_continue_on_update_failure.call_count) self.assertEqual(1, patch_get_enable.call_count) @@ -2328,6 +2320,8 @@ def test_uninstall_rc_env_var_should_report_not_run_for_non_update_calls_to_exth side_effect=[ExtensionError("Disable Failed"), "ok", ExtensionError("uninstall failed"), "ok", "ok", "New enable run ok"]) as patch_start_cmd: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + _, update_kwargs = patch_start_cmd.call_args_list[1] _, install_kwargs = patch_start_cmd.call_args_list[3] _, enable_kwargs = patch_start_cmd.call_args_list[4] @@ -2355,6 +2349,8 @@ def test_uninstall_rc_env_var_should_report_not_run_for_non_update_calls_to_exth protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + _, new_enable_kwargs = patch_start_cmd.call_args # Ensure the new run didn't have Disable Return Code env variable @@ -2375,6 +2371,7 @@ def test_ext_path_and_version_env_variables_set_for_ever_operation(self, *args): with patch.object(CGroupConfigurator.get_instance(), "start_extension_command") as patch_start_cmd: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() # Extension Path and Version should be set for all launch_command calls for args, kwargs in patch_start_cmd.call_args_list: @@ -2393,6 +2390,7 @@ def test_ext_sequence_no_should_be_set_for_every_command_call(self, _, *args): with patch("subprocess.Popen") as patch_popen: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() for _, kwargs in patch_popen.call_args_list: self.assertIn(ExtCommandEnvVariable.ExtensionSeqNumber, kwargs['env']) @@ -2409,6 +2407,7 @@ def test_ext_sequence_no_should_be_set_for_every_command_call(self, _, *args): with patch("subprocess.Popen") as patch_popen: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() for _, kwargs in patch_popen.call_args_list: self.assertIn(ExtCommandEnvVariable.ExtensionSeqNumber, kwargs['env']) @@ -2448,6 +2447,7 @@ def test_ext_sequence_no_should_be_set_from_within_extension(self, *args): with patch.object(ExtHandlerInstance, "load_manifest", return_value=manifest): with patch.object(ExtHandlerInstance, 'report_event') as mock_report_event: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() for _, kwargs in mock_report_event.call_args_list: # The output is of the format - 'testfile.sh\n[stdout]ConfigSequenceNumber=N\n[stderr]' @@ -2469,6 +2469,7 @@ def test_ext_sequence_no_should_be_set_from_within_extension(self, *args): with patch.object(ExtHandlerInstance, 'report_event') as mock_report_event: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() for _, kwargs in mock_report_event.call_args_list: # The output is of the format - 'testfile.sh\n[stdout]ConfigSequenceNumber=N\n[stderr]' @@ -2517,6 +2518,8 @@ def test_correct_exit_code_should_be_set_on_uninstall_cmd_failure(self, *args): with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.load_manifest", return_value=manifest): with patch.object(ExtHandlerInstance, 'report_event') as mock_report_event: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + exthandlers_handler.report_ext_handlers_status() _, disable_kwargs = mock_report_event.call_args_list[1] # pylint: disable=unused-variable _, update_kwargs = mock_report_event.call_args_list[2] @@ -2533,6 +2536,8 @@ def test_it_should_persist_goal_state_aggregate_status_until_new_incarnation(sel exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt_util, *args) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.0") args, _ = protocol.report_vm_status.call_args gs_aggregate_status = args[0].vmAgent.vm_artifacts_aggregate_status.goal_state_aggregate_status @@ -2540,18 +2545,12 @@ def test_it_should_persist_goal_state_aggregate_status_until_new_incarnation(sel self.assertEqual(gs_aggregate_status.status, GoalStateStatus.Success, "Wrong status reported") self.assertEqual(gs_aggregate_status.in_svd_seq_no, "1", "Incorrect seq no") - # Running handler again without incarnation change should report the same gs_aggregate_status - for _ in range(5): - exthandlers_handler.run() - args, _ = protocol.report_vm_status.call_args - self.assertEqual(gs_aggregate_status, - args[0].vmAgent.vm_artifacts_aggregate_status.goal_state_aggregate_status, - "Aggregate status different") - # Update incarnation and ensure the gs_aggregate_status is modified too test_data.set_incarnation(2) protocol.client.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self._assert_handler_status(protocol.report_vm_status, "Ready", expected_ext_count=1, version="1.0.0") args, _ = protocol.report_vm_status.call_args new_gs_aggregate_status = args[0].vmAgent.vm_artifacts_aggregate_status.goal_state_aggregate_status @@ -2578,6 +2577,8 @@ def test_it_should_fail_goal_state_if_required_features_not_supported(self, mock exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt_util, *args) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + args, _ = protocol.report_vm_status.call_args gs_aggregate_status = args[0].vmAgent.vm_artifacts_aggregate_status.goal_state_aggregate_status self.assertEqual(0, len(args[0].vmAgent.extensionHandlers), "No extensions should be reported") @@ -2661,6 +2662,7 @@ def get_ext_handling_status(ext): with patch.object(ExtHandlerInstance, "get_handler_status", ExtHandlerStatus): with patch('azurelinuxagent.ga.exthandlers._DEFAULT_EXT_TIMEOUT_MINUTES', 0.01): exthandlers_handler.run() + self._validate_extension_sequence(expected_sequence, exthandlers_handler) def test_handle_ext_handlers(self, *args): @@ -2820,6 +2822,7 @@ def _do_upgrade_scenario_and_get_order(first_ext, upgraded_ext): with enable_invocations(first_ext, upgraded_ext) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() invocation_record.compare( (first_ext, ExtensionCommandNames.INSTALL), @@ -2834,6 +2837,8 @@ def _do_upgrade_scenario_and_get_order(first_ext, upgraded_ext): with enable_invocations(first_ext, upgraded_ext) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + return invocation_record @@ -3360,8 +3365,7 @@ def manifest_location_handler(url, **kwargs): if manifests_used: # Still locations to try in the list; throw a fake # error to make sure all of the locations get called. - return Exception("Failing manifest fetch from uri '{0}' for testing purposes."\ - .format(url)) + return Exception("Failing manifest fetch from uri '{0}' for testing purposes.".format(url)) return None @@ -3369,6 +3373,7 @@ def manifest_location_handler(url, **kwargs): with mock_wire_protocol(self.test_data, http_get_handler=manifest_location_handler) as protocol: exthandlers_handler = get_exthandlers_handler(protocol) exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() def test_fetch_manifest_timeout_is_respected(self): diff --git a/tests/ga/test_multi_config_extension.py b/tests/ga/test_multi_config_extension.py index a6a61345e..00183bf7c 100644 --- a/tests/ga/test_multi_config_extension.py +++ b/tests/ga/test_multi_config_extension.py @@ -221,6 +221,7 @@ def get_message(msg): return msg if with_message else None exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(no_of_extensions, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), "incorrect extensions reported") @@ -252,6 +253,7 @@ def __setup_and_assert_disable_scenario(self, exthandlers_handler, protocol): protocol.mock_wire_data.set_incarnation(2) protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, handler_name="OSTCExtensions.ExampleHandlerLinux", @@ -285,6 +287,7 @@ def __setup_generic_test_env(self): with self._setup_test_env(mock_manifest=True) as (exthandlers_handler, protocol, no_of_extensions): with enable_invocations(first_ext, second_ext, third_ext, fourth_ext) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(no_of_extensions, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), "incorrect extensions reported") @@ -316,6 +319,7 @@ def test_it_should_execute_and_report_multi_config_extensions_properly(self): protocol.mock_wire_data.set_extensions_config_state(ExtHandlerRequestedState.Uninstall) protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(0, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), "No handler/extension status should be reported") @@ -329,6 +333,7 @@ def test_it_should_report_unregistered_version_error_per_extension(self): protocol.mock_wire_data.set_incarnation(2) protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(no_of_extensions, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), "incorrect extensions reported") @@ -370,6 +375,7 @@ def test_it_should_retry_handler_installation_per_extension_if_failed(self): sc_ext = extension_emulator(name="Microsoft.Powershell.ExampleExtension", install_action=fail_action) with enable_invocations(first_ext, second_ext, third_ext, sc_ext) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(no_of_extensions, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), "incorrect extensions reported") @@ -415,6 +421,7 @@ def test_it_should_only_disable_enabled_extensions_on_update(self): new_sc_ext = extension_emulator(name="Microsoft.Powershell.ExampleExtension", version=new_version) with enable_invocations(new_first_ext, new_second_ext, new_third_ext, new_sc_ext, *old_exts) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() old_first, old_second, old_third, old_fourth = old_exts invocation_record.compare( # Disable all enabled commands for MC before updating the Handler @@ -471,6 +478,7 @@ def test_it_should_retry_update_sequence_per_extension_if_previous_failed(self): with enable_invocations(new_first_ext, new_second_ext, new_third_ext, new_sc_ext, *old_exts) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() old_first, old_second, old_third, old_fourth = old_exts invocation_record.compare( # Disable all enabled commands for MC before updating the Handler @@ -537,6 +545,7 @@ def test_it_should_report_disabled_extension_errors_if_update_failed(self): with enable_invocations(new_first_ext, new_second_ext, new_third_ext, new_fourth_ext, *old_exts) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() old_first, _, _, old_fourth = old_exts invocation_record.compare( # Disable for firstExtension should fail 3 times, i.e., once per extension which tries to update the Handler @@ -590,6 +599,7 @@ def test_it_should_handle_and_report_enable_errors_properly(self): fourth_ext = extension_emulator(name="Microsoft.Powershell.ExampleExtension", enable_action=fail_action) with enable_invocations(first_ext, second_ext, third_ext, fourth_ext) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(no_of_extensions, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), "incorrect extensions reported") @@ -646,6 +656,7 @@ def __assert_state_file(handler_name, handler_version, extensions, state, not_pr protocol.update_goal_state() ext_handler.run() + ext_handler.report_ext_handlers_status() mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, handler_name="OSTCExtensions.ExampleHandlerLinux", expected_count=2, status="Ready") @@ -767,6 +778,7 @@ def mock_popen(cmd, *_, **kwargs): protocol.mock_wire_data.set_incarnation(2) protocol.update_goal_state() exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() mc_handlers = self._assert_and_get_handler_status(aggregate_status=protocol.aggregate_status, handler_name="OSTCExtensions.ExampleHandlerLinux", @@ -978,6 +990,7 @@ def test_it_should_fail_handler_if_handler_does_not_support_mc(self): with self._setup_test_env() as (exthandlers_handler, protocol, no_of_extensions): with enable_invocations(first_ext, second_ext, third_ext, fourth_ext) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(no_of_extensions, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), "incorrect extensions reported") @@ -1017,6 +1030,7 @@ def test_it_should_check_every_time_if_handler_supports_mc(self): with enable_invocations(*old_exts) as invocation_record: (_, _, _, fourth_ext) = old_exts exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(4, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), "incorrect extensions reported") @@ -1069,6 +1083,7 @@ def test_it_should_process_dependency_chain_extensions_properly(self): independent_sc_ext): with enable_invocations(first_ext, second_ext, third_ext, dependent_sc_ext, independent_sc_ext) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(no_of_extensions, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), "incorrect extensions reported") @@ -1144,6 +1159,7 @@ def test_it_should_report_extension_status_failures_for_all_dependent_extensions with enable_invocations(first_ext, second_ext, third_ext, dependent_sc_ext, independent_sc_ext) as invocation_record: exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(no_of_extensions, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), "incorrect extensions reported") @@ -1197,6 +1213,7 @@ def mock_popen(cmd, *_, **kwargs): with self._setup_test_env(mock_manifest=True) as (exthandlers_handler, protocol, no_of_extensions): with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() self.assertEqual(no_of_extensions, len(protocol.aggregate_status['aggregateStatus']['handlerAggregateStatus']), diff --git a/tests/ga/test_remoteaccess_handler.py b/tests/ga/test_remoteaccess_handler.py index dbe925cc5..1e8e76e21 100644 --- a/tests/ga/test_remoteaccess_handler.py +++ b/tests/ga/test_remoteaccess_handler.py @@ -469,7 +469,7 @@ def test_handle_remote_access_remove_and_add(self, _): def test_remote_access_handler_run_error(self, _): with patch("azurelinuxagent.ga.remoteaccess.get_osutil", return_value=MockOSUtil()): mock_protocol = WireProtocol("foo.bar") - mock_protocol.get_incarnation = MagicMock(side_effect=Exception("foobar!")) + mock_protocol.client.get_remote_access = MagicMock(side_effect=Exception("foobar!")) rah = RemoteAccessHandler(mock_protocol) rah.run() diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index d2bda25ae..e700f43dd 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -43,7 +43,7 @@ CHILD_LAUNCH_RESTART_MAX, CHILD_HEALTH_INTERVAL, UpdateHandler from tests.protocol.mocks import mock_wire_protocol from tests.protocol.mockwiredata import DATA_FILE, DATA_FILE_MULTIPLE_EXT -from tests.tools import AgentTestCase, call, data_dir, DEFAULT, patch, load_bin_data, load_data, Mock, MagicMock, \ +from tests.tools import AgentTestCase, data_dir, DEFAULT, patch, load_bin_data, load_data, Mock, MagicMock, \ clear_singleton_instances, mock_sleep NO_ERROR = { @@ -717,15 +717,16 @@ def setUp(self): UpdateTestCase.setUp(self) self.event_patch = patch('azurelinuxagent.common.event.add_event') self.update_handler = get_update_handler() + protocol = Mock() + protocol.get_ext_handlers = Mock(return_value=(Mock(), Mock())) self.update_handler.protocol_util = Mock() + self.update_handler.protocol_util.get_protocol = Mock(return_value=protocol) # Since ProtocolUtil is a singleton per thread, we need to clear it to ensure that the test cases do not reuse # a previous state clear_singleton_instances(ProtocolUtil) def test_creation(self): - self.assertTrue(self.update_handler.running) - self.assertEqual(None, self.update_handler.last_attempt_time) self.assertEqual(0, len(self.update_handler.agents)) @@ -1246,7 +1247,7 @@ def test_run_latest_exception_does_not_blacklist_if_terminating(self): self.assertEqual(0, latest_agent.error.failure_count) with patch('azurelinuxagent.ga.update.UpdateHandler.get_latest_agent', return_value=latest_agent): - self.update_handler.running = False + self.update_handler.is_running = False self._test_run_latest(mock_child=ChildMock(side_effect=Exception("Attempt blacklisting"))) self.assertTrue(latest_agent.is_available) @@ -1265,71 +1266,61 @@ def test_run_latest_creates_only_one_signal_handler(self, mock_signal): self._test_run_latest() self.assertEqual(0, mock_signal.call_count) - def _test_run(self, invocations=1, calls=None, enable_updates=False, sleep_interval=(6,)): - if calls is None: - calls = [call.run()] + def _test_run(self, invocations=1, calls=1, enable_updates=False, sleep_interval=(6,)): conf.get_autoupdate_enabled = Mock(return_value=enable_updates) - # Note: - # - Python only allows mutations of objects to which a function has - # a reference. Incrementing an integer directly changes the - # reference. Incrementing an item of a list changes an item to - # which the code has a reference. - # See http://stackoverflow.com/questions/26408941/python-nested-functions-and-variable-scope - iterations = [0] - - def iterator(*args, **kwargs): # pylint: disable=unused-argument - iterations[0] += 1 - if iterations[0] >= invocations: - self.update_handler.running = False - return + def iterator(*_, **__): + iterator.count += 1 + if iterator.count <= invocations: + return True + return False + iterator.count = 0 fileutil.write_file(conf.get_agent_pid_file_path(), ustr(42)) with patch('azurelinuxagent.ga.exthandlers.get_exthandlers_handler') as mock_handler: - with patch('azurelinuxagent.ga.remoteaccess.get_remote_access_handler') as mock_ra_handler: - with patch('azurelinuxagent.ga.update.get_monitor_handler') as mock_monitor: - with patch('azurelinuxagent.ga.update.get_env_handler') as mock_env: - with patch('azurelinuxagent.ga.update.get_collect_logs_handler') as mock_collect_logs: - with patch('azurelinuxagent.ga.update.get_send_telemetry_events_handler') as mock_telemetry_send_events: - with patch('azurelinuxagent.ga.update.get_collect_telemetry_events_handler') as mock_event_collector: - with patch('azurelinuxagent.ga.update.initialize_event_logger_vminfo_common_parameters'): - with patch('azurelinuxagent.ga.update.is_log_collection_allowed', return_value=True): - with patch('time.sleep', side_effect=iterator) as sleep_mock: - with patch('sys.exit') as mock_exit: - if isinstance(os.getppid, MagicMock): - self.update_handler.run() - else: - with patch('os.getppid', return_value=42): + mock_handler.run_ext_handlers = Mock() + with patch('azurelinuxagent.ga.update.get_monitor_handler') as mock_monitor: + with patch.object(UpdateHandler, 'is_running') as mock_is_running: + mock_is_running.__get__ = Mock(side_effect=iterator) + with patch('azurelinuxagent.ga.remoteaccess.get_remote_access_handler') as mock_ra_handler: + with patch('azurelinuxagent.ga.update.get_env_handler') as mock_env: + with patch('azurelinuxagent.ga.update.get_collect_logs_handler') as mock_collect_logs: + with patch('azurelinuxagent.ga.update.get_send_telemetry_events_handler') as mock_telemetry_send_events: + with patch('azurelinuxagent.ga.update.get_collect_telemetry_events_handler') as mock_event_collector: + with patch('azurelinuxagent.ga.update.initialize_event_logger_vminfo_common_parameters'): + with patch('azurelinuxagent.ga.update.is_log_collection_allowed', return_value=True): + with patch('time.sleep') as sleep_mock: + with patch('sys.exit') as mock_exit: + if isinstance(os.getppid, MagicMock): self.update_handler.run() - - self.assertEqual(1, mock_handler.call_count) - self.assertEqual(mock_handler.return_value.method_calls, calls) - self.assertEqual(1, mock_ra_handler.call_count) - self.assertEqual(mock_ra_handler.return_value.method_calls, calls) - self.assertEqual(invocations, sleep_mock.call_count) - if invocations > 0: - self.assertEqual(sleep_interval, sleep_mock.call_args[0]) - self.assertEqual(1, mock_monitor.call_count) - self.assertEqual(1, mock_env.call_count) - self.assertEqual(1, mock_collect_logs.call_count) - self.assertEqual(1, mock_telemetry_send_events.call_count) - self.assertEqual(1, mock_event_collector.call_count) - self.assertEqual(1, mock_exit.call_count) + else: + with patch('os.getppid', return_value=42): + self.update_handler.run() + + self.assertEqual(1, mock_handler.call_count) + self.assertEqual(calls, len([c for c in [call[0] for call in mock_handler.return_value.method_calls] if c == 'run'])) + self.assertEqual(1, mock_ra_handler.call_count) + self.assertEqual(calls, len(mock_ra_handler.return_value.method_calls)) + if calls > 0: + self.assertEqual(sleep_interval, sleep_mock.call_args[0]) + self.assertEqual(1, mock_monitor.call_count) + self.assertEqual(1, mock_env.call_count) + self.assertEqual(1, mock_collect_logs.call_count) + self.assertEqual(1, mock_telemetry_send_events.call_count) + self.assertEqual(1, mock_event_collector.call_count) + self.assertEqual(1, mock_exit.call_count) def test_run(self): self._test_run() - def test_run_keeps_running(self): - self._test_run(invocations=15, calls=[call.run()] * 15) - def test_run_stops_if_update_available(self): self.update_handler._upgrade_available = Mock(return_value=True) - self._test_run(invocations=0, calls=[], enable_updates=True) + self._test_run(invocations=0, calls=0, enable_updates=True) def test_run_stops_if_orphaned(self): with patch('os.getppid', return_value=1): - self._test_run(invocations=0, calls=[], enable_updates=True) + self._test_run(invocations=0, calls=0, enable_updates=True) def test_run_clears_sentinel_on_successful_exit(self): self._test_run() @@ -1337,7 +1328,7 @@ def test_run_clears_sentinel_on_successful_exit(self): def test_run_leaves_sentinel_on_unsuccessful_exit(self): self.update_handler._upgrade_available = Mock(side_effect=Exception) - self._test_run(invocations=0, calls=[], enable_updates=True) + self._test_run(invocations=1, calls=0, enable_updates=True) self.assertTrue(os.path.isfile(self.update_handler._sentinel_file_path())) def test_run_emits_restart_event(self): @@ -1376,13 +1367,13 @@ def test_set_sentinel_writes_current_agent(self): def test_shutdown(self): self.update_handler._set_sentinel() self.update_handler._shutdown() - self.assertFalse(self.update_handler.running) + self.assertFalse(self.update_handler.is_running) self.assertFalse(os.path.isfile(self.update_handler._sentinel_file_path())) def test_shutdown_ignores_missing_sentinel_file(self): self.assertFalse(os.path.isfile(self.update_handler._sentinel_file_path())) self.update_handler._shutdown() - self.assertFalse(self.update_handler.running) + self.assertFalse(self.update_handler.is_running) self.assertFalse(os.path.isfile(self.update_handler._sentinel_file_path())) def test_shutdown_ignores_exceptions(self): @@ -1505,7 +1496,7 @@ def test_update_happens_when_extensions_disabled(self, _): behavior never changes. """ self.update_handler._upgrade_available = Mock(return_value=True) - self._test_run(invocations=0, calls=[], enable_updates=True, sleep_interval=(300,)) + self._test_run(invocations=0, calls=0, enable_updates=True, sleep_interval=(300,)) @patch('azurelinuxagent.common.conf.get_extensions_enabled', return_value=False) def test_interval_changes_when_extensions_disabled(self, _): @@ -1513,7 +1504,7 @@ def test_interval_changes_when_extensions_disabled(self, _): When extension processing is disabled, the goal state interval should be larger. """ self.update_handler._upgrade_available = Mock(return_value=False) - self._test_run(invocations=15, calls=[call.run()] * 15, sleep_interval=(300,)) + self._test_run(invocations=1, calls=1, sleep_interval=(300,)) @patch("azurelinuxagent.common.logger.info") @patch("azurelinuxagent.ga.update.add_event") @@ -1561,7 +1552,7 @@ def check_running(*args, **kwargs): # pylint: disable=unused-argument update_handler._cur_iteration = 0 update_handler._iterations = 0 update_handler.set_iterations = lambda i: _set_iterations(i) # pylint: disable=unnecessary-lambda - type(update_handler).running = PropertyMock(side_effect=check_running) + type(update_handler).is_running = PropertyMock(side_effect=check_running) with patch("time.sleep", side_effect=lambda _: mock_sleep(0.001)): with patch('sys.exit'): # Setup the initial number of iterations @@ -1571,7 +1562,7 @@ def check_running(*args, **kwargs): # pylint: disable=unused-argument finally: # Since PropertyMock requires us to mock the type(ClassName).property of the object, # reverting it back to keep the state of the test clean - type(update_handler).running = True + type(update_handler).is_running = True @staticmethod def _get_test_ext_handler_instance(protocol, name="OSTCExtensions.ExampleHandlerLinux", version="1.0.0"): @@ -1701,34 +1692,36 @@ def setUp(self): AgentTestCase.setUp(self) self.event_patch = patch('azurelinuxagent.common.event.add_event') currentThread().setName("ExtHandler") + protocol = Mock() + protocol.get_ext_handlers = Mock(return_value=(Mock(), Mock())) self.update_handler = get_update_handler() self.update_handler.protocol_util = Mock() + self.update_handler.protocol_util.get_protocol = Mock(return_value=protocol) clear_singleton_instances(ProtocolUtil) def _test_run(self, invocations=1): - iterations = [0] - - def iterator(*args, **kwargs): # pylint: disable=unused-argument - iterations[0] += 1 - if iterations[0] >= invocations: - self.update_handler.running = False - return + def iterator(*_, **__): + iterator.count += 1 + if iterator.count <= invocations: + return True + return False + iterator.count = 0 with patch('os.getpid', return_value=42): with patch.object(UpdateHandler, '_is_orphaned') as mock_is_orphaned: mock_is_orphaned.__get__ = Mock(return_value=False) - with patch('azurelinuxagent.ga.exthandlers.get_exthandlers_handler'): - with patch('azurelinuxagent.ga.remoteaccess.get_remote_access_handler'): - with patch('azurelinuxagent.ga.update.initialize_event_logger_vminfo_common_parameters'): - with patch('azurelinuxagent.common.cgroupapi.CGroupsApi.cgroups_supported', return_value=False): # skip all cgroup stuff - with patch('azurelinuxagent.ga.update.is_log_collection_allowed', return_value=True): - with patch('time.sleep', side_effect=iterator): - with patch('sys.exit'): - self.update_handler.run() + with patch.object(UpdateHandler, 'is_running') as mock_is_running: + mock_is_running.__get__ = Mock(side_effect=iterator) + with patch('azurelinuxagent.ga.exthandlers.get_exthandlers_handler'): + with patch('azurelinuxagent.ga.remoteaccess.get_remote_access_handler'): + with patch('azurelinuxagent.ga.update.initialize_event_logger_vminfo_common_parameters'): + with patch('azurelinuxagent.common.cgroupapi.CGroupsApi.cgroups_supported', return_value=False): # skip all cgroup stuff + with patch('azurelinuxagent.ga.update.is_log_collection_allowed', return_value=True): + with patch('time.sleep'): + with patch('sys.exit'): + self.update_handler.run() def _setup_mock_thread_and_start_test_run(self, mock_thread, is_alive=True, invocations=0): - self.assertTrue(self.update_handler.running) - thread = MagicMock() thread.run = MagicMock() thread.is_alive = MagicMock(return_value=is_alive) @@ -1739,8 +1732,6 @@ def _setup_mock_thread_and_start_test_run(self, mock_thread, is_alive=True, invo return thread def test_start_threads(self, mock_env, mock_monitor, mock_collect_logs, mock_telemetry_send_events, mock_telemetry_collector): - self.assertTrue(self.update_handler.running) - def _get_mock_thread(): thread = MagicMock() thread.run = MagicMock() @@ -1758,7 +1749,7 @@ def _get_mock_thread(): self.assertEqual(1, thread().run.call_count) def test_check_if_monitor_thread_is_alive(self, _, mock_monitor, *args): # pylint: disable=unused-argument - mock_monitor_thread = self._setup_mock_thread_and_start_test_run(mock_monitor, is_alive=True, invocations=0) + mock_monitor_thread = self._setup_mock_thread_and_start_test_run(mock_monitor, is_alive=True, invocations=1) self.assertEqual(1, mock_monitor.call_count) self.assertEqual(1, mock_monitor_thread.run.call_count) self.assertEqual(1, mock_monitor_thread.is_alive.call_count) @@ -1786,14 +1777,14 @@ def test_restart_env_thread_if_not_alive(self, mock_env, *args): # pylint: disa self.assertEqual(1, mock_env_thread.start.call_count) def test_restart_monitor_thread(self, _, mock_monitor, *args): # pylint: disable=unused-argument - mock_monitor_thread = self._setup_mock_thread_and_start_test_run(mock_monitor, is_alive=False, invocations=0) + mock_monitor_thread = self._setup_mock_thread_and_start_test_run(mock_monitor, is_alive=False, invocations=1) self.assertEqual(True, mock_monitor.called) self.assertEqual(True, mock_monitor_thread.run.called) self.assertEqual(True, mock_monitor_thread.is_alive.called) self.assertEqual(True, mock_monitor_thread.start.called) def test_restart_env_thread(self, mock_env, *args): # pylint: disable=unused-argument - mock_env_thread = self._setup_mock_thread_and_start_test_run(mock_env, is_alive=False, invocations=0) + mock_env_thread = self._setup_mock_thread_and_start_test_run(mock_env, is_alive=False, invocations=1) self.assertEqual(True, mock_env.called) self.assertEqual(True, mock_env_thread.run.called) self.assertEqual(True, mock_env_thread.is_alive.called) @@ -1901,5 +1892,40 @@ def time(self): return current_time +class TestProcessGoalState(AgentTestCase): + """ + Tests for UpdateHandler._process_goal_state + """ + def test_it_should_process_goal_state_only_on_new_goal_state(self): + with mock_wire_protocol(DATA_FILE) as protocol: + update_handler = get_update_handler() + with patch.object(update_handler, "_upgrade_available", return_value=False): # skip the upgrade logic + exthandlers_handler = Mock() + remote_access_handler = Mock() + + def get_method_calls(mock, method): + return [call[0] for call in mock.method_calls if call[0] == method] + + # process a goal state + update_handler._process_goal_state(protocol, exthandlers_handler, remote_access_handler) + self.assertEqual(1, len(get_method_calls(exthandlers_handler, 'run')), "exthandlers_handler.run() should have been called on the first goal state") + self.assertEqual(1, len(get_method_calls(exthandlers_handler, 'report_ext_handlers_status')), "exthandlers_handler.report_ext_handlers_status() should have been called on the first goal state") + self.assertEqual(1, len(get_method_calls(remote_access_handler, 'run')), "remote_access_handler.run() should have been called on the first goal state") + + # process the same goal state + update_handler._process_goal_state(protocol, exthandlers_handler, remote_access_handler) + self.assertEqual(1, len(get_method_calls(exthandlers_handler, 'run')), "exthandlers_handler.run() should have not been called on the same goal state") + self.assertEqual(2, len(get_method_calls(exthandlers_handler, 'report_ext_handlers_status')), "exthandlers_handler.report_ext_handlers_status() should have been called on the same goal state") + self.assertEqual(1, len(get_method_calls(remote_access_handler, 'run')), "remote_access_handler.run() should not have been called on the same goal state") + + # process a new goal state + protocol.mock_wire_data.set_incarnation(999) + protocol.client.update_goal_state() + update_handler._process_goal_state(protocol, exthandlers_handler, remote_access_handler) + self.assertEqual(2, len(get_method_calls(exthandlers_handler, 'run')), "exthandlers_handler.run() should have been called on a new goal state") + self.assertEqual(3, len(get_method_calls(exthandlers_handler, 'report_ext_handlers_status')), "exthandlers_handler.report_ext_handlers_status() should have been called on a new goal state") + self.assertEqual(2, len(get_method_calls(remote_access_handler, 'run')), "remote_access_handler.run() should have been called on a new goal state") + + if __name__ == '__main__': unittest.main() diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py index 5efe0d46e..fd7a6300d 100644 --- a/tests/protocol/test_wire.py +++ b/tests/protocol/test_wire.py @@ -393,6 +393,8 @@ def mock_http_put(url, *args, **__): with patch("azurelinuxagent.common.agent_supported_feature._MultiConfigFeature.is_supported", True): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertIsNotNone(protocol.aggregate_status, "Aggregate status should not be None") self.assertIn("supportedFeatures", protocol.aggregate_status, "supported features not reported") multi_config_feature = get_supported_feature_by_name(SupportedFeatureNames.MultiConfig) @@ -406,6 +408,8 @@ def mock_http_put(url, *args, **__): # Feature should not be reported if not present with patch("azurelinuxagent.common.agent_supported_feature._MultiConfigFeature.is_supported", False): exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + self.assertIsNotNone(protocol.aggregate_status, "Aggregate status should not be None") if "supportedFeatures" not in protocol.aggregate_status: # In the case Multi-config was the only feature available, 'supportedFeatures' should not be From aaa0b6fd7a034484f18e0cfb43488750a5fab694 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Tue, 22 Jun 2021 17:32:44 -0700 Subject: [PATCH 27/35] Fix telemetry unicode errors (Re-add #1937) (#2278) --- azurelinuxagent/common/event.py | 18 +++++------ azurelinuxagent/common/protocol/wire.py | 30 ++++++++--------- azurelinuxagent/common/utils/restutil.py | 3 +- azurelinuxagent/common/utils/textutil.py | 34 ++++++++++++++++++++ tests/common/test_event.py | 41 +++++++++++++++++++++++- tests/ga/test_send_telemetry_events.py | 8 ++--- tests/protocol/test_wire.py | 14 ++++---- 7 files changed, 111 insertions(+), 37 deletions(-) diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py index 4f94ec488..1fe90214f 100644 --- a/azurelinuxagent/common/event.py +++ b/azurelinuxagent/common/event.py @@ -36,7 +36,7 @@ from azurelinuxagent.common.telemetryevent import TelemetryEventParam, TelemetryEvent, CommonTelemetryEventSchema, \ GuestAgentGenericLogsSchema, GuestAgentExtensionEventsSchema, GuestAgentPerfCounterEventsSchema from azurelinuxagent.common.utils import fileutil, textutil -from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, getattrib +from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, getattrib, str_to_encoded_ustr from azurelinuxagent.common.version import CURRENT_VERSION, CURRENT_AGENT, AGENT_NAME, DISTRO_NAME, DISTRO_VERSION, DISTRO_CODE_NAME, AGENT_EXECUTION_MODE from azurelinuxagent.common.protocol.imds import get_imds_client @@ -482,11 +482,11 @@ def add_event(self, name, op=WALAEventOperation.Unknown, is_success=True, durati _log_event(name, op, message, duration, is_success=is_success) event = TelemetryEvent(TELEMETRY_EVENT_EVENT_ID, TELEMETRY_EVENT_PROVIDER_ID) - event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Name, ustr(name))) - event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Version, ustr(version))) - event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Operation, ustr(op))) + event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Name, str_to_encoded_ustr(name))) + event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Version, str_to_encoded_ustr(version))) + event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Operation, str_to_encoded_ustr(op))) event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.OperationSuccess, bool(is_success))) - event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Message, ustr(message))) + event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Message, str_to_encoded_ustr(message))) event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Duration, int(duration))) self.add_common_event_parameters(event, datetime.utcnow()) @@ -500,7 +500,7 @@ def add_log_event(self, level, message): event = TelemetryEvent(TELEMETRY_LOG_EVENT_ID, TELEMETRY_LOG_PROVIDER_ID) event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.EventName, WALAEventOperation.Log)) event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.CapabilityUsed, logger.LogLevel.STRINGS[level])) - event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.Context1, self._clean_up_message(message))) + event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.Context1, str_to_encoded_ustr(self._clean_up_message(message)))) event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.Context2, datetime.utcnow().strftime(logger.Logger.LogTimeFormatInUTC))) event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.Context3, '')) self.add_common_event_parameters(event, datetime.utcnow()) @@ -526,9 +526,9 @@ def add_metric(self, category, counter, instance, value, log_event=False): _log_event(AGENT_NAME, "METRIC", message, 0) event = TelemetryEvent(TELEMETRY_METRICS_EVENT_ID, TELEMETRY_EVENT_PROVIDER_ID) - event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Category, str(category))) - event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Counter, str(counter))) - event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Instance, str(instance))) + event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Category, str_to_encoded_ustr(category))) + event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Counter, str_to_encoded_ustr(counter))) + event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Instance, str_to_encoded_ustr(instance))) event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Value, float(value))) self.add_common_event_parameters(event, datetime.utcnow()) diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 2ed0d8faa..7654b6750 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -540,7 +540,7 @@ def put_page_blob(self, url, data): def event_param_to_v1(param): - param_format = '' + param_format = ustr('') param_type = type(param.value) attr_type = "" if param_type is int: @@ -558,12 +558,12 @@ def event_param_to_v1(param): attr_type) -def event_to_v1(event): +def event_to_v1_encoded(event, encoding='utf-8'): params = "" for param in event.parameters: params += event_param_to_v1(param) - event_str = ''.format(event.eventId, params) - return event_str + event_str = ustr('').format(event.eventId, params) + return event_str.encode(encoding) class WireClient(object): @@ -1140,14 +1140,14 @@ def report_health(self, status, substatus, description): u",{0}: {1}").format(resp.status, resp.read())) - def send_event(self, provider_id, event_str): + def send_encoded_event(self, provider_id, event_str, encoding='utf8'): uri = TELEMETRY_URI.format(self.get_endpoint()) - data_format = ('' - '' - '{1}' - '' - '') - data = data_format.format(provider_id, event_str) + data_format_header = ustr('').format( + provider_id).encode(encoding) + data_format_footer = ustr('').encode(encoding) + # Event string should already be encoded by the time it gets here, to avoid double encoding, + # dividing it into parts. + data = data_format_header + event_str + data_format_footer try: header = self.get_header_for_xml_content() # NOTE: The call to wireserver requests utf-8 encoding in the headers, but the body should not @@ -1168,7 +1168,7 @@ def report_event(self, events_iterator): def _send_event(provider_id, debug_info): try: - self.send_event(provider_id, buf[provider_id]) + self.send_encoded_event(provider_id, buf[provider_id]) except UnicodeError as uni_error: debug_info.update_unicode_error(uni_error) except Exception as error: @@ -1178,8 +1178,8 @@ def _send_event(provider_id, debug_info): for event in events_iterator: try: if event.providerId not in buf: - buf[event.providerId] = "" - event_str = event_to_v1(event) + buf[event.providerId] = b"" + event_str = event_to_v1_encoded(event) if len(event_str) >= MAX_EVENT_BUFFER_SIZE: # Ignore single events that are too large to send out @@ -1196,7 +1196,7 @@ def _send_event(provider_id, debug_info): if len(buf[event.providerId] + event_str) >= MAX_EVENT_BUFFER_SIZE: logger.verbose("No of events this request = {0}".format(events_per_provider[event.providerId])) _send_event(event.providerId, debug_info) - buf[event.providerId] = "" + buf[event.providerId] = b"" events_per_provider[event.providerId] = 0 # Add encoded events to the buffer diff --git a/azurelinuxagent/common/utils/restutil.py b/azurelinuxagent/common/utils/restutil.py index 1b761d7ea..06dbde919 100644 --- a/azurelinuxagent/common/utils/restutil.py +++ b/azurelinuxagent/common/utils/restutil.py @@ -338,10 +338,11 @@ def _http_request(method, host, rel_uri, port=None, data=None, secure=False, if redact_data: payload = "[REDACTED]" + # Logger requires the msg to be a ustr to log properly, ensuring that the data string that we log is always ustr logger.verbose("HTTP connection [{0}] [{1}] [{2}] [{3}]", method, redact_sas_tokens_in_urls(url), - payload, + textutil.str_to_encoded_ustr(payload), headers) conn.request(method=method, url=url, body=data, headers=headers) diff --git a/azurelinuxagent/common/utils/textutil.py b/azurelinuxagent/common/utils/textutil.py index 405ee2bbc..21d0adf66 100644 --- a/azurelinuxagent/common/utils/textutil.py +++ b/azurelinuxagent/common/utils/textutil.py @@ -27,6 +27,8 @@ import xml.dom.minidom as minidom import zlib +from azurelinuxagent.common.future import ustr + def parse_doc(xml_text): """ @@ -396,3 +398,35 @@ def format_memory_value(unit, value): raise TypeError('Value must be convertible to a float') return int(value * units[unit]) + + +def str_to_encoded_ustr(s, encoding='utf-8'): + """ + This function takes the string and converts it into the corresponding encoded ustr if its not already a ustr. + The encoding is utf-8 by default if not specified. + Note: ustr() is a unicode object for Py2 and a str object for Py3. + :param s: The string to convert to ustr + :param encoding: Encoding to use. Utf-8 by default + :return: Returns the corresponding ustr string. Returns None if input is None. + """ + + # TODO: Import at the top of the file instead of a local import (using local import here to avoid cyclic dependency) + from azurelinuxagent.common.version import PY_VERSION_MAJOR + + if s is None or type(s) is ustr: + # If its already a ustr/None then return as is + return s + if PY_VERSION_MAJOR > 2: + try: + # For py3+, str() is unicode by default + if isinstance(s, bytes): + # str.encode() returns bytes which should be decoded to get the str. + return s.decode(encoding) + else: + # If its not encoded, just return the string + return ustr(s) + except Exception: + # If some issues in decoding, just return the string + return ustr(s) + # For Py2, explicitly convert the string to unicode with the specified encoding + return ustr(s, encoding=encoding) diff --git a/tests/common/test_event.py b/tests/common/test_event.py index 88dd483e6..72f4bcfe7 100644 --- a/tests/common/test_event.py +++ b/tests/common/test_event.py @@ -739,8 +739,10 @@ def get_event_message_from_event_file(event_file): raise ValueError('Could not find the Message for the telemetry event in {0}'.format(event_file)) - def get_event_message_from_http_request_body(http_request_body): + def get_event_message_from_http_request_body(event_body): # The XML for the event is sent over as a CDATA element ("Event") in the request's body + http_request_body = event_body if ( + event_body is None or type(event_body) is ustr) else textutil.str_to_encoded_ustr(event_body) request_body_xml_doc = textutil.parse_doc(http_request_body) event_node = textutil.find(request_body_xml_doc, "Event") @@ -786,6 +788,43 @@ def http_post_handler(url, body, **__): self.assertEqual(event_message, expected_message, "The Message in the HTTP request does not match the Message in the event's *.tld file") + def test_report_event_should_encode_events_correctly(self): + + def http_post_handler(url, body, **__): + if self.is_telemetry_request(url): + http_post_handler.request_body = body + return MockHttpResponse(status=200) + return None + http_post_handler.request_body = None + + with mock_wire_protocol(mockwiredata.DATA_FILE, http_post_handler=http_post_handler) as protocol: + test_messages = [ + 'Non-English message - 此文字不是英文的', + "Ξεσκεπάζω τὴν ψυχοφθόρα βδελυγμία", + "The quick brown fox jumps over the lazy dog", + "El pingüino Wenceslao hizo kilómetros bajo exhaustiva lluvia y frío, añoraba a su querido cachorro.", + "Portez ce vieux whisky au juge blond qui fume sur son île intérieure, à côté de l'alcôve ovoïde, où les bûches", + "se consument dans l'âtre, ce qui lui permet de penser à la cænogenèse de l'être dont il est question", + "dans la cause ambiguë entendue à Moÿ, dans un capharnaüm qui, pense-t-il, diminue çà et là la qualité de son œuvre.", + "D'fhuascail Íosa, Úrmhac na hÓighe Beannaithe, pór Éava agus Ádhaimh", + "Árvíztűrő tükörfúrógép", + "Kæmi ný öxi hér ykist þjófum nú bæði víl og ádrepa", + "Sævör grét áðan því úlpan var ónýt", + "いろはにほへとちりぬるを わかよたれそつねならむ うゐのおくやまけふこえて あさきゆめみしゑひもせす", + "? דג סקרן שט בים מאוכזב ולפתע מצא לו חברה איך הקליטה" + "Pchnąć w tę łódź jeża lub ośm skrzyń fig", + "Normal string event" + ] + for msg in test_messages: + add_event('TestEventEncoding', message=msg) + event_list = self._collect_events() + self._report_events(protocol, event_list) + # In Py2, encode() produces a str and in py3 it produces a bytes string. + # type(bytes) == type(str) for Py2 so this check is mainly for Py3 to ensure that the event is encoded properly. + self.assertIsInstance(http_post_handler.request_body, bytes, "The Event request body should be encoded") + self.assertIn(textutil.str_to_encoded_ustr(msg).encode('utf-8'), http_post_handler.request_body, + "Encoded message not found in body") + class TestMetrics(AgentTestCase): @patch('azurelinuxagent.common.event.EventLogger.save_event') diff --git a/tests/ga/test_send_telemetry_events.py b/tests/ga/test_send_telemetry_events.py index 5f14bab0b..1c92ba91e 100644 --- a/tests/ga/test_send_telemetry_events.py +++ b/tests/ga/test_send_telemetry_events.py @@ -33,7 +33,7 @@ from azurelinuxagent.common.future import ustr from azurelinuxagent.common.osutil.factory import get_osutil from azurelinuxagent.common.protocol.util import ProtocolUtil -from azurelinuxagent.common.protocol.wire import event_to_v1 +from azurelinuxagent.common.protocol.wire import event_to_v1_encoded from azurelinuxagent.common.telemetryevent import TelemetryEvent, TelemetryEventParam, \ GuestAgentExtensionEventsSchema from azurelinuxagent.common.utils import restutil, fileutil @@ -97,7 +97,7 @@ def _assert_test_data_in_event_body(self, telemetry_handler, test_events): TestSendTelemetryEventsHandler._stop_handler(telemetry_handler) for telemetry_event in test_events: - event_str = event_to_v1(telemetry_event) + event_str = event_to_v1_encoded(telemetry_event) found = False for _, event_body in telemetry_handler.event_calls: if event_str in event_body: @@ -257,7 +257,7 @@ def test_it_should_honour_the_incoming_order_of_events(self): self.assertTrue(telemetry_handler.is_alive(), "Thread not alive") TestSendTelemetryEventsHandler._stop_handler(telemetry_handler) _, event_body = telemetry_handler.event_calls[0] - event_orders = re.findall(r'', event_body) + event_orders = re.findall(r'', event_body.decode('utf-8')) self.assertEqual(sorted(event_orders), event_orders, "Events not ordered correctly") def test_send_telemetry_events_should_report_event_if_wireserver_returns_http_error(self): @@ -382,7 +382,7 @@ def test_it_should_enqueue_and_send_events_properly(self, mock_lib_dir, *_): '' \ ']]>'.format(AGENT_VERSION, CURRENT_AGENT, test_opcodename, test_eventtid, test_eventpid, test_taskname, osversion, int(osutil.get_total_mem()), - osutil.get_processor_cores()) + osutil.get_processor_cores()).encode('utf-8') self.assertIn(sample_message, collected_event) diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py index fd7a6300d..d85ed0954 100644 --- a/tests/protocol/test_wire.py +++ b/tests/protocol/test_wire.py @@ -429,12 +429,12 @@ def mock_http_put(url, *args, **__): self.assertFalse(found, "Multi-config name should be present in supportedFeatures") @patch("azurelinuxagent.common.utils.restutil.http_request") - def test_send_event(self, mock_http_request, *args): + def test_send_encoded_event(self, mock_http_request, *args): mock_http_request.return_value = MockResponse("", 200) event_str = u'a test string' client = WireProtocol(WIRESERVER_URL).client - client.send_event("foo", event_str.encode('utf-8')) + client.send_encoded_event("foo", event_str.encode('utf-8')) first_call = mock_http_request.call_args_list[0] args, kwargs = first_call @@ -443,10 +443,10 @@ def test_send_event(self, mock_http_request, *args): # the headers should include utf-8 encoding... self.assertTrue("utf-8" in headers['Content-Type']) - # the body is not encoded, just check for equality - self.assertIn(event_str, body_received) + # the body is encoded, decode and check for equality + self.assertIn(event_str, body_received.decode('utf-8')) - @patch("azurelinuxagent.common.protocol.wire.WireClient.send_event") + @patch("azurelinuxagent.common.protocol.wire.WireClient.send_encoded_event") def test_report_event_small_event(self, patch_send_event, *args): # pylint: disable=unused-argument event_list = [] client = WireProtocol(WIRESERVER_URL).client @@ -468,7 +468,7 @@ def test_report_event_small_event(self, patch_send_event, *args): # pylint: dis # It merges the messages into one message self.assertEqual(patch_send_event.call_count, 1) - @patch("azurelinuxagent.common.protocol.wire.WireClient.send_event") + @patch("azurelinuxagent.common.protocol.wire.WireClient.send_encoded_event") def test_report_event_multiple_events_to_fill_buffer(self, patch_send_event, *args): # pylint: disable=unused-argument event_list = [] client = WireProtocol(WIRESERVER_URL).client @@ -482,7 +482,7 @@ def test_report_event_multiple_events_to_fill_buffer(self, patch_send_event, *ar # It merges the messages into one message self.assertEqual(patch_send_event.call_count, 2) - @patch("azurelinuxagent.common.protocol.wire.WireClient.send_event") + @patch("azurelinuxagent.common.protocol.wire.WireClient.send_encoded_event") def test_report_event_large_event(self, patch_send_event, *args): # pylint: disable=unused-argument event_list = [] event_str = random_generator(2 ** 18) From b99ad5c5e33c937dc2dde364de2276263eb4fbe9 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Thu, 24 Jun 2021 10:20:45 -0700 Subject: [PATCH 28/35] release prepare-2.4.0.0 (#2280) --- azurelinuxagent/common/cgroupconfigurator.py | 2 +- azurelinuxagent/common/version.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azurelinuxagent/common/cgroupconfigurator.py b/azurelinuxagent/common/cgroupconfigurator.py index d7ebd67f3..b10c9426c 100644 --- a/azurelinuxagent/common/cgroupconfigurator.py +++ b/azurelinuxagent/common/cgroupconfigurator.py @@ -69,7 +69,7 @@ [Service] CPUQuota={0} """ -_AGENT_CPU_QUOTA = 5 +_AGENT_CPU_QUOTA = 100 _AGENT_THROTTLED_TIME_THRESHOLD = 120 # 2 minutes diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 39fdf1ba6..70bd61773 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -200,9 +200,9 @@ def has_logrotate(): # IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise # DCR may test the wrong agent version. # -# When doing a release, be sure to use the actual agent version. Current agent version: 2.3.0.2 +# When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '9.9.9.9' +AGENT_VERSION = '2.4.0.0' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From b6a06246ba9f86c92fe4bce2c5045ca3a0b2166d Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Fri, 25 Jun 2021 14:28:47 -0700 Subject: [PATCH 29/35] Fix bug with dependent extensions with no settings (#2285) --- azurelinuxagent/ga/exthandlers.py | 10 ++++-- ..._conf_dependencies_with_empty_settings.xml | 32 +++++++++++++++++++ tests/ga/test_extension.py | 32 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 tests/data/wire/ext_conf_dependencies_with_empty_settings.xml diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 5a44d5d95..641cf3892 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -574,9 +574,13 @@ def wait_for_handler_completion(handler_i, wait_until, extension=None): Check the status of the extension being handled. Wait until it has a terminal state or times out. :raises: Exception if it is not handled successfully. """ - extension_name = handler_i.get_extension_full_name(extension) + # If the handler had no settings, we should not wait at all for handler to report status. + if extension is None: + logger.info("No settings found for {0}, not waiting for it's status".format(extension_name)) + return + try: ext_completed, status = False, None @@ -1656,7 +1660,9 @@ def get_status_file_path(self, extension=None): def collect_ext_status(self, ext): self.logger.verbose("Collect extension status for {0}".format(self.get_extension_full_name(ext))) seq_no, ext_status_file = self.get_status_file_path(ext) - if seq_no == -1: + + # We should never try to read any status file if the handler has no settings, returning None in that case + if seq_no == -1 or ext is None: return None data = None diff --git a/tests/data/wire/ext_conf_dependencies_with_empty_settings.xml b/tests/data/wire/ext_conf_dependencies_with_empty_settings.xml new file mode 100644 index 000000000..402de6438 --- /dev/null +++ b/tests/data/wire/ext_conf_dependencies_with_empty_settings.xml @@ -0,0 +1,32 @@ + + + + + Prod + + http://mock-goal-state/manifest_of_ga.xml + + + + Test + + http://mock-goal-state/manifest_of_ga.xml + + + + + + + + + + + + + + {"runtimeSettings":[{"handlerSettings":{"protectedSettingsCertThumbprint":"4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3","protectedSettings":"MIICWgYJK","publicSettings":{"foo":"bar"}}}]} + + + https://test.blob.core.windows.net/vhds/test-cs12.test-cs12.test-cs12.status?sr=b&sp=rw&se=9999-01-01&sk=key1&sv=2014-02-14&sig=hfRh7gzUE7sUtYwke78IOlZOrTRCYvkec4hGZ9zZzXo + diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index 4a399bf4e..4432d21ba 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -992,6 +992,38 @@ def test_ext_handler_sequencing(self, *args): (dep_ext_level_5, ExtensionCommandNames.UNINSTALL) ) + def test_it_should_process_sequencing_properly_even_if_no_settings_for_dependent_extension( + self, mock_get, mock_crypt, *args): + test_data_file = DATA_FILE.copy() + test_data_file["ext_conf"] = "wire/ext_conf_dependencies_with_empty_settings.xml" + test_data = mockwiredata.WireProtocolData(test_data_file) + exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) + + ext_1 = extension_emulator(name="OSTCExtensions.ExampleHandlerLinux") + ext_2 = extension_emulator(name="OSTCExtensions.OtherExampleHandlerLinux") + + with enable_invocations(ext_1, ext_2) as invocation_record: + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + + # Ensure no extension status was reported for OtherExampleHandlerLinux as no settings provided for it + self._assert_handler_status(protocol.report_vm_status, "Ready", 0, "1.0.0", + expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux") + + # Ensure correct status reported back for the other extension with settings + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="OSTCExtensions.ExampleHandlerLinux") + self._assert_ext_status(protocol.report_vm_status, "success", 0, + expected_handler_name="OSTCExtensions.ExampleHandlerLinux") + + # Ensure the invocation order follows the dependency levels + invocation_record.compare( + (ext_2, ExtensionCommandNames.INSTALL), + (ext_2, ExtensionCommandNames.ENABLE), + (ext_1, ExtensionCommandNames.INSTALL), + (ext_1, ExtensionCommandNames.ENABLE) + ) + def test_ext_handler_sequencing_should_fail_if_handler_failed(self, mock_get, mock_crypt, *args): test_data = mockwiredata.WireProtocolData(mockwiredata.DATA_FILE_EXT_SEQUENCING) exthandlers_handler, protocol = self._create_mock(test_data, mock_get, mock_crypt, *args) From 469a168f9c7bef56017aa2f2f9ee2547b74dfd92 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Fri, 9 Jul 2021 12:16:14 -0700 Subject: [PATCH 30/35] update test-requirements to pin pylint. (#2288) (#2299) Co-authored-by: Kevin Clark --- test-requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 85d5d4263..d4d62738e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,4 +6,6 @@ mock==4.0.2; python_version >= '3.6' distro; python_version >= '3.8' nose nose-timer; python_version >= '2.7' -pylint; python_version > '2.6' +pylint; python_version > '2.6' and python_version < '3.6' +pylint==2.8.3; python_version >= '3.6' + From a9f572fac8f430272ffc121466f193ce63249abd Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Fri, 9 Jul 2021 14:08:56 -0700 Subject: [PATCH 31/35] Do not create placeholder status file for AKS extensions (#2298) --- azurelinuxagent/ga/exthandlers.py | 14 ++++- tests/data/wire/ext_conf_aks_extension.xml | 69 ++++++++++++++++++++++ tests/ga/test_extension.py | 50 ++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 tests/data/wire/ext_conf_aks_extension.xml diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 641cf3892..8ff84824c 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -735,7 +735,8 @@ def handle_enable(self, ext_handler_i, extension): # failures back to CRP. If a placeholder for an extension already exists with Transitioning status, we would # not override it, hence we only create a placeholder for enable/disable commands but the extensions have the # data to create their own if needed. - ext_handler_i.create_placeholder_status_file(extension) + if ext_handler_i.should_create_default_placeholder(extension): + ext_handler_i.create_placeholder_status_file(extension) self.__handle_extension(ext_handler_i, extension, uninstall_exit_code) @staticmethod @@ -1398,6 +1399,17 @@ def initialize(self): # Save HandlerEnvironment.json self.create_handler_env() + def should_create_default_placeholder(self, extension=None): + """ + There's a bug in the AKS extension where they dont update the status file if it exists. + This violates the contract we have with extensions. Until they fix their extension, + we're going to skip creating a placeholder for them to ensure they dont have any downtime. + For all other extensions, we should create a + """ + + ignore_extension_regex = r"Microsoft.AKS.Compute.AKS\S*" + return re.match(ignore_extension_regex, self.get_extension_full_name(extension)) is None + def create_placeholder_status_file(self, extension=None, status=ValidHandlerStatus.transitioning, code=0, operation="Enabling Extension", message="Install/Enable is in progress."): _, status_path = self.get_status_file_path(extension) diff --git a/tests/data/wire/ext_conf_aks_extension.xml b/tests/data/wire/ext_conf_aks_extension.xml new file mode 100644 index 000000000..5901c0e44 --- /dev/null +++ b/tests/data/wire/ext_conf_aks_extension.xml @@ -0,0 +1,69 @@ + + + + + Prod + + http://mock-goal-state/manifest_of_ga.xml + + + + Test + + http://mock-goal-state/manifest_of_ga.xml + + + + + + + + + + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling non-AKS"} + } + } + ] + } + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling AKSNode"} + } + } + ] + } + + + + + { + "runtimeSettings": [ + { + "handlerSettings": { + "publicSettings": {"message": "Enabling AKSBilling"} + } + } + ] + } + + + + + https://test.blob.core.windows.net/vhds/test-cs12.test-cs12.test-cs12.status?sr=b&sp=rw&se=9999-01-01&sk=key1&sv=2014-02-14&sig=hfRh7gzUE7sUtYwke78IOlZOrTRCYvkec4hGZ9zZzXo + + + diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index 4432d21ba..add41d61c 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -1664,6 +1664,56 @@ def mock_popen(cmd, *args, **kwargs): expected_msg="Dependent Extension OSTCExtensions.OtherExampleHandlerLinux did not reach a terminal state within the allowed timeout. Last status was {0}".format( ValidHandlerStatus.warning)) + def test_it_should_not_create_placeholder_for_aks_extension(self, mock_http_get, mock_crypt_util, *args): + original_popen = subprocess.Popen + + def mock_popen(cmd, *_, **kwargs): + if 'env' in kwargs: + if ExtensionCommandNames.ENABLE not in cmd: + # To force the test extension to not create a status file on Install, changing command + return original_popen(["echo", "not-enable"], *_, **kwargs) + + seq_no = kwargs['env'][ExtCommandEnvVariable.ExtensionSeqNumber] + ext_path = kwargs['env'][ExtCommandEnvVariable.ExtensionPath] + status_file_name = "{0}.status".format(seq_no) + status_file = os.path.join(ext_path, "status", status_file_name) + if "AKS" in cmd: + self.assertFalse(os.path.exists(status_file), "Placeholder file should not be created for AKS") + else: + self.assertTrue(os.path.exists(status_file), "Placeholder file should be created for all extensions") + + return original_popen(cmd, *_, **kwargs) + + aks_test_mock = DATA_FILE.copy() + aks_test_mock["ext_conf"] = "wire/ext_conf_aks_extension.xml" + + exthandlers_handler, protocol = self._create_mock(mockwiredata.WireProtocolData(aks_test_mock), + mock_http_get, mock_crypt_util, *args) + + with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): + exthandlers_handler.run() + exthandlers_handler.report_ext_handlers_status() + + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="OSTCExtensions.ExampleHandlerLinux") + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="Microsoft.AKS.Compute.AKS.Linux.AKSNode") + self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0", + expected_handler_name="Microsoft.AKS.Compute.AKS-Engine.Linux.Billing") + # Extension without settings + self._assert_handler_status(protocol.report_vm_status, "Ready", 0, "1.0.0", + expected_handler_name="Microsoft.AKS.Compute.AKS.Linux.Billing") + + self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.success, 0, + expected_handler_name="OSTCExtensions.ExampleHandlerLinux", + expected_msg="Enabling non-AKS") + self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.success, 0, + expected_handler_name="Microsoft.AKS.Compute.AKS.Linux.AKSNode", + expected_msg="Enabling AKSNode") + self._assert_ext_status(protocol.report_vm_status, ValidHandlerStatus.success, 0, + expected_handler_name="Microsoft.AKS.Compute.AKS-Engine.Linux.Billing", + expected_msg="Enabling AKSBilling") + def test_it_should_include_part_of_status_in_ext_handler_message(self, mock_http_get, mock_crypt_util, *args): """ Testing scenario when the status file is invalid, From 1344f619391a992a0b4c61a9edb201caa55b8537 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Thu, 15 Jul 2021 17:47:14 -0700 Subject: [PATCH 32/35] Exception for Linux Patch Extension for creating placeholder status file (#2307) --- azurelinuxagent/ga/exthandlers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index 8ff84824c..dac348fa6 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -1401,14 +1401,16 @@ def initialize(self): def should_create_default_placeholder(self, extension=None): """ - There's a bug in the AKS extension where they dont update the status file if it exists. - This violates the contract we have with extensions. Until they fix their extension, + There's a bug in the AKS extension where they dont update the status file if it exists and another bug in + LinuxPatchExtension where they inherently have dependency on creating the status file first. + This violates the contract we have with extensions. Until they fix their extensions, we're going to skip creating a placeholder for them to ensure they dont have any downtime. For all other extensions, we should create a """ - ignore_extension_regex = r"Microsoft.AKS.Compute.AKS\S*" - return re.match(ignore_extension_regex, self.get_extension_full_name(extension)) is None + ignore_extensions_regex = [r"Microsoft.AKS.Compute.AKS\S*", r"Microsoft.CPlat.Core.LinuxPatchExtension\S*"] + return all(re.match(ext_regex, self.get_extension_full_name(extension)) is None + for ext_regex in ignore_extensions_regex) def create_placeholder_status_file(self, extension=None, status=ValidHandlerStatus.transitioning, code=0, operation="Enabling Extension", message="Install/Enable is in progress."): From 92afd7f618ee66bab2d2443a2992fc1177aebec8 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Fri, 16 Jul 2021 10:04:11 -0700 Subject: [PATCH 33/35] update release version (#2308) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index 70bd61773..ded3e7608 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -202,7 +202,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '2.4.0.0' +AGENT_VERSION = '2.4.0.1' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux From 615d48a16a675d92ddffefdb6752ab0216337d05 Mon Sep 17 00:00:00 2001 From: Laveesh Rohra Date: Tue, 3 Aug 2021 13:39:47 -0700 Subject: [PATCH 34/35] Dont create default status file for Single-Config extensions (#2318) --- azurelinuxagent/ga/exthandlers.py | 19 +++++-------------- tests/ga/test_extension.py | 13 +++++-------- tests/ga/test_multi_config_extension.py | 9 +++------ 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index dac348fa6..9ccf2669b 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -735,7 +735,11 @@ def handle_enable(self, ext_handler_i, extension): # failures back to CRP. If a placeholder for an extension already exists with Transitioning status, we would # not override it, hence we only create a placeholder for enable/disable commands but the extensions have the # data to create their own if needed. - if ext_handler_i.should_create_default_placeholder(extension): + + # Note: Due to a bug in multiple extensions, we're only creating a default placeholder for Multi-Config extensions. + # A fix will follow soon where we will report transitioning status for extensions by default if no status file + # found instead of reporting an error. + if ext_handler_i.should_perform_multi_config_op(extension): ext_handler_i.create_placeholder_status_file(extension) self.__handle_extension(ext_handler_i, extension, uninstall_exit_code) @@ -1399,19 +1403,6 @@ def initialize(self): # Save HandlerEnvironment.json self.create_handler_env() - def should_create_default_placeholder(self, extension=None): - """ - There's a bug in the AKS extension where they dont update the status file if it exists and another bug in - LinuxPatchExtension where they inherently have dependency on creating the status file first. - This violates the contract we have with extensions. Until they fix their extensions, - we're going to skip creating a placeholder for them to ensure they dont have any downtime. - For all other extensions, we should create a - """ - - ignore_extensions_regex = [r"Microsoft.AKS.Compute.AKS\S*", r"Microsoft.CPlat.Core.LinuxPatchExtension\S*"] - return all(re.match(ext_regex, self.get_extension_full_name(extension)) is None - for ext_regex in ignore_extensions_regex) - def create_placeholder_status_file(self, extension=None, status=ValidHandlerStatus.transitioning, code=0, operation="Enabling Extension", message="Install/Enable is in progress."): _, status_path = self.get_status_file_path(extension) diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py index add41d61c..29546a1c7 100644 --- a/tests/ga/test_extension.py +++ b/tests/ga/test_extension.py @@ -450,7 +450,7 @@ def _assert_handler_status(self, report_vm_status, expected_status, self.assertNotEqual(0, len(vm_status.vmAgent.extensionHandlers)) handler_status = next( status for status in vm_status.vmAgent.extensionHandlers if status.name == expected_handler_name) - self.assertEqual(expected_status, handler_status.status) + self.assertEqual(expected_status, handler_status.status, get_properties(handler_status)) self.assertEqual(expected_handler_name, handler_status.name) self.assertEqual(version, handler_status.version) self.assertEqual(expected_ext_count, len([ext_handler for ext_handler in vm_status.vmAgent.extensionHandlers if @@ -1639,9 +1639,9 @@ def mock_popen(cmd, *args, **kwargs): if "sample.py" in cmd: status_path = os.path.join(kwargs['env'][ExtCommandEnvVariable.ExtensionPath], "status", "{0}.status".format(kwargs['env'][ExtCommandEnvVariable.ExtensionSeqNumber])) + mock_popen.deleted_status_file = status_path if os.path.exists(status_path): os.remove(status_path) - mock_popen.deleted_status_file = status_path return original_popen(["echo", "Yes"], *args, **kwargs) with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): @@ -1664,7 +1664,7 @@ def mock_popen(cmd, *args, **kwargs): expected_msg="Dependent Extension OSTCExtensions.OtherExampleHandlerLinux did not reach a terminal state within the allowed timeout. Last status was {0}".format( ValidHandlerStatus.warning)) - def test_it_should_not_create_placeholder_for_aks_extension(self, mock_http_get, mock_crypt_util, *args): + def test_it_should_not_create_placeholder_for_single_config_extensions(self, mock_http_get, mock_crypt_util, *args): original_popen = subprocess.Popen def mock_popen(cmd, *_, **kwargs): @@ -1677,10 +1677,7 @@ def mock_popen(cmd, *_, **kwargs): ext_path = kwargs['env'][ExtCommandEnvVariable.ExtensionPath] status_file_name = "{0}.status".format(seq_no) status_file = os.path.join(ext_path, "status", status_file_name) - if "AKS" in cmd: - self.assertFalse(os.path.exists(status_file), "Placeholder file should not be created for AKS") - else: - self.assertTrue(os.path.exists(status_file), "Placeholder file should be created for all extensions") + self.assertFalse(os.path.exists(status_file), "Placeholder file should not be created for single config extensions") return original_popen(cmd, *_, **kwargs) @@ -1732,7 +1729,7 @@ def mock_popen(cmd, *args, **kwargs): "{0}.status".format(kwargs['env'][ExtCommandEnvVariable.ExtensionSeqNumber])) invalid_json_path = os.path.join(data_dir, "ext", "sample-status-invalid-json-format.json") - if os.path.exists(status_path): + if 'enable' in cmd: invalid_json = fileutil.read_file(invalid_json_path) fileutil.write_file(status_path,invalid_json) diff --git a/tests/ga/test_multi_config_extension.py b/tests/ga/test_multi_config_extension.py index 00183bf7c..274fe4b11 100644 --- a/tests/ga/test_multi_config_extension.py +++ b/tests/ga/test_multi_config_extension.py @@ -894,7 +894,7 @@ def test_it_should_ignore_disable_errors_for_multi_config_extensions(self): fail_code in kwargs['message'] for args, kwargs in patch_report_event.call_args_list if kwargs['name'] == third_ext.name), "Error not reported") - def test_it_should_always_create_placeholder_for_all_extensions(self): + def test_it_should_always_create_placeholder_for_multi_config_extensions(self): original_popen = subprocess.Popen handler_statuses = {} @@ -903,9 +903,6 @@ def __assert_status_file_in_handlers(): file_name = "{0}.{1}.status".format(handler['runtimeSettingsStatus']['extensionName'], handler['runtimeSettingsStatus']['sequenceNumber']) __assert_status_file(handler['handlerName'], status_file=file_name) - for handler in sc_handler: - file_name = "{0}.status".format(handler['runtimeSettingsStatus']['sequenceNumber']) - __assert_status_file(handler['handlerName'], status_file=file_name) def __assert_status_file(handler_name, status_file): status = handler_statuses["{0}.{1}.enable".format(handler_name, status_file)] @@ -938,7 +935,7 @@ def mock_popen(cmd, *_, **kwargs): "ext_conf_multi_config_no_dependencies.xml") with self._setup_test_env(mock_manifest=True) as (exthandlers_handler, protocol, no_of_extensions): with patch('azurelinuxagent.common.cgroupapi.subprocess.Popen', side_effect=mock_popen): - mc_handlers, sc_handler = self.__run_and_assert_generic_case(exthandlers_handler, protocol, + mc_handlers, _ = self.__run_and_assert_generic_case(exthandlers_handler, protocol, no_of_extensions) # Ensure we dont create a placeholder for Install command @@ -953,7 +950,7 @@ def mock_popen(cmd, *_, **kwargs): __assert_status_file_in_handlers() # Update GS, remove 2 extensions and add 3 - mc_handlers, sc_handler = self.__setup_and_assert_disable_scenario(exthandlers_handler, protocol) + mc_handlers, _ = self.__setup_and_assert_disable_scenario(exthandlers_handler, protocol) __assert_status_file_in_handlers() def test_it_should_report_status_correctly_for_unsupported_goal_state(self): From 11cd6697b213fef8d9c57cf06902e4d23e90c8c2 Mon Sep 17 00:00:00 2001 From: Nageswara Nandigam <84482346+nagworld9@users.noreply.github.com> Date: Tue, 3 Aug 2021 14:06:11 -0700 Subject: [PATCH 35/35] version update (#2319) --- azurelinuxagent/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index ded3e7608..b686b8662 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -202,7 +202,7 @@ def has_logrotate(): # # When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 # -AGENT_VERSION = '2.4.0.1' +AGENT_VERSION = '2.4.0.2' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux