Merge pull request #2364 from Azure/release-2.4.0.0-dev

Merge release-2.4.0.0 into  master

CODEOWNERS

@@ -20,4 +20,4 @@
 #
 # Linux Agent team
 #
-* @narrieta @larohra @kevinclark19a @ZhidongPeng
+* @narrieta @larohra @kevinclark19a @ZhidongPeng @dhivyaganesan @nagworld9

README.md

@@ -1,29 +1,15 @@
 
 # Microsoft Azure Linux Agent
 
-## Develop branch status
+## Linux distributions support
+
+Our daily automation tests most of the [Linux distributions supported by Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/endorsed-distros); the Agent can be
+used on other distributions as well, but development, testing and support for those are done by the open source community.
+
+Testing is done using the develop branch, which can be unstable. For a stable build please use the master branch instead.
 
 [![CodeCov](https://codecov.io/gh/Azure/WALinuxAgent/branch/develop/graph/badge.svg)](https://codecov.io/gh/Azure/WALinuxAgent/branch/develop)
 
-Each badge below represents our basic validation tests for an image, which are executed several times each day. These include provisioning, user account, disk, extension and networking scenarios.
-
-Note: These badges represent testing to our develop branch which might not be stable. For a stable build please use master branch instead. 
-
-Image | Status |
-------|--------|
-Canonical UbuntuServer 14.04.5-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_14.04.5-LTS__agent--bvt.svg)
-Canonical UbuntuServer 14.04.5-DAILY-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_14.04.5-DAILY-LTS__agent--bvt.svg)
-Canonical UbuntuServer 16.04-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_16.04-LTS__agent--bvt.svg)
-Canonical UbuntuServer 16.04-DAILY-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_16.04-DAILY-LTS__agent--bvt.svg)
-Canonical UbuntuServer 18.04-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_18.04-LTS__agent--bvt.svg)
-Canonical UbuntuServer 18.04-DAILY-LTS|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Canonical_UbuntuServer_18.04-DAILY-LTS__agent--bvt.svg)
-Credativ Debian 8|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Credativ_Debian_8__agent--bvt.svg)
-Credativ Debian 9|![badge](https://dcrbadges.blob.core.windows.net/scenarios/Credativ_Debian_9__agent--bvt.svg)
-OpenLogic CentOS 6.9|![badge](https://dcrbadges.blob.core.windows.net/scenarios/OpenLogic_CentOS_6.9__agent--bvt.svg)
-OpenLogic CentOS 7.4|![badge](https://dcrbadges.blob.core.windows.net/scenarios/OpenLogic_CentOS_7.4__agent--bvt.svg)
-RedHat RHEL 6.9|![badge](https://dcrbadges.blob.core.windows.net/scenarios/RedHat_RHEL_6.9__agent--bvt.svg)
-RedHat RHEL 7-RAW|![badge](https://dcrbadges.blob.core.windows.net/scenarios/RedHat_RHEL_7-RAW__agent--bvt.svg)
-SUSE SLES 12-SP5|![badge](https://dcrbadges.blob.core.windows.net/scenarios/SUSE_SLES_12-SP5__agent--bvt.svg)
 
 ## Introduction
 
@@ -191,7 +177,7 @@ A sample configuration file is shown below:
 ```yml
 Extensions.Enabled=y
 Extensions.GoalStatePeriod=6
-Extensions.GoalStateHistoryCleanupPeriod=86400
+Extensions.GoalStateHistoryCleanupPeriod=1800
 Provisioning.Agent=auto
 Provisioning.DeleteRootPassword=n
 Provisioning.RegenerateSshHostKeyPair=y
@@ -260,7 +246,7 @@ setting affects how fast the agent starts executing extensions.
 #### __Extensions.GoalStateHistoryCleanupPeriod__
 
 _Type: Integer_  
-_Default: 86400 (24 hours)_
+_Default: 1800 (30 minutes)_
 
 How often to clean up the history folder of the agent. The agent keeps past goal
 states on this folder, each goal state represented with a set of small files. The

azurelinuxagent/common/agent_supported_feature.py

@@ -51,7 +51,7 @@ class _MultiConfigFeature(AgentSupportedFeature):
 
     __NAME = SupportedFeatureNames.MultiConfig
     __VERSION = "1.0"
-    __SUPPORTED = False
+    __SUPPORTED = True
 
     def __init__(self):
         super(_MultiConfigFeature, self).__init__(name=_MultiConfigFeature.__NAME,

azurelinuxagent/common/conf.py

@@ -145,7 +145,7 @@ def load_conf_from_file(conf_file_path, conf=__conf__):
 
 __INTEGER_OPTIONS__ = {
     "Extensions.GoalStatePeriod": 6,
-    "Extensions.GoalStateHistoryCleanupPeriod": 86400,
+    "Extensions.GoalStateHistoryCleanupPeriod": 1800,
     "OS.EnableFirewallPeriod": 30,
     "OS.RemovePersistentNetRulesPeriod": 30,
     "OS.RootDeviceScsiTimeoutPeriod": 30,
@@ -346,7 +346,7 @@ def get_goal_state_period(conf=__conf__):
 
 
 def get_goal_state_history_cleanup_period(conf=__conf__):
-    return conf.get_int("Extensions.GoalStateHistoryCleanupPeriod", 86400)
+    return conf.get_int("Extensions.GoalStateHistoryCleanupPeriod", 1800)
 
 
 def get_allow_reset_sys_user(conf=__conf__):

azurelinuxagent/common/event.py

@@ -36,7 +36,7 @@
 from azurelinuxagent.common.telemetryevent import TelemetryEventParam, TelemetryEvent, CommonTelemetryEventSchema, \
     GuestAgentGenericLogsSchema, GuestAgentExtensionEventsSchema, GuestAgentPerfCounterEventsSchema
 from azurelinuxagent.common.utils import fileutil, textutil
-from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, getattrib
+from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, getattrib, str_to_encoded_ustr
 from azurelinuxagent.common.version import CURRENT_VERSION, CURRENT_AGENT, AGENT_NAME, DISTRO_NAME, DISTRO_VERSION, DISTRO_CODE_NAME, AGENT_EXECUTION_MODE
 from azurelinuxagent.common.protocol.imds import get_imds_client
 
@@ -73,7 +73,6 @@ class WALAEventOperation:
     CGroupsCleanUp = "CGroupsCleanUp"
     CGroupsDisabled = "CGroupsDisabled"
     CGroupsInfo = "CGroupsInfo"
-    CGroupsLimitsCrossed = "CGroupsLimitsCrossed"
     CollectEventErrors = "CollectEventErrors"
     CollectEventUnicodeErrors = "CollectEventUnicodeErrors"
     ConfigurationChange = "ConfigurationChange"
@@ -106,7 +105,6 @@ class WALAEventOperation:
     PersistFirewallRules = "PersistFirewallRules"
     PluginSettingsVersionMismatch = "PluginSettingsVersionMismatch"
     InvalidExtensionConfig = "InvalidExtensionConfig"
-    ProcessGoalState = "ProcessGoalState"
     Provision = "Provision"
     ProvisionGuestAgent = "ProvisionGuestAgent"
     RemoteAccessHandling = "RemoteAccessHandling"
@@ -484,11 +482,11 @@ def add_event(self, name, op=WALAEventOperation.Unknown, is_success=True, durati
             _log_event(name, op, message, duration, is_success=is_success)
 
         event = TelemetryEvent(TELEMETRY_EVENT_EVENT_ID, TELEMETRY_EVENT_PROVIDER_ID)
-        event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Name, str(name)))
-        event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Version, str(version)))
-        event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Operation, str(op)))
+        event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Name, str_to_encoded_ustr(name)))
+        event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Version, str_to_encoded_ustr(version)))
+        event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Operation, str_to_encoded_ustr(op)))
         event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.OperationSuccess, bool(is_success)))
-        event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Message, str(message)))
+        event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Message, str_to_encoded_ustr(message)))
         event.parameters.append(TelemetryEventParam(GuestAgentExtensionEventsSchema.Duration, int(duration)))
         self.add_common_event_parameters(event, datetime.utcnow())
 
@@ -502,7 +500,7 @@ def add_log_event(self, level, message):
         event = TelemetryEvent(TELEMETRY_LOG_EVENT_ID, TELEMETRY_LOG_PROVIDER_ID)
         event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.EventName, WALAEventOperation.Log))
         event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.CapabilityUsed, logger.LogLevel.STRINGS[level]))
-        event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.Context1, self._clean_up_message(message)))
+        event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.Context1, str_to_encoded_ustr(self._clean_up_message(message))))
         event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.Context2, datetime.utcnow().strftime(logger.Logger.LogTimeFormatInUTC)))
         event.parameters.append(TelemetryEventParam(GuestAgentGenericLogsSchema.Context3, ''))
         self.add_common_event_parameters(event, datetime.utcnow())
@@ -528,9 +526,9 @@ def add_metric(self, category, counter, instance, value, log_event=False):
             _log_event(AGENT_NAME, "METRIC", message, 0)
 
         event = TelemetryEvent(TELEMETRY_METRICS_EVENT_ID, TELEMETRY_EVENT_PROVIDER_ID)
-        event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Category, str(category)))
-        event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Counter, str(counter)))
-        event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Instance, str(instance)))
+        event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Category, str_to_encoded_ustr(category)))
+        event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Counter, str_to_encoded_ustr(counter)))
+        event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Instance, str_to_encoded_ustr(instance)))
         event.parameters.append(TelemetryEventParam(GuestAgentPerfCounterEventsSchema.Value, float(value)))
         self.add_common_event_parameters(event, datetime.utcnow())
 

azurelinuxagent/common/exception.py

@@ -22,6 +22,15 @@
 """
 
 
+class ExitException(BaseException):
+    """
+    Used to exit the agent's process
+    """
+    def __init__(self, reason):
+        super(ExitException, self).__init__()
+        self.reason = reason
+
+
 class AgentError(Exception):
     """
     Base class of agent error.
@@ -100,6 +109,12 @@ class ExtensionConfigError(ExtensionError):
     """
 
 
+class MultiConfigExtensionEnableError(ExtensionError):
+    """
+    Error raised when enable for a Multi-Config extension is failing.
+    """
+
+
 class ProvisionError(AgentError):
     """
     When provision failed

azurelinuxagent/common/future.py

@@ -1,3 +1,4 @@
+import contextlib
 import platform
 import sys
 import os
@@ -35,6 +36,10 @@
     from collections import OrderedDict  # pylint: disable=W0611
     from queue import Queue, Empty  # pylint: disable=W0611,import-error
 
+    # unused-import<W0611> Disabled: python2.7 doesn't have subprocess.DEVNULL
+    # so this import is only used by python3.
+    import subprocess   # pylint: disable=unused-import
+
 elif sys.version_info[0] == 2:
     import httplib as httpclient  # pylint: disable=E0401,W0611
     from urlparse import urlparse  # pylint: disable=E0401
@@ -143,6 +148,22 @@ def is_file_not_found_error(exception):
 
     return isinstance(exception, FileNotFoundError)
 
+@contextlib.contextmanager
+def subprocess_dev_null():
+
+    if sys.version_info[0] == 3:
+        # Suppress no-member errors on python2.7
+        yield subprocess.DEVNULL # pylint: disable=no-member
+    else:
+        try:
+            devnull = open(os.devnull, "a+")
+            yield devnull
+        except Exception:
+            yield None
+        finally:
+            if devnull is not None:
+                devnull.close()
+
 def array_to_bytes(buff):
     # Python 3.9 removed the tostring() method on arrays, the new alias is tobytes()
     if sys.version_info[0] == 2:

azurelinuxagent/common/protocol/goal_state.py

@@ -32,7 +32,7 @@
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.protocol.restapi import Cert, CertList, Extension, ExtHandler, ExtHandlerList, \
     ExtHandlerVersionUri, RemoteAccessUser, RemoteAccessUsersList, VMAgentManifest, VMAgentManifestList, \
-    VMAgentManifestUri, InVMGoalStateMetaData, RequiredFeature
+    VMAgentManifestUri, InVMGoalStateMetaData, RequiredFeature, ExtensionState
 from azurelinuxagent.common.utils import fileutil
 from azurelinuxagent.common.utils.cryptutil import CryptUtil
 from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, findtext, getattrib, gettext
@@ -526,7 +526,7 @@ def to_lower(str_to_change): return str_to_change.lower() if str_to_change is no
         runtime_settings_nodes = findall(plugin_settings_node, "RuntimeSettings")
         extension_runtime_settings_nodes = findall(plugin_settings_node, "ExtensionRuntimeSettings")
 
-        if (runtime_settings_nodes != []) and (extension_runtime_settings_nodes != []):
+        if any(runtime_settings_nodes) and any(extension_runtime_settings_nodes):
             # There can only be a single RuntimeSettings node or multiple ExtensionRuntimeSettings nodes per Plugin
             msg = "Both RuntimeSettings and ExtensionRuntimeSettings found for the same handler: {0} and version: {1}".format(
                 handler_name, version)
@@ -652,27 +652,34 @@ def __parse_extension_runtime_settings(plugin_settings_node, extension_runtime_s
             dependency_level = ExtensionsConfig.__get_dependency_level_from_node(depends_on_node, extension_name)
             dependency_levels[extension_name] = dependency_level
 
+        ext_handler.supports_multi_config = True
         for extension_runtime_setting_node in extension_runtime_settings_nodes:
             # Name and State will only be set for ExtensionRuntimeSettings for Multi-Config
             extension_name = getattrib(extension_runtime_setting_node, "name")
             if extension_name in (None, ""):
                 raise ExtensionConfigError("Extension Name not specified for ExtensionRuntimeSettings for MultiConfig!")
-            # State can either be `enabled` (default) or `disabled`
+            # State can either be `ExtensionState.Enabled` (default) or `ExtensionState.Disabled`
             state = getattrib(extension_runtime_setting_node, "state")
-            state = state if state not in (None, "") else "enabled"
+            state = ustr(state.lower()) if state not in (None, "") else ExtensionState.Enabled
             ExtensionsConfig.__parse_and_add_extension_settings(extension_runtime_setting_node, extension_name,
                                                                 ext_handler, dependency_levels[extension_name],
                                                                 state=state)
 
     @staticmethod
-    def __parse_and_add_extension_settings(settings_node, name, ext_handler, depends_on_level, state="enabled"):
+    def __parse_and_add_extension_settings(settings_node, name, ext_handler, depends_on_level, state=ExtensionState.Enabled):
         seq_no = getattrib(settings_node, "seqNo")
         if seq_no in (None, ""):
             raise ExtensionConfigError("SeqNo not specified for the Extension: {0}".format(name))
+
         try:
             runtime_settings = json.loads(gettext(settings_node))
         except ValueError as error:
             logger.error("Invalid extension settings: {0}", ustr(error))
+            # Incase of invalid/no settings, add the name and seqNo of the Extension and treat it as an extension with
+            # no settings since we were able to successfully parse those data properly. Without this, we wont report
+            # anything for that sequence number and CRP would eventually have to timeout rather than fail fast.
+            ext_handler.properties.extensions.append(
+                Extension(name=name, sequenceNumber=seq_no, state=state, dependencyLevel=depends_on_level))
             return
 
         for plugin_settings_list in runtime_settings["runtimeSettings"]:

azurelinuxagent/common/protocol/restapi.py

@@ -89,6 +89,22 @@ def __init__(self, name, value=None):
         self.value = value
 
 
+class ExtensionState(object):
+    Enabled = ustr("enabled")
+    Disabled = ustr("disabled")
+
+
+class ExtHandlerRequestedState(object):
+    """
+    This is the state of the Handler as requested by the Goal State.
+    CRP only supports 2 states as of now - Enabled and Uninstall
+    Disabled was used for older XML extensions and we keep it to support backward compatibility.
+    """
+    Enabled = ustr("enabled")
+    Disabled = ustr("disabled")
+    Uninstall = ustr("uninstall")
+
+
 class Extension(DataContract):
     """
     The runtime settings associated with a Handler
@@ -105,7 +121,7 @@ def __init__(self,
                  protectedSettings=None,
                  certificateThumbprint=None,
                  dependencyLevel=0,
-                 state="enabled"):
+                 state=ExtensionState.Enabled):
         self.name = name
         self.sequenceNumber = sequenceNumber
         self.publicSettings = publicSettings
@@ -114,6 +130,16 @@ def __init__(self,
         self.dependencyLevel = dependencyLevel
         self.state = state
 
+    def dependency_level_sort_key(self, handler_state):
+        level = self.dependencyLevel
+        # Process uninstall or disabled before enabled, in reverse order
+        # Prioritize Handler state and Extension state both when sorting extensions
+        # remap 0 to -1, 1 to -2, 2 to -3, etc
+        if handler_state != ExtHandlerRequestedState.Enabled or self.state != ExtensionState.Enabled:
+            level = (0 - level) - 1
+
+        return level
+
 
 class ExtHandlerProperties(DataContract):
     def __init__(self):
@@ -139,6 +165,7 @@ def __init__(self, name=None):
         self.properties = ExtHandlerProperties()
         self.versionUris = DataContractList(ExtHandlerVersionUri)
         self.__invalid_handler_setting_reason = None
+        self.supports_multi_config = False
 
     @property
     def is_invalid_setting(self):
@@ -152,7 +179,7 @@ def invalid_setting_reason(self):
     def invalid_setting_reason(self, value):
         self.__invalid_handler_setting_reason = value
 
-    def sort_key(self):
+    def dependency_level_sort_key(self):
         levels = [e.dependencyLevel for e in self.properties.extensions]
         if len(levels) == 0:
             level = 0
@@ -243,12 +270,14 @@ def __init__(self, name=None, status=None, code=None, message=None):
 
 class ExtensionStatus(DataContract):
     def __init__(self,
+                 name=None,
                  configurationAppliedTime=None,
                  operation=None,
                  status=None,
                  seq_no=None,
                  code=None,
                  message=None):
+        self.name = name
         self.configurationAppliedTime = configurationAppliedTime
         self.operation = operation
         self.status = status
@@ -270,7 +299,8 @@ def __init__(self,
         self.status = status
         self.code = code
         self.message = message
-        self.extensions = DataContractList(ustr)
+        self.supports_multi_config = False
+        self.extension_status = None
 
 
 class VMAgentStatus(DataContract):
@@ -318,4 +348,3 @@ def __init__(self, name, encrypted_password, expiration):
 class RemoteAccessUsersList(DataContract):
     def __init__(self):
         self.users = DataContractList(RemoteAccessUser)
-