diff --git a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java index 2182dfc542d3..81cd7eeb29e3 100644 --- a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java +++ b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java @@ -166,4 +166,6 @@ public enum TapAgentsAction { void notifyMonitorsOfRemovedHost(long hostId, long clusterId); void propagateChangeToAgents(Map params); + + void updateCapacityOfHosts(); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 606a902dce7c..b57f4de4c86e 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Collections; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; @@ -34,6 +35,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.LongFunction; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -51,6 +53,7 @@ import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; +import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.lang3.BooleanUtils; import org.apache.log4j.Logger; @@ -81,7 +84,11 @@ import com.cloud.agent.transport.Request; import com.cloud.agent.transport.Response; import com.cloud.alert.AlertManager; +import com.cloud.capacity.Capacity; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.capacity.CapacityVO; import com.cloud.configuration.ManagementServiceConfiguration; +import com.cloud.configuration.ConfigurationManagerImpl; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; @@ -124,6 +131,8 @@ import com.cloud.utils.nio.Task; import com.cloud.utils.time.InaccurateClock; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.ThreadContext; +import org.apache.commons.collections.CollectionUtils; /** * Implementation of the Agent Manager. This class controls the connection to the agents. @@ -172,6 +181,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl @Inject protected IndirectAgentLB indirectAgentLB; + @Inject + protected CapacityDao capacityDao; protected int _retry = 2; @@ -1874,6 +1885,7 @@ public void processConnect(final Host host, final StartupCommand cmd, final bool params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString())); params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString())); params.put(NetworkOrchestrationService.TUNGSTEN_ENABLED.key(), String.valueOf(NetworkOrchestrationService.TUNGSTEN_ENABLED.valueIn(host.getDataCenterId()))); + params.put(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key(), String.valueOf(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(host.getClusterId()))); try { SetHostParamsCommand cmds = new SetHostParamsCommand(params); @@ -1932,12 +1944,10 @@ protected Map> getHostsPerZone() { return hostsByZone; } - private void sendCommandToAgents(Map> hostsPerZone, Map params) { - SetHostParamsCommand cmds = new SetHostParamsCommand(params); - for (Long zoneId : hostsPerZone.keySet()) { - List hostIds = hostsPerZone.get(zoneId); + private void sendCommandToAgents(Map> hostsPerZone, LongFunction> paramsGenerator ) { + for (List hostIds : hostsPerZone.values()) { for (Long hostId : hostIds) { - Answer answer = easySend(hostId, cmds); + Answer answer = easySend(hostId, new SetHostParamsCommand(paramsGenerator.apply(hostId))); if (answer == null || !answer.getResult()) { s_logger.error("Error sending parameters to agent " + hostId); } @@ -1945,12 +1955,76 @@ private void sendCommandToAgents(Map> hostsPerZone, Map 0) { + updateMemoriesInDb(host, newMemoryValue); + return true; + } + } catch (Exception e) { + s_logger.error("Unable to update the reserved memory capacity for host id " + host.getId() + " : " + e.getMessage()); + } + return false; + } + @Override public void propagateChangeToAgents(Map params) { if (params != null && ! params.isEmpty()) { s_logger.debug("Propagating changes on host parameters to the agents"); Map> hostsPerZone = getHostsPerZone(); - sendCommandToAgents(hostsPerZone, params); + sendCommandToAgents(hostsPerZone, id -> params); + } + } + + @Override + public void updateCapacityOfHosts() { + Map> hostsByZone = new HashMap<>(); + boolean allHostMemoryValuesAreValid = true; + + List allHosts = _resourceMgr.listAllHostsInAllZonesByType(Host.Type.Routing); + if (CollectionUtils.isEmpty(allHosts)) { + return; + } + + for (HostVO host : allHosts) { + boolean updateWasSuccessful = updateHostMemory(host); + + if (!updateWasSuccessful) { + allHostMemoryValuesAreValid = false; + continue; + } + + Long zoneId = host.getDataCenterId(); + List hostIds = hostsByZone.getOrDefault(zoneId, new ArrayList<>()); + hostIds.add(host.getId()); + hostsByZone.put(zoneId, hostIds); + } + + if (allHostMemoryValuesAreValid) { + sendCommandToAgents(hostsByZone, + hostId -> Collections.singletonMap( + ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key(), + ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(_hostDao.findById(hostId).getClusterId()).toString())); } } diff --git a/engine/schema/src/main/java/com/cloud/host/HostVO.java b/engine/schema/src/main/java/com/cloud/host/HostVO.java index 697401ad0696..b76bb7b0e1ef 100644 --- a/engine/schema/src/main/java/com/cloud/host/HostVO.java +++ b/engine/schema/src/main/java/com/cloud/host/HostVO.java @@ -782,4 +782,12 @@ public boolean checkHostServiceOfferingTags(ServiceOffering serviceOffering){ public PartitionType partitionType() { return PartitionType.Host; } + + public long getDom0MinMemory() { + return dom0MinMemory; + } + + public void setDom0MinMemory(long dom0MinMemory) { + this.dom0MinMemory = dom0MinMemory; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 342bb87cf41f..c7f4853676f2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -133,6 +133,7 @@ import com.cloud.agent.resource.virtualnetwork.VirtualRouterDeployer; import com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource; import com.cloud.configuration.Config; +import com.cloud.configuration.ConfigurationManagerImpl; import com.cloud.dc.Vlan; import com.cloud.exception.InternalErrorException; import com.cloud.host.Host.Type; @@ -1111,7 +1112,17 @@ public boolean configure(final String name, final Map params) th videoRam = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_VIDEO_RAM); // Reserve 1GB unless admin overrides - dom0MinMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB)); + long reservedMemory = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB); + + value = (String)params.get(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key()); + long clusterReservedMemoryValue = NumbersUtil.parseInt(value, 1024); + + if (clusterReservedMemoryValue != 1024) { + reservedMemory = clusterReservedMemoryValue; + } + + dom0MinMem = ByteScaleUtils.mebibytesToBytes(reservedMemory); + dom0MinCpuCores = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_CPU_CORE_COUNT); @@ -1453,6 +1464,15 @@ private void validateLocalStorageUUID(String localStorageUUID) throws Configurat } } + private void updateDom0MinMem(PropertiesStorage storage, final Map params) { + long value = Long.parseLong(params.get(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key())); + s_logger.info("Reserved memory for host is " + value + "MB"); + dom0MinMem = ByteScaleUtils.mebibytesToBytes(value); + if (!StringUtils.isEmpty(String.valueOf(value))) { + storage.persist(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key(), String.valueOf(value)); + } + } + public boolean configureHostParams(final Map params) { final File file = PropertiesUtil.findConfigFile("agent.properties"); if (file == null) { @@ -1468,7 +1488,7 @@ public boolean configureHostParams(final Map params) { } if (params.get(Config.MigrateWait.toString()) != null) { - String value = (String)params.get(Config.MigrateWait.toString()); + String value = (String) params.get(Config.MigrateWait.toString()); Integer intValue = NumbersUtil.parseInt(value, -1); storage.persist("vm.migrate.wait", String.valueOf(intValue)); migrateWait = intValue; @@ -1478,6 +1498,10 @@ public boolean configureHostParams(final Map params) { isTungstenEnabled = Boolean.parseBoolean(params.get(NetworkOrchestrationService.TUNGSTEN_ENABLED.key())); } + if (params.get(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key()) != null) { + updateDom0MinMem(storage, params); + } + return true; } diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 29579759c7f4..a0e5bd6cff78 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -111,7 +111,6 @@ import org.apache.cloudstack.framework.config.impl.ConfigurationSubGroupVO; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.framework.messagebus.MessageSubscriber; import org.apache.cloudstack.framework.messagebus.PublishScope; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.region.PortableIp; @@ -493,6 +492,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati public static final ConfigKey ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN = new ConfigKey(Boolean.class, "enable.domain.settings.for.child.domain", "Advanced", "false", "Indicates whether the settings of parent domain should be applied for child domain. If true, the child domain will get value from parent domain if its not configured in child domain else global value is taken.", true, ConfigKey.Scope.Global, null); + public static final ConfigKey HOST_RESERVED_MEM_MB = new ConfigKey<>("Advanced", Integer.class, "host.reserved.mem.mb", "1024", + "Set an upper limit for memory in megabytes which will be reserved for host and not used for VM allocation.", true, ConfigKey.Scope.Cluster); public static ConfigKey VM_SERVICE_OFFERING_MAX_CPU_CORES = new ConfigKey("Advanced", Integer.class, "vm.serviceoffering.cpu.cores.max", "0", "Maximum CPU cores " + "for vm service offering. If 0 - no limitation", true); @@ -595,23 +596,22 @@ private void overProvisioningFactorsForValidation() { } private void initMessageBusListener() { - messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, new MessageSubscriber() { - @Override - public void onPublishMessage(String serderAddress, String subject, Object args) { - String globalSettingUpdated = (String) args; - if (StringUtils.isEmpty(globalSettingUpdated)) { - return; - } - if (globalSettingUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) || - globalSettingUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBAlgorithm.key())) { - _indirectAgentLB.propagateMSListToAgents(); - } else if (globalSettingUpdated.equals(Config.RouterAggregationCommandEachTimeout.toString()) - || globalSettingUpdated.equals(Config.MigrateWait.toString())) { - Map params = new HashMap(); - params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString())); - params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString())); - _agentManager.propagateChangeToAgents(params); - } + messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, (serverAddress, subject, args) -> { + String globalSettingUpdated = (String) args; + if (StringUtils.isEmpty(globalSettingUpdated)) { + return; + } + if (globalSettingUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) || + globalSettingUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBAlgorithm.key())) { + _indirectAgentLB.propagateMSListToAgents(); + } else if (globalSettingUpdated.equals(Config.RouterAggregationCommandEachTimeout.toString()) + || globalSettingUpdated.equals(Config.MigrateWait.toString())) { + Map params = new HashMap(); + params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString())); + params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString())); + _agentManager.propagateChangeToAgents(params); + } else if (globalSettingUpdated.equalsIgnoreCase(HOST_RESERVED_MEM_MB.key())) { + _agentManager.updateCapacityOfHosts(); } }); } @@ -7809,7 +7809,7 @@ public ConfigKey[] getConfigKeys() { BYTES_MAX_READ_LENGTH, BYTES_MAX_WRITE_LENGTH, ADD_HOST_ON_SERVICE_RESTART_KVM, SET_HOST_DOWN_TO_MAINTENANCE, VM_SERVICE_OFFERING_MAX_CPU_CORES, VM_SERVICE_OFFERING_MAX_RAM_SIZE, MIGRATE_VM_ACROSS_CLUSTERS, ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN, ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN, - ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, DELETE_QUERY_BATCH_SIZE + ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, DELETE_QUERY_BATCH_SIZE, HOST_RESERVED_MEM_MB }; } diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 5d3ec62c56d3..ced14076f6c3 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -66,6 +66,7 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ObjectUtils; @@ -100,6 +101,7 @@ import com.cloud.cluster.ClusterManager; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; +import com.cloud.configuration.ConfigurationManagerImpl; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.dc.ClusterVO; @@ -2350,6 +2352,11 @@ protected HostVO createHostVO(final StartupCommand[] cmds, final ServerResource hostTags = implicitHostTags; } } + // Update host memory reported by agent + if (ssCmd.getHypervisorType().equals(HypervisorType.KVM) || + ssCmd.getHypervisorType().equals(HypervisorType.LXC)) { + host.setDom0MinMemory(ByteScaleUtils.mebibytesToBytes(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(host.getClusterId()))); + } } host.setDataCenterId(dc.getId());